diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6cf5b7c..25f195cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,10 @@ jobs: uses: golangci/golangci-lint-action@v9 with: version: v2.13.2 - only-new-issues: true + # The GitHub PR diff API rejects reviews larger than 20,000 lines. + # Keep the new-code gate independent of that API limit. + args: >- + --new-from-rev=${{ github.event_name == 'pull_request' && format('origin/{0}', github.base_ref) || 'HEAD^' }} test: name: Test diff --git a/.gitignore b/.gitignore index a61c2dce..268d7bff 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ configs/liveforge.local.yaml # Recordings /data/ /recordings/ +/module/record/recordings/ # Node (Playwright test tooling) node_modules/ diff --git a/README.md b/README.md index 1ef7c2b6..49c60dde 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ LiveForge is a modular live streaming media server that ingests, transmuxes, and - **Multi-protocol ingest** — Publish via RTMP, RTSP (TCP + UDP, separate eligible audio/video SETUP tracks), SRT, WebRTC WHIP, or GB28181 - **Multi-protocol playback** — Pull via RTMP, RTSP, SRT, WebRTC WHEP, HLS, LL-HLS, DASH, HTTP-FLV, HTTP-TS, FMP4, or WebSocket +- **Continuous HTTP integrity** — HTTP-FLV, HTTP-TS, FMP4, and their WebSocket outputs terminate on a ring overwrite and never bridge the media gap with a retained post-gap packet +- **Segment overwrite handling** — HLS and LL-HLS discard partial media, reopen the refreshed direct/transformed audio source when needed, and recover at live media with one discontinuity (video waits for a keyframe; audio-only resumes immediately); retained LL-HLS fMP4 media keeps matching immutable versioned init data, while DASH preserves completed single-Period media and retires the affected manager - **SRT** — Secure Reliable Transport with AES encryption, low-latency MPEG-TS delivery (pure Go via `datarhei/gosrt`) - **WebRTC** — WHIP/WHEP with a 1 MiB SDP offer limit, ICE Lite, GCC send-side bandwidth estimation, and browser-based publish - **Codec support** — H.264, H.265/HEVC, VP8, VP9, AV1, AAC, Opus, G.711 (μ-law/A-law), MP3 @@ -104,12 +106,14 @@ Multi-protocol forwarding and on-demand origin pull for building CDN-like topolo - **HTTP scheduler** — Dynamic target resolution via external HTTP callback, or static target lists - **Topologies** — Origin-edge, origin-multi-edge, origin-center-edge (three-tier) - **Retry & resilience** — Configurable retry count, interval, and backoff -- **Forwarding hot path** — Relay and WHEP readers use independent blocking waits; RTMP push reuses FLV encoding buffers, RTSP interleaving uses vectored writes, and relay byte metrics batch after the first observation to reduce per-frame overhead +- **Forwarding hot path** — Relay readers use independent blocking waits; WHEP uses one condition-backed pump per source or target-audio reader so readiness and atomic reads cannot race; RTMP push reuses FLV encoding buffers, RTSP interleaving uses vectored writes, and relay byte metrics batch after the first observation to reduce per-frame overhead > See [Wiki: Cluster Deployment](../../wiki/Cluster-Deployment) for topology examples and configuration. For focused forwarding measurements, run `go test -bench='BenchmarkRingReader|BenchmarkRTMPConn' -benchmem ./pkg/util ./module/cluster`. Benchmark values depend on the host and are not capacity guarantees. +For production-path regression measurements that include stable-publisher Stream admission, ring/GOP writes, complete RTMP FLV/chunk framing, RTSP H.264 packetization/RTP/interleaved framing, and bounded relay-byte accounting, run `go test -run '^$' -bench 'BenchmarkStreamIngressProduction|BenchmarkRTMPRelaySendMediaFrameProduction|BenchmarkRTSPRelaySendFrameProduction|BenchmarkRelayObservationAccounting' -benchmem -count=3 ./core ./module/cluster`. On an Apple M1 Pro with Go 1.26.0, the fixture measured stable Stream ingress at 65.86-67.28 ns/op (29 B/op, 0 allocs/op), RTMP H.264 at 155.1-155.6 ns/op (24 B/op, 3 allocs/op), RTMP AAC at 73.60-73.76 ns/op (21 B/op, 3 allocs/op), RTSP single-NAL H.264 at 1.825-1.833 us/op (4,044 B/op, 9 allocs/op), and RTSP three-packet FU-A H.264 at 4.593-4.605 us/op (9,892 B/op, 23 allocs/op). Stream ingress uses a preallocated 64-second monotonic 25 fps H.264/50 fps G.711A frame pool with shared immutable payloads, no subscribers, bitrate limiting disabled, two retained GOPs, a 300-frame GOP bound, and a 4,096-entry ring. RTMP uses fixed-timestamp media frames and payload-scoped relay accounting; RTSP uses fixed-timestamp RTP input and framed-byte accounting. Both egress fixtures terminate at bounded in-memory writers, excluding socket writes, TCP writev, deadlines, and kernel/network syscall cost. The isolated accounting ns/op values also exclude the production context lookup and are primarily allocation-regression evidence. These paths are deliberately not compared with the older narrower `BenchmarkStreamWriteFrame` microbenchmark, and none of these figures predict subscriber count, concurrency, or deployment capacity. + ### LL-HLS (Low-Latency HLS) Apple LL-HLS implementation for sub-second latency HLS delivery: @@ -121,25 +125,35 @@ Apple LL-HLS implementation for sub-second latency HLS delivery: - **fMP4 fragment parsing** — Complete media segments assembled from multiple `moof`/`mdat` fragments are parsed without dropping earlier fragments - **fMP4 AAC timing** — Omitted AAC sample rate and channel count are derived from the AudioSpecificConfig; the resolved sample rate is reused as the media timescale so DTS intervals remain stable - **Legacy player compat** — Graceful degradation for players without LL-HLS support (buffered segment delivery) -- **Keyframe-aligned startup** — Cached and live GOP frames remain continuous; HLS, LL-HLS, and DASH segmenters wait for the current publisher generation's required sequence headers and bind those headers, replay frames, and the live cursor from one startup snapshot. The initial Hls.js manifest waits for one complete segment without duplicating its parts. Its bounded wait covers the configured full-segment target plus one part (10-second floor, 30-second cap), and returns 503 instead of a part-only manifest if no full segment becomes available. Blocking reloads retain the latest completed part identities while consuming new low-latency parts. DASH also starts after one complete segment, uses a one-fragment live delay, and refreshes its MPD within two seconds. HLS, LL-HLS, and DASH manifests escape each stream-key segment, DASH URL attributes are XML-safe, and media-segment routing preserves valid keys at arbitrary path depth +- **Keyframe-aligned startup** — Cached and live GOP frames remain continuous; HLS, LL-HLS, and DASH segmenters wait for the current publisher generation's required sequence headers and bind those headers, replay frames, and the live cursor from one startup snapshot. The initial Hls.js manifest waits for one complete segment without duplicating its parts. Its bounded wait covers the configured full-segment target plus one part (10-second floor, 30-second cap), and returns 503 instead of a part-only manifest if no full segment becomes available. Blocking reloads retain the latest completed part identities while consuming new low-latency parts. DASH also starts after one complete segment, uses a one-fragment live delay, and refreshes its MPD within two seconds. Manifest and segment write deadlines start immediately before the response write, so delayed readiness does not consume the write window. HLS, LL-HLS, and DASH manifests escape each stream-key segment, DASH URL attributes are XML-safe, and media-segment routing preserves valid keys at arbitrary path depth +- **Generation-safe finalization** — Publisher stop retires the matching HLS, DASH, or LL-HLS manager from new request lookup while it drains every accepted frame through that generation's captured end cursor and finalizes once. A replacement publisher uses a distinct manager and cannot cross-contaminate the retired output. LL-HLS blocking reloads also terminate when the manager stops; HTTP module shutdown force-stops and joins active and draining manager workers ### Management & Operations - **Web console** — Seven permission-aware tabs with multi-protocol preview and WHIP publish: Streams, GB28181, Config, Cluster, SIP Calls, Storage, and Security. Recent Audit is a surface inside Security, not a separate tab. - **REST API** — Stream lifecycle, config refresh/status, cluster status, SIP call control, recording/DVR management, security/audit, GB28181, and public health probes - **Auth and RBAC** — Named viewer/operator/admin API tokens, console sessions, JWT/callback publish/subscribe auth, bounded redacted audit trail -- **Recording and DVR** — FLV, fragmented MP4, MP4, MPEG-TS, and HLS recording; new recordings default to fMP4/`.mp4`; fMP4 declares AAC directly, converts non-AAC source audio such as G.711, Opus, and MP3 to AAC through the optional `audiocodec`/FFmpeg build, and filters audio to keep playable video-only output when that path is unavailable; a stopped transformed recording drains source frames already committed before finalizing; DVR TS similarly normalizes audio unsupported by its target; segmentation, storage health, download/range/inline-play/delete management, zero-byte session protection, and time-shift status -- **Local protocol labs** — SIP and GB28181 pages run one-shot and persistent fake-device checks locally without another platform or device. SIP uses separate H.264 and PCMA/PCMU RTP/RTCP tracks and never mutates a receive-mode source stream. Receive mode waits for the selected publisher generation's required sequence headers before signaling, while known unsupported codecs are rejected immediately. GB28181 publish registers a listening fake device and exercises LiveForge's normal server-initiated live-play and real RTP/RTCP receive path; receive validates H.264 plus G.711A and admits its source subscriber before module-owned PS/RTP/RTCP egress becomes active. Subscriber-limit rejection fails startup synchronously, while a later media-send failure moves the Lab to `failed` and releases signaling and media resources. The dependency-free moving 160x90 test pattern runs at 25 fps with one IDR per second and audible 20 ms audio frames. Persistent GB28181 sessions renew Keepalive at roughly one-third of `gb28181.keepalive.timeout`. When both modules share one SIP listener, H.264 plus PCMA/PCMU RTP offers route to SIP Gateway while PS/90000 offers route to GB28181 +- **Recording and DVR** — FLV, fragmented MP4, MP4, MPEG-TS, and HLS recording; new recordings default to fMP4/`.mp4`; every rotated recording file retains the declared tracks and latest codec initialization, starts each track on a zero-based file-local timeline, and remains independently parseable; TS emits PAT/PMT before its first media PES, while classic MP4 calculates audio/video durations on their independent clocks, saturates version-0 timing fields instead of wrapping, uses signed `ctts` version 1 for negative B-frame composition offsets, and writes expandable AAC ESDS lengths; fMP4 declares AAC directly, converts non-AAC source audio such as G.711, Opus, and MP3 to AAC through the optional `audiocodec`/FFmpeg build, and filters audio to keep playable video-only output when that path is unavailable; a stopped transformed recording drains source frames already committed before finalizing, while publisher-generation completion flushes retained resampler samples, pads the final partial PCM frame, and emits delayed encoder packets exactly once before Record/DVR output closes; DVR TS similarly normalizes audio unsupported by its target; segmentation, storage health, download/range/inline-play/delete management, exact full-ID action routing, retryable cleanup-before-primary deletion, zero-byte session protection, and time-shift status. Recording play/download acquire the global connection budget before opening media and apply a 10-second write deadline. +- **Recording/DVR state and routing** — Only completed recordings are served by download or inline play; active and failed recordings return JSON `409` without media bytes. Record formats are `flv`, `fmp4`, `mp4`, `ts`, and `hls` (`hls` stores TS), with empty/zero or decimal `B`/`KB`/`MB`/`GB` size limits. Audio-only DVR publishes a segment at the audio DTS duration boundary while its publisher remains online. DVR nested stream-key routes preserve slash hierarchy, reject encoded separators and dot segments, and escape `?`, `#`, and `%` independently per key segment; `/api/v1/server/info` supplies the bound non-zero DVR port and its actual HTTP/TLS scheme. +- **Local protocol labs** — SIP and GB28181 pages run one-shot and persistent fake-device checks locally without another platform or device. SIP uses separate H.264 and PCMA/PCMU RTP/RTCP tracks and never mutates a receive-mode source stream. Receive mode waits for the selected publisher generation's required sequence headers before signaling and treats the selected PCMA/PCMU value as the outbound target codec; a differing source uses a generation-bound shared audio transcode reader when the tagged runtime can produce that target, while H.264 remains on its original live cursor. Known unsupported conversions are rejected immediately. GB28181 publish registers a listening fake device and exercises LiveForge's normal server-initiated live-play and real RTP/RTCP receive path; receive requires H.264 plus direct G.711A or audio that the tagged runtime can convert to G.711A, then admits its source subscriber before module-owned PS/RTP/RTCP egress becomes active; transformed audio uses an independent generation-bound reader while H.264 remains direct. Subscriber-limit rejection fails startup synchronously, while a later media-send failure moves the Lab to `failed` and releases signaling and media resources. The dependency-free moving 160x90 test pattern runs at 25 fps with one IDR per second and audible 20 ms audio frames. Persistent GB28181 sessions renew Keepalive at roughly one-third of `gb28181.keepalive.timeout`. When both modules share one SIP listener, H.264 plus PCMA/PCMU RTP offers route to SIP Gateway while PS/90000 offers route to GB28181 +- **Protocol lab admission** — `sip.gateway.max_lab_sessions` and `gb28181.max_lab_sessions` bound active persistent lab sessions independently; the default is 16, terminal history does not consume the limit, non-positive values use the default, and a full ceiling returns HTTP 429 before sockets or media resources are allocated +- **SIP RTP port ownership** — Gateway media pairs skip externally occupied ports and remain socket-bound throughout SDP negotiation; fake Lab endpoints avoid the configured gateway RTP range +- **SIP outbound retirement** — Requested PCMA/PCMU conversion uses an independent publisher-generation-bound audio reader. Each ready frame is packetized before final send admission, then rechecks cancellation and the current publisher generation under the terminal send gate. Terminal teardown closes admission and owned sockets, drains already admitted sends without holding lifecycle or admission locks, and only then publishes terminal state and callbacks; publisher retirement releases the transcode reader and subscriber, reclaims the RTP/RTCP pair, and emits one BYE even when another teardown arrives concurrently +- **SIP overwrite recovery** — Outbound SIP discards retained gap-crossing media and advances only the affected source or target-audio reader. A source gap keeps transformed audio flowing and gates direct H.264 until the newest same-generation sequence header plus IDR; a target-audio gap keeps direct video flowing and resumes audio at live media. Active-generation transformed-audio EOF fails the call as `network_lost`, and the dual-reader parent cancels and joins both media pumps before returning - **Protocol Lab stream keys** — SIP and GB28181 accept printable ASCII keys up to 256 bytes whose slash-separated segments are non-empty and are neither `.` nor `..`. GB28181 publish uses that requested key only for the loopback simulator; real devices retain `{stream_prefix}/{channel_id}` - **GB28181 PS compatibility** — Outbound PS converts internal AVCC/HVCC video samples to Annex-B so real GB28181 receivers can decode video +- **GB28181 overwrite recovery** — Outbound PS/RTP serializes source and transformed-audio control results ahead of pending output, discards overwritten and pending pre-gap media, advances only the affected reader, and keeps unaffected media flowing. Source gaps start fresh PS state without resetting RTP sequence and resume H.264 only at the latest post-gap header plus IDR; transformed-audio gaps preserve clean video and PS state without restarting its original 20 ms holdback deadline - **Lab diagnostics** — Managers retain all active sessions plus the newest 16 terminal records. Failed sessions expose a bounded `last_error` with SIP credentials and bearer tokens removed; session views expose receiver-side RTCP and separate audio/video counters. Playback paths escape each stream-key segment and use actual bound listeners for absolute RTMP/RTSP URLs; Console Lab Preview consumes those returned paths directly - **Startup rollback** — Listener or module initialization failures report the original error, close only modules whose initialization was attempted, and do not panic while rolling back later uninitialized modules - **Notifications** — HTTP webhook (HMAC-SHA256 signed) and WebSocket real-time events -- **Prometheus metrics** — Server-level and per-stream gauges: connections, bitrate, FPS, GOP cache, subscribers by protocol -- **Rate limiting** — Per-IP token bucket for connection flood protection +- **Prometheus metrics** — Server-level gauges are always available when enabled; per-stream bitrate/FPS/GOP/subscriber labels are opt-in. Without an allowlist, the configured limit is a Collector-lifetime cardinality budget: active keys are admitted in creation order, retained as scalar keys after their streams disappear, and never replaced by churn. An exact allowlist defines the only eligible keys and the limit still bounds each scrape. `stream_detail_limit: 0` disables per-stream series; negative configured limits are invalid and rejected. Use the management API for current stream detail or an exact allowlist for selected Prometheus labels +- **Rate limiting** — Per-IP token bucket for connection flood protection; trusted proxy chains are resolved right-to-left so attacker-controlled XFF prefixes cannot select new buckets +- **HTTP connection timeouts** — API, WebRTC signaling, and metrics listeners bound header parsing to 5 seconds and idle keep-alive connections to 2 minutes; existing write deadlines remain unchanged - **Slow consumer protection** — EWMA-based lag detection with progressive frame dropping - **GCC congestion control** — Send-side bandwidth estimation for WebRTC WHEP with adaptive bitrate pacing -- **Generation-bound startup** — SIP, GB28181, recording, DVR, and cluster egress capture one publisher snapshot, replay only the required current headers/GOP once, then continue from its live cursor. SIP inbound INVITEs run synchronous publish authorization before RTP allocation and emit matching start/stop lifecycle events after activation, so recording and DVR follow the call. Publisher replacement cancels old readers, pure-audio streams never replay retained history, and sequence-header-only recordings are failed rather than published as successful media +- **Generation-bound startup** — SIP, GB28181, recording, DVR, and cluster egress capture one publisher snapshot, replay only the required current headers/GOP once, then continue from its live cursor. DVR carries that validated snapshot through retained-index/storage recovery and rechecks the generation immediately before session installation; replacement during setup discards the candidate. DVR shutdown starts its absolute drain deadline before waiting for setup ownership, so blocked setup cannot extend the configured shutdown bound. SIP inbound INVITEs run synchronous publish authorization before RTP allocation and emit matching start/stop lifecycle events after activation, so recording and DVR follow the call. Publisher replacement cancels old readers, pure-audio streams never replay retained history, and sequence-header-only recordings are failed rather than published as successful media +- **Publisher ownership isolation** — Each non-empty publisher ID can create only one generation during a `Stream` object's lifetime. Reusing A after an intervening B is rejected before stream state changes, so delayed A frames, activity, and cleanup cannot affect the active owner; a newly created `Stream` starts a separate identity lifetime. Once stream destruction starts, late cleanup cannot return it to an attachable state or reopen its closed ring +- **Hot GOP-bound reload** — Tightening frame, duration, or byte bounds keeps the shortest keyframe-led playable prefix allowed by all active bounds and may seal it immediately. Duration uses the full unordered min/max DTS span without rewriting media order. With GOP caching enabled, at least one frame or byte bound must remain positive; zero disables only that bound. Relaxation lets only the active retained GOP admit future interleaved frames until the remaining bounds; older GOPs stay trimmed, omitted frames are not restored, and the next keyframe starts a new complete GOP ## Architecture @@ -256,9 +270,11 @@ ffmpeg -re -i input.mp4 -c copy -f mpegts "srt://localhost:6000?streamid=publish **WebRTC (Browser):** Open `http://localhost:8090/console`, click **"+ WebRTC Publish"**, select camera/mic, and start streaming. -The Console can publish H.265/HEVC video with Opus audio when the browser and platform expose an H.265 WebRTC encoder. WHIP maps audio and video RTP onto one session timeline, and HLS/DASH/FLV/TS use a combined transcode reader from the cached GOP source position so target audio history and live source video continue without a first-frame freeze or duplicate cached video. Its FMP4 preview preserves signed B-frame composition offsets on a near-zero timeline established when the shared muxer starts; later subscribers seek to their first buffered timestamp. WHEP Live replays the atomic cached GOP while source video continues from the matching ring cursor, with transcoded target audio read independently. The WebRTC transcode worker waits without consuming the source playback wakeup, so video pacing remains stable even when source audio pauses. The tagged audio build is the complete cross-protocol profile; see [WHIP H.265 + Opus playback verification](docs/recipes/whip-h265-opus-playback.md). +The Console can publish H.265/HEVC video with Opus audio when the browser and platform expose an H.265 WebRTC encoder. WHIP maps audio and video RTP onto one session timeline, and HLS/DASH/FLV/TS use a combined transcode reader from the cached GOP source position so target audio history and live source video continue without a first-frame freeze or duplicate cached video. Its FMP4 preview preserves signed B-frame composition offsets on a near-zero timeline established when the shared muxer starts; later subscribers seek to their first buffered timestamp. For G.711 sources, the Console declares AAC in the FMP4 SourceBuffer only when `GET /api/v1/server/info` reports that the configured process can actually transcode both G.711 variants to AAC; portable builds keep the video-only declaration. WHEP Live replays the atomic cached GOP while source video continues from the matching ring cursor, with transcoded target audio read independently. The WebRTC transcode worker waits without consuming the source playback wakeup, so video pacing remains stable even when source audio pauses. The tagged audio build is the complete cross-protocol profile; see [WHIP H.265 + Opus playback verification](docs/recipes/whip-h265-opus-playback.md). + +The Console's default WHEP preview uses the cached live startup path, so a normal H.264 GOP does not have to wait for a post-snapshot IDR. Protocol Lab returns distinct `whep`/`whep_live` (`mode=live`) and `whep_realtime` (`mode=realtime`) paths. A requested source audio or video track must negotiate successfully: an unsupported requested codec returns 415 and an internal track setup failure returns 500 instead of silently serving only the other track; disabled or non-receiving offer m-lines remain intentionally omitted. Receiving direction follows media-level attributes first and then session-level attributes, and codec matching requires an exact `rtpmap` name on a payload listed by that m-line. Explicit realtime mode reports a distinct waiting-keyframe state, including mixed feeds whose audio advances while video interframes are still being discarded before the first IDR. During active playback, WHEP binds each source or transformed-audio overwrite to its atomic reader result, discards the retained post-gap frame, and advances only that reader to live. One condition-backed pump exclusively owns each reader's readiness check, atomic read, and live advance; shutdown cancels and joins both pumps before releasing transformed-audio ownership once. A source overwrite preserves established audio while video returns to `waiting_keyframe`, resets pacing/DTS/PTS state, and resumes with the latest same-generation parameter sets plus a keyframe; audio-only playback resumes at the next live frame. A transformed target-audio overwrite leaves clean video continuous, while active expected target-audio EOF terminates as `target_audio_failed` instead of silently degrading to video-only. `GET /webrtc/session/{sessionId}/status` exposes expected media kinds, the first successful sample time and stable `first_media_wait_ms`, per-kind last-advance timestamps, generation, cursor, media counters, actual RTP packet/byte and received RTCP packet counters, and bounded sample-write errors. Dropped counters cover negotiated tracks only; session close preserves one final monotonic transport snapshot before storing the terminal status. Both requested kinds must advance before `playing`; after startup, eight seconds without advancement from any expected kind reports recoverable `media_stalled`, and every stale kind must advance before recovery. The Console names only the stale expected kinds using server timestamps. Real state transitions emit one structured log with generation, cursor, mode, previous/next state, and a bounded error when present; each overwrite emits one bounded warning with reader identity, exact overwrite count, and recovery action. Feed termination closes and releases the session automatically, while at most 64 terminal status records remain readable for up to two minutes. The tagged Chromium matrix verifies SIP publish to GB28181 receive plus WHEP, GB28181 publish to SIP receive plus WHEP, and WHIP H.264/Opus publish to both SIP and GB28181 receive plus WHEP. It requires expected decoded dimensions, advancing media time, increasing video/audio RTP and decoded-frame counters, connected ICE, and non-stalled server RTP/RTCP status; `LIVEFORGE_PROTOCOL_MATRIX_SOAK=60s` extends those checks per second without claiming deployment capacity. See [the technical risk record](docs/TECHNICAL-RISKS.md); SDP success or an `ontrack` callback alone is not proof of playback. -Known review issue: the Console's default realtime WHEP preview can report `No advancing media received (check codec support and keyframes)` while it waits for the next H.264 keyframe after `LiveCursor`; a long GOP can outlast the 8-second watchdog. WHEP Live and the current H.264 browser path decode successfully, but the default behavior, write-error diagnostics, and real GB28181/SIP H.264 browser coverage remain open. See [the technical risk record](docs/TECHNICAL-RISKS.md); SDP success or an `ontrack` callback alone is not proof of playback. +WHEP status also exposes `source_overwrites`. This is the number of source-ring positions lost during recovery and is intentionally separate from `dropped_video` and `dropped_audio`, because a mixed source ring cannot attribute each lost position to one media kind. Direct audio pacing is reset at the same recovery boundary; transformed target-audio pacing remains independent. **GB28181:** Configure your IP camera's SIP server to point at `localhost:5060`, or use the built-in simulator: @@ -310,6 +326,7 @@ The tabs, in order, are Streams, GB28181, Config, Cluster, SIP Calls, Storage, a - SIP and GB28181 local protocol Test Lab results, including unavailable-module states; both provider sessions can publish or receive persistent H.264 plus G.711 loopback media, show per-track RTP/RTCP/PS counters, stop cleanly, and preview through the available output protocols DVR playlist and segment GETs run synchronous subscribe authorization hooks only; they do not emit asynchronous subscribe lifecycle events. +Finite DVR playlist and segment responses use a 10-second server write bound. Every admitted success, error, canceled, or timed-out request releases exactly one global connection slot; range requests and `ServeContent` metadata are unchanged. Recording preview uses the authenticated management API session. DVR preview uses the separate `dvr.listen` HLS listener with non-credentialed CORS, so its subscribe authorization still applies; the Console does not persist or append bearer tokens. ## Configuration @@ -339,14 +356,16 @@ Key sections: | `metrics` | Prometheus metrics endpoint (default `:9090`) | | `limits` | Global connection, stream, and subscriber limits | | `tls` | TLS certificate and key for HTTPS/secure protocols | -| `stream` | GOP cache, ring buffer, idle timeout, slow consumer, feedback; Simulcast fields are deferred | +| `stream` | GOP cache and per-GOP frame/duration/byte bounds, ring buffer, idle timeout, slow consumer, feedback; Simulcast fields are deferred | | `runtime` | Background configuration refresh source: file, HTTP/HTTPS, Consul, or Redis | -Environment variable expansion is supported: `${API_TOKEN}`, `${AUTH_JWT_SECRET}`. +Trusted bootstrap/runtime source loading supports environment variable expansion such as `${API_TOKEN}` and `${AUTH_JWT_SECRET}`. Viewer-facing Config Validate never expands the server process environment: it treats references literally, accepts exactly one YAML/JSON document, and rejects unknown root or nested typed fields. Config Apply and trusted runtime source loading remain permissive for source fields not mapped by the typed runtime struct. ### Runtime configuration refresh -The bootstrap file is loaded once. A background manager then polls the selected `runtime.source` and atomically publishes validated snapshots. Application reads use the in-memory snapshot only, so they never block on file or network I/O. Source loads, Config Apply writes, and source close are serialized; Apply waits for the source write before returning 202 and schedules parsing/application/publication asynchronously. The Config page shows the complete versioned JSON Schema and retains raw desired source YAML, including comments and fields not represented by the typed runtime struct. Source failures retain the last valid snapshot. For HTTP sources, the selected `http` or `https` source must match the URL scheme, redirects are disabled, and ETag/Last-Modified validators advance only after a document is accepted; `X-Config-Version` is separate version metadata. `SIGHUP` and `POST /api/v1/server/config/refresh` schedule asynchronous refresh; listener/module/TLS/port changes are reported as restart-required and are not partially applied. Status and Prometheus expose accepted, rejected, application-failed, callback-failed, coalesced callback, and pending-restart state. See [`docs/recipes/runtime-config-sources.md`](docs/recipes/runtime-config-sources.md) for file, HTTP, HTTPS, Consul, Redis, Config Validate, and Config Apply examples. +The bootstrap file is loaded once. A background manager then polls the selected `runtime.source` and atomically publishes validated snapshots. Application reads use the in-memory snapshot only, so they never block on file or network I/O. Source loads, Config Apply writes, and source close are serialized; Apply waits for the source write before returning 202 with `written_and_refresh_scheduled` and schedules parsing/application/publication asynchronously. Every file, HTTP/HTTPS, Consul, and Redis source defaults to a 4 MiB complete-document/materialization limit, configurable with `runtime..max_bytes`; Redis hash reads prefer `HSCAN NOVALUES` and use bounded `HKEYS` fallback only for older servers. Flattened Consul/Redis leaves infer only safe booleans, nulls, canonical decimal integers, and finite decimal/exponent floats; leading-zero identifiers, durations, out-of-range values, and YAML-looking strings remain strings. Dotted/slashed flattened paths are canonicalized and sorted; duplicate paths and scalar/container prefix collisions fail closed deterministically. The Config page shows the complete versioned JSON Schema and retains raw desired source YAML, including comments and fields not represented by the typed runtime struct. Its redacted document preserves collection shape: opaque structured sensitive values retain only explicit stable identity fields such as `id`, `name`, `username`, `channel_id`, and `device_id`; valid absolute hierarchical URL scalars are recognized by value even under unmapped non-URL-shaped keys and retain safe scheme/host/port identity while replacing every non-root path with a stable opaque digest marker and removing userinfo/query/fragment. URL-shaped keys retain TURN/opaque, malformed/hostless fail-closed, and plain-address handling; ordinary strings, durations, IDs, and bare host/address values remain unchanged outside that key policy. Ambiguous restoration fails closed. Source failures retain the last valid snapshot. For HTTP sources, the selected `http` or `https` source must match the URL scheme, redirects are disabled, and ETag/Last-Modified validators advance only after a document is accepted; `X-Config-Version` is separate version metadata. Consul KV GET and PUT also reject redirects without dispatching to the target, so `X-Consul-Token` is never forwarded. `SIGHUP` and `POST /api/v1/server/config/refresh` schedule asynchronous refresh; listener/module/TLS/port changes are reported as restart-required and are not partially applied. Status and Prometheus expose accepted, rejected, application-failed, callback-failed, coalesced callback, and pending-restart state. See [`docs/recipes/runtime-config-sources.md`](docs/recipes/runtime-config-sources.md) for file, HTTP, HTTPS, Consul, Redis, Config Validate, and Config Apply examples. + +File Apply creates new targets with private mode `0600` and preserves the existing file's permission bits during atomic replacement. Redis Apply writes the document and optional version increment in one `MULTI/EXEC` transaction; transaction errors are returned rather than producing a false success. The refresh response is `202` with `status: scheduled`, while a successful Apply is `202` with `status: written_and_refresh_scheduled`. The Console tracks a monotonic editor revision so a newer local edit cannot be overwritten by a stale desired snapshot after Apply. Operators can inspect the redacted loader state at `GET /api/v1/server/config` (protected by the normal API authentication rules). diff --git a/README.zh-CN.md b/README.zh-CN.md index d9f6273a..aad9f43a 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -42,6 +42,8 @@ LiveForge 是一个模块化的直播流媒体服务器，支持实时音视频 - **多协议推流** — RTMP、RTSP（TCP + UDP，兼容符合会话条件的独立音视频轨 SETUP）、SRT、WebRTC WHIP、GB28181，兼容 OBS、FFmpeg、GStreamer 及浏览器 - **多协议拉流** — RTMP、RTSP、SRT、WebRTC WHEP、HLS、LL-HLS、DASH、HTTP-FLV、HTTP-TS、FMP4、WebSocket +- **连续 HTTP 流完整性** — HTTP-FLV、HTTP-TS、FMP4 及其 WebSocket 输出在 ring overwrite 时立即终止，不会发送保留的 gap 后数据来跨越媒体断点 +- **分片 overwrite 处理** — HLS 和 LL-HLS 丢弃未完成媒体，按需重新打开刷新后的直接/转码音频源，并以一次 discontinuity 从 live 位置恢复（视频等待关键帧，纯音频立即恢复）；LL-HLS 已保留的 fMP4 媒体继续引用匹配且不可变的版本化 init，DASH 则保留已完成的单 Period 媒体并退休受影响 manager - **SRT** — 安全可靠传输，AES 加密，低延迟 MPEG-TS 传输（纯 Go 实现 `datarhei/gosrt`） - **WebRTC** — WHIP/WHEP（SDP offer 上限 1 MiB）、ICE Lite、GCC 发送端带宽估计、浏览器推流 - **编解码** — H.264、H.265/HEVC、VP8、VP9、AV1、AAC、Opus、G.711（μ-law/A-law）、MP3 @@ -106,12 +108,14 @@ go run ./tools/gb28181-sim \ - **HTTP 调度器** — 通过外部 HTTP 回调动态解析目标节点，或使用静态目标列表 - **拓扑模式** — 单层（Origin-Edge）、多边缘（Origin-Multi-Edge）、三级级联（Origin-Center-Edge） - **重试与容错** — 可配置重试次数、间隔和退避 -- **转发热路径** — Relay 和 WHEP reader 使用独立阻塞等待；RTMP 转推复用 FLV 编码缓冲区，RTSP interleaved 使用向量写入，relay 字节指标首包即时提交、后续批量更新，降低逐帧开销 +- **转发热路径** — Relay reader 使用独立阻塞等待；WHEP 为 source 和 target-audio reader 各使用一个 condition-backed pump，使 readiness 与原子读取不会并发竞争；RTMP 转推复用 FLV 编码缓冲区，RTSP interleaved 使用向量写入，relay 字节指标首包即时提交、后续批量更新，降低逐帧开销 > 详见 [Wiki: 集群部署](../../wiki/Cluster-Deployment-zh)。 可用以下命令测量转发热路径：`go test -bench='BenchmarkRingReader|BenchmarkRTMPConn' -benchmem ./pkg/util ./module/cluster`。基准结果取决于运行机器，不代表固定容量保证。 +可用 `go test -run '^$' -bench 'BenchmarkStreamIngressProduction|BenchmarkRTMPRelaySendMediaFrameProduction|BenchmarkRTSPRelaySendFrameProduction|BenchmarkRelayObservationAccounting' -benchmem -count=3 ./core ./module/cluster` 测量更接近生产路径的回归基准，其中包括稳定 publisher 校验、Stream ring/GOP 写入、完整 RTMP FLV/chunk framing、RTSP H.264 packetizer/RTP/interleaved framing 和有界 relay 字节统计。在 Apple M1 Pro、Go 1.26.0 上，三次结果为：稳定 Stream ingress 65.86-67.28 ns/op（29 B/op，0 allocs/op），RTMP H.264 155.1-155.6 ns/op（24 B/op，3 allocs/op），RTMP AAC 73.60-73.76 ns/op（21 B/op，3 allocs/op），RTSP 单 NAL H.264 1.825-1.833 us/op（4,044 B/op，9 allocs/op），RTSP 三包 FU-A H.264 4.593-4.605 us/op（9,892 B/op，23 allocs/op）。Stream 使用共享只读 payload 的预分配 64 秒单调时间戳、25 fps H.264/50 fps G.711A 帧池，零 subscriber，关闭 bitrate limit，保留 2 个 GOP，单 GOP 上限 300 帧，ring 为 4,096 项。RTMP 使用固定时间戳媒体帧和 payload 字节统计，RTSP 使用固定时间戳 RTP 输入和 framed-byte 统计；两种 egress 都终止于有界内存 writer，不包含 socket write、TCP writev、deadline 和内核/网络 syscall。独立 accounting 的 ns/op 还排除了生产 context lookup，主要用于 allocation 回归。该 fixture 与更窄的旧 `BenchmarkStreamWriteFrame` 微基准不可直接比较，也不代表订阅数、并发或部署容量。 + ### LL-HLS（低延迟 HLS） Apple LL-HLS 标准实现，亚秒级延迟 HLS 分发： @@ -123,25 +127,35 @@ Apple LL-HLS 标准实现，亚秒级延迟 HLS 分发： - **fMP4 分片解析** — 可解析由多个 `moof`/`mdat` fragment 拼接成的完整媒体分片，不会丢弃前面的 fragment - **fMP4 AAC 时间** — 未显式提供 AAC 采样率和声道数时从 AudioSpecificConfig 推导，并复用解析出的采样率作为媒体 timescale，保持 DTS 间隔稳定 - **兼容旧播放器** — 无 LL-HLS 支持的播放器自动降级为缓冲分片模式 -- **关键帧对齐启动** — GOP 缓存与实时帧保持连续；HLS、LL-HLS 和 DASH 分段器会等待当前 publisher generation 的必要序列头，并从同一个启动快照绑定序列头、回放帧和实时游标。Hls.js 的初始清单等待一个完整分段但不重复公告其 part。等待上限覆盖配置的完整分段目标加一个 part（下限 10 秒、上限 30 秒）；若仍无完整分段则返回 503，而不是只包含 part 的清单。后续阻塞刷新保留最近已完成 part 的身份并继续消费新的低延迟 part；DASH 同样在一个完整分段后启动、采用一个 fragment 的直播延迟，且 MPD 最迟每两秒刷新。HLS、LL-HLS 和 DASH 清单会逐段转义流键，DASH URL 属性同时进行 XML 转义，媒体分片路由可保留任意深度的有效流键 +- **关键帧对齐启动** — GOP 缓存与实时帧保持连续；HLS、LL-HLS 和 DASH 分段器会等待当前 publisher generation 的必要序列头，并从同一个启动快照绑定序列头、回放帧和实时游标。Hls.js 的初始清单等待一个完整分段但不重复公告其 part。等待上限覆盖配置的完整分段目标加一个 part（下限 10 秒、上限 30 秒）；若仍无完整分段则返回 503，而不是只包含 part 的清单。后续阻塞刷新保留最近已完成 part 的身份并继续消费新的低延迟 part；DASH 同样在一个完整分段后启动、采用一个 fragment 的直播延迟，且 MPD 最迟每两秒刷新。清单和分片只在真正写响应前启动 write deadline，等待首段不会提前消耗写窗口。HLS、LL-HLS 和 DASH 清单会逐段转义流键，DASH URL 属性同时进行 XML 转义，媒体分片路由可保留任意深度的有效流键 +- **generation 安全完成** — Publisher stop 会先从新请求查找中移除匹配的 HLS、DASH 或 LL-HLS manager，但 manager 会继续排空该 generation 捕获的结束游标之前已经接纳的全部帧，并且只完成一次。替代 publisher 使用不同 manager，不会把新旧 generation 帧写入对方输出。LL-HLS 阻塞刷新也会在 manager 停止时退出；HTTP 模块关闭会强制停止并等待 active 和 draining manager worker ### 管理与运维 - **Web 控制台** — 七个权限感知标签页及多协议预览和 WHIP 推流：Streams, GB28181, Config, Cluster, SIP Calls, Storage, and Security。Recent Audit 是 Security 内部的界面，不是单独的第八个标签页。 - **REST API** — 流生命周期、配置刷新/状态、集群状态、SIP 呼叫、录制/DVR、安全/审计、GB28181 和公开健康探针 - **鉴权与 RBAC** — viewer/operator/admin 命名令牌、控制台会话、推拉流 JWT/回调鉴权，以及有界脱敏审计记录 -- **录制与 DVR** — FLV、FMP4、MP4、MPEG-TS、HLS 录制；新录像默认使用 fMP4/`.mp4`；fMP4 仅直接写入 AAC，启用可选 `audiocodec`/FFmpeg 构建时会将 G.711、Opus、MP3 等非 AAC 音频转为 AAC，未启用时过滤音频并保留可播放的纯视频输出；转码录制停止时会先排空停止边界前已经提交的源帧再完成文件；DVR TS 同样会将目标不支持的音频统一转换；支持分段、存储健康、下载/Range/在线预览/删除管理、零字节会话保护和时移状态 -- **本地协议实验室** — SIP 和 GB28181 页面可在不依赖其他平台或设备的情况下运行一次性及持久假设备检查。SIP 使用独立的 H.264 与 PCMA/PCMU RTP/RTCP 轨道，接收模式不会改写源流；接收模式会在发送信令前等待当前 publisher generation 的必要序列头，已知不支持的音频编码会立即拒绝。GB28181 发布模式注册一个可监听的假设备，并经过 LiveForge 正常的服务端主动点播及真实 RTP/RTCP 接收路径；接收模式先校验 H.264 加 G.711A，并在模块自己的 PS/RTP/RTCP 出站会话激活前同步接纳源流订阅者。订阅者上限拒绝会让启动同步失败；后续媒体发送失败会把 Lab 转为 `failed` 并释放信令及媒体资源。无外部依赖的 160x90 动态测试图以 25fps 运行、每秒一个 IDR，并生成可听的 20ms 音频帧。持久 GB28181 会话会按 `gb28181.keepalive.timeout` 的约三分之一持续发送 Keepalive。两者共用 SIP 监听端口时，H.264 加 PCMA/PCMU RTP offer 交给 SIP Gateway，PS/90000 offer 交给 GB28181 +- **录制与 DVR** — FLV、FMP4、MP4、MPEG-TS、HLS 录制；新录像默认使用 fMP4/`.mp4`；每个轮转录像文件都会保留已声明轨道和最新 codec 初始化，并让每条轨道从文件内零时间轴开始，确保文件可独立解析；TS 会在首个媒体 PES 前写入 PAT/PMT，经典 MP4 按音视频各自时钟计算 duration、对超出 version-0 表示范围的时间字段做饱和而不回绕、对负 B 帧合成偏移使用有符号 `ctts` version 1，并使用可扩展 AAC ESDS 长度编码；fMP4 仅直接写入 AAC，启用可选 `audiocodec`/FFmpeg 构建时会将 G.711、Opus、MP3 等非 AAC 音频转为 AAC，未启用时过滤音频并保留可播放的纯视频输出；转码录制停止时会先排空停止边界前已经提交的源帧再完成文件，publisher generation 结束时还会先排空重采样滤波器保留的样本，再用静音补齐最后一个不完整 PCM 帧，并在 Record/DVR 输出关闭前仅一次排空编码器延迟包；DVR TS 同样会将目标不支持的音频统一转换；支持分段、存储健康、下载/Range/在线预览/删除管理、精确完整 ID 操作路由、清理失败后可重试且最后删除主文件、零字节会话保护和时移状态。录制在线预览/下载会在打开媒体前占用全局连接配额，并设置 10 秒写期限。 +- **录制/DVR 状态与路由** — 只有 `completed` 录像可以下载或在线播放；`active` 和 `failed` 状态统一返回 JSON `409`，不会返回媒体字节。录制格式为 `flv`、`fmp4`、`mp4`、`ts` 和 `hls`（`hls` 按 TS 存储），`max_size` 只接受空值/零值或带 `B`/`KB`/`MB`/`GB` 的十进制字节数。纯音频 DVR 在 publisher 仍在线时按音频 DTS 达到分段时长立即发布分片。DVR 嵌套流键保留 `/` 层级，拒绝编码分隔符和点段，并对每个流键段独立转义 `?`、`#`、`%`；`/api/v1/server/info` 返回已绑定的非零 DVR 端口及实际 HTTP/TLS scheme。 +- **本地协议实验室** — SIP 和 GB28181 页面可在不依赖其他平台或设备的情况下运行一次性及持久假设备检查。SIP 使用独立的 H.264 与 PCMA/PCMU RTP/RTCP 轨道，接收模式不会改写源流；接收模式会在发送信令前等待当前 publisher generation 的必要序列头，并把所选 PCMA/PCMU 作为真实出站目标 codec。源 codec 不同时，带标签且具备能力的运行时使用 generation 绑定的共享音频转码 reader，H.264 仍从原始 live cursor 读取；不支持的转换会立即拒绝。GB28181 发布模式注册一个可监听的假设备，并经过 LiveForge 正常的服务端主动点播及真实 RTP/RTCP 接收路径；接收模式要求 H.264 加直接 G.711A，或带标签运行时可转换为 G.711A 的音频；转码音频使用独立且绑定 generation 的 reader，H.264 保持直接读取，并在模块自己的 PS/RTP/RTCP 出站会话激活前同步接纳源流订阅者。订阅者上限拒绝会让启动同步失败；后续媒体发送失败会把 Lab 转为 `failed` 并释放信令及媒体资源。无外部依赖的 160x90 动态测试图以 25fps 运行、每秒一个 IDR，并生成可听的 20ms 音频帧。持久 GB28181 会话会按 `gb28181.keepalive.timeout` 的约三分之一持续发送 Keepalive。两者共用 SIP 监听端口时，H.264 加 PCMA/PCMU RTP offer 交给 SIP Gateway，PS/90000 offer 交给 GB28181 +- **协议实验室接纳上限** — `sip.gateway.max_lab_sessions` 和 `gb28181.max_lab_sessions` 分别限制持久实验室的活跃会话；默认值为 16，终态历史不占用上限，非正值使用默认值，达到上限时会在分配 socket 或媒体资源前返回 HTTP 429 +- **SIP RTP 端口所有权** — Gateway 媒体端口会跳过外部占用并在 SDP 协商期间保持 socket 已绑定；Lab 假端点同时避开 Gateway 配置的 RTP 范围 +- **SIP 出站退役** — 请求的 PCMA/PCMU 转换使用独立且绑定 publisher generation 的音频 reader。每个就绪帧会先完成 packetize，再在终态 send gate 下进行最终发送准入，同时复查取消状态和当前 publisher generation。终态清理先关闭准入和自有 socket，在不持有 lifecycle 或 admission 锁时等待已准入发送退出，之后才发布终态与回调；publisher 退役会释放转码 reader 和订阅者、回收 RTP/RTCP 端口对，并在并发触发其他清理时仍只发送一个 BYE +- **SIP overwrite 恢复** — SIP 出站会丢弃跨越媒体断点的保留帧，并且只推进发生 overwrite 的 source 或 target-audio reader。source 断点不会中断有效转码音频，直接 H.264 会等待同一 generation 的最新序列头和 IDR；target-audio 断点不会中断直接视频，音频会从 live 位置恢复。generation 仍活跃时 target-audio EOF 会让呼叫以 `network_lost` 失败，双 reader 父循环会在返回前取消并等待两个媒体 pump 退出 - **协议实验室流键** — SIP 和 GB28181 接受最长 256 字节的可打印 ASCII 流键；以 `/` 分隔的每一段都不能为空，也不能是 `.` 或 `..`。GB28181 发布仅对 loopback 模拟器使用请求中的流键，真实设备仍使用 `{stream_prefix}/{channel_id}` - **GB28181 PS 兼容性** — PS 出站会把内部 AVCC/HVCC 视频样本转换为 Annex-B，保证真实 GB28181 接收端能解码视频 +- **GB28181 覆盖恢复** — PS/RTP 出站会在发送待发媒体前串行处理源与转码音频 control result，丢弃被覆盖值和 gap 前待发媒体，只推进发生覆盖的 reader，并保持未受影响媒体连续。源 gap 会在不重置 RTP 序列号的前提下创建新 PS 状态，等到 gap 后最新序列头加 IDR 才恢复 H.264；转码音频 gap 则保留干净的视频和 PS 状态，也不会重置其原有的 20ms holdback deadline - **实验室诊断** — Manager 保留全部活跃会话和最新 16 条终态记录。失败会话的有界 `last_error` 会先移除 SIP 凭据与 bearer token；会话视图展示接收端 RTCP 及独立音视频计数。播放路径会逐段转义流键，并按实际绑定监听器生成 RTMP/RTSP 绝对地址；Console 的 Lab Preview 直接使用这些返回路径 - **启动回滚** — 监听器或模块初始化失败时保留并报告原始错误，只关闭已经尝试初始化的模块，不会在回滚尚未初始化的后续模块时 panic - **通知** — HTTP Webhook（HMAC-SHA256 签名）和 WebSocket 实时事件 -- **Prometheus 监控** — 服务器级和流级指标：连接数、码率、帧率、GOP 缓存、各协议订阅者数 -- **限流** — IP 级令牌桶，防止连接洪泛 +- **Prometheus 监控** — 启用模块后始终提供服务器级指标，流级码率、帧率、GOP 和订阅者 label 默认关闭。未配置 allowlist 时，数量上限是单个 Collector 整个生命周期的 cardinality 预算：活跃流键按创建顺序接纳，流消失后仅保留标量键且槽位不因 churn 复用。精确 allowlist 定义唯一可选流键，上限仍约束每次抓取。`stream_detail_limit: 0` 会关闭流级 series；负数配置无效并会被拒绝。需要查看当前流请使用管理 API，需要固定 Prometheus label 请配置精确 allowlist +- **限流** — IP 级令牌桶，防止连接洪泛；可信代理链从右向左解析，攻击者控制的 XFF 左侧前缀不能切换限流桶 +- **HTTP 连接超时** — API、WebRTC 信令和 metrics 监听器将请求头解析限制为 5 秒，将空闲 keep-alive 连接限制为 2 分钟；现有写入 deadline 保持不变 - **慢消费者保护** — 基于 EWMA 的延迟检测，渐进式丢帧 - **GCC 拥塞控制** — WebRTC WHEP 发送端带宽估计，自适应码率 -- **按 generation 绑定起播** — SIP、GB28181、录制、DVR 和集群出站使用同一个 publisher 原子快照，只在协议需要时重放当前 headers/GOP 一次，再从 live cursor 接续。SIP inbound INVITE 会在分配 RTP 端口前执行同步发布鉴权，激活后发送匹配的 start/stop 生命周期事件，因此录制和 DVR 能跟随并收尾 SIP 会话。publisher 替换会取消旧 reader，纯音频不会重放保留历史，只有 sequence header 的录制会失败而不会发布为成功媒体 +- **按 generation 绑定起播** — SIP、GB28181、录制、DVR 和集群出站使用同一个 publisher 原子快照，只在协议需要时重放当前 headers/GOP 一次，再从 live cursor 接续。DVR 会将已校验快照贯穿保留索引/存储恢复，并在安装 session 前再次检查 generation；设置期间发生替代时会丢弃候选 session。DVR shutdown 会在等待 setup 所有权之前启动绝对 drain deadline，因此阻塞的 setup 不能延长配置的关闭边界。SIP inbound INVITE 会在分配 RTP 端口前执行同步发布鉴权，激活后发送匹配的 start/stop 生命周期事件，因此录制和 DVR 能跟随并收尾 SIP 会话。publisher 替换会取消旧 reader，纯音频不会重放保留历史，只有 sequence header 的录制会失败而不会发布为成功媒体 +- **Publisher 所有权隔离** — 每个非空 publisher ID 在一个 `Stream` 对象生命周期内只能创建一个 generation。即使中间出现 B，再次使用 A 也会在任何流状态变化前被拒绝，因此 A 的延迟帧、活动和清理回调不能影响当前 owner；新创建的 `Stream` 拥有独立的 identity 生命周期。流一旦开始销毁，延迟清理不能把它恢复为可挂接状态，也不能重开已关闭的 ring +- **GOP 上限热更新** — 收紧帧数、时长或字节上限会保留所有启用上限共同允许的最短关键帧起始可播放前缀，并可能立即封存；时长按观测到的 DTS 最小值与最大值之间的完整无序跨度计算，不重排媒体。启用 GOP 缓存时至少要保留一个正的帧数或字节硬上限，零只禁用对应上限。放宽后只有当前保留 GOP 能接纳后续交错音视频帧，旧 GOP 保持裁剪，已省略帧不会恢复，下一个关键帧会开始新的完整 GOP ## 架构 @@ -258,9 +272,11 @@ ffmpeg -re -i input.mp4 -c copy -f mpegts "srt://localhost:6000?streamid=publish **WebRTC（浏览器）：** 打开 `http://localhost:8090/console`，点击 **"+ WebRTC Publish"**，选择摄像头/麦克风后开始推流。 -当浏览器和操作系统提供 H.265 WebRTC 编码器时，控制台可以推送 H.265/HEVC 视频和 Opus 音频。WHIP 会把音频和视频 RTP 映射到同一个会话时间线，HLS/DASH/FLV/TS 使用从缓存 GOP 源游标开始的组合转码 reader，让目标音频历史和实时视频连续进入输出，避免首帧冻结和重复缓存视频。FMP4 预览在共享 muxer 启动时建立接近零的时间线并保留 B 帧的有符号合成偏移，晚加入的订阅从自身首个缓冲时间戳开始播放。WHEP Live 回放原子缓存 GOP 后，从与快照匹配的 ring 游标继续读取源视频，并通过独立 reader 获取转码后的目标音频。WebRTC 转码 worker 等待新帧时不会消费源播放唤醒信号，因此即使源音频暂停，视频节奏也能保持稳定。带 `audiocodec` 标签的构建是完整跨协议配置，验收步骤见 [WHIP H.265 + Opus 播放验证](docs/recipes/whip-h265-opus-playback.md)。 +当浏览器和操作系统提供 H.265 WebRTC 编码器时，控制台可以推送 H.265/HEVC 视频和 Opus 音频。WHIP 会把音频和视频 RTP 映射到同一个会话时间线，HLS/DASH/FLV/TS 使用从缓存 GOP 源游标开始的组合转码 reader，让目标音频历史和实时视频连续进入输出，避免首帧冻结和重复缓存视频。FMP4 预览在共享 muxer 启动时建立接近零的时间线并保留 B 帧的有符号合成偏移，晚加入的订阅从自身首个缓冲时间戳开始播放。对于 G.711 源，只有当 `GET /api/v1/server/info` 报告当前进程已配置且实际具备两种 G.711 到 AAC 的转码能力时，控制台才会在 FMP4 SourceBuffer 中声明 AAC；便携构建仍使用纯视频声明。WHEP Live 回放原子缓存 GOP 后，从与快照匹配的 ring 游标继续读取源视频，并通过独立 reader 获取转码后的目标音频。WebRTC 转码 worker 等待新帧时不会消费源播放唤醒信号，因此即使源音频暂停，视频节奏也能保持稳定。带 `audiocodec` 标签的构建是完整跨协议配置，验收步骤见 [WHIP H.265 + Opus 播放验证](docs/recipes/whip-h265-opus-playback.md)。 + +控制台默认的 WHEP 预览使用带缓存的 live 启动路径，正常 H.264 GOP 不必等待 snapshot 之后的 IDR。协议实验室分别返回 `whep`/`whep_live`（`mode=live`）和 `whep_realtime`（`mode=realtime`）路径。源中存在且 offer 实际请求的每条音视频轨都必须成功协商：不支持的请求 codec 返回 415，内部建轨失败返回 500，不能静默只保留另一条轨；端口为 0 或方向不接收的 m-line 仍视为有意省略。接收方向优先使用媒体级属性，否则继承会话级属性；codec 只与该 m-line 实际列出的 payload 的精确 `rtpmap` 名称匹配。显式 realtime 模式会显示可区分的“等待关键帧”状态；即使混合流音频已推进，只要视频仍在首个 IDR 前丢弃非关键帧，就不会误报 `media_stalled`。播放期间，WHEP 会把源 reader 或转码音频 reader 的覆盖事件绑定到各自的原子读取结果，丢弃覆盖后的保留帧，并且只把受影响的 reader 推进到 live。每个 reader 的 readiness、原子读取和 live 推进都由同一个 condition-backed pump 独占；关闭时先取消并等待两个 pump，再且仅一次释放转码音频所有权。源 reader 覆盖时，已建立的音频继续推进，视频回到 `waiting_keyframe`，重置 pacing/DTS/PTS 状态，并从同一 generation 的最新参数集加关键帧恢复；纯音频从下一帧 live 音频继续。目标音频 reader 覆盖不会扰动干净的视频；期望的目标音频在 active generation 中 EOF 时会以 `target_audio_failed` 终止，而不是静默降级为纯视频。`GET /webrtc/session/{sessionId}/status` 提供期望媒体种类、首个成功样本时间和固定的 `first_media_wait_ms`、每种媒体最后推进时间、generation、游标、媒体计数、真实 RTP 包/字节、收到的 RTCP 包和有界 sample-write 错误。dropped 只统计已协商轨；会话关闭会在保存终态前捕获一次最终的单调 transport 计数。所有请求轨都推进后才能进入 `playing`；启动后任一期望媒体连续 8 秒没有推进会进入可恢复的 `media_stalled`，所有过期媒体重新推进后才恢复；Console 使用服务端时间只列出实际过期的媒体种类。每次真实状态迁移只写一条结构化日志，包含 generation、游标、模式、前后状态以及存在时的有界错误；每次覆盖另写一条有界 warning，包含 reader 身份、精确覆盖计数和恢复动作。Feed 终止会自动关闭并释放会话，最多 64 条终态仍可读取两分钟。带标签的 Chromium 矩阵会验证 SIP 发布到 GB28181 接收加 WHEP、GB28181 发布到 SIP 接收加 WHEP，以及 WHIP H.264/Opus 发布到 SIP 与 GB28181 接收加 WHEP。验收要求预期解码尺寸、媒体时间、视频/音频 RTP 和解码帧计数持续推进、ICE 已连接且服务端 RTP/RTCP 状态未 stalled；设置 `LIVEFORGE_PROTOCOL_MATRIX_SOAK=60s` 可按秒延长检查，但不代表部署容量结论。详见[技术风险记录](docs/TECHNICAL-RISKS.md)；SDP 协商成功或触发 `ontrack` 都不能单独证明已经播放。 -当前已确认一个待关闭问题：控制台默认的 realtime WHEP 在 `LiveCursor` 之后等待下一个 H.264 关键帧，长 GOP 可能超过 8 秒 watchdog，从而显示 `No advancing media received (check codec support and keyframes)`。WHEP Live 和当前 H.264 浏览器路径可以正常解码，但默认行为、写入错误诊断以及真实 GB28181/SIP H.264 浏览器覆盖仍需补齐。详见[技术风险记录](docs/TECHNICAL-RISKS.md)；SDP 协商成功或触发 `ontrack` 都不能单独证明已经播放。 +WHEP 状态还提供 `source_overwrites`，表示恢复期间 source ring 丢失的 position 数。它不会计入 `dropped_video` 或 `dropped_audio`，因为混合源 ring 无法把每个丢失 position 可靠归类到单一媒体类型；直接音频 pacing 会在同一恢复边界重置，转码后的 target-audio pacing 保持独立。 **GB28181：** 将 IP 摄像头的 SIP 服务器指向 `localhost:5060`，或使用内置模拟器： @@ -311,6 +327,7 @@ go run ./tools/gb28181-sim -server 127.0.0.1:5060 - SIP 和 GB28181 本地协议实验室结果，以及模块不可用状态；两者都支持无需外部平台的持久 H.264 加 G.711 模拟设备发布/接收，会话显示分轨 RTP/RTCP/PS 计数，停止时清理资源，并可通过已启用的其他输出协议预览 DVR 播放列表和分片 GET 只运行同步订阅鉴权钩子，不会触发异步订阅生命周期事件。 +有限的 DVR 播放列表和分片响应使用 10 秒服务端写入上限。每个已接纳的成功、错误、取消或超时请求都只释放一个全局连接槽位；Range 请求和 `ServeContent` 元数据保持不变。 录制预览复用已认证的管理 API 会话；DVR 预览使用带非凭据 CORS 的独立 `dvr.listen` HLS 监听器，因此仍执行订阅鉴权，控制台不会持久化或拼接 bearer token。 ## 配置 @@ -340,14 +357,16 @@ LiveForge 使用 bootstrap YAML 配置，并可通过 runtime source 持续读 | `metrics` | Prometheus 监控端点（默认 `:9090`） | | `limits` | 全局连接数、流数、订阅者数限制 | | `tls` | TLS 证书和密钥配置 | -| `stream` | GOP 缓存、环形缓冲区、空闲超时、慢消费者、反馈；Simulcast 字段仍延期 | +| `stream` | GOP 缓存及单 GOP 帧数/时长/字节上限、环形缓冲区、空闲超时、慢消费者、反馈；Simulcast 字段仍延期 | | `runtime` | 后台配置刷新源：文件、HTTP/HTTPS、Consul 或 Redis | -支持环境变量展开：`${API_TOKEN}`、`${AUTH_JWT_SECRET}`。 +受信任的 bootstrap/runtime source 加载支持 `${API_TOKEN}`、`${AUTH_JWT_SECRET}` 等环境变量展开。面向 viewer 的 Config Validate 绝不会展开服务端进程环境变量，而是按字面值处理引用，只接受一个 YAML/JSON 文档，并拒绝 root 或 nested typed field 中的未知键。Config Apply 和受信任的 runtime source 加载仍允许 typed runtime struct 未映射的 source 字段。 ### 运行时配置刷新 -进程启动时只读取一次 bootstrap 配置文件，之后由后台管理器定期读取选定的 `runtime.source`，解析、校验后以原子快照发布。业务读取配置只做内存中的原子读取，不会触发文件/网络 I/O，也不会等待刷新。源加载、Config Apply 写入和关闭操作会串行执行；Apply 会等待数据源写入完成后返回 202，再异步执行解析、模块应用和发布。Config 页面展示完整的版本化 JSON Schema，并保留 source 原始 desired YAML，包括注释和 typed runtime struct 未映射的字段。配置源失败时继续使用最后一次有效快照。HTTP 源要求 `runtime.source` 的 `http` 或 `https` 与 URL 协议一致，禁止所有重定向，且 ETag/Last-Modified 仅在文档被接受后推进；`X-Config-Version` 是独立的版本元数据。`SIGHUP` 和 `POST /api/v1/server/config/refresh` 只会异步调度刷新。监听地址、模块开关、TLS、端口范围等变更会标记为需要重启，不会对运行中的监听器做部分切换。状态 API 和 Prometheus 会暴露接受、拒绝、应用失败、回调失败、回调合并丢弃和待重启状态。文件、HTTP、HTTPS、Consul、Redis 以及 Config Validate/Apply 示例见 [`docs/recipes/runtime-config-sources.md`](docs/recipes/runtime-config-sources.md)。 +进程启动时只读取一次 bootstrap 配置文件，之后由后台管理器定期读取选定的 `runtime.source`，解析、校验后以原子快照发布。业务读取配置只做内存中的原子读取，不会触发文件/网络 I/O，也不会等待刷新。源加载、Config Apply 写入和关闭操作会串行执行；Apply 会等待数据源写入完成后返回 202，并返回 `status: written_and_refresh_scheduled`，再异步执行解析、模块应用和发布。file、HTTP/HTTPS、Consul、Redis 源默认都有 4 MiB 的完整文档/物化上限，可通过对应的 `runtime..max_bytes` 配置；Redis hash 优先使用 `HSCAN NOVALUES`，旧 Redis 只在明确不支持时退回 `HKEYS`，并继续使用有界的 `HSTRLEN/HGET` 批量读取。扁平化 Consul/Redis 叶子值只会推断安全的布尔值、null、规范十进制整数以及有限的十进制/指数浮点数；前导零标识符、时长、越界数值和类似 YAML 的字符串仍保持字符串。点号/斜杠扁平路径会先规范化并排序；重复路径以及标量/容器前缀冲突会以确定性的错误 fail closed。Config 页面展示完整的版本化 JSON Schema，并保留 source 原始 desired YAML，包括注释和 typed runtime struct 未映射的字段。脱敏文档会保留集合形状：不透明的结构化敏感值仅保留 `id`、`name`、`username`、`channel_id`、`device_id` 等明确的稳定标识字段；有效的 absolute hierarchical URL scalar 即使位于名称不符合 URL heuristic 的未映射字段中，也会按 value 识别，保留安全的 scheme/host/port 标识，同时把所有非 root path 替换为稳定的不透明 digest marker，并移除 userinfo/query/fragment。URL-shaped key 继续执行 TURN/opaque、malformed/hostless fail-closed 和 plain-address 规则；该 key policy 之外的普通 string、duration、ID 和 bare host/address 保持不变；占位符恢复存在歧义时会拒绝写入。配置源失败时继续使用最后一次有效快照。HTTP 源要求 `runtime.source` 的 `http` 或 `https` 与 URL 协议一致，禁止所有重定向，且 ETag/Last-Modified 仅在文档被接受后推进；`X-Config-Version` 是独立的版本元数据。Consul KV GET 和 PUT 同样拒绝重定向且不会向目标发出请求，因此绝不会转发 `X-Consul-Token`。`SIGHUP` 和 `POST /api/v1/server/config/refresh` 只会异步调度刷新。监听地址、模块开关、TLS、端口范围等变更会标记为需要重启，不会对运行中的监听器做部分切换。状态 API 和 Prometheus 会暴露接受、拒绝、应用失败、回调失败、回调合并丢弃和待重启状态。文件、HTTP、HTTPS、Consul、Redis 以及 Config Validate/Apply 示例见 [`docs/recipes/runtime-config-sources.md`](docs/recipes/runtime-config-sources.md)。 + +文件 Apply 创建新目标时使用私有权限 `0600`，替换已存在文件时保留原有权限位。Redis Apply 会在一个 `MULTI/EXEC` 事务中写入文档并递增可选的 version key，事务或 EXEC 错误会返回给调用方而不会虚报成功。刷新接口成功返回 `202` 和 `status: scheduled`；Apply 成功返回 `202` 和 `status: written_and_refresh_scheduled`。Console 使用单调递增的编辑版本号，Apply 之后返回的过期 desired 快照不能覆盖更新后的本地编辑文本。 运维人员可通过 `GET /api/v1/server/config` 查看脱敏后的加载器状态（遵循 API 的现有鉴权规则）。 diff --git a/agent-manifest.json b/agent-manifest.json index aef1b37a..3e2f189b 100644 --- a/agent-manifest.json +++ b/agent-manifest.json @@ -20,10 +20,22 @@ "open_review": { "technical_risks": "docs/TECHNICAL-RISKS.md", "webrtc_regression": { - "status": "root_cause_confirmed_fix_open", - "symptom": "Console default realtime WHEP may report No advancing media received while waiting for the next H.264 keyframe after LiveCursor", - "automated_coverage": "Pion WHEP H.264 RTP and VP8 browser playback pass; current H.264 mode=live browser playback decodes, while real GB28181/SIP H.264 browser decode remains unverified", - "do_not_close_on": ["SDP success", "ontrack callback"] + "status": "default_fixed_fail_closed_stall_overwrite_recovery_cross_protocol_browser_matrix_verified", + "symptom": "Explicit realtime WHEP may wait for the next H.264 keyframe after LiveCursor; the Console default now uses the cached live startup path", + "automated_coverage": "Pion WHEP H.264 RTP and VP8 browser playback pass; requested mixed tracks fail closed; per-kind startup, stall, overwrite recovery, terminal precedence, watchdog exit, immutable replay snapshots, and source/target reader identity have race coverage; tiny real-ring RTP tests cover retained-frame discard, established-audio continuation, fresh parameter sets, pacing/PTS reset, active target-audio EOF, and replacement-generation exclusion; a unified Chromium matrix verifies SIP publish to GB28181 receive plus WHEP, GB28181 publish to SIP receive plus WHEP, and WHIP H.264/Opus publish to SIP and GB28181 receive plus WHEP when the browser offer advertises H.264; unsupported browser H.264 capability is an explicit environment skip, while Pion negotiation coverage remains required; LIVEFORGE_PROTOCOL_MATRIX_SOAK extends per-second advancement checks", + "do_not_close_on": ["SDP success", "ontrack callback"], + "status_additions": { + "source_overwrites": "Source-ring positions lost during overwrite recovery; kept separate from dropped_video and dropped_audio because mixed source positions cannot be attributed to one media kind." + } + }, + "ARCH-033": { + "status": "closed", + "contract": "The API, WebRTC signaling, and metrics HTTP servers set ReadHeaderTimeout to 5 seconds and IdleTimeout to 2 minutes; existing handlers and write-deadline behavior remain unchanged", + "verification": "go test ./module/api ./module/webrtc ./module/metrics -run '^TestHTTPServerTimeouts$' -count=1" + }, + "media_correctness": { + "ARCH-030": {"status": "closed", "shared_transcode_history": "Internal by-value output envelopes retain the original AVFrame pointer plus a valid source-ring SourceSpan; each snapshot reader applies its own SourceCursor floor using SourceSpan.Begin, drops crossing packets, and rejects stale audio epochs. A track retains the latest eight target headers by epoch; a lagging bridge replays only the header matching each accepted payload epoch and drops an AAC payload when that bounded cache has no match", "producer_source_overwrite": "An atomic source-ring overwrite discards the retained post-gap value, suppresses clean codec-tail finalization, records the exact overwritten count as a typed track cause, and closes only that generation-bound target track"}, + "ARCH-031": {"status": "closed", "continuous_http": "HTTP-FLV, HTTP-TS, fMP4, and their WebSocket outputs terminate on direct-source, transformed-audio, or shared-output ring overwrite and discard the retained post-gap value; fMP4 also discards pending partial media on overwrite while clean completion still flushes it", "http_segmenters": "HLS and LL-HLS discard abandoned partial media, advance to live same-generation input, refresh container state, reopen at the live direct or shared transformed audio source when that plan changes, gate refreshed video topology on the next keyframe, and mark the first recovered segment or part with a discontinuity; audio-only resumes on the next live audio frame. LL-HLS retains immutable init epochs for advertised fMP4 media and serves each retained version URL until its segments are evicted. DASH preserves completed single-Period media but discards current batches and retires the manager on overwrite", "sip": "Outbound SIP carries atomic source and target-audio overwrite results independently, discards each retained post-gap value, and advances only the affected reader. Source overwrite keeps transformed audio flowing and gates direct H.264 until the latest same-generation sequence header plus IDR; target-audio overwrite keeps direct video flowing and resumes audio at live media. Final RTP admission rechecks cancellation and the current generation under the terminal send gate. Active-generation target-audio EOF fails the call as network_lost; terminal paths close admission and owned sockets, drain admitted sends before publishing terminal state or callbacks, and cancel and join both media pumps", "gb28181": "Outbound GB28181 wait-only pumps queue reader readiness while the merge owns each atomic read and drains queued control before pending output. It preserves source versus target-audio identity and exact overwrite counts, discards every retained post-gap value, and advances only the affected reader. Source overwrite clears pending video, replaces the PS muxer while preserving SSRC and monotonic RTP sequence, keeps valid target audio flowing, and gates H.264 until the latest same-generation sequence header plus IDR. Target-audio overwrite clears pending audio while preserving clean source video, PS state, and that video's original holdback deadline. Active-generation target-audio EOF fails before pending RTP, and terminal paths cancel and join both media pumps", "whep": "WHEP reads source and transformed target audio with independent atomic results, discards every retained post-gap value, and advances only the affected reader. One pump exclusively owns each reader's condition wait, atomic read, and live advance. Source overwrite preserves the original generation, resets video pacing/DTS/PTS state, enters the TrackSender keyframe gate, refreshes the latest same-generation parameter sets, and keeps established audio moving; audio-only resumes at the next live frame. Target-audio overwrite keeps clean source video continuous. Active expected target-audio EOF is terminal target_audio_failed, and close cancels and joins both reader pumps before releasing target ownership once", "rtmp": "RTMP relay aborts on packetization, marshal, write, and short-write errors", "rtsp": "RTSP relay fails closed on packetization, marshal, and write errors", "srt": "SRT relay terminates on source continuity loss", "cluster": "Cluster relay consumers fail closed on source continuity loss", "record": "Record terminates on source continuity loss without clean tail", "dvr": "DVR terminates on source continuity loss without publishing retained post-gap media"} } }, "configuration": { @@ -34,10 +46,19 @@ "read_path": "atomic immutable snapshot; no file/network I/O or blocking waits", "failure_behavior": "retain last valid snapshot and expose source status", "http_policy": "runtime.source http requires an http URL and https requires an https URL; redirects are disabled; ETag and Last-Modified validators advance only after a document is accepted", + "consul_policy": "Consul KV GET and PUT reject redirects without dispatching to the redirect target; X-Consul-Token is never forwarded", + "document_limits": {"default_bytes": 4194304, "configurable_paths": ["runtime.file.max_bytes", "runtime.http.max_bytes", "runtime.consul.max_bytes", "runtime.redis.max_bytes"], "non_positive": "selects the 4 MiB default; file and network reads reject oversized documents, while Redis flattened materialization is checked before and after bounded value reads"}, + "redis_hash_read": "prefers HSCAN NOVALUES and falls back to HKEYS only for unsupported-command or syntax errors; values use bounded HSTRLEN/HGET batches and never HGETALL", "reload": "SIGHUP schedules asynchronous refresh", - "management": "Config Console reads the complete redacted effective/desired document, retains raw source comments/unmapped fields, displays the embedded versioned JSON Schema, and validates; apply writes only when the selected source implements ConfigWriter", - "writable_sources": {"file": "atomic local replacement", "http": "authenticated HTTP PUT", "https": "authenticated HTTPS PUT", "consul": "Consul KV PUT at prefix/config.yaml", "redis": "Redis hash or prefix config.yaml write plus optional version increment"}, + "management": "Config Console reads the complete redacted effective/desired document, retains raw source comments/unmapped fields, and displays the embedded versioned JSON Schema; viewer Validate treats environment references literally, accepts exactly one YAML/JSON document, and rejects unknown typed fields; Apply and trusted runtime source loading remain permissive for unmapped source fields; secret collections preserve map/sequence shape and restore placeholders by stable identity, while ambiguous identity is rejected; Apply captures the submitted document and monotonic editor revision so a stale desired refresh cannot overwrite newer local text; apply writes only when the selected source implements ConfigWriter", + "stream_cache": {"gop_cache_max_frames": 300, "gop_cache_max_duration": "10s", "gop_cache_max_bytes": 33554432, "zero_value": "zero disables the corresponding per-GOP bound; when GOP caching is enabled with a positive gop_cache_num, at least one positive frame or byte bound is required; duration-only configuration is rejected", "duration_policy": "admission and hot trimming use the overflow-safe full unordered min/max observed DTS span while preserving insertion/media order", "direct_constructor_fallback": "an unvalidated GOP-enabled stream with no hard frame or byte bound receives a 300-frame limit", "reload": "tightening recomputes and may seal the retained current-GOP prefix; relaxation permits only future frames from that prefix and does not restore omitted or trimmed frames"}, + "trusted_proxy_policy": "Forwarded client-IP headers are accepted only when the direct peer matches limits.rate_limit.trusted_proxies; X-Forwarded-For is parsed right-to-left, trusted proxy hops are stripped, and the first untrusted hop owns the rate-limit identity; invalid IP/CIDR entries fail configuration validation", + "writable_sources": {"file": "atomic local replacement; new targets use mode 0600 and existing permission bits are preserved", "http": "authenticated HTTP PUT", "https": "authenticated HTTPS PUT", "consul": "Consul KV PUT at prefix/config.yaml", "redis": "Redis hash or prefix config.yaml write plus optional version increment in one MULTI/EXEC transaction; transaction/EXEC errors fail the write"}, "read_only_behavior": "Sources without a writer return HTTP 409 from config apply; source credentials and config secrets are never returned", + "flattened_scalar_policy": "Consul/Redis dotted or slash-separated leaves infer booleans, null, canonical base-10 integers, and finite decimal/exponent floats; leading-zero identifiers, durations, non-finite/out-of-range numbers, and YAML-looking strings remain strings", + "flattened_collision_policy": "Consul/Redis flattened keys are canonicalized and sorted; duplicate dotted/slashed paths and scalar/container prefix collisions fail closed with deterministic errors", + "apply_status": {"apply": "written_and_refresh_scheduled", "refresh": "scheduled", "apply_contract": "Apply returns 202 only after the serialized source write succeeds; refresh and publication remain asynchronous"}, + "redaction_policy": "Schema x-liveforge-secret fields, api_key, key_file, and sensitive collection descendants are redacted while collection shape and only stable id/name/username/channel_id/device_id fields are retained; valid absolute hierarchical URL scalars are recognized by value even under unmapped non-URL-shaped keys and expose safe scheme/host/port identity, but every non-root URL path is replaced with a stable opaque digest marker and userinfo/query/fragment are removed; URL-shaped keys retain TURN/opaque, malformed/hostless fail-closed, and plain-address handling; ordinary strings, durations, IDs, and bare host/address values remain unchanged outside that key policy; apply restores by stable public identity and rejects missing, ambiguous, or shape-mismatched originals", "restart_required": ["module enablement", "listener addresses", "TLS files/mode", "port ranges", "audio codec enablement"], "status_counters": ["config_changes_accepted", "config_changes_rejected", "config_changes_application_failed", "callback_failures", "dropped_callbacks"], "docs": "docs/recipes/runtime-config-sources.md" @@ -46,6 +67,11 @@ "runtime": { "go": ">=1.26", "startup_failure": "report the original listener or module error and roll back only modules whose initialization was attempted; uninitialized later modules are not closed", + "shutdown": "idempotent; stop and join alive events, close attempted modules in reverse order, then drain all accepted asynchronous hooks up to server.drain_timeout (30s fallback)", + "lifecycle_admission": "bounded per stream/client/consumer; start reserves the complete matching terminal-hook capacity, rejected starts roll back protocol resources, generation-bound subscribers cannot attach to a replacement publisher, each non-empty publisher ID is generation-unique within one Stream lifetime, and a Destroying stream cannot be revived by late publisher cleanup or admission", + "http_segment_lifecycle": "HLS, DASH, and LL-HLS publish-stop retires the exact generation manager from lookup, drains accepted frames through its captured generation end cursor, and finalizes once; active-generation transformed EOF is abnormal and never clean-flushes partial state. HLS and LL-HLS recover same-generation overwrite with a discontinuity, while DASH retires; replacement generations use distinct managers, while HTTP module shutdown force-stops and joins all active or draining managers", + "dvr_lifecycle": "DVR publish admission carries one validated publisher-generation snapshot through storage/index recovery and revalidates that same generation before installation; stale candidates close resources they acquired and cannot replace a newer session; Close starts one absolute drain deadline before waiting for admission/setup ownership, returns a timeout at that bound, and lets already-started cleanup finish in the background; finite playlist and segment writes have a 10-second server timeout and every admitted response releases exactly one global connection slot; audio-only sessions rotate and publish when audio reaches segment duration while the publisher remains online, without waiting for a video keyframe; nested stream-key routes preserve slash hierarchy, reject encoded separators and dot segments, and escape reserved characters per path segment", + "http_server_timeouts": {"servers": ["api", "webrtc", "metrics"], "read_header_timeout": "5s", "idle_timeout": "2m", "write_deadline_policy": "Existing handler and media write deadlines are unchanged; these servers do not add a WriteTimeout"}, "default_build": { "cgo": false, "tags": [], @@ -63,19 +89,20 @@ {"id": "rtmp", "direction": ["publish", "play"], "status": "stable", "port": 1935, "url_templates": ["rtmp://HOST:1935/STREAM_KEY"]}, {"id": "rtsp", "direction": ["publish", "play"], "status": "stable", "port": 8554, "url_templates": ["rtsp://HOST:8554/STREAM_KEY"], "setup_policy": "track IDs must be unique, valid, in range, and eligible for the announced or described media before transport allocation"}, {"id": "srt", "direction": ["publish", "play"], "status": "stable", "port": 6000, "url_templates": ["srt://HOST:6000?streamid=publish:STREAM_KEY", "srt://HOST:6000?streamid=subscribe:STREAM_KEY"]}, - {"id": "webrtc", "direction": ["publish", "play"], "status": "stable_with_open_console_regression", "port": 8443, "entrypoints": ["POST /webrtc/whip/{stream_key}", "POST /webrtc/whep/{stream_key}"], "audio_codecs": ["opus", "PCMA", "PCMU"], "max_sdp_offer_bytes": 1048576, "requires": ["HTTPS or a browser-compatible secure context for browser camera and microphone access"], "known_issue": "Console WHEP may report No advancing media received for real H.264 input; automated Pion/VP8 paths pass while real H.264 browser decode is under investigation"}, - {"id": "hls", "direction": ["play"], "status": "stable", "port": 8080, "url_templates": ["http://HOST:8080/STREAM_KEY.m3u8", "http://HOST:8080/STREAM_KEY/0.ts"], "startup_policy": "waits for the current publisher generation's required sequence headers and binds headers, GOP replay, and live cursor from one snapshot", "audio_only_segmentation": "elapsed media time; completed TS is available before source shutdown"}, - {"id": "ll-hls", "direction": ["play"], "status": "stable", "port": 8080, "url_templates": ["http://HOST:8080/STREAM_KEY.m3u8", "http://HOST:8080/STREAM_KEY/0.ts", "http://HOST:8080/STREAM_KEY/0.m4s"], "requires": ["llhls.enabled=true"], "startup_policy": "waits for the current publisher generation's required sequence headers and binds headers, GOP replay, and live cursor from one snapshot; initial manifest waits for one complete segment without completed PART tags; blocking reloads retain the latest completed PART identity", "segment_policy": {"part_duration": "partial segment target", "segment_duration": "completed full-segment target", "segment_duration_default_seconds": 1.0, "segment_duration_schema_minimum_seconds": 0.1, "reload": "hot"}, "audio_only_segmentation": "elapsed media time; completed TS or fMP4 is available before source shutdown"}, - {"id": "dash", "direction": ["play"], "status": "stable", "port": 8080, "url_templates": ["http://HOST:8080/STREAM_KEY.mpd", "http://HOST:8080/STREAM_KEY/audio_init.mp4", "http://HOST:8080/STREAM_KEY/a1.m4s"], "startup_policy": "waits for the current publisher generation's required sequence headers and binds headers, GOP replay, and live cursor from one snapshot", "audio_only_segmentation": "elapsed media time; audio-only MPD omits video adaptation and completed m4s is available before source shutdown"}, + {"id": "webrtc", "direction": ["publish", "play"], "status": "stable", "port": 8443, "entrypoints": ["POST /webrtc/whip/{stream_key}", "POST /webrtc/whep/{stream_key}", "GET /webrtc/session/{session_id}/status"], "audio_codecs": ["opus", "PCMA", "PCMU"], "max_sdp_offer_bytes": 1048576, "requires": ["HTTPS or a browser-compatible secure context for browser camera and microphone access"], "startup": "Console and omitted mode use WHEP live GOP replay; explicit realtime waits for the next keyframe", "negotiation": "Every source media kind requested by a non-zero receiving offer m-line must negotiate; media direction inherits session direction when absent, codec names must exactly match an rtpmap payload listed by that m-line, unsupported requested codecs return 415, internal track setup returns 500, and disabled or non-receiving source kinds remain intentionally omitted", "overwrite_recovery": "Atomic source and target-audio results preserve reader identity and exact overwrite counts; the retained result is discarded and only that reader advances to live. One pump exclusively owns each reader's condition wait, atomic read, and live advance. Source overwrite resets video pacing/DTS/PTS state, requests the TrackSender keyframe gate, refreshes latest same-generation parameter sets, and keeps established audio moving; audio-only resumes at the next live frame. Target-audio overwrite preserves clean video. Active expected target-audio EOF is target_audio_failed; close cancels and joins both reader pumps and releases target ownership once", "diagnostics": "WHEP status reports expected audio/video, first sample time and stable first_media_wait_ms, per-kind last-advance timestamps, generation, cursor, mode, recoverable no-input and media-stalled states, keyframe gate, target_audio_failed, negotiated-track media counters, actual RTP packets/bytes, received RTCP packets, codec validation, and bounded sample-write errors; real state transitions emit one structured contextual log while same-state frame updates do not; close captures one final monotonic transport snapshot; every expected kind must advance before playing and after a stall; mixed feeds remain waiting-keyframe while video interframes are discarded before the first IDR; Console names only stale expected kinds from server timestamps; terminal states reject ordinary late updates; closed sessions retain at most 64 status tombstones for up to two minutes", "verification": "The tagged Chromium matrix covers SIP publish to GB28181 receive plus WHEP, GB28181 publish to SIP receive plus WHEP, and WHIP H.264/Opus publish to SIP and GB28181 receive plus WHEP when Chromium advertises H.264; missing browser H.264 capability is an explicit environment skip, and Pion negotiation tests remain mandatory; set LIVEFORGE_PROTOCOL_MATRIX_SOAK to extend per-second advancement checks"}, + {"id": "hls", "direction": ["play"], "status": "stable", "port": 8080, "url_templates": ["http://HOST:8080/STREAM_KEY.m3u8", "http://HOST:8080/STREAM_KEY/0.ts"], "startup_policy": "waits for the current publisher generation's required sequence headers and binds headers, GOP replay, and live cursor from one snapshot; response write deadlines are refreshed immediately before writing and do not include segment readiness waits", "audio_only_segmentation": "elapsed media time; completed TS is available before source shutdown", "overwrite_recovery": "discard partial media and the retained overwrite frame, advance to live, refresh same-generation headers and TS state, then resume video at a keyframe or audio-only at the next audio frame; mark the first recovered segment with EXT-X-DISCONTINUITY"}, + {"id": "ll-hls", "direction": ["play"], "status": "stable", "port": 8080, "url_templates": ["http://HOST:8080/STREAM_KEY.m3u8", "http://HOST:8080/STREAM_KEY/0.ts", "http://HOST:8080/STREAM_KEY/0.m4s"], "requires": ["llhls.enabled=true"], "startup_policy": "waits for the current publisher generation's required sequence headers and binds headers, GOP replay, and live cursor from one snapshot; initial manifest waits for one complete segment without completed PART tags; blocking reloads retain the latest completed PART identity", "segment_policy": {"part_duration": "partial segment target", "segment_duration": "completed full-segment target", "segment_duration_default_seconds": 1.0, "segment_duration_schema_minimum_seconds": 0.1, "reload": "hot"}, "audio_only_segmentation": "elapsed media time; completed TS or fMP4 is available before source shutdown", "overwrite_recovery": "abandon current parts and one MSN per recovery epoch, wake blocked reloads, advance to live, refresh same-generation init/container state, and mark the first recovered independent part or segment with EXT-X-DISCONTINUITY"}, + {"id": "dash", "direction": ["play"], "status": "stable", "port": 8080, "url_templates": ["http://HOST:8080/STREAM_KEY.mpd", "http://HOST:8080/STREAM_KEY/audio_init.mp4", "http://HOST:8080/STREAM_KEY/a1.m4s"], "startup_policy": "waits for the current publisher generation's required sequence headers and binds headers, GOP replay, and live cursor from one snapshot; response write deadlines are refreshed immediately before writing and do not include segment readiness waits", "audio_only_segmentation": "elapsed media time; audio-only MPD omits video adaptation and completed m4s is available before source shutdown", "overwrite_recovery": "preserve completed single-Period init and timeline media, discard current video/audio batches, retire the manager, and terminate future segment waits without publishing post-gap media"}, {"id": "http-flv", "direction": ["play"], "status": "stable", "port": 8080, "url_templates": ["http://HOST:8080/STREAM_KEY.flv"]}, {"id": "fmp4", "direction": ["play"], "status": "stable", "port": 8080, "url_templates": ["http://HOST:8080/STREAM_KEY.mp4"], "fragment_policy": "concatenated moof/mdat fragments are parsed as one complete media segment without dropping earlier fragments"}, - {"id": "sipgateway", "direction": ["publish", "play"], "status": "stable", "entrypoints": ["SIP Gateway provider StartLabSession/ListLabSessions/StopLabSession"], "requires": ["initialized SIP transport and enabled gateway", "H.264 plus PCMA or PCMU"], "lab_modes": {"publish": "fake device sends H.264 video and PCMA/PCMU audio on separate inbound RTP/RTCP tracks into gateway-bound receivers", "receive": "fake endpoint accepts the gateway outbound INVITE, leaves the selected source stream unchanged, counts audio/video RTP/RTCP, and sends periodic per-track receiver reports"}, "inbound_publish": "synchronous EventPublish authorization runs before RTP allocation; successful inbound sessions emit generation-matched asynchronous EventPublish and EventPublishStop events for Record/DVR consumers", "startup": "outbound signaling, ACK, generation admission, and media startup use one publisher-generation snapshot; retirement before activation aborts the stale call; if a 2xx dialog was accepted before retirement, cleanup sends one BYE through the dialog path", "rtcp": "inbound audio/video RTCP sockets parse valid packets; outbound sender reports use each track SSRC, RFC NTP time, per-track RTP payload packet/octet counts, and periodic emission", "shutdown": "Lab-owned SIP transport readers release their non-reader references and close before the fake UA is closed, preventing UDP reference underflow and closed-socket cleanup warnings", "docs": "docs/recipes/protocol-test-lab.md"}, - {"id": "gb28181", "direction": ["publish", "play"], "status": "stable", "ports": [5060, "40000-50000"], "entrypoints": ["SIP REGISTER/INVITE on 5060", "POST /api/v1/gb28181/channels/{channel_id}/play", "POST /api/v1/gb28181/lab/sessions"], "requires": ["SIP and RTP network reachability"], "lab_modes": {"publish": "fake device registers, announces a channel, accepts LiveForge's server-initiated live-play INVITE/ACK/BYE, and sends H.264 plus 8 kHz mono G.711A in PS/RTP with RTCP to LiveForge's bound receiver; the requested validated stream_key is honored only on loopback Lab INVITEs, while ordinary devices use {stream_prefix}/{channel_id}", "receive": "LiveForge validates an H.264/G.711A source before activation and a module-owned outbound media session sends PS/RTP/RTCP to the fake device"}, "startup": "outbound INVITE wait and ACK are bound to the captured publisher generation; retirement prevents stale ACK/activation; if a 2xx dialog was accepted before retirement, cleanup sends one BYE through the dialog/session path", "outbound_video": "AVCC/HVCC video samples are converted to Annex-B before PS muxing for GB28181 receivers", "shutdown": "Lab-owned SIP transport readers release their non-reader references and close before the fake client UA is closed; peer listeners stop before their UA is closed", "docs": "docs/recipes/protocol-test-lab.md"}, + {"id": "sipgateway", "direction": ["publish", "play"], "status": "stable", "entrypoints": ["SIP Gateway provider StartLabSession/ListLabSessions/StopLabSession"], "requires": ["initialized SIP transport and enabled gateway", "H.264 plus PCMA or PCMU"], "lab_modes": {"publish": "fake device sends H.264 video and PCMA/PCMU audio on separate inbound RTP/RTCP tracks into gateway-bound receivers", "receive": "fake endpoint accepts the gateway outbound INVITE, leaves the selected source stream unchanged, counts audio/video RTP/RTCP, sends periodic per-track receiver reports, and treats requested PCMA/PCMU as the actual target codec; a differing source uses the optional generation-bound shared audio transcoder"}, "inbound_publish": "synchronous EventPublish authorization runs before RTP allocation; successful inbound sessions emit generation-matched asynchronous EventPublish and EventPublishStop events for Record/DVR consumers", "startup": "outbound signaling, ACK, generation admission, and media startup use one publisher-generation snapshot; retirement before activation aborts the stale call; if a 2xx dialog was accepted before retirement, cleanup sends one BYE through the dialog path", "port_binding": "RTP/RTCP pairs skip ports occupied outside the allocator and remain socket-bound from SDP negotiation through session cleanup; local Lab endpoint pairs avoid the configured gateway range", "outbound_media": "direct H.264 uses the source LiveCursor while transformed audio uses an independent target-codec reader; unavailable requested conversions fail before signaling; each ready transformed frame rechecks generation immediately before RTP, and retirement releases transcode, subscriber, and socket ownership before one BYE", "overwrite_recovery": "atomic source and target-audio results retain reader identity and exact skipped counts; the retained result is discarded and only that reader advances to live. Source overwrite gates H.264 until the latest same-generation sequence header plus IDR while transformed audio continues; target-audio overwrite resumes audio without disturbing direct video. Active-generation target-audio EOF is network_lost, and terminal paths cancel and join both pumps", "rtcp": "inbound audio/video RTCP sockets parse valid packets; outbound sender reports use each track SSRC, RFC NTP time, per-track RTP payload packet/octet counts, and periodic emission", "shutdown": "Lab-owned SIP transport readers release their non-reader references and close before the fake UA is closed, preventing UDP reference underflow and closed-socket cleanup warnings", "docs": "docs/recipes/protocol-test-lab.md"}, + {"id": "gb28181", "direction": ["publish", "play"], "status": "stable", "ports": [5060, "40000-50000"], "entrypoints": ["SIP REGISTER/INVITE on 5060", "POST /api/v1/gb28181/channels/{channel_id}/play", "POST /api/v1/gb28181/lab/sessions"], "requires": ["SIP and RTP network reachability"], "lab_modes": {"publish": "fake device registers, announces a channel, accepts LiveForge's server-initiated live-play INVITE/ACK/BYE, and sends H.264 plus 8 kHz mono G.711A in PS/RTP with RTCP to LiveForge's bound receiver; the requested validated stream_key is honored only on loopback Lab INVITEs, while ordinary devices use {stream_prefix}/{channel_id}", "receive": "LiveForge validates H.264 plus direct G.711A or audio that the tagged runtime can transform to G.711A before activation; a module-owned outbound media session sends PS/RTP/RTCP to the fake device"}, "startup": "outbound INVITE wait and ACK are bound to the captured publisher generation; retirement prevents stale ACK/activation; if a 2xx dialog was accepted before retirement, cleanup sends one BYE through the dialog/session path", "outbound_audio": "Direct G.711A uses the source reader; other supported source codecs use an independent generation-bound shared G.711A reader and fail before signaling when conversion is unavailable", "outbound_video": "AVCC/HVCC video samples are converted to Annex-B before PS muxing for GB28181 receivers", "overwrite_recovery": "wait-only pumps queue source and target-audio readiness; the merge performs each atomic read and drains queued control before pending output. Reader identity and exact skipped counts are preserved, every retained overwrite value and affected pending holdback are discarded, and only that reader advances to live. Source overwrite creates fresh PS state while preserving SSRC and monotonic RTP sequence, keeps unaffected audio flowing, and resumes H.264 only at the latest same-generation sequence header plus IDR. Target-audio overwrite preserves clean source video, PS state, and its original holdback deadline. Active-generation target-audio EOF fails before pending RTP, and terminal paths cancel and join both pumps", "shutdown": "Lab-owned SIP transport readers release their non-reader references and close before the fake client UA is closed; peer listeners stop before their UA is closed", "docs": "docs/recipes/protocol-test-lab.md"}, {"id": "websocket", "direction": ["play"], "status": "stable", "default_enabled": false, "port": 8080, "url_templates": ["ws://HOST:8080/ws/STREAM_KEY.flv"]} ], "media_behavior": { "fmp4_aac_timing": "Muxer.Init derives omitted AAC sample rate and channels from AudioSpecificConfig, uses 48 kHz stereo and 44.1 kHz stereo defaults for headerless Opus and MP3, preserves explicit positive arguments, and reuses the resolved sample rate as the audio media timescale for every fragment", - "fmp4_recording_audio": "fMP4 recordings declare AAC directly; non-AAC source audio including G.711, Opus, and MP3 is converted through the generation-bound audiocodec/FFmpeg path when available, otherwise audio is filtered and playable video-only output is retained; transformed recordings capture a stop source cursor, wait with a bounded timeout for the shared AAC track to consume it, and drain generated output before finalization" + "fmp4_recording_audio": "fMP4 recordings declare AAC directly; non-AAC source audio including G.711, Opus, and MP3 is converted through the generation-bound audiocodec/FFmpeg path when available, otherwise audio is filtered and playable video-only output is retained; transformed recordings capture a stop source cursor, wait with a bounded timeout for the shared AAC track to consume it, and drain generated output before finalization; publisher-generation completion flushes retained resampler samples, silence-pads final partial fixed-size PCM, and emits delayed encoder packets exactly once before Record/DVR output closes", + "classic_mp4_timing": "Audio and video keep independent DTS state and media timescales; movie and track durations are normalized and saturated at version-0 limits; CTTS uses version 1 when any composition offset is negative and version 0 otherwise" }, "install": [ {"id": "source", "status": "available", "command": "go build -o bin/liveforge ./cmd/liveforge", "docs": "docs/recipes/source-build.md"}, @@ -86,26 +113,37 @@ "verification": { "docs": "tools/check-agent-docs_test.sh", "build": "CGO_ENABLED=1 go build -tags audiocodec ./cmd/liveforge", - "tests": "CGO_ENABLED=1 go test -tags audiocodec -race -coverprofile=coverage.out -covermode=atomic ./...", + "tests": "CGO_ENABLED=1 go test -tags audiocodec -race -coverprofile=coverage.out -covermode=atomic ./...", + "protocol_browser_matrix": "CGO_ENABLED=1 go test -tags audiocodec ./test/integration -run '^TestSIPGB28181WHIPBrowserBridgeMatrix$' -count=1; set LIVEFORGE_PROTOCOL_MATRIX_SOAK=60s for extended per-second advancement checks", "integration_cli": "go build -o bin/lf-test ./tools/lf-test", - "forwarding_benchmarks": "go test -bench='BenchmarkRingReader|BenchmarkRTMPConn' -benchmem ./pkg/util ./module/cluster" + "forwarding_benchmarks": "go test -bench='BenchmarkRingReader|BenchmarkRTMPConn' -benchmem ./pkg/util ./module/cluster", + "metrics_benchmarks": "go test ./module/metrics -run '^$' -bench '^BenchmarkCollectorGatherStreamDetails$' -benchmem", + "media_hot_path_benchmarks": "go test -run '^$' -bench 'BenchmarkStreamIngressProduction|BenchmarkRTMPRelaySendMediaFrameProduction|BenchmarkRTSPRelaySendFrameProduction|BenchmarkRelayObservationAccounting' -benchmem -count=3 ./core ./module/cluster", + "webrtc_status_benchmarks": "go test ./module/webrtc -run '^$' -bench '^BenchmarkWHEPFeedStatus' -benchmem -count=3", + "stream_microbenchmarks": "go test -run '^$' -bench '^BenchmarkStreamWriteFrame$' -benchmem ./core", + "rtp_output_benchmarks": "go test -run '^$' -bench 'BenchmarkGBOutboundSendFrame|BenchmarkSIPOutboundSendFrame' -benchmem ./module/gb28181 ./module/sipgateway" + }, + "benchmark_evidence": { + "media_hot_path_fixture": "Preallocated monotonic H.264/G.711A Stream frames with limits disabled and no subscribers; fixed-timestamp 1200-byte H.264 and 160-byte AAC RTMP frames; fixed-timestamp 1200-byte single-NAL plus 3000-byte FU-A RTSP frames through real production functions; both egress paths end at bounded in-memory writers and exclude socket/syscall costs; regression evidence only, not capacity evidence" }, "api": { "openapi": "docs/api/openapi.yaml", "base_path": "/api/v1", "authentication": "Management accepts api.auth.bearer_token, named viewer/operator/admin tokens, or a console session when configured; GET /api/v1/server/health remains public; publish and subscribe auth are configured under auth.", - "runtime_config_status": "GET /api/v1/server/config returns redacted source/version/failure/callback/restart state; GET /api/v1/server/config/document and /schema expose complete redacted configuration, raw desired source document, and embedded versioned JSON Schema; POST /api/v1/server/config/validate is read-only; POST /api/v1/server/config/apply serializes source write with loads/close, returns 202 after the write, and schedules refresh; POST /api/v1/server/config/refresh schedules asynchronous refresh.", + "runtime_config_status": "GET /api/v1/server/config returns redacted source/version/failure/callback/restart state; GET /api/v1/server/config/document and /schema expose complete redacted configuration, raw desired source document, and embedded versioned JSON Schema; POST /api/v1/server/config/validate is read-only; POST /api/v1/server/config/apply serializes source write with loads/close, returns 202 with status=written_and_refresh_scheduled only after the write, and schedules refresh; POST /api/v1/server/config/refresh returns 202 with status=scheduled and schedules asynchronous refresh.", "runtime_config_permissions": {"read": "config:read (viewer/operator/admin)", "validate": "config:read (viewer/operator/admin)", "apply": "config:reload (operator/admin)", "refresh": "config:reload (operator/admin)"}, "protocol_self_tests": ["GET /api/v1/sipgateway/test", "GET /api/v1/gb28181/test"], - "sip_lab_provider": {"status": "available", "methods": ["StartLabSession(ctx, LabSessionRequest)", "ListLabSessions()", "StopLabSession(id)"], "modes": ["publish", "receive"], "media": "moving 160x90 constrained-baseline H.264 at 25 fps plus audible 20 ms PCMA/PCMU frames on separate RTP tracks", "codecs": ["H264", "PCMA", "PCMU"], "counters": ["aggregate RTP/RTCP bytes and packets", "receiver-side RTCP packets", "audio RTP packets", "video RTP packets"], "diagnostics": "active sessions plus the newest 16 stopped/failed sessions are retained; last_error is redacted before its 256-rune bound", "receive_source": "read-only; the lab never injects generated frames", "shutdown": "start signaling derives its operation context from the caller and session stop context; Stop and Gateway.Close cancel unanswered starts and release audio/video RTP/RTCP sockets", "standalone_manager": "NewLabManager returns contract-only sessions with state=contract and never reports transportless sessions as active", "docs": "docs/recipes/protocol-test-lab.md"}, + "protocol_lab_capacity": "Each SIP and GB28181 provider admits at most max_lab_sessions active starting/active sessions (and SIP contract sessions); the default is 16, terminal history is excluded, and a full ceiling returns HTTP 429 before resource allocation", + "sip_lab_provider": {"status": "available", "methods": ["StartLabSession(ctx, LabSessionRequest)", "ListLabSessions()", "StopLabSession(id)"], "modes": ["publish", "receive"], "media": "moving 160x90 constrained-baseline H.264 at 25 fps plus audible 20 ms PCMA/PCMU frames on separate RTP tracks", "codecs": ["H264", "PCMA", "PCMU"], "counters": ["aggregate RTP/RTCP bytes and packets", "receiver-side RTCP packets", "audio RTP packets", "video RTP packets"], "diagnostics": "active sessions plus the newest 16 stopped/failed sessions are retained; last_error is redacted before its 256-rune bound", "receive_source": "read-only; the lab never injects generated frames; requested PCMA/PCMU is negotiated as the target and may use optional source-to-target transcoding", "shutdown": "start signaling derives its operation context from the caller and session stop context; Stop and Gateway.Close cancel unanswered starts and release audio/video RTP/RTCP sockets", "standalone_manager": "NewLabManager returns contract-only sessions with state=contract and never reports transportless sessions as active", "docs": "docs/recipes/protocol-test-lab.md"}, "gb28181_lab_provider": {"status": "available", "methods": ["StartLabSession(ctx, LabSessionRequest)", "ListLabSessions()", "StopLabSession(id)"], "modes": ["publish", "receive"], "media": "moving 160x90 constrained-baseline H.264 at 25 fps plus audible 20 ms 8 kHz mono G.711A in PS/RTP payload type 96 with RTCP", "counters": ["RTP/RTCP packets and bytes", "PS frames", "audio frames", "video frames"], "diagnostics": "active sessions plus the newest 16 stopped/failed sessions are retained; SIP credentials and bearer tokens are redacted before the 256-rune last_error bound", "receive_failure": "source subscriber admission completes before activation; admission rejection is synchronous, while later outbound media failure transitions the Lab to failed and releases signaling, session, subscriber, sockets, and ports", "signaling": ["REGISTER", "Keepalive", "Catalog", "server-initiated INVITE", "ACK", "BYE", "unregister"], "keepalive": "persistent sessions renew at roughly one-third of gb28181.keepalive.timeout, with bounded practical interval", "stream_key": "publish honors printable ASCII keys up to 256 bytes through a loopback-only private SIP header; ordinary devices use {stream_prefix}/{channel_id}", "shutdown": "stop and module close cancel SIP/control/media loops and release separate fake client/peer UAs plus loopback SIP, RTP, and RTCP sockets", "docs": "docs/recipes/protocol-test-lab.md"}, "management_surfaces": ["streams", "runtime config", "cluster status", "SIP Gateway calls", "recordings and DVR", "security status", "audit", "GB28181"], - "response_contracts": {"management": "code/message/data envelope, including GB28181 lab 400/404 responses", "gb28181": "protocol-specific device/channel endpoints may return direct JSON", "webrtc": "SDP success and plain-text errors"} + "response_contracts": {"management": "code/message/data envelope, including GB28181 lab 400/404 responses", "server_info": "GET /api/v1/server/info includes capabilities.audio_transcoding, which is true only when configured G.711 A-law and mu-law to AAC paths are available in this process; endpoint_schemes.dvr reports the actual http/https scheme of the bound DVR listener and endpoints.dvr reports its non-zero bound port after initialization", "recording_media": "Only completed recordings are downloadable or playable; active and failed recordings return HTTP 409 with a JSON error and no media body", "gb28181": "protocol-specific device/channel endpoints may return direct JSON", "webrtc": "SDP success and plain-text errors; requested WHEP codec mismatch returns 415, internal track setup returns 500, and WHEP session status returns JSON diagnostics"} }, "operations": { "protocol_lab_startup": "SIP and GB28181 receive labs wait for the selected publisher generation's required sequence headers before outbound signaling; a known unsupported SIP audio codec is rejected before waiting. Sources with late headers remain cancellable through the caller context.", - "console": {"status": "available", "views": ["Streams", "GB28181", "Config", "Cluster", "SIP Calls", "Storage", "Security"], "groups": {"Workspace": ["Streams", "GB28181", "SIP Calls", "Storage"], "Operations": ["Cluster"], "System": ["Config", "Security"]}, "recent_audit": "inside Security; not a separate tab", "permission_aware_actions": true, "config_editor": "desired redacted source document is editable only when the selected file/http/https/consul/redis source implements ConfigWriter; effective applied document, pending restart paths, complete schema, and redacted details for all source kinds are displayed separately", "media_cache": "Streams reports keyframe-driven GOP generation with interleaved video/audio frame counts and duration; audio-only streams report startup GOP as not applicable", "protocol_labs": ["SIP H.264 plus PCMA/PCMU RTP/RTCP loopback", "GB28181 H.264 plus 8 kHz mono G.711A PS/RTP/RTCP loopback"], "persistent_provider_labs": {"sip": "available", "gb28181": "available"}, "preview_protocols": ["http-flv", "ws-flv", "http-ts", "fmp4", "hls", "dash", "whep-realtime", "whep-live"], "protocol_lab_playback": "each stream-key path segment is URL-escaped; RTMP/RTSP use bound endpoint discovery when available and replace wildcard bind hosts with the management request host", "g711_preview": "Audio-only PCMA/PCMU streams use the audio element and WHEP; HTTP muxers do not promise G.711 browser playback", "whep_autoplay": "WHEP preview starts muted when asynchronous audio delivery would otherwise be blocked by browser autoplay policy and exposes an explicit Unmute/Mute control", "media_endpoint_discovery": "GET /api/v1/server/info reports the active bound HTTP/WebRTC listener when available; wildcard listeners are normalized by the Console to the host serving the Console. A different process on that host and port can still intercept browser media requests."}, - "recording": {"status": "available", "default_format": "fmp4", "default_extension": ".mp4", "formats": ["flv", "fmp4", "mp4", "ts", "hls"], "management": ["list", "status", "detail", "range download", "inline range playback", "delete"], "disabled_status": "Storage returns HTTP 200 with state=disabled when the record module is absent; recording item routes still return 503", "startup": "Record and DVR capture one publisher-generation startup snapshot; expected tracks come from snapshot.MediaInfo; direct readers start at LiveCursor; generation completion cancels old readers; pure-audio sessions have no retained replay history", "audio_compatibility": "fMP4 Record and DVR TS normalize G.711 and other TS-incompatible audio to AAC through the shared audiocodec/FFmpeg path when available; without that optional dependency they filter the incompatible audio and keep playable video-only output", "empty_output": "sequence-header-only or empty Record sessions are failed, not completed playable files; DVR does not publish a successful segment without media", "dvr_status": true, "dvr_media_authorization": "synchronous subscribe authorization hooks only; no asynchronous subscribe lifecycle emission", "dvr_media_cors": "non-credentialed wildcard CORS for split-port HLS playlist/segment fetches", "console_playback": {"recordings": "authenticated management API session; browser-native MP4/fMP4 and mpegts.js FLV/TS", "dvr": "HLS on the separate dvr.listen media listener; Console does not persist or append bearer tokens"}, "docs": "docs/recipes/recording-dvr-management.md"}, + "metrics": {"stream_detail_default": false, "aggregate_metrics": "available whenever the metrics module is enabled", "stream_detail_limit_validation": "zero disables per-stream series; negative configured values are invalid and rejected", "without_allowlist": "active scalar stream keys are admitted in creation order up to stream_detail_limit for one Collector lifetime; admitted keys are never evicted or replaced by churn", "with_allowlist": "only exact configured keys are eligible; the list is deduplicated and sorted once, and stream_detail_limit bounds each gather", "operator_tradeoff": "use the management API for current stream detail or an exact allowlist for selected Prometheus labels"}, + "console": {"status": "available", "views": ["Streams", "GB28181", "Config", "Cluster", "SIP Calls", "Storage", "Security"], "groups": {"Workspace": ["Streams", "GB28181", "SIP Calls", "Storage"], "Operations": ["Cluster"], "System": ["Config", "Security"]}, "recent_audit": "inside Security; not a separate tab", "permission_aware_actions": true, "config_editor": "desired redacted source document is editable only when the selected file/http/https/consul/redis source implements ConfigWriter; effective applied document, pending restart paths, complete schema, and redacted details for all source kinds are displayed separately", "media_cache": "Streams reports keyframe-driven GOP generation with interleaved video/audio frame counts and duration; audio-only streams report startup GOP as not applicable", "protocol_labs": ["SIP H.264 plus PCMA/PCMU RTP/RTCP loopback", "GB28181 H.264 plus 8 kHz mono G.711A PS/RTP/RTCP loopback"], "persistent_provider_labs": {"sip": "available", "gb28181": "available"}, "preview_protocols": ["http-flv", "ws-flv", "http-ts", "fmp4", "hls", "dash", "whep-realtime", "whep-live"], "protocol_lab_playback": "each stream-key path segment is URL-escaped; RTMP/RTSP use bound endpoint discovery when available and replace wildcard bind hosts with the management request host; whep/whep_live use mode=live and whep_realtime uses mode=realtime, and Console buttons consume the matching field", "g711_preview": "Audio-only PCMA/PCMU streams use the audio element and WHEP; HTTP muxers do not promise G.711 browser playback", "fmp4_effective_audio": "For G.711 sources the Console declares AAC to MSE only when server-info capabilities.audio_transcoding is true; portable builds retain video-only output and MIME declaration", "whep_autoplay": "WHEP preview starts muted when asynchronous audio delivery would otherwise be blocked by browser autoplay policy and exposes an explicit Unmute/Mute control", "media_endpoint_discovery": "GET /api/v1/server/info reports the active bound HTTP/WebRTC listener when available; wildcard listeners are normalized by the Console to the host serving the Console. A different process on that host and port can still intercept browser media requests."}, + "recording": {"status": "available", "default_format": "fmp4", "default_extension": ".mp4", "formats": ["flv", "fmp4", "mp4", "ts", "hls"], "management": ["list", "status", "detail", "range download", "inline range playback", "delete"], "action_routing": "plain GET and DELETE use the complete recording ID; ?action=play and ?action=download explicitly act on that full ID; legacy /play and /download suffix actions apply only when no exact ID exists", "deletion": "exact owned TS sidecars and metadata are removed before the authoritative primary; cleanup failure leaves the primary retriable and already removed cleanup artifacts are idempotent", "disabled_status": "Storage returns HTTP 200 with state=disabled when the record module is absent; recording item routes still return 503", "startup": "Record and DVR capture one publisher-generation startup snapshot; DVR carries its validated snapshot through storage/index recovery and rejects the candidate if that generation is no longer current before installation; expected tracks come from snapshot.MediaInfo; direct readers start at LiveCursor; generation completion cancels old readers; pure-audio sessions have no retained replay history", "rotation_initialization": "every rotated FLV, fMP4, MP4, or TS file restores declared tracks and deep-copied latest video/audio sequence headers, then rebases each track to a zero-based file-local timeline; TS writes PAT/PMT before first media and classic MP4 uses independent track clocks", "audio_compatibility": "fMP4 Record and DVR TS normalize G.711 and other TS-incompatible audio to AAC through the shared audiocodec/FFmpeg path when available; at publisher-generation end they retain flushed resampler samples, silence-padded partial PCM, and delayed encoder packets exactly once before output close; without that optional dependency they filter the incompatible audio and keep playable video-only output", "dvr_segmentation": "audio-only DVR rotates and publishes a media segment at the audio DTS duration threshold while its publisher remains online; video DVR still uses a valid keyframe boundary, and nested/reserved stream-key paths are segment-wise escaped", "format_validation": "record format accepts flv, fmp4, mp4, ts, and hls (hls stores TS); segment.max_size accepts empty/zero or non-negative decimal bytes with B, KB, MB, or GB suffixes and rejects fractional, negative, unknown-suffix, and overflow values", "empty_output": "sequence-header-only or empty Record sessions are failed, not completed playable files; DVR does not publish a successful segment without media", "recording_media_write_bound": "inline play and download acquire one global connection slot before opening media, release it exactly once on every return path, and set a 10-second write deadline immediately before ServeContent", "dvr_status": true, "dvr_media_authorization": "synchronous subscribe authorization hooks only; no asynchronous subscribe lifecycle emission", "dvr_media_cors": "non-credentialed wildcard CORS for split-port HLS playlist/segment fetches", "dvr_media_write_bound": "finite playlist and segment responses use a 10-second server WriteTimeout and release exactly one global connection slot on success, error, cancellation, or timeout", "console_playback": {"recordings": "authenticated management API session; browser-native MP4/fMP4 and mpegts.js FLV/TS", "dvr": "HLS on the separate dvr.listen media listener; Console does not persist or append bearer tokens"}, "docs": "docs/recipes/recording-dvr-management.md"}, "sipgateway": {"status": "available", "management": ["list", "dial", "detail", "hangup"], "self_test": "GET /api/v1/sipgateway/test runs an in-process fake-peer REGISTER/401/digest, INVITE/200/ACK/BYE, incompatible rejection/timeout, RTP media, and RTCP control loop without a remote platform", "docs": "docs/recipes/sipgateway-management.md"}, "gb28181": {"status": "available", "self_test": "GET /api/v1/gb28181/test runs an in-process fake-device REGISTER, Keepalive, Catalog, PS/90000 INVITE/SDP/ACK/BYE, rejection/timeout, PS-over-UDP media, and RTCP control loop without a remote device", "persistent_lab": "POST/GET/DELETE /api/v1/gb28181/lab/sessions runs server-initiated publish and module-owned receive egress with H.264 plus G.711A PS/RTP/RTCP and separate audio/video counters", "docs": "docs/recipes/protocol-test-lab.md"}, "cluster": {"status": "available", "protocols": ["rtmp", "srt", "rtsp", "rtp", "gb28181"], "credential_resolution": "atomic per request; api.auth.bearer_token then first named admin token", "peer_errors": "bounded and redacted", "forwarding_hot_path": "cluster readers use per-reader context-aware condition waits; each push binds one atomic publisher-generation startup snapshot, sends required headers/replay once, reads from LiveCursor, cancels on GenerationDone, and rejects a raced replacement frame; GB28181 PS video sequence-header send errors are propagated before replay/live media; RTMP push reuses FLV encoding state; RTSP TCP interleaving uses net.Buffers; relay byte metrics bind labels once, flush after 64 KiB and on operation completion", "audio_only_startup": "no replay history and no retained-ring startup", "docs": "docs/recipes/cluster-relay-operations.md"}, diff --git a/config/config.go b/config/config.go index fbba1f89..c79bf5dd 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,10 @@ package config import "time" +// DefaultRuntimeSourceMaxBytes is the default complete-document limit for +// file and network-backed runtime configuration sources. +const DefaultRuntimeSourceMaxBytes int64 = 4 << 20 + // Config is the root configuration for the streaming server. type Config struct { Server ServerConfig `yaml:"server"` @@ -41,7 +45,8 @@ type RuntimeConfig struct { } type RuntimeFileSourceConfig struct { - Path string `yaml:"path"` + Path string `yaml:"path"` + MaxBytes int64 `yaml:"max_bytes"` } type RuntimeHTTPSourceConfig struct { @@ -66,6 +71,7 @@ type RuntimeRedisSourceConfig struct { Hash string `yaml:"hash"` VersionKey string `yaml:"version_key"` TLS bool `yaml:"tls"` + MaxBytes int64 `yaml:"max_bytes"` } // AudioCodecConfig controls audio transcoding between protocols. @@ -103,9 +109,10 @@ type LimitsConfig struct { // RateLimitConfig holds per-IP HTTP rate limiting settings. type RateLimitConfig struct { - Enabled bool `yaml:"enabled"` - Rate float64 `yaml:"rate"` // requests per second per IP - Burst int `yaml:"burst"` // max burst size per IP + Enabled bool `yaml:"enabled"` + Rate float64 `yaml:"rate"` // requests per second per IP + Burst int `yaml:"burst"` // max burst size per IP + TrustedProxies []string `yaml:"trusted_proxies"` } // RTMPConfig holds RTMP module settings. @@ -231,11 +238,12 @@ type SIPAuth struct { // SIPGatewayConfig holds SIP-to-stream gateway settings. type SIPGatewayConfig struct { - Enabled bool `yaml:"enabled"` - StreamPrefix string `yaml:"stream_prefix"` // stream key prefix (default "sip") - RTPPortRange []int `yaml:"rtp_port_range"` // [min, max] for RTP port allocation - Codecs []string `yaml:"codecs"` // preferred codecs (default: opus, PCMA, PCMU) - MaxCalls int `yaml:"max_calls"` // max concurrent calls (default 100) + Enabled bool `yaml:"enabled"` + StreamPrefix string `yaml:"stream_prefix"` // stream key prefix (default "sip") + RTPPortRange []int `yaml:"rtp_port_range"` // [min, max] for RTP port allocation + Codecs []string `yaml:"codecs"` // preferred codecs (default: opus, PCMA, PCMU) + MaxCalls int `yaml:"max_calls"` // max concurrent calls (default 100) + MaxLabSessions int `yaml:"max_lab_sessions"` // max active local protocol labs (default 16) } // GB28181Config holds GB28181 module settings. @@ -243,6 +251,7 @@ type GB28181Config struct { Enabled bool `yaml:"enabled"` StreamPrefix string `yaml:"stream_prefix"` RTPPortRange []int `yaml:"rtp_port_range"` + MaxLabSessions int `yaml:"max_lab_sessions"` // max active local protocol labs (default 16) SSRC SSRCConfig `yaml:"ssrc"` Keepalive KeepaliveConfig `yaml:"keepalive"` AutoInvite bool `yaml:"auto_invite"` @@ -283,15 +292,22 @@ type SlowConsumerConfig struct { // StreamConfig holds stream-level settings. type StreamConfig struct { - GOPCache bool `yaml:"gop_cache"` - GOPCacheNum int `yaml:"gop_cache_num"` - RingBufferSize int `yaml:"ring_buffer_size"` - IdleTimeout time.Duration `yaml:"idle_timeout"` - NoPublisherTimeout time.Duration `yaml:"no_publisher_timeout"` - SlowConsumer SlowConsumerConfig `yaml:"slow_consumer"` - Simulcast SimulcastConfig `yaml:"simulcast"` - Feedback FeedbackConfig `yaml:"feedback"` -} + GOPCache bool `yaml:"gop_cache"` + GOPCacheNum int `yaml:"gop_cache_num"` + GOPCacheMaxFrames int `yaml:"gop_cache_max_frames"` + GOPCacheMaxDuration time.Duration `yaml:"gop_cache_max_duration"` + GOPCacheMaxBytes int64 `yaml:"gop_cache_max_bytes"` + RingBufferSize int `yaml:"ring_buffer_size"` + IdleTimeout time.Duration `yaml:"idle_timeout"` + NoPublisherTimeout time.Duration `yaml:"no_publisher_timeout"` + SlowConsumer SlowConsumerConfig `yaml:"slow_consumer"` + Simulcast SimulcastConfig `yaml:"simulcast"` + Feedback FeedbackConfig `yaml:"feedback"` +} + +// DefaultGOPCacheMaxFrames is the defensive cardinality bound used when a +// stream is constructed directly without passing through config validation. +const DefaultGOPCacheMaxFrames = 300 // SimulcastConfig holds simulcast layer settings. type SimulcastConfig struct { @@ -499,9 +515,12 @@ type DVRConfig struct { // MetricsConfig holds Prometheus metrics settings. type MetricsConfig struct { - Enabled bool `yaml:"enabled"` - Listen string `yaml:"listen"` - Path string `yaml:"path"` + Enabled bool `yaml:"enabled"` + Listen string `yaml:"listen"` + Path string `yaml:"path"` + StreamDetail bool `yaml:"stream_detail"` + StreamDetailLimit int `yaml:"stream_detail_limit"` + StreamDetailAllowlist []string `yaml:"stream_detail_allowlist"` } // APIConfig holds the management API settings. diff --git a/config/config_test.go b/config/config_test.go index 3aa8391a..b9bad70d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "time" ) @@ -72,6 +73,109 @@ func TestLoadConfigDefaults(t *testing.T) { if cfg.Stream.RingBufferSize != 1024 { t.Errorf("expected default ring_buffer_size 1024, got %d", cfg.Stream.RingBufferSize) } + if cfg.Stream.GOPCacheMaxFrames <= 0 || cfg.Stream.GOPCacheMaxDuration <= 0 || cfg.Stream.GOPCacheMaxBytes <= 0 { + t.Fatalf("GOP cache bounds must have positive defaults: frames=%d duration=%s bytes=%d", cfg.Stream.GOPCacheMaxFrames, cfg.Stream.GOPCacheMaxDuration, cfg.Stream.GOPCacheMaxBytes) + } + if cfg.SIP.Gateway.MaxLabSessions != 16 || cfg.GB28181.MaxLabSessions != 16 { + t.Fatalf("expected default protocol lab session ceilings of 16, got SIP=%d GB=%d", cfg.SIP.Gateway.MaxLabSessions, cfg.GB28181.MaxLabSessions) + } +} + +func TestLoadConfigGOPCacheBounds(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + doc := "stream:\n gop_cache_max_frames: 7\n gop_cache_max_duration: 2s\n gop_cache_max_bytes: 8192\n" + if err := os.WriteFile(path, []byte(doc), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if cfg.Stream.GOPCacheMaxFrames != 7 || cfg.Stream.GOPCacheMaxDuration != 2*time.Second || cfg.Stream.GOPCacheMaxBytes != 8192 { + t.Fatalf("loaded GOP bounds = frames=%d duration=%s bytes=%d", cfg.Stream.GOPCacheMaxFrames, cfg.Stream.GOPCacheMaxDuration, cfg.Stream.GOPCacheMaxBytes) + } +} + +func TestValidateRejectsNonPositiveRingBufferSize(t *testing.T) { + cfg := Defaults() + cfg.Stream.RingBufferSize = 0 + if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "stream.ring_buffer_size") { + t.Fatalf("Validate() error = %v, want ring buffer size rejection", err) + } +} + +func TestValidateMetricsStreamDetailLimitAllowsZeroAndRejectsNegative(t *testing.T) { + zero := Defaults() + zero.Metrics.StreamDetailLimit = 0 + if err := Validate(zero); err != nil { + t.Fatalf("Validate() rejected metrics.stream_detail_limit=0: %v", err) + } + + negative := Defaults() + negative.Metrics.StreamDetailLimit = -1 + if err := Validate(negative); err == nil || !strings.Contains(err.Error(), "metrics.stream_detail_limit must not be negative") { + t.Fatalf("Validate() error = %v, want negative metrics stream detail limit rejection", err) + } +} + +func TestValidateRejectsInvalidTrustedProxy(t *testing.T) { + cfg := Defaults() + cfg.Limits.RateLimit.TrustedProxies = []string{"127.0.0.1", "not-a-network"} + if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "limits.rate_limit.trusted_proxies[1]") { + t.Fatalf("Validate() error = %v, want invalid trusted proxy rejection", err) + } +} + +func TestValidateRejectsUnboundedEnabledGOPCache(t *testing.T) { + cfg := Defaults() + cfg.Stream.GOPCache = true + cfg.Stream.GOPCacheMaxFrames = 0 + cfg.Stream.GOPCacheMaxBytes = 0 + if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "gop_cache_max_frames or stream.gop_cache_max_bytes") { + t.Fatalf("Validate() error = %v, want hard GOP bound rejection", err) + } +} + +func TestValidateRecordFormatAndMaxSize(t *testing.T) { + for _, format := range []string{"flv", "fmp4", "mp4", "ts", "hls", " HLS "} { + cfg := Defaults() + cfg.Record.Format = format + if err := Validate(cfg); err != nil { + t.Errorf("Validate() rejected record format %q: %v", format, err) + } + } + + for _, maxSize := range []string{"", "0", "0MB", "512KB", "1GB", " 256mb "} { + cfg := Defaults() + cfg.Record.Segment.MaxSize = maxSize + if err := Validate(cfg); err != nil { + t.Errorf("Validate() rejected record.segment.max_size %q: %v", maxSize, err) + } + } + + for _, test := range []struct { + name string + field string + value string + }{ + {name: "format", field: "record.format", value: "webm"}, + {name: "fractional size", field: "record.segment.max_size", value: "1.5MB"}, + {name: "negative size", field: "record.segment.max_size", value: "-1MB"}, + {name: "unknown suffix", field: "record.segment.max_size", value: "1TB"}, + {name: "overflow size", field: "record.segment.max_size", value: "9223372036854775808B"}, + } { + t.Run(test.name, func(t *testing.T) { + cfg := Defaults() + if test.field == "record.format" { + cfg.Record.Format = test.value + } else { + cfg.Record.Segment.MaxSize = test.value + } + if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), test.field) { + t.Fatalf("Validate() error = %v, want %s rejection", err, test.field) + } + }) + } } func TestLoadConfigEnvExpansion(t *testing.T) { @@ -229,7 +333,9 @@ http_stream: container: "ts" ` tmpFile := filepath.Join(t.TempDir(), "test.yaml") - os.WriteFile(tmpFile, []byte(yaml), 0644) + if err := os.WriteFile(tmpFile, []byte(yaml), 0600); err != nil { + t.Fatal(err) + } cfg, err := Load(tmpFile) if err != nil { t.Fatalf("Load: %v", err) @@ -257,7 +363,9 @@ http_stream: listen: ":8080" ` tmpFile := filepath.Join(t.TempDir(), "test.yaml") - os.WriteFile(tmpFile, []byte(yaml), 0644) + if err := os.WriteFile(tmpFile, []byte(yaml), 0600); err != nil { + t.Fatal(err) + } cfg, err := Load(tmpFile) if err != nil { t.Fatalf("Load: %v", err) @@ -303,7 +411,9 @@ http_stream: container: "mpegts" ` tmpFile := filepath.Join(t.TempDir(), "test.yaml") - os.WriteFile(tmpFile, []byte(yaml), 0644) + if err := os.WriteFile(tmpFile, []byte(yaml), 0600); err != nil { + t.Fatal(err) + } cfg, err := Load(tmpFile) if err != nil { t.Fatalf("Load: %v", err) @@ -320,7 +430,9 @@ http_stream: container: "mpeg-ts" ` tmpFile := filepath.Join(t.TempDir(), "test.yaml") - os.WriteFile(tmpFile, []byte(yaml), 0644) + if err := os.WriteFile(tmpFile, []byte(yaml), 0600); err != nil { + t.Fatal(err) + } cfg, err := Load(tmpFile) if err != nil { t.Fatalf("Load: %v", err) @@ -401,7 +513,9 @@ func TestLoadConfigInvalidPath(t *testing.T) { func TestLoadConfigInvalidYAML(t *testing.T) { tmpFile := filepath.Join(t.TempDir(), "bad.yaml") - os.WriteFile(tmpFile, []byte("{{invalid yaml"), 0644) + if err := os.WriteFile(tmpFile, []byte("{{invalid yaml"), 0600); err != nil { + t.Fatal(err) + } _, err := Load(tmpFile) if err == nil { t.Error("expected error for invalid YAML") diff --git a/config/loader.go b/config/loader.go index 2fabc410..ec3612d0 100644 --- a/config/loader.go +++ b/config/loader.go @@ -203,11 +203,15 @@ func defaults() *Config { SIP: SIPConfig{ Listen: ":5060", Transport: []string{"udp", "tcp"}, + Gateway: SIPGatewayConfig{MaxLabSessions: 16}, }, Stream: StreamConfig{ - GOPCache: true, - GOPCacheNum: 1, - RingBufferSize: 1024, + GOPCache: true, + GOPCacheNum: 1, + GOPCacheMaxFrames: DefaultGOPCacheMaxFrames, + GOPCacheMaxDuration: 10 * time.Second, + GOPCacheMaxBytes: 32 * 1024 * 1024, + RingBufferSize: 1024, SlowConsumer: SlowConsumerConfig{ Enabled: true, LagWarnRatio: 0.5, @@ -236,13 +240,18 @@ func defaults() *Config { Audit: AuditConfig{MaxEntries: 1000}, }, Metrics: MetricsConfig{ - Listen: ":9090", - Path: "/metrics", + Listen: ":9090", + Path: "/metrics", + StreamDetailLimit: 100, }, Runtime: RuntimeConfig{ Source: "file", PollInterval: 30 * time.Second, LoadTimeout: 10 * time.Second, + File: RuntimeFileSourceConfig{MaxBytes: DefaultRuntimeSourceMaxBytes}, + HTTP: RuntimeHTTPSourceConfig{MaxBytes: DefaultRuntimeSourceMaxBytes}, + Consul: RuntimeConsulSourceConfig{MaxBytes: DefaultRuntimeSourceMaxBytes}, + Redis: RuntimeRedisSourceConfig{MaxBytes: DefaultRuntimeSourceMaxBytes}, }, Record: RecordConfig{Format: "fmp4"}, DVR: DVRConfig{ @@ -252,6 +261,7 @@ func defaults() *Config { SegmentDuration: 6 * time.Second, CleanupInterval: 30 * time.Second, }, + GB28181: GB28181Config{MaxLabSessions: 16}, Cluster: ClusterConfig{ SRT: ClusterSRTConfig{ Latency: 120 * time.Millisecond, diff --git a/config/runtime/error_redaction.go b/config/runtime/error_redaction.go new file mode 100644 index 00000000..7c4f2a25 --- /dev/null +++ b/config/runtime/error_redaction.go @@ -0,0 +1,40 @@ +package runtime + +import ( + "net/url" + "regexp" + "strings" +) + +var errorURLPattern = regexp.MustCompile(`(?i)(?:https?|rediss?|consul)://[^\s"'<>]+`) + +// RedactError removes URL credentials, query values, fragments, and line +// breaks before an error crosses a logging or management API boundary. +func RedactError(err error) string { + if err == nil { + return "" + } + message := strings.NewReplacer("\r", " ", "\n", " ").Replace(err.Error()) + return errorURLPattern.ReplaceAllStringFunc(message, redactErrorURL) +} + +func redactErrorURL(raw string) string { + trailing := "" + for len(raw) > 0 && strings.ContainsRune(".,;:)", rune(raw[len(raw)-1])) { + trailing = string(raw[len(raw)-1]) + trailing + raw = raw[:len(raw)-1] + } + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "REDACTED_URL" + trailing + } + parsed.User = nil + if parsed.Path != "" && parsed.Path != "/" { + parsed.Path = "/REDACTED" + parsed.RawPath = "" + } + parsed.RawQuery = "__liveforge_redacted__=1" + parsed.ForceQuery = false + parsed.Fragment = "" + return parsed.String() + trailing +} diff --git a/config/runtime/error_redaction_test.go b/config/runtime/error_redaction_test.go new file mode 100644 index 00000000..172f1ccf --- /dev/null +++ b/config/runtime/error_redaction_test.go @@ -0,0 +1,20 @@ +package runtime + +import ( + "errors" + "strings" + "testing" +) + +func TestRedactErrorHidesURLPathCredentialsAndPreservesHost(t *testing.T) { + const rawURL = "https://hooks.slack.com/services/T111/B111/error-path-token" + redacted := RedactError(errors.New("send webhook " + rawURL + ": connection refused")) + for _, secret := range []string{"T111", "B111", "error-path-token"} { + if strings.Contains(redacted, secret) { + t.Fatalf("redacted error leaked URL path credential %q: %s", secret, redacted) + } + } + if !strings.Contains(redacted, "hooks.slack.com") { + t.Fatalf("redacted error lost safe URL host: %s", redacted) + } +} diff --git a/config/runtime/manager.go b/config/runtime/manager.go index 9e13833e..e5f0847b 100644 --- a/config/runtime/manager.go +++ b/config/runtime/manager.go @@ -154,7 +154,7 @@ func (m *Manager) run(ctx context.Context) { m.initialResult <- nil } if err != nil && !isContextError(err) { - slog.Warn("runtime config source initial refresh failed; keeping bootstrap config", "source", m.sourceName, "error", err) + slog.Warn("runtime config source initial refresh failed; keeping bootstrap config", "source", m.sourceName, "error", RedactError(err)) } ticker := time.NewTicker(m.pollInterval) defer ticker.Stop() @@ -337,7 +337,13 @@ func (m *Manager) Refresh(ctx context.Context) error { // ValidateDocument parses and validates a complete source document without // changing the active snapshot or the backing source. func ValidateDocument(data []byte) (*config.Config, error) { - return ParseDocument(data) + return parseDocument(data, false, false) +} + +// ValidateKnownDocument validates an untrusted candidate without expanding +// process environment variables and rejects fields outside the typed contract. +func ValidateKnownDocument(data []byte) (*config.Config, error) { + return parseDocument(data, false, true) } // Write validates and persists a complete configuration document when the @@ -430,7 +436,7 @@ func (m *Manager) callbackLoop() { m.statusMu.Lock() m.status.CallbackFailures++ m.statusMu.Unlock() - slog.Error("runtime config callback failed", "error", err) + slog.Error("runtime config callback failed", "error", RedactError(err)) } } } @@ -494,7 +500,7 @@ func (m *Manager) setAcceptedVersion(v Version, pending []string) { func (m *Manager) setRejected(err error) { m.statusMu.Lock() m.status.ConsecutiveFailures++ - m.status.LastError = err.Error() + m.status.LastError = RedactError(err) m.status.ConfigChangesRejected++ m.statusMu.Unlock() } @@ -502,7 +508,7 @@ func (m *Manager) setRejected(err error) { func (m *Manager) setApplicationFailure(err error) { m.statusMu.Lock() m.status.ConsecutiveFailures++ - m.status.LastError = err.Error() + m.status.LastError = RedactError(err) m.status.ConfigChangesApplicationFailed++ m.statusMu.Unlock() } diff --git a/config/runtime/manager_test.go b/config/runtime/manager_test.go index 0c3a602e..33acdd18 100644 --- a/config/runtime/manager_test.go +++ b/config/runtime/manager_test.go @@ -1,8 +1,11 @@ package runtime import ( + "bytes" "context" "errors" + "log/slog" + "strings" "sync" "sync/atomic" "testing" @@ -49,6 +52,35 @@ func (s *blockingSource) Load(ctx context.Context, previous Version) (Snapshot, func (s *blockingSource) Close() error { return nil } +type countingErrorSource struct { + err error + loads atomic.Int32 +} + +func (s *countingErrorSource) Load(context.Context, Version) (Snapshot, error) { + s.loads.Add(1) + return Snapshot{}, s.err +} + +func (s *countingErrorSource) Close() error { return nil } + +type synchronizedBuffer struct { + mu sync.Mutex + buffer bytes.Buffer +} + +func (b *synchronizedBuffer) Write(data []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.Write(data) +} + +func (b *synchronizedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.String() +} + type serializedWriterSource struct { active atomic.Int32 overlap atomic.Bool @@ -154,9 +186,11 @@ func TestManagerWriteHonorsContextWhileWaitingForSourceIO(t *testing.T) { func TestSnapshotReadDoesNotWaitForRefresh(t *testing.T) { source := &blockingSource{release: make(chan struct{})} + initial := config.Defaults() + initial.Server.Name = "initial" m, err := NewManager(Options{ Source: source, - Initial: &config.Config{Server: config.ServerConfig{Name: "initial"}}, + Initial: initial, }) if err != nil { t.Fatal(err) @@ -186,9 +220,11 @@ func TestSnapshotReadDoesNotWaitForRefresh(t *testing.T) { func TestFailedRefreshRetainsLastValidSnapshot(t *testing.T) { source := &blockingSource{release: make(chan struct{}), err: errors.New("source unavailable")} + initial := config.Defaults() + initial.Server.Name = "initial" m, err := NewManager(Options{ Source: source, - Initial: &config.Config{Server: config.ServerConfig{Name: "initial"}}, + Initial: initial, }) if err != nil { t.Fatal(err) @@ -210,6 +246,67 @@ func TestFailedRefreshRetainsLastValidSnapshot(t *testing.T) { } } +func TestManagerRedactsURLCredentialsFromFailureStatus(t *testing.T) { + release := make(chan struct{}) + close(release) + source := &blockingSource{ + release: release, + err: errors.New(`Get "https://user:password@example.test/config.yaml?token=source-secret": connection refused`), + } + initial := config.Defaults() + initial.Server.Name = "initial" + m, err := NewManager(Options{Source: source, Initial: initial}) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if err := m.load(context.Background()); err == nil { + t.Fatal("expected source failure") + } + lastError := m.Status().LastError + for _, secret := range []string{"user", "password", "source-secret", "token="} { + if strings.Contains(lastError, secret) { + t.Fatalf("status error leaked %q: %s", secret, lastError) + } + } + if !strings.Contains(lastError, "__liveforge_redacted__") { + t.Fatalf("status error did not retain an explicit redaction marker: %s", lastError) + } +} + +func TestManagerRedactsBackgroundSourceErrorsInStatusAndLogs(t *testing.T) { + var logs synchronizedBuffer + previousLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previousLogger) }) + + source := &countingErrorSource{err: errors.New("source load failed for https://source-user:source-password@config.example.test/live.yaml?token=query-secret\nretry denied")} + m, err := NewManager(Options{Source: source, Initial: config.Defaults(), PollInterval: time.Hour}) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if err := m.Start(context.Background()); err != nil { + t.Fatal(err) + } + waitForManagerTest(t, func() bool { + return m.Status().ConsecutiveFailures >= 1 && strings.Contains(logs.String(), "config.example.test") + }) + assertRuntimeDiagnosticRedacted(t, m.Status().LastError) + if strings.ContainsAny(m.Status().LastError, "\r\n") { + t.Fatalf("runtime status retained line breaks: %q", m.Status().LastError) + } + assertRuntimeDiagnosticRedacted(t, logs.String()) + + if err := m.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + waitForManagerTest(t, func() bool { + return source.loads.Load() >= 2 && m.Status().ConsecutiveFailures >= 2 + }) + assertRuntimeDiagnosticRedacted(t, m.Status().LastError) +} + func TestManagerPublishesEffectiveConfigAndKeepsDesiredRestartValuesPending(t *testing.T) { initial := config.Defaults() initial.RTMP.Listen = ":1935" @@ -353,6 +450,71 @@ func TestManagerApplicationFailureDoesNotPublishCandidateOrTypedKeys(t *testing. } } +func TestManagerRedactsApplicationFailureInStatus(t *testing.T) { + initial := config.Defaults() + desired := config.Defaults() + desired.Limits.MaxStreams = 42 + data, err := normalizedBytes(desired) + if err != nil { + t.Fatal(err) + } + applyErr := errors.New("application failed at https://apply-user:apply-password@config.example.test/apply?token=query-secret\nrollback required") + m, err := NewManager(Options{ + Source: &mutableSource{snapshot: Snapshot{Data: data, Version: "application-error"}}, + Initial: initial, + Apply: func(*ConfigSnapshot, ChangeSet) error { + return applyErr + }, + }) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if err := m.load(context.Background()); !errors.Is(err, applyErr) { + t.Fatalf("load error=%v want=%v", err, applyErr) + } + assertRuntimeDiagnosticRedacted(t, m.Status().LastError) + if strings.ContainsAny(m.Status().LastError, "\r\n") { + t.Fatalf("application status retained line breaks: %q", m.Status().LastError) + } +} + +func TestManagerRedactsCallbackFailureInLogs(t *testing.T) { + var logs synchronizedBuffer + previousLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previousLogger) }) + + desired := config.Defaults() + desired.Limits.MaxStreams++ + data, err := normalizedBytes(desired) + if err != nil { + t.Fatal(err) + } + callbackErr := errors.New("callback failed at https://callback-user:callback-password@config.example.test/callback?token=query-secret\nretry queued") + m, err := NewManager(Options{ + Source: &mutableSource{snapshot: Snapshot{Data: data, Version: "callback-error"}}, + Initial: config.Defaults(), + OnChange: func(ChangeSet) error { + return callbackErr + }, + }) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if err := m.load(context.Background()); err != nil { + t.Fatal(err) + } + waitForManagerTest(t, func() bool { + return m.Status().CallbackFailures == 1 && strings.Contains(logs.String(), "config.example.test") + }) + assertRuntimeDiagnosticRedacted(t, logs.String()) + if strings.Contains(logs.String(), `\nretry queued`) { + t.Fatalf("callback log retained an injected line break: %q", logs.String()) + } +} + func TestManagerCoalescesNotificationsWithoutLosingLatestSnapshot(t *testing.T) { initial := config.Defaults() source := &mutableSource{} @@ -462,7 +624,7 @@ func TestManagerCountsAcceptedRejectedAndApplicationFailedChanges(t *testing.T) } func BenchmarkSnapshotRead(b *testing.B) { - m, err := NewManager(Options{Source: &blockingSource{release: make(chan struct{})}, Initial: &config.Config{}}) + m, err := NewManager(Options{Source: &blockingSource{release: make(chan struct{})}, Initial: config.Defaults()}) if err != nil { b.Fatal(err) } @@ -473,3 +635,26 @@ func BenchmarkSnapshotRead(b *testing.B) { } } } + +func waitForManagerTest(t *testing.T, condition func() bool) { + t.Helper() + deadline := time.Now().Add(time.Second) + for !condition() { + if time.Now().After(deadline) { + t.Fatal("timed out waiting for manager condition") + } + time.Sleep(time.Millisecond) + } +} + +func assertRuntimeDiagnosticRedacted(t *testing.T, diagnostic string) { + t.Helper() + for _, secret := range []string{"source-user", "source-password", "apply-user", "apply-password", "callback-user", "callback-password", "query-secret", "token="} { + if strings.Contains(diagnostic, secret) { + t.Fatalf("runtime diagnostic leaked %q: %q", secret, diagnostic) + } + } + if !strings.Contains(diagnostic, "config.example.test") { + t.Fatalf("runtime diagnostic lost useful host identity: %q", diagnostic) + } +} diff --git a/config/runtime/parser.go b/config/runtime/parser.go index da34f711..cad18ec4 100644 --- a/config/runtime/parser.go +++ b/config/runtime/parser.go @@ -4,7 +4,9 @@ import ( "bytes" "crypto/sha256" "encoding/hex" + "errors" "fmt" + "io" "os" "reflect" "sort" @@ -17,10 +19,10 @@ import ( // ParseDocument parses YAML or JSON into a defaulted and normalized config. func ParseDocument(data []byte) (*config.Config, error) { - return parseDocument(data, true) + return parseDocument(data, true, false) } -func parseDocument(data []byte, expandEnvironment bool) (*config.Config, error) { +func parseDocument(data []byte, expandEnvironment, rejectUnknown bool) (*config.Config, error) { if len(bytes.TrimSpace(data)) == 0 { return nil, fmt.Errorf("configuration document is empty") } @@ -31,7 +33,25 @@ func parseDocument(data []byte, expandEnvironment bool) (*config.Config, error) return nil, err } cfg := config.Defaults() - if err := yaml.Unmarshal(data, cfg); err != nil { + var err error + if rejectUnknown { + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + err = decoder.Decode(cfg) + if err == nil { + var extra yaml.Node + if extraErr := decoder.Decode(&extra); !errors.Is(extraErr, io.EOF) { + if extraErr != nil { + err = extraErr + } else { + err = fmt.Errorf("configuration must contain exactly one YAML or JSON document") + } + } + } + } else { + err = yaml.Unmarshal(data, cfg) + } + if err != nil { return nil, fmt.Errorf("parse config: %w", err) } config.Normalize(cfg) @@ -104,7 +124,7 @@ func cloneConfig(cfg *config.Config) (*config.Config, error) { if err != nil { return nil, err } - return parseDocument(b, false) + return parseDocument(b, false, false) } func configMap(cfg *config.Config) (map[string]any, error) { @@ -198,7 +218,7 @@ func applyHotChanges(current, desired *config.Config, changes []Change) (*config if err != nil { return nil, fmt.Errorf("marshal effective config: %w", err) } - return parseDocument(data, false) + return parseDocument(data, false, false) } func mapPathValue(root map[string]any, parts []string) (any, bool) { diff --git a/config/runtime/parser_test.go b/config/runtime/parser_test.go index 87f95668..d9172fad 100644 --- a/config/runtime/parser_test.go +++ b/config/runtime/parser_test.go @@ -72,6 +72,13 @@ auth: } } +func TestValidateKnownDocumentRejectsSecondYAMLDocument(t *testing.T) { + _, err := ValidateKnownDocument([]byte("server:\n name: liveforge\n---\nmalicious_or_unknown:\n value: ignored\n")) + if err == nil { + t.Fatal("expected second YAML document to be rejected") + } +} + func TestParseDocumentAppliesDefaultsAndNormalizesContainer(t *testing.T) { cfg, err := ParseDocument([]byte("http_stream:\n llhls:\n container: mpeg-ts\n")) if err != nil { diff --git a/config/runtime/schema_contract_test.go b/config/runtime/schema_contract_test.go index 0114ff31..a167d8e3 100644 --- a/config/runtime/schema_contract_test.go +++ b/config/runtime/schema_contract_test.go @@ -78,13 +78,79 @@ func TestConfigSchemaRequiresPracticalLLHLSSegmentDuration(t *testing.T) { } } +func TestConfigSchemaAndRuntimeRejectNegativeMetricsStreamDetailLimit(t *testing.T) { + schema := loadConfigSchema(t) + zero := map[string]any{"metrics": map[string]any{"stream_detail_limit": 0}} + if err := validateSchemaValue(schema, schema, zero, "$"); err != nil { + t.Fatalf("schema rejected metrics.stream_detail_limit=0: %v", err) + } + if _, err := ParseDocument([]byte("metrics:\n stream_detail_limit: 0\n")); err != nil { + t.Fatalf("runtime parser rejected metrics.stream_detail_limit=0: %v", err) + } + + negative := map[string]any{"metrics": map[string]any{"stream_detail_limit": -1}} + if err := validateSchemaValue(schema, schema, negative, "$"); err == nil { + t.Fatal("schema accepted negative metrics.stream_detail_limit") + } + if _, err := ParseDocument([]byte("metrics:\n stream_detail_limit: -1\n")); err == nil || !strings.Contains(err.Error(), "metrics.stream_detail_limit must not be negative") { + t.Fatalf("runtime parser error = %v, want negative metrics stream detail limit rejection", err) + } +} + +func TestConfigSchemaRejectsUnboundedEnabledGOPCache(t *testing.T) { + schema := loadConfigSchema(t) + document := map[string]any{ + "stream": map[string]any{ + "gop_cache": true, + "gop_cache_num": 1, + "gop_cache_max_frames": 0, + "gop_cache_max_duration": "10s", + "gop_cache_max_bytes": 0, + }, + } + if err := validateSchemaValue(schema, schema, document, "$"); err == nil { + t.Fatal("schema accepted an enabled GOP cache without a hard bound") + } +} + +func TestConfigSchemaMatchesRecordMaxSizeContract(t *testing.T) { + schema := loadConfigSchema(t) + valid := []string{"", " ", "0", "0B", "512KB", "1gb", "42b"} + for _, value := range valid { + document := map[string]any{ + "record": map[string]any{ + "segment": map[string]any{"max_size": value}, + }, + } + if err := validateSchemaValue(schema, schema, document, "$"); err != nil { + t.Errorf("schema rejected record.segment.max_size=%q: %v", value, err) + } + } + + for _, value := range []string{"K", "1K", "1M", "1G", "1TB", "1.5MB", "-1MB"} { + document := map[string]any{ + "record": map[string]any{ + "segment": map[string]any{"max_size": value}, + }, + } + if err := validateSchemaValue(schema, schema, document, "$"); err == nil { + t.Errorf("schema accepted invalid record.segment.max_size=%q", value) + } + } +} + func TestConfigSchemaAcceptsNegativeScalarDefaultSentinels(t *testing.T) { schema := loadConfigSchema(t) document := map[string]any{ "sip": map[string]any{"gateway": map[string]any{"max_calls": -1}}, "cluster": map[string]any{"health_check": map[string]any{"evict_threshold": -1}}, "api": map[string]any{"audit": map[string]any{"max_entries": -1}}, - "runtime": map[string]any{"http": map[string]any{"max_bytes": -1}, "consul": map[string]any{"max_bytes": -1}}, + "runtime": map[string]any{ + "file": map[string]any{"max_bytes": -1}, + "http": map[string]any{"max_bytes": -1}, + "consul": map[string]any{"max_bytes": -1}, + "redis": map[string]any{"max_bytes": -1}, + }, } if err := validateSchemaValue(schema, schema, document, "$"); err != nil { t.Fatalf("schema rejected source-supported negative default sentinels: %v", err) @@ -101,20 +167,25 @@ api: audit: max_entries: -1 runtime: + file: + max_bytes: -1 http: max_bytes: -1 consul: max_bytes: -1 + redis: + max_bytes: -1 ` cfg, err := ParseDocument([]byte(yamlDocument)) if err != nil { t.Fatalf("runtime parser rejected source-supported negative default sentinels: %v", err) } if cfg.SIP.Gateway.MaxCalls != -1 || cfg.Cluster.HealthCheck.EvictThreshold != -1 || - cfg.API.Audit.MaxEntries != -1 || cfg.Runtime.HTTP.MaxBytes != -1 || cfg.Runtime.Consul.MaxBytes != -1 { - t.Fatalf("runtime parser did not preserve negative sentinels: max_calls=%d evict_threshold=%d max_entries=%d http_max_bytes=%d consul_max_bytes=%d", + cfg.API.Audit.MaxEntries != -1 || cfg.Runtime.File.MaxBytes != -1 || cfg.Runtime.HTTP.MaxBytes != -1 || + cfg.Runtime.Consul.MaxBytes != -1 || cfg.Runtime.Redis.MaxBytes != -1 { + t.Fatalf("runtime parser did not preserve negative sentinels: max_calls=%d evict_threshold=%d max_entries=%d file_max_bytes=%d http_max_bytes=%d consul_max_bytes=%d redis_max_bytes=%d", cfg.SIP.Gateway.MaxCalls, cfg.Cluster.HealthCheck.EvictThreshold, cfg.API.Audit.MaxEntries, - cfg.Runtime.HTTP.MaxBytes, cfg.Runtime.Consul.MaxBytes) + cfg.Runtime.File.MaxBytes, cfg.Runtime.HTTP.MaxBytes, cfg.Runtime.Consul.MaxBytes, cfg.Runtime.Redis.MaxBytes) } } @@ -177,6 +248,20 @@ func TestConfigSchemaAcceptsCheckedInSample(t *testing.T) { } } +func TestConfigSchemaExposesTrustedProxyPolicy(t *testing.T) { + schema := loadConfigSchema(t) + document := map[string]any{ + "limits": map[string]any{ + "rate_limit": map[string]any{ + "trusted_proxies": []any{"127.0.0.1", "10.0.0.0/8"}, + }, + }, + } + if err := validateSchemaValue(schema, schema, document, "$"); err != nil { + t.Fatalf("schema rejected trusted proxy policy: %v", err) + } +} + func loadConfigSchema(t *testing.T) map[string]any { t.Helper() data, err := os.ReadFile("../../docs/config/config.schema.json") @@ -193,6 +278,22 @@ func loadConfigSchema(t *testing.T) map[string]any { // validateSchemaValue executes the JSON Schema keywords used by the focused // sentinel fixtures, including local refs and combinators. func validateSchemaValue(root, schema map[string]any, value any, path string) error { + if condition, ok := schema["if"].(map[string]any); ok { + if validateSchemaValue(root, condition, value, path) == nil { + if thenSchema, ok := schema["then"].(map[string]any); ok { + if err := validateSchemaValue(root, thenSchema, value, path); err != nil { + return err + } + } + } else if elseSchema, ok := schema["else"].(map[string]any); ok { + if err := validateSchemaValue(root, elseSchema, value, path); err != nil { + return err + } + } + } + if expected, ok := schema["const"]; ok && !schemaValuesEqual(expected, value) { + return fmt.Errorf("%s is %v, want %v", path, value, expected) + } if ref, ok := schema["$ref"].(string); ok { resolved, err := resolveLocalSchemaRef(root, ref) if err != nil { @@ -200,6 +301,13 @@ func validateSchemaValue(root, schema map[string]any, value any, path string) er } return validateSchemaValue(root, resolved, value, path) } + if clauses, ok := schema["allOf"].([]any); ok { + for _, clause := range clauses { + if err := validateSchemaValue(root, clause.(map[string]any), value, path); err != nil { + return err + } + } + } if branches, ok := schema["anyOf"].([]any); ok { if !anySchemaBranchAccepts(root, branches, value, path) { return fmt.Errorf("%s does not satisfy anyOf", path) diff --git a/config/runtime/source.go b/config/runtime/source.go index 3cb82a0c..5037c40e 100644 --- a/config/runtime/source.go +++ b/config/runtime/source.go @@ -2,8 +2,26 @@ package runtime import ( "fmt" + "math" "net/url" + "regexp" + "sort" + "strconv" "strings" + + "github.com/im-pingo/liveforge/config" +) + +var ( + canonicalDecimalInteger = regexp.MustCompile(`^(?:0|-[1-9][0-9]*|[1-9][0-9]*)$`) + canonicalDecimalFloat = regexp.MustCompile(`^-?(?:0|[1-9][0-9]*)(?:\.[0-9]+(?:[eE][+-]?[0-9]+)?|[eE][+-]?[0-9]+)$`) +) + +const ( + // DefaultSourceMaxBytes bounds one complete configuration document when a + // source does not provide an explicit limit. + DefaultSourceMaxBytes int64 = config.DefaultRuntimeSourceMaxBytes + maxSourceEntries = 65536 ) func requireURL(raw, field string) (*url.URL, error) { @@ -18,20 +36,61 @@ func requireURL(raw, field string) (*url.URL, error) { } func documentFromKeyValues(values map[string]string) ([]byte, error) { + return documentFromKeyValuesWithLimit(values, DefaultSourceMaxBytes) +} + +func documentFromKeyValuesWithLimit(values map[string]string, maxBytes int64) ([]byte, error) { + if maxBytes <= 0 { + maxBytes = DefaultSourceMaxBytes + } if len(values) == 0 { return nil, fmt.Errorf("configuration key snapshot is empty") } for _, key := range []string{"config", "config.yaml", "config.yml", "config.json"} { if value, ok := values[key]; ok { + if int64(len(value)) > maxBytes { + return nil, fmt.Errorf("configuration document exceeds %d bytes", maxBytes) + } return []byte(value), nil } } root := make(map[string]any) + var materializedBytes int64 + type flattenedValue struct { + key string + value string + parts []string + canonical string + } + flattened := make([]flattenedValue, 0, len(values)) for key, value := range values { parts := strings.FieldsFunc(strings.Trim(key, "./"), func(r rune) bool { return r == '.' || r == '/' }) if len(parts) == 0 { continue } + flattened = append(flattened, flattenedValue{key: key, value: value, parts: parts, canonical: strings.Join(parts, ".")}) + } + sort.Slice(flattened, func(i, j int) bool { + if flattened[i].canonical != flattened[j].canonical { + return flattened[i].canonical < flattened[j].canonical + } + return flattened[i].key < flattened[j].key + }) + for index := 1; index < len(flattened); index++ { + previous, current := flattened[index-1], flattened[index] + if previous.canonical == current.canonical { + return nil, fmt.Errorf("flattened configuration keys %q and %q collide at %q", previous.key, current.key, current.canonical) + } + if strings.HasPrefix(current.canonical, previous.canonical+".") { + return nil, fmt.Errorf("flattened configuration keys %q and %q collide at %q", previous.key, current.key, previous.canonical) + } + } + for _, item := range flattened { + parts, value := item.parts, item.value + materializedBytes += int64(len(item.key) + len(value)) + if materializedBytes > maxBytes { + return nil, fmt.Errorf("configuration materialization exceeds %d bytes", maxBytes) + } current := root for _, part := range parts[:len(parts)-1] { next, ok := current[part].(map[string]any) @@ -43,11 +102,19 @@ func documentFromKeyValues(values map[string]string) ([]byte, error) { } current[parts[len(parts)-1]] = parseScalar(value) } - return marshalDeterministic(root) + data, err := marshalDeterministic(root) + if err != nil { + return nil, err + } + if int64(len(data)) > maxBytes { + return nil, fmt.Errorf("configuration document exceeds %d bytes", maxBytes) + } + return data, nil } func parseScalar(value string) any { - switch strings.ToLower(strings.TrimSpace(value)) { + trimmed := strings.TrimSpace(value) + switch strings.ToLower(trimmed) { case "true": return true case "false": @@ -55,5 +122,15 @@ func parseScalar(value string) any { case "null": return nil } + if canonicalDecimalInteger.MatchString(trimmed) { + if parsed, err := strconv.ParseInt(trimmed, 10, 64); err == nil { + return parsed + } + } + if canonicalDecimalFloat.MatchString(trimmed) { + if parsed, err := strconv.ParseFloat(trimmed, 64); err == nil && !math.IsInf(parsed, 0) && !math.IsNaN(parsed) { + return parsed + } + } return value } diff --git a/config/runtime/source_config.go b/config/runtime/source_config.go index a0be81bf..d81127f9 100644 --- a/config/runtime/source_config.go +++ b/config/runtime/source_config.go @@ -22,7 +22,7 @@ func BuildSource(cfg config.RuntimeConfig, bootstrapPath string) (NamedSource, e if path == "" { path = bootstrapPath } - return NewFileSource(path) + return NewFileSourceWithOptions(FileSourceOptions{Path: path, MaxBytes: cfg.File.MaxBytes}) case "http", "https": return NewHTTPSource(HTTPSourceOptions{URL: cfg.HTTP.URL, Token: cfg.HTTP.Token, Scheme: kind, MaxBytes: cfg.HTTP.MaxBytes}) case "consul": @@ -32,7 +32,7 @@ func BuildSource(cfg config.RuntimeConfig, bootstrapPath string) (NamedSource, e if cfg.Redis.TLS { tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12} } - return NewRedisSource(RedisSourceOptions{Addr: cfg.Redis.Addr, Username: cfg.Redis.Username, Password: cfg.Redis.Password, DB: cfg.Redis.DB, Prefix: cfg.Redis.Prefix, Hash: cfg.Redis.Hash, VersionKey: cfg.Redis.VersionKey, TLSConfig: tlsConfig}) + return NewRedisSource(RedisSourceOptions{Addr: cfg.Redis.Addr, Username: cfg.Redis.Username, Password: cfg.Redis.Password, DB: cfg.Redis.DB, Prefix: cfg.Redis.Prefix, Hash: cfg.Redis.Hash, VersionKey: cfg.Redis.VersionKey, TLSConfig: tlsConfig, MaxBytes: cfg.Redis.MaxBytes}) default: return nil, fmt.Errorf("unsupported runtime config source %q", kind) } diff --git a/config/runtime/source_consul.go b/config/runtime/source_consul.go index d97cc8f9..fa101138 100644 --- a/config/runtime/source_consul.go +++ b/config/runtime/source_consul.go @@ -43,10 +43,14 @@ func NewConsulSource(opts ConsulSourceOptions) (*ConsulSource, error) { if opts.Client == nil { opts.Client = http.DefaultClient } + client := *opts.Client + client.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } if opts.MaxBytes <= 0 { opts.MaxBytes = 4 << 20 } - return &ConsulSource{address: strings.TrimRight(opts.Address, "/"), prefix: strings.Trim(opts.Prefix, "/"), token: opts.Token, client: opts.Client, maxBytes: opts.MaxBytes}, nil + return &ConsulSource{address: strings.TrimRight(opts.Address, "/"), prefix: strings.Trim(opts.Prefix, "/"), token: opts.Token, client: &client, maxBytes: opts.MaxBytes}, nil } func (s *ConsulSource) Name() string { return "consul" } diff --git a/config/runtime/source_file.go b/config/runtime/source_file.go index cc276270..8dc8e043 100644 --- a/config/runtime/source_file.go +++ b/config/runtime/source_file.go @@ -3,19 +3,36 @@ package runtime import ( "context" "fmt" + "io" "os" "path/filepath" "time" ) +// FileSourceOptions configures one local YAML/JSON document source. +type FileSourceOptions struct { + Path string + MaxBytes int64 +} + // FileSource reads one local YAML/JSON document. -type FileSource struct{ path string } +type FileSource struct { + path string + maxBytes int64 +} func NewFileSource(path string) (*FileSource, error) { - if path == "" { + return NewFileSourceWithOptions(FileSourceOptions{Path: path}) +} + +func NewFileSourceWithOptions(opts FileSourceOptions) (*FileSource, error) { + if opts.Path == "" { return nil, fmt.Errorf("config file path is required") } - return &FileSource{path: path}, nil + if opts.MaxBytes <= 0 { + opts.MaxBytes = DefaultSourceMaxBytes + } + return &FileSource{path: opts.Path, maxBytes: opts.MaxBytes}, nil } func (s *FileSource) Name() string { return "file" } @@ -33,10 +50,18 @@ func (s *FileSource) Load(ctx context.Context, previous Version) (Snapshot, erro if info.IsDir() { return Snapshot{}, fmt.Errorf("config path is a directory") } - data, err := os.ReadFile(s.path) + file, err := os.Open(s.path) + if err != nil { + return Snapshot{}, fmt.Errorf("open config file: %w", err) + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, s.maxBytes+1)) if err != nil { return Snapshot{}, fmt.Errorf("read config file: %w", err) } + if int64(len(data)) > s.maxBytes { + return Snapshot{}, fmt.Errorf("config file exceeds %d bytes", s.maxBytes) + } if len(data) == 0 { return Snapshot{}, fmt.Errorf("config file is empty") } @@ -50,7 +75,7 @@ func (s *FileSource) Write(ctx context.Context, data []byte) error { return err } info, err := os.Stat(s.path) - mode := os.FileMode(0644) + mode := os.FileMode(0600) if err == nil { mode = info.Mode().Perm() } else if !os.IsNotExist(err) { diff --git a/config/runtime/source_redis.go b/config/runtime/source_redis.go index 044f3546..545ca89c 100644 --- a/config/runtime/source_redis.go +++ b/config/runtime/source_redis.go @@ -3,6 +3,7 @@ package runtime import ( "context" "crypto/tls" + "errors" "fmt" "sort" "strings" @@ -22,6 +23,7 @@ type RedisSourceOptions struct { VersionKey string TLSConfig *tls.Config Client *redis.Client + MaxBytes int64 } type RedisSource struct { @@ -29,6 +31,7 @@ type RedisSource struct { prefix string hash string versionKey string + maxBytes int64 } func NewRedisSource(opts RedisSourceOptions) (*RedisSource, error) { @@ -41,79 +44,327 @@ func NewRedisSource(opts RedisSourceOptions) (*RedisSource, error) { if opts.Hash == "" && opts.Prefix == "" { return nil, fmt.Errorf("redis hash or prefix is required") } - return &RedisSource{client: opts.Client, prefix: opts.Prefix, hash: opts.Hash, versionKey: opts.VersionKey}, nil + if opts.MaxBytes <= 0 { + opts.MaxBytes = DefaultSourceMaxBytes + } + return &RedisSource{client: opts.Client, prefix: opts.Prefix, hash: opts.Hash, versionKey: opts.VersionKey, maxBytes: opts.MaxBytes}, nil } func (s *RedisSource) Name() string { return "redis" } func (s *RedisSource) Load(ctx context.Context, previous Version) (Snapshot, error) { - values := make(map[string]string) + var ( + values map[string]string + err error + ) if s.hash != "" { - fields, err := s.client.HGetAll(ctx, s.hash).Result() + values, err = s.loadHash(ctx) + } else { + values, err = s.loadPrefix(ctx) + } + if err != nil { + return Snapshot{}, err + } + data, err := documentFromKeyValuesWithLimit(values, s.maxBytes) + if err != nil { + return Snapshot{}, err + } + version := "" + if s.versionKey != "" { + version, err = s.readString(ctx, s.versionKey) + if err != nil { + return Snapshot{}, fmt.Errorf("read redis config version: %w", err) + } + } + return Snapshot{Data: data, Version: version}, nil +} + +var completeConfigKeys = []string{"config", "config.yaml", "config.yml", "config.json"} + +func (s *RedisSource) loadHash(ctx context.Context) (map[string]string, error) { + for _, field := range completeConfigKeys { + exists, err := s.client.HExists(ctx, s.hash, field).Result() if err != nil { - return Snapshot{}, fmt.Errorf("read redis hash: %w", err) + return nil, fmt.Errorf("check redis hash field: %w", err) } - for key, value := range fields { - values[key] = value + if !exists { + continue } - } else { - var keys []string - var cursor uint64 - for { - batch, next, err := s.client.Scan(ctx, cursor, s.prefix+"*", 256).Result() + value, err := s.readHashString(ctx, field) + if err != nil { + return nil, err + } + return map[string]string{field: value}, nil + } + + fields, err := s.hashFieldNames(ctx) + if err != nil { + return nil, err + } + values := make(map[string]string, len(fields)) + var materializedBytes int64 + for start := 0; start < len(fields); start += 128 { + end := start + 128 + if end > len(fields) { + end = len(fields) + } + lengthPipe := s.client.Pipeline() + lengthCommands := make([]*redis.Cmd, 0, end-start) + for _, field := range fields[start:end] { + lengthCommands = append(lengthCommands, lengthPipe.Do(ctx, "HSTRLEN", s.hash, field)) + } + if _, err := lengthPipe.Exec(ctx); err != nil { + return nil, fmt.Errorf("read redis hash field lengths: %w", err) + } + preflightBytes := materializedBytes + for i, field := range fields[start:end] { + length, err := lengthCommands[i].Int64() if err != nil { - return Snapshot{}, fmt.Errorf("scan redis config keys: %w", err) + return nil, fmt.Errorf("read redis hash field %q length: %w", field, err) } - keys = append(keys, batch...) - cursor = next - if cursor == 0 { - break + preflightBytes += int64(len(field)) + length + if preflightBytes > s.maxBytes { + return nil, fmt.Errorf("redis configuration materialization exceeds %d bytes", s.maxBytes) } } - sort.Strings(keys) - pipe := s.client.Pipeline() - commands := make([]*redis.StringCmd, 0, len(keys)) - for _, key := range keys { - commands = append(commands, pipe.Get(ctx, key)) - } - if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil { - return Snapshot{}, fmt.Errorf("read redis config keys: %w", err) - } - for i, key := range keys { - value, err := commands[i].Result() - if err == nil { - values[strings.TrimPrefix(key, s.prefix)] = value + + valuePipe := s.client.Pipeline() + valueCommands := make([]*redis.StringCmd, 0, end-start) + for _, field := range fields[start:end] { + valueCommands = append(valueCommands, valuePipe.HGet(ctx, s.hash, field)) + } + if _, err := valuePipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) { + return nil, fmt.Errorf("read redis hash fields: %w", err) + } + for i, field := range fields[start:end] { + value, err := valueCommands[i].Result() + if errors.Is(err, redis.Nil) { + continue + } + if err != nil { + return nil, fmt.Errorf("read redis hash field %q: %w", field, err) + } + materializedBytes += int64(len(field) + len(value)) + if materializedBytes > s.maxBytes { + return nil, fmt.Errorf("redis configuration materialization exceeds %d bytes", s.maxBytes) } + values[field] = value } } - data, err := documentFromKeyValues(values) + return values, nil +} + +func (s *RedisSource) hashFieldNames(ctx context.Context) ([]string, error) { + fields, err := s.scanHashFieldNames(ctx) + if err == nil { + return fields, nil + } + if !redisHashScanWithoutValuesUnsupported(err) { + return nil, fmt.Errorf("scan redis hash field names: %w", err) + } + + // HSCAN NOVALUES was added after HSCAN. HKEYS keeps the compatibility path + // value-free; values are still fetched later in bounded HSTRLEN/HGET batches. + fields, err = s.client.HKeys(ctx, s.hash).Result() if err != nil { - return Snapshot{}, err + return nil, fmt.Errorf("list redis hash field names: %w", err) } - version := "" - if s.versionKey != "" { - version, err = s.client.Get(ctx, s.versionKey).Result() - if err != nil && err != redis.Nil { - return Snapshot{}, fmt.Errorf("read redis config version: %w", err) + if len(fields) > maxSourceEntries { + return nil, fmt.Errorf("redis configuration exceeds %d entries", maxSourceEntries) + } + var fieldBytes int64 + for _, field := range fields { + fieldBytes += int64(len(field)) + if fieldBytes > s.maxBytes { + return nil, fmt.Errorf("redis configuration materialization exceeds %d bytes", s.maxBytes) } } - return Snapshot{Data: data, Version: version}, nil + sort.Strings(fields) + return fields, nil +} + +func (s *RedisSource) scanHashFieldNames(ctx context.Context) ([]string, error) { + fields := make([]string, 0) + var cursor uint64 + for { + page, next, err := s.client.HScanNoValues(ctx, s.hash, cursor, "", 128).Result() + if err != nil { + return nil, err + } + if len(fields)+len(page) > maxSourceEntries { + return nil, fmt.Errorf("redis configuration exceeds %d entries", maxSourceEntries) + } + for _, field := range page { + if int64(len(field)) > s.maxBytes { + return nil, fmt.Errorf("redis configuration materialization exceeds %d bytes", s.maxBytes) + } + fields = append(fields, field) + } + cursor = next + if cursor == 0 { + sort.Strings(fields) + return fields, nil + } + } +} + +func redisHashScanWithoutValuesUnsupported(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + return strings.Contains(message, "unknown command") || + strings.Contains(message, "unknown subcommand") || + strings.Contains(message, "syntax error") +} + +func (s *RedisSource) loadPrefix(ctx context.Context) (map[string]string, error) { + for _, field := range completeConfigKeys { + value, found, err := s.readStringIfPresent(ctx, s.prefix+field) + if err != nil { + return nil, err + } + if found { + return map[string]string{field: value}, nil + } + } + + values := make(map[string]string) + var cursor uint64 + var materializedBytes int64 + for { + keys, next, err := s.client.Scan(ctx, cursor, s.prefix+"*", 128).Result() + if err != nil { + return nil, fmt.Errorf("scan redis config keys: %w", err) + } + sort.Strings(keys) + if len(values)+len(keys) > maxSourceEntries { + return nil, fmt.Errorf("redis configuration exceeds %d entries", maxSourceEntries) + } + for start := 0; start < len(keys); start += 128 { + end := start + 128 + if end > len(keys) { + end = len(keys) + } + lengthPipe := s.client.Pipeline() + lengthCommands := make([]*redis.IntCmd, 0, end-start) + for _, key := range keys[start:end] { + lengthCommands = append(lengthCommands, lengthPipe.StrLen(ctx, key)) + } + if _, err := lengthPipe.Exec(ctx); err != nil { + return nil, fmt.Errorf("read redis config key lengths: %w", err) + } + preflightBytes := materializedBytes + for i, key := range keys[start:end] { + length, err := lengthCommands[i].Result() + if err != nil { + return nil, fmt.Errorf("read redis config key %q length: %w", key, err) + } + field := strings.TrimPrefix(key, s.prefix) + preflightBytes += int64(len(field)) + length + if preflightBytes > s.maxBytes { + return nil, fmt.Errorf("redis configuration materialization exceeds %d bytes", s.maxBytes) + } + } + + pipe := s.client.Pipeline() + commands := make([]*redis.StringCmd, 0, end-start) + for _, key := range keys[start:end] { + commands = append(commands, pipe.Get(ctx, key)) + } + if _, err := pipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) { + return nil, fmt.Errorf("read redis config keys: %w", err) + } + for i, key := range keys[start:end] { + value, err := commands[i].Result() + if errors.Is(err, redis.Nil) { + continue + } + if err != nil { + return nil, fmt.Errorf("read redis config key %q: %w", key, err) + } + field := strings.TrimPrefix(key, s.prefix) + materializedBytes += int64(len(field) + len(value)) + if materializedBytes > s.maxBytes { + return nil, fmt.Errorf("redis configuration materialization exceeds %d bytes", s.maxBytes) + } + values[field] = value + } + } + cursor = next + if cursor == 0 { + return values, nil + } + } +} + +func (s *RedisSource) readHashString(ctx context.Context, field string) (string, error) { + length, err := s.client.Do(ctx, "HSTRLEN", s.hash, field).Int64() + if err != nil { + return "", fmt.Errorf("read redis hash field length: %w", err) + } + if length > s.maxBytes { + return "", fmt.Errorf("redis configuration value exceeds %d bytes", s.maxBytes) + } + value, err := s.client.HGet(ctx, s.hash, field).Result() + if err != nil { + return "", fmt.Errorf("read redis hash field: %w", err) + } + if int64(len(value)) > s.maxBytes { + return "", fmt.Errorf("redis configuration value exceeds %d bytes", s.maxBytes) + } + return value, nil +} + +func (s *RedisSource) readStringIfPresent(ctx context.Context, key string) (string, bool, error) { + exists, err := s.client.Exists(ctx, key).Result() + if err != nil { + return "", false, fmt.Errorf("check redis config key: %w", err) + } + if exists == 0 { + return "", false, nil + } + value, err := s.readString(ctx, key) + return value, true, err +} + +func (s *RedisSource) readString(ctx context.Context, key string) (string, error) { + length, err := s.client.StrLen(ctx, key).Result() + if err != nil { + return "", fmt.Errorf("read Redis value length: %w", err) + } + if length > s.maxBytes { + return "", fmt.Errorf("redis configuration value exceeds %d bytes", s.maxBytes) + } + value, err := s.client.Get(ctx, key).Result() + if errors.Is(err, redis.Nil) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("read Redis value: %w", err) + } + if int64(len(value)) > s.maxBytes { + return "", fmt.Errorf("redis configuration value exceeds %d bytes", s.maxBytes) + } + return value, nil } func (s *RedisSource) Close() error { return s.client.Close() } func (s *RedisSource) Write(ctx context.Context, data []byte) error { - if s.hash != "" { - if err := s.client.HSet(ctx, s.hash, "config.yaml", string(data)).Err(); err != nil { - return fmt.Errorf("write redis config hash: %w", err) + _, err := s.client.TxPipelined(ctx, func(pipe redis.Pipeliner) error { + if s.hash != "" { + pipe.HSet(ctx, s.hash, "config.yaml", string(data)) + } else { + pipe.Set(ctx, s.prefix+"config.yaml", data, 0) } - } else if err := s.client.Set(ctx, s.prefix+"config.yaml", data, 0).Err(); err != nil { - return fmt.Errorf("write redis config key: %w", err) - } - if s.versionKey != "" { - if err := s.client.Incr(ctx, s.versionKey).Err(); err != nil { - return fmt.Errorf("write redis config version: %w", err) + if s.versionKey != "" { + pipe.Incr(ctx, s.versionKey) } + return nil + }) + if err != nil { + return fmt.Errorf("write redis config transaction: %w", err) } return nil } diff --git a/config/runtime/source_test.go b/config/runtime/source_test.go index f45dbe17..fe3e86a3 100644 --- a/config/runtime/source_test.go +++ b/config/runtime/source_test.go @@ -1,16 +1,22 @@ package runtime import ( + "bufio" + "bytes" "context" "encoding/base64" + "errors" "fmt" "io" + "net" "net/http" "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" + "time" "github.com/im-pingo/liveforge/config" "github.com/redis/go-redis/v9" @@ -39,6 +45,21 @@ func TestFileSourceReturnsDocumentAndModificationMetadata(t *testing.T) { } } +func TestFileSourceLoadRejectsDocumentOverConfiguredLimit(t *testing.T) { + path := filepath.Join(t.TempDir(), "oversized.yaml") + if err := os.WriteFile(path, []byte("server:\n name: oversized\n"), 0o600); err != nil { + t.Fatal(err) + } + source, err := NewFileSourceWithOptions(FileSourceOptions{Path: path, MaxBytes: 8}) + if err != nil { + t.Fatal(err) + } + defer source.Close() + if _, err := source.Load(context.Background(), Version{}); err == nil || !strings.Contains(err.Error(), "exceeds 8 bytes") { + t.Fatalf("oversized file load error = %v, want configured byte-limit error", err) + } +} + func TestFileSourceWriteReplacesDocumentAtomically(t *testing.T) { path := filepath.Join(t.TempDir(), "liveforge.yaml") if err := os.WriteFile(path, []byte("server:\n name: old\n"), 0o600); err != nil { @@ -48,8 +69,8 @@ func TestFileSourceWriteReplacesDocumentAtomically(t *testing.T) { if err != nil { t.Fatal(err) } - if err := source.Write(context.Background(), []byte("server:\n name: new\n")); err != nil { - t.Fatal(err) + if writeErr := source.Write(context.Background(), []byte("server:\n name: new\n")); writeErr != nil { + t.Fatal(writeErr) } data, err := os.ReadFile(path) if err != nil || string(data) != "server:\n name: new\n" { @@ -61,6 +82,25 @@ func TestFileSourceWriteReplacesDocumentAtomically(t *testing.T) { } } +func TestFileSourceWriteUsesPrivateModeForNewDocument(t *testing.T) { + path := filepath.Join(t.TempDir(), "new-liveforge.yaml") + source, err := NewFileSource(path) + if err != nil { + t.Fatal(err) + } + defer source.Close() + if writeErr := source.Write(context.Background(), []byte("server:\n name: new\n")); writeErr != nil { + t.Fatal(writeErr) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("new config mode=%#o, want %#o", got, 0o600) + } +} + func TestHTTPSourceUsesETagAndAcceptsNotModified(t *testing.T) { requests := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -405,6 +445,60 @@ func TestConsulSourceWriteUsesCompleteConfigKey(t *testing.T) { } } +func TestConsulSourceLoadRejectsRedirectWithoutForwardingToken(t *testing.T) { + targetRequests := make(chan string, 1) + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + targetRequests <- r.Header.Get("X-Consul-Token") + _, _ = fmt.Fprint(w, `[]`) + })) + defer target.Close() + redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // #nosec G710 -- this test intentionally redirects to a loopback server to verify redirect rejection. + http.Redirect(w, r, target.URL+r.URL.RequestURI(), http.StatusTemporaryRedirect) + })) + defer redirect.Close() + + source, err := NewConsulSource(ConsulSourceOptions{Address: redirect.URL, Prefix: "liveforge", Token: "consul-secret"}) + if err != nil { + t.Fatal(err) + } + if _, err := source.Load(context.Background(), Version{}); err == nil { + t.Fatal("Consul load followed redirect") + } + select { + case token := <-targetRequests: + t.Fatalf("redirect target received X-Consul-Token %q", token) + default: + } +} + +func TestConsulSourceWriteRejectsRedirectWithoutForwardingToken(t *testing.T) { + targetRequests := make(chan string, 1) + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + targetRequests <- r.Header.Get("X-Consul-Token") + w.WriteHeader(http.StatusOK) + })) + defer target.Close() + redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // #nosec G710 -- this test intentionally redirects to a loopback server to verify redirect rejection. + http.Redirect(w, r, target.URL+r.URL.RequestURI(), http.StatusTemporaryRedirect) + })) + defer redirect.Close() + + source, err := NewConsulSource(ConsulSourceOptions{Address: redirect.URL, Prefix: "liveforge", Token: "consul-secret"}) + if err != nil { + t.Fatal(err) + } + if err := source.Write(context.Background(), []byte("server:\n name: consul\n")); err == nil { + t.Fatal("Consul write followed redirect") + } + select { + case token := <-targetRequests: + t.Fatalf("redirect target received X-Consul-Token %q", token) + default: + } +} + func TestRedisSourceBuildsNestedDocumentFromKeys(t *testing.T) { doc, err := documentFromKeyValues(map[string]string{ "server.name": "redis", @@ -429,3 +523,426 @@ func TestRedisSourceBuildsNestedDocumentFromKeys(t *testing.T) { t.Fatal(err) } } + +func TestRedisHashSourceFallsBackWhenHScanNoValuesIsUnavailable(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + serverErrors := make(chan error, 1) + go serveRedisHashFallbackTestServer(listener, serverErrors) + + client := redis.NewClient(&redis.Options{ + Addr: listener.Addr().String(), + Protocol: 2, + DisableIndentity: true, + Dialer: func(context.Context, string, string) (net.Conn, error) { + return net.DialTimeout("tcp", listener.Addr().String(), time.Second) + }, + }) + source, err := NewRedisSource(RedisSourceOptions{Client: client, Hash: "config"}) + if err != nil { + t.Fatal(err) + } + defer source.Close() + + snapshot, err := source.Load(context.Background(), Version{}) + if err != nil { + t.Fatalf("load Redis hash with legacy HSCAN support: %v", err) + } + select { + case serverErr := <-serverErrors: + t.Fatal(serverErr) + default: + } + cfg, err := ParseDocument(snapshot.Data) + if err != nil { + t.Fatalf("parse Redis hash document: %v\n%s", err, snapshot.Data) + } + if cfg.Server.Name != "redis-legacy" || !cfg.HTTP.Enabled { + t.Fatalf("Redis hash config = %+v, want flattened server.name and http_stream.enabled", cfg) + } +} + +func serveRedisHashFallbackTestServer(listener net.Listener, serverErrors chan<- error) { + conn, err := listener.Accept() + if err != nil { + serverErrors <- err + return + } + defer conn.Close() + reader := bufio.NewReader(conn) + values := map[string]string{ + "server.name": "redis-legacy", + "http_stream.enabled": "true", + } + for { + command, err := readRedisCommand(reader) + if err != nil { + if !errors.Is(err, io.EOF) { + select { + case serverErrors <- err: + default: + } + } + return + } + if len(command) == 0 { + continue + } + switch strings.ToUpper(command[0]) { + case "HELLO": + _, _ = io.WriteString(conn, "*4\r\n$6\r\nserver\r\n$5\r\nredis\r\n$5\r\nproto\r\n:2\r\n") + case "CLIENT": + _, _ = io.WriteString(conn, "+OK\r\n") + case "HEXISTS": + _, _ = io.WriteString(conn, ":0\r\n") + case "HSCAN": + _, _ = io.WriteString(conn, "-ERR unknown command 'HSCAN'\r\n") + case "HKEYS": + _, _ = io.WriteString(conn, "*2\r\n$11\r\nserver.name\r\n$19\r\nhttp_stream.enabled\r\n") + case "HSTRLEN": + value, ok := values[command[2]] + if !ok { + _, _ = io.WriteString(conn, ":0\r\n") + continue + } + _, _ = fmt.Fprintf(conn, ":%d\r\n", len(value)) + case "HGET": + value, ok := values[command[2]] + if !ok { + _, _ = io.WriteString(conn, "$-1\r\n") + continue + } + _, _ = fmt.Fprintf(conn, "$%d\r\n%s\r\n", len(value), value) + default: + select { + case serverErrors <- fmt.Errorf("unexpected Redis command %q", command[0]): + default: + } + _, _ = io.WriteString(conn, "-ERR unexpected command\r\n") + return + } + } +} + +func TestRedisSourceRejectsOversizedCompleteConfigValue(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + go serveRedisBoundedStringTestServer(listener, []byte("server:\n name: oversized\n")) + + client := redis.NewClient(&redis.Options{Addr: listener.Addr().String(), Protocol: 2, DisableIndentity: true}) + source, err := NewRedisSource(RedisSourceOptions{Client: client, Prefix: "liveforge:", MaxBytes: 8}) + if err != nil { + t.Fatal(err) + } + defer source.Close() + if _, err := source.Load(context.Background(), Version{}); err == nil || !strings.Contains(err.Error(), "exceeds 8 bytes") { + t.Fatalf("oversized Redis load error = %v, want configured byte-limit error", err) + } +} + +func serveRedisBoundedStringTestServer(listener net.Listener, value []byte) { + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + reader := bufio.NewReader(conn) + for { + command, err := readRedisCommand(reader) + if err != nil { + return + } + switch strings.ToUpper(command[0]) { + case "HELLO": + _, _ = io.WriteString(conn, "*2\r\n$6\r\nserver\r\n$5\r\nredis\r\n") + case "EXISTS": + _, _ = io.WriteString(conn, ":1\r\n") + case "STRLEN": + _, _ = fmt.Fprintf(conn, ":%d\r\n", len(value)) + case "GET": + _, _ = fmt.Fprintf(conn, "$%d\r\n%s\r\n", len(value), value) + default: + _, _ = io.WriteString(conn, "+OK\r\n") + } + } +} + +func TestParseFlattenedScalar(t *testing.T) { + tests := []struct { + name string + value string + want any + }{ + {name: "positive integer", value: "100", want: int64(100)}, + {name: "negative integer", value: "-42", want: int64(-42)}, + {name: "decimal float", value: "1.25", want: 1.25}, + {name: "exponent float", value: "6.25e-2", want: 0.0625}, + {name: "true", value: "true", want: true}, + {name: "false", value: "FALSE", want: false}, + {name: "null", value: "null", want: nil}, + {name: "ordinary string", value: "liveforge", want: "liveforge"}, + {name: "duration", value: "15s", want: "15s"}, + {name: "leading zero identifier", value: "00123", want: "00123"}, + {name: "negative leading zero identifier", value: "-01", want: "-01"}, + {name: "leading zero float identifier", value: "01.5", want: "01.5"}, + {name: "plus-prefixed integer", value: "+12", want: "+12"}, + {name: "underscored integer", value: "1_000", want: "1_000"}, + {name: "hexadecimal integer", value: "0x10", want: "0x10"}, + {name: "integer overflow", value: "9223372036854775808", want: "9223372036854775808"}, + {name: "float overflow", value: "1e309", want: "1e309"}, + {name: "not a number", value: "NaN", want: "NaN"}, + {name: "infinity", value: ".inf", want: ".inf"}, + {name: "YAML sequence", value: "[one, two]", want: "[one, two]"}, + {name: "YAML mapping", value: "{key: value}", want: "{key: value}"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := parseScalar(test.value); got != test.want { + t.Fatalf("parseScalar(%q) = %#v (%T), want %#v (%T)", test.value, got, got, test.want, test.want) + } + }) + } +} + +func TestFlattenedKeyValueDocumentsDecodeTypedNumericConfig(t *testing.T) { + tests := []struct { + name string + values map[string]string + }{ + { + name: "consul slash paths", + values: map[string]string{ + "limits/max_connections": "100", + "limits/max_streams": "25", + "http_stream/llhls/segment_duration": "1.25", + "http_stream/llhls/part_duration": "0.2", + "http_stream/llhls/segment_count": "6", + "http_stream/llhls/enabled": "true", + }, + }, + { + name: "redis dotted paths", + values: map[string]string{ + "limits.max_connections": "100", + "limits.max_streams": "25", + "http_stream.llhls.segment_duration": "1.25", + "http_stream.llhls.part_duration": "0.2", + "http_stream.llhls.segment_count": "6", + "http_stream.llhls.enabled": "true", + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + document, err := documentFromKeyValues(test.values) + if err != nil { + t.Fatal(err) + } + cfg, err := ParseDocument(document) + if err != nil { + t.Fatalf("parse generated document: %v\n%s", err, document) + } + if cfg.Limits.MaxConnections != 100 || cfg.Limits.MaxStreams != 25 { + t.Fatalf("typed limits = %+v, want max_connections=100 max_streams=25", cfg.Limits) + } + if cfg.HTTP.LLHLS.SegmentDuration != 1.25 || cfg.HTTP.LLHLS.PartDuration != 0.2 || cfg.HTTP.LLHLS.SegmentCount != 6 { + t.Fatalf("typed LL-HLS config = %+v", cfg.HTTP.LLHLS) + } + }) + } +} + +func TestFlattenedKeyValueDocumentSerializationIsDeterministic(t *testing.T) { + first, err := documentFromKeyValues(map[string]string{ + "limits.max_connections": "100", + "http_stream.llhls.segment_duration": "1.25", + "server.name": "liveforge", + }) + if err != nil { + t.Fatal(err) + } + second, err := documentFromKeyValues(map[string]string{ + "server.name": "liveforge", + "http_stream.llhls.segment_duration": "1.25", + "limits.max_connections": "100", + }) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(first, second) { + t.Fatalf("flattened documents differ:\nfirst:\n%s\nsecond:\n%s", first, second) + } +} + +func TestFlattenedKeyValueDocumentRejectsMaterializedSizeOverLimit(t *testing.T) { + _, err := documentFromKeyValuesWithLimit(map[string]string{ + "server.name": "materialized-size-limit", + }, 8) + if err == nil || !strings.Contains(err.Error(), "exceeds 8 bytes") { + t.Fatalf("materialized document error = %v, want configured byte-limit error", err) + } +} + +func TestFlattenedKeyValueDocumentsRejectDottedAndSlashedKeyCollisions(t *testing.T) { + values := map[string]string{ + "server.name": "dotted", + "server/name": "slashed", + } + _, err := documentFromKeyValues(values) + if err == nil { + t.Fatal("expected dotted and slashed key collision") + } + if !strings.Contains(err.Error(), "server.name") || !strings.Contains(err.Error(), "server/name") { + t.Fatalf("collision error=%q, want both source keys", err) + } +} + +func TestFlattenedKeyValueDocumentsRejectScalarAndNestedKeyCollisions(t *testing.T) { + _, err := documentFromKeyValues(map[string]string{ + "server": "scalar", + "server.name": "nested", + }) + if err == nil { + t.Fatal("expected scalar and nested key collision") + } +} + +func TestFlattenedKeyValueCollisionErrorsAreDeterministic(t *testing.T) { + _, err := documentFromKeyValues(map[string]string{ + "server/name": "slashed", + "server.name": "dotted", + }) + if err == nil { + t.Fatal("expected first collision") + } + _, secondErr := documentFromKeyValues(map[string]string{ + "server.name": "dotted", + "server/name": "slashed", + }) + if secondErr == nil { + t.Fatal("expected second collision") + } + if err.Error() != secondErr.Error() { + t.Fatalf("collision errors differ: %q vs %q", err, secondErr) + } +} + +func TestRedisSourceWriteUsesAtomicDocumentAndVersionTransaction(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + commands := make(chan []string, 8) + serverErrors := make(chan error, 1) + go serveRedisTransactionTestServer(listener, commands, serverErrors) + + client := redis.NewClient(&redis.Options{ + Addr: listener.Addr().String(), + Protocol: 2, + DisableIndentity: true, + Dialer: func(context.Context, string, string) (net.Conn, error) { + return net.DialTimeout("tcp", listener.Addr().String(), time.Second) + }, + }) + source, err := NewRedisSource(RedisSourceOptions{Client: client, Hash: "config", VersionKey: "config:version"}) + if err != nil { + t.Fatal(err) + } + defer source.Close() + if err := source.Write(context.Background(), []byte("server:\n name: transaction\n")); err != nil { + t.Fatal(err) + } + + var got []string + for len(got) < 4 { + select { + case command := <-commands: + name := strings.ToUpper(command[0]) + if name != "HELLO" { + got = append(got, name) + } + case err := <-serverErrors: + t.Fatal(err) + case <-time.After(time.Second): + t.Fatalf("timed out waiting for Redis command %d", len(got)+1) + } + } + want := []string{"MULTI", "HSET", "INCR", "EXEC"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("Redis write commands=%v, want %v", got, want) + } +} + +func serveRedisTransactionTestServer(listener net.Listener, commands chan<- []string, serverErrors chan<- error) { + conn, err := listener.Accept() + if err != nil { + serverErrors <- err + return + } + defer conn.Close() + reader := bufio.NewReader(conn) + for { + command, err := readRedisCommand(reader) + if err != nil { + if !errors.Is(err, io.EOF) { + select { + case serverErrors <- err: + default: + } + } + return + } + commands <- command + switch strings.ToUpper(command[0]) { + case "MULTI": + _, _ = io.WriteString(conn, "+OK\r\n") + case "HSET", "SET", "INCR": + _, _ = io.WriteString(conn, "+QUEUED\r\n") + case "EXEC": + _, _ = io.WriteString(conn, "*3\r\n:1\r\n:1\r\n:2\r\n") + default: + _, _ = io.WriteString(conn, "-ERR unexpected command\r\n") + } + } +} + +func readRedisCommand(reader *bufio.Reader) ([]string, error) { + line, err := reader.ReadString('\n') + if err != nil { + return nil, err + } + if len(line) < 2 || line[0] != '*' { + return nil, fmt.Errorf("invalid Redis array header %q", line) + } + count, err := strconv.Atoi(strings.TrimSpace(line[1:])) + if err != nil { + return nil, err + } + command := make([]string, 0, count) + for i := 0; i < count; i++ { + line, err = reader.ReadString('\n') + if err != nil { + return nil, err + } + if len(line) < 2 || line[0] != '$' { + return nil, fmt.Errorf("invalid Redis bulk header %q", line) + } + length, err := strconv.Atoi(strings.TrimSpace(line[1:])) + if err != nil { + return nil, err + } + value := make([]byte, length+2) + if _, err := io.ReadFull(reader, value); err != nil { + return nil, err + } + command = append(command, string(value[:length])) + } + return command, nil +} diff --git a/config/validate.go b/config/validate.go index 2bcc09be..b94740da 100644 --- a/config/validate.go +++ b/config/validate.go @@ -2,6 +2,8 @@ package config import ( "fmt" + "net" + "strconv" "strings" ) @@ -25,6 +27,45 @@ func Validate(cfg *Config) error { if cfg.HTTP.LLHLS.Enabled && cfg.HTTP.LLHLS.SegmentDuration <= 0 { return fmt.Errorf("http_stream.llhls.segment_duration must be greater than zero") } + if cfg.Stream.RingBufferSize <= 0 { + return fmt.Errorf("stream.ring_buffer_size must be greater than zero") + } + if cfg.Stream.GOPCacheNum < 0 { + return fmt.Errorf("stream.gop_cache_num must not be negative") + } + if cfg.Stream.GOPCacheMaxFrames < 0 { + return fmt.Errorf("stream.gop_cache_max_frames must not be negative") + } + if cfg.Stream.GOPCacheMaxDuration < 0 { + return fmt.Errorf("stream.gop_cache_max_duration must not be negative") + } + if cfg.Stream.GOPCacheMaxBytes < 0 { + return fmt.Errorf("stream.gop_cache_max_bytes must not be negative") + } + if cfg.Stream.GOPCache && cfg.Stream.GOPCacheNum > 0 && + cfg.Stream.GOPCacheMaxFrames == 0 && cfg.Stream.GOPCacheMaxBytes == 0 { + return fmt.Errorf("stream.gop_cache_max_frames or stream.gop_cache_max_bytes must be positive when GOP cache is enabled") + } + if err := ValidateRecordConfig(cfg.Record); err != nil { + return err + } + if cfg.Metrics.StreamDetailLimit < 0 { + return fmt.Errorf("metrics.stream_detail_limit must not be negative") + } + for i, value := range cfg.Limits.RateLimit.TrustedProxies { + value = strings.TrimSpace(value) + if value == "" { + return fmt.Errorf("limits.rate_limit.trusted_proxies[%d] must not be empty", i) + } + if strings.Contains(value, "/") { + if _, _, err := net.ParseCIDR(value); err == nil { + continue + } + } else if net.ParseIP(value) != nil { + continue + } + return fmt.Errorf("limits.rate_limit.trusted_proxies[%d] must be an IP address or CIDR network", i) + } seenTokens := make(map[string]struct{}, len(cfg.API.Auth.Tokens)) for i, binding := range cfg.API.Auth.Tokens { @@ -45,6 +86,61 @@ func Validate(cfg *Config) error { return nil } +// ValidateRecordConfig checks recording-specific values without requiring a +// complete root configuration. Empty and zero max_size values disable size +// rotation; otherwise the value is a non-negative decimal byte count with an +// optional B, KB, MB, or GB suffix. +func ValidateRecordConfig(cfg RecordConfig) error { + switch format := strings.ToLower(strings.TrimSpace(cfg.Format)); format { + case "", "flv", "fmp4", "mp4", "ts", "hls": + default: + return fmt.Errorf("record.format must be flv, fmp4, mp4, ts, or hls") + } + if cfg.Segment.MaxSize == "" { + return nil + } + if _, err := ParseByteSize(cfg.Segment.MaxSize); err != nil { + return fmt.Errorf("record.segment.max_size: %w", err) + } + return nil +} + +// ParseByteSize parses a decimal byte count with an optional binary suffix. +func ParseByteSize(value string) (int64, error) { + value = strings.TrimSpace(strings.ToUpper(value)) + if value == "" { + return 0, nil + } + + multiplier := int64(1) + switch { + case strings.HasSuffix(value, "GB"): + value = strings.TrimSuffix(value, "GB") + multiplier = 1024 * 1024 * 1024 + case strings.HasSuffix(value, "MB"): + value = strings.TrimSuffix(value, "MB") + multiplier = 1024 * 1024 + case strings.HasSuffix(value, "KB"): + value = strings.TrimSuffix(value, "KB") + multiplier = 1024 + case strings.HasSuffix(value, "B"): + value = strings.TrimSuffix(value, "B") + } + if value == "" { + return 0, fmt.Errorf("must contain decimal digits") + } + for _, digit := range value { + if digit < '0' || digit > '9' { + return 0, fmt.Errorf("must be a non-negative integer with optional B, KB, MB, or GB suffix") + } + } + n, err := strconv.ParseInt(value, 10, 63) + if err != nil || n > (1<<63-1)/multiplier { + return 0, fmt.Errorf("is too large") + } + return n * multiplier, nil +} + func validAPIRole(role string) bool { switch role { case "viewer", "operator", "admin": diff --git a/configs/liveforge.yaml b/configs/liveforge.yaml index e4f90b71..892a9b9d 100644 --- a/configs/liveforge.yaml +++ b/configs/liveforge.yaml @@ -8,8 +8,10 @@ runtime: source: file poll_interval: 30s load_timeout: 10s + # Each source defaults to a 4 MiB complete-document/materialization limit. file: path: "" + max_bytes: 4194304 tls: cert_file: "" key_file: "" @@ -23,6 +25,8 @@ limits: enabled: false rate: 50 # requests per second per IP burst: 100 # max burst per IP + # Forwarded client-IP headers are ignored unless the direct peer is listed. + trusted_proxies: [] rtmp: enabled: true listen: ":1935" @@ -107,10 +111,12 @@ sip: rtp_port_range: [30000, 30100] codecs: [opus, PCMA, PCMU] max_calls: 100 + max_lab_sessions: 16 gb28181: enabled: true stream_prefix: "gb28181" rtp_port_range: [40000, 50000] + max_lab_sessions: 16 ssrc: prefix: "34020" keepalive: @@ -124,6 +130,9 @@ audio_codec: stream: gop_cache: true gop_cache_num: 1 + gop_cache_max_frames: 300 + gop_cache_max_duration: 10s + gop_cache_max_bytes: 33554432 ring_buffer_size: 1024 idle_timeout: 30s no_publisher_timeout: 15s @@ -231,6 +240,12 @@ metrics: enabled: false listen: ":9090" path: "/metrics" + # Per-stream labels are opt-in. Without an allowlist, the limit is a + # Collector-lifetime budget whose admitted stream keys are never replaced. + # Set the limit to 0 to disable details; negative values are rejected. + stream_detail: false + stream_detail_limit: 100 + stream_detail_allowlist: [] api: enabled: true listen: ":8090" diff --git a/core/event_bus.go b/core/event_bus.go index 8c59c25e..59de1b01 100644 --- a/core/event_bus.go +++ b/core/event_bus.go @@ -1,6 +1,7 @@ package core import ( + "context" "errors" "fmt" "log/slog" @@ -12,12 +13,15 @@ import ( const ( maxLifecycleQueueDepth = 8 maxLifecycleLanes = 4096 + maxAsyncDispatches = 1024 ) var ErrAsyncBackpressure = errors.New("event bus async lifecycle capacity exceeded") type AsyncDispatchStats struct { Rejected uint64 + InFlight int + Capacity int } // EventBus dispatches events to registered hook handlers. @@ -29,6 +33,9 @@ type EventBus struct { lifecycleLanes map[lifecycleLaneKey]*lifecycleLane autoConsumers map[autoConsumerKey]uint64 asyncRejected atomic.Uint64 + dispatchMu sync.Mutex + pendingAsync int + asyncIdle chan struct{} } type lifecycleLaneKey struct { @@ -39,13 +46,15 @@ type lifecycleLaneKey struct { } type lifecycleDispatch struct { - ctx *EventContext - hook HookRegistration + ctx *EventContext + hook HookRegistration + terminal bool } type lifecycleLane struct { - queue []lifecycleDispatch - running bool + queue []lifecycleDispatch + running bool + closeWhenEmpty bool } type autoConsumerKey struct { @@ -55,10 +64,13 @@ type autoConsumerKey struct { // NewEventBus creates a new EventBus. func NewEventBus() *EventBus { + idle := make(chan struct{}) + close(idle) return &EventBus{ hooks: make(map[EventType][]HookRegistration), lifecycleLanes: make(map[lifecycleLaneKey]*lifecycleLane), autoConsumers: make(map[autoConsumerKey]uint64), + asyncIdle: idle, } } @@ -111,20 +123,95 @@ func (b *EventBus) EmitSync(event EventType, ctx *EventContext) error { // than a partial start/stop delivery when capacity is exhausted. func (b *EventBus) EmitAsync(event EventType, ctx *EventContext) error { hooks := asyncHooks(b.snapshot(event)) + if key, ok := eventLifecycleKey(event, ctx); ok { + terminalCounts := b.terminalConsumerCounts(event) + if len(hooks) > 0 || len(terminalCounts) > 0 { + return b.enqueueLifecycle(event, key, hooks, ctx, terminalCounts) + } + return nil + } if len(hooks) == 0 { return nil } - if key, ok := eventLifecycleKey(event, ctx); ok { - return b.enqueueLifecycle(key, hooks, ctx) + if !b.tryBeginAsync(len(hooks)) { + b.asyncRejected.Add(1) + return ErrAsyncBackpressure } for _, hook := range hooks { - go runAsyncHook(hook, cloneEventContext(ctx)) + go b.runTrackedAsyncHook(hook, cloneEventContext(ctx)) } return nil } +// Drain waits for all asynchronous hook dispatches accepted before the bus +// becomes idle. Callers must stop event producers before relying on an idle +// result as a shutdown barrier. +func (b *EventBus) Drain(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + b.dispatchMu.Lock() + idle := b.asyncIdle + b.dispatchMu.Unlock() + select { + case <-idle: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (b *EventBus) beginAsync(count int) { + if count <= 0 { + return + } + b.dispatchMu.Lock() + if b.pendingAsync == 0 { + b.asyncIdle = make(chan struct{}) + } + b.pendingAsync += count + b.dispatchMu.Unlock() +} + +func (b *EventBus) tryBeginAsync(count int) bool { + if count <= 0 { + return true + } + b.dispatchMu.Lock() + defer b.dispatchMu.Unlock() + if b.pendingAsync+count > maxAsyncDispatches { + return false + } + if b.pendingAsync == 0 { + b.asyncIdle = make(chan struct{}) + } + b.pendingAsync += count + return true +} + +func (b *EventBus) completeAsync() { + b.dispatchMu.Lock() + b.pendingAsync-- + if b.pendingAsync == 0 { + close(b.asyncIdle) + } + b.dispatchMu.Unlock() +} + +func (b *EventBus) runTrackedAsyncHook(hook HookRegistration, ctx *EventContext) { + defer b.completeAsync() + runAsyncHook(hook, ctx) +} + func (b *EventBus) AsyncStats() AsyncDispatchStats { - return AsyncDispatchStats{Rejected: b.asyncRejected.Load()} + b.dispatchMu.Lock() + inFlight := b.pendingAsync + b.dispatchMu.Unlock() + return AsyncDispatchStats{ + Rejected: b.asyncRejected.Load(), + InFlight: inFlight, + Capacity: maxAsyncDispatches, + } } func asyncHooks(hooks []HookRegistration) []HookRegistration { @@ -168,24 +255,58 @@ func lifecycleFamily(event EventType) (uint8, bool) { } } -func (b *EventBus) enqueueLifecycle(base lifecycleLaneKey, hooks []HookRegistration, ctx *EventContext) error { +func (b *EventBus) terminalConsumerCounts(event EventType) map[string]int { + var terminal EventType + switch event { + case EventPublish: + terminal = EventPublishStop + case EventSubscribe: + terminal = EventSubscribeStop + default: + return nil + } + consumers := make(map[string]int) + for _, hook := range asyncHooks(b.snapshot(terminal)) { + consumers[hook.Consumer]++ + } + return consumers +} + +func (b *EventBus) enqueueLifecycle(event EventType, base lifecycleLaneKey, hooks []HookRegistration, ctx *EventContext, terminalCounts map[string]int) error { type laneStart struct { key lifecycleLaneKey lane *lifecycleLane } counts := make(map[lifecycleLaneKey]int, len(hooks)) + holdOpen := make(map[lifecycleLaneKey]bool, len(hooks)) + terminal := event == EventPublishStop || event == EventSubscribeStop for _, hook := range hooks { key := base key.consumer = hook.Consumer counts[key]++ + holdOpen[key] = terminalCounts[hook.Consumer] > 0 + } + if !terminal { + for consumer := range terminalCounts { + key := base + key.consumer = consumer + if _, exists := counts[key]; !exists { + counts[key] = 0 + } + holdOpen[key] = true + } } b.asyncMu.Lock() newLanes := 0 for key, count := range counts { lane := b.lifecycleLanes[key] + limit := maxLifecycleQueueDepth + if !terminal { + limit -= terminalCounts[key.consumer] + } if lane == nil { - if count > maxLifecycleQueueDepth { + if count > limit { b.asyncMu.Unlock() b.asyncRejected.Add(1) return ErrAsyncBackpressure @@ -193,7 +314,7 @@ func (b *EventBus) enqueueLifecycle(base lifecycleLaneKey, hooks []HookRegistrat newLanes++ continue } - if len(lane.queue)+count > maxLifecycleQueueDepth { + if len(lane.queue)+count > limit { b.asyncMu.Unlock() b.asyncRejected.Add(1) return ErrAsyncBackpressure @@ -205,17 +326,29 @@ func (b *EventBus) enqueueLifecycle(base lifecycleLaneKey, hooks []HookRegistrat return ErrAsyncBackpressure } starts := make([]laneStart, 0, newLanes) - for _, hook := range hooks { - key := base - key.consumer = hook.Consumer + for key, count := range counts { lane := b.lifecycleLanes[key] if lane == nil { - lane = &lifecycleLane{running: true} + lane = &lifecycleLane{running: count > 0, closeWhenEmpty: terminal || !holdOpen[key]} b.lifecycleLanes[key] = lane + if lane.running { + starts = append(starts, laneStart{key: key, lane: lane}) + } + } else if count > 0 && !lane.running { + lane.running = true starts = append(starts, laneStart{key: key, lane: lane}) } - lane.queue = append(lane.queue, lifecycleDispatch{ctx: cloneEventContext(ctx), hook: hook}) + if !terminal && holdOpen[key] { + lane.closeWhenEmpty = false + } } + for _, hook := range hooks { + key := base + key.consumer = hook.Consumer + lane := b.lifecycleLanes[key] + lane.queue = append(lane.queue, lifecycleDispatch{ctx: cloneEventContext(ctx), hook: hook, terminal: terminal}) + } + b.beginAsync(len(hooks)) b.asyncMu.Unlock() for _, start := range starts { go b.runLifecycleLane(start.key, start.lane) @@ -228,7 +361,7 @@ func (b *EventBus) runLifecycleLane(key lifecycleLaneKey, lane *lifecycleLane) { b.asyncMu.Lock() if len(lane.queue) == 0 { lane.running = false - if b.lifecycleLanes[key] == lane { + if lane.closeWhenEmpty && b.lifecycleLanes[key] == lane { delete(b.lifecycleLanes, key) } b.asyncMu.Unlock() @@ -239,7 +372,12 @@ func (b *EventBus) runLifecycleLane(key lifecycleLaneKey, lane *lifecycleLane) { lane.queue = lane.queue[1:] b.asyncMu.Unlock() - runAsyncHook(dispatch.hook, dispatch.ctx) + b.runTrackedAsyncHook(dispatch.hook, dispatch.ctx) + if dispatch.terminal { + b.asyncMu.Lock() + lane.closeWhenEmpty = true + b.asyncMu.Unlock() + } } } diff --git a/core/event_bus_test.go b/core/event_bus_test.go index 6bfab549..5e0fe47f 100644 --- a/core/event_bus_test.go +++ b/core/event_bus_test.go @@ -1,6 +1,7 @@ package core import ( + "context" "errors" "sync" "sync/atomic" @@ -8,6 +9,169 @@ import ( "time" ) +func TestEventBusReservesTerminalOnlyConsumerLaneOnStart(t *testing.T) { + bus := NewEventBus() + started := make(chan struct{}, 1) + recordStopped := make(chan struct{}, 1) + httpStopped := make(chan struct{}, 1) + bus.Register(HookRegistration{Event: EventPublish, Mode: HookAsync, Consumer: "record", Handler: func(*EventContext) error { + started <- struct{}{} + return nil + }}) + bus.Register(HookRegistration{Event: EventPublishStop, Mode: HookAsync, Consumer: "record", Handler: func(*EventContext) error { + recordStopped <- struct{}{} + return nil + }}) + bus.Register(HookRegistration{Event: EventPublishStop, Mode: HookAsync, Consumer: "httpstream", Handler: func(*EventContext) error { + httpStopped <- struct{}{} + return nil + }}) + ctx := &EventContext{StreamKey: "live/terminal-reservation", PublisherID: "publisher-1"} + + if err := bus.EmitAsync(EventPublish, ctx); err != nil { + t.Fatal(err) + } + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("publish start hook did not run") + } + + bus.asyncMu.Lock() + reserved := len(bus.lifecycleLanes) + bus.asyncMu.Unlock() + if reserved != 2 { + t.Fatalf("reserved lifecycle lanes = %d, want start and terminal-only consumers", reserved) + } + + if err := bus.EmitAsync(EventPublishStop, ctx); err != nil { + t.Fatal(err) + } + for name, done := range map[string]<-chan struct{}{"record": recordStopped, "httpstream": httpStopped} { + select { + case <-done: + case <-time.After(time.Second): + t.Fatalf("%s terminal hook did not run", name) + } + } + waitEventBusLanesReleased(t, bus) +} + +func TestEventBusReservesTerminalOnlyLaneWithoutStartHooks(t *testing.T) { + bus := NewEventBus() + stopped := make(chan struct{}, 1) + bus.Register(HookRegistration{Event: EventPublishStop, Mode: HookAsync, Consumer: "record", Handler: func(*EventContext) error { + stopped <- struct{}{} + return nil + }}) + ctx := &EventContext{StreamKey: "live/terminal-only", PublisherID: "publisher-1"} + + if err := bus.EmitAsync(EventPublish, ctx); err != nil { + t.Fatal(err) + } + bus.asyncMu.Lock() + reserved := len(bus.lifecycleLanes) + bus.asyncMu.Unlock() + if reserved != 1 { + t.Fatalf("reserved lifecycle lanes = %d, want terminal-only consumer", reserved) + } + + if err := bus.EmitAsync(EventPublishStop, ctx); err != nil { + t.Fatal(err) + } + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("terminal-only hook did not run") + } + waitEventBusLanesReleased(t, bus) +} + +func TestEventBusReservesEveryTerminalHookSlotForConsumer(t *testing.T) { + bus := NewEventBus() + entered := make(chan struct{}) + release := make(chan struct{}) + var starts atomic.Int32 + var stops atomic.Int32 + bus.Register(HookRegistration{Event: EventPublish, Mode: HookAsync, Consumer: "record", Handler: func(*EventContext) error { + if starts.Add(1) == 1 { + close(entered) + <-release + } + return nil + }}) + for range 2 { + bus.Register(HookRegistration{Event: EventPublishStop, Mode: HookAsync, Consumer: "record", Handler: func(*EventContext) error { + stops.Add(1) + return nil + }}) + } + ctx := &EventContext{StreamKey: "live/terminal-slots", PublisherID: "publisher-1"} + if err := bus.EmitAsync(EventPublish, ctx); err != nil { + t.Fatal(err) + } + <-entered + for i := 0; i < maxLifecycleQueueDepth-2; i++ { + if err := bus.EmitAsync(EventPublish, ctx); err != nil { + t.Fatalf("start queue admission %d: %v", i, err) + } + } + if err := bus.EmitAsync(EventPublish, ctx); !errors.Is(err, ErrAsyncBackpressure) { + t.Fatalf("start consumed terminal reservation: %v", err) + } + if err := bus.EmitAsync(EventPublishStop, ctx); err != nil { + t.Fatalf("terminal hooks did not fit reserved slots: %v", err) + } + close(release) + waitEventBusLanesReleased(t, bus) + if got := stops.Load(); got != 2 { + t.Fatalf("terminal hook calls = %d, want 2", got) + } +} + +func TestEventBusDrainWaitsForAsyncHooksAndHonorsContext(t *testing.T) { + bus := NewEventBus() + entered := make(chan struct{}) + release := make(chan struct{}) + bus.Register(HookRegistration{Event: EventStreamAlive, Mode: HookAsync, Handler: func(*EventContext) error { + close(entered) + <-release + return nil + }}) + if err := bus.EmitAsync(EventStreamAlive, &EventContext{StreamKey: "live/drain"}); err != nil { + t.Fatal(err) + } + <-entered + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := bus.Drain(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Drain() error = %v, want context deadline", err) + } + + close(release) + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := bus.Drain(ctx); err != nil { + t.Fatalf("Drain() after release: %v", err) + } +} + +func TestEventBusDrainCompletesAfterPanickingHook(t *testing.T) { + bus := NewEventBus() + bus.Register(HookRegistration{Event: EventStreamAlive, Mode: HookAsync, Handler: func(*EventContext) error { + panic("expected test panic") + }}) + if err := bus.EmitAsync(EventStreamAlive, &EventContext{StreamKey: "live/panic-drain"}); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := bus.Drain(ctx); err != nil { + t.Fatalf("Drain() after panic: %v", err) + } +} + func TestEventBusSyncHook(t *testing.T) { bus := NewEventBus() var called int32 @@ -161,12 +325,15 @@ func TestEventBusLifecycleQueueBackpressureIsBoundedAndObservable(t *testing.T) } return nil }}) + bus.Register(HookRegistration{Event: EventPublishStop, Mode: HookAsync, Consumer: "blocked", Handler: func(*EventContext) error { + return nil + }}) ctx := &EventContext{StreamKey: "live/saturated", PublisherID: "publisher-1"} if err := bus.EmitAsync(EventPublish, ctx); err != nil { t.Fatalf("initial lifecycle admission: %v", err) } <-entered - for i := 0; i < maxLifecycleQueueDepth; i++ { + for i := 0; i < maxLifecycleQueueDepth-1; i++ { if err := bus.EmitAsync(EventPublish, ctx); err != nil { t.Fatalf("queue admission %d: %v", i, err) } @@ -174,15 +341,82 @@ func TestEventBusLifecycleQueueBackpressureIsBoundedAndObservable(t *testing.T) if err := bus.EmitAsync(EventPublish, ctx); !errors.Is(err, ErrAsyncBackpressure) { t.Fatalf("saturated admission error = %v, want %v", err, ErrAsyncBackpressure) } + if err := bus.EmitAsync(EventPublishStop, ctx); err != nil { + t.Fatalf("terminal lifecycle event did not use its reserved queue slot: %v", err) + } if got := bus.AsyncStats().Rejected; got != 1 { t.Fatalf("rejected dispatches = %d, want 1", got) } close(release) waitEventBusLanesReleased(t, bus) - if got := calls.Load(); got != int32(maxLifecycleQueueDepth+1) { - t.Fatalf("accepted lifecycle calls = %d, want %d", got, maxLifecycleQueueDepth+1) + if got := calls.Load(); got != int32(maxLifecycleQueueDepth) { + t.Fatalf("accepted publish calls = %d, want %d", got, maxLifecycleQueueDepth) + } +} + +func TestEventBusRetainsIdleLifecycleLaneUntilTerminalEvent(t *testing.T) { + bus := NewEventBus() + started := make(chan struct{}) + stopped := make(chan struct{}) + bus.Register(HookRegistration{Event: EventPublish, Mode: HookAsync, Consumer: "record", Handler: func(*EventContext) error { + close(started) + return nil + }}) + bus.Register(HookRegistration{Event: EventPublishStop, Mode: HookAsync, Consumer: "record", Handler: func(*EventContext) error { + close(stopped) + return nil + }}) + ctx := &EventContext{StreamKey: "live/idle-lane", PublisherID: "publisher-1"} + if err := bus.EmitAsync(EventPublish, ctx); err != nil { + t.Fatal(err) + } + <-started + + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + bus.asyncMu.Lock() + laneCount := len(bus.lifecycleLanes) + var running bool + for _, lane := range bus.lifecycleLanes { + running = lane.running + } + bus.asyncMu.Unlock() + if laneCount == 1 && !running { + break + } + time.Sleep(time.Millisecond) + } + bus.asyncMu.Lock() + retained := len(bus.lifecycleLanes) == 1 + bus.asyncMu.Unlock() + if !retained { + t.Fatal("lifecycle lane was released before its terminal event") + } + + if err := bus.EmitAsync(EventPublishStop, ctx); err != nil { + t.Fatal(err) + } + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("terminal lifecycle event did not run") + } + waitEventBusLanesReleased(t, bus) +} + +func TestEventBusReleasesStartOnlyConsumerLane(t *testing.T) { + bus := NewEventBus() + done := make(chan struct{}) + bus.Register(HookRegistration{Event: EventSubscribe, Mode: HookAsync, Consumer: "origin-pull", Handler: func(*EventContext) error { + close(done) + return nil + }}) + if err := bus.EmitAsync(EventSubscribe, &EventContext{StreamKey: "live/pull", SubscriberID: "subscriber-1"}); err != nil { + t.Fatal(err) } + <-done + waitEventBusLanesReleased(t, bus) } func TestEventBusLifecycleWorkerRecoversPanicAndReleasesLane(t *testing.T) { @@ -256,15 +490,24 @@ func TestEventBusLifecycleGenerationsRunIndependentlyAndReleaseState(t *testing. }, }) - bus.EmitAsync(EventPublish, &EventContext{StreamKey: "live/generations", PublisherID: "publisher-1"}) + if err := bus.EmitAsync(EventPublish, &EventContext{StreamKey: "live/generations", PublisherID: "publisher-1"}); err != nil { + t.Fatal(err) + } <-blocked - bus.EmitAsync(EventPublishStop, &EventContext{StreamKey: "live/generations", PublisherID: "publisher-1"}) - bus.EmitAsync(EventPublish, &EventContext{StreamKey: "live/generations", PublisherID: "publisher-2"}) + if err := bus.EmitAsync(EventPublishStop, &EventContext{StreamKey: "live/generations", PublisherID: "publisher-1"}); err != nil { + t.Fatal(err) + } + if err := bus.EmitAsync(EventPublish, &EventContext{StreamKey: "live/generations", PublisherID: "publisher-2"}); err != nil { + t.Fatal(err) + } select { case <-secondRan: case <-time.After(time.Second): t.Fatal("independent publisher generation was blocked") } + if err := bus.EmitAsync(EventPublishStop, &EventContext{StreamKey: "live/generations", PublisherID: "publisher-2"}); err != nil { + t.Fatal(err) + } close(release) select { case <-firstDone: @@ -390,6 +633,36 @@ func TestEventBusAsyncHook(t *testing.T) { } } +func TestEventBusBoundsNonLifecycleAsyncDispatch(t *testing.T) { + bus := NewEventBus() + release := make(chan struct{}) + bus.Register(HookRegistration{ + Event: EventStreamAlive, + Mode: HookAsync, + Handler: func(*EventContext) error { + <-release + return nil + }, + }) + ctx := &EventContext{StreamKey: "live/async-capacity"} + for i := 0; i < maxAsyncDispatches; i++ { + if err := bus.EmitAsync(EventStreamAlive, ctx); err != nil { + t.Fatalf("async admission %d: %v", i, err) + } + } + if err := bus.EmitAsync(EventStreamAlive, ctx); !errors.Is(err, ErrAsyncBackpressure) { + t.Fatalf("over-capacity async admission = %v, want %v", err, ErrAsyncBackpressure) + } + stats := bus.AsyncStats() + if stats.InFlight != maxAsyncDispatches || stats.Capacity != maxAsyncDispatches { + t.Fatalf("async pressure stats = %+v, want in-flight/capacity %d", stats, maxAsyncDispatches) + } + close(release) + if err := bus.Drain(context.Background()); err != nil { + t.Fatalf("drain after releasing async hooks: %v", err) + } +} + func TestEventBusSeparatesSyncAuthorizationFromAsyncLifecycle(t *testing.T) { bus := NewEventBus() actionStarted := false diff --git a/core/frame_reader.go b/core/frame_reader.go new file mode 100644 index 00000000..f4ed43c6 --- /dev/null +++ b/core/frame_reader.go @@ -0,0 +1,63 @@ +package core + +import ( + "context" + "errors" + "fmt" + + "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/im-pingo/liveforge/pkg/util" +) + +// ErrReaderOverwritten identifies a reader that lost continuity before it +// could consume the retained frame at its cursor. +var ErrReaderOverwritten = errors.New("stream reader overwritten") + +// ReaderOverwrittenError reports how many ring positions were lost. The +// reader is advanced to the live cursor before this error is returned. +type ReaderOverwrittenError struct { + Count int64 +} + +func (e *ReaderOverwrittenError) Error() string { + return fmt.Sprintf("%s by %d frame(s)", ErrReaderOverwritten, e.Count) +} + +func (e *ReaderOverwrittenError) Is(target error) bool { + return target == ErrReaderOverwritten +} + +// ReadFrameContext reads one source frame while preserving the atomic +// overwrite result. A continuity loss never returns the retained slot as if +// it were valid media. +func ReadFrameContext(ctx context.Context, reader *util.RingReader[*avframe.AVFrame]) (*avframe.AVFrame, bool, error) { + if reader == nil { + return nil, false, errors.New("nil stream reader") + } + result := reader.ReadResultContext(ctx) + if !result.OK { + return nil, false, nil + } + if result.Overwritten > 0 { + reader.AdvanceToLive() + return nil, false, &ReaderOverwrittenError{Count: result.Overwritten} + } + return result.Value, true, nil +} + +// TryReadFrame reads one immediately available source frame with the same +// overwrite contract as ReadFrameContext. +func TryReadFrame(reader *util.RingReader[*avframe.AVFrame]) (*avframe.AVFrame, bool, error) { + if reader == nil { + return nil, false, errors.New("nil stream reader") + } + result := reader.TryReadResult() + if !result.OK { + return nil, false, nil + } + if result.Overwritten > 0 { + reader.AdvanceToLive() + return nil, false, &ReaderOverwrittenError{Count: result.Overwritten} + } + return result.Value, true, nil +} diff --git a/core/frame_reader_test.go b/core/frame_reader_test.go new file mode 100644 index 00000000..5dbc33b7 --- /dev/null +++ b/core/frame_reader_test.go @@ -0,0 +1,35 @@ +package core + +import ( + "context" + "errors" + "testing" + + "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/im-pingo/liveforge/pkg/util" +) + +func TestReadFrameContextRejectsOverwrittenRetainedFrame(t *testing.T) { + ring := util.NewRingBuffer[*avframe.AVFrame](2) + reader := ring.NewReaderAt(0) + for i := 0; i < 4; i++ { + ring.Write(avframe.NewAVFrame( + avframe.MediaTypeVideo, + avframe.CodecH264, + avframe.FrameTypeInterframe, + int64(i), int64(i), []byte{byte(i)}, + )) + } + + frame, ok, err := ReadFrameContext(context.Background(), reader) + if frame != nil || ok { + t.Fatalf("overwritten read = frame=%v ok=%v, want no retained frame", frame, ok) + } + var overwritten *ReaderOverwrittenError + if !errors.As(err, &overwritten) || overwritten.Count != 2 { + t.Fatalf("overwritten error = %v, want count 2", err) + } + if got, want := reader.ReadCursor(), ring.WriteCursor(); got != want { + t.Fatalf("reader cursor = %d, want live cursor %d", got, want) + } +} diff --git a/core/module.go b/core/module.go index 92b83f3d..47d7dca9 100644 --- a/core/module.go +++ b/core/module.go @@ -39,13 +39,15 @@ const ( // EventContext carries event data passed to hook handlers. type EventContext struct { - StreamKey string - PublisherID string - SubscriberID string - Protocol string - RemoteAddr string - Params map[string]string // URL query params (e.g. "token" -> "xxx") - Extra map[string]any + StreamKey string + StreamInstanceID uint64 + PublisherGeneration uint64 + PublisherID string + SubscriberID string + Protocol string + RemoteAddr string + Params map[string]string // URL query params (e.g. "token" -> "xxx") + Extra map[string]any } // EventHandler is a function that handles an event. diff --git a/core/muxer_manager.go b/core/muxer_manager.go index 68c55aeb..a85c4d7a 100644 --- a/core/muxer_manager.go +++ b/core/muxer_manager.go @@ -78,6 +78,23 @@ func (mm *MuxerManager) GetOrCreateMuxer(format string) (*SharedBufferReader, *M defer mm.mu.Unlock() generation, active := mm.stream.activePublisherGeneration() + return mm.getOrCreateMuxerLocked(format, generation, active) +} + +// GetOrCreateMuxerForGeneration returns a muxer only while the requested +// publisher generation is still active. +func (mm *MuxerManager) GetOrCreateMuxerForGeneration(format string, generation uint64) (*SharedBufferReader, *MuxerInstance) { + mm.mu.Lock() + defer mm.mu.Unlock() + + activeGeneration, active := mm.stream.activePublisherGeneration() + if !active || activeGeneration != generation { + return nil, nil + } + return mm.getOrCreateMuxerLocked(format, generation, true) +} + +func (mm *MuxerManager) getOrCreateMuxerLocked(format string, generation uint64, active bool) (*SharedBufferReader, *MuxerInstance) { inst, ok := mm.muxers[format] if !active { if ok { diff --git a/core/muxer_manager_test.go b/core/muxer_manager_test.go index 0456a817..71921d3d 100644 --- a/core/muxer_manager_test.go +++ b/core/muxer_manager_test.go @@ -259,6 +259,33 @@ func TestMuxerManagerGenerationReplacementAndInstanceRelease(t *testing.T) { } } +func TestMuxerManagerRejectsStaleRequestedGeneration(t *testing.T) { + stream := NewStream("live/muxer-stale-request", newTestStreamConfig(), config.LimitsConfig{}, NewEventBus()) + pubA := &testPublisher{id: "publisher-a", info: &avframe.MediaInfo{AudioCodec: avframe.CodecMP3}} + if err := stream.SetPublisher(pubA); err != nil { + t.Fatal(err) + } + generationA := stream.StartupSnapshot().Generation + if !stream.RemovePublisherIf(pubA) { + t.Fatal("publisher A was not removed") + } + pubB := &testPublisher{id: "publisher-b", info: &avframe.MediaInfo{AudioCodec: avframe.CodecMP3}} + if err := stream.SetPublisher(pubB); err != nil { + t.Fatal(err) + } + mm := NewMuxerManager(stream, 256) + starts := 0 + mm.RegisterMuxerStart("flv", func(*MuxerInstance, *Stream) { starts++ }) + + reader, inst := mm.GetOrCreateMuxerForGeneration("flv", generationA) + if reader != nil || inst != nil { + t.Fatalf("stale generation muxer = (%p, %p), want nil", reader, inst) + } + if starts != 0 || mm.SubscriberCount("flv") != 0 { + t.Fatalf("stale request started muxer: starts=%d subscribers=%d", starts, mm.SubscriberCount("flv")) + } +} + func TestMuxerManagerSubscriberCountNonExistent(t *testing.T) { bus := NewEventBus() cfg := newTestStreamConfig() diff --git a/core/production_hotpath_bench_test.go b/core/production_hotpath_bench_test.go new file mode 100644 index 00000000..35395c4b --- /dev/null +++ b/core/production_hotpath_bench_test.go @@ -0,0 +1,133 @@ +package core + +import ( + "bytes" + "testing" + "time" + + "github.com/im-pingo/liveforge/config" + "github.com/im-pingo/liveforge/pkg/avframe" +) + +const productionIngressFixtureDurationMillis int64 = 64_000 + +type productionIngressBenchmarkFrame struct { + frame *avframe.AVFrame + dts int64 +} + +func BenchmarkStreamIngressProductionStablePublisher(b *testing.B) { + stream, publisher, frames, wantPayloads, startCursor := newProductionIngressBenchmark(b) + frameIndex := 4 // The first four frames form the validated startup GOP. + var cycleOffset int64 + + b.ReportAllocs() + b.ResetTimer() + for range b.N { + fixture := frames[frameIndex] + fixture.frame.DTS = fixture.dts + cycleOffset + fixture.frame.PTS = fixture.dts + cycleOffset + if !stream.WriteFrameForPublisher(publisher, fixture.frame) { + b.Fatal("production ingress rejected the active publisher frame") + } + frameIndex++ + if frameIndex == len(frames) { + frameIndex = 0 + cycleOffset += productionIngressFixtureDurationMillis + } + } + b.StopTimer() + + if advanced := stream.RingBuffer().WriteCursor() - startCursor; advanced != int64(b.N) { + b.Fatalf("production ingress advanced ring by %d frames, want %d", advanced, b.N) + } + snapshot := stream.StartupSnapshot() + if !snapshot.Ready || len(snapshot.ReplayFrames) == 0 || + snapshot.ReplayFrames[0].FrameType != avframe.FrameTypeKeyframe || + len(snapshot.ReplayFrames) > 300 { + b.Fatalf("production ingress left invalid bounded GOP state: ready=%v replay=%d", snapshot.Ready, len(snapshot.ReplayFrames)) + } + if !bytes.Equal(frames[0].frame.Payload, wantPayloads[0]) || + !bytes.Equal(frames[1].frame.Payload, wantPayloads[1]) || + !bytes.Equal(frames[3].frame.Payload, wantPayloads[2]) { + b.Fatal("production ingress mutated a fixture payload") + } +} + +func newProductionIngressBenchmark(b *testing.B) (*Stream, Publisher, []productionIngressBenchmarkFrame, [3][]byte, int64) { + b.Helper() + cfg := config.StreamConfig{ + GOPCache: true, + GOPCacheNum: 2, + GOPCacheMaxFrames: 300, + GOPCacheMaxDuration: 10 * time.Second, + GOPCacheMaxBytes: 32 * 1024 * 1024, + RingBufferSize: 4096, + } + stream := NewStream("bench/production-ingress", cfg, config.LimitsConfig{}, NewEventBus()) + b.Cleanup(stream.Close) + publisher := &testPublisher{id: "production-benchmark-publisher", info: &avframe.MediaInfo{ + VideoCodec: avframe.CodecH264, + AudioCodec: avframe.CodecG711A, + SampleRate: 8000, + Channels: 1, + VideoSequenceHeader: []byte{1, 0x42, 0, 0x1f, 0xff, 0xe1, 0, 1, 0x67, 1, 0, 1, 0x68}, + }} + if err := stream.SetPublisher(publisher); err != nil { + b.Fatal(err) + } + + frames, wantPayloads := productionIngressFrames() + for _, fixture := range frames[:4] { + if !stream.WriteFrameForPublisher(publisher, fixture.frame) { + b.Fatal("production ingress fixture rejected its preflight frame") + } + } + snapshot := stream.StartupSnapshot() + if !snapshot.Ready || len(snapshot.ReplayFrames) != 4 || + !snapshot.ReplayFrames[0].MediaType.IsVideo() || + !snapshot.ReplayFrames[1].MediaType.IsAudio() || + !snapshot.ReplayFrames[2].MediaType.IsAudio() || + !snapshot.ReplayFrames[3].MediaType.IsVideo() { + b.Fatalf("production ingress preflight did not create an interleaved playable GOP: ready=%v replay=%d", snapshot.Ready, len(snapshot.ReplayFrames)) + } + return stream, publisher, frames, wantPayloads, stream.RingBuffer().WriteCursor() +} + +func productionIngressFrames() ([]productionIngressBenchmarkFrame, [3][]byte) { + keyPayload := make([]byte, 1200) + keyPayload[0] = 0x65 + interPayload := make([]byte, 1200) + interPayload[0] = 0x41 + audioPayload := make([]byte, 160) + for index := range audioPayload { + audioPayload[index] = byte(index) + } + + frames := make([]productionIngressBenchmarkFrame, 0, 4800) + for timestamp := int64(0); timestamp < productionIngressFixtureDurationMillis; timestamp += 20 { + if timestamp%40 == 0 { + frameType := avframe.FrameTypeInterframe + payload := interPayload + if timestamp%1000 == 0 { + frameType = avframe.FrameTypeKeyframe + payload = keyPayload + } + frames = append(frames, productionIngressBenchmarkFrame{ + frame: avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, frameType, timestamp, timestamp, payload), + dts: timestamp, + }) + } + frames = append(frames, productionIngressBenchmarkFrame{ + frame: avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecG711A, avframe.FrameTypeInterframe, timestamp, timestamp, audioPayload), + dts: timestamp, + }) + } + + wantPayloads := [3][]byte{ + append([]byte(nil), keyPayload...), + append([]byte(nil), audioPayload...), + append([]byte(nil), interPayload...), + } + return frames, wantPayloads +} diff --git a/core/reload_test.go b/core/reload_test.go index 3e6e10c6..a832abc6 100644 --- a/core/reload_test.go +++ b/core/reload_test.go @@ -67,3 +67,221 @@ func TestStreamUpdatePolicyReconcilesGOPCacheAndTimeouts(t *testing.T) { t.Fatal("updated no-publisher timeout did not replace the active timer") } } + +func TestStreamUpdatePolicyRelaxationReopensCurrentGOPForFutureFrames(t *testing.T) { + tests := []struct { + name string + tighten func(*config.StreamConfig) + relax func(*config.StreamConfig) + }{ + { + name: "frames", + tighten: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxFrames = 2 + }, + relax: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxFrames = 4 + }, + }, + { + name: "bytes", + tighten: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxBytes = 4 + }, + relax: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxBytes = 7 + }, + }, + { + name: "duration", + tighten: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxDuration = 20 * time.Millisecond + }, + relax: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxDuration = 60 * time.Millisecond + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := newTestStreamConfig() + stream := NewStream("live/policy-relax-"+test.name, cfg, config.LimitsConfig{}, NewEventBus()) + publisher := &testPublisher{ + id: "publisher-" + test.name, + info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264, AudioCodec: avframe.CodecAAC}, + } + if err := stream.SetPublisher(publisher); err != nil { + t.Fatal(err) + } + + keyframe := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 0, 0, []byte{1, 2}) + initialAudio := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 20, 20, []byte{3, 4}) + omittedVideo := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 40, 40, []byte{5}) + futureAudio := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 40, 40, []byte{6}) + futureVideo := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 60, 60, []byte{7, 8}) + afterRelaxedBound := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 80, 80, []byte{9}) + assertGOP := func(stage string, want ...*avframe.AVFrame) { + t.Helper() + got := stream.GOPCache() + if len(got) != len(want) { + t.Fatalf("%s GOP length = %d, want %d", stage, len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("%s GOP frame[%d] = %p, want %p", stage, i, got[i], want[i]) + } + } + } + + stream.WriteFrame(keyframe) + stream.WriteFrame(initialAudio) + tightened := cfg + test.tighten(&tightened) + stream.UpdatePolicy(tightened, config.LimitsConfig{}) + stream.WriteFrame(omittedVideo) + assertGOP("tightened and sealed", keyframe, initialAudio) + + relaxed := cfg + test.relax(&relaxed) + stream.UpdatePolicy(relaxed, config.LimitsConfig{}) + assertGOP("relaxed before future frames", keyframe, initialAudio) + stream.WriteFrame(futureAudio) + stream.WriteFrame(futureVideo) + stream.WriteFrame(afterRelaxedBound) + assertGOP("relaxed at new bound", keyframe, initialAudio, futureAudio, futureVideo) + }) + } +} + +func TestStreamUpdatePolicyCombinesGOPBoundsAndRetainsOnlyPlayablePrefix(t *testing.T) { + cfg := newTestStreamConfig() + stream := NewStream("live/policy-combined-bounds", cfg, config.LimitsConfig{}, NewEventBus()) + + keyframe := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 0, 0, []byte{1, 2}) + initialAudio := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 20, 20, []byte{3, 4}) + omittedVideo := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 40, 40, []byte{5, 6, 7}) + omittedAudio := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 60, 60, []byte{8}) + for _, frame := range []*avframe.AVFrame{keyframe, initialAudio, omittedVideo, omittedAudio} { + stream.WriteFrame(frame) + } + + tightened := cfg + tightened.GOPCacheMaxFrames = 4 + tightened.GOPCacheMaxBytes = 6 + tightened.GOPCacheMaxDuration = 60 * time.Millisecond + stream.UpdatePolicy(tightened, config.LimitsConfig{}) + assertCachedFrames(t, stream, "combined bounds use shortest playable prefix", keyframe, initialAudio) + + relaxed := tightened + relaxed.GOPCacheMaxBytes = 8 + stream.UpdatePolicy(relaxed, config.LimitsConfig{}) + futureVideo := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 40, 40, []byte{9, 10, 11}) + futureAudio := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 60, 60, []byte{12}) + afterAllBounds := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 80, 80, []byte{13}) + stream.WriteFrame(futureVideo) + stream.WriteFrame(futureAudio) + stream.WriteFrame(afterAllBounds) + assertCachedFrames(t, stream, "relaxed byte bound preserves all remaining bounds", keyframe, initialAudio, futureVideo, futureAudio) +} + +func TestStreamUpdatePolicyRelaxationReopensOnlyActiveRetainedGOP(t *testing.T) { + cfg := newTestStreamConfig() + cfg.GOPCacheNum = 3 + stream := NewStream("live/policy-multiple-gops", cfg, config.LimitsConfig{}, NewEventBus()) + + var retained []*avframe.AVFrame + for gop := int64(0); gop < 3; gop++ { + base := gop * 100 + keyframe := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, base, base, []byte{byte(gop + 1)}) + audio := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, base+20, base+20, []byte{byte(gop + 11)}) + video := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, base+40, base+40, []byte{byte(gop + 21)}) + retained = append(retained, keyframe, audio) + stream.WriteFrame(keyframe) + stream.WriteFrame(audio) + stream.WriteFrame(video) + } + + tightened := cfg + tightened.GOPCacheMaxFrames = 2 + stream.UpdatePolicy(tightened, config.LimitsConfig{}) + assertCachedFrames(t, stream, "all retained GOPs trimmed", retained...) + + relaxed := cfg + relaxed.GOPCacheMaxFrames = 3 + stream.UpdatePolicy(relaxed, config.LimitsConfig{}) + futureActiveAudio := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 260, 260, []byte{31}) + stream.WriteFrame(futureActiveAudio) + retained = append(retained, futureActiveAudio) + assertCachedFrames(t, stream, "only active GOP accepts future frame", retained...) +} + +func TestStreamUpdatePolicyZeroDisablesOnlySelectedGOPBound(t *testing.T) { + tests := []struct { + name string + configure func(*config.StreamConfig) + zero func(*config.StreamConfig) + }{ + { + name: "frames", + configure: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxFrames = 2 + cfg.GOPCacheMaxBytes = 7 + }, + zero: func(cfg *config.StreamConfig) { cfg.GOPCacheMaxFrames = 0 }, + }, + { + name: "bytes", + configure: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxBytes = 4 + cfg.GOPCacheMaxFrames = 4 + }, + zero: func(cfg *config.StreamConfig) { cfg.GOPCacheMaxBytes = 0 }, + }, + { + name: "duration", + configure: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxDuration = 20 * time.Millisecond + cfg.GOPCacheMaxFrames = 4 + }, + zero: func(cfg *config.StreamConfig) { cfg.GOPCacheMaxDuration = 0 }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := newTestStreamConfig() + test.configure(&cfg) + stream := NewStream("live/policy-zero-"+test.name, cfg, config.LimitsConfig{}, NewEventBus()) + keyframe := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 0, 0, []byte{1, 2}) + initialAudio := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 20, 20, []byte{3, 4}) + stream.WriteFrame(keyframe) + stream.WriteFrame(initialAudio) + assertCachedFrames(t, stream, "selected nonzero bound seals active GOP", keyframe, initialAudio) + + reloaded := cfg + test.zero(&reloaded) + stream.UpdatePolicy(reloaded, config.LimitsConfig{}) + futureVideo := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 40, 40, []byte{5, 6}) + futureAudio := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 60, 60, []byte{7}) + afterRemainingBound := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 80, 80, []byte{8}) + stream.WriteFrame(futureVideo) + stream.WriteFrame(futureAudio) + stream.WriteFrame(afterRemainingBound) + assertCachedFrames(t, stream, "zero disables selected bound only", keyframe, initialAudio, futureVideo, futureAudio) + }) + } +} + +func assertCachedFrames(t *testing.T, stream *Stream, stage string, want ...*avframe.AVFrame) { + t.Helper() + got := stream.GOPCache() + if len(got) != len(want) { + t.Fatalf("%s: cached frames = %d, want %d", stage, len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("%s: cached frame[%d] = %p, want %p", stage, i, got[i], want[i]) + } + } +} diff --git a/core/server.go b/core/server.go index 955470d0..34519e8a 100644 --- a/core/server.go +++ b/core/server.go @@ -1,6 +1,7 @@ package core import ( + "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -38,6 +39,8 @@ type Server struct { startTime time.Time connCount atomic.Int64 done chan struct{} + shutdownOnce sync.Once + aliveWG sync.WaitGroup apiMu sync.RWMutex apiHandlers map[string]http.Handler @@ -216,18 +219,35 @@ func (s *Server) Init() error { } } - go s.aliveLoop() + s.aliveWG.Add(1) + go func() { + defer s.aliveWG.Done() + s.aliveLoop() + }() return nil } // Shutdown stops the alive loop and closes all modules in reverse registration order. func (s *Server) Shutdown() { - close(s.done) - modules := s.attemptedModuleSnapshot() - for i := len(modules) - 1; i >= 0; i-- { - modules[i].Close() //nolint:errcheck - } + s.shutdownOnce.Do(func() { + close(s.done) + s.aliveWG.Wait() + modules := s.attemptedModuleSnapshot() + for i := len(modules) - 1; i >= 0; i-- { + modules[i].Close() //nolint:errcheck + } + + timeout := s.Config().Server.DrainTimeout + if timeout <= 0 { + timeout = 30 * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + if err := s.eventBus.Drain(ctx); err != nil { + slog.Error("event bus drain timed out during shutdown", "timeout", timeout, "error", err) + } + }) } // StartTime returns when the server was created. @@ -282,18 +302,34 @@ func (s *Server) APIHandlers() map[string]http.Handler { // AcquireConn increments the connection counter. Returns false if max_connections is exceeded. func (s *Server) AcquireConn() bool { max := s.Config().Limits.MaxConnections - if max > 0 { - if s.connCount.Load() >= int64(max) { + if max <= 0 { + s.connCount.Add(1) + return true + } + + limit := int64(max) + for { + current := s.connCount.Load() + if current >= limit { return false } + if s.connCount.CompareAndSwap(current, current+1) { + return true + } } - s.connCount.Add(1) - return true } // ReleaseConn decrements the connection counter. func (s *Server) ReleaseConn() { - s.connCount.Add(-1) + for { + current := s.connCount.Load() + if current <= 0 { + return + } + if s.connCount.CompareAndSwap(current, current-1) { + return + } + } } // ConnectionCount returns the current number of active connections. diff --git a/core/server_test.go b/core/server_test.go index 95ca56b9..c3102f70 100644 --- a/core/server_test.go +++ b/core/server_test.go @@ -15,6 +15,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -388,6 +389,53 @@ func TestServerConnectionTrackingUnlimited(t *testing.T) { } } +func TestServerConnectionLimitIsAtomicUnderConcurrency(t *testing.T) { + cfg := &config.Config{} + cfg.Limits.MaxConnections = 4 + s := NewServer(cfg) + + const callers = 128 + start := make(chan struct{}) + results := make(chan bool, callers) + var wg sync.WaitGroup + for range callers { + wg.Add(1) + go func() { + defer wg.Done() + <-start + results <- s.AcquireConn() + }() + } + close(start) + wg.Wait() + close(results) + + acquired := 0 + for ok := range results { + if ok { + acquired++ + } + } + if acquired != cfg.Limits.MaxConnections { + t.Fatalf("acquired %d connections, want exactly %d", acquired, cfg.Limits.MaxConnections) + } + if got := s.ConnectionCount(); got != int64(acquired) { + t.Fatalf("connection count = %d, want %d", got, acquired) + } + for range acquired { + s.ReleaseConn() + } +} + +func TestServerReleaseConnDoesNotUnderflow(t *testing.T) { + s := NewServer(&config.Config{}) + s.ReleaseConn() + s.ReleaseConn() + if got := s.ConnectionCount(); got != 0 { + t.Fatalf("connection count = %d after releasing without acquire, want 0", got) + } +} + func TestTLSConfigConfigured(t *testing.T) { tests := []struct { name string @@ -551,3 +599,87 @@ func TestServerUpdateConfigSnapshotReturnsReloadFailure(t *testing.T) { t.Fatalf("rejected pending restart paths were published: %v", pending) } } + +type shutdownDispatchModule struct { + bus *EventBus + entered chan struct{} + release chan struct{} +} + +func (m *shutdownDispatchModule) Name() string { return "shutdown-dispatch" } +func (m *shutdownDispatchModule) Init(server *Server) error { + m.bus = server.GetEventBus() + return nil +} +func (m *shutdownDispatchModule) Hooks() []HookRegistration { + return []HookRegistration{{ + Event: EventStreamAlive, + Mode: HookAsync, + Handler: func(*EventContext) error { + close(m.entered) + <-m.release + return nil + }, + }} +} +func (m *shutdownDispatchModule) Close() error { + return m.bus.EmitAsync(EventStreamAlive, &EventContext{StreamKey: "live/shutdown"}) +} + +func TestServerShutdownDrainsAcceptedAsyncHooks(t *testing.T) { + cfg := config.Defaults() + cfg.Server.DrainTimeout = time.Second + server := NewServer(cfg) + module := &shutdownDispatchModule{entered: make(chan struct{}), release: make(chan struct{})} + server.RegisterModule(module) + if err := server.Init(); err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + go func() { + server.Shutdown() + close(done) + }() + select { + case <-module.entered: + case <-time.After(time.Second): + t.Fatal("shutdown hook did not start") + } + select { + case <-done: + t.Fatal("Shutdown returned before accepted async hook completed") + case <-time.After(20 * time.Millisecond): + } + close(module.release) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Shutdown did not return after async hook completed") + } +} + +func TestServerShutdownDrainTimeoutIsBounded(t *testing.T) { + cfg := config.Defaults() + cfg.Server.DrainTimeout = 25 * time.Millisecond + server := NewServer(cfg) + module := &shutdownDispatchModule{entered: make(chan struct{}), release: make(chan struct{})} + server.RegisterModule(module) + if err := server.Init(); err != nil { + t.Fatal(err) + } + + started := time.Now() + server.Shutdown() + elapsed := time.Since(started) + close(module.release) + if elapsed < 20*time.Millisecond || elapsed > 500*time.Millisecond { + t.Fatalf("Shutdown elapsed = %s, want bounded drain timeout", elapsed) + } +} + +func TestServerShutdownIsIdempotent(t *testing.T) { + server := NewServer(config.Defaults()) + server.Shutdown() + server.Shutdown() +} diff --git a/core/shared_buffer.go b/core/shared_buffer.go index 269f75d9..fc70d578 100644 --- a/core/shared_buffer.go +++ b/core/shared_buffer.go @@ -39,9 +39,29 @@ type SharedBufferReader struct { reader *util.RingReader[[]byte] } +// SharedBufferReadResult binds overwrite metadata to the packet returned by +// one SharedBuffer read. +type SharedBufferReadResult struct { + Data []byte + OK bool + Overwritten int64 +} + // Read returns the next packet, blocking until data is available. func (r *SharedBufferReader) Read() ([]byte, bool) { - return r.reader.Read() + result := r.ReadResult() + return result.Data, result.OK +} + +// ReadResult returns the next packet and its overwrite metadata, blocking +// until data is available or the buffer or reader is closed. +func (r *SharedBufferReader) ReadResult() SharedBufferReadResult { + result := r.reader.ReadResult() + return SharedBufferReadResult{ + Data: result.Value, + OK: result.OK, + Overwritten: result.Overwritten, + } } // Close marks this reader as closed, unblocking any in-progress Read(). @@ -51,5 +71,17 @@ func (r *SharedBufferReader) Close() { // TryRead attempts a non-blocking read. func (r *SharedBufferReader) TryRead() ([]byte, bool) { - return r.reader.TryRead() + result := r.TryReadResult() + return result.Data, result.OK +} + +// TryReadResult attempts a non-blocking read and returns overwrite metadata +// from the same operation as the packet. +func (r *SharedBufferReader) TryReadResult() SharedBufferReadResult { + result := r.reader.TryReadResult() + return SharedBufferReadResult{ + Data: result.Value, + OK: result.OK, + Overwritten: result.Overwritten, + } } diff --git a/core/shared_buffer_test.go b/core/shared_buffer_test.go index 2d36e84b..b12ac537 100644 --- a/core/shared_buffer_test.go +++ b/core/shared_buffer_test.go @@ -55,6 +55,60 @@ func TestSharedBufferOverflow(t *testing.T) { } } +func TestSharedBufferReadResultReportsRetainedPacketOverwrite(t *testing.T) { + sb := NewSharedBuffer(2) + reader := sb.NewReader() + + for _, packet := range [][]byte{{0}, {1}, {2}, {3}} { + sb.Write(packet) + } + + result := reader.ReadResult() + if !result.OK || !bytes.Equal(result.Data, []byte{2}) || result.Overwritten != 2 { + t.Fatalf("ReadResult = %+v, want data [2], OK, and 2 overwritten packets", result) + } + result = reader.ReadResult() + if !result.OK || !bytes.Equal(result.Data, []byte{3}) || result.Overwritten != 0 { + t.Fatalf("next ReadResult = %+v, want data [3], OK, and no overwrite", result) + } +} + +func TestSharedBufferTryReadResultReportsRetainedPacketOverwrite(t *testing.T) { + sb := NewSharedBuffer(2) + reader := sb.NewReader() + + for _, packet := range [][]byte{{0}, {1}, {2}, {3}, {4}} { + sb.Write(packet) + } + + result := reader.TryReadResult() + if !result.OK || !bytes.Equal(result.Data, []byte{3}) || result.Overwritten != 3 { + t.Fatalf("TryReadResult = %+v, want data [3], OK, and 3 overwritten packets", result) + } + result = reader.TryReadResult() + if !result.OK || !bytes.Equal(result.Data, []byte{4}) || result.Overwritten != 0 { + t.Fatalf("next TryReadResult = %+v, want data [4], OK, and no overwrite", result) + } +} + +func TestSharedBufferLegacyReadWrappersRetainBehavior(t *testing.T) { + sb := NewSharedBuffer(2) + reader := sb.NewReader() + + for _, packet := range [][]byte{{0}, {1}, {2}, {3}} { + sb.Write(packet) + } + + data, ok := reader.Read() + if !ok || !bytes.Equal(data, []byte{2}) { + t.Fatalf("Read = (%v, %v), want ([2], true)", data, ok) + } + data, ok = reader.TryRead() + if !ok || !bytes.Equal(data, []byte{3}) { + t.Fatalf("TryRead = (%v, %v), want ([3], true)", data, ok) + } +} + func TestSharedBufferTryRead(t *testing.T) { sb := NewSharedBuffer(64) r := sb.NewReader() diff --git a/core/slow_consumer.go b/core/slow_consumer.go index 9959ff7d..405cc23c 100644 --- a/core/slow_consumer.go +++ b/core/slow_consumer.go @@ -13,7 +13,7 @@ import ( type ConsumerState uint8 const ( - ConsumerStateNormal ConsumerState = iota + ConsumerStateNormal ConsumerState = iota ConsumerStateDropNonKey ConsumerStateSkipToKey ) @@ -43,6 +43,14 @@ type SlowConsumerFilter struct { dropped int64 // total dropped frame count } +// SlowConsumerFrameResult binds overwrite metadata to one frame read. Frame is +// always nil when OK is false; Overwritten remains set on threshold disconnect. +type SlowConsumerFrameResult struct { + Frame *avframe.AVFrame + OK bool + Overwritten int64 +} + // NewSlowConsumerFilter creates a new filter. If cfg.Enabled is false, // the filter acts as a passthrough (no dropping). // skipCfg controls the ring buffer skip tracker; nil disables skip tracking. @@ -68,59 +76,98 @@ func NewSlowConsumerFilter( // or the subscriber exceeded the skip threshold. func (f *SlowConsumerFilter) NextFrame() (*avframe.AVFrame, bool) { for { - frame, ok := f.reader.TryRead() - if !ok { - frame, ok = f.reader.Read() - if !ok { - return nil, false - } + result := f.NextFrameResult() + if !result.OK { + return nil, false } - if frame == nil { + if result.Frame == nil { continue } + if result.Overwritten == 0 || f.applyPolicy(result.Frame) { + return result.Frame, true + } + } +} - // Check if ring buffer frames were skipped (overwritten) - if f.skipTracker != nil && f.reader.Skipped() > 0 { - if f.skipTracker.RecordSkip() { - slog.Warn("subscriber exceeded skip threshold, disconnecting", - "skipped", f.reader.Skipped()) - return nil, false +// NextFrameResult reads the next frame and returns overwrite metadata from the +// same Ring read. An overwrite is returned before frame-drop policy is applied +// so callers can discard the retained frame and advance to live safely. +func (f *SlowConsumerFilter) NextFrameResult() SlowConsumerFrameResult { + for { + read := f.reader.TryReadResult() + if !read.OK { + read = f.reader.ReadResult() + if !read.OK { + return SlowConsumerFrameResult{} } } - // If filter is disabled, pass through all frames - if !f.config.Enabled { - return frame, true + result := SlowConsumerFrameResult{ + Frame: read.Value, + OK: true, + Overwritten: read.Overwritten, + } + if result.Overwritten > 0 { + if f.recordOverwrite(result.Overwritten) { + result.Frame = nil + result.OK = false + } + return result } + if result.Frame == nil { + continue + } + if f.applyPolicy(result.Frame) { + return result + } + } +} + +func (f *SlowConsumerFilter) recordOverwrite(overwritten int64) bool { + if f.skipTracker == nil { + return false + } + if !f.skipTracker.RecordSkip() { + return false + } + slog.Warn("subscriber exceeded skip threshold, disconnecting", "skipped", overwritten) + return true +} - // Update state based on current lag - f.updateState() +func (f *SlowConsumerFilter) applyPolicy(frame *avframe.AVFrame) bool { + // If filter is disabled, pass through all frames + if !f.config.Enabled { + return true + } - // Apply drop policy based on current state - switch f.state { - case ConsumerStateNormal: - return frame, true + // Update state based on current lag + f.updateState() - case ConsumerStateDropNonKey: - if f.shouldDeliver(frame) { - return frame, true - } - f.dropped++ - continue // skip this frame, read next + // Apply drop policy based on current state + switch f.state { + case ConsumerStateNormal: + return true - case ConsumerStateSkipToKey: - if frame.MediaType.IsVideo() && frame.FrameType.IsKeyframe() { - f.state = ConsumerStateDropNonKey - return frame, true - } - // Also deliver audio and sequence headers even in skip-to-key mode - if frame.MediaType.IsAudio() || frame.FrameType == avframe.FrameTypeSequenceHeader { - return frame, true - } - f.dropped++ - continue + case ConsumerStateDropNonKey: + if f.shouldDeliver(frame) { + return true + } + f.dropped++ + return false + + case ConsumerStateSkipToKey: + if frame.MediaType.IsVideo() && frame.FrameType.IsKeyframe() { + f.state = ConsumerStateDropNonKey + return true } + // Also deliver audio and sequence headers even in skip-to-key mode + if frame.MediaType.IsAudio() || frame.FrameType == avframe.FrameTypeSequenceHeader { + return true + } + f.dropped++ + return false } + return false } // ReportSendTime updates the EWMA with the duration of the last frame send. diff --git a/core/slow_consumer_test.go b/core/slow_consumer_test.go index 6d8dea90..2403fbaf 100644 --- a/core/slow_consumer_test.go +++ b/core/slow_consumer_test.go @@ -40,6 +40,256 @@ func buildSlow(f *SlowConsumerFilter) { } } +func TestSlowConsumerFilterReportsFirstOverwriteBeforeDelivery(t *testing.T) { + // Mutation caught: dropping RingReadResult.Overwritten or deriving it from a + // later Skipped call hides the first integrity-loss event. + cfg := testSlowConsumerConfig() + cfg.Enabled = false + rb := util.NewRingBuffer[*avframe.AVFrame](2) + reader := rb.NewReaderAt(0) + overwrittenFirst := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeKeyframe) + overwrittenSecond := makeFrame(avframe.MediaTypeAudio, avframe.FrameTypeInterframe) + retained := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeInterframe) + next := makeFrame(avframe.MediaTypeAudio, avframe.FrameTypeInterframe) + rb.Write(overwrittenFirst) + rb.Write(overwrittenSecond) + rb.Write(retained) + rb.Write(next) + + filter := NewSlowConsumerFilter(reader, cfg, nil) + result := filter.NextFrameResult() + if !result.OK || result.Frame != retained || result.Overwritten != 2 { + t.Fatalf("first result = %+v, want retained frame %p, OK, overwritten=2", result, retained) + } + + result = filter.NextFrameResult() + if !result.OK || result.Frame != next || result.Overwritten != 0 { + t.Fatalf("continuous result = %+v, want next frame %p, OK, overwritten=0", result, next) + } +} + +func TestSlowConsumerFilterReportsOverwriteBeforePolicy(t *testing.T) { + // Mutation caught: applying DropNonKey/SkipToKey before returning overwrite + // metadata can hide the retained interframe from recovery callers. + tests := []struct { + name string + enabled bool + skipCfg *config.SkipTrackerConfig + state ConsumerState + wantEvents int + }{ + {name: "nil_tracker", enabled: false, state: ConsumerStateNormal}, + { + name: "disabled_tracker", + enabled: false, + skipCfg: &config.SkipTrackerConfig{MaxCount: 0, Window: 10 * time.Second}, + state: ConsumerStateNormal, + }, + { + name: "below_threshold_tracker", + enabled: false, + skipCfg: &config.SkipTrackerConfig{MaxCount: 2, Window: 10 * time.Second}, + state: ConsumerStateNormal, + wantEvents: 1, + }, + { + name: "drop_non_key_would_hide_retained_interframe", + enabled: true, + state: ConsumerStateDropNonKey, + }, + { + name: "skip_to_key_would_hide_retained_interframe", + enabled: true, + state: ConsumerStateSkipToKey, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := testSlowConsumerConfig() + cfg.Enabled = tt.enabled + rb := util.NewRingBuffer[*avframe.AVFrame](2) + reader := rb.NewReaderAt(0) + overwritten := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeKeyframe) + retained := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeInterframe) + nextKeyframe := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeKeyframe) + rb.Write(overwritten) + rb.Write(retained) + rb.Write(nextKeyframe) + + filter := NewSlowConsumerFilter(reader, cfg, tt.skipCfg) + filter.state = tt.state + result := filter.NextFrameResult() + if !result.OK || result.Frame != retained || result.Overwritten != 1 { + t.Fatalf("result = %+v, want retained frame %p, OK, overwritten=1", result, retained) + } + if filter.Dropped() != 0 { + t.Fatalf("overwrite result was filtered before delivery: dropped=%d", filter.Dropped()) + } + if filter.skipTracker != nil && len(filter.skipTracker.events) != tt.wantEvents { + t.Fatalf("recorded events = %d, want %d", len(filter.skipTracker.events), tt.wantEvents) + } + }) + } +} + +func TestSlowConsumerFilterRecordsOverwriteOnceAndPreservesThreshold(t *testing.T) { + // Mutation caught: recording once in the result helper and again in the + // compatibility wrapper disconnects one overwrite too early. + cfg := testSlowConsumerConfig() + cfg.Enabled = false + rb := util.NewRingBuffer[*avframe.AVFrame](2) + reader := rb.NewReaderAt(0) + filter := NewSlowConsumerFilter(reader, cfg, &config.SkipTrackerConfig{ + MaxCount: 1, + Window: 10 * time.Second, + }) + + firstRetained := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeInterframe) + continuous := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeKeyframe) + rb.Write(makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeInterframe)) + rb.Write(firstRetained) + rb.Write(continuous) + + frame, ok := filter.NextFrame() + if !ok || frame != firstRetained { + t.Fatalf("first compatibility read = (%p, %v), want retained frame %p", frame, ok, firstRetained) + } + if got := len(filter.skipTracker.events); got != 1 { + t.Fatalf("events after one compatibility overwrite read = %d, want 1", got) + } + continuousResult := filter.NextFrameResult() + if !continuousResult.OK || continuousResult.Frame != continuous || continuousResult.Overwritten != 0 { + t.Fatalf("continuous result = %+v, want frame %p, OK, overwritten=0", continuousResult, continuous) + } + if got := len(filter.skipTracker.events); got != 1 { + t.Fatalf("events after one overwrite and one continuous result = %d, want 1", got) + } + + secondRetained := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeInterframe) + rb.Write(makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeInterframe)) + rb.Write(secondRetained) + rb.Write(makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeKeyframe)) + second := filter.NextFrameResult() + // Mutation caught: retaining the Ring value after threshold breach lets a + // frame-first protocol branch send terminal media before checking status. + wouldSend := second.Frame != nil + if wouldSend { + t.Fatalf("threshold result is structurally sendable before status check: %+v", second) + } + if second.OK || second.Overwritten != 1 { + t.Fatalf("threshold result = %+v, want nil frame, not OK, overwritten=1", second) + } + if got := len(filter.skipTracker.events); got != 2 { + t.Fatalf("events at disconnect boundary = %d, want 2", got) + } +} + +func TestSlowConsumerFilterNextFrameCompatibilityAfterResultAPI(t *testing.T) { + // Mutation caught: returning an overwrite result directly from NextFrame + // bypasses the established DropNonKey policy. + cfg := testSlowConsumerConfig() + rb := util.NewRingBuffer[*avframe.AVFrame](2) + reader := rb.NewReaderAt(0) + retainedInterframe := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeInterframe) + keyframe := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeKeyframe) + rb.Write(makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeInterframe)) + rb.Write(retainedInterframe) + rb.Write(keyframe) + rb.Close() + + filter := NewSlowConsumerFilter(reader, cfg, nil) + filter.state = ConsumerStateDropNonKey + frame, ok := filter.NextFrame() + if !ok || frame != keyframe { + t.Fatalf("NextFrame = (%p, %v), want keyframe %p", frame, ok, keyframe) + } + if got := filter.Dropped(); got != 1 { + t.Fatalf("dropped = %d, want retained interframe counted once", got) + } + if frame, ok = filter.NextFrame(); ok || frame != nil { + t.Fatalf("NextFrame after close = (%p, %v), want (nil, false)", frame, ok) + } +} + +func TestSlowConsumerFilterNextFrameCompatibilitySkipsNil(t *testing.T) { + // Mutation caught: making NextFrame a direct result-field adapter exposes + // nil ring values that compatibility callers have always skipped. + cfg := testSlowConsumerConfig() + cfg.Enabled = false + rb := util.NewRingBuffer[*avframe.AVFrame](2) + reader := rb.NewReaderAt(0) + want := makeFrame(avframe.MediaTypeAudio, avframe.FrameTypeInterframe) + rb.Write(nil) + rb.Write(want) + rb.Close() + + filter := NewSlowConsumerFilter(reader, cfg, nil) + frame, ok := filter.NextFrame() + if !ok || frame != want { + t.Fatalf("NextFrame = (%p, %v), want non-nil frame %p", frame, ok, want) + } +} + +func TestSlowConsumerFilterNextFrameCompatibilityCloseUnblocks(t *testing.T) { + // Mutation caught: replacing the blocking Ring result read with polling can + // lose the reader-close wakeup used by existing compatibility callers. + cfg := testSlowConsumerConfig() + rb := util.NewRingBuffer[*avframe.AVFrame](2) + filter := NewSlowConsumerFilter(rb.NewReaderAt(0), cfg, nil) + result := make(chan bool, 1) + go func() { + _, ok := filter.NextFrame() + result <- ok + }() + + filter.Close() + select { + case ok := <-result: + if ok { + t.Fatal("NextFrame returned OK after filter close") + } + case <-time.After(time.Second): + t.Fatal("NextFrame did not unblock after filter close") + } +} + +func TestSlowConsumerFilterNextFrameResultClosedHasNoEvent(t *testing.T) { + // Mutation caught: carrying prior overwrite state into an empty/closed read + // fabricates an integrity event with no available Ring result. + cfg := testSlowConsumerConfig() + rb := util.NewRingBuffer[*avframe.AVFrame](2) + rb.Close() + filter := NewSlowConsumerFilter(rb.NewReaderAt(0), cfg, nil) + + result := filter.NextFrameResult() + if result.OK || result.Frame != nil || result.Overwritten != 0 { + t.Fatalf("closed result = %+v, want zero result", result) + } +} + +func TestSlowConsumerFilterNextFrameResultAllocations(t *testing.T) { + // Mutation caught: boxing the result, copying the payload, or allocating + // per-read metadata regresses the immediate normal delivery hot path. + cfg := testSlowConsumerConfig() + rb := util.NewRingBuffer[*avframe.AVFrame](2) + filter := NewSlowConsumerFilter(rb.NewReaderAt(0), cfg, nil) + frame := makeFrame(avframe.MediaTypeVideo, avframe.FrameTypeKeyframe) + var result SlowConsumerFrameResult + + allocs := testing.AllocsPerRun(1000, func() { + rb.Write(frame) + result = filter.NextFrameResult() + }) + if allocs != 0 { + t.Fatalf("NextFrameResult immediate delivery allocations = %f, want 0", allocs) + } + t.Logf("NextFrameResult immediate delivery allocations = %f", allocs) + if !result.OK || result.Frame != frame || result.Overwritten != 0 { + t.Fatalf("last allocation-run result = %+v, want frame %p, OK, overwritten=0", result, frame) + } +} + func TestSlowConsumerFilterDisabled(t *testing.T) { cfg := testSlowConsumerConfig() cfg.Enabled = false @@ -257,7 +507,7 @@ func TestSlowConsumerFilterHysteresis(t *testing.T) { // Test hysteresis: in DropNonKey, lag 0.6 should NOT recover (0.6 > 0.5 recover threshold) t.Run("drop_stays_in_dead_zone", func(t *testing.T) { rb := util.NewRingBuffer[*avframe.AVFrame](100) - writeFrames(rb, 60) // writer at 60 + writeFrames(rb, 60) // writer at 60 reader := rb.NewReaderAt(0) // reader at 0, lag = 0.6 filter := NewSlowConsumerFilter(reader, cfg, nil) filter.state = ConsumerStateDropNonKey // force into DropNonKey @@ -273,7 +523,7 @@ func TestSlowConsumerFilterHysteresis(t *testing.T) { // Test hysteresis: in DropNonKey, lag 0.4 should recover (0.4 < 0.5 recover threshold) t.Run("drop_recovers_below_threshold", func(t *testing.T) { rb := util.NewRingBuffer[*avframe.AVFrame](100) - writeFrames(rb, 40) // writer at 40 + writeFrames(rb, 40) // writer at 40 reader := rb.NewReaderAt(0) // reader at 0, lag = 0.4 filter := NewSlowConsumerFilter(reader, cfg, nil) filter.state = ConsumerStateDropNonKey @@ -312,8 +562,10 @@ func TestSlowConsumerFilterSkipTrackerDisconnect(t *testing.T) { } } else { // 3rd skip: should disconnect - if ok { - t.Fatalf("skip %d: expected disconnect (ok=false), got frame", i+1) + // Mutation caught: adapting NextFrame to return a terminal result's + // retained frame would violate the legacy (nil, false) contract. + if ok || frame != nil { + t.Fatalf("skip %d: expected disconnect (nil, false), got (%p, %v)", i+1, frame, ok) } return } diff --git a/core/stream.go b/core/stream.go index d43db3c0..833c404d 100644 --- a/core/stream.go +++ b/core/stream.go @@ -6,6 +6,7 @@ import ( "fmt" "reflect" "sync" + "sync/atomic" "time" "github.com/im-pingo/liveforge/config" @@ -82,19 +83,27 @@ func (st *SkipTracker) RecordSkip() bool { // Stream manages the lifecycle, publisher, subscribers, and frame distribution for a stream key. type Stream struct { - key string - config config.StreamConfig - limits config.LimitsConfig - - mu sync.RWMutex - state StreamState - publisher Publisher + key string + instanceID uint64 + config config.StreamConfig + limits config.LimitsConfig + + mu sync.RWMutex + state StreamState + publisher Publisher + usedPublisherIDs map[string]struct{} + lastPublisherID string + lastPublisherGeneration uint64 ringBuffer *util.RingBuffer[*avframe.AVFrame] muxerManager *MuxerManager gopCache [][]*avframe.AVFrame gopStarts []int64 + gopBytes []int64 + gopMinDTS []int64 + gopMaxDTS []int64 gopGeneration uint64 + gopCacheSealed bool subscribers map[string]int // protocol -> count (e.g. "rtmp" -> 2) generationSubscribers map[uint64]map[string]int @@ -107,6 +116,7 @@ type Stream struct { mediaInfo avframe.MediaInfo audioCodecEpoch uint64 generationDone chan struct{} + generationBoundary *streamGenerationBoundary startupStateChanged chan struct{} startupReady bool @@ -116,18 +126,47 @@ type Stream struct { idleTimer *time.Timer feedbackRouter *FeedbackRouter transcodeManager *TranscodeManager + destroyCallback func() + destroyOnce sync.Once +} + +var streamInstanceSequence atomic.Uint64 + +// streamGenerationBoundary retains the immutable end position of one +// publisher generation after that publisher detaches. Snapshots keep this +// object alive without retaining the Stream or its publisher. +type streamGenerationBoundary struct { + endCursor atomic.Int64 + ended atomic.Bool +} + +func (b *streamGenerationBoundary) end() (int64, bool) { + if b == nil || !b.ended.Load() { + return 0, false + } + return b.endCursor.Load(), true +} + +func normalizeGOPConfig(cfg config.StreamConfig) config.StreamConfig { + if cfg.GOPCache && cfg.GOPCacheNum > 0 && cfg.GOPCacheMaxFrames <= 0 && cfg.GOPCacheMaxBytes <= 0 { + cfg.GOPCacheMaxFrames = config.DefaultGOPCacheMaxFrames + } + return cfg } // NewStream creates a new Stream in idle state. func NewStream(key string, cfg config.StreamConfig, limits config.LimitsConfig, bus *EventBus) *Stream { + cfg = normalizeGOPConfig(cfg) s := &Stream{ key: key, + instanceID: streamInstanceSequence.Add(1), config: cfg, limits: limits, state: StreamStateIdle, ringBuffer: util.NewRingBuffer[*avframe.AVFrame](cfg.RingBufferSize), eventBus: bus, subscribers: make(map[string]int), + usedPublisherIDs: make(map[string]struct{}), generationSubscribers: make(map[uint64]map[string]int), seqHeaderReady: make(chan struct{}), startupStateChanged: make(chan struct{}), @@ -137,6 +176,27 @@ func NewStream(key string, cfg config.StreamConfig, limits config.LimitsConfig, return s } +// InstanceID identifies this concrete stream object without retaining the +// object itself in registries or lifecycle events. +func (s *Stream) InstanceID() uint64 { return s.instanceID } + +func (s *Stream) setDestroyCallback(callback func()) { + s.mu.Lock() + s.destroyCallback = callback + s.mu.Unlock() +} + +func (s *Stream) notifyDestroy() { + s.destroyOnce.Do(func() { + s.mu.RLock() + callback := s.destroyCallback + s.mu.RUnlock() + if callback != nil { + callback() + } + }) +} + // Key returns the stream key. func (s *Stream) Key() string { return s.key @@ -153,16 +213,25 @@ func (s *Stream) Config() config.StreamConfig { // muxer capacity are intentionally not resized in place; new streams receive // those structural values from StreamHub. func (s *Stream) UpdatePolicy(cfg config.StreamConfig, limits config.LimitsConfig) { + cfg = normalizeGOPConfig(cfg) s.mu.Lock() s.config = cfg s.limits = limits if !cfg.GOPCache || cfg.GOPCacheNum <= 0 { s.gopCache = nil s.gopStarts = nil + s.gopBytes = nil + s.gopMinDTS = nil + s.gopMaxDTS = nil + s.gopCacheSealed = false } else if len(s.gopCache) > cfg.GOPCacheNum { s.gopCache = append([][]*avframe.AVFrame(nil), s.gopCache[len(s.gopCache)-cfg.GOPCacheNum:]...) s.gopStarts = append([]int64(nil), s.gopStarts[len(s.gopStarts)-cfg.GOPCacheNum:]...) + s.gopBytes = append([]int64(nil), s.gopBytes[len(s.gopBytes)-cfg.GOPCacheNum:]...) + s.gopMinDTS = append([]int64(nil), s.gopMinDTS[len(s.gopMinDTS)-cfg.GOPCacheNum:]...) + s.gopMaxDTS = append([]int64(nil), s.gopMaxDTS[len(s.gopMaxDTS)-cfg.GOPCacheNum:]...) } + s.trimGOPCacheLocked() if s.noPublisherTimer != nil { s.noPublisherTimer.Stop() s.noPublisherTimer = nil @@ -170,10 +239,16 @@ func (s *Stream) UpdatePolicy(cfg config.StreamConfig, limits config.LimitsConfi if s.state == StreamStateNoPublisher && cfg.NoPublisherTimeout > 0 { s.noPublisherTimer = time.AfterFunc(cfg.NoPublisherTimeout, func() { s.mu.Lock() - defer s.mu.Unlock() + notify := false if s.state == StreamStateNoPublisher { + s.usedPublisherIDs = nil s.state = StreamStateDestroying s.signalStartupStateChangedLocked() + notify = true + } + s.mu.Unlock() + if notify { + s.notifyDestroy() } }) } @@ -208,6 +283,13 @@ func (s *Stream) SetPublisher(pub Publisher) error { if s.state == StreamStatePublishing { return errors.New("stream already has a publisher") } + if s.state == StreamStateDestroying { + return errors.New("stream is destroying") + } + publisherID := pub.ID() + if _, used := s.usedPublisherIDs[publisherID]; publisherID != "" && used { + return fmt.Errorf("publisher ID %q was already used by an earlier generation", publisherID) + } // Cancel no-publisher timer if republishing if s.noPublisherTimer != nil { @@ -230,7 +312,11 @@ func (s *Stream) SetPublisher(pub Publisher) error { s.generationStartCursor = s.ringBuffer.WriteCursor() s.gopCache = nil s.gopStarts = nil + s.gopBytes = nil + s.gopMinDTS = nil + s.gopMaxDTS = nil s.gopGeneration = 0 + s.gopCacheSealed = false s.videoSeqHeader = nil s.audioSeqHeader = nil s.seqHeaderReady = make(chan struct{}) @@ -238,6 +324,7 @@ func (s *Stream) SetPublisher(pub Publisher) error { s.audioCodecEpoch = 0 s.startupReady = false s.generationDone = make(chan struct{}) + s.generationBoundary = &streamGenerationBoundary{} s.mergePublisherMediaInfoLocked(pub.MediaInfo()) if s.mediaInfo.AudioCodec != 0 { s.audioCodecEpoch = 1 @@ -247,10 +334,15 @@ func (s *Stream) SetPublisher(pub Publisher) error { } } s.publisher = pub + s.lastPublisherID = publisherID + s.lastPublisherGeneration = s.publisherGeneration s.state = StreamStatePublishing s.startupReady = s.startupReadyLocked() s.stats.initStats() s.signalStartupStateChangedLocked() + if publisherID != "" { + s.usedPublisherIDs[publisherID] = struct{}{} + } return nil } @@ -267,7 +359,7 @@ func (s *Stream) RemovePublisher() { func (s *Stream) RemovePublisherIf(pub Publisher) bool { s.mu.Lock() defer s.mu.Unlock() - if !samePublisher(s.publisher, pub) { + if !s.publisherMatchesLocked(pub) { return false } s.removePublisherLocked() @@ -275,6 +367,9 @@ func (s *Stream) RemovePublisherIf(pub Publisher) bool { } func (s *Stream) removePublisherLocked() { + if s.state == StreamStateDestroying { + return + } s.closeGenerationLocked() s.publisher = nil s.state = StreamStateNoPublisher @@ -284,10 +379,16 @@ func (s *Stream) removePublisherLocked() { if s.config.NoPublisherTimeout > 0 { s.noPublisherTimer = time.AfterFunc(s.config.NoPublisherTimeout, func() { s.mu.Lock() - defer s.mu.Unlock() + notify := false if s.state == StreamStateNoPublisher { + s.usedPublisherIDs = nil s.state = StreamStateDestroying s.signalStartupStateChangedLocked() + notify = true + } + s.mu.Unlock() + if notify { + s.notifyDestroy() } }) } @@ -306,6 +407,20 @@ func samePublisher(left, right Publisher) bool { return left == right } +// publisherMatchesLocked uses the publisher contract's stable ID on the frame +// hot path. The reflective comparison remains only for legacy publishers that +// do not provide an ID, which are outside the normal protocol adapters. +func (s *Stream) publisherMatchesLocked(pub Publisher) bool { + if s.state != StreamStatePublishing || isNilPublisher(s.publisher) || isNilPublisher(pub) { + return false + } + candidateID := pub.ID() + if s.lastPublisherID != "" || candidateID != "" { + return s.lastPublisherID != "" && s.lastPublisherID == candidateID + } + return samePublisher(s.publisher, pub) +} + func isNilPublisher(pub Publisher) bool { if pub == nil { return true @@ -320,14 +435,14 @@ func isNilPublisher(pub Publisher) bool { } func (s *Stream) closeGenerationLocked() { - if s.generationDone == nil { + if s.generationDone == nil || s.generationBoundary == nil || s.generationBoundary.ended.Load() { return } - select { - case <-s.generationDone: - default: - close(s.generationDone) - } + // Frame writes and publisher removal are serialized by s.mu, so this is + // the exact exclusive upper bound for this generation in the shared ring. + s.generationBoundary.endCursor.Store(s.ringBuffer.WriteCursor()) + s.generationBoundary.ended.Store(true) + close(s.generationDone) } func (s *Stream) signalStartupStateChangedLocked() { @@ -422,9 +537,10 @@ func trackReady(codec avframe.CodecType, hasSequenceHeader bool) bool { // and transitions to destroying state. func (s *Stream) Close() { s.mu.Lock() - defer s.mu.Unlock() - if s.state == StreamStateDestroying { + s.ringBuffer.Close() + s.mu.Unlock() + s.notifyDestroy() return } @@ -438,16 +554,36 @@ func (s *Stream) Close() { s.idleTimer = nil } - if s.publisher != nil { - s.publisher.Close() //nolint:errcheck - s.publisher = nil - } + publisher := s.publisher + s.publisher = nil s.closeGenerationLocked() s.startupReady = false + s.usedPublisherIDs = nil s.state = StreamStateDestroying s.signalStartupStateChangedLocked() s.ringBuffer.Close() + s.mu.Unlock() + if publisher != nil { + publisher.Close() //nolint:errcheck + } + s.notifyDestroy() +} + +// LastPublisherID returns the most recent publisher identity. It is used only +// to scope delayed stream-destroy cleanup after the publisher has detached. +func (s *Stream) LastPublisherID() string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.lastPublisherID +} + +// LastPublisherGeneration returns the most recent publisher generation after +// the publisher has detached. +func (s *Stream) LastPublisherGeneration() uint64 { + s.mu.RLock() + defer s.mu.RUnlock() + return s.lastPublisherGeneration } // Publisher returns the current publisher, if any. @@ -460,6 +596,9 @@ func (s *Stream) Publisher() Publisher { // WriteFrame writes a media frame to the ring buffer and updates caches. // Returns false if the frame was rejected due to bitrate limit. func (s *Stream) WriteFrame(frame *avframe.AVFrame) bool { + if frame == nil { + return false + } s.mu.Lock() defer s.mu.Unlock() return s.writeFrameLocked(frame) @@ -467,9 +606,12 @@ func (s *Stream) WriteFrame(frame *avframe.AVFrame) bool { // WriteFrameForPublisher writes a frame only when pub still owns the active generation. func (s *Stream) WriteFrameForPublisher(pub Publisher, frame *avframe.AVFrame) bool { + if frame == nil { + return false + } s.mu.Lock() defer s.mu.Unlock() - if isNilPublisher(pub) || !samePublisher(s.publisher, pub) || s.state != StreamStatePublishing { + if !s.publisherMatchesLocked(pub) || s.state != StreamStatePublishing { return false } return s.writeFrameLocked(frame) @@ -481,7 +623,7 @@ func (s *Stream) WriteFrameForPublisher(pub Publisher, frame *avframe.AVFrame) b func (s *Stream) WithActivePublisher(pub Publisher, activity func()) bool { s.mu.Lock() defer s.mu.Unlock() - if isNilPublisher(pub) || !samePublisher(s.publisher, pub) || s.state != StreamStatePublishing { + if !s.publisherMatchesLocked(pub) || s.state != StreamStatePublishing { return false } activity() @@ -538,24 +680,31 @@ func (s *Stream) writeFrameLocked(frame *avframe.AVFrame) bool { s.updateStartupReadyLocked() // Update GOP cache for video frames - if s.config.GOPCache { + if s.config.GOPCache && s.config.GOPCacheNum > 0 { if frame.MediaType.IsVideo() { if frame.FrameType.IsKeyframe() { // Start new GOP s.gopGeneration++ + s.gopCacheSealed = false gopStart := s.ringBuffer.WriteCursor() s.gopCache = append(s.gopCache, []*avframe.AVFrame{frame}) s.gopStarts = append(s.gopStarts, gopStart) + s.gopBytes = append(s.gopBytes, int64(len(frame.Payload))) + s.gopMinDTS = append(s.gopMinDTS, frame.DTS) + s.gopMaxDTS = append(s.gopMaxDTS, frame.DTS) if len(s.gopCache) > s.config.GOPCacheNum { s.gopCache = s.gopCache[len(s.gopCache)-s.config.GOPCacheNum:] s.gopStarts = s.gopStarts[len(s.gopStarts)-s.config.GOPCacheNum:] + s.gopBytes = s.gopBytes[len(s.gopBytes)-s.config.GOPCacheNum:] + s.gopMinDTS = s.gopMinDTS[len(s.gopMinDTS)-s.config.GOPCacheNum:] + s.gopMaxDTS = s.gopMaxDTS[len(s.gopMaxDTS)-s.config.GOPCacheNum:] } } else if frame.FrameType != avframe.FrameTypeSequenceHeader && len(s.gopCache) > 0 { - s.gopCache[len(s.gopCache)-1] = append(s.gopCache[len(s.gopCache)-1], frame) + s.appendGOPFrameLocked(frame) } } else if frame.MediaType.IsAudio() && frame.FrameType != avframe.FrameTypeSequenceHeader && len(s.gopCache) > 0 { // Interleave audio into GOP cache for DTS ordering - s.gopCache[len(s.gopCache)-1] = append(s.gopCache[len(s.gopCache)-1], frame) + s.appendGOPFrameLocked(frame) } } @@ -564,8 +713,184 @@ func (s *Stream) writeFrameLocked(frame *avframe.AVFrame) bool { return true } +// appendGOPFrameLocked adds a frame to the current GOP when doing so stays +// within every configured per-GOP bound. The keyframe that starts a GOP is +// always retained, even when its payload alone exceeds max bytes. +func (s *Stream) appendGOPFrameLocked(frame *avframe.AVFrame) { + if len(s.gopCache) == 0 || s.gopCacheSealed { + return + } + index := len(s.gopCache) - 1 + gop := s.gopCache[index] + if s.config.GOPCacheMaxFrames > 0 && len(gop) >= s.config.GOPCacheMaxFrames { + s.gopCacheSealed = true + return + } + if s.config.GOPCacheMaxBytes > 0 && len(gop) > 0 && s.gopBytes[index]+int64(len(frame.Payload)) > s.config.GOPCacheMaxBytes { + s.gopCacheSealed = true + return + } + minDTS, maxDTS := s.gopMinDTS[index], s.gopMaxDTS[index] + if frame.DTS < minDTS { + minDTS = frame.DTS + } + if frame.DTS > maxDTS { + maxDTS = frame.DTS + } + if s.config.GOPCacheMaxDuration > 0 && len(gop) > 0 && gopDurationExceeded(minDTS, maxDTS, s.config.GOPCacheMaxDuration) { + s.gopCacheSealed = true + return + } + s.gopCache[index] = append(gop, frame) + s.gopBytes[index] += int64(len(frame.Payload)) + s.gopMinDTS[index] = minDTS + s.gopMaxDTS[index] = maxDTS + if s.currentGOPAtBoundLocked() { + s.gopCacheSealed = true + } +} + +func (s *Stream) currentGOPAtBoundLocked() bool { + if len(s.gopCache) == 0 { + return false + } + index := len(s.gopCache) - 1 + gop := s.gopCache[index] + if s.config.GOPCacheMaxFrames > 0 && len(gop) >= s.config.GOPCacheMaxFrames { + return true + } + if s.config.GOPCacheMaxBytes > 0 && s.gopBytes[index] >= s.config.GOPCacheMaxBytes { + return true + } + return s.config.GOPCacheMaxDuration > 0 && len(gop) > 0 && + gopDurationAtLeast(s.gopMinDTS[index], s.gopMaxDTS[index], s.config.GOPCacheMaxDuration) +} + +func gopDurationExceeded(minDTS, maxDTS int64, limit time.Duration) bool { + if limit <= 0 || maxDTS < minDTS { + return false + } + return dtsSpanMillis(minDTS, maxDTS) > int64(limit/time.Millisecond) +} + +func gopDurationAtLeast(minDTS, maxDTS int64, limit time.Duration) bool { + if limit <= 0 || maxDTS < minDTS { + return false + } + return dtsSpanMillis(minDTS, maxDTS) >= int64(limit/time.Millisecond) +} + +func dtsSpanMillis(minDTS, maxDTS int64) int64 { + if maxDTS < minDTS { + minDTS, maxDTS = maxDTS, minDTS + } + if minDTS < 0 && maxDTS > (1<<63-1)+minDTS { + return 1<<63 - 1 + } + return maxDTS - minDTS +} + +// trimGOPCacheLocked repairs cache entries after a policy update. It keeps a +// playable prefix beginning at each GOP keyframe and never trims a keyframe. +func (s *Stream) trimGOPCacheLocked() { + s.gopCacheSealed = false + if !s.config.GOPCache || s.config.GOPCacheNum <= 0 { + s.gopCache = nil + s.gopStarts = nil + s.gopBytes = nil + s.gopMinDTS = nil + s.gopMaxDTS = nil + return + } + if len(s.gopCache) > s.config.GOPCacheNum { + start := len(s.gopCache) - s.config.GOPCacheNum + s.gopCache = s.gopCache[start:] + s.gopStarts = s.gopStarts[start:] + s.gopBytes = s.gopBytes[start:] + s.gopMinDTS = s.gopMinDTS[start:] + s.gopMaxDTS = s.gopMaxDTS[start:] + } + for i, gop := range s.gopCache { + if len(gop) == 0 { + s.gopBytes[i] = 0 + s.gopMinDTS[i] = 0 + s.gopMaxDTS[i] = 0 + continue + } + limit := len(gop) + if s.config.GOPCacheMaxFrames > 0 && limit > s.config.GOPCacheMaxFrames { + limit = s.config.GOPCacheMaxFrames + } + if s.config.GOPCacheMaxBytes > 0 { + bytes := int64(0) + byteLimit := 0 + for n, frame := range gop[:limit] { + frameBytes := int64(len(frame.Payload)) + if n > 0 && bytes+frameBytes > s.config.GOPCacheMaxBytes { + break + } + bytes += frameBytes + byteLimit = n + 1 + if bytes >= s.config.GOPCacheMaxBytes { + break + } + } + if byteLimit < limit { + limit = byteLimit + } + } + if s.config.GOPCacheMaxDuration > 0 { + durationLimit := 1 + minDTS, maxDTS := gop[0].DTS, gop[0].DTS + for n := 1; n < limit; n++ { + candidateMin, candidateMax := minDTS, maxDTS + if gop[n].DTS < candidateMin { + candidateMin = gop[n].DTS + } + if gop[n].DTS > candidateMax { + candidateMax = gop[n].DTS + } + if gopDurationExceeded(candidateMin, candidateMax, s.config.GOPCacheMaxDuration) { + break + } + minDTS, maxDTS = candidateMin, candidateMax + durationLimit = n + 1 + } + if durationLimit < limit { + limit = durationLimit + } + } + if limit < 1 { + limit = 1 + } + if i == len(s.gopCache)-1 && limit < len(gop) { + s.gopCacheSealed = true + } + s.gopCache[i] = gop[:limit] + bytes := int64(0) + for _, frame := range s.gopCache[i] { + bytes += int64(len(frame.Payload)) + } + s.gopBytes[i] = bytes + minDTS, maxDTS := s.gopCache[i][0].DTS, s.gopCache[i][0].DTS + for _, frame := range s.gopCache[i][1:] { + if frame.DTS < minDTS { + minDTS = frame.DTS + } + if frame.DTS > maxDTS { + maxDTS = frame.DTS + } + } + s.gopMinDTS[i], s.gopMaxDTS[i] = minDTS, maxDTS + } + if s.currentGOPAtBoundLocked() { + s.gopCacheSealed = true + } +} + // StreamStartupSnapshot is an atomic view of the current publisher generation's startup state. type StreamStartupSnapshot struct { + StreamInstanceID uint64 Generation uint64 PublisherID string GenerationStartCursor int64 @@ -578,6 +903,13 @@ type StreamStartupSnapshot struct { GenerationDone <-chan struct{} Ready bool audioCodecEpoch uint64 + generationBoundary *streamGenerationBoundary +} + +// GenerationEndCursor returns the exclusive source-ring boundary captured +// when this snapshot's publisher generation ended. +func (s StreamStartupSnapshot) GenerationEndCursor() (int64, bool) { + return s.generationBoundary.end() } // StartupSnapshot captures media information, headers, replay frames, and cursors atomically. @@ -602,6 +934,7 @@ func (s *Stream) startupSnapshotLocked() StreamStartupSnapshot { publisherID = s.publisher.ID() } return StreamStartupSnapshot{ + StreamInstanceID: s.instanceID, Generation: s.publisherGeneration, PublisherID: publisherID, GenerationStartCursor: s.generationStartCursor, @@ -614,6 +947,7 @@ func (s *Stream) startupSnapshotLocked() StreamStartupSnapshot { GenerationDone: s.generationDone, Ready: s.startupReady, audioCodecEpoch: s.audioCodecEpoch, + generationBoundary: s.generationBoundary, } } @@ -712,7 +1046,7 @@ func (s *Stream) GOPCacheDetail() GOPCacheDetail { } } if dtsSet { - d.DurationMs = maxDTS - minDTS + d.DurationMs = dtsSpanMillis(minDTS, maxDTS) } return d } @@ -936,9 +1270,16 @@ func (s *Stream) checkIdleTimeout() { if s.idleTimer == nil { s.idleTimer = time.AfterFunc(s.config.IdleTimeout, func() { s.mu.Lock() - defer s.mu.Unlock() + notify := false if s.publisher == nil && s.totalSubscribers() == 0 { + s.usedPublisherIDs = nil s.state = StreamStateDestroying + s.signalStartupStateChangedLocked() + notify = true + } + s.mu.Unlock() + if notify { + s.notifyDestroy() } }) } diff --git a/core/stream_hub.go b/core/stream_hub.go index afe69ae7..0564df01 100644 --- a/core/stream_hub.go +++ b/core/stream_hub.go @@ -1,6 +1,7 @@ package core import ( + "container/list" "fmt" "sync" @@ -12,19 +13,28 @@ import ( type StreamHub struct { mu sync.RWMutex streams map[string]*Stream + streamOrder *list.List + streamOrderByKey map[string]*list.Element config config.StreamConfig limits config.LimitsConfig eventBus *EventBus audioCodecEnabled bool } +type orderedStream struct { + key string + stream *Stream +} + // NewStreamHub creates a new StreamHub. func NewStreamHub(cfg config.StreamConfig, limits config.LimitsConfig, bus *EventBus) *StreamHub { return &StreamHub{ - streams: make(map[string]*Stream), - config: cfg, - limits: limits, - eventBus: bus, + streams: make(map[string]*Stream), + streamOrder: list.New(), + streamOrderByKey: make(map[string]*list.Element), + config: cfg, + limits: limits, + eventBus: bus, } } @@ -64,25 +74,41 @@ func (h *StreamHub) Limits() config.LimitsConfig { // Returns an error if max_streams limit is reached and the stream does not already exist. func (h *StreamHub) GetOrCreate(key string) (*Stream, error) { h.mu.Lock() - defer h.mu.Unlock() - + var replacing *Stream if s, ok := h.streams[key]; ok { if s.State() != StreamStateDestroying { + h.mu.Unlock() return s, nil } // Stream is being destroyed; replace it with a fresh one. delete(h.streams, key) + h.removeOrderedStreamLocked(key, s) + replacing = s } if max := h.limits.MaxStreams; max > 0 && len(h.streams) >= max { + h.mu.Unlock() return nil, fmt.Errorf("max streams limit reached (%d)", max) } s := NewStream(key, h.config, h.limits, h.eventBus) + s.setDestroyCallback(func() { h.removeIfCurrent(key, s) }) if h.audioCodecEnabled { s.transcodeManager = NewTranscodeManager(s, audiocodec.Global(), h.config.RingBufferSize) } h.streams[key] = s + h.addOrderedStreamLocked(key, s) + h.mu.Unlock() + + if replacing != nil { + replacing.Close() + _ = h.eventBus.Emit(EventStreamDestroy, &EventContext{ + StreamKey: key, + StreamInstanceID: replacing.InstanceID(), + PublisherGeneration: replacing.LastPublisherGeneration(), + PublisherID: replacing.LastPublisherID(), + }) //nolint:errcheck + } h.eventBus.Emit(EventStreamCreate, &EventContext{StreamKey: key}) //nolint:errcheck @@ -100,11 +126,44 @@ func (h *StreamHub) Find(key string) (*Stream, bool) { // Remove deletes a stream from the hub and emits EventStreamDestroy. func (h *StreamHub) Remove(key string) { h.mu.Lock() - defer h.mu.Unlock() - if _, ok := h.streams[key]; ok { + stream, ok := h.streams[key] + if ok { delete(h.streams, key) - h.eventBus.Emit(EventStreamDestroy, &EventContext{StreamKey: key}) //nolint:errcheck + h.removeOrderedStreamLocked(key, stream) } + h.mu.Unlock() + if !ok { + return + } + stream.Close() + _ = h.eventBus.Emit(EventStreamDestroy, &EventContext{ + StreamKey: key, + StreamInstanceID: stream.InstanceID(), + PublisherGeneration: stream.LastPublisherGeneration(), + PublisherID: stream.LastPublisherID(), + }) //nolint:errcheck +} + +func (h *StreamHub) removeIfCurrent(key string, stream *Stream) { + h.mu.Lock() + current, ok := h.streams[key] + if ok && current == stream { + delete(h.streams, key) + h.removeOrderedStreamLocked(key, stream) + } + h.mu.Unlock() + if !ok || current != stream { + return + } + // The timer already transitioned the stream to Destroying. Close the ring + // without re-entering the once-guarded destroy callback. + stream.ringBuffer.Close() + _ = h.eventBus.Emit(EventStreamDestroy, &EventContext{ + StreamKey: key, + StreamInstanceID: stream.InstanceID(), + PublisherGeneration: stream.LastPublisherGeneration(), + PublisherID: stream.LastPublisherID(), + }) //nolint:errcheck } // Count returns the number of active streams. @@ -124,3 +183,40 @@ func (h *StreamHub) Keys() []string { } return keys } + +// StableStreams returns at most limit non-destroying streams in creation +// order. The returned slice is detached from Hub state and bounded by limit. +func (h *StreamHub) StableStreams(limit int) []*Stream { + if limit <= 0 { + return nil + } + h.mu.RLock() + capacity := min(limit, len(h.streams)) + streams := make([]*Stream, 0, capacity) + for element := h.streamOrder.Front(); element != nil && len(streams) < limit; element = element.Next() { + entry := element.Value.(orderedStream) + if entry.stream.State() == StreamStateDestroying { + continue + } + streams = append(streams, entry.stream) + } + h.mu.RUnlock() + return streams +} + +func (h *StreamHub) addOrderedStreamLocked(key string, stream *Stream) { + h.streamOrderByKey[key] = h.streamOrder.PushBack(orderedStream{key: key, stream: stream}) +} + +func (h *StreamHub) removeOrderedStreamLocked(key string, stream *Stream) { + element, ok := h.streamOrderByKey[key] + if !ok { + return + } + entry := element.Value.(orderedStream) + if entry.stream != stream { + return + } + h.streamOrder.Remove(element) + delete(h.streamOrderByKey, key) +} diff --git a/core/stream_hub_test.go b/core/stream_hub_test.go index 96040689..ab6bb7d7 100644 --- a/core/stream_hub_test.go +++ b/core/stream_hub_test.go @@ -2,6 +2,7 @@ package core import ( "testing" + "time" "github.com/im-pingo/liveforge/config" ) @@ -63,6 +64,35 @@ func TestStreamHubList(t *testing.T) { } } +func TestStreamHubStableStreamsReturnsBoundedCreationOrder(t *testing.T) { + hub := NewStreamHub(newTestStreamConfig(), config.LimitsConfig{}, NewEventBus()) + for _, key := range []string{"live/z", "live/a", "live/m"} { + if _, err := hub.GetOrCreate(key); err != nil { + t.Fatal(err) + } + } + + assertStreamKeys := func(want []string) { + t.Helper() + streams := hub.StableStreams(len(want)) + if len(streams) != len(want) { + t.Fatalf("stable stream count = %d, want %d", len(streams), len(want)) + } + for index, stream := range streams { + if got := stream.Key(); got != want[index] { + t.Fatalf("stable stream %d = %q, want %q", index, got, want[index]) + } + } + } + + assertStreamKeys([]string{"live/z", "live/a"}) + hub.Remove("live/z") + assertStreamKeys([]string{"live/a", "live/m"}) + if got := hub.StableStreams(0); len(got) != 0 { + t.Fatalf("zero-limit stable streams = %d, want 0", len(got)) + } +} + func TestStreamHubFind(t *testing.T) { bus := NewEventBus() cfg := newTestStreamConfig() @@ -102,6 +132,28 @@ func TestStreamHubGetOrCreateReplacesDestroying(t *testing.T) { } } +func TestStreamHubRemovesStreamAfterPublisherTimeout(t *testing.T) { + cfg := config.StreamConfig{RingBufferSize: 8, NoPublisherTimeout: 20 * time.Millisecond} + hub := NewStreamHub(cfg, config.LimitsConfig{}, NewEventBus()) + stream, err := hub.GetOrCreate("live/timeout-removal") + if err != nil { + t.Fatal(err) + } + if err := stream.SetPublisher(&testPublisher{id: "timeout-publisher"}); err != nil { + t.Fatal(err) + } + stream.RemovePublisher() + + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if _, ok := hub.Find("live/timeout-removal"); !ok { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("publisher-timeout stream remained in hub") +} + func TestStreamHubMaxStreams(t *testing.T) { bus := NewEventBus() cfg := newTestStreamConfig() diff --git a/core/stream_stats.go b/core/stream_stats.go index 813ef0bd..60bed0de 100644 --- a/core/stream_stats.go +++ b/core/stream_stats.go @@ -13,30 +13,33 @@ type StreamStats struct { bytesIn atomic.Int64 videoFrames atomic.Int64 audioFrames atomic.Int64 - lastFrame atomic.Value // time.Time - // windowMu also guards startTime: initStats runs on the publisher - // goroutine while snapshot() runs on API handler goroutines. + // recordFrame updates the window counters atomically. windowMu only + // serializes snapshot rotations, so frame ingestion never waits for a stats + // reader. windowMu sync.Mutex - startTime time.Time - windowBytes int64 - windowVideo int64 - windowStart time.Time - snapBytes int64 - snapVideo int64 - snapTime time.Time + startTime atomic.Int64 + windowBytes atomic.Int64 + windowVideo atomic.Int64 + windowStart atomic.Int64 + snapBytes atomic.Int64 + snapVideo atomic.Int64 + snapTime atomic.Int64 } const statsWindowDuration = 2 * time.Second // initStats sets the start time. Called once when the stream begins publishing. func (s *StreamStats) initStats() { - now := time.Now() - s.lastFrame.Store(now) + now := time.Now().UnixNano() s.windowMu.Lock() - s.startTime = now - s.windowStart = now - s.snapTime = now + s.startTime.Store(now) + s.windowStart.Store(now) + s.snapTime.Store(now) + s.windowBytes.Store(0) + s.windowVideo.Store(0) + s.snapBytes.Store(0) + s.snapVideo.Store(0) s.windowMu.Unlock() } @@ -48,15 +51,11 @@ func (s *StreamStats) recordFrame(payloadSize int, isVideo bool) { } else { s.audioFrames.Add(1) } - s.lastFrame.Store(time.Now()) - - // Update sliding window counters. - s.windowMu.Lock() - s.windowBytes += int64(payloadSize) + // Update sliding window counters without contending with stats readers. + s.windowBytes.Add(int64(payloadSize)) if isVideo { - s.windowVideo++ + s.windowVideo.Add(1) } - s.windowMu.Unlock() } // StreamStatsSnapshot is a point-in-time copy of stream statistics. @@ -81,30 +80,35 @@ func (s *StreamStats) snapshot() StreamStatsSnapshot { AudioFrames: s.audioFrames.Load(), } - // Compute instantaneous bitrate and FPS from sliding window. + // Compute instantaneous bitrate and FPS from the sliding window. Rotation is + // serialized, but frame ingestion uses atomic counters and does not wait. s.windowMu.Lock() - snap.StartTime = s.startTime - elapsed := now.Sub(s.startTime) + startNano := s.startTime.Load() + windowStartNano := s.windowStart.Load() + snapTimeNano := s.snapTime.Load() + startTime := time.Unix(0, startNano) + windowStart := time.Unix(0, windowStartNano) + snapTime := time.Unix(0, snapTimeNano) + snap.StartTime = startTime + elapsed := now.Sub(startTime) snap.Uptime = elapsed - windowElapsed := now.Sub(s.windowStart) + windowElapsed := now.Sub(windowStart) if windowElapsed >= statsWindowDuration { // Window has enough data: compute rates from current window, // then rotate: current window becomes the new snapshot. if ms := windowElapsed.Milliseconds(); ms > 0 { - snap.BitrateKbps = s.windowBytes * 8 / ms - snap.FPS = float64(s.windowVideo) / windowElapsed.Seconds() + snap.BitrateKbps = s.windowBytes.Load() * 8 / ms + snap.FPS = float64(s.windowVideo.Load()) / windowElapsed.Seconds() } - s.snapBytes = s.windowBytes - s.snapVideo = s.windowVideo - s.snapTime = s.windowStart - s.windowBytes = 0 - s.windowVideo = 0 - s.windowStart = now - } else if s.snapTime != s.startTime || windowElapsed > 0 { + s.snapBytes.Store(s.windowBytes.Swap(0)) + s.snapVideo.Store(s.windowVideo.Swap(0)) + s.snapTime.Store(windowStartNano) + s.windowStart.Store(now.UnixNano()) + } else if snapTimeNano != startNano || windowElapsed > 0 { // Window too short: use snapshot + current window combined. - totalBytes := s.snapBytes + s.windowBytes - totalVideo := s.snapVideo + s.windowVideo - totalElapsed := now.Sub(s.snapTime) + totalBytes := s.snapBytes.Load() + s.windowBytes.Load() + totalVideo := s.snapVideo.Load() + s.windowVideo.Load() + totalElapsed := now.Sub(snapTime) if ms := totalElapsed.Milliseconds(); ms > 0 { snap.BitrateKbps = totalBytes * 8 / ms snap.FPS = float64(totalVideo) / totalElapsed.Seconds() diff --git a/core/stream_test.go b/core/stream_test.go index a8d23370..5b8e959e 100644 --- a/core/stream_test.go +++ b/core/stream_test.go @@ -3,11 +3,14 @@ package core import ( "context" "encoding/binary" + "reflect" "runtime" + "sync/atomic" "testing" "time" "github.com/im-pingo/liveforge/config" + "github.com/im-pingo/liveforge/pkg/audiocodec" "github.com/im-pingo/liveforge/pkg/avframe" ) @@ -30,6 +33,17 @@ func (p *testPublisher) ID() string { return p.id } func (p *testPublisher) MediaInfo() *avframe.MediaInfo { return p.info } func (p *testPublisher) Close() error { return nil } +type reentrantClosePublisher struct { + stream *Stream +} + +func (p *reentrantClosePublisher) ID() string { return "reentrant-close" } +func (p *reentrantClosePublisher) MediaInfo() *avframe.MediaInfo { return nil } +func (p *reentrantClosePublisher) Close() error { + _ = p.stream.State() + return nil +} + type typedNilTestPublisher struct{} func (*typedNilTestPublisher) ID() string { return "typed-nil" } @@ -62,6 +76,34 @@ func TestStreamStateTransitions(t *testing.T) { } } +func TestStreamCloseDoesNotHoldLockWhileClosingPublisher(t *testing.T) { + s := NewStream("live/reentrant-close", newTestStreamConfig(), config.LimitsConfig{}, NewEventBus()) + pub := &reentrantClosePublisher{stream: s} + if err := s.SetPublisher(pub); err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + go func() { + s.Close() + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("stream close deadlocked while closing publisher") + } +} + +func TestStreamRejectsPublisherAfterClose(t *testing.T) { + s := NewStream("live/closed", newTestStreamConfig(), config.LimitsConfig{}, NewEventBus()) + s.Close() + if err := s.SetPublisher(&testPublisher{id: "late"}); err == nil { + t.Fatal("SetPublisher accepted a publisher after stream close") + } +} + func TestStreamRejectsNilPublisherWithoutMutation(t *testing.T) { var typedNil *typedNilTestPublisher tests := []struct { @@ -119,6 +161,24 @@ func TestStreamRejectsNilPublisherWithoutMutation(t *testing.T) { } } +func TestStreamRejectsNilFrameWithoutMutation(t *testing.T) { + s := NewStream("live/nil-frame", newTestStreamConfig(), config.LimitsConfig{}, NewEventBus()) + pub := &testPublisher{id: "nil-frame-publisher"} + if err := s.SetPublisher(pub); err != nil { + t.Fatal(err) + } + beforeCursor := s.RingBuffer().WriteCursor() + if s.WriteFrame(nil) { + t.Fatal("WriteFrame accepted a nil frame") + } + if s.WriteFrameForPublisher(pub, nil) { + t.Fatal("WriteFrameForPublisher accepted a nil frame") + } + if got := s.RingBuffer().WriteCursor(); got != beforeCursor { + t.Fatalf("nil frame advanced ring cursor from %d to %d", beforeCursor, got) + } +} + func TestRemovePublisherIfKeepsReplacement(t *testing.T) { s := NewStream("live/reorder", newTestStreamConfig(), config.LimitsConfig{}, NewEventBus()) oldPublisher := &testPublisher{id: "old"} @@ -138,6 +198,219 @@ func TestRemovePublisherIfKeepsReplacement(t *testing.T) { } } +func TestStreamRejectsPublisherIDReuseAfterInterveningGeneration(t *testing.T) { + cfg := newTestStreamConfig() + cfg.NoPublisherTimeout = time.Hour + s := NewStream("live/reused-publisher-id", cfg, config.LimitsConfig{}, NewEventBus()) + tm := NewTranscodeManager(s, audiocodec.Global(), cfg.RingBufferSize) + SetTranscodeManagerForTest(s, tm) + firstA := &testPublisher{ + id: "publisher-a", + info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}, + } + if err := s.SetPublisher(firstA); err != nil { + t.Fatal(err) + } + if !s.RemovePublisherIf(firstA) { + t.Fatal("first publisher A was not removed") + } + + publisherB := &testPublisher{ + id: "publisher-b", + info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264, AudioCodec: avframe.CodecG711U}, + } + if err := s.SetPublisher(publisherB); err != nil { + t.Fatal(err) + } + videoHeader := avframe.NewAVFrame( + avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeSequenceHeader, + 0, 0, []byte{0x67, 0x64, 0x00, 0x1f}, + ) + audioHeader := avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711U, avframe.FrameTypeSequenceHeader, + 0, 0, []byte{0x00, 0x01}, + ) + keyframe := avframe.NewAVFrame( + avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, + 100, 100, []byte{0x65, 0x01, 0x02}, + ) + audio := avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711U, avframe.FrameTypeInterframe, + 120, 120, []byte{0x11, 0x22}, + ) + interframe := avframe.NewAVFrame( + avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, + 140, 140, []byte{0x41}, + ) + for _, frame := range []*avframe.AVFrame{videoHeader, audioHeader, keyframe, audio, interframe} { + if !s.WriteFrameForPublisher(publisherB, frame) { + t.Fatalf("publisher B frame was rejected: %+v", frame) + } + } + _, releaseTranscode, err := tm.GetOrCreateReaderAt(avframe.CodecG711A, s.GOPCacheSourceStart()) + if err != nil { + t.Fatal(err) + } + defer releaseTranscode() + if !s.RemovePublisherIf(publisherB) { + t.Fatal("publisher B was not removed") + } + + type stableStatsSnapshot struct { + BytesIn int64 + VideoFrames int64 + AudioFrames int64 + StartTime time.Time + } + stableStats := func() stableStatsSnapshot { + stats := s.Stats() + return stableStatsSnapshot{ + BytesIn: stats.BytesIn, VideoFrames: stats.VideoFrames, + AudioFrames: stats.AudioFrames, StartTime: stats.StartTime, + } + } + type transcodeTrackSnapshot struct { + Track *TranscodedTrack + TrackCount int + AudioTrackCount int + TargetCodec avframe.CodecType + SourceStart int64 + SubscriberCount int + GenerationDone <-chan struct{} + GenerationBoundary *streamGenerationBoundary + } + transcodeState := func() transcodeTrackSnapshot { + tm.mu.Lock() + defer tm.mu.Unlock() + track := tm.tracks[avframe.CodecG711A] + snapshot := transcodeTrackSnapshot{ + Track: track, TrackCount: len(tm.tracks), AudioTrackCount: len(tm.audioTracks), + } + if track != nil { + snapshot.TargetCodec = track.targetCodec + snapshot.SourceStart = track.sourceStart + snapshot.SubscriberCount = track.subCount + snapshot.GenerationDone = track.generationDone + snapshot.GenerationBoundary = track.generationBoundary + } + return snapshot + } + + before := s.StartupSnapshot() + beforeNoPublisherTimer := s.noPublisherTimer + beforeIdleTimer := s.idleTimer + beforeCursor := s.RingBuffer().WriteCursor() + beforeGOP := s.GOPCache() + beforeGOPDetail := s.GOPCacheDetail() + beforeVideoHeader := s.VideoSeqHeader() + beforeAudioHeader := s.AudioSeqHeader() + beforeStats := stableStats() + beforeTranscode := transcodeState() + if len(beforeGOP) == 0 || beforeVideoHeader == nil || beforeAudioHeader == nil { + t.Fatal("publisher B did not populate retained GOP and sequence-header state") + } + if beforeStats.BytesIn == 0 || beforeStats.VideoFrames == 0 || beforeStats.AudioFrames == 0 { + t.Fatalf("publisher B did not populate stream stats: %+v", beforeStats) + } + if beforeTranscode.Track == nil || beforeTranscode.SubscriberCount != 1 { + t.Fatalf("publisher B did not populate transcode state: %+v", beforeTranscode) + } + reusedA := &testPublisher{ + id: firstA.ID(), + info: &avframe.MediaInfo{VideoCodec: avframe.CodecH265}, + } + if err := s.SetPublisher(reusedA); err == nil { + t.Fatal("SetPublisher accepted A -> B -> A publisher ID reuse") + } + if s.Publisher() != nil || s.State() != StreamStateNoPublisher { + t.Fatal("rejected publisher ID reuse mutated stream ownership") + } + after := s.StartupSnapshot() + if !reflect.DeepEqual(after, before) { + t.Fatalf("rejected publisher ID reuse mutated startup state: before=%+v after=%+v", before, after) + } + if s.noPublisherTimer != beforeNoPublisherTimer || s.idleTimer != beforeIdleTimer { + t.Fatal("rejected publisher ID reuse changed stream timers") + } + if got := s.RingBuffer().WriteCursor(); got != beforeCursor { + t.Fatalf("rejected publisher ID reuse advanced ring cursor from %d to %d", beforeCursor, got) + } + if got := s.GOPCache(); !reflect.DeepEqual(got, beforeGOP) { + t.Fatalf("rejected publisher ID reuse changed GOP cache: before=%+v after=%+v", beforeGOP, got) + } + if got := s.GOPCacheDetail(); got != beforeGOPDetail { + t.Fatalf("rejected publisher ID reuse changed GOP detail: before=%+v after=%+v", beforeGOPDetail, got) + } + if got := s.VideoSeqHeader(); got != beforeVideoHeader || !reflect.DeepEqual(got, beforeVideoHeader) { + t.Fatalf("rejected publisher ID reuse changed video sequence header: before=%+v after=%+v", beforeVideoHeader, got) + } + if got := s.AudioSeqHeader(); got != beforeAudioHeader || !reflect.DeepEqual(got, beforeAudioHeader) { + t.Fatalf("rejected publisher ID reuse changed audio sequence header: before=%+v after=%+v", beforeAudioHeader, got) + } + if got := stableStats(); got != beforeStats { + t.Fatalf("rejected publisher ID reuse changed stable stats: before=%+v after=%+v", beforeStats, got) + } + if got := transcodeState(); got != beforeTranscode { + t.Fatalf("rejected publisher ID reuse changed transcode state: before=%+v after=%+v", beforeTranscode, got) + } + + active := &testPublisher{id: "publisher-c", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}} + if err := s.SetPublisher(active); err != nil { + t.Fatal(err) + } + activeCursor := s.RingBuffer().WriteCursor() + staleFrame := avframe.NewAVFrame( + avframe.MediaTypeVideo, + avframe.CodecH264, + avframe.FrameTypeKeyframe, + 0, + 0, + []byte{0x65}, + ) + if s.WriteFrameForPublisher(firstA, staleFrame) { + t.Fatal("stale publisher A frame was accepted after rejected ID reuse") + } + if got := s.RingBuffer().WriteCursor(); got != activeCursor { + t.Fatalf("stale publisher A advanced ring cursor from %d to %d", activeCursor, got) + } + staleActivityRan := false + if s.WithActivePublisher(firstA, func() { staleActivityRan = true }) { + t.Fatal("stale publisher A activity was accepted after rejected ID reuse") + } + if staleActivityRan { + t.Fatal("stale publisher A activity callback ran") + } + if s.RemovePublisherIf(firstA) { + t.Fatal("stale publisher A removed the active publisher") + } + if s.Publisher() != active || s.State() != StreamStatePublishing { + t.Fatal("stale publisher A callbacks changed active ownership") + } +} + +func TestRemovePublisherIfDoesNotRepeatCleanupForDetachedPublisher(t *testing.T) { + cfg := newTestStreamConfig() + cfg.NoPublisherTimeout = time.Second + s := NewStream("live/idempotent-remove", cfg, config.LimitsConfig{}, NewEventBus()) + pub := &testPublisher{id: "publisher"} + if err := s.SetPublisher(pub); err != nil { + t.Fatal(err) + } + if !s.RemovePublisherIf(pub) { + t.Fatal("active publisher was not removed") + } + timer := s.noPublisherTimer + if timer == nil { + t.Fatal("first removal did not start no-publisher timer") + } + if s.RemovePublisherIf(pub) { + t.Fatal("detached publisher repeated stream cleanup") + } + if s.noPublisherTimer != timer { + t.Fatal("detached publisher reset the no-publisher timer") + } +} + func TestWithActivePublisherLinearizesActivityAndReplacement(t *testing.T) { s := NewStream("live/linearized-activity", newTestStreamConfig(), config.LimitsConfig{}, NewEventBus()) oldPublisher := &testPublisher{id: "old"} @@ -281,6 +554,71 @@ func TestStreamPublisherGenerationIsolation(t *testing.T) { } } +func TestStreamStartupSnapshotRetainsGenerationEndCursor(t *testing.T) { + s := NewStream("live/generation-end-cursor", newTestStreamConfig(), config.LimitsConfig{}, NewEventBus()) + pubA := &testPublisher{id: "publisher-a", info: &avframe.MediaInfo{AudioCodec: avframe.CodecG711A}} + if err := s.SetPublisher(pubA); err != nil { + t.Fatal(err) + } + snapshotA := s.StartupSnapshot() + if _, ended := snapshotA.GenerationEndCursor(); ended { + t.Fatal("active generation reported an end cursor") + } + for timestamp := int64(0); timestamp < 60; timestamp += 20 { + s.WriteFrameForPublisher(pubA, avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711A, avframe.FrameTypeInterframe, + timestamp, timestamp, []byte{1}, + )) + } + wantEnd := s.RingBuffer().WriteCursor() + s.RemovePublisher() + if got, ended := snapshotA.GenerationEndCursor(); !ended || got != wantEnd { + t.Fatalf("old generation end cursor = (%d, %v), want (%d, true)", got, ended, wantEnd) + } + + pubB := &testPublisher{id: "publisher-b", info: &avframe.MediaInfo{AudioCodec: avframe.CodecG711A}} + if err := s.SetPublisher(pubB); err != nil { + t.Fatal(err) + } + s.WriteFrameForPublisher(pubB, avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711A, avframe.FrameTypeInterframe, + 0, 0, []byte{2}, + )) + if got, _ := snapshotA.GenerationEndCursor(); got != wantEnd { + t.Fatalf("replacement changed old generation end cursor from %d to %d", wantEnd, got) + } + if _, ended := s.StartupSnapshot().GenerationEndCursor(); ended { + t.Fatal("replacement active generation inherited old end cursor") + } +} + +func TestStreamPublisherReplacementClosesGenerationBeforeReset(t *testing.T) { + stream := NewStream("live/replacement-reset-order", newTestStreamConfig(), config.LimitsConfig{}, NewEventBus()) + defer stream.Close() + if err := stream.SetPublisher(&testPublisher{id: "old", info: &avframe.MediaInfo{AudioCodec: avframe.CodecG711A}}); err != nil { + t.Fatal(err) + } + snapshot := stream.StartupSnapshot() + var resetCause error + tm := NewTranscodeManager(stream, nil, 1) + tm.tracks[avframe.CodecAAC] = &TranscodedTrack{ + generationBoundary: snapshot.generationBoundary, + cancel: func(err error) { resetCause = err }, + } + SetTranscodeManagerForTest(stream, tm) + stream.RemovePublisher() + + if err := stream.SetPublisher(&testPublisher{id: "replacement", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}}); err != nil { + t.Fatal(err) + } + if resetCause != nil { + t.Fatalf("old generation transcode track reset with %v before generation close", resetCause) + } + if _, ended := snapshot.GenerationEndCursor(); !ended { + t.Fatal("old publisher generation was not closed before replacement") + } +} + func TestStreamWaitForStartupContextCancellation(t *testing.T) { t.Run("not ready", func(t *testing.T) { s := NewStream("live/startup-cancel", newTestStreamConfig(), config.LimitsConfig{}, NewEventBus()) @@ -858,6 +1196,49 @@ func TestStreamMaxBitrateDisabled(t *testing.T) { } } +func BenchmarkStreamWriteFrame(b *testing.B) { + cfg := newTestStreamConfig() + cfg.GOPCache = false + s := NewStream("bench/write-frame", cfg, config.LimitsConfig{}, NewEventBus()) + pub := &testPublisher{id: "bench-publisher", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}} + if err := s.SetPublisher(pub); err != nil { + b.Fatal(err) + } + frame := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 40, 40, make([]byte, 512)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if !s.WriteFrameForPublisher(pub, frame) { + b.Fatal("benchmark frame was rejected") + } + } +} + +func BenchmarkStreamWriteFrameWithGOPCache(b *testing.B) { + cfg := newTestStreamConfig() + cfg.GOPCache = true + cfg.GOPCacheNum = 1 + s := NewStream("bench/write-frame-gop-cache", cfg, config.LimitsConfig{}, NewEventBus()) + pub := &testPublisher{id: "bench-publisher", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}} + if err := s.SetPublisher(pub); err != nil { + b.Fatal(err) + } + keyframe := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 0, 0, make([]byte, 1024)) + if !s.WriteFrameForPublisher(pub, keyframe) { + b.Fatal("benchmark keyframe was rejected") + } + frame := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 40, 40, make([]byte, 512)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + frame.DTS = int64(i + 1) + frame.PTS = frame.DTS + if !s.WriteFrameForPublisher(pub, frame) { + b.Fatal("benchmark frame was rejected") + } + } +} + func TestStreamStateString(t *testing.T) { tests := []struct { state StreamState @@ -930,6 +1311,168 @@ func TestStreamClose(t *testing.T) { s.Close() } +func TestStreamDestructionIsIrreversibleAgainstLatePublisherCleanup(t *testing.T) { + routes := []struct { + name string + configure func(*config.StreamConfig) + destroy func(*testing.T, *Stream, Publisher) + }{ + { + name: "explicit close", + configure: func(cfg *config.StreamConfig) { + cfg.NoPublisherTimeout = 0 + cfg.IdleTimeout = 0 + }, + destroy: func(_ *testing.T, stream *Stream, _ Publisher) { + stream.Close() + }, + }, + { + name: "no-publisher timeout", + configure: func(cfg *config.StreamConfig) { + cfg.NoPublisherTimeout = time.Millisecond + cfg.IdleTimeout = 0 + }, + destroy: func(t *testing.T, stream *Stream, publisher Publisher) { + t.Helper() + if !stream.RemovePublisherIf(publisher) { + t.Fatal("active publisher was not removed") + } + }, + }, + { + name: "idle timeout", + configure: func(cfg *config.StreamConfig) { + cfg.NoPublisherTimeout = 0 + cfg.IdleTimeout = time.Millisecond + }, + destroy: func(t *testing.T, stream *Stream, publisher Publisher) { + t.Helper() + if !stream.RemovePublisherIf(publisher) { + t.Fatal("active publisher was not removed") + } + }, + }, + { + name: "policy no-publisher timeout", + configure: func(cfg *config.StreamConfig) { + cfg.NoPublisherTimeout = 0 + cfg.IdleTimeout = 0 + }, + destroy: func(t *testing.T, stream *Stream, publisher Publisher) { + t.Helper() + if !stream.RemovePublisherIf(publisher) { + t.Fatal("active publisher was not removed") + } + next := stream.Config() + next.NoPublisherTimeout = time.Millisecond + stream.UpdatePolicy(next, config.LimitsConfig{}) + }, + }, + } + + for _, route := range routes { + t.Run(route.name, func(t *testing.T) { + cfg := newTestStreamConfig() + route.configure(&cfg) + stream := NewStream("live/destroy-"+route.name, cfg, config.LimitsConfig{}, NewEventBus()) + publisher := &testPublisher{id: "publisher", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}} + if err := stream.SetPublisher(publisher); err != nil { + t.Fatal(err) + } + if !stream.WriteFrameForPublisher(publisher, avframe.NewAVFrame( + avframe.MediaTypeVideo, + avframe.CodecH264, + avframe.FrameTypeKeyframe, + 0, + 0, + []byte{0x65}, + )) { + t.Fatal("active publisher frame was rejected") + } + cursorBeforeDestroy := stream.RingBuffer().WriteCursor() + + var destroyCalls atomic.Int32 + destroyed := make(chan struct{}, 1) + stream.setDestroyCallback(func() { + destroyCalls.Add(1) + stream.RingBuffer().Close() + select { + case destroyed <- struct{}{}: + default: + } + }) + route.destroy(t, stream, publisher) + select { + case <-destroyed: + case <-time.After(time.Second): + t.Fatal("stream destruction callback did not run") + } + + if got := stream.State(); got != StreamStateDestroying { + t.Fatalf("state after destruction = %s, want destroying", got) + } + if !stream.RingBuffer().IsClosed() { + t.Fatal("ring remained open after destruction") + } + if got := destroyCalls.Load(); got != 1 { + t.Fatalf("destroy callback calls = %d, want 1", got) + } + if stream.RemovePublisherIf(publisher) { + t.Error("late conditional cleanup removed a publisher after destruction") + } + if got := stream.State(); got != StreamStateDestroying { + t.Errorf("state after late conditional cleanup = %s, want destroying", got) + } + + stream.RemovePublisher() + if got := stream.State(); got != StreamStateDestroying { + t.Errorf("state after late unconditional cleanup = %s, want destroying", got) + } + + trySetPublisher := func(candidate Publisher) (panicValue any, err error) { + defer func() { panicValue = recover() }() + err = stream.SetPublisher(candidate) + return panicValue, err + } + nonEmpty := &testPublisher{id: "late-non-empty", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH265}} + if panicValue, err := trySetPublisher(nonEmpty); panicValue != nil { + t.Errorf("non-empty publisher reattach panicked: %v", panicValue) + } else if err == nil { + t.Error("non-empty publisher reattached after destruction") + } + if got := stream.State(); got != StreamStateDestroying { + t.Errorf("state after non-empty reattach attempt = %s, want destroying", got) + } + + stream.RemovePublisher() + emptyID := &testPublisher{info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}} + if panicValue, err := trySetPublisher(emptyID); panicValue != nil { + t.Errorf("empty-ID publisher reattach panicked: %v", panicValue) + } else if err == nil { + t.Error("empty-ID publisher reattached after destruction") + } + if got := stream.State(); got != StreamStateDestroying { + t.Errorf("state after empty-ID reattach attempt = %s, want destroying", got) + } + if stream.Publisher() != nil { + t.Error("destroyed stream retained a reattached publisher") + } + if !stream.RingBuffer().IsClosed() { + t.Error("late cleanup or reattach reopened the ring") + } + if got := stream.RingBuffer().WriteCursor(); got != cursorBeforeDestroy { + t.Errorf("closed ring cursor = %d, want %d", got, cursorBeforeDestroy) + } + + stream.Close() + if got := destroyCalls.Load(); got != 1 { + t.Errorf("destroy callback calls after late cleanup and close = %d, want 1", got) + } + }) + } +} + func TestStreamSubscribers(t *testing.T) { bus := NewEventBus() cfg := newTestStreamConfig() @@ -1106,6 +1649,211 @@ func TestStreamGOPCacheDetailIncludesZeroDTS(t *testing.T) { } } +func TestStreamGOPCacheMaxFramesKeepsPlayablePrefix(t *testing.T) { + cfg := newTestStreamConfig() + cfg.GOPCacheNum = 1 + cfg.GOPCacheMaxFrames = 3 + stream := NewStream("live/gop-max-frames", cfg, config.LimitsConfig{}, NewEventBus()) + if err := stream.SetPublisher(&testPublisher{id: "frames", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264, AudioCodec: avframe.CodecAAC}}); err != nil { + t.Fatal(err) + } + + frames := []*avframe.AVFrame{ + avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 0, 0, []byte{1}), + avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 20, 20, []byte{2}), + avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 40, 40, []byte{3}), + avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 60, 60, []byte{4}), + } + for _, frame := range frames { + stream.WriteFrame(frame) + } + + got := stream.GOPCache() + if len(got) != 3 || got[0] != frames[0] || got[2] != frames[2] { + t.Fatalf("frame-bounded GOP = %v, want first three interleaved frames", got) + } +} + +func TestStreamGOPCacheMaxDurationKeepsPlayablePrefix(t *testing.T) { + cfg := newTestStreamConfig() + cfg.GOPCacheNum = 1 + cfg.GOPCacheMaxDuration = 40 * time.Millisecond + stream := NewStream("live/gop-max-duration", cfg, config.LimitsConfig{}, NewEventBus()) + if err := stream.SetPublisher(&testPublisher{id: "duration", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}}); err != nil { + t.Fatal(err) + } + + key := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 100, 100, []byte{1}) + inter := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 140, 140, []byte{2}) + tooLate := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 180, 180, []byte{3}) + stream.WriteFrame(key) + stream.WriteFrame(inter) + stream.WriteFrame(tooLate) + + got := stream.GOPCache() + if len(got) != 2 || got[0] != key || got[1] != inter { + t.Fatalf("duration-bounded GOP = %v, want frames through 140ms", got) + } +} + +func TestStreamGOPCacheMaxBytesKeepsKeyframeAndInterleavedFrames(t *testing.T) { + cfg := newTestStreamConfig() + cfg.GOPCacheNum = 1 + cfg.GOPCacheMaxBytes = 5 + stream := NewStream("live/gop-max-bytes", cfg, config.LimitsConfig{}, NewEventBus()) + if err := stream.SetPublisher(&testPublisher{id: "bytes", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264, AudioCodec: avframe.CodecAAC}}); err != nil { + t.Fatal(err) + } + + key := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 0, 0, []byte{1, 2, 3}) + audio := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 20, 20, []byte{4, 5}) + video := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 40, 40, []byte{6}) + stream.WriteFrame(key) + stream.WriteFrame(audio) + stream.WriteFrame(video) + + got := stream.GOPCache() + if len(got) != 2 || got[0] != key || got[1] != audio { + t.Fatalf("byte-bounded GOP = %v, want keyframe plus audio", got) + } +} + +func TestStreamGOPCacheStopsAtFirstRejectedFrameUntilNextKeyframe(t *testing.T) { + tests := []struct { + name string + configure func(*config.StreamConfig) + rejected *avframe.AVFrame + later *avframe.AVFrame + }{ + { + name: "bytes", + configure: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxBytes = 5 + }, + rejected: avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 40, 40, []byte{4, 5, 6}), + later: avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 60, 60, []byte{7}), + }, + { + name: "duration", + configure: func(cfg *config.StreamConfig) { + cfg.GOPCacheMaxDuration = 40 * time.Millisecond + }, + rejected: avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 80, 80, []byte{4}), + later: avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 20, 20, []byte{5}), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := newTestStreamConfig() + cfg.GOPCacheNum = 1 + test.configure(&cfg) + stream := NewStream("live/gop-sealed-"+test.name, cfg, config.LimitsConfig{}, NewEventBus()) + if err := stream.SetPublisher(&testPublisher{id: test.name, info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264, AudioCodec: avframe.CodecAAC}}); err != nil { + t.Fatal(err) + } + + key := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 0, 0, []byte{1, 2, 3}) + stream.WriteFrame(key) + stream.WriteFrame(test.rejected) + stream.WriteFrame(test.later) + if got := stream.GOPCache(); len(got) != 1 || got[0] != key { + t.Fatalf("truncated GOP accepted a later frame: %v", got) + } + + nextKey := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 100, 100, []byte{8}) + nextInter := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 120, 120, []byte{9}) + stream.WriteFrame(nextKey) + stream.WriteFrame(nextInter) + if got := stream.GOPCache(); len(got) != 2 || got[0] != nextKey || got[1] != nextInter { + t.Fatalf("next GOP did not reopen cache: %v", got) + } + }) + } +} + +func TestStreamGOPCacheUsesUnorderedDTSSpanForAdmission(t *testing.T) { + cfg := newTestStreamConfig() + cfg.GOPCacheMaxFrames = 0 + cfg.GOPCacheMaxBytes = 0 + cfg.GOPCacheMaxDuration = 100 * time.Millisecond + stream := NewStream("live/gop-unordered-dts", cfg, config.LimitsConfig{}, NewEventBus()) + + key := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 100, 100, []byte{1}) + late := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 140, 140, []byte{2}) + older := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 0, 0, []byte{3}) + tooWide := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 150, 150, []byte{4}) + for _, frame := range []*avframe.AVFrame{key, late, older, tooWide} { + stream.WriteFrame(frame) + } + + if got := stream.GOPCache(); len(got) != 2 || got[0] != key || got[1] != late { + t.Fatalf("unordered DTS cache = %v, want insertion-order prefix before older frame", got) + } + stream.WriteFrame(avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 50, 50, []byte{5})) + if got := stream.GOPCache(); len(got) != 2 { + t.Fatalf("sealed GOP accepted a frame after the exceeding span: %v", got) + } +} + +func TestStreamGOPCacheDTSExtremesDoNotOverflowDuration(t *testing.T) { + cfg := newTestStreamConfig() + cfg.GOPCacheMaxFrames = 0 + cfg.GOPCacheMaxBytes = 0 + cfg.GOPCacheMaxDuration = time.Millisecond + stream := NewStream("live/gop-extreme-dts", cfg, config.LimitsConfig{}, NewEventBus()) + + key := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, -1<<63, -1<<63, []byte{1}) + far := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 1<<63-1, 1<<63-1, []byte{2}) + stream.WriteFrame(key) + stream.WriteFrame(far) + + if got := stream.GOPCache(); len(got) != 1 || got[0] != key { + t.Fatalf("extreme DTS cache = %v, want only keyframe after overflow-safe rejection", got) + } +} + +func TestNewStreamAppliesHardGOPFallbackForUnvalidatedConfig(t *testing.T) { + const wantFallbackFrames = 300 + cfg := newTestStreamConfig() + cfg.GOPCacheMaxFrames = 0 + cfg.GOPCacheMaxBytes = 0 + cfg.GOPCacheMaxDuration = 0 + stream := NewStream("live/gop-defensive-fallback", cfg, config.LimitsConfig{}, NewEventBus()) + if got := stream.Config().GOPCacheMaxFrames; got != wantFallbackFrames { + t.Fatalf("defensive frame bound = %d, want %d", got, wantFallbackFrames) + } + key := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 0, 0, []byte{1}) + stream.WriteFrame(key) + for i := 1; i < wantFallbackFrames+20; i++ { + stream.WriteFrame(avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, int64(i), int64(i), []byte{2})) + } + if got := stream.GOPCacheLen(); got != wantFallbackFrames { + t.Fatalf("defensive fallback cache length = %d, want %d", got, wantFallbackFrames) + } +} + +func TestStreamGOPCacheRuntimeTrimUsesUnorderedDTSSpan(t *testing.T) { + cfg := newTestStreamConfig() + cfg.GOPCacheMaxFrames = 10 + cfg.GOPCacheMaxBytes = 0 + cfg.GOPCacheMaxDuration = 0 + stream := NewStream("live/gop-trim-unordered-dts", cfg, config.LimitsConfig{}, NewEventBus()) + key := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 100, 100, []byte{1}) + late := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 140, 140, []byte{2}) + older := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 0, 0, []byte{3}) + for _, frame := range []*avframe.AVFrame{key, late, older} { + stream.WriteFrame(frame) + } + + tightened := cfg + tightened.GOPCacheMaxDuration = 100 * time.Millisecond + stream.UpdatePolicy(tightened, config.LimitsConfig{}) + if got := stream.GOPCache(); len(got) != 2 || got[0] != key || got[1] != late { + t.Fatalf("unordered DTS trim = %v, want playable prefix before older frame", got) + } +} + func TestStreamRepublishBeforeTimeout(t *testing.T) { bus := NewEventBus() cfg := newTestStreamConfig() diff --git a/core/transcode_envelope_test.go b/core/transcode_envelope_test.go new file mode 100644 index 00000000..3d8f7260 --- /dev/null +++ b/core/transcode_envelope_test.go @@ -0,0 +1,820 @@ +package core + +import ( + "bytes" + "context" + "errors" + "slices" + "sync" + "testing" + "time" + + "github.com/im-pingo/liveforge/config" + "github.com/im-pingo/liveforge/pkg/audiocodec" + "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/im-pingo/liveforge/pkg/util" +) + +// Catches replacing source-cursor identity with output order/DTS or copying an +// AVFrame/payload merely to attach attribution metadata. +func TestTranscodeEnvelopePreservesDirectFrameIdentityAndSourceSpan(t *testing.T) { + stream := newTranscodeTestStream(avframe.CodecG711U) + defer stream.Close() + tm := stream.TranscodeManager() + reader, release, err := tm.GetOrCreateReaderAtFromHistory(avframe.CodecG711A, stream.StartupSnapshot()) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + defer release() + + videoPayload := []byte{0x01, 0x02, 0x03} + video := avframe.NewAVFrame( + avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, + 9000, 9000, videoPayload, + ) + audioPayload := []byte{0x11, 0x22, 0x33} + audio := avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711A, avframe.FrameTypeInterframe, + 9020, 9020, audioPayload, + ) + stream.WriteFrame(video) + stream.WriteFrame(audio) + + endCursor := stream.RingBuffer().WriteCursor() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if !tm.WaitForSourceCursor(avframe.CodecG711A, endCursor, ctx) { + t.Fatal("transcode producer did not consume direct source frames") + } + + tm.mu.Lock() + track := tm.tracks[avframe.CodecG711A] + tm.mu.Unlock() + if track == nil { + t.Fatal("combined transcode track disappeared") + } + outputReader := track.ringBuffer.NewReader() + defer outputReader.Close() + + wants := []struct { + frame *avframe.AVFrame + span audiocodec.SourceSpan + }{ + {frame: video, span: audiocodec.SourceSpan{Begin: 0, End: 1}}, + {frame: audio, span: audiocodec.SourceSpan{Begin: 1, End: 2}}, + } + for i, want := range wants { + output, ok := outputReader.TryRead() + if !ok { + t.Fatalf("internal output ended at record %d", i) + } + if output.frame != want.frame { + t.Fatalf("record %d frame pointer = %p, want %p", i, output.frame, want.frame) + } + if output.sourceSpan != want.span { + t.Fatalf("record %d source span = %+v, want %+v", i, output.sourceSpan, want.span) + } + if len(output.frame.Payload) == 0 || &output.frame.Payload[0] != &want.frame.Payload[0] { + t.Fatalf("record %d payload backing storage was copied", i) + } + } +} + +// Catches filtering with SourceSpan.End > floor: the crossing [3,5) packet +// contains pre-floor samples and must not reach the late reader. +func TestTranscodeSourceFloorUsesSpanBeginAndPreservesExistingReader(t *testing.T) { + tm := &TranscodeManager{bufSize: 16} + track := &TranscodedTrack{ + targetCodec: avframe.CodecAAC, + ringBuffer: util.NewRingBuffer[transcodeOutput](16), + } + existing, stopExisting := tm.newTrackReader(track, true, 0, 0) + defer stopExisting() + + staleHeader := transcodeHeaderOutput(6, []byte{0x06}) + currentHeader := transcodeHeaderOutput(7, []byte{0x07}) + old := transcodeMediaOutput(7, 20, []byte{0x20}, audiocodec.SourceSpan{Begin: 2, End: 3}) + crossing := transcodeMediaOutput(7, 30, []byte{0x30}, audiocodec.SourceSpan{Begin: 3, End: 5}) + accepted := transcodeMediaOutput(7, 40, []byte{0x40}, audiocodec.SourceSpan{Begin: 4, End: 5}) + track.ringBuffer.Write(staleHeader) + track.ringBuffer.Write(currentHeader) + track.ringBuffer.Write(old) + track.ringBuffer.Write(crossing) + + late, stopLate := tm.newTrackReader(track, true, 7, 4) + defer stopLate() + track.ringBuffer.Write(accepted) + + for i, want := range []*avframe.AVFrame{ + staleHeader.frame, currentHeader.frame, old.frame, crossing.frame, accepted.frame, + } { + if got := readTranscodeFrame(t, existing); got != want { + t.Fatalf("existing reader record %d = %p, want %p", i, got, want) + } + } + if got := readTranscodeFrame(t, late); got != currentHeader.frame { + t.Fatalf("late reader first record = %p, want current header %p", got, currentHeader.frame) + } + if got := readTranscodeFrame(t, late); got != accepted.frame { + t.Fatalf("late reader first media = %p, want accepted media %p", got, accepted.frame) + } +} + +// Catches relying on the generated header still being retained in the output +// ring when a late reader accepts its first same-epoch payload. +func TestTranscodeSourceFloorReplaysCurrentHeaderWhenRingHistoryLostIt(t *testing.T) { + tm := &TranscodeManager{bufSize: 4} + track := &TranscodedTrack{ + targetCodec: avframe.CodecAAC, + ringBuffer: util.NewRingBuffer[transcodeOutput](2), + } + header := transcodeHeaderOutput(7, []byte{0x07}) + track.cacheSequenceHeader(header) + track.ringBuffer.Write(header) + track.ringBuffer.Write(transcodeMediaOutput( + 7, 30, []byte{0x30}, audiocodec.SourceSpan{Begin: 3, End: 4}, + )) + accepted := transcodeMediaOutput( + 7, 40, []byte{0x40}, audiocodec.SourceSpan{Begin: 4, End: 5}, + ) + track.ringBuffer.Write(accepted) + + late, stopLate := tm.newTrackReader(track, true, 7, 4) + defer stopLate() + if got := readTranscodeFrame(t, late); got != header.frame { + t.Fatalf("late reader first record = %p, want cached current header %p", got, header.frame) + } + if got := readTranscodeFrame(t, late); got != accepted.frame { + t.Fatalf("late reader first media = %p, want accepted payload %p", got, accepted.frame) + } +} + +// Catches replacing a matching retained header when the producer advances its +// cache before a lagging bridge handles same-epoch media still in output history. +func TestTranscodeSourceFloorReplaysMatchingHeaderAfterCacheAdvances(t *testing.T) { + tm := &TranscodeManager{bufSize: 4} + track := &TranscodedTrack{ + targetCodec: avframe.CodecAAC, + ringBuffer: util.NewRingBuffer[transcodeOutput](2), + } + header7 := transcodeHeaderOutput(7, []byte{0x07}) + track.cacheSequenceHeader(header7) + track.ringBuffer.Write(header7) + track.ringBuffer.Write(transcodeMediaOutput( + 7, 30, []byte{0x30}, audiocodec.SourceSpan{Begin: 3, End: 4}, + )) + payload7 := transcodeMediaOutput( + 7, 40, []byte{0x40}, audiocodec.SourceSpan{Begin: 4, End: 5}, + ) + track.ringBuffer.Write(payload7) + track.cacheSequenceHeader(transcodeHeaderOutput(8, []byte{0x08})) + + late, stopLate := tm.newTrackReader(track, true, 7, 4) + defer stopLate() + if got := readTranscodeFrame(t, late); got != header7.frame { + t.Fatalf("late reader first record epoch = %d, want matching header epoch 7", got.AudioCodecEpoch) + } + if got := readTranscodeFrame(t, late); got != payload7.frame { + t.Fatalf("late reader first media epoch = %d, want retained payload epoch 7", got.AudioCodecEpoch) + } +} + +func transcodeHeaderOutput(epoch uint64, payload []byte) transcodeOutput { + frame := avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeSequenceHeader, + 0, 0, payload, + ) + frame.AudioCodecEpoch = epoch + frame.AudioProvenance = avframe.FrameProvenanceTranscoded + return transcodeOutput{ + frame: frame, kind: transcodeOutputSequenceHeader, audioEpoch: epoch, + } +} + +func transcodeMediaOutput(epoch uint64, dts int64, payload []byte, span audiocodec.SourceSpan) transcodeOutput { + frame := avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, + dts, dts, payload, + ) + frame.AudioCodecEpoch = epoch + frame.AudioProvenance = avframe.FrameProvenanceTranscoded + return transcodeOutput{ + frame: frame, sourceSpan: span, kind: transcodeOutputMedia, audioEpoch: epoch, + } +} + +func readTranscodeFrame(t *testing.T, reader *util.RingReader[*avframe.AVFrame]) *avframe.AVFrame { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + frame, ok := reader.ReadContext(ctx) + if !ok { + t.Fatal("transcode reader closed before expected frame") + } + return frame +} + +type provenanceEncoder struct { + spans []audiocodec.SourceSpan +} + +type stereoProvenanceEncoder struct { + encoded [][]int16 + spans []audiocodec.SourceSpan +} + +type terminalAttributionDecoder struct { + calls int +} + +func (d *terminalAttributionDecoder) SetExtradata([]byte) {} +func (d *terminalAttributionDecoder) Decode([]byte) (*audiocodec.PCMFrame, error) { + d.calls++ + return &audiocodec.PCMFrame{ + Samples: []int16{int16(d.calls)}, SampleRate: 4000, Channels: 1, // #nosec G115 -- decoder call count is a bounded test fixture. + }, nil +} +func (d *terminalAttributionDecoder) SampleRate() int { return 4000 } +func (d *terminalAttributionDecoder) Channels() int { return 1 } +func (d *terminalAttributionDecoder) Close() {} + +type terminalAttributedResampler struct { + calls int + attributedDrainCalls int + legacyCalls int + legacyDrainCalls int +} + +func (r *terminalAttributedResampler) Resample(*audiocodec.PCMFrame) *audiocodec.PCMFrame { + r.legacyCalls++ + return nil +} + +func (r *terminalAttributedResampler) ResampleAttributed(_ *audiocodec.PCMFrame, span audiocodec.SourceSpan) (*audiocodec.AttributedPCMFrame, error) { + r.calls++ + if r.calls == 1 { + return &audiocodec.AttributedPCMFrame{}, nil + } + return &audiocodec.AttributedPCMFrame{ + PCMFrame: audiocodec.PCMFrame{Samples: []int16{1, 2, 3}, SampleRate: 8000, Channels: 1}, + SourceSpan: audiocodec.SourceSpan{Begin: 20, End: span.End}, + }, nil +} + +func (r *terminalAttributedResampler) Drain() *audiocodec.PCMFrame { + r.legacyDrainCalls++ + return nil +} + +func (r *terminalAttributedResampler) DrainAttributed() (*audiocodec.AttributedPCMFrame, error) { + r.attributedDrainCalls++ + return &audiocodec.AttributedPCMFrame{ + PCMFrame: audiocodec.PCMFrame{Samples: []int16{4, 5}, SampleRate: 8000, Channels: 1}, + SourceSpan: audiocodec.SourceSpan{Begin: 21, End: 22}, + }, nil +} + +func (r *terminalAttributedResampler) Close() {} + +type terminalAttributedEncoder struct { + encoded [][]int16 + spans []audiocodec.SourceSpan + attributedDrainCalls int + legacyCalls int + legacyDrainCalls int +} + +func (e *terminalAttributedEncoder) Encode(*audiocodec.PCMFrame) ([]byte, error) { + e.legacyCalls++ + return nil, nil +} + +func (e *terminalAttributedEncoder) EncodeAttributed(pcm *audiocodec.PCMFrame, span audiocodec.SourceSpan) ([]audiocodec.AttributedPacket, error) { + e.encoded = append(e.encoded, append([]int16(nil), pcm.Samples...)) + e.spans = append(e.spans, span) + return []audiocodec.AttributedPacket{{Payload: []byte{byte(0xa0 + len(e.encoded))}, SourceSpan: span}}, nil // #nosec G115 -- encoded fixture count stays below one byte. +} + +func (e *terminalAttributedEncoder) Drain() ([][]byte, error) { + e.legacyDrainCalls++ + return nil, nil +} + +func (e *terminalAttributedEncoder) DrainAttributed() ([]audiocodec.AttributedPacket, error) { + e.attributedDrainCalls++ + return []audiocodec.AttributedPacket{ + {Payload: []byte{0xd1}, SourceSpan: audiocodec.SourceSpan{Begin: 20, End: 22}}, + {Payload: []byte{0xd2}, SourceSpan: audiocodec.SourceSpan{Begin: 21, End: 22}}, + }, nil +} + +func (e *terminalAttributedEncoder) SampleRate() int { return 8000 } +func (e *terminalAttributedEncoder) Channels() int { return 1 } +func (e *terminalAttributedEncoder) FrameSize() int { return 4 } +func (e *terminalAttributedEncoder) Close() {} + +func (e *provenanceEncoder) Encode(*audiocodec.PCMFrame) ([]byte, error) { + return []byte{0xee}, nil +} + +func (e *provenanceEncoder) EncodeAttributed(_ *audiocodec.PCMFrame, span audiocodec.SourceSpan) ([]audiocodec.AttributedPacket, error) { + e.spans = append(e.spans, span) + return []audiocodec.AttributedPacket{{Payload: []byte{byte(len(e.spans))}, SourceSpan: span}}, nil // #nosec G115 -- provenance fixture count stays below one byte. +} + +func (e *provenanceEncoder) SampleRate() int { return 8000 } +func (e *provenanceEncoder) Channels() int { return 1 } +func (e *provenanceEncoder) FrameSize() int { return 4 } +func (e *provenanceEncoder) Close() {} + +func (e *stereoProvenanceEncoder) Encode(*audiocodec.PCMFrame) ([]byte, error) { + return nil, nil +} + +func (e *stereoProvenanceEncoder) EncodeAttributed(pcm *audiocodec.PCMFrame, span audiocodec.SourceSpan) ([]audiocodec.AttributedPacket, error) { + e.encoded = append(e.encoded, append([]int16(nil), pcm.Samples...)) + e.spans = append(e.spans, span) + return []audiocodec.AttributedPacket{{Payload: []byte{byte(len(e.spans))}, SourceSpan: span}}, nil // #nosec G115 -- provenance fixture count stays below one byte. +} + +func (*stereoProvenanceEncoder) SampleRate() int { return 8000 } +func (*stereoProvenanceEncoder) Channels() int { return 2 } +func (*stereoProvenanceEncoder) FrameSize() int { return 4 } +func (*stereoProvenanceEncoder) Close() {} + +// Catches attributing fixed encoder frames to only the newest PCM chunk or +// retaining an already-consumed old span in every later output. +func TestTranscodePCMProvenanceConsumesOnlyFrameContributors(t *testing.T) { + track := &TranscodedTrack{ + targetCodec: avframe.CodecAAC, + ringBuffer: util.NewRingBuffer[transcodeOutput](8), + } + encoder := &provenanceEncoder{} + pipeline := &audioTranscodePipeline{ + track: track, sourceEpoch: 3, encoder: encoder, + } + pipeline.ts.Init(0, encoder.SampleRate()) + pipeline.tsInited = true + pipeline.encodePCM( + &audiocodec.PCMFrame{Samples: []int16{1, 2, 3}, SampleRate: 8000, Channels: 1}, + audiocodec.SourceSpan{Begin: 10, End: 11}, + ) + pipeline.encodePCM( + &audiocodec.PCMFrame{Samples: []int16{4, 5, 6}, SampleRate: 8000, Channels: 1}, + audiocodec.SourceSpan{Begin: 11, End: 12}, + ) + pipeline.encodePCM( + &audiocodec.PCMFrame{Samples: []int16{7, 8}, SampleRate: 8000, Channels: 1}, + audiocodec.SourceSpan{Begin: 12, End: 13}, + ) + + wantSpans := []audiocodec.SourceSpan{ + {Begin: 10, End: 12}, + {Begin: 11, End: 13}, + } + if len(encoder.spans) != len(wantSpans) { + t.Fatalf("attributed encoder calls = %d, want %d", len(encoder.spans), len(wantSpans)) + } + outputReader := track.ringBuffer.NewReader() + defer outputReader.Close() + for i, want := range wantSpans { + if encoder.spans[i] != want { + t.Fatalf("encoder span %d = %+v, want %+v", i, encoder.spans[i], want) + } + output, ok := outputReader.TryRead() + if !ok { + t.Fatalf("encoded output ended at packet %d", i) + } + if output.sourceSpan != want { + t.Fatalf("output span %d = %+v, want %+v", i, output.sourceSpan, want) + } + } +} + +// Catches counting interleaved stereo samples as samples per channel. Each +// encoder frame crosses contributor boundaries and leaves a partial segment +// that must age into exactly one later frame. +func TestTranscodeStereoPCMProvenanceUsesSamplesPerChannel(t *testing.T) { + track := &TranscodedTrack{ + targetCodec: avframe.CodecAAC, + ringBuffer: util.NewRingBuffer[transcodeOutput](8), + } + encoder := &stereoProvenanceEncoder{} + pipeline := &audioTranscodePipeline{track: track, sourceEpoch: 4, encoder: encoder} + pipeline.ts.Init(0, encoder.SampleRate()) + pipeline.tsInited = true + chunks := []struct { + samples []int16 + span audiocodec.SourceSpan + }{ + {samples: []int16{1, 101, 2, 102, 3, 103}, span: audiocodec.SourceSpan{Begin: 30, End: 31}}, + {samples: []int16{4, 104, 5, 105, 6, 106}, span: audiocodec.SourceSpan{Begin: 31, End: 32}}, + {samples: []int16{7, 107, 8, 108, 9, 109, 10, 110}, span: audiocodec.SourceSpan{Begin: 32, End: 33}}, + {samples: []int16{11, 111, 12, 112}, span: audiocodec.SourceSpan{Begin: 33, End: 34}}, + } + for _, chunk := range chunks { + pipeline.encodePCM(&audiocodec.PCMFrame{ + Samples: chunk.samples, SampleRate: 8000, Channels: 2, + }, chunk.span) + } + + wantPCM := [][]int16{ + {1, 101, 2, 102, 3, 103, 4, 104}, + {5, 105, 6, 106, 7, 107, 8, 108}, + {9, 109, 10, 110, 11, 111, 12, 112}, + } + wantSpans := []audiocodec.SourceSpan{ + {Begin: 30, End: 32}, + {Begin: 31, End: 33}, + {Begin: 32, End: 34}, + } + if len(encoder.spans) != len(wantSpans) { + t.Fatalf("stereo attributed encoder calls = %d, want %d", len(encoder.spans), len(wantSpans)) + } + outputReader := track.ringBuffer.NewReader() + defer outputReader.Close() + for i, want := range wantSpans { + if !slices.Equal(encoder.encoded[i], wantPCM[i]) { + t.Fatalf("stereo PCM %d = %v, want %v", i, encoder.encoded[i], wantPCM[i]) + } + if encoder.spans[i] != want { + t.Fatalf("stereo encoder span %d = %+v, want %+v", i, encoder.spans[i], want) + } + output, ok := outputReader.TryRead() + if !ok { + t.Fatalf("stereo output ended at packet %d", i) + } + if output.sourceSpan != want { + t.Fatalf("stereo output span %d = %+v, want %+v", i, output.sourceSpan, want) + } + } +} + +// Catches dropping zero-output resampler provenance or assigning zero/newest- +// only attribution to padded PCM and later encoder-drain packets. +func TestTranscodeAttributedTerminalDrainPreservesSpansAndContentOnce(t *testing.T) { + track := &TranscodedTrack{ + targetCodec: avframe.CodecAAC, + ringBuffer: util.NewRingBuffer[transcodeOutput](8), + } + decoder := &terminalAttributionDecoder{} + resampler := &terminalAttributedResampler{} + encoder := &terminalAttributedEncoder{} + pipeline := &audioTranscodePipeline{ + track: track, sourceEpoch: 9, decoder: decoder, encoder: encoder, + resampler: resampler, resampled: true, + } + for i, span := range []audiocodec.SourceSpan{{Begin: 20, End: 21}, {Begin: 21, End: 22}} { + pipeline.encode(avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711U, avframe.FrameTypeInterframe, + int64(1000+i*20), int64(1000+i*20), []byte{byte(i)}, + ), span) + } + pipeline.finalize() + pipeline.finalize() + + if resampler.legacyCalls != 0 || resampler.legacyDrainCalls != 0 { + t.Fatalf("legacy resampler calls = %d/%d, want 0/0", resampler.legacyCalls, resampler.legacyDrainCalls) + } + if resampler.calls != 2 || resampler.attributedDrainCalls != 1 { + t.Fatalf("attributed resampler calls/drain = %d/%d, want 2/1", resampler.calls, resampler.attributedDrainCalls) + } + if encoder.legacyCalls != 0 || encoder.legacyDrainCalls != 0 { + t.Fatalf("legacy encoder calls = %d/%d, want 0/0", encoder.legacyCalls, encoder.legacyDrainCalls) + } + if encoder.attributedDrainCalls != 1 { + t.Fatalf("attributed encoder drain calls = %d, want 1", encoder.attributedDrainCalls) + } + wantPCM := [][]int16{{1, 2, 3, 4}, {5, 0, 0, 0}} + wantEncodeSpans := []audiocodec.SourceSpan{{Begin: 20, End: 22}, {Begin: 21, End: 22}} + for i := range wantPCM { + if !slices.Equal(encoder.encoded[i], wantPCM[i]) { + t.Fatalf("encoded PCM %d = %v, want %v", i, encoder.encoded[i], wantPCM[i]) + } + if encoder.spans[i] != wantEncodeSpans[i] { + t.Fatalf("encoded span %d = %+v, want %+v", i, encoder.spans[i], wantEncodeSpans[i]) + } + } + + wantPayloads := [][]byte{{0xa1}, {0xa2}, {0xd1}, {0xd2}} + wantOutputSpans := []audiocodec.SourceSpan{ + {Begin: 20, End: 22}, {Begin: 21, End: 22}, + {Begin: 20, End: 22}, {Begin: 21, End: 22}, + } + outputReader := track.ringBuffer.NewReader() + defer outputReader.Close() + for i := range wantPayloads { + output, ok := outputReader.TryRead() + if !ok { + t.Fatalf("terminal output ended at packet %d", i) + } + if !bytes.Equal(output.frame.Payload, wantPayloads[i]) || output.sourceSpan != wantOutputSpans[i] { + t.Fatalf("packet %d payload/span = %x/%+v, want %x/%+v", i, output.frame.Payload, output.sourceSpan, wantPayloads[i], wantOutputSpans[i]) + } + } + if _, ok := outputReader.TryRead(); ok { + t.Fatal("terminal media was published more than once") + } +} + +// Catches calling attributed drain before any PCM submission, which has no +// valid source span and makes the reviewed encoder fail with ErrInvalidSourceSpan. +func TestTranscodeAttributedEncoderWithoutSubmissionDoesNotDrain(t *testing.T) { + encoder := &terminalAttributedEncoder{} + pipeline := &audioTranscodePipeline{encoder: encoder} + pipeline.drainEncoder(false) + if encoder.attributedDrainCalls != 0 || encoder.legacyDrainCalls != 0 { + t.Fatalf("drain calls without submission = %d/%d, want 0/0", encoder.attributedDrainCalls, encoder.legacyDrainCalls) + } +} + +type overwritePendingDecoder struct{} + +func (*overwritePendingDecoder) SetExtradata([]byte) {} +func (*overwritePendingDecoder) Decode([]byte) (*audiocodec.PCMFrame, error) { + return &audiocodec.PCMFrame{Samples: []int16{1, 2, 3}, SampleRate: 8000, Channels: 1}, nil +} +func (*overwritePendingDecoder) SampleRate() int { return 8000 } +func (*overwritePendingDecoder) Channels() int { return 1 } +func (*overwritePendingDecoder) Close() {} + +type overwriteDelayedEncoder struct { + entered chan struct{} + release chan struct{} + enteredOnce sync.Once + encodeCalls int + drainCalls int + legacyCalls int + delayedSource audiocodec.SourceSpan +} + +func (e *overwriteDelayedEncoder) Encode(*audiocodec.PCMFrame) ([]byte, error) { + e.legacyCalls++ + return nil, nil +} + +func (e *overwriteDelayedEncoder) EncodeAttributed(_ *audiocodec.PCMFrame, span audiocodec.SourceSpan) ([]audiocodec.AttributedPacket, error) { + e.encodeCalls++ + if e.encodeCalls == 1 { + e.delayedSource = span + e.enteredOnce.Do(func() { close(e.entered) }) + <-e.release + return nil, nil + } + return []audiocodec.AttributedPacket{{Payload: []byte{0xf1}, SourceSpan: span}}, nil +} + +func (e *overwriteDelayedEncoder) Drain() ([][]byte, error) { + e.legacyCalls++ + return nil, nil +} + +func (e *overwriteDelayedEncoder) DrainAttributed() ([]audiocodec.AttributedPacket, error) { + e.drainCalls++ + return []audiocodec.AttributedPacket{{Payload: []byte{0xd1}, SourceSpan: e.delayedSource}}, nil +} + +func (*overwriteDelayedEncoder) SampleRate() int { return 8000 } +func (*overwriteDelayedEncoder) Channels() int { return 1 } +func (*overwriteDelayedEncoder) FrameSize() int { return 4 } +func (*overwriteDelayedEncoder) Close() {} + +// Catches ignoring RingReadResult.Overwritten, finalizing pending PCM or +// delayed encoder output as a clean tail, or bleeding termination into a +// replacement generation. +func TestTranscodeProducerSourceOverwriteTerminatesWithExactCause(t *testing.T) { + const ( + overwriteSourceCodec avframe.CodecType = 240 + overwriteTargetCodec avframe.CodecType = 241 + ) + stream := NewStream( + "transcode-overwrite", + config.StreamConfig{RingBufferSize: 2}, + config.LimitsConfig{}, + NewEventBus(), + ) + if err := stream.SetPublisher(&testPublisher{ + id: "overwrite-source", info: &avframe.MediaInfo{AudioCodec: overwriteSourceCodec}, + }); err != nil { + t.Fatal(err) + } + defer stream.Close() + snapshot := stream.StartupSnapshot() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + track := &TranscodedTrack{ + targetCodec: overwriteTargetCodec, + ringBuffer: util.NewRingBuffer[transcodeOutput](8), + sourceStart: 0, + sourceAdvance: make(chan struct{}), + generationDone: snapshot.GenerationDone, + generationBoundary: snapshot.generationBoundary, + } + track.sourceCursor.Store(0) + registry := audiocodec.Global() + encoder := &overwriteDelayedEncoder{entered: make(chan struct{}), release: make(chan struct{})} + registry.RegisterDecoder(overwriteSourceCodec, func() audiocodec.Decoder { return &overwritePendingDecoder{} }) + registry.RegisterEncoder(overwriteTargetCodec, func() audiocodec.Encoder { return encoder }) + tm := NewTranscodeManager(stream, registry, 8) + done := make(chan struct{}) + var releaseEncoder sync.Once + unblockEncoder := func() { releaseEncoder.Do(func() { close(encoder.release) }) } + defer unblockEncoder() + + go func() { + defer close(done) + tm.transcodeLoop(ctx, track, 0, 0, false) + }() + outputReader := track.ringBuffer.NewReader() + defer outputReader.Close() + for i := 0; i < 2; i++ { + stream.WriteFrame(avframe.NewAVFrame( + avframe.MediaTypeAudio, overwriteSourceCodec, avframe.FrameTypeInterframe, + int64(i*20), int64(i*20), []byte{byte(i)}, + )) + } + select { + case <-encoder.entered: + case <-time.After(time.Second): + t.Fatal("producer did not reach the deterministic delayed-encoder barrier") + } + for i := 2; i < 6; i++ { + stream.WriteFrame(avframe.NewAVFrame( + avframe.MediaTypeAudio, overwriteSourceCodec, avframe.FrameTypeInterframe, + int64(i*20), int64(i*20), []byte{byte(i)}, + )) + } + stream.RemovePublisher() + unblockEncoder() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("overwritten transcode producer did not terminate") + } + var overwrite *transcodeSourceOverwriteError + cause := track.terminationCause() + if !errors.As(cause, &overwrite) { + t.Fatalf("termination cause = %T %v, want typed source overwrite", cause, cause) + } + if overwrite.Overwritten != 2 { + t.Fatalf("overwritten count = %d, want 2", overwrite.Overwritten) + } + if errors.Is(cause, errTranscodeGenerationComplete) { + t.Fatal("source overwrite was reported as clean generation completion") + } + if output, ok := outputReader.TryRead(); ok { + t.Fatalf("overwrite published padded/drained tail payload %x with span %+v", output.frame.Payload, output.sourceSpan) + } + if encoder.encodeCalls != 1 || encoder.drainCalls != 1 || encoder.legacyCalls != 0 { + t.Fatalf("overwrite encoder calls encode/drain/legacy = %d/%d/%d, want 1/1/0", encoder.encodeCalls, encoder.drainCalls, encoder.legacyCalls) + } + + if err := stream.SetPublisher(&testPublisher{ + id: "overwrite-replacement", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}, + }); err != nil { + t.Fatalf("install replacement publisher: %v", err) + } + replacementSnapshot := stream.StartupSnapshot() + replacement := &TranscodedTrack{ + targetCodec: overwriteTargetCodec, + ringBuffer: util.NewRingBuffer[transcodeOutput](8), + sourceStart: replacementSnapshot.SourceCursor, + sourceAdvance: make(chan struct{}), + generationDone: replacementSnapshot.GenerationDone, + generationBoundary: replacementSnapshot.generationBoundary, + } + replacement.sourceCursor.Store(replacementSnapshot.SourceCursor) + replacementDone := make(chan struct{}) + go func() { + defer close(replacementDone) + tm.transcodeLoop(ctx, replacement, replacementSnapshot.SourceCursor, 0, false) + }() + replacementReader := replacement.ringBuffer.NewReader() + defer replacementReader.Close() + replacementFrame := avframe.NewAVFrame( + avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, + 1000, 1000, []byte{0xaa}, + ) + stream.WriteFrame(replacementFrame) + if output := readTranscodeOutput(t, replacementReader); output.frame != replacementFrame { + t.Fatalf("replacement output frame = %p, want %p", output.frame, replacementFrame) + } + stream.RemovePublisher() + select { + case <-replacementDone: + case <-time.After(time.Second): + t.Fatal("replacement transcode producer did not terminate") + } + if cause := replacement.terminationCause(); !errors.Is(cause, errTranscodeGenerationComplete) { + t.Fatalf("replacement termination cause = %v, want clean generation completion", cause) + } +} + +func readTranscodeOutput(t *testing.T, reader *util.RingReader[transcodeOutput]) transcodeOutput { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + output, ok := reader.ReadContext(ctx) + if !ok { + t.Fatal("transcode output closed before expected record") + } + return output +} + +// Catches forwarding the retained envelope returned with an internal bridge +// overwrite or closing the shared track used by an unrelated peer reader. +func TestTranscodeBridgeOverwriteClosesOnlyThatReader(t *testing.T) { + tm := &TranscodeManager{bufSize: 4} + track := &TranscodedTrack{ + targetCodec: avframe.CodecG711A, + ringBuffer: util.NewRingBuffer[transcodeOutput](2), + } + outputs := []transcodeOutput{ + transcodeMediaOutput(1, 0, []byte{0}, audiocodec.SourceSpan{Begin: 0, End: 1}), + transcodeMediaOutput(1, 1, []byte{1}, audiocodec.SourceSpan{Begin: 1, End: 2}), + transcodeMediaOutput(1, 2, []byte{2}, audiocodec.SourceSpan{Begin: 2, End: 3}), + } + for i := range outputs { + outputs[i].frame.Codec = avframe.CodecG711A + track.ringBuffer.Write(outputs[i]) + } + + lappedShared := track.ringBuffer.NewReaderAt(0) + lapped, stopLapped := tm.bridgeTrackReader(track, lappedShared, 0, 0) + defer stopLapped() + peerShared := track.ringBuffer.NewReader() + peer, stopPeer := tm.bridgeTrackReader(track, peerShared, 0, 0) + defer stopPeer() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + result := lapped.ReadResultContext(ctx) + if result.OK { + t.Fatalf("lapped bridge forwarded retained post-gap frame %p", result.Value) + } + if ctx.Err() != nil { + t.Fatal("lapped bridge did not close after overwrite") + } + if got := readTranscodeFrame(t, peer); got != outputs[1].frame { + t.Fatalf("peer first retained frame = %p, want %p", got, outputs[1].frame) + } + if track.ringBuffer.IsClosed() { + t.Fatal("one bridge overwrite closed the shared track") + } +} + +// Catches collapsing last-subscriber release and manager Reset into one +// indistinguishable cancellation cause. +func TestTranscodeTrackCancellationDistinguishesReleaseAndReset(t *testing.T) { + releaseStream := newTranscodeTestStream(avframe.CodecG711U) + releaseReader, release, err := releaseStream.TranscodeManager().GetOrCreateReader(avframe.CodecG711A) + if err != nil { + t.Fatal(err) + } + releaseStream.TranscodeManager().mu.Lock() + releasedTrack := releaseStream.TranscodeManager().tracks[avframe.CodecG711A] + releaseStream.TranscodeManager().mu.Unlock() + release() + release() + waitForTranscodeTrackClose(t, releasedTrack) + releaseReader.Close() + if cause := releasedTrack.terminationCause(); !errors.Is(cause, errTranscodeSubscriberReleased) { + t.Fatalf("last-subscriber cause = %v, want subscriber release", cause) + } + releaseStream.Close() + + resetStream := newTranscodeTestStream(avframe.CodecG711U) + resetReader, resetRelease, err := resetStream.TranscodeManager().GetOrCreateReader(avframe.CodecG711A) + if err != nil { + t.Fatal(err) + } + defer resetRelease() + resetStream.TranscodeManager().mu.Lock() + resetTrack := resetStream.TranscodeManager().tracks[avframe.CodecG711A] + resetStream.TranscodeManager().mu.Unlock() + resetStream.TranscodeManager().Reset() + waitForTranscodeTrackClose(t, resetTrack) + resetReader.Close() + if cause := resetTrack.terminationCause(); !errors.Is(cause, errTranscodeManagerReset) { + t.Fatalf("manager-reset cause = %v, want manager reset", cause) + } + resetStream.Close() +} + +func waitForTranscodeTrackClose(t *testing.T, track *TranscodedTrack) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + reader := track.ringBuffer.NewReaderAt(track.ringBuffer.WriteCursor()) + defer reader.Close() + for { + if _, ok := reader.ReadContext(ctx); !ok { + break + } + } + if ctx.Err() != nil { + t.Fatal("transcode track did not close after termination") + } +} diff --git a/core/transcode_manager.go b/core/transcode_manager.go index fa23aae2..896a210c 100644 --- a/core/transcode_manager.go +++ b/core/transcode_manager.go @@ -2,6 +2,7 @@ package core import ( "context" + "errors" "fmt" "log/slog" "sync" @@ -12,16 +13,102 @@ import ( "github.com/im-pingo/liveforge/pkg/util" ) -// TranscodedTrack holds a ring buffer for a specific target codec. +// TranscodedTrack holds source-attributed output for a specific target codec. type TranscodedTrack struct { - targetCodec avframe.CodecType - ringBuffer *util.RingBuffer[*avframe.AVFrame] - sourceStart int64 - sourceCursor atomic.Int64 - sourceMu sync.Mutex - sourceAdvance chan struct{} - subCount int - cancel context.CancelFunc + targetCodec avframe.CodecType + ringBuffer *util.RingBuffer[transcodeOutput] + sourceStart int64 + sourceCursor atomic.Int64 + sourceMu sync.Mutex + sourceAdvance chan struct{} + generationDone <-chan struct{} + generationBoundary *streamGenerationBoundary + headerMu sync.RWMutex + sequenceHeaders []transcodeOutput + terminationMu sync.Mutex + termination error + subCount int + cancel context.CancelCauseFunc +} + +const transcodeSequenceHeaderCacheLimit = 8 + +var ( + errTranscodeGenerationComplete = errors.New("transcode generation complete") + errTranscodeSubscriberReleased = errors.New("transcode subscriber released") + errTranscodeManagerReset = errors.New("transcode manager reset") + errTranscodeSourceClosed = errors.New("transcode source closed") +) + +type transcodeSourceOverwriteError struct { + Overwritten int64 +} + +func (e *transcodeSourceOverwriteError) Error() string { + return fmt.Sprintf("transcode source overwritten by %d frames", e.Overwritten) +} + +func (track *TranscodedTrack) setTerminationCause(cause error) { + if cause == nil { + return + } + track.terminationMu.Lock() + defer track.terminationMu.Unlock() + if track.termination == nil { + track.termination = cause + } +} + +func (track *TranscodedTrack) terminationCause() error { + track.terminationMu.Lock() + defer track.terminationMu.Unlock() + return track.termination +} + +func (track *TranscodedTrack) cacheSequenceHeader(output transcodeOutput) { + if output.frame == nil || output.kind != transcodeOutputSequenceHeader { + return + } + track.headerMu.Lock() + defer track.headerMu.Unlock() + for i := len(track.sequenceHeaders) - 1; i >= 0; i-- { + if track.sequenceHeaders[i].audioEpoch == output.audioEpoch { + track.sequenceHeaders[i] = output + return + } + } + track.sequenceHeaders = append(track.sequenceHeaders, output) + if len(track.sequenceHeaders) > transcodeSequenceHeaderCacheLimit { + copy(track.sequenceHeaders, track.sequenceHeaders[len(track.sequenceHeaders)-transcodeSequenceHeaderCacheLimit:]) + track.sequenceHeaders = track.sequenceHeaders[:transcodeSequenceHeaderCacheLimit] + } +} + +func (track *TranscodedTrack) sequenceHeaderForEpoch(epoch uint64) (transcodeOutput, bool) { + track.headerMu.RLock() + defer track.headerMu.RUnlock() + for i := len(track.sequenceHeaders) - 1; i >= 0; i-- { + if output := track.sequenceHeaders[i]; output.audioEpoch == epoch { + return output, output.frame != nil + } + } + return transcodeOutput{}, false +} + +type transcodeOutputKind uint8 + +const ( + transcodeOutputMedia transcodeOutputKind = iota + transcodeOutputSequenceHeader +) + +// transcodeOutput keeps source identity beside a frame without copying its +// payload. Configuration records intentionally carry no media source span. +type transcodeOutput struct { + frame *avframe.AVFrame + sourceSpan audiocodec.SourceSpan + kind transcodeOutputKind + audioEpoch uint64 } // TranscodeManager creates and manages on-demand audio transcoding goroutines. @@ -47,6 +134,12 @@ func NewTranscodeManager(stream *Stream, registry *audiocodec.Registry, bufSize } } +// CanTranscode reports whether this manager's configured registry can convert +// one audio codec to another in the current build. +func (tm *TranscodeManager) CanTranscode(from, to avframe.CodecType) bool { + return tm != nil && tm.registry != nil && tm.registry.CanTranscode(from, to) +} + // GetOrCreateReader returns a reader for the given target codec. // If the publisher's codec matches, it returns the original ring buffer reader (zero overhead). // Otherwise it creates or reuses a shared TranscodedTrack. @@ -64,24 +157,25 @@ func (tm *TranscodeManager) GetOrCreateReaderAt(targetCodec avframe.CodecType, s } // GetOrCreateReaderAtFromHistory returns a combined audio/video transcode -// reader that includes retained source video and target audio no older than -// snapshot's audio codec epoch. HLS, LL-HLS, and DASH use this compatibility -// path to transform the captured GOP without replaying stale audio epochs. +// reader that includes retained output whose source span begins at or after the +// snapshot source cursor and whose audio is no older than the snapshot codec +// epoch. HLS, LL-HLS, and DASH use this compatibility path. func (tm *TranscodeManager) GetOrCreateReaderAtFromHistory(targetCodec avframe.CodecType, snapshot StreamStartupSnapshot) (*util.RingReader[*avframe.AVFrame], func(), error) { return tm.getOrCreateReaderAt(targetCodec, snapshot.SourceCursor, snapshot.audioCodecEpoch, false, true, false, &snapshot) } // GetOrCreateAudioReaderAt returns a target-codec audio-only reader sourced // from snapshot. RTMP and WHEP use it when direct video has a separate cursor. -// The reader cannot emit audio older than snapshot's current codec epoch. +// The reader cannot emit media attributed before snapshot's source cursor or +// audio older than snapshot's current codec epoch. func (tm *TranscodeManager) GetOrCreateAudioReaderAt(targetCodec avframe.CodecType, snapshot StreamStartupSnapshot) (*util.RingReader[*avframe.AVFrame], func(), error) { return tm.getOrCreateReaderAt(targetCodec, snapshot.SourceCursor, snapshot.audioCodecEpoch, true, false, true, &snapshot) } // GetOrCreateAudioReaderAtFromHistory returns an audio-only target-codec -// reader at retained output from snapshot's current audio epoch. Snapshot -// muxers use it to transform cached audio without pulling direct video behind -// the snapshot's live cursor. +// reader at retained output from snapshot's source cursor and current audio +// epoch. Snapshot muxers use it to transform cached audio without pulling +// direct video behind the snapshot's live cursor. func (tm *TranscodeManager) GetOrCreateAudioReaderAtFromHistory(targetCodec avframe.CodecType, snapshot StreamStartupSnapshot) (*util.RingReader[*avframe.AVFrame], func(), error) { return tm.getOrCreateReaderAt(targetCodec, snapshot.SourceCursor, snapshot.audioCodecEpoch, true, true, true, &snapshot) } @@ -100,18 +194,33 @@ func (tm *TranscodeManager) getOrCreateReaderAt( // snapshot generation check and map mutation one atomic acquisition. tm.stream.mu.RLock() defer tm.stream.mu.RUnlock() - if tm.stream.state != StreamStatePublishing || isNilPublisher(tm.stream.publisher) { + activeGeneration := tm.stream.state == StreamStatePublishing && !isNilPublisher(tm.stream.publisher) + if snapshot == nil && !activeGeneration { return nil, func() {}, fmt.Errorf("no publisher on stream") } - if snapshot != nil && tm.stream.publisherGeneration != snapshot.Generation { + if snapshot != nil && (tm.stream.instanceID != snapshot.StreamInstanceID || + tm.stream.publisherGeneration != snapshot.Generation || + tm.stream.generationBoundary != snapshot.generationBoundary) { return nil, func() {}, fmt.Errorf( "stale stream startup snapshot generation %d (active %d)", snapshot.Generation, tm.stream.publisherGeneration, ) } + if snapshot != nil && !activeGeneration { + if _, ended := snapshot.GenerationEndCursor(); !ended { + return nil, func() {}, fmt.Errorf("no publisher on stream") + } + } sourceCodec := tm.stream.mediaInfo.AudioCodec + generationDone := (<-chan struct{})(tm.stream.generationDone) + generationBoundary := tm.stream.generationBoundary + if snapshot != nil { + sourceCodec = snapshot.MediaInfo.AudioCodec + generationDone = snapshot.GenerationDone + generationBoundary = snapshot.generationBoundary + } // Zero-overhead path: target matches source, no transcoding needed. if targetCodec == sourceCodec && !forceTrack { @@ -129,7 +238,11 @@ func (tm *TranscodeManager) getOrCreateReaderAt( if track, ok := tracks[targetCodec]; ok { track.subCount++ - reader, stopReader := tm.newTrackReader(track, fromHistory, audioEpochFloor) + sourceFloor := int64(0) + if snapshot != nil { + sourceFloor = snapshot.SourceCursor + } + reader, stopReader := tm.newTrackReader(track, fromHistory, audioEpochFloor, sourceFloor) var once sync.Once release := func() { once.Do(func() { @@ -140,21 +253,27 @@ func (tm *TranscodeManager) getOrCreateReaderAt( return reader, release, nil } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancelCause(context.Background()) track := &TranscodedTrack{ - targetCodec: targetCodec, - ringBuffer: util.NewRingBuffer[*avframe.AVFrame](tm.bufSize), - sourceStart: sourceStart, - subCount: 1, - cancel: cancel, - sourceAdvance: make(chan struct{}), + targetCodec: targetCodec, + ringBuffer: util.NewRingBuffer[transcodeOutput](tm.bufSize), + sourceStart: sourceStart, + subCount: 1, + cancel: cancel, + sourceAdvance: make(chan struct{}), + generationDone: generationDone, + generationBoundary: generationBoundary, } track.sourceCursor.Store(sourceStart) tracks[targetCodec] = track // Attach the first reader before starting the producer so a non-history // subscriber cannot race and miss the generated target sequence header. - reader, stopReader := tm.newTrackReader(track, fromHistory, audioEpochFloor) + sourceFloor := int64(0) + if snapshot != nil { + sourceFloor = snapshot.SourceCursor + } + reader, stopReader := tm.newTrackReader(track, fromHistory, audioEpochFloor, sourceFloor) go tm.transcodeLoop(ctx, track, sourceStart, audioEpochFloor, audioOnly) var once sync.Once release := func() { @@ -213,18 +332,23 @@ func (tm *TranscodeManager) newTrackReader( track *TranscodedTrack, fromHistory bool, audioEpochFloor uint64, + sourceFloor int64, ) (*util.RingReader[*avframe.AVFrame], func()) { sharedReader := track.ringBuffer.NewReaderAt(track.ringBuffer.WriteCursor()) if fromHistory { sharedReader = track.ringBuffer.NewReader() } - if audioEpochFloor == 0 { - return sharedReader, func() { sharedReader.Close() } - } + return tm.bridgeTrackReader(track, sharedReader, audioEpochFloor, sourceFloor) +} - // A reader-local ring preserves shared producer ownership while enforcing - // the snapshot's audio epoch floor. Source video remains unfiltered for the - // legacy combined-track consumers. +func (tm *TranscodeManager) bridgeTrackReader( + track *TranscodedTrack, + sharedReader *util.RingReader[transcodeOutput], + audioEpochFloor uint64, + sourceFloor int64, +) (*util.RingReader[*avframe.AVFrame], func()) { + // A reader-local ring preserves the public RingReader[*AVFrame] contract + // while the shared producer retains source attribution internally. filtered := util.NewRingBuffer[*avframe.AVFrame](tm.bufSize) reader := filtered.NewReader() ctx, cancel := context.WithCancel(context.Background()) @@ -233,14 +357,46 @@ func (tm *TranscodeManager) newTrackReader( defer close(done) defer filtered.Close() defer sharedReader.Close() + var forwardedHeaderEpoch uint64 for { - frame, ok := sharedReader.ReadContext(ctx) - if !ok { + result := sharedReader.ReadResultContext(ctx) + if !result.OK || result.Overwritten > 0 { + return + } + output := result.Value + if output.kind == transcodeOutputSequenceHeader { + if output.audioEpoch < audioEpochFloor { + continue + } + filtered.Write(output.frame) + forwardedHeaderEpoch = output.audioEpoch + continue + } + if !output.sourceSpan.Valid() { return } - if frame != nil && frame.MediaType.IsAudio() && frame.AudioCodecEpoch < audioEpochFloor { + if output.sourceSpan.Begin < sourceFloor { continue } + frame := output.frame + if frame == nil { + return + } + if frame.MediaType.IsAudio() && output.audioEpoch < audioEpochFloor { + continue + } + if frame.MediaType.IsAudio() { + if frame.FrameType == avframe.FrameTypeSequenceHeader { + forwardedHeaderEpoch = output.audioEpoch + } else if track.targetCodec == avframe.CodecAAC && forwardedHeaderEpoch != output.audioEpoch { + header, ok := track.sequenceHeaderForEpoch(output.audioEpoch) + if !ok || header.audioEpoch < audioEpochFloor { + continue + } + filtered.Write(header.frame) + forwardedHeaderEpoch = header.audioEpoch + } + } filtered.Write(frame) } }() @@ -269,7 +425,7 @@ func (tm *TranscodeManager) releaseTrack(targetCodec avframe.CodecType, audioOnl } track.subCount-- if track.subCount <= 0 { - track.cancel() + track.cancel(errTranscodeSubscriberReleased) delete(tracks, targetCodec) } } @@ -286,6 +442,58 @@ type audioTranscodePipeline struct { ts audiocodec.TsTracker tsInited bool pcmBuf []int16 + sourceSpan audiocodec.SourceSpan + pcmSpans pcmSourceSpanQueue + submitted bool + finalized bool +} + +type pcmSourceSpanSegment struct { + samples int + span audiocodec.SourceSpan +} + +// pcmSourceSpanQueue accounts provenance in samples per channel, matching the +// unit used by Encoder.FrameSize. +type pcmSourceSpanQueue struct { + segments []pcmSourceSpanSegment +} + +func (q *pcmSourceSpanQueue) append(samples int, span audiocodec.SourceSpan) bool { + if samples <= 0 || !span.Valid() { + return false + } + q.segments = append(q.segments, pcmSourceSpanSegment{samples: samples, span: span}) + return true +} + +func (q *pcmSourceSpanQueue) consume(samples int) audiocodec.SourceSpan { + var span audiocodec.SourceSpan + for samples > 0 && len(q.segments) > 0 { + segment := &q.segments[0] + take := samples + if take > segment.samples { + take = segment.samples + } + if !span.Valid() { + span = segment.span + } else { + span = span.Union(segment.span) + } + segment.samples -= take + samples -= take + if segment.samples == 0 { + q.segments = q.segments[1:] + } + } + if samples != 0 { + return audiocodec.SourceSpan{} + } + return span +} + +func (q *pcmSourceSpanQueue) discard(samples int) bool { + return q.consume(samples).Valid() } func (tm *TranscodeManager) newAudioTranscodePipeline( @@ -319,6 +527,11 @@ func (tm *TranscodeManager) newAudioTranscodePipeline( } func (p *audioTranscodePipeline) close() { + if !p.finalized { + p.finalized = true + p.pcmBuf = nil + p.drainEncoder(false) + } if p.resampler != nil { p.resampler.Close() } @@ -341,20 +554,99 @@ func writeTranscodeSequenceHeader(registry *audiocodec.Registry, track *Transcod ) frame.AudioCodecEpoch = epoch frame.AudioProvenance = avframe.FrameProvenanceTranscoded - track.ringBuffer.Write(frame) + output := transcodeOutput{ + frame: frame, kind: transcodeOutputSequenceHeader, audioEpoch: epoch, + } + track.cacheSequenceHeader(output) + track.ringBuffer.Write(output) } -func (p *audioTranscodePipeline) writeEncoded(dts int64, payload []byte) { +func (p *audioTranscodePipeline) writeEncoded(dts int64, payload []byte, sourceSpan audiocodec.SourceSpan) { + if len(payload) == 0 || p.track.ringBuffer.IsClosed() { + return + } + if !sourceSpan.Valid() { + return + } frame := avframe.NewAVFrame( avframe.MediaTypeAudio, p.track.targetCodec, avframe.FrameTypeInterframe, dts, dts, payload, ) frame.AudioCodecEpoch = p.sourceEpoch frame.AudioProvenance = avframe.FrameProvenanceTranscoded - p.track.ringBuffer.Write(frame) + p.track.ringBuffer.Write(transcodeOutput{ + frame: frame, sourceSpan: sourceSpan, kind: transcodeOutputMedia, audioEpoch: p.sourceEpoch, + }) } -func (p *audioTranscodePipeline) encode(frame *avframe.AVFrame) { +func (p *audioTranscodePipeline) drainEncoder(publish bool) { + frameSamples := p.encoder.FrameSize() + if frameSamples <= 0 || !p.submitted { + return + } + if drainer, ok := p.encoder.(audiocodec.AttributedDrainingEncoder); ok { + packets, err := drainer.DrainAttributed() + if err != nil { + slog.Warn("transcode: encoder drain failed", "codec", p.track.targetCodec, "epoch", p.sourceEpoch, "error", err) + } + if !publish { + return + } + for _, packet := range packets { + p.writeEncoded(p.ts.Next(frameSamples), packet.Payload, packet.SourceSpan) + } + return + } + drainer, ok := p.encoder.(audiocodec.DrainingEncoder) + if !ok { + return + } + packets, err := drainer.Drain() + if err != nil { + slog.Warn("transcode: encoder drain failed", "codec", p.track.targetCodec, "epoch", p.sourceEpoch, "error", err) + } + if !publish { + return + } + for _, packet := range packets { + p.writeEncoded(p.ts.Next(frameSamples), packet, p.sourceSpan) + } +} + +func (p *audioTranscodePipeline) finalize() { + if p.finalized { + return + } + p.finalized = true + if drainer, ok := p.resampler.(audiocodec.AttributedDrainingResampler); ok { + pcm, err := drainer.DrainAttributed() + if err == nil && pcm != nil { + p.encodePCM(&pcm.PCMFrame, pcm.SourceSpan) + } + } else if drainer, ok := p.resampler.(audiocodec.DrainingResampler); ok { + p.encodePCM(drainer.Drain(), p.sourceSpan) + } + + frameSamples := p.encoder.FrameSize() + channels := p.encoder.Channels() + frameSize := frameSamples * channels + if frameSize > 0 && len(p.pcmBuf) > 0 { + padded := make([]int16, frameSize) + copy(padded, p.pcmBuf) + span := p.pcmSpans.consume((len(p.pcmBuf) + channels - 1) / channels) + p.pcmBuf = nil + p.encodePackets(&audiocodec.PCMFrame{ + Samples: padded, SampleRate: p.encoder.SampleRate(), Channels: channels, + }, span, frameSamples) + } + p.drainEncoder(true) +} + +func (p *audioTranscodePipeline) encode(frame *avframe.AVFrame, sourceSpan audiocodec.SourceSpan) { + p.sourceSpan = p.sourceSpan.Union(sourceSpan) + if !p.sourceSpan.Valid() { + p.sourceSpan = sourceSpan + } if !p.tsInited { p.ts.Init(frame.DTS, p.encoder.SampleRate()) p.tsInited = true @@ -374,37 +666,84 @@ func (p *audioTranscodePipeline) encode(frame *avframe.AVFrame) { p.resampled = true } if p.resampler != nil { + if resampler, ok := p.resampler.(audiocodec.AttributedResampler); ok { + attributed, resampleErr := resampler.ResampleAttributed(pcm, sourceSpan) + if resampleErr != nil || attributed == nil { + return + } + p.encodePCM(&attributed.PCMFrame, attributed.SourceSpan) + return + } pcm = p.resampler.Resample(pcm) + p.encodePCM(pcm, p.sourceSpan) + return } + p.encodePCM(pcm, sourceSpan) +} +func (p *audioTranscodePipeline) encodePCM(pcm *audiocodec.PCMFrame, sourceSpan audiocodec.SourceSpan) { + if pcm == nil || len(pcm.Samples) == 0 { + return + } + channels := p.encoder.Channels() + if channels <= 0 || len(pcm.Samples)%channels != 0 || !sourceSpan.Valid() { + return + } + if !p.sourceSpan.Valid() { + p.sourceSpan = sourceSpan + } else { + p.sourceSpan = p.sourceSpan.Union(sourceSpan) + } frameSize := p.encoder.FrameSize() * p.encoder.Channels() if frameSize == 0 { - encoded, encErr := p.encoder.Encode(&audiocodec.PCMFrame{ + p.encodePackets(&audiocodec.PCMFrame{ Samples: pcm.Samples, SampleRate: p.encoder.SampleRate(), Channels: p.encoder.Channels(), - }) - if encErr != nil { - return - } - samplesPerChannel := len(pcm.Samples) / p.encoder.Channels() - p.writeEncoded(p.ts.Next(samplesPerChannel), encoded) + }, sourceSpan, len(pcm.Samples)/channels) return } p.pcmBuf = append(p.pcmBuf, pcm.Samples...) + p.pcmSpans.append(len(pcm.Samples)/channels, sourceSpan) const maxPCMBufSamples = 48000 * 2 if len(p.pcmBuf) > maxPCMBufSamples { - p.pcmBuf = p.pcmBuf[len(p.pcmBuf)-maxPCMBufSamples:] + drop := len(p.pcmBuf) - maxPCMBufSamples + drop -= drop % channels + p.pcmBuf = p.pcmBuf[drop:] + p.pcmSpans.discard(drop / channels) } for len(p.pcmBuf) >= frameSize { - encoded, encErr := p.encoder.Encode(&audiocodec.PCMFrame{ + span := p.pcmSpans.consume(p.encoder.FrameSize()) + p.encodePackets(&audiocodec.PCMFrame{ Samples: p.pcmBuf[:frameSize], SampleRate: p.encoder.SampleRate(), Channels: p.encoder.Channels(), - }) + }, span, p.encoder.FrameSize()) p.pcmBuf = p.pcmBuf[frameSize:] - if encErr != nil { - continue + } +} + +func (p *audioTranscodePipeline) encodePackets(pcm *audiocodec.PCMFrame, sourceSpan audiocodec.SourceSpan, samplesPerPacket int) { + if !sourceSpan.Valid() || samplesPerPacket <= 0 { + return + } + if encoder, ok := p.encoder.(audiocodec.AttributedEncoder); ok { + packets, err := encoder.EncodeAttributed(pcm, sourceSpan) + if err != nil { + return + } + p.submitted = true + for _, packet := range packets { + if !packet.SourceSpan.Valid() { + continue + } + p.writeEncoded(p.ts.Next(samplesPerPacket), packet.Payload, packet.SourceSpan) } - p.writeEncoded(p.ts.Next(p.encoder.FrameSize()), encoded) + return + } + encoded, err := p.encoder.Encode(pcm) + if err != nil { + return } + p.submitted = true + p.writeEncoded(p.ts.Next(samplesPerPacket), encoded, sourceSpan) } // transcodeLoop is the core decode-resample-encode pipeline for a single target codec. @@ -447,15 +786,20 @@ func (tm *TranscodeManager) transcodeLoop( // can miss wakeups and emit video in bursts. RingReader.Read blocks on the // ring condition variable instead; close the reader when this track is // cancelled so the blocking read remains interruptible. + readCtx, cancelRead := context.WithCancel(ctx) + defer cancelRead() stopReader := make(chan struct{}) go func() { select { case <-ctx.Done(): - reader.Close() + cancelRead() + case <-track.generationDone: + cancelRead() case <-stopReader: } }() defer close(stopReader) + defer reader.Close() for { // Inline processing: handle each frame as it arrives. Video passes @@ -465,29 +809,72 @@ func (tm *TranscodeManager) transcodeLoop( // N × encode_time. Chrome's jitter estimator accumulates delivery // irregularities via EWMA, so even small periodic delays from batch // encoding compound over minutes into large jitter buffer growth. - frame, ok := reader.Read() - if !ok { + if ctx.Err() != nil { + track.setTerminationCause(context.Cause(ctx)) return } - for { - if frame.MediaType.IsVideo() { - if !audioOnly { - // Legacy reader: pass video through without encoding. - track.ringBuffer.Write(frame) - } - } else if frame.MediaType.IsAudio() { - frameEpoch := frame.AudioCodecEpoch - if frameEpoch == 0 { - frameEpoch = sourceEpoch - } - if frameEpoch < audioEpochFloor { - track.advanceSourceCursor(reader.ReadCursor()) - frame, ok = reader.TryRead() - if !ok { - break - } - continue - } + endCursor, generationEnded := track.generationBoundary.end() + if generationEnded && reader.ReadCursor() >= endCursor { + if pipeline != nil { + pipeline.finalize() + } + track.advanceSourceCursor(endCursor) + track.setTerminationCause(errTranscodeGenerationComplete) + return + } + + var read util.RingReadResult[*avframe.AVFrame] + if generationEnded { + read = reader.TryReadResult() + } else { + read = reader.ReadResultContext(readCtx) + } + if !read.OK { + if ctx.Err() != nil { + track.setTerminationCause(context.Cause(ctx)) + return + } + // A generation close cancels the blocking read. Re-evaluate its + // now-published finite boundary and drain retained source frames. + if _, ended := track.generationBoundary.end(); ended { + continue + } + track.setTerminationCause(errTranscodeSourceClosed) + return + } + if read.Overwritten > 0 { + cause := &transcodeSourceOverwriteError{Overwritten: read.Overwritten} + track.setTerminationCause(cause) + slog.Warn("transcode: source ring overwritten", "codec", track.targetCodec, "overwritten", read.Overwritten) + return + } + frame := read.Value + consumedCursor := reader.ReadCursor() + sourceSpan := audiocodec.SourceSpan{Begin: consumedCursor - 1, End: consumedCursor} + if endCursor, ended := track.generationBoundary.end(); ended && consumedCursor > endCursor { + // The shared source ring may already contain a replacement. Never + // emit a frame beyond this track's publisher-generation boundary. + if pipeline != nil { + pipeline.finalize() + } + track.advanceSourceCursor(endCursor) + track.setTerminationCause(errTranscodeGenerationComplete) + return + } + + if frame.MediaType.IsVideo() { + if !audioOnly { + // Legacy reader: pass video through without encoding. + track.ringBuffer.Write(transcodeOutput{ + frame: frame, sourceSpan: sourceSpan, kind: transcodeOutputMedia, + }) + } + } else if frame.MediaType.IsAudio() { + frameEpoch := frame.AudioCodecEpoch + if frameEpoch == 0 { + frameEpoch = sourceEpoch + } + if frameEpoch >= audioEpochFloor { if frame.Codec != sourceCodec || frameEpoch != sourceEpoch { firstAudioEpoch := sourceCodec == 0 if pipeline != nil { @@ -508,40 +895,42 @@ func (tm *TranscodeManager) transcodeLoop( } if frame.Codec == track.targetCodec { - track.ringBuffer.Write(frame) + output := transcodeOutput{ + frame: frame, sourceSpan: sourceSpan, kind: transcodeOutputMedia, audioEpoch: frameEpoch, + } + if frame.FrameType == avframe.FrameTypeSequenceHeader { + track.cacheSequenceHeader(transcodeOutput{ + frame: frame, kind: transcodeOutputSequenceHeader, audioEpoch: frameEpoch, + }) + } + track.ringBuffer.Write(output) } else if pipeline != nil && frame.FrameType == avframe.FrameTypeSequenceHeader { pipeline.decoder.SetExtradata(frame.Payload) } else if pipeline != nil { - pipeline.encode(frame) + pipeline.encode(frame, sourceSpan) } } - track.advanceSourceCursor(reader.ReadCursor()) - - frame, ok = reader.TryRead() - if !ok { - break - } } + track.advanceSourceCursor(consumedCursor) } } -// Reset cancels all active transcode goroutines and removes all tracks. -// Called when a new publisher replaces the old one. +// Reset removes track mappings before a replacement publisher starts. Active +// generations are canceled; ended-generation producers retain ownership of +// their output rings until finite readers have consumed their terminal output. func (tm *TranscodeManager) Reset() { tm.mu.Lock() defer tm.mu.Unlock() for codec, track := range tm.tracks { - if track.cancel != nil { - track.cancel() + if _, ended := track.generationBoundary.end(); !ended && track.cancel != nil { + track.cancel(errTranscodeManagerReset) } - track.ringBuffer.Close() delete(tm.tracks, codec) } for codec, track := range tm.audioTracks { - if track.cancel != nil { - track.cancel() + if _, ended := track.generationBoundary.end(); !ended && track.cancel != nil { + track.cancel(errTranscodeManagerReset) } - track.ringBuffer.Close() delete(tm.audioTracks, codec) } } diff --git a/core/transcode_manager_audiocodec_test.go b/core/transcode_manager_audiocodec_test.go index 69acc761..0ac1de06 100644 --- a/core/transcode_manager_audiocodec_test.go +++ b/core/transcode_manager_audiocodec_test.go @@ -180,8 +180,9 @@ func TestTranscodeManagerUnsupportedStartupEpochDoesNotPoisonSharedTrack(t *test } writeG711Frames(stream, 1000, 12, bytes.Repeat([]byte{0xff}, 160)) + lateSnapshot := stream.StartupSnapshot() lateReader, releaseLate, err := stream.TranscodeManager().GetOrCreateAudioReaderAtFromHistory( - avframe.CodecAAC, stream.StartupSnapshot(), + avframe.CodecAAC, lateSnapshot, ) if err != nil { t.Fatal(err) @@ -190,16 +191,17 @@ func TestTranscodeManagerUnsupportedStartupEpochDoesNotPoisonSharedTrack(t *test lateReader.Close() releaseLate() }) + writeDirectAACFrames(stream, 3000, 2) for _, frame := range readTranscodedAudioFrames(t, reader, 2) { if frame.DTS < 1000 { t.Fatalf("existing reader emitted stale DTS %d before the supported epoch", frame.DTS) } } - lateFrames := readCurrentEpochAudioWithHeader(t, lateReader, stream.StartupSnapshot().audioCodecEpoch, 2) + lateFrames := readCurrentEpochAudioWithHeader(t, lateReader, lateSnapshot.audioCodecEpoch, 2) for _, frame := range lateFrames { - if frame.DTS < 1000 { - t.Fatalf("new reader emitted stale DTS %d before the supported epoch", frame.DTS) + if frame.DTS < 3000 { + t.Fatalf("new reader emitted pre-floor DTS %d before post-snapshot direct AAC", frame.DTS) } } @@ -256,6 +258,7 @@ func TestTranscodeManagerLateSnapshotReaderStartsAtCurrentAudioEpoch(t *testing. lateReader.Close() releaseLate() }() + writeDirectAACFrames(stream, 3000, 2) first := readTranscodedAudioFrames(t, lateReader, 1)[0] if first.AudioCodecEpoch < current[0].AudioCodecEpoch { @@ -264,6 +267,146 @@ func TestTranscodeManagerLateSnapshotReaderStartsAtCurrentAudioEpoch(t *testing. if first.DTS < current[0].DTS { t.Fatalf("late reader first DTS = %d, want at least current DTS %d", first.DTS, current[0].DTS) } + if first.DTS < 3000 { + t.Fatalf("late reader emitted pre-floor retained DTS %d", first.DTS) + } +} + +func TestTranscodeManagerGenerationEndFinalizesPartialAACAndDelayedPackets(t *testing.T) { + stream := newTranscodeTestStream(avframe.CodecG711U) + defer stream.Close() + tm := stream.TranscodeManager() + snapshot := stream.StartupSnapshot() + reader, release, err := tm.GetOrCreateReaderAtFromHistory(avframe.CodecAAC, snapshot) + if err != nil { + t.Fatal(err) + } + defer release() + defer reader.Close() + + const firstDTS = int64(1000) + stream.WriteFrame(avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711U, avframe.FrameTypeInterframe, + firstDTS, firstDTS, bytes.Repeat([]byte{0xff}, 160), + )) + endCursor := stream.RingBuffer().WriteCursor() + waitCtx, cancelWait := context.WithTimeout(context.Background(), 3*time.Second) + if !tm.WaitForSourceCursor(avframe.CodecAAC, endCursor, waitCtx) { + cancelWait() + t.Fatal("transcoder did not consume the partial source frame") + } + cancelWait() + + var mediaBeforeEnd int + for { + frame, ok := reader.TryRead() + if !ok { + break + } + if frame != nil && frame.MediaType.IsAudio() && frame.FrameType != avframe.FrameTypeSequenceHeader { + mediaBeforeEnd++ + } + } + if mediaBeforeEnd != 0 { + t.Fatalf("partial PCM produced %d AAC packets before generation end, want 0", mediaBeforeEnd) + } + + tm.mu.Lock() + track := tm.tracks[avframe.CodecAAC] + tm.mu.Unlock() + if track == nil { + t.Fatal("AAC track disappeared before generation retirement") + } + stream.RemovePublisher() + if err := stream.SetPublisher(&testPublisher{ + id: "replacement-after-audio-tail", + info: &avframe.MediaInfo{AudioCodec: avframe.CodecG711U}, + }); err != nil { + t.Fatalf("install replacement publisher: %v", err) + } + + readCtx, cancelRead := context.WithTimeout(context.Background(), 3*time.Second) + defer cancelRead() + var tail []*avframe.AVFrame + for { + frame, ok := reader.ReadContext(readCtx) + if !ok { + break + } + if frame != nil && frame.MediaType.IsAudio() && frame.FrameType != avframe.FrameTypeSequenceHeader { + tail = append(tail, frame) + } + } + if err := readCtx.Err(); err != nil { + t.Fatalf("transcoded generation tail did not close: %v", err) + } + if len(tail) == 0 { + t.Fatal("generation tail produced no AAC packets") + } + for i, frame := range tail { + if i > 0 && frame.DTS <= tail[i-1].DTS { + t.Fatalf("generation tail DTS[%d] = %d after %d, want strictly increasing", i, frame.DTS, tail[i-1].DTS) + } + if frame.AudioCodecEpoch != snapshot.audioCodecEpoch { + t.Fatalf("generation tail epoch[%d] = %d, want %d", i, frame.AudioCodecEpoch, snapshot.audioCodecEpoch) + } + if frame.AudioProvenance != avframe.FrameProvenanceTranscoded { + t.Fatalf("generation tail provenance[%d] = %d, want transcoded", i, frame.AudioProvenance) + } + } + if !track.ringBuffer.IsClosed() { + t.Fatal("transcode output ring remained open after generation tail") + } + if got := track.sourceCursor.Load(); got != endCursor { + t.Fatalf("transcode source cursor = %d, want generation end %d", got, endCursor) + } + closedCursor := track.ringBuffer.WriteCursor() + time.Sleep(20 * time.Millisecond) + if got := track.ringBuffer.WriteCursor(); got != closedCursor { + t.Fatalf("transcode output advanced from %d to %d after ring close", closedCursor, got) + } +} + +func TestTranscodeManagerLastConsumerCancellationDiscardsPartialTail(t *testing.T) { + stream := newTranscodeTestStream(avframe.CodecG711U) + defer stream.Close() + tm := stream.TranscodeManager() + reader, release, err := tm.GetOrCreateReaderAtFromHistory(avframe.CodecAAC, stream.StartupSnapshot()) + if err != nil { + t.Fatal(err) + } + + stream.WriteFrame(avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711U, avframe.FrameTypeInterframe, + 1000, 1000, bytes.Repeat([]byte{0xff}, 160), + )) + sourceCursor := stream.RingBuffer().WriteCursor() + waitCtx, cancelWait := context.WithTimeout(context.Background(), 3*time.Second) + if !tm.WaitForSourceCursor(avframe.CodecAAC, sourceCursor, waitCtx) { + cancelWait() + t.Fatal("transcoder did not consume the partial source frame") + } + cancelWait() + + tm.mu.Lock() + track := tm.tracks[avframe.CodecAAC] + tm.mu.Unlock() + if track == nil { + t.Fatal("AAC track disappeared before consumer cancellation") + } + beforeCancel := track.ringBuffer.WriteCursor() + release() + reader.Close() + deadline := time.Now().Add(3 * time.Second) + for !track.ringBuffer.IsClosed() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if !track.ringBuffer.IsClosed() { + t.Fatal("transcode output ring remained open after last consumer cancellation") + } + if got := track.ringBuffer.WriteCursor(); got != beforeCancel { + t.Fatalf("last-consumer cancellation published %d unowned tail frames", got-beforeCancel) + } } func writeG711Frames(stream *Stream, startDTS int64, count int, payload []byte) { @@ -276,6 +419,20 @@ func writeG711Frames(stream *Stream, startDTS int64, count int, payload []byte) } } +func writeDirectAACFrames(stream *Stream, startDTS int64, count int) { + stream.WriteFrame(avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeSequenceHeader, + startDTS, startDTS, []byte{0x12, 0x10}, + )) + for i := range count { + dts := startDTS + int64((i+1)*20) + stream.WriteFrame(avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, + dts, dts, []byte{0x21, byte(i)}, + )) + } +} + func readTranscodedAudioFrames(t *testing.T, reader interface { ReadContext(context.Context) (*avframe.AVFrame, bool) }, count int) []*avframe.AVFrame { @@ -302,7 +459,7 @@ func readCurrentEpochAudioWithHeader(t *testing.T, reader interface { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() frames := make([]*avframe.AVFrame, 0, count) - headerSeen := false + var headerEpoch uint64 for len(frames) < count { frame, ok := reader.ReadContext(ctx) if !ok { @@ -315,11 +472,11 @@ func readCurrentEpochAudioWithHeader(t *testing.T, reader interface { t.Fatalf("reader emitted audio epoch %d below floor %d", frame.AudioCodecEpoch, epoch) } if frame.FrameType == avframe.FrameTypeSequenceHeader { - headerSeen = true + headerEpoch = frame.AudioCodecEpoch continue } - if !headerSeen { - t.Fatalf("reader emitted epoch %d media before its target sequence header", epoch) + if headerEpoch != frame.AudioCodecEpoch { + t.Fatalf("reader emitted epoch %d media after target sequence header epoch %d", frame.AudioCodecEpoch, headerEpoch) } frames = append(frames, frame) } diff --git a/core/transcode_manager_test.go b/core/transcode_manager_test.go index 35ec8ab4..c011189c 100644 --- a/core/transcode_manager_test.go +++ b/core/transcode_manager_test.go @@ -1,14 +1,72 @@ package core import ( + "bytes" "testing" "time" "github.com/im-pingo/liveforge/config" "github.com/im-pingo/liveforge/pkg/audiocodec" "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/im-pingo/liveforge/pkg/util" ) +type partialPCMDecoder struct { + closed bool +} + +func (d *partialPCMDecoder) SetExtradata([]byte) {} +func (d *partialPCMDecoder) Decode([]byte) (*audiocodec.PCMFrame, error) { + return &audiocodec.PCMFrame{Samples: []int16{11, 22, 33}, SampleRate: 50, Channels: 1}, nil +} +func (d *partialPCMDecoder) SampleRate() int { return 50 } +func (d *partialPCMDecoder) Channels() int { return 1 } +func (d *partialPCMDecoder) Close() { d.closed = true } + +type observableDrainingEncoder struct { + ring *util.RingBuffer[transcodeOutput] + encoded [][]int16 + drainCalls int + drainedAfterClose bool + closed bool +} + +func (e *observableDrainingEncoder) Encode(pcm *audiocodec.PCMFrame) ([]byte, error) { + e.encoded = append(e.encoded, append([]int16(nil), pcm.Samples...)) + return []byte{0x10}, nil +} +func (e *observableDrainingEncoder) Drain() ([][]byte, error) { + e.drainCalls++ + e.drainedAfterClose = e.drainedAfterClose || e.ring.IsClosed() + return [][]byte{{0x20}, {0x30}}, nil +} +func (e *observableDrainingEncoder) SampleRate() int { return 50 } +func (e *observableDrainingEncoder) Channels() int { return 1 } +func (e *observableDrainingEncoder) FrameSize() int { return 4 } +func (e *observableDrainingEncoder) Close() { e.closed = true } + +type observableDrainingResampler struct { + drainCalls int + drained bool +} + +func (r *observableDrainingResampler) Resample(pcm *audiocodec.PCMFrame) *audiocodec.PCMFrame { + return &audiocodec.PCMFrame{ + Samples: append([]int16(nil), pcm.Samples...), SampleRate: 50, Channels: 1, + } +} + +func (r *observableDrainingResampler) Drain() *audiocodec.PCMFrame { + r.drainCalls++ + if r.drained { + return &audiocodec.PCMFrame{SampleRate: 50, Channels: 1} + } + r.drained = true + return &audiocodec.PCMFrame{Samples: []int16{44, 55}, SampleRate: 50, Channels: 1} +} + +func (r *observableDrainingResampler) Close() {} + // TestTranscodeManagerZeroOverhead verifies no TranscodedTrack is created // when the subscriber requests the same codec as the publisher. func TestTranscodeManagerZeroOverhead(t *testing.T) { @@ -283,6 +341,90 @@ func TestTranscodeManagerNoPublisher(t *testing.T) { } } +func TestAudioTranscodePipelineFinalizesPartialPCMBeforeRingClose(t *testing.T) { + ring := util.NewRingBuffer[transcodeOutput](8) + track := &TranscodedTrack{targetCodec: avframe.CodecAAC, ringBuffer: ring} + decoder := &partialPCMDecoder{} + encoder := &observableDrainingEncoder{ring: ring} + resampler := &observableDrainingResampler{} + pipeline := &audioTranscodePipeline{ + track: track, + sourceCodec: avframe.CodecG711U, + sourceEpoch: 7, + decoder: decoder, + encoder: encoder, + resampler: resampler, + resampled: true, + } + pipeline.encode(avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711U, avframe.FrameTypeInterframe, + 500, 500, []byte{0xff}, + ), audiocodec.SourceSpan{Begin: 10, End: 11}) + if got := ring.WriteCursor(); got != 0 { + t.Fatalf("partial PCM wrote %d packets before finalization, want 0", got) + } + + pipeline.finalize() + if len(encoder.encoded) != 2 { + t.Fatalf("encoder calls = %d, want 1 full resampler-tail frame plus 1 padded frame", len(encoder.encoded)) + } + wantPCM := [][]int16{{11, 22, 33, 44}, {55, 0, 0, 0}} + for frameIndex := range wantPCM { + if got := encoder.encoded[frameIndex]; len(got) != len(wantPCM[frameIndex]) { + t.Fatalf("encoded PCM frame %d length = %d, want %d", frameIndex, len(got), len(wantPCM[frameIndex])) + } else { + for i := range wantPCM[frameIndex] { + if got[i] != wantPCM[frameIndex][i] { + t.Fatalf("encoded PCM frame %d sample[%d] = %d, want %d", frameIndex, i, got[i], wantPCM[frameIndex][i]) + } + } + } + } + if resampler.drainCalls != 1 { + t.Fatalf("resampler drain calls = %d, want exactly 1", resampler.drainCalls) + } + if encoder.drainCalls != 1 || encoder.drainedAfterClose { + t.Fatalf("drain calls/after-close = %d/%v, want 1/false", encoder.drainCalls, encoder.drainedAfterClose) + } + if got := ring.WriteCursor(); got != 4 { + t.Fatalf("finalized packet count = %d, want 2 encoded packets plus 2 delayed packets", got) + } + + ring.Close() + closedCursor := ring.WriteCursor() + pipeline.finalize() + if got := ring.WriteCursor(); got != closedCursor { + t.Fatalf("ring cursor advanced from %d to %d after close", closedCursor, got) + } + if encoder.drainCalls != 1 { + t.Fatalf("repeated finalization called drain %d times, want exactly once", encoder.drainCalls) + } + if resampler.drainCalls != 1 { + t.Fatalf("repeated finalization called resampler drain %d times, want exactly once", resampler.drainCalls) + } + + reader := ring.NewReader() + wantDTS := []int64{500, 580, 660, 740} + wantPayload := [][]byte{{0x10}, {0x10}, {0x20}, {0x30}} + for i := range wantDTS { + output, ok := reader.TryRead() + if !ok { + t.Fatalf("finalized output ended at packet %d", i) + } + frame := output.frame + if frame.DTS != wantDTS[i] || !bytes.Equal(frame.Payload, wantPayload[i]) { + t.Fatalf("packet %d = DTS %d payload %x, want DTS %d payload %x", i, frame.DTS, frame.Payload, wantDTS[i], wantPayload[i]) + } + if frame.AudioCodecEpoch != 7 || frame.AudioProvenance != avframe.FrameProvenanceTranscoded { + t.Fatalf("packet %d epoch/provenance = %d/%d, want 7/transcoded", i, frame.AudioCodecEpoch, frame.AudioProvenance) + } + } + pipeline.close() + if !decoder.closed || !encoder.closed { + t.Fatal("pipeline close did not release decoder and encoder") + } +} + // newTranscodeTestStream creates a test stream with a publisher of the given audio codec. func newTranscodeTestStream(codec avframe.CodecType) *Stream { cfg := config.StreamConfig{RingBufferSize: 64} diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index fc92656e..12b6d6e3 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -2,18 +2,18 @@ > Source-aligned project status. Update this file only after implementation and a passing verification path exist. > -> Last updated: 2026-08-28 +> Last updated: 2026-08-30 ## Current Status LiveForge is a Go 1.26+ modular streaming server with multi-protocol ingest/playback, protocol bridging, management operations, optional FFmpeg audio transcoding, runtime configuration refresh, and multi-node relay. -Previously identified incomplete or unclosed runtime features are implemented and documented except Simulcast layer selection. `stream.simulcast` remains configuration-only, restart-required, explicitly deferred, and unsupported by the WebRTC runtime. SIP and GB28181 persistent fake-device publish/receive signaling and RTP/RTCP loopback are implemented and verified at their providers. A new Console WHEP regression for real H.264 input is open; the project is not considered fully complete until WEBRTC-001 in [docs/TECHNICAL-RISKS.md](TECHNICAL-RISKS.md) is reproduced, fixed, and browser-verified. +Previously identified incomplete or unclosed runtime features are implemented and documented except Simulcast layer selection. `stream.simulcast` remains configuration-only, restart-required, explicitly deferred, and unsupported by the WebRTC runtime. SIP and GB28181 persistent fake-device publish/receive signaling and RTP/RTCP loopback are implemented and verified at their providers. The Console WHEP H.264 regression is fixed and browser-verified; the remaining work is limited to the explicitly documented Simulcast boundary and long-duration/concurrency capacity testing. -## Open Review Items +## Review Items -- **WEBRTC-001 (P0)**: Console WHEP can show `No advancing media received (check codec support and keyframes)`. Root cause is confirmed: the default Console path requests realtime mode, which starts after `LiveCursor` and discards media until a later keyframe; the 8-second watchdog can report failure before a long GOP's next IDR arrives. `mode=live` and the current H.264 browser path can decode, but the default behavior, error state, media-write diagnostics, and real GB28181/SIP H.264 browser coverage remain open. -- **WEBRTC-002 (P1)**: Add a browser regression path for real H.264 inputs and distinguish no keyframe, codec negotiation, malformed payload, and `WriteSample` failure. +- **WEBRTC-001 (P0)**: Closed. The default Console and protocol-lab WHEP path uses atomic `mode=live` GOP startup; explicit realtime mode retains its waiting-keyframe semantics, while feed status and bounded diagnostics distinguish waiting, codec mismatch, write failure, generation end, and media stall. +- **WEBRTC-002 (P1)**: Closed for the supported matrix. Chromium coverage verifies real H.264/VP8 playback when the browser advertises H.264, SIP/GB28181/WHIP cross-protocol paths, decoded dimensions, advancing media time, RTP/RTCP counters, and non-stalled server status; browsers without H.264 receive an explicit environment skip, while Pion negotiation coverage remains required. Long-duration and high-concurrency capacity remain separate operational work. - Performance, lifecycle, resource, security, and functional-boundary findings are recorded with source locations in [docs/TECHNICAL-RISKS.md](TECHNICAL-RISKS.md). They are not silently treated as completed work. Release artifacts remain conditional: source builds are available from the repository; versioned binaries and GHCR images exist only after a `v*` tag completes the Release workflow. Portable release binaries use `CGO_ENABLED=0` and do not provide audio transcoding. Tagged source builds and the Dockerfile use `audiocodec` plus FFmpeg. @@ -68,7 +68,7 @@ Release artifacts remain conditional: source builds are available from the repos - Authenticated recording list/status/detail, HTTP range download, inline browser playback, and admin delete operations. - Storage Console actions preview completed recordings and DVR sessions with available segments; recording media uses the management session while DVR media remains on its separate listener without browser bearer-token persistence. - DVR playlist/segment serving with synchronous-only subscribe authorization and no asynchronous subscribe lifecycle emission, retention cleanup, storage/session status, and Prometheus metrics. -- DVR media registration is valid on Go 1.26 and strictly dispatches only `GET /dvr/{app}/{key}.m3u8` and `GET /dvr/{app}/{key}/{filename}`; malformed or nested resources return 404 before playback authorization or storage lookup. +- DVR media registration is valid on Go 1.26 and dispatches `GET /dvr/{app}/{key}.m3u8` and `GET /dvr/{app}/{key}/{filename}`, including nested stream keys. Encoded path separators, dot segments, and extra resource levels return 404 before playback authorization or storage lookup. ### SIP Gateway @@ -115,7 +115,9 @@ CGO_ENABLED=1 go test -tags audiocodec -race \ | WHIP H.265 + Opus eight-protocol browser playback | Codec-specific Annex-B tests, atomic WHEP Live snapshot test, and `docs/recipes/whip-h265-opus-playback.md` | | Storage recording availability and unified fMP4 playback | `module/record/record_test.go`, `module/api/recording_test.go`, and `RecordingStatusResponse` contract | | Config document/schema/validate/apply and five runtime sources | `module/api/config_api_test.go`, `config/runtime/source_test.go`, and `docs/recipes/runtime-config-sources.md` | +| CONFIG-004 through CONFIG-008 runtime/API hardening | `config/runtime/source_test.go`, `module/api/console_management_test.go`, `module/api/openapi_contract_test.go`, and the synchronized schema/recipe/OpenAPI docs | | SIP/GB28181 fast self-tests and persistent provider labs | `module/api/config_api_test.go`, `module/api/protocol_testlab_api_test.go`, `module/sipgateway/lab_test.go`, `module/gb28181/lab_test.go`, and `docs/recipes/protocol-test-lab.md` | +| ARCH-033 unified HTTP header and idle timeouts | `module/api`, `module/webrtc`, and `module/metrics` `TestHTTPServerTimeouts`; `docs/recipes/auth-and-tls.md` | ## Operations Documentation diff --git a/docs/TECHNICAL-RISKS.md b/docs/TECHNICAL-RISKS.md index c8494de1..24dd667e 100644 --- a/docs/TECHNICAL-RISKS.md +++ b/docs/TECHNICAL-RISKS.md @@ -1,6 +1,6 @@ # 技术风险、性能瓶颈与问题记录 -> 记录日期：2026-08-28 +> 记录日期：2026-08-29 > > 本文是源码审查和当前复现结果的工作记录。`已确认` 表示已经从源码、测试或稳定复现得到证据；`待复现` 表示代码路径明确但还需要真实控制台/协议输入确认；`功能边界` 表示当前没有实现或受构建条件限制，不能当作已支持能力。 @@ -8,56 +8,99 @@ ### WEBRTC-001：控制台 WHEP 播放报 `No advancing media received` -- **等级**：P0，用户可见，状态为 `根因已确认，修复未关闭`。 +- **等级**：P0，用户可见，状态为 `默认 Console 与 SIP/GB28181 协议实验室路径已修复并完成真实浏览器验收`。 - **现象**：控制台在 8 秒后显示 `No advancing media received (check codec support and keyframes)`，用户看不到视频。 -- **已确认的数据流**：控制台 `module/api/console.html` 的 `playWHEP` 默认请求 `mode=realtime`；`module/webrtc/whep_feed.go` 从 `startup.LiveCursor` 创建 reader，并在 `gotKeyframe` 变为 true 前丢弃所有视频非关键帧。 +- **已确认的数据流**：控制台和协议 lab 的默认 WHEP 请求现在使用 `mode=live`；显式 `mode=realtime` 仍从 `startup.LiveCursor` 创建 reader，并在 `gotKeyframe` 变为 true 前丢弃所有视频非关键帧。 - **已确认断点**：如果 `LiveCursor` 位于最近一个关键帧之后，而输入源下一个 IDR 间隔较长、没有继续发送 IDR，或输入源不响应 PLI，则 feed loop 会持续读取并丢弃视频，浏览器在 watchdog 窗口内收不到可解码的首个视频访问单元。`mode=live` 会先发送快照中的 GOP，因此可作为对照组。 -- **第二个断点**：`whep_feed.go` 中 `video.WriteSample` 的错误被转换成 `false` 后由调用方忽略，track 关闭、协商 payload type 不匹配、编码器拒绝样本等情况不会进入 session 状态或日志，最终只表现为前端 watchdog 超时。 -- **测试证据**：当前 Pion WHEP H.264 RTP、GCC、AAC->Opus、H.264+PCMA，以及 VP8 headless Chrome 测试通过。对当前运行实例的 `live/h264-test`（H.264 + AAC，观测到 GOP 约 3.8-6.1 秒）实测，`mode=realtime` 在 5 秒窗口只有 240 个视频帧，而 `mode=live` 有 1395 个视频帧；Console 浏览器实测 `mode=live` 在约 3.5 秒内得到 640x360、`currentTime` 递增的视频，`mode=realtime` 在后续 IDR 到来前停留在等待状态，8 秒 watchdog 可能先报错，关键帧到达后才恢复 `Playing`。这确认了首帧门控/超时问题，但仍未覆盖真实 GB28181/SIP H.264 的浏览器解码器路径。 -- **必须补齐的验证**：记录 WHEP offer/answer 中实际 video codec、publisher codec、startup generation、`LiveCursor`、首个关键帧时间、丢弃帧数量、`WriteSample` 错误和每个 sender 的 RTP 计数；分别验证 `mode=realtime`、`mode=live`、稀疏关键帧、无 GOP cache、GB28181 H.264 和 SIP H.264。 +- **第二个断点（已修复）**：`whep_feed.go` 中 `video.WriteSample`/`audio.WriteSample` 错误现在进入 WHEP feed 状态和结构化日志；每次真实迁移包含 generation、cursor、mode、前后状态和有界错误，同状态逐帧更新不重复记录；`GET /webrtc/session/{sessionId}/status` 可读取首媒体时间、固定等待毫秒数和有界诊断。 +- **测试证据**：当前 Pion WHEP H.264 RTP、GCC、AAC->Opus、H.264+PCMA，以及 VP8 headless Chrome 测试通过。默认 Console/lab 路径已切到 `mode=live`，显式 `mode=realtime` 仍会在后续 IDR 到来前处于 `waiting_keyframe`；状态 endpoint 已覆盖 generation/cursor/error 读取。新增真实 H.264 Annex-B fixture -> AVCC -> WHEP -> Chromium 回归在浏览器 offer 宣告 H.264 时验证 320x180 解码尺寸、ICE connected、无 media error 且 currentTime 连续推进；不具备 H.264 接收能力的 Chromium 环境会明确 skip，而不是把服务端正确的 415 判为产品故障。2026-08-28 独立端口验收进一步验证 SIP 与 GB28181 假设备生成的 H.264 均在 Console WHEP 中得到 160x90、`readyState=4`、无 media error 且 `currentTime` 连续推进。 +- **剩余验证**：显式 `mode=realtime`、稀疏关键帧和无 GOP cache 的状态区分回归已保留；2026-08-30 的 60 秒统一矩阵 soak 已通过，但更长时长、背压和并发容量仍需独立运行，不能由正确性矩阵替代。 - **验收标准**：默认 Console WHEP 必须在 8 秒内收到可解码视频帧并推进 `currentTime`；首帧前允许等待关键帧，但不能因正常的 GOP 间隔先显示误导性的失败状态，也不能静默丢包或永久等待；显式 realtime 模式若无法及时获得关键帧，必须展示可区分的等待/无关键帧状态；失败时服务端日志必须指出是无关键帧、编码不匹配还是样本写入错误。 -### WEBRTC-002：真实 H.264 浏览器覆盖不足 +### WEBRTC-002：真实 H.264 浏览器覆盖 -- **等级**：P1，状态为 `测试缺口`。 -- 当前 browser jitter 测试主要使用 VP8，H.264 相关端到端测试主要验证 Pion 对端的 RTP，不验证 Chrome/浏览器 H.264 实际解码、profile-level-id、SPS/PPS 和访问单元结构。 -- GB28181 输入的 PS 解封装、SIP 输入的 RTP 解包和 WHIP 输入的 RTP 解包可能生成不同的 H.264 payload/关键帧形态；没有一条真实输入到浏览器解码的统一回归路径。 +- **等级**：P1，状态为 `SIP/GB28181/WHIP 统一自动化矩阵已实现并通过短时 soak`。 +- 统一 Chromium 矩阵覆盖 SIP publish -> GB28181 receive + WHEP、GB28181 publish -> SIP receive + WHEP、WHIP H.264/Opus publish -> SIP receive + GB28181 receive + WHEP。它校验真实解码尺寸、媒体时钟、音视频 RTP/解码帧、RTCP、ICE 和服务端非 stalled 状态。 +- `LIVEFORGE_PROTOCOL_MATRIX_SOAK` 可逐秒扩展推进检查；2026-08-29 已通过 15 秒/场景的自动化运行。Chrome 缺失或 Chromium offer 不具备 H.264 接收能力时测试会明确 skip，默认 soak 为零，因此需要具备 Chromium/H.264 才能把浏览器矩阵当作发布门禁；Pion 协商覆盖仍是强制路径。该矩阵证明协议正确性，不证明并发会话、长时背压或部署容量。 -## 已确认的性能瓶颈 +## 性能风险处置状态 以下问题不会因为删除 `audioCache` 自动消失，需要单独处理和基准验证。 -| ID | 风险 | 证据位置 | 影响 | +| ID | 状态 | 当前结论 | 剩余影响 | | --- | --- | --- | --- | -| PERF-001 | `Stream.WriteFrame` 在 publisher 校验、反射比较、媒体信息、GOP、统计和 ring 写入期间持有 stream 锁 | `core/stream.go` 的 `WriteFrame`/`writeFrameLocked` | 所有协议推流共享串行临界区，帧率和并发 publisher 增加时锁竞争放大 | -| PERF-002 | `samePublisher` 在帧热路径使用 reflection | `core/stream.go` 的 `samePublisher` | 每帧产生额外类型/可比性判断，削弱高帧率输入吞吐 | -| PERF-003 | 码率限制每帧调用 stats snapshot，包含窗口锁和 `time.Now` | `core/stream.go`、`core/stream_stats.go` | 码率限制打开时 CPU、锁竞争和时间调用开销按帧增长 | -| PERF-004 | 音频转码可能为每个 subscriber 创建 reader-local RingBuffer 和 goroutine | `core/transcode_manager.go`、`module/httpstream/muxer_worker.go`、`module/rtmp/subscriber.go` | 订阅者数量增加时内存、goroutine 和重复搬运增长 | -| PERF-005 | SIP/GB28181 出站 RTP 按 fragment 分配、marshal 和 UDP syscall | `module/gb28181/outbound_media.go`、`module/sipgateway/call_session.go` | 监控流/呼叫数增加时系统调用和 GC 压力高 | -| PERF-006 | Consul/Redis refresh 会完整读取、解析、hash、diff，并在一个 worker 中串行应用 | `config/runtime/manager.go`、`source_consul.go`、`source_redis.go` | 大配置或高刷新频率下阻塞后续 refresh/callback，造成配置延迟 | -| PERF-007 | Prometheus 使用任意 `stream_key` 作为 label | `module/metrics/collector.go` | 高基数流键导致时间序列 churn、内存增长和查询退化 | - -## 已确认的架构与可靠性风险 - -| ID | 风险 | 证据位置 | 影响 | -| --- | --- | --- | --- | -| ARCH-001 | GOP cache 只有 GOP 数量上限，没有单 GOP 的帧数、持续时间和字节上限 | `core/stream.go`、`config/config.go` | 异常稀疏关键帧或超大帧会导致单个 GOP 占用过多内存；`gop_cache_num=1` 不能保证内存有界 | -| ARCH-002 | GB28181 RTP receiver 复用 UDP buffer，重排队列保留 Payload slice | `module/gb28181/rtp_receiver.go` | 后续 ReadFrom 会覆盖已排队 payload，造成偶发 PS/RTP 损坏和难以复现的解码失败 | -| ARCH-003 | 无 publisher 超时只将 stream 标为 `Destroying`，未完整从 StreamHub 移除和释放资源 | `core/stream.go`、`core/stream_hub.go` | 空流对象和关联资源可能长期保留，流键复用时状态边界复杂 | -| ARCH-004 | HTTP module 的 `registered map[*core.Stream]bool` 保留历史 Stream 指针 | `module/httpstream/module.go` | 长时间运行和大量动态流键下内存泄漏式增长 | -| ARCH-005 | `AcquireConn` 使用 Load-then-Add，存在并发超限竞态 | `core/server.go` | 峰值并发可能超过 `max_connections` | -| ARCH-006 | `max_connections` 未覆盖所有会产生连接的路径，DVR 没有 `AcquireConn` | `module/dvr/module.go`、`README.md` | 限流语义不一致，DVR 可绕过全局容量保护 | -| ARCH-007 | HTTP-FLV/TS/fMP4/WebSocket 播放未统一使用 generation-aware subscriber admission | `module/httpstream/handler.go`、`ws_handler.go` | publisher 替换期间可能跨 generation 计数或绕过 per-stream subscriber limit | -| ARCH-008 | `ring_buffer_size=0` 通过 Go validation，但 RingBuffer 取模时可除零/panic | `config/validate.go`、`pkg/util/ringbuffer.go` | 错误配置导致进程崩溃而不是启动期拒绝 | -| ARCH-009 | GB28181 DeviceRegistry 对外暴露可变 `*Device` 和 `Channels` map | `module/gb28181/device_registry.go`、`api.go` | Keepalive/Catalog 更新与 API 读取可能 data race 或观察到半更新状态 | -| ARCH-010 | HTTP streaming 没有清晰的写超时、读 header 超时和慢消费者断开策略 | `module/httpstream/module.go`、`handler.go` | 客户端不读或网络异常时 goroutine、连接和 buffer 可能长时间占用 | -| ARCH-011 | HLS/LL-HLS 阻塞等待使用 `time.Sleep`，没有绑定 request cancellation | `module/httpstream/handler_hls.go` | 客户端断开后请求仍可能等待到超时，浪费 goroutine 和调度时间 | -| ARCH-012 | HLS/DASH/LL-HLS manager cleanup 只按 stream key，不按 publisher generation | `module/httpstream/module.go` | 旧异步 destroy 事件可能删除新 generation 的 manager | -| ARCH-013 | 多处异步 lifecycle event 错误被忽略，背压时 stop/cleanup 事件可能丢失 | 各协议模块 lifecycle 调用点 | 录制、DVR、审计和监控可能与实际 session 状态不一致 | -| ARCH-014 | 配置 URL 中的账号密码可能绕过仅按字段名的脱敏逻辑 | `module/api/config.go` | desired/effective 文档或错误响应可能泄漏 source credentials | -| ARCH-015 | 限流器信任可伪造的 `X-Forwarded-For`/`X-Real-IP` | `pkg/ratelimit/ratelimit.go` | 未配置可信代理时攻击者可绕过 IP 限流 | -| ARCH-016 | `DeviceRegistry.Stop` 和 `ratelimit.Limiter.Close` 非幂等 | 对应模块的 `Stop`/`Close` | 重复 shutdown 或失败回滚可能 panic/重复 close | -| ARCH-017 | WHEP feed loop 的媒体错误和首帧门控状态没有统一的可观测状态模型 | `module/webrtc/whep_feed.go`、`track_sender.go` | 浏览器只能看到笼统的 watchdog 错误，诊断依赖猜测 | +| PERF-001 | 部分缓解 | 协议热路径使用稳定 publisher ID，统计写入改成 atomic；Apple M1 Pro、Go 1.26.0 的真实 `WriteFrameForPublisher` + ring + 交错 GOP fixture 为 65.86-67.28 ns/op、29 B/op、0 alloc/op；它使用共享只读 payload 的预分配 64 秒单调时间戳帧池，零 subscriber，关闭 bitrate limit，保留 2 个 GOP，单 GOP 上限 300 帧，ring 为 4096 项；该路径覆盖 publisher 身份、ring 和 GOP，不与旧的直接 `BenchmarkStreamWriteFrame` 数字比较 | 媒体信息、GOP 和 ring 写入仍由 stream 单写者锁保证顺序；启用 `max_bitrate_per_stream` 的额外成本见 PERF-003；多 publisher 争用不是正常单流拓扑，真实多流/多订阅者容量仍需负载测试 | +| PERF-002 | 已关闭 | 正常协议 publisher 的每帧 identity 校验不再 reflection；仅空 ID 的 legacy/test publisher 回退到反射比较 | 不应让生产 adapter 使用空 publisher ID | +| PERF-003 | 部分缓解 | 每帧 stats 更新不再等待窗口锁 | 启用 `max_bitrate_per_stream` 时仍会在每帧读取完整 snapshot 和时钟，后续可改成周期更新的原子 bitrate | +| PERF-004 | 未关闭 | 共享 transcode track 已做引用计数，但 reader/goroutine 数仍随独立消费者增长 | 大量不同输出/订阅者仍需内存和 goroutine 容量测试 | +| PERF-005 | 部分缓解 | SIP/GB28181 RTP 改用 session-owned marshal buffer，分别降到 264 B/3 alloc 和 1880 B/6 alloc 每测试帧 | packetizer fragment 分配与每 packet UDP syscall 仍在，批量发送需按平台验证 | +| PERF-006 | 部分缓解 | source I/O 保持串行；相同 source version 或相同 hash 会跳过 diff/application，snapshot 读取为原子且约 0.54ns、0 alloc | 后端仍返回完整变化文档时必须解析/hash，大文档高频刷新仍可能排队 | +| PERF-007 | 部分缓解 | per-stream Prometheus series 默认关闭；无 allowlist 时 Collector 按创建顺序进行生命周期接纳，容量满后不驱逐标量 key，重复 gather 与并发 race 回归证明 churn 不会产生超过 limit 的新 `stream_key`；exact allowlist 仍只允许配置键并在 Collector 创建时去重排序；Apple M1 Pro、128 个活跃流、limit 32 的 Gather-only 微基准中，首次接纳为 126892-127899 ns/op、169326 B/op、2683 allocs/op，稳定 Gather 为 133365-136777 ns/op、164580-164581 B/op、2667 allocs/op | 较大 limit 或较大 exact allowlist 的 cardinality 与采集成本仍由部署方承担；这些数字只描述单机固定 fixture 的 Collector Gather 路径，不能作为 stream、scrape、并发或部署容量结论 | +| PERF-008 | 未关闭 | 同机生产 egress fixture 覆盖完整 RTMP FLV/chunk framing、RTSP H.264 packetizer/RTP/interleaved framing 和 relay accounting：RTMP H.264 155.1-155.6 ns/op、24 B/op、3 allocs/op，RTMP AAC 73.60-73.76 ns/op、21 B/op、3 allocs/op，RTSP 单 NAL H.264 1.825-1.833 us/op、4044 B/op、9 allocs/op，三包 FU-A H.264 4.593-4.605 us/op、9892 B/op、23 allocs/op；relay first/batch/threshold/terminal 分别为 7.700-7.751、6.256-6.289、31.50-31.52、7.765-7.792 ns/op 且均为 0 alloc | 两种 egress 都使用固定时间戳媒体帧并终止于有界内存 writer，不含 socket write、deadline、TCP writev 和内核/网络 syscall；RTMP 按 payload、RTSP 按 framed bytes 统计，独立 accounting 数字排除 context lookup，主要用于 allocation 回归；RTSP packetization/marshal 分配仍明显，仍需真实连接、并发订阅和背压负载测试 | + +## 架构与可靠性风险处置状态 + +| ID | 状态 | 处置 | +| --- | --- | --- | +| ARCH-001 | 已关闭 | GOP cache 增加单 GOP 帧数、持续时间和 payload 字节上限，保留关键帧与可播放交错前缀 | +| ARCH-002 | 已关闭 | RTP receiver 在进入重排队列前取得 payload 所有权，并有 buffer alias 回归测试 | +| ARCH-003 | 已关闭 | idle/no-publisher timeout 通过带 instance/generation 的 callback 从 StreamHub 删除匹配对象 | +| ARCH-004 | 已关闭 | HTTP 注册表改为 stream key/instance/generation 元数据，不保留历史 Stream 指针 | +| ARCH-005 | 已关闭 | `AcquireConn` 使用 CAS 严格接纳，并通过并发测试验证不超限 | +| ARCH-006 | 已关闭 | DVR handler 在 session 前申请全局连接配额，成功、错误、客户端取消和写超时路径均 release-once；有限 playlist/segment 响应使用 10 秒 server `WriteTimeout`，stalled peer 不能无限持有槽位 | +| ARCH-007 | 已关闭 | HTTP-FLV/TS/fMP4/WebSocket、RTMP、RTSP、SRT subscriber 均绑定一个 startup generation lease | +| ARCH-008 | 已关闭 | typed config 拒绝非正 ring size，工具层构造函数对非法容量使用一槽 fallback | +| ARCH-009 | 已关闭 | DeviceRegistry 对外返回深拷贝 snapshot，内部 channel map 不再逃逸 | +| ARCH-010 | 已关闭 | HTTP server 配置 header/idle timeout；请求入口不提前设置 write deadline，HLS/DASH 等待完成后才在 manifest/init/segment 实际写入前刷新 10 秒期限；HTTP-FLV/TS/fMP4 每次 write/flush 与 WebSocket 每次 write 同样使用逐次期限，stream loop 响应 request cancellation | +| ARCH-011 | 已关闭 | HLS/DASH/LL-HLS 等待使用 context-aware timer/condition，客户端取消立即退出 | +| ARCH-012 | 已关闭 | manager/muxer cleanup 校验 stream instance 和 publisher generation，旧事件不能删除替代 generation | +| ARCH-013 | 已关闭 | lifecycle start 在 EventBus admission 成功后才标记 started；失败回滚资源；stop lane 按 consumer 的全部 terminal hooks 预留，shutdown 有界 drain | +| ARCH-014 | 已关闭 | Config 文档、source details 和 runtime last error 脱敏 URL userinfo/query/fragment；secret map/sequence 保留结构，token/ICE/endpoint 集合按稳定身份恢复，增删不能错配密文，身份歧义拒绝 Apply；编辑 URL 只恢复旧 secret 组件 | +| ARCH-015 | 已关闭 | 默认忽略 forwarded headers；仅可信代理 IP/CIDR 可提供 client IP；XFF 从右向左剥离可信跳点并选择首个不可信来源，攻击者左前缀不能切换限流桶；非法配置启动期拒绝 | +| ARCH-016 | 已关闭 | DeviceRegistry、Limiter 和 Server shutdown 使用 once/幂等关闭语义 | +| ARCH-017 | 已关闭 | WHEP 对非零且接收方向的每条请求源轨 fail closed：媒体级方向优先并继承会话级方向，codec 只精确匹配该 m-line 列出的 payload `rtpmap`；codec 不兼容返回 415，内部 track/AddTrack 失败返回 500，并释放 generation lease、连接槽、PeerConnection 和 session；禁用/非接收 m-line 可有意省略且不增加 dropped。状态区分 waiting/playing/no-input/media-stalled/codec/write/generation/closed，公开 expected 音视频、首个成功样本时间及固定等待毫秒数、分轨最后推进时间、实际 RTP 包/字节和收到的 RTCP 包；真实状态迁移写一次带上下文的结构化日志，同状态逐帧更新不写；Close 在终态记录前捕获一次最终单调 transport snapshot；混合流全部期望轨推进后才 playing，视频首个 IDR 前即使音频推进也保持 waiting-keyframe，启动后任一轨 8 秒不推进才 stalled，全部恢复才 playing，Console 仅显示实际过期轨；原子终态拒绝普通迟到媒体/watchdog/transport 更新，feed 终止自动释放全部资源，最多 64 条终态保留两分钟 | +| ARCH-018 | 已关闭 | 录像轮转保留 publisher 声明轨道和深拷贝的最新音视频序列头，按轨道归零文件内时间轴；TS 首媒体前写 PAT/PMT，经典 MP4 独立计算音视频 duration、将 `mvhd/tkhd` 归一到 movie timescale、保留 `mdhd` 轨道 timescale、边界值饱和而不回绕、负 PTS-DTS 使用 `ctts` version 1、非负保持 version 0，并使用可扩展 AAC ESDS 长度；逐格式解析回归覆盖，超长单文件仍需保留轮转 | +| ARCH-019 | 已关闭 | Server info 公开当前进程真实音频转码能力；Console fMP4 根据有效输出 codec 而非 G.711 源 codec 创建 MSE SourceBuffer，避免含 AAC 初始化段被视频-only MIME 拒绝 | +| ARCH-020 | 已关闭 | SIP receive 将所选 PCMA/PCMU 作为真实协商目标；源 codec 不同时使用 generation 绑定的独立目标音频 reader，H.264 保持原始 live cursor，并在无可用转换时信令前失败 | +| ARCH-021 | 已关闭 | GB28181 live/playback 成功 INVITE 将托管 dialog 交给 MediaSession；停止、receiver failure 和回滚汇聚到一次 BYE/Close，重复停止幂等 | +| ARCH-022 | 已关闭 | publisher identity 匹配要求流仍处于 publishing 且当前 publisher 非空；旧 `lastPublisherID` 不能重复解绑 generation 或重置 no-publisher timer | +| ARCH-023 | 已关闭 | SIP Gateway RTP/RTCP pair 在 allocator 锁内完成双 socket 绑定，跳过外部占用，并从 SDP 协商前持有到 session cleanup；本地 Lab 假端点避开配置范围，消除编号分配到实际 bind 之间的 TOCTOU | +| ARCH-024 | 已关闭 | GB28181 入站 INVITE 在 2xx 前完成异步 publish-start admission，backpressure 回滚 publisher/session/socket/stream/port 且不发未配对 stop；GB28181 receive Lab 原子分配并绑定 RTP/RTCP，SIP/GB 一键自测也实际绑定配置 pair 并检测外部端口耗尽 | +| ARCH-025 | 已关闭 | HLS/DASH/LL-HLS publish-stop 仅退休匹配 generation 的请求查找，manager 排空捕获的 generation end cursor 后一次完成；替代 generation 使用独立 manager 且无帧串入。LL-HLS 条件等待同时响应 request/hold 取消和 manager stop，HTTP 模块关闭仍强制停止并等待 active/draining worker | +| ARCH-026 | 已关闭 | DVR publish admission 将一次校验通过的 publisher-generation snapshot 贯穿索引/存储恢复和 session 构造，安装前再次校验相同 stream generation；设置期间替代 publisher 会丢弃候选并只关闭候选取得的资源，不能组合旧 identity 与新 media，也不能覆盖新 session | +| ARCH-027 | 已关闭 | 非空 publisher ID 在单个 Stream 生命周期内只能使用一次；A -> B -> A 在修改 timer/state/generation/media/GOP/ring 前拒绝，旧 A 的延迟写入、活动和清理不能命中新 owner；身份集合不跨 Stream 保存 | +| ARCH-028 | 已关闭 | GOP 帧数、时长和字节上限热更新会从保留前缀清除并重新计算当前 seal；收紧保持关键帧开头的可播放前缀，放宽只接纳后续交错帧，不恢复已省略或裁剪历史，下一个关键帧开始完整新 GOP | +| ARCH-029 | 已关闭 | `Destroying` 是 Stream 的不可逆终态；显式关闭、idle/no-publisher timeout 和策略触发销毁后的延迟 publisher 清理不能恢复 `NoPublisher`，不能挂接非空或空 ID publisher，也不会重复销毁通知或重开 ring | +| ARCH-030 | 已关闭 | 共享转码输出使用携带 source-ring `SourceSpan` 的内部 envelope；snapshot reader 以 reader-local `SourceCursor` floor 按 `Begin >= floor` 过滤同 epoch 历史，跨 floor packet 也丢弃；track 按值保留最近 8 个 epoch 的目标序列头，lagging bridge 在首个可接受 payload 前只补发相同 epoch 的头，匹配头已超出边界时丢弃 AAC payload 而不把 miss 视为满足；直接帧保持原指针/载荷，decode/resample/PCM 聚合/encode/drain 保留有效保守归因 | +| ARCH-031 | 已关闭 | 共享转码、连续 HTTP/WebSocket、HLS/LL-HLS、DASH、SIP、GB28181、WHEP、RTMP、RTSP、SRT、cluster、Record 和 DVR 均已使用 generation-aware reader 与 fail-closed overwrite 处理；各协议丢弃 retained post-gap media，视频等待同代参数集加关键帧，纯音频按 live media 恢复，终态不 clean flush。验证覆盖 focused/race 测试、协议实验室与带浏览器的跨协议矩阵。 | +| ARCH-032 | 已关闭 | GOP duration admission/热更新使用溢出安全的无序 min/max DTS span，保留插入顺序并在越界前封存；启用 GOP 时至少要求正的帧数或字节硬上限，直接构造缺失硬上限时使用 300 帧防御默认，避免等 DTS 帧导致无界增长 | +| ARCH-033 | 已关闭 | API、WebRTC 和 metrics HTTP server 统一配置 `ReadHeaderTimeout=5s`、`IdleTimeout=2m`，慢 header 在限流前有界、空闲 keep-alive 连接不会长期占用资源；现有 handler/media write deadline 和行为保持不变，三模块 focused tests 已验证 | +| ARCH-034 | 已关闭 | EventBus lifecycle lane 使用有界队列和 admission；Alive 与通知 WebSocket 使用有界发送队列，并暴露 dropped/pressure 计数，避免每个异步事件无限创建 goroutine | +| ARCH-035 | 已关闭 | file、HTTP/HTTPS、Consul 和 Redis source 统一使用默认 4 MiB 上限；网络/file 在解析前受限，Redis hash/prefix 使用长度预检和有界批量 materialization | +| ARCH-036 | 已关闭 | SIP 出站在最终发送前 fail closed 处理 packetize、空输出、nil packet、marshal、UDP write 和 short write 错误，并将有界错误保存在 call status 的 `last_error` | +| ARCH-037 | 已关闭 | SIP Gateway 与 GB28181 Protocol Lab 使用显式 active-session ceiling，超限在资源分配和信令前拒绝；终态记录保留有界，停止与清理幂等 | + +## Config、Storage 与 Record 已验证项 + +| ID | 等级/状态 | 当前问题 | +| --- | --- | --- | +| CONFIG-001 | 已关闭 | viewer Validate 不展开服务端进程环境变量；受信任的 runtime source 加载保留环境变量展开 | +| CONFIG-002 | 已关闭 | viewer Validate 只接受单个 YAML/JSON 文档并拒绝未知 root/nested typed fields；Apply/source loading 对未映射 source 字段保持宽松 | +| CONFIG-003 | 已关闭 | schema secrets、`api_key`、`tls.key_file` 和 URL path-token 已不透明脱敏并稳定恢复；原始 source 中即使路径符合 digest marker 语法也会再次 hash，只有 candidate 将该语法视为占位符且必须匹配当前原始路径计算出的 digest；未映射字段中的有效 absolute hierarchical URL scalar 也按 value 识别，reorder 按 stable digest/public identity 恢复且 edit/ambiguity fail closed；普通 string、duration、ID、bare host/address 保持不变；Consul GET/PUT 拒绝 redirect 且不转发 `X-Consul-Token` | +| CONFIG-004 | 已关闭 | Redis document 与可选 version increment 在一个 `MULTI/EXEC` 事务中排队；事务/EXEC 错误由 `RedisSource.Write` 返回，Apply 只有写入成功才返回 202 | +| CONFIG-005 | 已关闭 | FileSource 新目标明确使用 `0600`，原有文件替换时保留既有 permission bits；`TestFileSourceWriteUsesPrivateModeForNewDocument` 覆盖新文件路径 | +| CONFIG-006 | 已关闭 | Consul/Redis flattened dotted/slashed key 先规范化、排序，再拒绝重复路径和 scalar/container 前缀冲突；错误顺序由测试固定 | +| CONFIG-007 | 已关闭 | Console Apply 捕获提交文本和单调 editor revision，过期 desired refresh 只有在 revision 未变化时才能回填；browser race regression 覆盖新编辑优先 | +| CONFIG-008 | 已关闭 | OpenAPI Apply 202 使用 `ConfigApplyResponse` 的 `written_and_refresh_scheduled`，独立 refresh 仍使用 `scheduled`；contract test 校验引用和 schema | +| STORAGE-001 | 已关闭 | Record 与 DVR 都在等待 admission/setup 锁前捕获唯一绝对 drain deadline；调用方在该边界返回 timeout，已经启动的清理继续后台完成 | +| STORAGE-002 | 已关闭 | recording play/download 在打开媒体前申请全局连接槽，每条成功/错误路径 release-once，并在 `ServeContent` 前设置 10 秒写期限；metadata/list/status/delete 不额外占用媒体槽 | +| STORAGE-003 | 已关闭 | 自动轮转在阈值后立即停止时不创建空后继；视频在阈值后首个有效关键帧前轮转，纯音频在阈值后首个音频帧前轮转；无 `{time}` 的固定模板也会为后继生成唯一且排他创建的路径 | +| STORAGE-004 | 已关闭 | audio-only DVR 按音频媒体时间达到 segment duration 时轮转并立即发布分片，publisher 持续在线不依赖视频关键帧；带媒体的分片经过在线 demux 验证 | +| STORAGE-005 | 已关闭 | DVR route 严格拒绝非法/编码分隔符；嵌套 stream key 保持层级，playlist 对每个 key segment 单独 `PathEscape`，保留 `?`、`#`、`%` 不改变资源边界 | +| STORAGE-006 | 已关闭 | Record 与 FileWriter 共用格式和字节大小校验；支持 `flv`、`fmp4`、`mp4`、`ts`，并将 `hls` 作为 TS 存储 alias；空值/零值关闭 max-size，其他值只接受非负十进制 B/KB/MB/GB | +| STORAGE-007 | 已关闭 | `/api/v1/server/info` 从已初始化 DVR 的绑定 listener 发现非零端口，并通过 `endpoint_schemes.dvr` 报告实际 `http`/`https` scheme；未初始化时才回退配置值 | +| STORAGE-008 | 已关闭 | recording download 只服务 `completed`；active 或 failed recording 返回 JSON `409` 且不返回媒体，OpenAPI 与 inline play 保持同一语义 | +| STORAGE-009 | 已关闭 | `!audiocodec` 自动化覆盖 H.264 + G.711 DVR fallback：TS 分片可 demux 为视频、没有音频帧；带 FFmpeg 的构建仍由 tagged transcode 测试覆盖 | ## 功能边界和未完成项 @@ -66,8 +109,8 @@ | FUNC-001 | WebRTC simulcast layer selection 和 automatic layer pausing 未实现 | `stream.simulcast.*` 明确标记 deferred/unsupported，不得宣传为已支持 | | FUNC-002 | 未使用 `audiocodec`/FFmpeg 时，非 AAC 录制和部分输出可能过滤音频并保留纯视频 | 保持可播放视频输出，并在 UI/文档标明构建前提 | | FUNC-003 | SIP 主要覆盖 H.264 + PCMA/PCMU，GB28181 主要覆盖 H.264 + G.711A | 协议实验室和 API 应对不支持 codec fail closed，并展示原因 | -| FUNC-004 | 当前 WebRTC 浏览器回归没有覆盖真实 GB28181/SIP H.264 输入 | 在 WEBRTC-002 关闭前不能把“Pion RTP 测试通过”当作浏览器播放完整证明 | -| FUNC-005 | 各输出协议对同一 stream 的 codec 能力和音频转码前提仍不完全一致 | 需要建立 capability matrix 和跨协议自动化测试，尤其是 G.711/Opus/AAC | +| FUNC-004 | SIP/GB28181/WHIP 已形成统一 Chromium 正确性矩阵，但 Chromium 可缺席或不提供 H.264 接收能力且默认不 soak | 发布门禁必须提供带 H.264 接收能力的 Chromium 并显式运行长时 soak；不能把环境 skip 或短时矩阵当作容量证明 | +| FUNC-005 | G.711A 源已实测 HTTP-FLV/WS-FLV/HTTP-TS/fMP4/HLS/DASH/WHEP；矩阵覆盖 SIP/GB28181/WHIP H.264 和 PCMA/PCMU/G.711A/Opus 的关键转换，其他 codec 组合仍未穷举 | 继续扩展 capability matrix，尤其是 AAC/H.265 和无 FFmpeg fallback | ## `audioCache` 删除后的设计记录 @@ -77,14 +120,25 @@ ## 后续验证顺序 -1. 完成 WEBRTC-001 Phase 1：用控制台真实请求采集 SDP、generation、游标、关键帧、丢弃帧、WriteSample 错误和 RTP 计数。 -2. 为已确认的断点添加最小失败测试，优先覆盖 realtime 模式在快照后等待关键帧、稀疏/无关键帧、以及 H.264 真实 payload。 -3. 修复并验证 WHEP 后，再按 PERF-001/PERF-003/PERF-007 和 ARCH-001/ARCH-002/ARCH-005/ARCH-008 的风险顺序做基准、race 和故障注入。 -4. 关闭功能边界前补齐文档、OpenAPI/schema（若契约变化）、控制台状态和跨协议验收矩阵。 +1. 对已关闭的 ARCH-031 媒体正确性路径继续安排长时、背压和高并发容量验证；这类测试不能由短时正确性回归替代。 +2. 持续检查 Simulcast 层选择这一明确功能边界，并在实现前保持 schema、Console 和 release 文档中的 deferred 标识。 +3. 显式运行 60 秒统一协议 soak，再对 PERF-001/PERF-003/PERF-004/PERF-005/PERF-006 做多 publisher/多 subscriber 长时容量测试；微基准和短时矩阵都不能替代容量测试。 ## 当前验证记录 - `go test ./module/webrtc -run 'WHEP|whep|Browser' -count=1 -v`：通过。 - `go test -tags audiocodec ./module/webrtc -run 'TestWHEPPayloadTypeCorrectness|TestWHEPWithGCC|TestWHEPAudioTranscoding|TestValidVideoRTPDelta' -count=1 -v`：通过。 - `go test -tags audiocodec ./module/webrtc -run TestWHEPBrowserJitterDiagnostic -count=1 -v`：通过；VP8 视频和 VP8+AAC->Opus 场景均有推进帧、无丢包、无冻结。 -- 当前运行实例的 H.264 对照：`lf-test` WHEP `mode=realtime` 与 `mode=live` 的 5 秒帧数分别为 240 和 1395；浏览器 `mode=live` 已观察到 640x360、`readyState=4`、`paused=false` 且 `currentTime` 递增，浏览器 `mode=realtime` 在首个后续关键帧前保持等待。这些结果关闭了“所有 H.264 RTP 都不可解码”的假设，但 WEBRTC-001 仍未关闭，因为默认 Console 行为和真实 GB28181/SIP H.264 浏览器路径仍需修复与覆盖。 +- `go test ./module/record -count=1`、`go test -race ./module/record -count=1`、`CGO_ENABLED=1 go test -tags audiocodec -race ./module/record -count=1`：通过；覆盖 fMP4/FLV/MP4/TS 轮转后的完整轨道初始化。 +- `go test ./module/sipgateway -count=1`、`go test -race ./module/sipgateway -count=1`、`CGO_ENABLED=1 go test -tags audiocodec -race ./module/sipgateway -count=1`：通过；包含 PCMA 源到请求 PCMU 的真实 RTP/RTCP Lab 转码回归。 +- 2026-08-28 独立端口 Console 验收：GB28181 G.711A 源的 HTTP-FLV、WS-FLV、HTTP-TS、fMP4、HLS、DASH、WHEP 均解码为 160x90 且媒体时钟推进；SIP WHEP 同样为 160x90、`readyState=4` 并推进。GB28181->SIP PCMU receive 音频/视频/RTCP 计数增长，SIP PCMA->GB28181 receive 的 RTP/RTCP/PS 发送和接收计数一致。SIP 11 项与 GB28181 13 项一键自测全部通过。 +- 当前运行实例的 H.264 对照：`lf-test` WHEP `mode=realtime` 与 `mode=live` 的 5 秒帧数分别为 240 和 1395；浏览器 `mode=live` 已观察到 640x360、`readyState=4`、`paused=false` 且 `currentTime` 递增，浏览器 `mode=realtime` 在首个后续关键帧前保持等待。新增 fixture 浏览器回归通过，关闭了“所有 H.264 RTP 都不可解码”的假设。 +- 2026-08-29 统一 Chromium 矩阵以 `LIVEFORGE_PROTOCOL_MATRIX_SOAK=15s` 通过三种场景，每个场景约 19 秒；SIP/GB28181 为 160x90，WHIP 为 640x360，全部校验音视频 RTP、decoded frames、媒体时钟、RTCP、ICE、浏览器错误和服务端 stall。随后 `CGO_ENABLED=1 go test -tags audiocodec -race ./module/gb28181 ./tools/testkit/push ./tools/testkit/testutil ./test/integration -count=1` 通过。该结果是短时正确性证据，不是并发/容量结论。 +- WHEP 音频样本写入失败从缓存、直读和转码 reader 三条路径立即终止 feed；连接后 8 秒完全没有输入进入可恢复的 `no_media_input`，首帧后任一期望轨 8 秒不推进进入 `media_stalled`，全部期望轨重新推进后恢复；无效 H.264/H.265 参数集和空访问单元进入 `codec_mismatch`。状态公开首个成功媒体时间和不会被 watchdog/后续帧改写的 `first_media_wait_ms`；媒体热路径使用原子计数/时间戳，基准命令为 `go test ./module/webrtc -run '^$' -bench '^BenchmarkWHEPFeedStatus' -benchmem -count=3`，结果只用于同机回归。 +- WHEP 源 reader 与目标音频 reader 独立消费原子覆盖结果：覆盖后的保留帧不会进入 RTP，只有发生覆盖的 reader 推进到 live。源覆盖会保留原 publisher generation，重置视频 pacing/DTS/PTS 状态，复用 TrackSender 的关键帧门并刷新同 generation 最新参数集；已经建立的直通或转码音频继续推进，纯音频从下一帧 live 音频恢复。目标音频覆盖不扰动干净视频；active generation 中期望目标音频 EOF 会立即进入 `target_audio_failed`，不会等待 8 秒 watchdog 或静默降级。终止路径取消并 join reader waiter，且目标 reader ownership 只释放一次；每次覆盖 warning 只记录 reader 身份、精确覆盖数和恢复动作。 +- `go test ./pkg/muxer/mp4 ./module/gb28181 ./module/httpstream ./pkg/ratelimit ./core ./module/metrics -count=1` 的对应包级回归均通过；覆盖负 CTS、GB28181 回放 BYE、延迟 HLS/DASH 写 deadline、XFF 前缀绕过、稳定有界指标迭代和重复 publisher 清理。 +- `go test ./module/sipgateway -count=5 -timeout=120s`：通过；覆盖外部占用 pair 跳过、SDP 前 socket 绑定、Lab 范围避让和失败清理顺序。 +- 2026-08-28 Apple M1 Pro 微基准：Stream write 54.8-55.0ns/0 alloc；Ring TryRead 37.6-37.7ns/0 alloc；Ring immediate context read 40.1-40.5ns/0 alloc；GB28181 outbound 6.43-6.62us/1880 B/6 alloc；SIP outbound 4.49-4.57us/264 B/3 alloc。结果仅用于同机相对回归。 +- 2026-08-29 Apple M1 Pro Prometheus Collector fixture（128 个活跃流，detail limit 32，3 次运行）：首次接纳 gather 为 157.5-158.1us、约 179.6KB/2822 alloc；接纳满后的 steady gather 为 134.9-136.9us、约 164.6KB/2667 alloc。结果只描述该真实 Collector gather/admission fixture 的分配与延迟，不是容量结论。 +- 2026-08-30 fresh verification：`go test ./...`、重点模块 race、`CGO_ENABLED=1 go test -tags audiocodec -race -coverprofile=coverage.out -covermode=atomic ./...`、agent-doc/schema/diff 检查全部通过；ring reader 为约 40-50ns/0 alloc，核心生产写入为 67.93-71.27ns/0 alloc，RTMP egress 为约 75-164ns/3 alloc，RTSP egress 为约 1.8-5.1us/9-23 alloc，relay accounting 为约 6.3-31.8ns/0 alloc。数字仅作为本机回归基线，不是部署容量承诺。 +- 2026-08-30 `LIVEFORGE_PROTOCOL_MATRIX_SOAK=60s go test -tags audiocodec -race ./test/integration -run '^TestSIPGB28181WHIPBrowserBridgeMatrix$' -count=1 -timeout=8m -v` 通过；三个 SIP/GB28181/WHIP 跨协议 Chromium 场景各自保持约 60 秒媒体推进，WHEP 进入 `playing`，并在结束时完成 SIP、GB28181、WHIP/WHEP 清理。当前本地 Console 页面 smoke 检查也显示正确的 Workspace/Operations/System 分组、完整 Config 文档/schema、两类协议 Lab 入口，浏览器日志无 error/warning。 diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 5dbbe6fb..9f294d71 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -105,7 +105,7 @@ paths: tags: [server] operationId: getRuntimeConfigDocument summary: Read the complete effective and desired configuration document - description: Sensitive values are redacted. Desired document text is retained from the accepted source so comments and unmapped fields remain editable. The writable flag describes whether the selected runtime source supports POST apply. + description: Sensitive values are redacted. Valid absolute hierarchical URL scalars are recognized by value even under unmapped non-URL-shaped keys; safe scheme/host/port identity remains visible while non-root paths, userinfo, query, and fragment are opaque. URL-shaped keys retain TURN/opaque, malformed/hostless fail-closed, and plain-address handling; ordinary strings, durations, IDs, and bare host/address values remain unchanged outside that key policy. Desired document text is retained from the accepted source so comments and unmapped fields remain editable. The writable flag describes whether the selected runtime source supports POST apply. x-liveforge-permission: config:read responses: '200': {description: Configuration documents, content: {application/json: {schema: {$ref: '#/components/schemas/ConfigDocumentResponse'}}}} @@ -128,6 +128,7 @@ paths: tags: [server] operationId: validateRuntimeConfigDocument summary: Validate a complete YAML or JSON configuration document + description: Viewer-accessible validation treats environment references literally without reading the server process environment, accepts exactly one YAML/JSON document, and rejects unknown root or nested typed fields. x-liveforge-permission: config:read requestBody: required: true @@ -145,7 +146,7 @@ paths: tags: [server] operationId: applyRuntimeConfigDocument summary: Persist a complete document to a writable runtime source and schedule refresh - description: The source write completes and is serialized with source loads before the 202 response; parsing, module application, and publication remain on the background manager worker. Listener, module, TLS, port, and audio-codec topology changes remain restart-required. + description: The source write completes and is serialized with source loads before the 202 response; parsing, module application, and publication remain on the background manager worker. Apply and trusted runtime source loading remain permissive for source fields not mapped by the typed runtime struct. Consul KV PUT rejects redirects and never forwards X-Consul-Token. Listener, module, TLS, port, and audio-codec topology changes remain restart-required. x-liveforge-permission: config:reload requestBody: required: true @@ -153,7 +154,7 @@ paths: application/yaml: {schema: {$ref: '#/components/schemas/ConfigYAMLDocument'}} application/json: {schema: {$ref: '#/components/schemas/ConfigJSONDocumentRequest'}} responses: - '202': {description: Document written and refresh scheduled, content: {application/json: {schema: {$ref: '#/components/schemas/ScheduledResponse'}}}} + '202': {description: Document written and refresh scheduled, content: {application/json: {schema: {$ref: '#/components/schemas/ConfigApplyResponse'}}}} '400': {$ref: '#/components/responses/BadRequest'} '401': {$ref: '#/components/responses/Unauthorized'} '403': {$ref: '#/components/responses/Forbidden'} @@ -298,7 +299,7 @@ paths: tags: [sipgateway] operationId: startSIPGatewayLabSession summary: Start a persistent local fake SIP device - description: Publish mode sends real H.264 plus PCMA or PCMU RTP/RTCP into LiveForge receivers. Receive mode accepts LiveForge's outbound INVITE, leaves the selected source stream unchanged, and sends periodic per-track receiver reports. No external PBX is required. + description: Publish mode sends real H.264 plus PCMA or PCMU RTP/RTCP into LiveForge receivers. Receive mode accepts LiveForge's outbound INVITE, leaves the selected source stream unchanged, sends periodic per-track receiver reports, and uses the requested PCMA/PCMU target codec through the optional configured audio transcoder when the source codec differs. The configured active lab-session ceiling is enforced atomically; terminal history does not consume it and an exhausted ceiling returns 429. No external PBX is required. x-liveforge-permission: sip:calls requestBody: required: true @@ -309,7 +310,7 @@ paths: '401': {$ref: '#/components/responses/Unauthorized'} '403': {$ref: '#/components/responses/Forbidden'} '409': {$ref: '#/components/responses/Conflict'} - '429': {$ref: '#/components/responses/RateLimited'} + '429': {$ref: '#/components/responses/ProtocolLabCapacity'} '502': {$ref: '#/components/responses/BadGateway'} '503': {$ref: '#/components/responses/Unavailable'} /api/v1/sipgateway/lab/sessions/{labSessionId}: @@ -400,15 +401,25 @@ paths: get: tags: [recording] operationId: getRecording - summary: Read recording metadata + summary: Read recording metadata or explicitly play/download the full recording ID + description: Without action, an existing exact recordingPath returns metadata. Set action=play or action=download to address that full ID unambiguously, including an ID whose final segment is play or download. Existing exact IDs take precedence over the backward-compatible suffix action forms. Media actions acquire the global connection budget before opening the recording and apply a 10-second write deadline; exhaustion returns 503. Only completed recordings are served; active or failed recordings return 409 with a JSON error and no media body. + parameters: + - name: action + in: query + required: false + description: Explicit media action for the complete recordingPath. + schema: {type: string, enum: [play, download]} x-liveforge-permission: recordings:read responses: - '200': {description: Recording metadata, content: {application/json: {schema: {$ref: '#/components/schemas/RecordingResponse'}}}} + '200': {description: Recording metadata or complete media, content: {application/json: {schema: {$ref: '#/components/schemas/RecordingResponse'}}, application/octet-stream: {schema: {type: string, format: binary}}, video/mp4: {schema: {type: string, format: binary}}, video/x-flv: {schema: {type: string, format: binary}}, video/mp2t: {schema: {type: string, format: binary}}}} + '206': {description: Requested media byte range, content: {application/octet-stream: {schema: {type: string, format: binary}}, video/mp4: {schema: {type: string, format: binary}}, video/x-flv: {schema: {type: string, format: binary}}, video/mp2t: {schema: {type: string, format: binary}}}} + '304': {description: Action response not modified} '400': {$ref: '#/components/responses/BadRequest'} '401': {$ref: '#/components/responses/Unauthorized'} '403': {$ref: '#/components/responses/Forbidden'} '404': {$ref: '#/components/responses/NotFound'} - '409': {$ref: '#/components/responses/Conflict'} + '409': {$ref: '#/components/responses/RecordingNotReady'} + '416': {description: Requested action range is not satisfiable} '429': {$ref: '#/components/responses/RateLimited'} '500': {$ref: '#/components/responses/InternalError'} '503': {$ref: '#/components/responses/Unavailable'} @@ -416,6 +427,7 @@ paths: tags: [recording] operationId: deleteRecording summary: Delete a completed recording + description: Always treats the complete recordingPath as the ID, even when its final segment is play or download. Local cleanup removes exact owned sidecars and metadata before the primary; a cleanup error leaves the primary available for retry. x-liveforge-permission: recordings:delete responses: '200': {$ref: '#/components/responses/Success'} @@ -434,6 +446,7 @@ paths: tags: [recording] operationId: downloadRecording summary: Download a completed recording with HTTP range support + description: Backward-compatible suffix action used only when no exact recording ID includes the final /download segment. Use action=download on /api/v1/recordings/{recordingPath} for an unambiguous full-ID request. Only completed recordings are served; active or failed recordings return 409 with a JSON error and no media body. The media response acquires the global connection budget before opening the recording and applies a 10-second write deadline; exhaustion returns 503. x-liveforge-permission: recordings:read responses: '200': {description: Complete recording, content: {application/octet-stream: {schema: {type: string, format: binary}}}} @@ -443,7 +456,7 @@ paths: '401': {$ref: '#/components/responses/Unauthorized'} '403': {$ref: '#/components/responses/Forbidden'} '404': {$ref: '#/components/responses/NotFound'} - '409': {$ref: '#/components/responses/Conflict'} + '409': {$ref: '#/components/responses/RecordingNotReady'} '416': {description: Requested range is not satisfiable} '429': {$ref: '#/components/responses/RateLimited'} '500': {$ref: '#/components/responses/InternalError'} @@ -455,7 +468,7 @@ paths: tags: [recording] operationId: playRecording summary: Stream a completed recording inline with HTTP range support - description: Returns the recording with a media MIME type and Content-Disposition inline. The endpoint accepts HTTP Range requests for browser seeking; active or not-ready recordings return 409. + description: Backward-compatible suffix action used only when no exact recording ID includes the final /play segment. Use action=play on /api/v1/recordings/{recordingPath} for an unambiguous full-ID request. Returns the recording with a media MIME type and Content-Disposition inline. The endpoint accepts HTTP Range requests for browser seeking; active or failed recordings return 409 with a JSON error and no media body. The media response acquires the global connection budget before opening the recording and applies a 10-second write deadline; exhaustion returns 503. x-liveforge-permission: recordings:read responses: '200': {description: Complete recording media, content: {video/mp4: {schema: {type: string, format: binary}}, video/x-flv: {schema: {type: string, format: binary}}, video/mp2t: {schema: {type: string, format: binary}}}} @@ -465,7 +478,7 @@ paths: '401': {$ref: '#/components/responses/Unauthorized'} '403': {$ref: '#/components/responses/Forbidden'} '404': {$ref: '#/components/responses/NotFound'} - '409': {$ref: '#/components/responses/Conflict'} + '409': {$ref: '#/components/responses/RecordingNotReady'} '416': {description: Requested range is not satisfiable} '429': {$ref: '#/components/responses/RateLimited'} '500': {$ref: '#/components/responses/InternalError'} @@ -506,15 +519,16 @@ paths: tags: [dvr] operationId: getDVRPlaylist summary: Read a DVR HLS playlist - description: The configured DVR media listener authorizes this request through synchronous subscribe hooks. It returns non-credentialed CORS headers for a split-port browser Console. + description: The configured DVR media listener authorizes this request through synchronous subscribe hooks. It returns non-credentialed CORS headers for a split-port browser Console. The key may contain nested slash-separated stream-key segments; each segment is escaped independently in playlist URIs. Audio-only sessions publish a playlist as soon as audio reaches the segment duration and do not wait for a video keyframe. parameters: - {name: app, in: path, required: true, schema: {type: string}} - - {name: key, in: path, required: true, schema: {type: string}} + - {name: key, in: path, required: true, description: URL-decoded nested stream-key suffix; each path segment is escaped independently. Encoded separators, dot segments, and backslashes are rejected., schema: {type: string}} responses: '200': {description: HLS playlist, content: {application/vnd.apple.mpegurl: {schema: {type: string}}}} '401': {description: Subscribe authentication required} '403': {description: Subscribe authorization denied} '404': {description: Session or playlist not found} + '503': {description: Global connection limit reached} /dvr/{app}/{key}/{filename}: servers: - url: http://127.0.0.1:8070 @@ -523,16 +537,17 @@ paths: tags: [dvr] operationId: getDVRSegment summary: Read a DVR HLS segment - description: The configured DVR media listener authorizes this request through synchronous subscribe hooks and returns non-credentialed CORS headers for a split-port browser Console. + description: The configured DVR media listener authorizes this request through synchronous subscribe hooks and returns non-credentialed CORS headers for a split-port browser Console. Nested stream-key segments remain part of the key, while `?`, `#`, and `%` are escaped per segment so they cannot change the resource boundary. parameters: - {name: app, in: path, required: true, schema: {type: string}} - - {name: key, in: path, required: true, schema: {type: string}} + - {name: key, in: path, required: true, description: URL-decoded nested stream-key suffix; each path segment is escaped independently. Encoded separators, dot segments, and backslashes are rejected., schema: {type: string}} - {name: filename, in: path, required: true, schema: {type: string}} responses: '200': {description: MPEG-TS segment, content: {video/mp2t: {schema: {type: string, format: binary}}}} '401': {description: Subscribe authentication required} '403': {description: Subscribe authorization denied} '404': {description: Segment not found} + '503': {description: Global connection limit reached} /api/v1/gb28181/devices: get: tags: [gb28181] @@ -729,7 +744,7 @@ paths: tags: [gb28181] operationId: startGB28181LabSession summary: Start a persistent local fake GB28181 device - description: Publish mode performs REGISTER, Keepalive, Catalog, accepts LiveForge's server-initiated INVITE/ACK/BYE, and sends deterministic H.264 plus G.711A in PS/RTP with RTCP to LiveForge's receiver. Receive mode validates an H.264/G.711A source and admits its subscriber before activation, then uses a module-owned outbound session to send PS/RTP/RTCP to the fake device. Subscriber admission rejection fails startup synchronously; a later outbound media failure transitions the Lab to failed with a bounded redacted last_error and releases its signaling, session, subscriber, sockets, and ports. No external GB28181 platform is required. + description: Publish mode performs REGISTER, Keepalive, Catalog, accepts LiveForge's server-initiated INVITE/ACK/BYE, and sends deterministic H.264 plus G.711A in PS/RTP with RTCP to LiveForge's receiver. Receive mode validates H.264 plus direct G.711A or audio that the tagged runtime can convert to G.711A and admits its subscriber before activation, then uses a module-owned outbound session with direct H.264 and an independent generation-bound target-audio reader to send PS/RTP/RTCP to the fake device; unavailable conversion fails before signaling. The configured active lab-session ceiling is enforced atomically; terminal history does not consume it and an exhausted ceiling returns 429. Subscriber admission rejection fails startup synchronously; a later outbound media failure transitions the Lab to failed with a bounded redacted last_error and releases its signaling, session, subscriber, sockets, and ports. No external GB28181 platform is required. x-liveforge-permission: gb28181:control requestBody: required: true @@ -740,7 +755,7 @@ paths: '401': {$ref: '#/components/responses/GBUnauthorized'} '403': {$ref: '#/components/responses/GBForbidden'} '409': {$ref: '#/components/responses/Conflict'} - '429': {$ref: '#/components/responses/GBRateLimited'} + '429': {$ref: '#/components/responses/ProtocolLabCapacity'} '502': {$ref: '#/components/responses/BadGateway'} '503': {$ref: '#/components/responses/Unavailable'} /api/v1/gb28181/lab/sessions/{labSessionId}: @@ -869,7 +884,7 @@ paths: operationId: playWHEP summary: Play with a WHEP SDP offer security: [{streamBearerAuth: []}, {streamToken: []}, {}] - requestBody: {required: true, description: SDP offer body; limited to 1 MiB (1048576 bytes)., content: {application/sdp: {schema: {type: string}}}} + requestBody: {required: true, description: SDP offer body; limited to 1 MiB (1048576 bytes). Every source media kind requested by a non-zero receiving m-line must negotiate. Media direction inherits the session-level direction when absent; codecs require an exact rtpmap name on a payload listed by that m-line. Unsupported requested codecs fail with 415, while disabled or non-receiving m-lines are intentionally omitted., content: {application/sdp: {schema: {type: string}}}} responses: '201': {$ref: '#/components/responses/SDPAnswer'} '400': {$ref: '#/components/responses/MediaBadRequest'} @@ -924,6 +939,25 @@ paths: responses: '204': {description: CORS preflight accepted} '429': {$ref: '#/components/responses/MediaRateLimited'} + /webrtc/session/{sessionId}/status: + servers: + - url: '{scheme}://127.0.0.1:8443' + description: Local WebRTC signaling listener; the checked-in sample uses plain HTTP. + variables: + scheme: {default: http, enum: [http, https], description: Select https when WebRTC TLS is enabled.} + parameters: + - {$ref: '#/components/parameters/SessionId'} + get: + tags: [webrtc] + operationId: getWebRTCSessionStatus + summary: Read WHEP media startup diagnostics + description: Returns the generation, cursor, startup mode, media counters, actual RTP packet/byte and received RTCP packet counters, keyframe gate, and bounded terminal error for an active or recently closed WHEP session. Closed status is retained in a bounded short-lived tombstone. + security: [] + responses: + '200': {description: WHEP feed status, content: {application/json: {schema: {$ref: '#/components/schemas/WHEPSessionStatusResponse'}}}} + '404': {$ref: '#/components/responses/MediaNotFound'} + '409': {description: Session does not own a WHEP feed} + '429': {$ref: '#/components/responses/MediaRateLimited'} components: securitySchemes: bearerAuth: {type: http, scheme: bearer, bearerFormat: opaque management token} @@ -946,6 +980,8 @@ components: Forbidden: {description: Principal lacks the required permission, content: {application/json: {schema: {$ref: '#/components/schemas/ApiError'}}}} NotFound: {description: Resource not found, content: {application/json: {schema: {$ref: '#/components/schemas/ApiError'}}}} Conflict: {description: Resource state conflicts with the operation, content: {application/json: {schema: {$ref: '#/components/schemas/ApiError'}}}} + RecordingNotReady: {description: Recording is active or failed and has no downloadable media; the response is JSON and never contains the media body, content: {application/json: {schema: {$ref: '#/components/schemas/ApiError'}}}} + ProtocolLabCapacity: {description: Configured active SIP or GB28181 protocol-lab session ceiling reached; terminal history is not counted, content: {application/json: {schema: {$ref: '#/components/schemas/ApiError'}}}} RateLimited: {description: Per-IP request limit exceeded, content: {application/json: {schema: {$ref: '#/components/schemas/ApiError'}}}} Unprocessable: {description: Valid JSON but missing target data or a compatible codec, content: {application/json: {schema: {$ref: '#/components/schemas/ApiError'}}}} BadGateway: {description: Upstream SIP call setup failed; internal details are redacted, content: {application/json: {schema: {$ref: '#/components/schemas/ApiError'}}}} @@ -986,10 +1022,16 @@ components: allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {type: object, properties: {status: {type: string, const: healthy}}}}}] ScheduledResponse: allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {type: object, properties: {status: {type: string, const: scheduled}}}}}] + ConfigApplyResponse: + allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {type: object, properties: {status: {type: string, const: written_and_refresh_scheduled}}}}}] ServerInfo: type: object - required: [version, uptime_sec, modules] - properties: {version: {type: string}, uptime_sec: {type: integer, format: int64}, modules: {type: array, items: {type: string}}, endpoints: {type: object, additionalProperties: {type: string}}} + required: [version, uptime_sec, modules, capabilities] + properties: {version: {type: string}, uptime_sec: {type: integer, format: int64}, modules: {type: array, items: {type: string}}, endpoints: {type: object, additionalProperties: {type: string}, description: Bound host:port values for listener-backed endpoints when initialized; otherwise configured values.}, endpoint_schemes: {type: object, additionalProperties: {type: string, enum: [http, https]}, description: Transport scheme for HTTP/WebRTC/DVR endpoints; the DVR value is required to build media URLs when its listener is TLS-enabled.}, capabilities: {$ref: '#/components/schemas/ServerCapabilities'}} + ServerCapabilities: + type: object + required: [audio_transcoding] + properties: {audio_transcoding: {type: boolean, description: True only when audio transcoding is configured and this process can transcode both G.711 A-law and mu-law to AAC.}} ServerInfoResponse: {allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {$ref: '#/components/schemas/ServerInfo'}}}]} ServerStats: {type: object, required: [streams, connections], properties: {streams: {type: integer}, connections: {type: integer, format: int64}}} ServerStatsResponse: {allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {$ref: '#/components/schemas/ServerStats'}}}]} @@ -1030,7 +1072,7 @@ components: desired: {type: object, additionalProperties: true} effective_document: {type: string} desired_document: {type: string} - source_details: {type: object, additionalProperties: true, description: Credentials are omitted.} + source_details: {type: object, additionalProperties: true, description: Credentials are omitted; URL userinfo/query/fragment and non-root paths are opaque while safe host/address identity remains visible. Consul KV GET and PUT reject redirects and never forward X-Consul-Token.} schema: {type: object, additionalProperties: true, description: The complete versioned docs/config/config.schema.json object.} writable: {type: boolean} ConfigDocumentResponse: {allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {$ref: '#/components/schemas/ConfigDocument'}}}]} @@ -1057,8 +1099,8 @@ components: properties: {legacy_bearer_configured: {type: boolean}, tokens: {type: array, items: {type: object, properties: {name: {type: string}, role: {type: string}}}}, console_configured: {type: boolean}, console_role: {type: string}, audit_enabled: {type: boolean}, audit_entries: {type: integer}, audit_events_total: {type: integer, format: int64}} SecurityStatusResponse: {allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {$ref: '#/components/schemas/SecurityStatus'}}}]} SIPDialRequest: {type: object, additionalProperties: false, required: [target_uri, stream_key], properties: {target_uri: {type: string}, stream_key: {type: string}}} - SIPLabRequest: {type: object, additionalProperties: false, required: [mode, device_id, stream_key, codec], properties: {mode: {type: string, enum: [publish, receive]}, device_id: {type: string, maxLength: 128, pattern: '^[A-Za-z0-9._-]+$', description: 'Letters, digits, hyphens, underscores, and periods only.'}, stream_key: {type: string, maxLength: 256, pattern: '^(?!/)(?!.*\/$)(?!.*//)(?!\.{1,2}(?:/|$))(?!.*\/\.{1,2}(?:/|$))[\x21-\x7E]+$', description: 'Printable ASCII without whitespace; slash-separated segments must be non-empty and cannot be dot or dot-dot segments.'}, codec: {type: string, enum: [PCMA, PCMU]}}} - ProtocolLabPlayback: {type: object, required: [stream_key, available], description: Playback paths URL-escape each stream-key segment; absolute RTMP/RTSP URLs use bound listener discovery and normalize wildcard bind hosts to the management request host., properties: {stream_key: {type: string}, available: {type: boolean}, rtmp: {type: string}, rtsp: {type: string}, http_flv: {type: string}, ws_flv: {type: string}, http_ts: {type: string}, fmp4: {type: string}, hls: {type: string}, dash: {type: string}, whep: {type: string}, whep_live: {type: string}}} + SIPLabRequest: {type: object, additionalProperties: false, required: [mode, device_id, stream_key, codec], properties: {mode: {type: string, enum: [publish, receive]}, device_id: {type: string, maxLength: 128, pattern: '^[A-Za-z0-9._-]+$', description: 'Letters, digits, hyphens, underscores, and periods only.'}, stream_key: {type: string, maxLength: 256, pattern: '^(?!/)(?!.*\/$)(?!.*//)(?!\.{1,2}(?:/|$))(?!.*\/\.{1,2}(?:/|$))[\x21-\x7E]+$', description: 'Printable ASCII without whitespace; slash-separated segments must be non-empty and cannot be dot or dot-dot segments.'}, codec: {type: string, enum: [PCMA, PCMU], description: 'Publish source codec or receive target codec. Receive requires direct compatibility or the configured audiocodec runtime.'}}} + ProtocolLabPlayback: {type: object, required: [stream_key, available], description: Playback paths URL-escape each stream-key segment; absolute RTMP/RTSP URLs use bound listener discovery and normalize wildcard bind hosts to the management request host. WHEP and WHEP Live use mode=live; WHEP Realtime uses mode=realtime., properties: {stream_key: {type: string}, available: {type: boolean}, rtmp: {type: string}, rtsp: {type: string}, http_flv: {type: string}, ws_flv: {type: string}, http_ts: {type: string}, fmp4: {type: string}, hls: {type: string}, dash: {type: string}, whep: {type: string}, whep_live: {type: string}, whep_realtime: {type: string}}} SIPLabSession: {type: object, properties: {id: {type: string}, identity: {type: string}, device_id: {type: string}, stream_key: {type: string}, mode: {type: string, enum: [publish, receive]}, state: {type: string, enum: [starting, active, contract, stopped, failed]}, direction: {type: string, enum: [inbound, outbound]}, codec: {type: string}, last_error: {type: string, description: Bounded redacted setup or terminal error}, rtp_packets_sent: {type: integer, format: int64}, rtp_packets_received: {type: integer, format: int64}, audio_rtp_packets_sent: {type: integer, format: int64}, audio_rtp_packets_received: {type: integer, format: int64}, video_rtp_packets_sent: {type: integer, format: int64}, video_rtp_packets_received: {type: integer, format: int64}, rtp_bytes_sent: {type: integer, format: int64}, rtp_bytes_received: {type: integer, format: int64}, rtcp_packets_sent: {type: integer, format: int64}, rtcp_packets_received: {type: integer, format: int64}, started_at: {type: string, format: date-time}, updated_at: {type: string, format: date-time}, last_media_at: {type: string, format: date-time}, stopped_at: {type: string, format: date-time}}} SIPLabSessionView: {type: object, required: [session, playback], properties: {session: {$ref: '#/components/schemas/SIPLabSession'}, playback: {$ref: '#/components/schemas/ProtocolLabPlayback'}}} SIPLabSessionsResponse: {allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {type: object, required: [sessions], properties: {sessions: {type: array, items: {$ref: '#/components/schemas/SIPLabSessionView'}}}}}}]} @@ -1100,7 +1142,7 @@ components: ClusterStatusResponse: {allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {$ref: '#/components/schemas/ClusterStatus'}}}]} Recording: type: object - properties: {id: {type: string}, stream_key: {type: string}, format: {type: string, enum: [flv, fmp4, mp4, ts, hls]}, state: {type: string}, size_bytes: {type: integer, format: int64}, started_at: {type: string, format: date-time}, completed_at: {type: string, format: date-time}, duration_sec: {type: number}, error: {type: string}} + properties: {id: {type: string}, stream_key: {type: string}, format: {type: string, enum: [flv, fmp4, mp4, ts, hls]}, state: {type: string, enum: [active, completed, failed]}, size_bytes: {type: integer, format: int64}, started_at: {type: string, format: date-time}, completed_at: {type: string, format: date-time}, duration_sec: {type: number}, error: {type: string}} StorageHealth: {type: object, properties: {backend: {type: string}, root: {type: string}, healthy: {type: boolean}, low_space: {type: boolean}, total_bytes: {type: integer, format: int64}, available_bytes: {type: integer, format: int64}, error: {type: string}}} RecordingSession: {type: object, properties: {stream_key: {type: string}, recording_id: {type: string}, state: {type: string}, started_at: {type: string, format: date-time}, completed_at: {type: string, format: date-time}, duration_sec: {type: number}, bytes: {type: integer, format: int64}, write_retries: {type: integer, format: int64}, last_error: {type: string}}} RecordingMetrics: {type: object, properties: {files_completed: {type: integer}, files_failed: {type: integer}, write_retries: {type: integer}, write_failures: {type: integer}, files_deleted: {type: integer}, bytes_written: {type: integer}, storage_errors: {type: integer}}} @@ -1136,3 +1178,5 @@ components: GBLabSessionView: {type: object, required: [session, playback], properties: {session: {$ref: '#/components/schemas/GBLabSession'}, playback: {$ref: '#/components/schemas/ProtocolLabPlayback'}}} GBLabSessionsResponse: {allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {type: object, required: [sessions], properties: {sessions: {type: array, items: {$ref: '#/components/schemas/GBLabSessionView'}}}}}}]} GBLabSessionResponse: {allOf: [{$ref: '#/components/schemas/ApiResponse'}, {type: object, properties: {data: {$ref: '#/components/schemas/GBLabSessionView'}}}]} + WHEPFeedStatus: {type: object, required: [generation, cursor, mode, state, first_media_wait_ms, updated_at, expected_video, expected_audio], properties: {generation: {type: integer, format: int64}, cursor: {type: integer, format: int64}, mode: {type: string, enum: [live, realtime]}, state: {type: string, enum: [waiting_keyframe, playing, no_media_input, media_stalled, codec_mismatch, sample_write_failed, generation_ended, closed]}, first_media_at: {type: string, format: date-time, description: Time of the first successfully emitted audio or video sample.}, first_media_wait_ms: {type: integer, format: int64, minimum: 0, description: Stable delay from feed status creation to the first successfully emitted audio or video sample; zero before the first sample.}, last_video_at: {type: string, format: date-time, description: Last successfully emitted video sample for this feed.}, last_audio_at: {type: string, format: date-time, description: Last successfully emitted audio sample for this feed.}, updated_at: {type: string, format: date-time}, expected_video: {type: boolean}, expected_audio: {type: boolean}, video_frames: {type: integer, format: int64}, audio_frames: {type: integer, format: int64}, dropped_video: {type: integer, format: int64, description: Dropped frames for the negotiated video track only.}, dropped_audio: {type: integer, format: int64, description: Dropped frames for the negotiated audio track only.}, source_overwrites: {type: integer, format: int64, description: Source-ring positions lost during overwrite recovery; mixed source positions are not classified as video or audio drops.}, rtp_packets_sent: {type: integer, format: int64, description: RTP packets successfully written by the PeerConnection interceptor for this feed; session close captures one final monotonic snapshot.}, rtp_bytes_sent: {type: integer, format: int64, description: RTP header plus payload bytes successfully written for this feed; session close captures one final monotonic snapshot.}, rtcp_packets_received: {type: integer, format: int64, description: Valid RTCP packets read by the outbound track sender; session close captures one final monotonic snapshot.}, last_error: {type: string, description: Bounded terminal media error}}} + WHEPSessionStatusResponse: {type: object, required: [session_id, stream_key, role, feed], properties: {session_id: {type: string}, stream_key: {type: string}, role: {type: string, enum: [whep]}, feed: {$ref: '#/components/schemas/WHEPFeedStatus'}}} diff --git a/docs/config/config.schema.json b/docs/config/config.schema.json index a60b4f36..1b1e6889 100644 --- a/docs/config/config.schema.json +++ b/docs/config/config.schema.json @@ -74,7 +74,7 @@ "max_bitrate_per_stream": {"type": "integer", "minimum": 0}, "rate_limit": { "type": "object", "additionalProperties": false, - "properties": {"enabled": {"type": "boolean"}, "rate": {"type": "number", "minimum": 0}, "burst": {"type": "integer", "minimum": 0}} + "properties": {"enabled": {"type": "boolean"}, "rate": {"type": "number", "minimum": 0}, "burst": {"type": "integer", "minimum": 0}, "trusted_proxies": {"type": "array", "items": {"type": "string", "minLength": 1}, "uniqueItems": true, "default": [], "description": "IP addresses or CIDR networks allowed to supply X-Forwarded-For or X-Real-IP. Forwarded headers are ignored for every other direct peer."}} } } }, @@ -141,13 +141,13 @@ "properties": { "enabled": {"type": "boolean"}, "listen": {"type": "string"}, "transport": {"type": "array", "items": {"type": "string", "enum": ["udp", "tcp"]}}, "server_id": {"type": "string"}, "domain": {"type": "string"}, "auth": {"type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean"}, "password": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}}}, - "gateway": {"type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean"}, "stream_prefix": {"type": "string"}, "rtp_port_range": {"$ref": "#/$defs/port_range"}, "codecs": {"type": "array", "items": {"type": "string"}}, "max_calls": {"type": "integer", "description": "Maximum concurrent calls; any non-positive value selects the gateway default of 100."}}} + "gateway": {"type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean"}, "stream_prefix": {"type": "string"}, "rtp_port_range": {"$ref": "#/$defs/port_range"}, "codecs": {"type": "array", "items": {"type": "string"}}, "max_calls": {"type": "integer", "description": "Maximum concurrent calls; any non-positive value selects the gateway default of 100."}, "max_lab_sessions": {"type": "integer", "minimum": 1, "description": "Maximum active persistent local SIP protocol-lab sessions; non-positive values use the default of 16.", "x-liveforge-reload": "restart_required"}}} } }, "gb28181": { "type": "object", "additionalProperties": false, "properties": { - "enabled": {"type": "boolean"}, "stream_prefix": {"type": "string"}, "rtp_port_range": {"$ref": "#/$defs/defaultable_port_range", "description": "An empty list selects the GB28181 module fallback range 40000-50000."}, + "enabled": {"type": "boolean"}, "stream_prefix": {"type": "string"}, "rtp_port_range": {"$ref": "#/$defs/defaultable_port_range", "description": "An empty list selects the GB28181 module fallback range 40000-50000."}, "max_lab_sessions": {"type": "integer", "minimum": 1, "description": "Maximum active persistent local GB28181 protocol-lab sessions; non-positive values use the default of 16.", "x-liveforge-reload": "restart_required"}, "ssrc": {"type": "object", "additionalProperties": false, "properties": {"prefix": {"type": "string"}}}, "keepalive": {"type": "object", "additionalProperties": false, "properties": {"interval": {"$ref": "#/$defs/duration"}, "timeout": {"$ref": "#/$defs/duration"}}}, "auto_invite": {"type": "boolean"}, "catalog_interval": {"$ref": "#/$defs/duration"}, "dump_file": {"type": "string"} @@ -174,11 +174,12 @@ "stream": { "type": "object", "additionalProperties": false, "properties": { - "gop_cache": {"type": "boolean", "x-liveforge-reload": "hot_reload"}, "gop_cache_num": {"type": "integer", "minimum": 0, "x-liveforge-reload": "hot_reload"}, + "gop_cache": {"type": "boolean", "x-liveforge-reload": "hot_reload"}, "gop_cache_num": {"type": "integer", "minimum": 0, "x-liveforge-reload": "hot_reload"}, "gop_cache_max_frames": {"type": "integer", "minimum": 0, "description": "Maximum frames in one cached GOP; zero disables this bound. Hot reload recomputes the retained prefix; relaxation affects future frames only.", "x-liveforge-reload": "hot_reload"}, "gop_cache_max_duration": {"$ref": "#/$defs/duration", "description": "Maximum duration of one cached GOP; zero disables this bound. Hot reload recomputes the retained prefix; relaxation affects future frames only.", "x-liveforge-reload": "hot_reload"}, "gop_cache_max_bytes": {"type": "integer", "minimum": 0, "description": "Maximum payload bytes in one cached GOP; zero disables this bound. Hot reload recomputes the retained prefix; relaxation affects future frames only.", "x-liveforge-reload": "hot_reload"}, "ring_buffer_size": {"type": "integer", "minimum": 1, "x-liveforge-reload": "restart_required"}, "idle_timeout": {"$ref": "#/$defs/duration", "x-liveforge-reload": "hot_reload"}, "no_publisher_timeout": {"$ref": "#/$defs/duration", "x-liveforge-reload": "hot_reload"}, "slow_consumer": {"$ref": "#/$defs/slow_consumer", "x-liveforge-reload": "hot_reload"}, "feedback": {"$ref": "#/$defs/feedback", "x-liveforge-reload": "hot_reload"}, "simulcast": {"$ref": "#/$defs/simulcast", "x-liveforge-reload": "restart_required", "x-liveforge-support": "deferred"} - } + }, + "allOf": [{"if": {"properties": {"gop_cache": {"const": true}, "gop_cache_num": {"minimum": 1}}, "required": ["gop_cache", "gop_cache_num"]}, "then": {"anyOf": [{"required": ["gop_cache_max_frames"], "properties": {"gop_cache_max_frames": {"minimum": 1}}}, {"required": ["gop_cache_max_bytes"], "properties": {"gop_cache_max_bytes": {"minimum": 1}}}]}}] }, "auth_rule": { "type": "object", "additionalProperties": false, @@ -236,8 +237,8 @@ "record": { "type": "object", "additionalProperties": false, "properties": { - "enabled": {"type": "boolean", "x-liveforge-reload": "restart_required"}, "stream_pattern": {"type": "string", "x-liveforge-reload": "hot_reload"}, "format": {"type": "string", "enum": ["flv", "fmp4", "mp4", "ts", "hls"], "default": "fmp4", "description": "Recording container. The default is fMP4 and the default extension is .mp4; fMP4/MP4 are the browser-friendly unified recording formats.", "x-liveforge-reload": "hot_reload"}, "path": {"type": "string", "description": "Recording path template. Supported placeholders are {stream_key}, {date}, {time}, and {ext}.", "x-liveforge-reload": "restart_required"}, - "segment": {"type": "object", "additionalProperties": false, "x-liveforge-reload": "hot_reload", "properties": {"mode": {"type": "string", "description": "Operator label; segmentation is activated by positive duration and/or max_size values."}, "duration": {"$ref": "#/$defs/duration"}, "max_size": {"type": "string"}}}, + "enabled": {"type": "boolean", "x-liveforge-reload": "restart_required"}, "stream_pattern": {"type": "string", "x-liveforge-reload": "hot_reload"}, "format": {"type": "string", "enum": ["flv", "fmp4", "mp4", "ts", "hls"], "default": "fmp4", "description": "Recording container. The default is fMP4 and the default extension is .mp4; fMP4/MP4 are browser-friendly unified recording formats. hls is an alias for TS storage and uses a .ts extension.", "x-liveforge-reload": "hot_reload"}, "path": {"type": "string", "description": "Recording path template. Supported placeholders are {stream_key}, {date}, {time}, and {ext}.", "x-liveforge-reload": "restart_required"}, + "segment": {"type": "object", "additionalProperties": false, "x-liveforge-reload": "hot_reload", "properties": {"mode": {"type": "string", "description": "Operator label; segmentation is activated by positive duration and/or max_size values."}, "duration": {"$ref": "#/$defs/duration"}, "max_size": {"type": "string", "pattern": "^\\s*(?:[0-9]+(?:[bB]|[kK][bB]|[mM][bB]|[gG][bB])?)?\\s*$", "description": "Optional non-negative decimal byte count with suffix B, KB, MB, or GB. Empty or zero disables size rotation; fractional, negative, unknown-suffix, and overflow values are invalid."}}}, "on_file_complete": {"type": "object", "additionalProperties": false, "x-liveforge-reload": "hot_reload", "properties": {"url": {"type": "string"}}} } }, @@ -245,7 +246,7 @@ "type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean", "x-liveforge-reload": "restart_required"}, "listen": {"type": "string", "x-liveforge-reload": "restart_required"}, "stream_pattern": {"type": "string", "x-liveforge-reload": "hot_reload"}, "path": {"type": "string", "x-liveforge-reload": "restart_required"}, "window": {"$ref": "#/$defs/duration", "x-liveforge-reload": "hot_reload"}, "segment_duration": {"$ref": "#/$defs/duration", "x-liveforge-reload": "hot_reload"}, "cleanup_interval": {"$ref": "#/$defs/duration", "x-liveforge-reload": "hot_reload"}} }, - "metrics": {"type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean"}, "listen": {"type": "string"}, "path": {"type": "string"}}}, + "metrics": {"type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean"}, "listen": {"type": "string"}, "path": {"type": "string"}, "stream_detail": {"type": "boolean", "default": false, "description": "Opt in to per-stream Prometheus series carrying stream_key labels. Server-level aggregate metrics remain available when disabled."}, "stream_detail_limit": {"type": "integer", "minimum": 0, "default": 100, "description": "Without an allowlist, maximum distinct stream keys admitted for one Collector lifetime; admitted keys are not evicted or replaced after streams disappear. With an allowlist, maximum keys exported per scrape. Zero exports no per-stream series; negative values are invalid."}, "stream_detail_allowlist": {"type": "array", "items": {"type": "string", "minLength": 1}, "uniqueItems": true, "default": [], "description": "Optional authoritative exact stream-key universe, deduplicated and sorted when the Collector is created, then subject to stream_detail_limit per scrape. When empty, lifetime creation-order admission applies."}}}, "api": { "type": "object", "additionalProperties": false, "properties": { @@ -263,10 +264,10 @@ "description": "Bootstrap-controlled background source. Loads run on the manager worker; snapshot and typed-key reads are atomic and non-blocking. Config writes are serialized with loads and close, complete before Apply returns 202, and then schedule background parse/apply/publication. The Config API exposes the complete versioned JSON Schema, raw source document, redacted document, validation, and apply operations. Apply is writable for file, HTTP/HTTPS, Consul, and Redis sources when their backend accepts writes; other sources return a read-only conflict.", "properties": { "source": {"type": "string", "enum": ["file", "http", "https", "consul", "redis"]}, "poll_interval": {"$ref": "#/$defs/duration"}, "load_timeout": {"$ref": "#/$defs/duration"}, - "file": {"type": "object", "additionalProperties": false, "description": "Writable by atomic replacement of the configured path.", "properties": {"path": {"type": "string"}}}, + "file": {"type": "object", "additionalProperties": false, "description": "Writable by atomic replacement of the configured path. New targets use private mode 0600; an existing target's permission bits are preserved. Loading is capped by max_bytes.", "properties": {"path": {"type": "string"}, "max_bytes": {"type": "integer", "description": "Maximum file document bytes; any non-positive value selects the 4 MiB source default."}}}, "http": {"type": "object", "additionalProperties": false, "description": "HTTP configuration source. runtime.source=http requires an http:// URL and runtime.source=https requires an https:// URL. Scheme mismatches are rejected before dispatch, redirects are disabled, and ETag/Last-Modified validators advance only after a document is accepted. Apply uses authenticated PUT.", "properties": {"url": {"type": "string", "description": "Complete source URL whose scheme must exactly match the selected http or https runtime.source."}, "token": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}, "max_bytes": {"type": "integer", "description": "Maximum response bytes; any non-positive value selects the 4 MiB source default."}}}, - "consul": {"type": "object", "additionalProperties": false, "description": "Apply writes the complete document to prefix/config.yaml through the Consul KV API.", "properties": {"address": {"type": "string"}, "prefix": {"type": "string", "minLength": 1}, "token": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}, "max_bytes": {"type": "integer", "description": "Maximum response bytes; any non-positive value selects the 4 MiB source default."}}}, - "redis": {"type": "object", "additionalProperties": false, "description": "Apply writes config.yaml in hash or prefix mode and increments version_key when configured.", "properties": {"addr": {"type": "string"}, "username": {"type": "string"}, "password": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}, "db": {"type": "integer", "minimum": 0}, "prefix": {"type": "string"}, "hash": {"type": "string"}, "version_key": {"type": "string"}, "tls": {"type": "boolean"}}} + "consul": {"type": "object", "additionalProperties": false, "description": "Apply writes the complete document to prefix/config.yaml through the Consul KV API. Flattened dotted/slashed keys are canonicalized and any duplicate path or scalar/container prefix collision is rejected deterministically before materialization.", "properties": {"address": {"type": "string"}, "prefix": {"type": "string", "minLength": 1}, "token": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}, "max_bytes": {"type": "integer", "description": "Maximum response bytes; any non-positive value selects the 4 MiB source default."}}}, + "redis": {"type": "object", "additionalProperties": false, "description": "Apply writes config.yaml in hash or prefix mode and increments version_key when configured. The document write and optional version increment are queued in one MULTI/EXEC transaction; transaction and EXEC errors are returned to Apply. Flattened dotted/slashed keys reject duplicate paths and scalar/container prefix collisions deterministically. Hash field names prefer HSCAN NOVALUES and fall back to HKEYS only when that subcommand is unsupported; values are read in bounded HSTRLEN/HGET batches. Prefix keys use bounded SCAN batches and length preflight.", "properties": {"addr": {"type": "string"}, "username": {"type": "string"}, "password": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}, "db": {"type": "integer", "minimum": 0}, "prefix": {"type": "string"}, "hash": {"type": "string"}, "version_key": {"type": "string"}, "tls": {"type": "boolean"}, "max_bytes": {"type": "integer", "description": "Maximum Redis configuration document/materialization bytes; any non-positive value selects the 4 MiB source default."}}} } } } diff --git a/docs/recipes/auth-and-tls.md b/docs/recipes/auth-and-tls.md index 061b23e1..5b9ada37 100644 --- a/docs/recipes/auth-and-tls.md +++ b/docs/recipes/auth-and-tls.md @@ -32,7 +32,11 @@ api: audit: max_entries: 1000 limits: - rate_limit: {enabled: true, rate: 20, burst: 40} + rate_limit: + enabled: true + rate: 20 + burst: 40 + trusted_proxies: [] auth: enabled: true publish: @@ -43,6 +47,23 @@ auth: token: {secret: "${SUBSCRIBE_JWT_SECRET}", algorithm: HS256} ``` +Client-IP forwarding headers are ignored by default. Add only the exact reverse +proxy IP addresses or CIDR networks you operate to `trusted_proxies`; malformed +entries are rejected during configuration validation. Never trust a broad +network merely to make `X-Forwarded-For` appear in logs, because that lets any +peer in the network choose a new rate-limit identity. +For a trusted direct peer, LiveForge scans `X-Forwarded-For` from right to left, +removes configured trusted proxy hops, and uses the first untrusted hop. This +prevents an attacker-controlled left prefix from creating a fresh rate-limit +bucket on every request. A malformed non-empty chain falls back to the direct +peer. + +The API, WebRTC signaling, and metrics HTTP listeners use fixed transport +guards: `ReadHeaderTimeout` is 5 seconds and `IdleTimeout` is 2 minutes. This +limits slow header parsing and idle keep-alive connections without replacing +the existing handler or media write deadlines; no server-level `WriteTimeout` +is added by this policy. + Global TLS files/mode, `api.listen`, `api.tls`, `auth.enabled`, and audit capacity require restart. Named management tokens, the legacy management bearer, console credentials/role, and publish/subscribe rule details are hot-reloadable. The deprecated management token path is `auth.api.bearer_token`. Move it to `api.auth.bearer_token`. Normalization uses the deprecated value only when the current path is empty; if both exist, `api.auth.bearer_token` wins. They do not create two active credentials. New deployments should prefer named `api.auth.tokens` for attribution and least privilege. diff --git a/docs/recipes/cluster-relay-operations.md b/docs/recipes/cluster-relay-operations.md index 53df6868..00a093f5 100644 --- a/docs/recipes/cluster-relay-operations.md +++ b/docs/recipes/cluster-relay-operations.md @@ -66,6 +66,8 @@ Status returns 200, including active forward/origin counts, bounded relay snapsh The default internal endpoints are `POST /api/relay/push`, `POST /api/relay/pull`, `POST /api/relay/gb/push`, and `POST /api/relay/gb/pull`. They require `server:mutate` and are node-to-node contracts, not operator workflows. RTP signaling uses SDP; GB signaling exchanges stream/port query values. Expected failures include 400 for invalid input, 404 for a missing pull stream, and 503 for allocation or setup failure. +RTP relay media admission fails closed: packetizer errors, empty packetizer output, nil packets, RTP marshal errors, UDP write errors, and short writes terminate the affected relay instead of being counted as successful media. Push cancellation remains a normal shutdown path; non-cancellation send failures are reported in bounded logs and relay status. + ## Credential Selection And Hot Rotation For every RTP/GB peer request, the node loads the current atomic configuration and selects credentials in this order: diff --git a/docs/recipes/protocol-test-lab.md b/docs/recipes/protocol-test-lab.md index f7971a1d..e89e4662 100644 --- a/docs/recipes/protocol-test-lab.md +++ b/docs/recipes/protocol-test-lab.md @@ -6,6 +6,13 @@ The Console includes local one-shot protocol self-tests and persistent fake-devi sessions so SIP and GB28181 workflows can be validated without a PBX, cloud platform, or camera. Enable SIP, its `gateway` block, and GB28181, keep the API listener on loopback, and use a viewer or operator token. +Each provider admits at most `max_lab_sessions` active `starting`, `active`, or +SIP `contract` sessions at once. The default is 16; terminal history is retained +for diagnosis but does not consume the ceiling. Set a positive value in +`sip.gateway.max_lab_sessions` or `gb28181.max_lab_sessions`; non-positive values +are normalized to the default. When the ceiling is full, the start API returns +HTTP 429 with the `ProtocolLabCapacity` error instead of allocating sockets or +publishing a partially active session. The checked-in sample includes a loopback-safe gateway port range so this page is usable immediately in local development: @@ -27,7 +34,9 @@ fake SIP peer through REGISTER and 401 digest challenge, authenticated registration, INVITE/200/ACK/BYE, incompatible-codec rejection, timeout handling, RTP media, and RTCP control. It also checks SDP parsing and codec negotiation against the configured gateway codecs and an RTP/RTCP port pair. -The Console SIP page renders every phase and its failure detail. +The port check binds both configured UDP sockets, fails when every pair is +occupied by another process, and closes/frees a successful reservation before +returning. The Console SIP page renders every phase and its failure detail. For a persistent provider session, call the `SIPGatewayProvider` methods `StartLabSession(ctx, LabSessionRequest)`, `ListLabSessions()`, and @@ -35,7 +44,11 @@ For a persistent provider session, call the `SIPGatewayProvider` methods SIP call and sends deterministic H.264 video plus PCMA/PCMU audio on separate RTP tracks, with RTCP, into the gateway-created stream. The gateway binds and parses both real RTCP receiver sockets, and the Lab counter reflects packets -accepted there rather than successful UDP writes. In `receive` mode, the fake +accepted there rather than successful UDP writes. Gateway RTP/RTCP allocation +skips pairs already occupied by another local process and keeps both sockets +bound before SDP is accepted or offered, closing the allocation only during +setup rollback or session cleanup. Fake Lab endpoint pairs avoid the configured +gateway RTP range. In `receive` mode, the fake SIP endpoint accepts the gateway outbound INVITE, receives the existing source without writing generated frames into that stream, counts audio/video RTP and RTCP, and sends periodic receiver reports for each track. Gateway sender reports @@ -47,9 +60,23 @@ transport reader before closing its fake SIP UA, so normal cleanup does not underflow sipgo UDP references or report an already-closed socket. This provider workflow requires an initialized SIP transport and enabled gateway. Receive mode waits for the selected publisher generation to become startup-ready before -sending its INVITE; a source with a known unsupported audio codec is rejected -before signaling, while a source with late sequence headers is waited on or -canceled with the request context. +sending its INVITE. The requested PCMA or PCMU value is the actual outbound +target codec, not just a display hint. A matching source is passed through; +when the source differs, the generation-bound shared audio transcoder supplies +an independent target-codec reader while H.264 continues from the original +live cursor. That conversion requires `audio_codec.enabled=true`, the +`audiocodec` build tag, and FFmpeg development libraries. A source without a +direct or available transformed path is rejected before signaling, while a +source with late sequence headers is waited on or canceled with the request +context. + +Gateway RTP/RTCP pairs are socket-bound before SDP and remain owned by the call +until teardown. For a transcoded outbound call, every ready target-audio frame +rechecks its captured publisher generation immediately before RTP send. Source +retirement closes and releases the target reader, generation subscriber, and +media sockets, returns the exact port pair to the allocator, and sends one BYE +even if another local teardown races with retirement. + The publish stream contains a dependency-free moving 160x90 constrained-baseline H.264 pattern at 25 fps, with one IDR per 25-frame loop, plus audible 20 ms PCMA/PCMU frames. The Console uses the video player and prefers WebRTC/WHEP for @@ -97,8 +124,10 @@ The report is returned by `GET /api/v1/gb28181/test` and runs an in-process fake device through SIP registration, Keepalive, Catalog query/response, playback INVITE/200 SDP/ACK/BYE, missing-SDP rejection, timeout handling, PS/90000 media over localhost UDP, and RTCP control. It also checks an RTP/RTCP port pair and -local PS mux/demux of an H.264 keyframe. It does not contact a platform or -camera. The Console GB28181 page renders every phase and its detail. +local PS mux/demux of an H.264 keyframe. The configured pair is accepted only +after both UDP sockets bind; external exhaustion fails this check, while a +successful check closes both sockets and frees the pair. It does not contact a +platform or camera. The Console GB28181 page renders every phase and its detail. Persistent GB28181 sessions use the same control shape and perform real REGISTER, Keepalive, Catalog, INVITE, ACK, BYE, and unregister signaling. In @@ -107,19 +136,36 @@ that Contact during registration. LiveForge then uses its normal invite client to initiate live play; the fake device accepts INVITE, consumes ACK without a response, handles BYE, and sends deterministic H.264 plus G.711A in PS/RTP payload type 96 with RTCP into LiveForge's real RTP/RTCP receiver. In `receive` -mode LiveForge validates an existing H.264/G.711A source before activation and -a module-owned outbound media session admits a source subscriber before it -sends that source as PS/RTP/RTCP. Subscriber-limit rejection is returned +mode LiveForge requires H.264 plus direct G.711A or source audio that the +tagged runtime can convert to G.711A. A module-owned outbound media session +admits a source subscriber before activation, keeps H.264 on the source live +cursor, and uses an independent generation-bound target-audio reader when +conversion is needed. Unsupported conversion fails before signaling, while a +supported source is sent as PS/RTP/RTCP. Subscriber-limit rejection is returned synchronously and the Lab is never published as active. If the sender later fails, the Lab transitions to `failed`, records a bounded redacted diagnostic, and releases its dialog, module session, subscriber, sockets, and ports. The fake device only receives and counts RTP, RTCP, PS, audio, and video frames. The -simulator binds only loopback sockets, requires no FFmpeg or external platform, -and releases both SIP UAs, dialogs, RTP/RTCP ports, and session resources on -stop or module close. The fake-client transport reader exits before its UA and +simulator binds only loopback sockets and requires no external platform. Direct +G.711A requires no FFmpeg; converting Opus, AAC, or another supported source +codec requires the `audiocodec` build. The Lab releases both SIP UAs, dialogs, +RTP/RTCP ports, target-audio readers, and session resources on stop or module +close. The fake-client transport reader exits before its UA and the fake-peer listener exits before its peer UA, avoiding sipgo UDP reference underflow and closed-socket cleanup warnings. +Inbound device INVITEs complete asynchronous publish-start admission before +LiveForge exposes `200 OK`. Backpressure returns a non-2xx response and removes +the publisher, session, receiver sockets, newly created stream, and allocated +pair without emitting an unmatched publish-stop. GB28181 receive-mode outbound +media also allocates and binds its RTP/RTCP pair atomically, so an externally +occupied first pair is skipped in favor of a later configured pair. + +Successful server-initiated live and playback INVITEs transfer dialog ownership +to the media session. Local stop, receiver failure, rollback after an accepted +2xx response, and repeated cleanup converge on one managed dialog, so at most +one BYE is sent and the transaction is closed once. + After the initial registration, each persistent fake device continues sending Keepalive messages at roughly one-third of the configured `gb28181.keepalive.timeout` (bounded to a practical interval), so a session @@ -136,8 +182,9 @@ curl -fsS -H "Authorization: Bearer $VIEWER_TOKEN" \ `GET` requires `gb28181:read`; `POST` and `DELETE` require `gb28181:control`. Receive mode requires the requested stream to already have -an H.264 video and G.711A audio publisher with the required startup sequence -headers. The receive path waits for that publisher generation to become ready +H.264 video and either direct G.711A audio or an audio codec the running tagged +build can transform to G.711A, with the required startup sequence headers. The +receive path waits for that publisher generation to become ready before sending its INVITE; a late header is not treated as a playable source until it arrives. The publish sample includes a moving constrained-baseline SPS/PPS/IDR/interframe pattern at 25 fps and audible 8 kHz mono G.711A audio, so the @@ -251,7 +298,21 @@ go test ./module/api ./module/sipgateway ./module/gb28181 go test ./pkg/rtp -run TestH264DepacketizerEmitsSequenceHeaderForSeparateSPSAndPPSPackets -count=1 go test ./module/webrtc -run 'Test(RegisterCodecs|WHEPPCMAudioPassthroughDeliversRTP)$' -count=1 go test -race ./module/gb28181 -run 'Lab|SelfTest' -v +CGO_ENABLED=1 go test -tags audiocodec ./test/integration -run '^TestSIPGB28181WHIPBrowserBridgeMatrix$' -count=1 -v +LIVEFORGE_PROTOCOL_MATRIX_SOAK=60s CGO_ENABLED=1 go test -tags audiocodec ./test/integration -run '^TestSIPGB28181WHIPBrowserBridgeMatrix$' -count=1 -v ``` -The self-tests bind only ephemeral localhost UDP sockets and release their port -pairs before returning. They do not write recordings or configuration. +The Chromium matrix checks SIP publish to GB28181 receive plus WHEP, +GB28181 publish to SIP receive plus WHEP, and WHIP H.264/Opus publish to both +SIP and GB28181 receive plus WHEP. It requires expected decoded dimensions, +connected ICE, no browser media error, increasing video/audio RTP and decoded +frame counters, an advancing media clock, and WHEP server RTP/RTCP state that +never enters `media_stalled`. The browser checks run when Chromium advertises +H.264 receive support; otherwise the test reports an environment skip, while +Pion negotiation tests remain mandatory. The soak duration is a correctness +soak; it is not evidence of leak freedom, concurrency capacity, or deployment +capacity. + +The self-tests bind their configured RTP/RTCP pair plus ephemeral localhost UDP +sockets and release every pair before returning. They do not write recordings +or configuration. diff --git a/docs/recipes/recording-dvr-management.md b/docs/recipes/recording-dvr-management.md index c43f6b3f..b6029d24 100644 --- a/docs/recipes/recording-dvr-management.md +++ b/docs/recipes/recording-dvr-management.md @@ -45,12 +45,27 @@ api: `record.enabled`, `record.path`, `dvr.enabled`, `dvr.listen`, and `dvr.path` require a restart. Recording format, stream pattern, segmentation, DVR window, segment duration, and cleanup interval are hot-reload candidates. Formats are `flv`, `fmp4`, `mp4`, `ts`, and `hls`. +Record format validation accepts only `flv`, `fmp4`, `mp4`, `ts`, or `hls`; `hls` is a TS storage alias and uses a `.ts` extension. `record.segment.max_size` accepts an empty/whitespace value or `0` to disable size rotation, or a non-negative decimal byte count with an optional `B`, `KB`, `MB`, or `GB` suffix. Fractional values, negatives, unknown suffixes, and values that overflow the byte counter are rejected consistently by runtime validation and the configuration schema. + The default recording format is fMP4 and the default extension is `.mp4`. fMP4 and MP4 are the preferred unified browser playback formats; media tracks are initialized lazily so a late audio track is not silently omitted. fMP4 writes AAC directly. Its init metadata derives omitted AAC sample rate and channel count from the AudioSpecificConfig and reuses the resolved sample rate as the media -timescale, preserving source DTS intervals. When the record module is not enabled, +timescale, preserving source DTS intervals. File rotation retains the publisher's +declared tracks and deep-copied latest video/audio sequence headers; each new FLV, +fMP4, MP4, or TS file therefore writes its own complete container initialization +instead of depending on an earlier file. Each file also rebases its audio and video +decode timelines independently to zero. TS writes PAT/PMT before the first media +PES even when audio arrives first, and classic MP4 computes sample durations with +separate video and audio clocks. Classic MP4 saturates sample composition, +duration, and version-0 movie-duration fields at their representable limits +instead of wrapping. It emits `ctts` version 1 when any PTS-DTS composition +offset is negative and keeps version 0 for non-negative offsets, so B-frame +timing is not decoded as a huge unsigned delay. It also writes expandable AAC +ESDS descriptor lengths. Keep +rotation enabled for files that could approach the version-0 duration limit. +When the record module is not enabled, `GET /api/v1/recordings/status` still returns HTTP 200 with `enabled=false`, `available=true`, and `state=disabled`, allowing Storage to render an explicit unavailable state. Recording item, download, and play routes return @@ -64,7 +79,14 @@ DVR MPEG-TS applies the same conversion to audio unsupported by its target. When a transformed fMP4 recording is stopped, its source-cursor boundary is captured and generated output already owed for frames before that boundary is drained before the file is finalized; this prevents an immediate stop from producing a -zero-media recording while the asynchronous AAC transform is catching up. +zero-media recording while the asynchronous AAC transform is catching up. At a +publisher-generation boundary, a fixed-size transform flushes samples retained +by its resampling filter, encodes complete frames, pads its final partial PCM +frame with silence, and emits every delayed encoder packet exactly once with +monotonic target-frame-size DTS before its output ring closes. Record +and DVR generation-tail drains therefore retain all transformed audio owed by +that finite source generation. Last-consumer cancellation can still discard a +tail that no remaining consumer owns. AAC remains direct in fMP4, and SIP/GB28181 G.711 recordings retain the existing G.711-to-AAC behavior without claiming audio transcoding in a portable no-CGO build. @@ -75,13 +97,53 @@ and the matching publisher stop event finalizes the active session. A SIP INVITE is rejected before RTP allocation when synchronous publish authorization fails. +DVR validates one publisher-generation startup snapshot and carries that exact +snapshot through retained-index and storage recovery into session construction. +It checks the same stream generation immediately before installation. If a +replacement publisher arrives during setup, the stale candidate is discarded, +resources opened by that candidate are closed, and the newer session is not +replaced. + +DVR shutdown captures one absolute drain deadline before waiting for active +publish setup to release module ownership. If setup or finalization exceeds the +configured `server.drain_timeout`, `Close` returns a timeout at that original +deadline while the already-started cleanup continues in the background; a +delayed lock acquisition does not start a second full drain window. + A publish session that ends before any media frame arrives is preserved as `state=failed` and is never offered as a completed playable recording. This prevents sequence-header-only or zero-byte files from returning a misleading successful playback response. +DVR video rotation waits for a valid video keyframe boundary after the duration +threshold. An audio-only session has no such boundary: it rotates when audio +DTS reaches `segment_duration` and publishes the non-empty segment immediately +while its publisher remains online. Portable `!audiocodec` builds retain this +behavior for H.264 plus G.711 by filtering unsupported audio and publishing a +demuxable video-only TS; tagged builds can normalize the audio to AAC when the +shared FFmpeg path is available. + +DVR media routes preserve nested stream-key hierarchy. The application prefix +and each stream-key segment are validated before authorization; encoded `/` or +`\\`, empty segments, and `.`/`..` segments are rejected without redirecting or +looking up storage. Playlist URIs escape each key segment independently, so +reserved `?`, `#`, and `%` characters remain part of the key rather than +starting a query, fragment, or second path component. + DVR playlist and segment GETs run only synchronous `EventSubscribe` authorization hooks. They do not emit asynchronous subscribe lifecycle, notification, or cluster-origin work. Authorization denial keeps the existing 401/403 response behavior. +Finite DVR playlist and segment responses have a 10-second server write bound. +Successful, error, client-canceled, and timed-out requests each release exactly +one global connection slot. The bound does not change range handling, +`ServeContent` metadata, CORS, authorization, or media routing. + +Recording inline-play and download responses use the management listener but +apply the same resource discipline: they acquire one global connection slot +before opening the recording, release it exactly once on every return path, and +set a 10-second write deadline immediately before `ServeContent`. Metadata, +list, status, and delete requests are not media responses and do not consume +this additional media slot. + ## Inspect And Download ```bash @@ -92,9 +154,9 @@ curl -fsS -H "Authorization: Bearer $VIEWER_TOKEN" \ curl -fsS -H "Authorization: Bearer $VIEWER_TOKEN" \ "$LIVEFORGE_API/api/v1/recordings/live/camera.mp4" curl -fS -H "Authorization: Bearer $VIEWER_TOKEN" -H 'Range: bytes=0-1023' \ - "$LIVEFORGE_API/api/v1/recordings/live/camera.mp4/download" -o /tmp/liveforge-recording.part + "$LIVEFORGE_API/api/v1/recordings/live/camera.mp4?action=download" -o /tmp/liveforge-recording.part curl -fS -H "Authorization: Bearer $VIEWER_TOKEN" -H 'Range: bytes=0-1023' \ - "$LIVEFORGE_API/api/v1/recordings/live/camera.mp4/play" -o /tmp/liveforge-recording-preview.part + "$LIVEFORGE_API/api/v1/recordings/live/camera.mp4?action=play" -o /tmp/liveforge-recording-preview.part curl -fsS -H "Authorization: Bearer $VIEWER_TOKEN" \ "$LIVEFORGE_API/api/v1/dvr/status" curl -fsS -H "Authorization: Bearer $VIEWER_TOKEN" \ @@ -104,8 +166,19 @@ curl -fS http://127.0.0.1:8070/dvr/live/camera.m3u8 -o /tmp/liveforge-dvr.m3u8 Successful metadata/status requests return 200. A complete download or inline play returns 200, a valid range returns 206, a cache validator can return 304, and an invalid range can return 416. Inline play sets a media MIME type and `Content-Disposition: inline`, so the Console can preview MP4/fMP4 natively and FLV/TS through mpegts.js. Invalid/traversing IDs return 400, missing objects 404, active/not-ready recordings 409, storage failures 500, and absent modules 503. Authentication failures return 401; a valid token without permission returns 403; rate limiting can return 429. +The active and failed recording states both use the 409 JSON error response for +download and inline play; no media body is written before this state check. + +The explicit `?action=play` and `?action=download` forms apply to the complete URL-decoded recording ID and are safe when that ID itself ends in `/play` or `/download`. The older `/{recordingPath}/play` and `/{recordingPath}/download` forms remain compatible only when no exact ID includes that final action-looking segment. A plain GET always returns an existing exact ID's metadata first. The Console uses the explicit query form. + The Storage view exposes Play for completed recordings and for DVR sessions with available segments. Recording playback is served by the authenticated management API and reuses the Console session cookie. DVR playback is an HLS URL on the separate `dvr.listen` media listener; its playlist and segment requests run the normal synchronous subscribe authorization hooks. The media listener returns non-credentialed CORS headers so a Console on another port can fetch HLS resources. A Console session cookie is not automatically shared with that listener, and the Console never stores or appends a bearer token. Configure DVR subscribe authorization accordingly when using the online browser action. +The Console obtains the DVR URL from `/api/v1/server/info`: after DVR +initialization, `endpoints.dvr` is the actual bound host and non-zero port, and +`endpoint_schemes.dvr` is `http` or `https` according to the listener. Before +initialization, the configured address is only a fallback and must not be used +as evidence that a listener is ready. + ## Delete A Recording Deletion requires `recordings:delete`, which only the admin role has. Confirm the recording ID and state before issuing the request. @@ -115,7 +188,9 @@ curl -fsS -X DELETE -H "Authorization: Bearer $ADMIN_TOKEN" \ "$LIVEFORGE_API/api/v1/recordings/live/camera.mp4" ``` -Success is 200. The same 400/404/409/500/503 storage states apply. A viewer or operator receives 403. +Success is 200. DELETE always treats the complete path as the recording ID, including an ID ending in `/play` or `/download`. The same 400/404/409/500/503 storage states apply. A viewer or operator receives 403. + +Local TS deletion recognizes only `.ts.segment_.ts` and `.ts.m3u8`, plus their `.partial`, `.failed`, and `.orphan--.failed` recovery variants, as owned sidecars. Arbitrary longer names such as `.ts.notes` remain independent recordings. Deletion removes owned sidecars and metadata before the primary. If cleanup returns 500, the primary remains authoritative; repair the filesystem problem and retry the same DELETE. Already removed cleanup artifacts do not make the retry fail. ## Metrics And Diagnostics diff --git a/docs/recipes/rtmp-to-hls.md b/docs/recipes/rtmp-to-hls.md index c348ff40..ebeb7ed1 100644 --- a/docs/recipes/rtmp-to-hls.md +++ b/docs/recipes/rtmp-to-hls.md @@ -31,3 +31,11 @@ curl -sv --noproxy '*' http://127.0.0.1:8080/live/demo.m3u8 The response should be from LiveForge with an HLS content type. A `404` from `nginx` or another server means the loopback media port is occupied by a different process; `ffplay` on RTMP and WHEP on their separate ports can still succeed. Release the conflicting port or change `http_stream.listen`, then reload the Console. HLS, LL-HLS, and DASH wait for the active publisher's required codec sequence headers before creating a playable segmenter. If a publisher is connected but has not sent its video or AAC configuration header yet, the playlist can remain empty until that header arrives; this avoids advertising a segment initialized with the wrong codec metadata. + +When a publisher disconnects, LiveForge immediately retires that generation's HLS, LL-HLS, and DASH managers from new request lookup, then lets each manager drain frames already accepted through the captured generation boundary and finalize them once. A replacement publisher receives a distinct manager. Server or HTTP module shutdown still force-stops and joins active or draining managers, and a stopped LL-HLS manager releases blocking playlist reloads. + +HTTP-FLV and fMP4 use a bounded one-second initialization wait. If the active publisher has not supplied enough codec metadata by then, LiveForge returns HTTP 503 instead of an empty successful response; retry after the sequence header arrives. + +Continuous HTTP-FLV, HTTP-TS, fMP4, and matching WebSocket outputs fail closed if either their direct/transformed media input or shared muxed-output ring is overwritten. Bytes already delivered remain visible, but LiveForge discards the retained post-gap value and ends that response instead of bridging the media gap. WebSocket clients receive a retry-later continuity-loss close; a clean producer end remains a normal close. + +HLS and LL-HLS instead discard their unfinished segment/part, advance to live input in the same publisher generation, refresh sequence headers and container state, and put one `#EXT-X-DISCONTINUITY` before the first recovered output. If the refreshed audio plan changes between direct and shared transformed input, the old reader is closed and released once and the replacement opens at the refreshed live cursor without GOP-history replay. Video emits nothing until the next keyframe, including when video first appears in the refreshed same-generation topology; audio-only resumes on the next live audio frame. LL-HLS abandons the affected MSN, removes its current-part URLs, and wakes blocked reloads. Retained fMP4 media keeps its matching immutable versioned init bytes; those init URLs remain available until the corresponding media leaves the playlist window, while unknown or evicted versions return 404. DASH does not continue across the gap in its existing single Period: it keeps already completed segments but retires that manager, so clients must reacquire playback. An unexpectedly closed transformed reader while the source generation remains active follows the same no-flush terminal rule rather than clean end-of-generation finalization. diff --git a/docs/recipes/runtime-config-sources.md b/docs/recipes/runtime-config-sources.md index e7137d62..2e958a67 100644 --- a/docs/recipes/runtime-config-sources.md +++ b/docs/recipes/runtime-config-sources.md @@ -2,7 +2,7 @@ The checked-in sample configuration is for local development only: it disables TLS and authentication and uses the console credentials `admin/admin`. Never expose it publicly unchanged. -LiveForge reads the bootstrap YAML at startup. A single background worker then loads the selected source immediately, polls it periodically, and publishes immutable snapshots. A runtime configuration read is an atomic in-memory load: it never performs file/network I/O, waits for refresh, or takes the manager status lock. Source loads, Config Apply writes, and source close are serialized; Apply waits for its source write to complete before returning 202, then schedules background parse, module application, and publication. +LiveForge reads the bootstrap YAML at startup. A single background worker then loads the selected source immediately, polls it periodically, and publishes immutable snapshots. A runtime configuration read is an atomic in-memory load: it never performs file/network I/O, waits for refresh, or takes the manager status lock. Source loads, Config Apply writes, and source close are serialized; Apply waits for its source write to complete before returning 202 with `written_and_refresh_scheduled`, then schedules background parse, module application, and publication. ## Prerequisites @@ -19,8 +19,9 @@ export OPERATOR_TOKEN='replace-me' The source, poll interval, load timeout, and source connection settings are bootstrap-controlled and require restart to change. Configuration accepts explicit default sentinels where the owning module defines -one. Any non-positive `runtime.http.max_bytes` or `runtime.consul.max_bytes` -selects a 4 MiB limit. Any non-positive `sip.gateway.max_calls`, +one. Any non-positive `runtime.file.max_bytes`, `runtime.http.max_bytes`, +`runtime.consul.max_bytes`, or `runtime.redis.max_bytes` selects a 4 MiB source +limit. Any non-positive `sip.gateway.max_calls`, `cluster.health_check.evict_threshold`, or `api.audit.max_entries` selects 100 calls, 3 failures, or 1000 entries respectively. Explicit empty RTSP, WebRTC, and GB28181 port ranges select their documented module fallback behavior; @@ -35,11 +36,58 @@ that GOP once, then continues from the atomically captured live cursor. A pure-audio stream has no GOP startup history: it starts at the live cursor and receives the next frame without a separate audio startup cache. +Each cached GOP also has independent optional bounds: `stream.gop_cache_max_frames` +(default 300), `stream.gop_cache_max_duration` (default 10s), and +`stream.gop_cache_max_bytes` (default 32 MiB). A value of zero disables that +specific bound without disabling the others. Combined bounds keep the shortest +playable prefix allowed by all enabled bounds. When a bound is reached, the +cache keeps the keyframe and that interleaved prefix and stops growing until +the next video keyframe starts a new GOP. `stream.ring_buffer_size` must be +positive; direct low-level RingBuffer callers are protected with a one-slot +fallback, while configuration loading rejects zero or negative values. + +When GOP caching is enabled with a positive `gop_cache_num`, at least one of +the frame or byte bounds must be positive; a duration-only policy is rejected +because equal-DTS frames would otherwise have no hard memory limit. Duration +admission uses the full unordered DTS span between the minimum and maximum +observed timestamps, without reordering the retained media. A directly +constructed, unvalidated stream with no hard bound receives the defensive +300-frame limit. + +Hot reload trims every retained GOP under the new policy and recomputes the +active GOP seal from the frames still retained. Tightening a frame, duration, +or byte bound may shorten those playable prefixes and seal the active GOP +immediately. Relaxing a bound permits only the active retained GOP to accept +later interleaved audio/video frames while it remains within every enabled +bound; reaching any enabled bound seals that GOP. +Older retained GOPs stay trimmed, and frames already omitted or trimmed are not +restored. The next video keyframe starts a new complete GOP under the current +policy. + The removed `stream.audio_cache_ms` setting is rejected before typed parsing, including when YAML mapping aliases or `<<` merge mappings and sequences introduce it. Validation follows repeated merge aliases with bounded work and terminates safely on recursive alias graphs. +## Prometheus Stream Detail Cardinality + +Server-level aggregate metrics remain available whenever the metrics module is +enabled. Per-stream `stream_key` labels require `metrics.stream_detail: true`. +Without `metrics.stream_detail_allowlist`, `stream_detail_limit` is the maximum +number of distinct keys one Collector can admit over its entire lifetime. +Active keys are admitted in creation order; removing an admitted stream does +not free its slot for a later key. The Collector retains only scalar keys and +resolves active streams during each gather. + +This deliberately favors bounded Prometheus cardinality over recency. Use the +management streams API for the current active set, or configure an exact +allowlist for selected Prometheus labels. The allowlist is deduplicated and +sorted once at Collector creation, only exact configured keys are eligible, and +`stream_detail_limit` still caps each gather. Set `stream_detail_limit: 0` to +export no per-stream series; negative configured values are invalid and +rejected. A directly constructed Collector still treats any non-positive limit +as disabled defensively. + ## Local File ```yaml @@ -49,9 +97,10 @@ runtime: load_timeout: 10s file: path: "" + max_bytes: 4194304 ``` -An empty `file.path` uses the path passed with `-c`. A changed file is parsed, normalized, validated, and considered only when normalized content changes. +An empty `file.path` uses the path passed with `-c`. A changed file is parsed, normalized, validated, and considered only when normalized content changes. Atomic replacement gives a new target private mode `0600`; when replacing an existing file, its permission bits are preserved exactly. ## HTTP Or HTTPS @@ -83,11 +132,16 @@ runtime: max_bytes: 4194304 ``` -One KV prefix is loaded per attempt. Dotted or slash-separated keys map to configuration paths; a complete `config`, `config.yaml`, `config.yml`, or `config.json` value is also accepted. The Consul index is used as source version when available. +One KV prefix is loaded per attempt. Dotted or slash-separated keys map to configuration paths; a complete `config`, `config.yaml`, `config.yml`, or `config.json` value is also accepted. Dotted and slash-separated spellings are canonicalized before materialization. Duplicate paths and scalar/container prefix collisions fail closed with deterministic errors rather than relying on map iteration order. The Consul index is used as source version when available. Consul KV GET and PUT reject every redirect without dispatching a request to the target, so `X-Consul-Token` is sent only to the exact configured endpoint and is never forwarded. A caller-supplied HTTP client's other behavior is preserved without mutating its redirect policy. ## Redis -Hash mode uses one `HGETALL`: +Hash mode prefers a value-free `HSCAN ... NOVALUES` field scan, then reads field +lengths and values in bounded batches. On Redis versions that reject +`HSCAN NOVALUES` with an unsupported-command or syntax error, it falls back to +`HKEYS` for field names and keeps the same bounded `HSTRLEN`/`HGET` materialization +checks. It never uses `HGETALL`, which could materialize an oversized hash before +the configured limit is known: ```yaml runtime: @@ -100,6 +154,7 @@ runtime: hash: "liveforge:config" version_key: "liveforge:config:version" tls: true + max_bytes: 4194304 ``` Prefix mode uses `SCAN` and pipelined `GET` operations: @@ -110,26 +165,59 @@ runtime: redis: addr: "127.0.0.1:6379" prefix: "liveforge:config:" + max_bytes: 4194304 ``` -Redis fields use dotted or slash-separated paths such as `server.log_level` and `limits.max_connections`. Prefer hash mode when an atomic producer can update the hash and version key together. +Redis fields use dotted or slash-separated paths such as `server.log_level` and `limits.max_connections`. Prefer hash mode when an atomic producer can update the hash and version key together. Each source has a 4 MiB default document/materialization limit, configurable with `runtime.redis.max_bytes`; the limit is checked before complete values are read when Redis exposes `STRLEN`/`HSTRLEN`. Config Apply queues the `config.yaml` write and optional `version_key` increment in one `MULTI/EXEC` transaction; transaction or command errors are returned and Apply does not report success. Redis executes transactions atomically with respect to interleaving clients, but a command error inside `EXEC` is not rolled back, so conflicting Redis key types must be corrected before retrying. + +For flattened Consul and Redis snapshots, leaf values are interpreted +conservatively. Case-insensitive booleans and `null`, canonical base-10 +integers without leading zeroes, and finite decimal or exponent floats become +typed YAML scalars. Leading-zero identifiers, durations, non-finite or +out-of-range numbers, and YAML-looking collection text remain strings. Supply +maps and sequences through a complete `config.yaml`/`config.json` value rather +than encoding YAML syntax in a flattened leaf. ## Config Console And Apply The Config view reads the complete effective and desired configuration document from `GET /api/v1/server/config/document`, fetches the complete versioned JSON Schema from -`GET /api/v1/server/config/schema`, and redacts values whose field names contain -`token`, `password`, `secret`, `credential`, `passphrase`, or `private_key`. The +`GET /api/v1/server/config/schema`, and redacts every schema +`x-liveforge-secret` field plus sensitive names including `api_key` and +`tls.key_file`. Valid absolute hierarchical URL scalars are recognized by value +even under unmapped non-URL-shaped keys. They retain safe public scheme, host, +and port identity while removing userinfo, query parameters, and fragments; +every non-root path becomes a stable opaque digest marker in documents/source +details and an opaque marker in failure status. URL-shaped keys retain the +existing TURN/opaque, malformed/hostless fail-closed, and plain-address policy. +Ordinary strings, durations, IDs, and bare host/address values remain unchanged +outside that key policy. The desired document is retained from the selected source so comments and fields not represented by the typed runtime struct remain visible in the editable source pane. The effective applied document is shown in a separate read-only pane; pending restart paths identify desired values that have not yet changed the effective configuration. Source details show the selected kind plus redacted file, HTTP, Consul, and Redis settings. The page can edit the YAML, run a -read-only Validate, and use Apply & Refresh. Viewer +read-only Validate, and use Apply & Refresh. Validate never expands the server +process environment: references remain literal, exactly one YAML/JSON document +is accepted, and unknown root or nested typed fields are rejected. This strict +boundary is viewer-specific; Apply and trusted runtime source loading remain +permissive for fields not mapped by the typed runtime struct, preserving the +editable desired source, while trusted source loading retains environment +expansion. Viewer tokens have `config:read`; Apply and Refresh require `config:reload` (operator or admin). +Sensitive containers are redacted recursively: collection shape and stable +identity fields (`id`, `name`, `username`, `channel_id`, and `device_id`) remain +visible, while every other scalar descendant is replaced. Structured +URL/address values remain under opaque traversal. Address keys retain bare +IPv4/IPv6 values accepted by `net.ParseIP` and validated plain `host:port` +values. Apply restores placeholders +from the current desired source document; reordered structured collections are matched by stable +public identity, and missing, ambiguous, or marked shape-mismatched originals +are rejected instead of guessing. + The editor starts fail-closed while source metadata is loading. Read-only sources and failed refreshes keep the editor read-only and Apply disabled; the page only enables writing after a successful response confirms that the selected source @@ -149,10 +237,16 @@ The request must contain a complete valid YAML/JSON document. YAML requests carr raw YAML text. JSON requests must use `{"document":"..."}`; a raw JSON object is not accepted as the request envelope. `[REDACTED]` placeholders are replaced with the currently effective sensitive values before a -write, so the editor never needs to receive secrets. A source that only implements -`ConfigSource` is read-only and Apply returns HTTP 409. The response is 202 only -after the serialized source write succeeds; parsing, module application, and -publication still run asynchronously and are visible in the status endpoint. +write, so the editor never needs to receive secrets. Secret maps and sequences +retain their original structure. For named tokens, +ICE servers, and notification endpoint collections, placeholder restoration +matches reordered items by a stable non-secret identity instead of by array +index. Inserting or deleting an item cannot inherit another item's old secret; +an ambiguous identity rejects Apply. Leaving a URL path digest marker unchanged +restores only the matching original path; explicitly replacing it with a new +URL location retains that new location while restoring only matching secret +components. +A source that only implements `ConfigSource` is read-only and Apply returns HTTP 409. The response is 202 with `status: written_and_refresh_scheduled` only after the serialized source write succeeds; parsing, module application, and publication still run asynchronously and are visible in the status endpoint. After Apply completes, the Console repopulates the editor with the submitted document only if its monotonic editor revision is unchanged. A newer local edit wins over the stale desired snapshot returned by the scheduled refresh. ## Refresh And Observe @@ -163,7 +257,7 @@ curl -fsS -H "Authorization: Bearer $OPERATOR_TOKEN" \ "$LIVEFORGE_API/api/v1/server/config" ``` -Refresh success is 202 and means scheduled, not already loaded. Status success is 200. The refresh route returns 401 for invalid credentials, 403 for a viewer, 429 when rate limited, and 503 when the manager is unavailable, closed, or not started. `SIGHUP` has the same asynchronous enqueue semantics and performs no source I/O in the signal loop. +Refresh success is 202 with `status: scheduled` and means scheduled, not already loaded. Config Apply success is 202 with `status: written_and_refresh_scheduled`. Status success is 200. The refresh route returns 401 for invalid credentials, 403 for a viewer, 429 when rate limited, and 503 when the manager is unavailable, closed, or not started. `SIGHUP` has the same asynchronous enqueue semantics and performs no source I/O in the signal loop. Status reports source/version/hash, last attempt/success, consecutive failures, redacted last error, pending restart paths, callback failures, superseded callback count, and accepted/rejected/application-failed counters. Coalescing retains the newest accepted callback and increments `dropped_callbacks` for each superseded pending notification. diff --git a/docs/recipes/whip-h265-opus-playback.md b/docs/recipes/whip-h265-opus-playback.md index 6f5771f8..e28f172b 100644 --- a/docs/recipes/whip-h265-opus-playback.md +++ b/docs/recipes/whip-h265-opus-playback.md @@ -40,9 +40,55 @@ Click **Preview** and verify each mode: | FMP4 | Fragmented MP4; Opus can pass through | | HLS | HLS or LL-HLS according to `http_stream.llhls.enabled`; the Console enables Hls.js low-latency part consumption | | DASH | Separate fMP4 representations; Opus can pass through; the Console keeps one fragment of live delay | -| WebRTC | WHEP realtime mode; waits for the next live keyframe | +| WebRTC | WHEP realtime mode; explicit low-latency path that waits for the next live keyframe | | WebRTC-Live | WHEP Live mode; replays an atomic GOP snapshot, then continues from the matching ring-buffer cursor | +When the WHEP mode is omitted, the server and Console use `mode=live` so an available +GOP keyframe is sent immediately. After the SDP answer, read the session `Location` +and append `/status` to inspect `feed.state`: `waiting_keyframe`, `playing`, +`no_media_input`, `media_stalled`, `codec_mismatch`, `sample_write_failed`, +`target_audio_failed`, `generation_ended`, and `closed` are distinct states. The status also contains +`expected_video`, `expected_audio`, `first_media_at`, the stable millisecond +`first_media_wait_ms`, `last_video_at`, `last_audio_at`, the startup +generation/cursor, sent and dropped media counters, and a bounded `last_error`. +Dropped counters cover negotiated tracks only, and session close captures one +final monotonic transport snapshot before storing terminal status. +Both expected tracks must advance before a mixed feed reports `playing`. Complete +startup silence becomes `no_media_input` after eight seconds; after startup, any +expected track that does not advance for eight seconds makes the feed +`media_stalled`. A mixed realtime feed remains `waiting_keyframe` while audio +advances but video interframes are discarded before the first IDR. Recovery +requires every stale expected track to advance again. Console derives the displayed +stale media names from server timestamps instead of listing every expected kind. +Every real feed-state transition emits one structured log containing generation, +cursor, mode, previous/next state, and a bounded error when present. Same-state +per-frame updates do not emit transition logs. + +An active WHEP reader overwrite is recovered per reader. The retained atomic +read result is never sent, and only that reader advances to live. One +condition-backed pump owns each reader's readiness check, atomic read, and +live advance; shutdown cancels and joins both pumps before releasing target +audio ownership once. A source +overwrite keeps established direct or transformed audio moving while video +returns to `waiting_keyframe`; video pacing/DTS/PTS state is reset and recovery +starts with the latest same-generation parameter sets plus a keyframe. An +audio-only feed resumes at the next live audio frame. A transformed target-audio +overwrite leaves clean source video continuous and resumes at the next valid +target frame. If that expected target reader closes while the publisher +generation is active, the feed terminates promptly as `target_audio_failed` +instead of waiting for `media_stalled` or silently becoming video-only. Each +overwrite warning identifies `reader=source|target_audio`, the exact +`overwritten` count, and `action=wait_keyframe|continue_audio`. + +WHEP negotiation is fail closed per requested source media kind. A source track on +a non-zero receiving offer m-line that has no compatible codec returns HTTP 415; +an internal local-track or AddTrack failure returns HTTP 500. Setup releases its +subscriber lease, connection slot, PeerConnection, and session entry on either +failure. A source kind omitted by the offer, disabled with port zero, or marked +`inactive`/`sendonly` is intentionally omitted and does not block another compatible +requested kind. Media-level direction overrides session-level direction, and codec +matching requires an exact `rtpmap` name on a payload listed by that m-line. + The plain HTTP FMP4 path establishes a near-zero timeline when the shared muxer starts, with the first cached GOP rebased once and later fragments preserving their relative DTS and PTS. Subscribers share already-muxed bytes, so a late subscriber is not independently rebased to zero. The Console appends fragments to an MSE `segments` SourceBuffer so explicit `tfdt` values and signed HEVC B-frame composition offsets remain authoritative, then starts playback at that subscriber's first buffered timestamp. An MSE failure aborts and releases the live response; a finite response reaches `endOfStream()` only after queued appends drain. For every mode, require all of the following: @@ -67,10 +113,12 @@ Check these failure signatures: - WHEP receives packets but decodes zero frames: inspect HEVC VPS/SPS/PPS conversion and confirm the keyframe carries Annex-B parameter sets. - WHEP Live renders only the cached GOP: verify the GOP cache and source-ring cursor are captured atomically before cache replay, source video always uses that cursor, and a separate transcode reader contributes only negotiated target audio. +- WHEP sends a post-gap P-frame, stalls after a source overwrite, or loses only transformed audio: verify atomic source/target read results are used, the retained result is discarded, only the named reader advances to live, source recovery requests the TrackSender keyframe gate with current parameter sets, and active target-audio EOF reports `target_audio_failed`. - FMP4 fails on the second fragment or reports a large starting timestamp: verify the shared muxer rebases both DTS and PTS by one baseline, the Console SourceBuffer remains in `segments` mode, and playback seeks to the first buffered range. - HLS fails at a segment boundary: verify each advertised independent segment starts at a video keyframe. The initial manifest must omit completed PART tags, while blocking reloads retain the latest completed PART identities so Hls.js neither appends cold-start media twice nor reloads a full segment after consuming its parts. - WHIP H.265 + Opus has a fixed audio offset across HLS, DASH, FLV, or TS: inspect the WHIP session timeline. Audio and video must be mapped onto one session clock from their packet arrival offsets; each track must not independently reset DTS to zero. For transcoded output, confirm the audio reader starts at the cached GOP source position. - WHEP video arrives in `80ms + 0ms` bursts after source audio pauses: the WebRTC audio transcode worker must wait on its ring reader condition and must not consume the shared source playback wakeup. Run `CGO_ENABLED=1 go test -tags audiocodec ./module/webrtc -run TestWHEPJitterDiagnostic/video_audio_transcode -count=1 -timeout=180s -v` and confirm zero bursts, no sequence gaps, and a stable jitter trend. +- `lf-test` realtime WHIP pacing rebases the current packet when processing falls behind the media timeline. This prevents an old absolute anchor from emitting a catch-up burst; the regression is covered by `TestWHIPRealtimePacerRebasesAfterFallingBehind`. - HLS or LL-HLS pauses after cached playback: verify the segmenter-specific compatibility path uses its combined historical transcode reader and filters only duplicated cached video by video DTS. Shared HTTP FLV/TS/fMP4 workers instead use independent direct-video and transformed-audio readers. The bundled Hls.js requires one completed segment in its initial manifest, then consumes low-latency parts; the server must not wait for the old three-segment buffer. The live reader begins at the atomic snapshot cursor, so no cross-track DTS watermark should discard a valid frame. - DASH startup takes multiple GOPs or stalls after a segment: the initial MPD should return after one complete keyframe-bounded segment, advertise a `minimumUpdatePeriod` of at most two seconds, preserve measured GOP durations in `SegmentTimeline`, and use one fragment of player live delay. A cold DASH manager still needs the next keyframe to close its first segment; for a 30 fps publisher with `keyint=250`, this bounded wait can approach 8.3 seconds but must not multiply across three segments. - FLV/TS playback has no audio: confirm the binary was built with `CGO_ENABLED=1 -tags audiocodec` and that FFmpeg libraries were found. For a same-session codec transition, confirm direct-AAC startup still has a shared target reader, generated target headers never claim direct ownership, a later G.711/Opus epoch creates fresh transform state, and unsupported epochs neither close the mapped track nor poison later readers. A late reader must begin at its snapshot epoch floor rather than retained older audio. diff --git a/docs/superpowers/plans/2026-08-28-stream-reliability-and-playback.md b/docs/superpowers/plans/2026-08-28-stream-reliability-and-playback.md new file mode 100644 index 00000000..662b98a3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-stream-reliability-and-playback.md @@ -0,0 +1,181 @@ +# Stream Reliability And Playback Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the confirmed WHEP playback regression and harden the documented cache, lifecycle, resource, configuration, performance, and Console gaps with reproducible verification. + +**Architecture:** Keep `core.StreamStartupSnapshot` as the only cross-module startup contract. WebRTC, HTTP, DVR, SIP, and GB28181 consumers use generation-aware admission and readers; diagnostics are attached to the owning session instead of inferred by the Console watchdog. Changes are split into independently testable phases so each can be reverted without changing the media model or reintroducing an audio cache. + +**Tech Stack:** Go 1.26, Pion WebRTC, Chromium/chromedp, Go race detector, Prometheus, YAML/JSON Schema, local UDP protocol labs. + +**Spec:** `docs/superpowers/specs/2026-08-28-stream-reliability-and-playback-design.md` + +## Global Constraints + +- Use `go 1.26` and keep `CGO_ENABLED=1` plus the `audiocodec` tag for the full baseline. +- Run `tools/check-agent-docs_test.sh` after every source change and `CHECK_AGENT_DOCS_DIFF=1 tools/check-agent-docs.sh` before integration. +- Preserve the single interleaved GOP cache; do not add or restore `audioCache`. +- Keep the sample configuration local-only and never commit secrets, recordings, binaries, or private URLs. +- Update `agent-manifest.json`, `llms-full.txt`, `README.md`, `README.zh-CN.md`, schema/OpenAPI/recipes when the changed behavior affects them. +- Commit as `im-pingo `; never use the `Pingos` identity for authored commits. + +--- + +### Task 1: WHEP startup state and H.264 first-frame path + +**Files:** +- Modify: `module/webrtc/whep.go` +- Modify: `module/webrtc/whep_feed.go` +- Modify: `module/webrtc/track_sender.go` +- Modify: `module/webrtc/session.go` +- Modify: `module/api/protocol_lab.go` +- Modify: `module/api/console.html` +- Test: `module/webrtc/whep_feed_test.go` +- Test: `module/webrtc/whep_e2e_test.go` +- Test: `module/webrtc/whep_browser_test.go` + +**Interfaces:** +- `whepFeedLoop` produces a terminal/ongoing `WHEPFeedStatus` containing generation, startup cursor, mode, readiness, dropped frames, sent frames, first-media time, RTP counters when available, and a redacted terminal error. +- `Session` exposes a concurrency-safe diagnostic snapshot for the Console/API path without exposing mutable internals. +- The Console uses `mode=live` for its default preview and renders explicit waiting/error states from the returned session status. + +- [x] **Step 1: Write failing tests** for realtime waiting-keyframe diagnostics, live cached-keyframe startup, empty-cache behavior, source generation termination, H.264 Annex-B output, and propagated `WriteSample` failure. +- [x] **Step 2: Run focused WebRTC tests** with `go test ./module/webrtc -run 'WHEP|whep' -count=1 -v`; confirm each new regression test fails for the expected missing state/error behavior. +- [x] **Step 3: Implement the status model and make feed writes return structured errors**; preserve audio-only live-cursor behavior and the existing single GOP cache. +- [x] **Step 4: Change only the Console default to `mode=live`**, keep explicit realtime semantics, and render waiting-keyframe versus terminal failure distinctly. +- [x] **Step 5: Run focused tests and the browser H.264 test**; inspect SDP codec, dimensions, advancing `currentTime`, RTP counts, and media errors. +- [x] **Step 6: Update WebRTC/OpenAPI/recipe/AI-facing documentation** for the status fields and default mode, then run the agent-doc checks. +- [x] **Step 7: Commit** with `git -c user.name='im-pingo' -c user.email='cczjp89@gmail.com' commit` after verification. + +### Task 2: Core cache bounds and generation-safe consumers + +**Files:** +- Modify: `config/config.go` +- Modify: `config/validate.go` +- Modify: `docs/config/config.schema.json` +- Modify: `core/stream.go` +- Modify: `core/stream_hub.go` +- Modify: `core/transcode_manager.go` +- Modify: `module/httpstream/module.go` +- Modify: HLS/DASH/LL-HLS manager files under `module/httpstream/` +- Test: `core/stream_test.go` +- Test: `core/stream_hub_test.go` +- Test: `pkg/util/ringbuffer_test.go` +- Test: `module/httpstream/*_test.go` + +**Interfaces:** +- Stream config exposes validated `gop_cache_max_frames`, `gop_cache_max_duration`, and `gop_cache_max_bytes` with bounded defaults. +- Cleanup APIs take `streamKey` and optional publisher generation/identity and never remove a newer generation. +- `NewRingBuffer` is safe for direct zero/negative-capacity callers while config validation remains fail-closed. + +- [x] **Step 1: Add failing tests** for each GOP bound, zero/negative ring capacity, stale destroy after republish, and historical HTTP stream registry cleanup. +- [x] **Step 2: Run `go test ./core ./pkg/util ./module/httpstream -run 'GOP|Ring|Destroy|Republish' -count=1`** and record the expected failures. +- [x] **Step 3: Implement bounded cache eviction and ring constructor protection** without changing interleaved audio ownership. +- [x] **Step 4: Replace pointer-retaining HTTP registration and make manager cleanup generation-aware**; make publisher timeout remove idle streams only when the generation still matches. +- [x] **Step 5: Run package tests, `go test -race ./core ./pkg/util ./module/httpstream`, and targeted allocation benchmarks. +- [x] **Step 6: Update schema, config recipe, manifest and llms docs** with defaults and memory-bound semantics. +- [x] **Step 7: Commit** the independently verified core reliability phase as `im-pingo`. + +### Task 3: Strict connection, RTP ownership, and shutdown + +**Files:** +- Modify: `core/server.go` +- Modify: `module/dvr/handler.go` +- Modify: `module/dvr/session.go` +- Modify: `module/gb28181/rtp_receiver.go` +- Modify: `module/gb28181/device_registry.go` +- Modify: `module/gb28181/module.go` +- Modify: `pkg/ratelimit/ratelimit.go` +- Modify: `module/httpstream/handler.go` +- Modify: `module/httpstream/handler_hls.go` +- Test: `core/server_test.go` +- Test: `module/dvr/*_test.go` +- Test: `module/gb28181/*_test.go` +- Test: `pkg/ratelimit/ratelimit_test.go` + +**Interfaces:** +- `Server.AcquireConn` is an atomic admission operation that never returns true above the configured limit. +- Device registry readers receive immutable snapshots; registry and limiter close operations are idempotent. +- Forwarded client IP is accepted only through an explicit trusted-proxy policy. +- All stream responses observe request cancellation and bounded write/header deadlines. + +- [x] **Step 1: Add failing concurrency, buffer-aliasing, snapshot-race, double-close, trusted-proxy, and cancellation tests.** +- [x] **Step 2: Run focused tests with `-race` and confirm the regressions reproduce.** +- [x] **Step 3: Implement CAS connection admission, DVR release-once, owned RTP payloads, immutable registry snapshots, and idempotent close.** +- [x] **Step 4: Replace cancellable streaming sleeps and add bounded HTTP deadlines without changing valid playlist contents.** +- [x] **Step 5: Run `go test -race ./core ./module/dvr ./module/gb28181 ./module/httpstream ./pkg/ratelimit` and inspect goroutine/resource cleanup.** +- [x] **Step 6: Update security/operations docs and manifest entries for trusted proxies, connection coverage, and shutdown guarantees.** +- [x] **Step 7: Commit** with the required `im-pingo` author. + +### Task 4: Hot-path and configuration-source hardening + +**Files:** +- Modify: `core/stream.go` +- Modify: `core/stream_stats.go` +- Modify: `module/gb28181/outbound_media.go` +- Modify: `module/sipgateway/call_session.go` +- Modify: `config/runtime/manager.go` +- Modify: `config/runtime/source_consul.go` +- Modify: `config/runtime/source_redis.go` +- Modify: `module/api/config.go` +- Modify: `module/metrics/collector.go` +- Test: corresponding package tests and new `*_bench_test.go` files beside changed packages + +**Interfaces:** +- `Stream.WriteFrame` keeps media ordering while using stable publisher identity and a narrower critical section. +- Runtime source reads/writes retain serialization and immutable snapshots, but unchanged versions do not repeat full application work. +- Configuration redaction covers URL userinfo and error values; metrics expose bounded labels or a documented opt-in stream detail mode. + +- [x] **Step 1: Add failing behavior tests** for URL/userinfo redaction, unchanged refresh, publisher identity hot path, and bounded metric labels. +- [x] **Step 2: Add baseline benchmarks** for `WriteFrame`, ring readers, RTMP/RTSP/RTP output and config refresh; capture before numbers. +- [x] **Step 3: Implement one optimization at a time**, running its focused tests after each change; preserve packet timing and byte counts. +- [x] **Step 4: Run race tests and benchmarks** with `go test -bench='BenchmarkRingReader|BenchmarkRTMPConn' -benchmem ./pkg/util ./module/cluster` plus focused new benchmarks. +- [x] **Step 5: Update configuration/security/metrics docs** and record measured limits without claiming unmeasured capacity. +- [x] **Step 6: Commit** with the required `im-pingo` author. + +### Task 5: Console, protocol matrix, and release verification + +**Files:** +- Modify: `module/api/console.html` +- Modify: `module/api/protocol_lab.go` +- Modify: `module/api/config.go` +- Modify: `module/api/recording.go` +- Modify: `module/api/console_management_test.go` +- Modify: `module/api/protocol_testlab_api_test.go` +- Modify: `module/api/config_api_test.go` +- Modify: `module/api/recording_test.go` +- Modify: `agent-manifest.json` +- Modify: `llms.txt` +- Modify: `llms-full.txt` +- Modify: `README.md` +- Modify: `README.zh-CN.md` +- Modify: `docs/api/openapi.yaml` +- Modify: `docs/recipes/protocol-test-lab.md` +- Modify: `docs/recipes/runtime-config-sources.md` +- Modify: `docs/recipes/recording-dvr-management.md` +- Modify: `docs/TECHNICAL-RISKS.md` + +**Interfaces:** +- Console group hierarchy has System-level Config/Security and Workspace-level media/lab/storage views. +- Config exposes complete redacted effective/desired documents, schema, source details, validation, apply/refresh state and pending restart paths for file/http/https/consul/redis. +- Protocol lab responses expose separate source/target stream, codec, audio/video/RTCP counters, generation and cross-protocol playback links. +- Storage distinguishes disabled record module, empty/incomplete output, complete playable recording, and DVR availability. + +- [x] **Step 1: Add failing DOM/API contract tests** for navigation groups, all config fields/source kinds, redaction, lab counters/links, and disabled storage. +- [x] **Step 2: Run focused API tests and browser smoke checks** to verify the failures represent missing behavior. +- [x] **Step 3: Implement the smallest UI/data changes** and keep labels tied to actual API capability states. +- [x] **Step 4: Run local SIP/GB28181 publish and receive labs**, then test WHEP, HTTP-FLV/TS/fMP4, HLS/DASH and recording playback where codecs allow it. +- [x] **Step 5: Run the complete verification matrix:** + +```bash +go test ./... +CGO_ENABLED=1 go build -tags audiocodec ./cmd/liveforge +CGO_ENABLED=1 go test -tags audiocodec -race -coverprofile=coverage.out -covermode=atomic ./... +tools/check-agent-docs_test.sh +CHECK_AGENT_DOCS_DIFF=1 tools/check-agent-docs.sh +git diff --check +jq empty agent-manifest.json +``` + +- [x] **Step 6: Review the final diff against the spec and risk table**, run `git status`, and verify no recordings, secrets, binaries or generated profiles are staged. +- [x] **Step 7: Commit** only after all commands above have fresh successful output, then push/merge only when CI is green. diff --git a/docs/superpowers/specs/2026-08-28-stream-reliability-and-playback-design.md b/docs/superpowers/specs/2026-08-28-stream-reliability-and-playback-design.md new file mode 100644 index 00000000..4948fc37 --- /dev/null +++ b/docs/superpowers/specs/2026-08-28-stream-reliability-and-playback-design.md @@ -0,0 +1,86 @@ +# LiveForge 流媒体可靠性与播放设计 + +**日期：** 2026-08-28 +**状态：** 已批准执行 +**范围：** WebRTC 首帧与真实 H.264、核心缓存与生命周期、连接容量与协议出站、运行时配置与指标、Console 与跨协议验收 + +## 目标 + +1. 控制台和 API 创建的 WHEP 播放在正常 GOP 周期内稳定收到可解码首帧，不再把“等待关键帧”误报为“无媒体”。 +2. SIP/GB28181/WHIP/RTMP 等输入经过服务器后，能够通过 WebRTC 和其他已启用输出进行可重复的跨协议验证。 +3. 缓存、连接限制、publisher generation、RTP 接收/发送和模块 shutdown 在异常、并发和重启场景下有界且无 data race。 +4. 热路径在保持媒体时序和协议语义不变的前提下减少不必要的锁、反射、分配和系统调用，并提供可重复基准。 +5. Config 页面、协议实验室和 Storage 页面只展示已实现且可验证的能力，配置源、权限、敏感信息和重启语义保持一致。 + +## 设计决策 + +### 1. WebRTC 启动模型 + +WHEP 使用一次性的 `StreamStartupSnapshot`，其中包含 publisher identity、generation、MediaInfo、sequence headers、交错 GOP replay frames、LiveCursor 和 GenerationDone。`mode=live` 发送该 snapshot 中的完整可用 GOP，然后从 LiveCursor 读取新帧；`mode=realtime` 不发送 replay frames，但等待后续 video keyframe。纯音频两种模式都从 LiveCursor 直接开始，不等待视频关键帧，也不引入独立 audio cache。 + +Console 的默认 WHEP 链接使用 `mode=live`，显式 `mode=realtime` 仍然可用并显示“等待关键帧”状态。首帧门控状态定义为 `waiting_keyframe`、`playing`、`no_media_input`、`codec_mismatch`、`sample_write_failed`、`generation_ended` 和 `closed`。feed loop 每次状态变化写结构化日志，并把首帧时间、等待时长、generation、cursor、丢弃帧、发送音视频帧、RTP 计数和最后错误绑定到 session 诊断；`WriteSample` 错误不再静默丢弃。 + +H.264/H.265 输入统一走 AVCC/HVCC 到 Annex-B 的访问单元转换。关键帧携带缓存的 SPS/PPS/VPS；空访问单元、缺失参数集、协商 codec 不匹配和发送错误分别失败关闭。浏览器回归使用真实 Chromium H.264 解码结果判定：`readyState`、视频尺寸、`currentTime` 推进、音频帧计数和 media error 必须同时满足，SDP 成功或 `ontrack` 不能单独算通过。 + +### 2. 核心缓存与 generation + +GOP cache 仍然是以视频关键帧开始、包含该 GOP 内交错音频的单一 replay cache；纯音频只使用 ring live cursor。GOP cache 在 `GOPCacheNum` 之外增加单 GOP 帧数、持续时间和字节上限，三者任一达到上限时保留从当前关键帧开始的可播放内容并停止继续增长；配置值经过 schema 和运行时校验，默认值保持当前行为的有界版本。 + +`Stream.WriteFrame` 在 generation/publisher 校验和 ring 写入之间保持单写者顺序，但把不需要保护的统计更新移出大锁；publisher identity 使用稳定的接口 identity 或显式 token，不在每帧使用 reflection。所有 subscriber admission、replay reader 和异步 manager cleanup 必须带 generation；旧 generation 的 stop 事件不得清理新 generation 的资源。 + +### 3. 资源与连接边界 + +`AcquireConn` 使用 CAS 循环实现严格的 max connection 上限，所有 HTTP、WebRTC、DVR 和协议 session 路径使用同一个 release-once 约定。DVR 请求在创建 session 前占用连接配额，取消、错误和正常结束均释放。RingBuffer 对非法容量进行构造期保护，避免直接调用工具包时除零或越界。 + +GB28181 RTP receiver 对从 UDP buffer 交给重排队列的 payload 做拥有式复制，DeviceRegistry 对外只返回 immutable snapshot，Stop/Close 均幂等。HTTP streaming 设置 header/write deadline 和 request cancellation 响应；HLS/DASH/LL-HLS 等等待路径使用 context-aware condition，不再用无法取消的固定 sleep。 + +### 4. 性能与观测 + +协议出站优先复用 packet/fragment buffer，在不改变所有权的地方使用 `net.Buffers` 或批量写；配置 refresh 保持 source I/O 串行，但解析/hash/diff/application 不阻塞下一次调度，重复版本快速返回。Prometheus 的 stream labels 使用受控、可配置的采集策略，默认不把任意高基数 stream key 扩散到无限时间序列；必要的流明细通过管理 API 获取。 + +所有优化必须有行为测试和 benchmark，benchmark 只用来比较相对变化，不能替代容量验收。错误日志不得包含 source URL credentials、bearer token 或 SIP 密码；限流器只有在明确配置可信代理时才读取 forwarded headers，否则使用 RemoteAddr。 + +### 5. Console 和配置 UX + +Console 顶层分为 Workspace、Operations、System；Config 和 Security 只属于 System，Streams、GB28181、SIP Calls、Storage 属于 Workspace，Cluster 属于 Operations。Config 页面分别显示完整 redacted effective document、desired source document、schema、source details、pending restart、校验结果和 apply 状态，不把不可写 source 伪装成可编辑。 + +SIP/GB28181 lab 的 publish 与 receive 都展示 source/target stream、RTP/RTCP、音视频帧、generation、codec、错误和跨协议 playback links。lab 只使用 loopback fake device，不依赖外部平台；H.264/G.711/Opus/AAC 的输出能力通过 capability matrix 明确显示“可用、需 FFmpeg、视频-only 或不支持”。录像默认 fMP4/MP4，Storage 在 record module 缺失时显示 disabled 状态而不是模块错误。 + +## 阶段与验收 + +### 阶段 A：WHEP 首帧与 H.264 + +- 添加 realtime/live/纯音频/稀疏关键帧/无 cache/样本写入失败测试。 +- 添加真实 GB28181、SIP 和 WHIP H.264 输入到 WHEP 的浏览器回归。 +- 修正默认 Console 模式和状态展示。 +- 验收：默认 WHEP 在 8 秒内推进 `currentTime`；失败时日志和 UI 能区分四类根因。 + +### 阶段 B：核心可靠性 + +- 增加 GOP 三类上限、ring 非法容量保护、publisher identity 热路径优化。 +- 修复 generation-aware cleanup、publisher timeout、HTTP 注册表历史指针和 subscriber admission。 +- 验收：race 测试、替换 publisher 压力测试、缓存内存上限测试全部通过。 + +### 阶段 C：连接、RTP 和 shutdown + +- 修复严格连接上限、DVR 配额、GB28181 packet ownership、DeviceRegistry snapshot、幂等 close、HTTP cancellation。 +- 验收：并发超限永不超过配置值，所有路径释放资源，`go test -race` 无数据竞争和 close panic。 + +### 阶段 D：性能与配置 + +- 低锁热路径、出站 buffer、配置刷新调度、指标高基数策略和基准。 +- 修复 config source 脱敏和 trusted proxy 语义。 +- 验收：基准报告保存在 PR/变更说明中，功能测试与配置源 contract 测试通过。 + +### 阶段 E：Console 与发布验收 + +- 完成页面分组、Config 全量字段/源适配、协议 lab 能力矩阵和 Storage disabled/回放状态。 +- 更新 manifest、llms、README、schema、OpenAPI 和 recipes。 +- 验收：本地无外部平台完成 SIP/GB28181 publish/receive、跨协议播放、录像回放，并通过完整构建、race、文档检查和 CI。 + +## 非目标 + +- 本轮不实现 WebRTC simulcast layer selection；配置继续标记为 deferred/unsupported。 +- 不把没有 FFmpeg 的构建描述成支持非 AAC 音频转码；无依赖构建只保证其声明的 codec 和视频-only fallback。 +- 不改变已公开的 stream-key escaping、权限和 bearer token 语义，除非测试证明当前行为违反安全契约。 + diff --git a/internal/localfs/root.go b/internal/localfs/root.go index 6a84c825..07eae2b1 100644 --- a/internal/localfs/root.go +++ b/internal/localfs/root.go @@ -329,6 +329,15 @@ func (d *Dir) MoveToUnique(base string, candidate func(int) string) (string, err } func (d *Dir) List(ctx context.Context) ([]Entry, error) { + return d.list(ctx, false) +} + +// ListAll reports every direct child without following symbolic links. +func (d *Dir) ListAll(ctx context.Context) ([]Entry, error) { + return d.list(ctx, true) +} + +func (d *Dir) list(ctx context.Context, includeNonRegular bool) ([]Entry, error) { dup, err := unix.Openat(d.fd, ".", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) if err != nil { return nil, mapPathError(err) @@ -352,19 +361,32 @@ func (d *Dir) List(ctx context.Context) ([]Entry, error) { if err := unix.Fstatat(d.fd, entry.Name(), &stat, unix.AT_SYMLINK_NOFOLLOW); err != nil { return nil, mapPathError(err) } - if stat.Mode&unix.S_IFMT != unix.S_IFREG { + if stat.Mode&unix.S_IFMT != unix.S_IFREG && !includeNonRegular { continue } result = append(result, Entry{ RelPath: joinRel(d.rel, entry.Name()), Size: stat.Size, - Mode: os.FileMode(stat.Mode), + Mode: entryFileMode(os.FileMode(stat.Mode)), ModTime: statModTime(stat), }) } return result, nil } +func entryFileMode(mode os.FileMode) os.FileMode { + switch mode & os.FileMode(unix.S_IFMT) { + case os.FileMode(unix.S_IFREG): + return mode + case os.FileMode(unix.S_IFDIR): + return mode | os.ModeDir + case os.FileMode(unix.S_IFLNK): + return mode | os.ModeSymlink + default: + return mode | os.ModeIrregular + } +} + func (r *Root) Fstatfs(stat *unix.Statfs_t) error { return unix.Fstatfs(r.fd, stat) } func (p *Pending) Name() string { @@ -425,6 +447,30 @@ func (p *Pending) StatSibling(base string) (os.FileInfo, error) { return info, nil } +// CreateSiblingPending creates an exclusive pending file in the directory +// pinned by p. Later path replacement cannot redirect the new object. +func (p *Pending) CreateSiblingPending(base string, perm os.FileMode) (*Pending, error) { + if !validBase(base) { + return nil, ErrInvalidPath + } + dirFD, err := unix.Openat(p.dirFD, ".", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return nil, mapPathError(err) + } + fd, err := unix.Openat(dirFD, base, unix.O_CREAT|unix.O_EXCL|unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOFOLLOW, uint32(perm.Perm())) + if err != nil { + _ = unix.Close(dirFD) + return nil, mapPathError(err) + } + file := os.NewFile(uintptr(fd), filepath.Join(p.rootPath, filepath.FromSlash(joinRel(p.dirRel, base)))) + if file == nil { + _ = unix.Close(fd) + _ = unix.Close(dirFD) + return nil, fmt.Errorf("create sibling pending file") + } + return &Pending{File: file, dirFD: dirFD, dirRel: p.dirRel, base: base, rootPath: p.rootPath}, nil +} + func (p *Pending) WriteSiblingAtomic(base string, data []byte, perm os.FileMode) error { if !validBase(base) { return ErrInvalidPath diff --git a/internal/localfs/root_test.go b/internal/localfs/root_test.go index 8923b777..28e60020 100644 --- a/internal/localfs/root_test.go +++ b/internal/localfs/root_test.go @@ -15,6 +15,14 @@ func rejectHardLinks(t *testing.T) { t.Cleanup(func() { linkAt = original }) } +func TestEntryFileModeAcceptsNamedFileMode(t *testing.T) { + mode := os.FileMode(unix.S_IFDIR | 0o750) + got := entryFileMode(mode) + if !got.IsDir() || got.Perm() != 0o750 { + t.Fatalf("entryFileMode(%#o) = %#o, want directory mode 0750", mode, got) + } +} + func TestOpenRootRejectsUnsupportedHardLinksWithoutProbeArtifacts(t *testing.T) { path := t.TempDir() rejectHardLinks(t) diff --git a/llms-full.txt b/llms-full.txt index fab0bfd0..96eb1284 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -8,7 +8,9 @@ LiveForge is an MIT-licensed Go 1.26+ live streaming server. It ingests, transmu For a source-oriented Chinese architecture walkthrough, see [docs/architecture.zh-CN.md](docs/architecture.zh-CN.md). It documents the current module boundaries, AVFrame flow, GOP snapshot/cursor consistency, SPMC RingBuffer semantics, SharedBuffer/MuxerManager, protocol egress, cluster forwarding/origin pull, and on-demand audio transcoding. It is intentionally based on implemented code; older design drafts are not treated as runtime behavior. -The current review record, including performance bottlenecks, lifecycle and resource risks, functional boundaries, and the open Console WHEP regression, is [docs/TECHNICAL-RISKS.md](docs/TECHNICAL-RISKS.md). The WHEP regression is not closed by SDP success or an `ontrack` callback: the browser must receive a decodable frame and an advancing media clock. +The current review record, including performance bottlenecks, lifecycle and resource risks, functional boundaries, and the Console WHEP regression record, is [docs/TECHNICAL-RISKS.md](docs/TECHNICAL-RISKS.md). The WHEP regression is not closed by SDP success or an `ontrack` callback: the browser must receive a decodable frame and an advancing media clock. + +The API, WebRTC signaling, and metrics HTTP servers use the same transport bounds: `ReadHeaderTimeout` is 5 seconds and `IdleTimeout` is 2 minutes. These bounds protect slow header parsing and idle keep-alive connections; existing handler and media write deadlines remain unchanged, and no server-level `WriteTimeout` is added by this policy. Startup readiness requires a sequence header only for H.264, H.265, and AAC. AV1, VP8, VP9, Opus, MP3, G.711, G.722, and G.729 can start from the publisher @@ -23,16 +25,24 @@ demuxed DTS intervals retain the source timing. Headerless Opus and MP3 use 48 kHz stereo and 44.1 kHz stereo defaults respectively; explicit positive rate and channel arguments remain authoritative. -The core publisher lifecycle rejects nil and typed-nil `Publisher` values before changing timers, state, or generation. `MuxerManager.GetOrCreateMuxer` creates or reuses an instance only for an active publishing generation; before first publish or after publisher removal it returns no reader/instance and retires any mapped instance from the ended generation. +The core publisher lifecycle rejects nil and typed-nil `Publisher` values before changing timers, state, or generation. Every non-empty publisher ID is generation-unique for one `Stream` lifetime: A -> B -> A reuse is rejected before ownership, timers, generation, media, sequence headers, stats, transcode tracks, GOP, or ring state changes, while a newly created `Stream` has an independent identity lifetime. `StreamStateDestroying` is terminal: late conditional or unconditional publisher cleanup cannot move the stream back to `NoPublisher`, later admission cannot attach either an identified or empty-ID publisher, and destruction notification remains single-shot against the closed ring. `MuxerManager.GetOrCreateMuxer` creates or reuses an instance only for an active publishing generation; before first publish or after publisher removal it returns no reader/instance and retires any mapped instance from the ended generation. Every production network or session ingress writer binds frames to the publisher that owns its connection through `Stream.WriteFrameForPublisher`. Delayed callbacks from a replaced publisher are rejected before they can mutate the replacement generation's media information, startup cache, or ring buffer; stream-oriented stale receive loops terminate, while datagram callbacks may drop frames after teardown. Within one publisher generation, accepted audio frames are stamped with source provenance and a codec epoch that increments whenever the source audio codec changes. RTSP refreshes publisher-session activity for active TCP-interleaved RTP/RTCP and UDP RTP at a linearizable boundary under the stream publisher-generation lock: replacement before that boundary prevents the old session's timeout refresh and terminates its TCP/UDP ingress, while replacement after it follows activity accepted for the then-active publisher. RTP parsing and publisher-bound media writes remain outside the activity callback; callbacks under the generation lock must not re-enter `Stream`. Stale RTCP and stale valid or malformed RTP therefore terminate without refreshing the old session. Cluster SRT origin pull treats the read error caused by intentional connection close after publisher-generation replacement as normal termination, while a read error for the still-active relay publisher remains a transport failure for health and retry accounting. Raw `Stream.WriteFrame` remains available only for tests and explicit internal injection. -Direct RTMP, RTSP, SRT, and WHEP playback and shared HTTP FLV/TS/fMP4 muxer workers bind startup to one ready `StreamStartupSnapshot`. Initial media information, sequence headers, replay frames, the post-snapshot `LiveCursor`, the historical transform-input `SourceCursor`, the current private audio-epoch floor, publisher generation, and `GenerationDone` therefore come from one lock-consistent view. Direct ring readers start at `LiveCursor`; only audio-transform input starts at `SourceCursor`. Snapshot-bound transformed readers expose retained source video where required but filter audio below that epoch floor, so late readers cannot emit stale audio or move DTS backward before current output. Their generation check is atomic with transcode track lookup and creation under the stream-to-transcode lock order: a stale snapshot returns no reader and cannot create, reuse, decrement, close, or poison a replacement-generation track. RTMP, WHEP, and shared HTTP muxers use independent direct-media and transformed-audio readers when transcoding is required, so transformed history cannot replay direct video or duplicate a startup header. Shared HTTP FLV/TS/fMP4 workers also acquire the shared target-AAC track when startup is already direct AAC; source-provenance copies are rejected in favor of the direct reader, but the retained subscription observes a later G.711/Opus epoch without missing its first transform frame or stopping direct video. A shared HTTP muxer has exactly one audio owner per source codec epoch: transformed AAC owns G.711/Opus epochs, compatible source AAC owns direct epochs, and ownership can return to transformed AAC without replacing the publisher generation. Transcoder-generated headers and encoded frames carry transformed provenance, while target-codec frames copied from the source retain source provenance. Therefore the generated startup AAC header is suppressed as already represented by muxer init data and can never masquerade as a direct-source handoff; a real source AAC header transfers ownership, stale queued output from older epochs is discarded, and a later transformed epoch reacquires ownership exactly once. Each HTTP worker keeps its own transformed reader until worker shutdown, so handoff does not release or close a shared producer used by another FLV/TS/fMP4 worker, RTMP/WHEP reader, or HLS/LL-HLS/DASH combined reader. Reader release captures the exact acquired shared-track instance, so a delayed old-generation release cannot decrement or cancel a replacement generation's same-codec track. The shared target-codec producer remains reference counted across transitions: each transcode-required source epoch creates fresh decoder, encoder, resampler, timestamp tracker, and PCM state; target-codec source frames pass through; incompatible unsupported epochs are dropped without closing or removing the track; a later supported epoch emits a current target header and media to existing and new readers; combined tracks continue source video. Direct video remains on its `LiveCursor` reader throughout. TS emits the real direct header's refreshed PAT/PMT before the first direct AAC PES and retains the AAC declaration when a direct-start worker changes to transformed AAC. Readers close when that generation ends and re-check `IsPublisherGeneration` after each blocking read before processing its frame, so the first frame from a replacement publisher cannot reach an old subscriber. Snapshot headers and replay frames are emitted once. RTMP forwards later live sequence headers. RTSP builds DESCRIBE SDP from one ready snapshot stored under the session lock; PLAY uses that same snapshot and rejects the session if its publisher generation has retired, while sequence-header RTP remains omitted because SDP carries parameter sets. SRT and shared HTTP TS rebuild MPEG-TS track configuration when a live sequence header changes known tracks and emit the refreshed PAT/PMT before the first media frame on the new track. SRT uses the snapshot cursor as its sole replay/live duplicate boundary and does not apply a cross-track maximum-DTS filter, so lower-DTS live audio remains deliverable after a higher-DTS cached video frame. HTTP requests release the exact `MuxerInstance` they acquired. A not-yet-ready worker watches the `GenerationDone` captured for that instance and terminates on removal instead of waiting into a replacement generation; ready workers use that same generation-bound snapshot, preventing an old request from decrementing or feeding a replacement-generation muxer. +Direct RTMP, RTSP, SRT, and WHEP playback and shared HTTP FLV/TS/fMP4 muxer workers bind startup to one ready `StreamStartupSnapshot`. Initial media information, sequence headers, replay frames, the post-snapshot `LiveCursor`, the historical transform-input `SourceCursor`, the current private audio-epoch floor, publisher generation, and `GenerationDone` therefore come from one lock-consistent view. Direct ring readers start at `LiveCursor`; only audio-transform input starts at `SourceCursor`. Snapshot-bound transformed readers expose retained source video where required but filter audio below that epoch floor, so late readers cannot emit stale audio or move DTS backward before current output. Their generation check is atomic with transcode track lookup and creation under the stream-to-transcode lock order: a stale snapshot returns no reader and cannot create, reuse, decrement, close, or poison a replacement-generation track. RTMP, WHEP, and shared HTTP muxers use independent direct-media and transformed-audio readers when transcoding is required, so transformed history cannot replay direct video or duplicate a startup header. Shared HTTP FLV/TS/fMP4 workers also acquire the shared target-AAC track when startup is already direct AAC; source-provenance copies are rejected in favor of the direct reader, but the retained subscription observes a later G.711/Opus epoch without missing its first transform frame or stopping direct video. A shared HTTP muxer has exactly one audio owner per source codec epoch: transformed AAC owns G.711/Opus epochs, compatible source AAC owns direct epochs, and ownership can return to transformed AAC without replacing the publisher generation. Transcoder-generated headers and encoded frames carry transformed provenance, while target-codec frames copied from the source retain source provenance. Therefore the generated startup AAC header is suppressed as already represented by muxer init data and can never masquerade as a direct-source handoff; a real source AAC header transfers ownership, stale queued output from older epochs is discarded, and a later transformed epoch reacquires ownership exactly once. Each HTTP worker keeps its own transformed reader until worker shutdown, so handoff does not release or close a shared producer used by another FLV/TS/fMP4 worker, RTMP/WHEP reader, or HLS/LL-HLS/DASH combined reader. Reader release captures the exact acquired shared-track instance, so a delayed old-generation release cannot decrement or cancel a replacement generation's same-codec track. The shared target-codec producer remains reference counted across transitions: each transcode-required source epoch creates fresh decoder, encoder, resampler, timestamp tracker, and PCM state; target-codec source frames pass through; incompatible unsupported epochs are dropped without closing or removing the track; a later supported epoch emits a current target header and media to existing and new readers; combined tracks continue source video. Direct video remains on its `LiveCursor` reader throughout. TS emits the real direct header's refreshed PAT/PMT before the first direct AAC PES and retains the AAC declaration when a direct-start worker changes to transformed AAC. Readers close when that generation ends and re-check `IsPublisherGeneration` after each blocking read before processing its frame, so the first frame from a replacement publisher cannot reach an old subscriber. Snapshot headers and replay frames are emitted once. RTMP forwards later live sequence headers. RTSP builds DESCRIBE SDP from one ready snapshot stored under the session lock; PLAY uses that same snapshot and rejects the session if its publisher generation has retired, while sequence-header RTP remains omitted because SDP carries parameter sets. SRT and shared HTTP TS rebuild MPEG-TS track configuration when a live sequence header changes known tracks and emit the refreshed PAT/PMT before the first media frame on the new track. SRT uses the snapshot cursor as its sole replay/live duplicate boundary and does not apply a cross-track maximum-DTS filter, so lower-DTS live audio remains deliverable after a higher-DTS cached video frame. HTTP requests release the exact `MuxerInstance` they acquired. A not-yet-ready worker watches the `GenerationDone` captured for that instance and terminates on removal instead of waiting into a replacement generation; ready workers use that same generation-bound snapshot, preventing an old request from decrementing or feeding a replacement-generation muxer. FLV and fMP4 requests that cannot obtain initialization data within the bounded startup wait return HTTP 503 instead of an empty HTTP 200. HTTP-FLV, TS, and fMP4 refresh a 10-second write/flush deadline for every media chunk, and WebSocket streaming bounds every message write with a 10-second context deadline. + +Continuous HTTP-FLV, HTTP-TS, and fMP4 muxer inputs consume atomic overwrite results from both the direct-source and transformed-audio readers. The first overwrite records its exact input kind and count, cancels and joins both pumps, and makes the retained post-gap frame unsendable. FLV and TS stop without muxing it; fMP4 discards pending partial media on overwrite but preserves the clean-completion flush. HTTP and WebSocket output readers also discard a retained overwritten packet before writing it. HTTP ends the response, while WebSocket closes with a bounded `TryAgainLater` continuity-loss reason; clean producer end remains a normal WebSocket closure. + +SIP, GB28181, Record, DVR, and cluster push egress all bind startup to one atomic publisher-generation snapshot. SIP inbound INVITEs run synchronous `EventPublish` authorization before RTP allocation, then emit matching asynchronous publish-start and publish-stop events after the publisher is active, so Record/DVR consumers follow and finalize SIP sessions. SIP and GB28181 keep that snapshot through signaling, response wait, ACK, admission, and media activation; retirement before ACK/activation aborts the stale setup rather than pairing old signaling with a replacement publisher. DVR likewise carries the validated snapshot through retained-index and storage recovery, then revalidates that same stream generation immediately before installing the session; replacement during setup discards the candidate and closes only resources it acquired. If a 2xx has already accepted the SIP dialog when generation retirement wins, cleanup sends one BYE through the accepted-dialog/session path before releasing the transaction; cancellation before acceptance remains close-only. Protocols emit only the captured headers/replay required by their container or signaling contract, then create direct readers at `LiveCursor`; `GenerationDone` cancels the reader and a generation check after wakeup discards a raced replacement frame. Pure-audio startup has no replay frames and never starts at the retained ring oldest position. SIP and GB28181 subscriber releases are generation-scoped. Record and DVR derive expected tracks from `snapshot.MediaInfo`; sequence-header-only or empty Record sessions fail, and DVR does not publish a successful segment without media. Cluster RTMP/PS preserves header/container order; GB28181 PS header-send errors are returned with their startup stage before replay/live continues, while RTP/RTSP omit sequence-header media carried by SDP. See [docs/cluster-guide.md](docs/cluster-guide.md), [docs/cluster-guide.zh-CN.md](docs/cluster-guide.zh-CN.md), and [docs/architecture.zh-CN.md](docs/architecture.zh-CN.md). -SIP, GB28181, Record, DVR, and cluster push egress all bind startup to one atomic publisher-generation snapshot. SIP inbound INVITEs run synchronous `EventPublish` authorization before RTP allocation, then emit matching asynchronous publish-start and publish-stop events after the publisher is active, so Record/DVR consumers follow and finalize SIP sessions. SIP and GB28181 keep that snapshot through signaling, response wait, ACK, admission, and media activation; retirement before ACK/activation aborts the stale setup rather than pairing old signaling with a replacement publisher. If a 2xx has already accepted the SIP dialog when generation retirement wins, cleanup sends one BYE through the accepted-dialog/session path before releasing the transaction; cancellation before acceptance remains close-only. Protocols emit only the captured headers/replay required by their container or signaling contract, then create direct readers at `LiveCursor`; `GenerationDone` cancels the reader and a generation check after wakeup discards a raced replacement frame. Pure-audio startup has no replay frames and never starts at the retained ring oldest position. SIP and GB28181 subscriber releases are generation-scoped. Record and DVR derive expected tracks from `snapshot.MediaInfo`; sequence-header-only or empty Record sessions fail, and DVR does not publish a successful segment without media. Cluster RTMP/PS preserves header/container order; GB28181 PS header-send errors are returned with their startup stage before replay/live continues, while RTP/RTSP omit sequence-header media carried by SDP. See [docs/cluster-guide.md](docs/cluster-guide.md), [docs/cluster-guide.zh-CN.md](docs/cluster-guide.zh-CN.md), and [docs/architecture.zh-CN.md](docs/architecture.zh-CN.md). +SIP Gateway reserves and binds each RTP/RTCP pair before SDP and transfers socket ownership to the admitted call. A requested PCMA/PCMU target may use an independent generation-bound transcode reader; every ready outbound frame is packetized before final admission, then rechecks cancellation and publisher generation under the terminal send gate immediately before RTP send. Outbound media carries atomic source and target-audio read results independently, discards a retained post-gap value, and advances only the affected reader. Source overwrite keeps transformed audio flowing and gates direct H.264 until the latest same-generation sequence header followed by an IDR; target-audio overwrite keeps direct video continuous and resumes audio at live media. Active-generation target-audio EOF fails the call as `network_lost`; terminal paths close send admission and owned sockets, wait for admitted sends without holding lifecycle or admission locks, and only then publish terminal state and callbacks. The dual-reader parent cancels and joins both media pumps before returning. Publisher retirement closes and releases the transcode reader, generation subscriber, and sockets, frees the pair for exact reuse, and converges with late teardown triggers on one BYE. Protocol Lab receive workflows use the same readiness rule: a known unsupported SIP audio codec is rejected before waiting, while SIP and GB28181 wait for the captured publisher generation's required sequence headers before sending outbound signaling. A late header can therefore be canceled by the caller instead of creating a partially negotiated call; receive-mode test fixtures must provide the source header when they expect synchronous activation. +ARCH-030 is closed at the shared transcode boundary. Shared transformed output now uses an internal by-value envelope containing the original `AVFrame` pointer, a valid source-ring `SourceSpan` for media, and separate target-header kind/epoch metadata. Each snapshot-bound bridge applies its own `SourceCursor` floor and emits media only when `SourceSpan.Begin >= floor`; packets crossing the floor are dropped and stale audio epochs remain filtered. Each track retains the latest eight target sequence headers by epoch, so a lagging bridge replays only the header whose epoch equals its first accepted payload; when that bounded cache has no matching header, the AAC payload is dropped and the miss is not treated as satisfied. Direct target audio and pass-through video retain their exact source-frame span and payload backing, while decode, attributed resampling/encoding, fixed-frame PCM aggregation, padding, and terminal drain preserve valid conservative spans. A source-ring overwrite terminates only that generation-bound shared producer with the exact typed overwrite count, discards the retained post-gap frame, and suppresses clean codec-tail finalization; an internal output-bridge overwrite similarly closes only that bridge before forwarding its retained value. ARCH-031 is closed after protocol-local overwrite and keyframe/discontinuity recovery was implemented and covered by focused and race tests. + +The continuous HTTP/WebSocket, HTTP segmenter, SIP, and GB28181 portions of ARCH-031 now handle overwrite explicitly. HLS and LL-HLS discard abandoned partial state, advance the affected reader to live, refresh same-generation headers/container state, keyframe-gate video, resume audio-only on the next live audio frame, and mark the first recovered output with a discontinuity; LL-HLS also abandons one MSN per recovery epoch and wakes blocked reloads. DASH preserves completed single-Period media but discards current batches and retires the manager. SIP preserves reader identity, advances only the overwritten source or target-audio reader, recovers direct H.264 at a fresh same-generation header plus IDR while unaffected audio continues, and treats active-generation transformed-audio EOF as terminal `network_lost`. GB28181 applies the same reader-local overwrite rule: wait-only pumps queue reader readiness, the merge performs each atomic read and drains queued control before pending output, so an observed source overwrite, target-audio overwrite, or active target EOF cannot be overtaken by pre-gap RTP. Source loss clears pending video, replaces PS state without resetting SSRC or RTP sequence, and resumes H.264 only at the newest post-gap same-generation header plus IDR while unaffected audio continues; target-audio loss clears only pending target audio, preserves clean source video and PS state, and keeps that video's original holdback deadline. Active target-audio EOF fails the session, and every terminal path cancels and joins both pumps. ARCH-031 is now complete across WHEP, RTMP, RTSP, SRT, cluster, Record, and DVR; long-duration and high-concurrency capacity remain operational follow-up tests. + ## Capability matrix | Capability | Publish | Play | Default port or path | Prerequisites | @@ -51,7 +61,17 @@ Protocol Lab receive workflows use the same readiness rule: a known unsupported The implementation supports protocol bridging through the shared stream hub. Exact codec compatibility depends on the source, destination, and whether the audio transcoding build is enabled. -Known WebRTC regression status: the Console can report `No advancing media received (check codec support and keyframes)` because its default realtime feed starts after the captured live cursor and gates video until a later keyframe; a long GOP can outlast the eight-second watchdog. Current H.264 `mode=live` browser playback decodes and automated Pion/VP8 paths pass, but the default behavior, write-error diagnostics, and real GB28181/SIP H.264-to-browser path still require a fix and regression coverage. See the technical risk record before changing the feed loop. +The stream startup cache remains one interleaved GOP cache: it begins at a video keyframe and includes the audio and video frames that follow it. Each GOP is bounded independently by `stream.gop_cache_max_frames` (300 by default), `stream.gop_cache_max_duration` (10s), and `stream.gop_cache_max_bytes` (32 MiB); zero disables only that bound, and combined bounds retain the shortest permitted playable prefix. With GOP caching enabled, at least one positive frame or byte bound is required; duration-only configuration is rejected because equal-DTS frames would otherwise be unbounded. Duration admission uses the full unordered min/max DTS span with overflow-safe comparison and preserves insertion/media order. Reaching a bound retains the keyframe and playable prefix until the next keyframe. A hot reload trims every retained GOP under the new policy and recomputes the active GOP seal: tightening may shorten and seal those playable prefixes, while relaxation allows only the active retained GOP to admit future interleaved frames under every remaining bound. Older retained GOPs stay trimmed, and frames already omitted or trimmed are not restored; the next keyframe starts a new complete GOP. Pure-audio streams use only the live cursor and never use an independent audio cache. `stream.ring_buffer_size` is rejected when non-positive during configuration validation, while direct RingBuffer construction uses a one-slot safety fallback and direct streams without a hard GOP bound receive a 300-frame fallback. + +The Console's default WHEP path uses the atomic live GOP startup, while explicit realtime mode may wait for the next keyframe. Protocol Lab exposes `whep` and `whep_live` as `mode=live` and a distinct `whep_realtime` as `mode=realtime`; Console buttons consume those matching metadata fields. Every source media kind actually requested by a receiving, non-zero SDP m-line must negotiate: media direction inherits session direction when no media-level direction is present, and a codec matches only an exact `rtpmap` name whose payload is listed by that m-line. An unsupported requested codec fails the WHEP POST with 415, an internal track/AddTrack failure returns 500, and all setup resources are released; an omitted, disabled, inactive, or send-only source kind does not fail another requested kind. `GET /webrtc/session/{sessionId}/status` reports expected media kinds, first successful sample time and stable `first_media_wait_ms`, per-kind last-advance timestamps, generation, cursor, mode, feed state, negotiated-track media counters, actual RTP packets/bytes, received RTCP packets, and bounded sample-write errors. Unrequested source kinds do not inflate dropped counters, and Session close captures one final monotonic transport snapshot before the tombstone is stored. Feed termination closes the WHEP session and releases its generation lease, connection slot, lifecycle lane, PeerConnection, and active map entry; at most 64 terminal status tombstones remain for two minutes. Complete startup silence for eight seconds reports recoverable `no_media_input`; realtime interframes dropped before the first IDR remain `waiting_keyframe`, including mixed feeds whose audio is already advancing. A requested mixed feed does not become `playing` until every expected kind advances. After any media starts, eight seconds without advancement from any expected kind reports recoverable `media_stalled`, and all stale kinds must advance before recovery; Console derives and names only stale expected kinds from server timestamps. Real state transitions emit one structured log with generation, cursor, mode, previous/next state, and a bounded error when present; same-state frame updates do not log. Terminal states reject ordinary late media, watchdog, and transport-stat updates. Invalid H.264/H.265 parameter sets and empty video access units terminate as `codec_mismatch`, and audio sample-write failures stop every direct, cached, or transformed feed path immediately. The tagged Chromium matrix verifies SIP publish to GB28181 receive plus WHEP, GB28181 publish to SIP receive plus WHEP, and WHIP H.264/Opus publish to both SIP and GB28181 receive plus WHEP when the browser offer advertises H.264. It requires expected decoded dimensions, advancing media time, increasing audio/video RTP and decoded-frame counters, connected ICE, and non-stalled server RTP/RTCP status; a browser without H.264 receive support is reported as an environment skip while Pion negotiation coverage remains mandatory. `LIVEFORGE_PROTOCOL_MATRIX_SOAK=60s` extends those checks per second without becoming a deployment-capacity claim. SDP success or an `ontrack` callback alone is not proof of playback. See the technical risk record before changing the feed loop. + +WHEP overwrite recovery uses independent atomic source and transformed target-audio reads. One pump exclusively owns each reader's condition wait, atomic read, and live advance, so readiness observation cannot race another goroutine consuming the same cursor. Every retained post-gap value is discarded and only the affected reader advances to a captured live cursor. Source overwrite retains the original publisher generation, keeps established direct or transformed audio moving, resets video pacing/DTS/PTS state, enters the existing TrackSender keyframe gate, refreshes the latest same-generation H.264/H.265 parameter sets, and returns to `playing` only after current video and every expected track advance; audio-only feeds resume at the next live frame. Target-audio overwrite preserves clean source video and resumes at the next valid target frame. An active expected target-audio EOF, including shared-producer source overwrite, terminates promptly as `target_audio_failed`; cancellation closes and joins both reader pumps and releases target ownership once. Each overwrite logs `protocol=whep`, `reader=source|target_audio`, the exact atomic overwrite count, and `action=wait_keyframe|continue_audio` without payload or stream-key labels. + +WHEP status includes `source_overwrites`, a source-ring continuity-loss count +kept separate from per-track dropped counters because mixed source positions +cannot be attributed to video or audio. Direct source audio pacing resets at +the same overwrite boundary; transformed target-audio pacing remains +independent. WebRTC registers Opus, PCMA, and PCMU. SIP Lab publish sessions carry H.264 video plus PCMA/PCMU audio on separate RTP tracks; the Console selects a video @@ -59,12 +79,20 @@ element and uses WHEP for direct browser playback without FFmpeg. HTTP-FLV, HTTP-TS, fMP4, HLS, and DASH remain available as protocol outputs but do not promise browser playback of G.711 audio without a compatible muxer or transcode path. +SIP Gateway allocates RTP/RTCP pairs by binding both UDP sockets before SDP is +accepted or offered, skips pairs already occupied by another local process, and +keeps those sockets bound through session cleanup. Local SIP Lab media sockets +also avoid the configured gateway RTP range. Server startup is transactional across registered modules. If a listener or module fails to initialize, shutdown preserves the original error and closes only modules whose initialization was attempted, in reverse order. Later uninitialized modules are not closed, and SRT close remains safe before initialization. Normal RTSP listener closure is not logged as an accept error. +Asynchronous publish/subscribe lifecycle dispatch is bounded and ordered per stream/client/consumer. Start admission reserves every queue slot needed by matching terminal hooks, including consumers that only observe stop; a rejected start never marks a protocol session active and triggers resource rollback. Accepted stop hooks therefore cannot be displaced by ordinary start traffic. Server shutdown is idempotent, stops and joins the alive-event producer, closes attempted modules in reverse order, then drains accepted asynchronous hooks up to `server.drain_timeout` (30 seconds when unset). + +HTTP rate limiting uses the direct peer address unless that peer matches an explicit `limits.rate_limit.trusted_proxies` IP/CIDR entry. Only then may `X-Forwarded-For` or `X-Real-IP` supply the client identity. XFF is parsed from right to left: configured trusted proxy hops are stripped and the first untrusted hop owns the bucket, so an attacker-controlled left prefix cannot rotate identities. A malformed non-empty XFF chain falls back to the direct peer. Invalid or empty trusted-proxy entries are rejected during bootstrap and runtime validation. + Console preview URLs are built from the active bound listener returned by `GET /api/v1/server/info`; wildcard bind addresses are resolved to the host that served the Console. If `127.0.0.1:8080` is served by nginx or another helper, HTTP preview requests can return that process's 404 while RTMP on 1935 and WHEP on 8443 continue to work. Confirm the media response status, `Content-Type`, and `Server` header, then release the conflicting port or configure `http_stream.listen` to an unused address. -HLS, LL-HLS, and DASH keep the atomic GOP-cache snapshot continuous with subsequent live interframes and only advertise video segments that begin at a keyframe. AAC-only streams close segments by elapsed media time before appending the boundary frame, so the boundary starts the next segment exactly once and completed HLS TS, DASH audio m4s, and LL-HLS TS/fMP4 segments become available while the source is live. LL-HLS `part_duration` controls partial segments; hot-reloadable `segment_duration` controls completed full segments, defaults to 1.0 second, and has a schema minimum of 0.1 second. The initial LL-HLS playlist waits for one completed segment because the bundled Hls.js rejects a part-only initial level, and it omits that completed segment's PART tags so cold startup cannot append the same media twice. Its millisecond-rounded wait covers `segment_duration + part_duration`, preserves a 10-second floor, caps at 30 seconds, and returns HTTP 503 rather than a part-only manifest when the bound expires or the manager ends. Subsequent blocking reloads retain PART tags for the latest completed segment so a client that already consumed those parts correlates the completed segment instead of fetching and appending its full URI again. This avoids the old three-segment wait and segment-boundary `bufferAppendError` while preserving low-latency part consumption. DASH returns its MPD after the first completed segment, omits the video adaptation for audio-only streams, caps `minimumUpdatePeriod` at two seconds for timely long-GOP discovery, retains exact per-segment durations in `SegmentTimeline`, and configures dash.js with a one-fragment live delay instead of a fixed number of seconds. Its fMP4 fragments retain one continuous relative decode timeline when the source DTS origin is zero or non-zero. HLS, LL-HLS, and DASH manifests URL-escape every stream-key segment, DASH XML-escapes generated URL attributes, and segment routing treats the final path component as the media filename while preserving arbitrarily deep valid preceding stream keys. Pure-audio verification uses `/STREAM_KEY.m3u8`, `/STREAM_KEY/0.ts` or `/0.m4s`, `/STREAM_KEY.mpd`, `/STREAM_KEY/audio_init.mp4`, and `/STREAM_KEY/a1.m4s`; see [docs/recipes/protocol-test-lab.md](docs/recipes/protocol-test-lab.md). +HLS, LL-HLS, and DASH keep the atomic GOP-cache snapshot continuous with subsequent live interframes and only advertise video segments that begin at a keyframe. AAC-only streams close segments by elapsed media time before appending the boundary frame, so the boundary starts the next segment exactly once and completed HLS TS, DASH audio m4s, and LL-HLS TS/fMP4 segments become available while the source is live. On an atomic ring overwrite, HLS and LL-HLS discard the retained value and all uncommitted media, advance to a same-generation live cursor, refresh sequence headers and muxer state, and mark exactly the first recovered segment/part with `#EXT-X-DISCONTINUITY`; video waits for a new keyframe while audio-only resumes immediately. If the refreshed audio plan changes between direct and shared transformed input, each segmenter closes and releases the old reader once and opens the new source at the refreshed live cursor without replaying GOP history. Refreshed LL-HLS topology resets keyframe gating, including when video first appears in the same generation. LL-HLS removes abandoned current-part URLs, advances and broadcasts its MSN once per recovery epoch, and content-versions changed fMP4 init bytes. Each advertised retained fMP4 segment or part references its immutable matching init epoch; old versioned init URLs continue serving those bytes until no retained media references them, and unknown or evicted versions return 404. DASH cannot bridge a gap inside its single Period, so it preserves completed init/timeline segments, discards current batches, retires, and ends future segment waits. If a transformed combined reader ends while its source generation remains active, all three discard partial state instead of performing clean finalization. A generation-matched publish-stop removes the manager from request lookup without interrupting its producer; the manager drains every accepted frame through the captured exclusive generation end cursor and finalizes once. A replacement publisher uses a distinct manager and cannot enter the retired generation output. HTTP module shutdown and segment-policy reload still force-stop tracked managers, and module shutdown joins both active and already-retired workers. LL-HLS `part_duration` controls partial segments; hot-reloadable `segment_duration` controls completed full segments, defaults to 1.0 second, and has a schema minimum of 0.1 second. The initial LL-HLS playlist waits for one completed segment because the bundled Hls.js rejects a part-only initial level, and it omits that completed segment's PART tags so cold startup cannot append the same media twice. Its millisecond-rounded wait covers `segment_duration + part_duration`, preserves a 10-second floor, caps at 30 seconds, and returns HTTP 503 rather than a part-only manifest when the bound expires or the manager ends. Subsequent blocking reloads retain PART tags for the latest completed segment so a client that already consumed those parts correlates the completed segment instead of fetching and appending its full URI again. Blocking and initial LL-HLS condition waits terminate when the request/hold is canceled or the manager stops, so module shutdown cannot leave a reload waiting after its stop wake. This avoids the old three-segment wait and segment-boundary `bufferAppendError` while preserving low-latency part consumption. DASH returns its MPD after the first completed segment, omits the video adaptation for audio-only streams, caps `minimumUpdatePeriod` at two seconds for timely long-GOP discovery, retains exact per-segment durations in `SegmentTimeline`, and configures dash.js with a one-fragment live delay instead of a fixed number of seconds. Its fMP4 fragments retain one continuous relative decode timeline when the source DTS origin is zero or non-zero. HTTP stream requests do not set a write deadline before readiness waits; manifest, init, segment, and streaming chunk paths refresh a 10-second deadline immediately before each actual write, so a valid delayed HLS/DASH response cannot expire while waiting for its first segment. HLS, LL-HLS, and DASH manifests URL-escape every stream-key segment, DASH XML-escapes generated URL attributes, and segment routing treats the final path component as the media filename while preserving arbitrarily deep valid preceding stream keys. Pure-audio verification uses `/STREAM_KEY.m3u8`, `/STREAM_KEY/0.ts` or `/0.m4s`, `/STREAM_KEY.mpd`, `/STREAM_KEY/audio_init.mp4`, and `/STREAM_KEY/a1.m4s`; see [docs/recipes/rtmp-to-hls.md](docs/recipes/rtmp-to-hls.md). The fMP4 demuxer also parses complete media segments assembled by concatenating multiple `moof`/`mdat` fragments, as produced by some LL-HLS full-segment paths. @@ -73,9 +101,15 @@ recording is stopped. Before finalization it waits, with a bounded timeout, for the shared AAC transcode track to consume input through that cursor and then drains generated frames. This preserves frames submitted before an immediate stop without keeping the recording open indefinitely; generation replacement -still cancels the old session. +still cancels the old session. When a publisher generation reaches its finite +source boundary, a fixed-size audio transform flushes samples retained by its +resampling filter, encodes complete frames, silence-pads the remaining PCM, and +emits every delayed encoder packet exactly once with target-frame-size DTS steps +before closing the output ring. Record and DVR can +therefore drain the complete old-generation tail; cancellation caused only by +the last consumer disappearing may discard output that no consumer owns. -The Console has a verified browser workflow for a WHIP source carrying H.265/HEVC video and Opus audio. Its Preview surface covers HTTP-FLV, WS-FLV, HTTP-TS, FMP4, HLS, DASH, WHEP realtime, and WHEP Live. WHIP maps the independent audio/video RTP clocks onto one session timeline using packet-arrival offsets, so a track callback or codec clock cannot introduce a fixed multi-second DTS offset. Shared HTTP FLV/TS/fMP4 muxers keep direct video on a `LiveCursor` reader and obtain transformed audio history from an independent `SourceCursor` reader. The segmenting HLS, LL-HLS, and DASH compatibility paths still use their combined historical transcode reader and video-only duplicate filter; they otherwise continue from the atomic snapshot cursor without a cross-track DTS watermark, so a later audio DTS cannot hide a valid live video frame. Plain HTTP FMP4 creates a near-zero timeline when its shared muxer starts and preserves relative DTS/PTS across fragments; later subscribers receive shared already-muxed bytes rather than a private timestamp rewrite. The Console uses MSE `segments` mode and starts at the first buffered timestamp so explicit `tfdt` values and signed HEVC B-frame composition offsets remain intact. MSE failure tears down the fetch, reader, queue, SourceBuffer, and object URL; end-of-stream waits for queued appends to drain. WHEP Live uses the atomic source-ring cursor for uninterrupted video and a separate target-codec reader for transcoded audio. The audio transcode worker blocks on its reader condition instead of consuming the shared source wakeup, preventing video bursts when source audio pauses. SIP Lab streams carry H.264 plus PCMA/PCMU; the Console selects a video element and binds both remote tracks. Truly audio-only G.711 streams still select the audio element, rebind the remote MediaStream after the track arrives, and start playback monitoring immediately. WHEP preview starts asynchronously received media muted when browser autoplay policy requires it and exposes an explicit Unmute/Mute control, so video becomes visible without dropping the audio track. Verification requires a decoded frame, non-zero video dimensions, an advancing media clock, and no media error; a `Playing` status string alone is not evidence of playback. WHEP also exposes decoded-frame, FPS, keyframe, loss, and audio RTP statistics. Use [docs/recipes/whip-h265-opus-playback.md](docs/recipes/whip-h265-opus-playback.md). +The Console has a verified browser workflow for a WHIP source carrying H.265/HEVC video and Opus audio. Its Preview surface covers HTTP-FLV, WS-FLV, HTTP-TS, FMP4, HLS, DASH, WHEP realtime, and WHEP Live. WHIP maps the independent audio/video RTP clocks onto one session timeline using packet-arrival offsets, so a track callback or codec clock cannot introduce a fixed multi-second DTS offset. Shared HTTP FLV/TS/fMP4 muxers keep direct video on a `LiveCursor` reader and obtain transformed audio history from an independent `SourceCursor` reader. The segmenting HLS, LL-HLS, and DASH compatibility paths still use their combined historical transcode reader and video-only duplicate filter; they otherwise continue from the atomic snapshot cursor without a cross-track DTS watermark, so a later audio DTS cannot hide a valid live video frame. Plain HTTP FMP4 creates a near-zero timeline when its shared muxer starts and preserves relative DTS/PTS across fragments; later subscribers receive shared already-muxed bytes rather than a private timestamp rewrite. The Console uses MSE `segments` mode and starts at the first buffered timestamp so explicit `tfdt` values and signed HEVC B-frame composition offsets remain intact. For G.711 sources it declares AAC in the FMP4 SourceBuffer only when `GET /api/v1/server/info` reports `capabilities.audio_transcoding=true`; that flag requires configuration plus working G.711 A-law and mu-law to AAC paths in the current process. MSE failure tears down the fetch, reader, queue, SourceBuffer, and object URL; end-of-stream waits for queued appends to drain. WHEP Live uses the atomic source-ring cursor for uninterrupted video and a separate target-codec reader for transcoded audio. The audio transcode worker blocks on its reader condition instead of consuming the shared source wakeup, preventing video bursts when source audio pauses. SIP Lab streams carry H.264 plus PCMA/PCMU; the Console selects a video element and binds both remote tracks. Truly audio-only G.711 streams still select the audio element, rebind the remote MediaStream after the track arrives, and start playback monitoring immediately. WHEP preview starts asynchronously received media muted when browser autoplay policy requires it and exposes an explicit Unmute/Mute control, so video becomes visible without dropping the audio track. Verification requires a decoded frame, non-zero video dimensions, an advancing media clock, and no media error; a `Playing` status string alone is not evidence of playback. WHEP also exposes decoded-frame, FPS, keyframe, loss, and audio RTP statistics. Use [docs/recipes/whip-h265-opus-playback.md](docs/recipes/whip-h265-opus-playback.md). ## Build profiles @@ -128,29 +162,45 @@ The local Docker Compose workflow builds the image from source by default. A rel The complete HTTP contract is [docs/api/openapi.yaml](docs/api/openapi.yaml). Management responses, including GB28181 Lab responses and their 400/404 errors, use a JSON envelope with `code`, `message`, and optional `data`. Protocol-specific GB28181 device/session/control endpoints can return direct GB JSON errors, while WebRTC uses SDP/plain text. Management authentication accepts `api.auth.bearer_token`, named viewer/operator/admin tokens, or an authenticated console session. `GET /api/v1/server/health` remains public. TLS API listeners set `Secure` on the HttpOnly, SameSite=Strict `lf_session` cookie; plain HTTP listeners leave it unset for local development. The permission-aware console tabs, in order, are Streams, GB28181, Config, Cluster, SIP Calls, Storage, and Security. Recent Audit is a surface inside Security, not a separate tab. Visual groups are Workspace (Streams, GB28181, SIP Calls, Storage), Operations (Cluster), and System (Config, Security). Viewer reads config document/schema and validates without writing; operator controls apply/refresh/calls/kick/live-playback; admin owns deletions/debug/internal mutation. -Recording supports FLV, FMP4, MP4, TS, and HLS plus authenticated listing, status, metadata, range download, inline range playback, and deletion. New recordings default to fMP4 and `.mp4`, with delayed media-track initialization so audio is retained when it arrives after video. fMP4 Record declares only AAC directly; G.711, Opus, MP3, and other non-AAC source audio use the generation-bound AAC transform through the shared `audiocodec`/FFmpeg path when available. Without that optional dependency, unsupported audio is filtered and the result remains playable video-only. DVR TS applies the corresponding target-codec normalization. Sessions that end before a media frame arrives are preserved as failed and are never exposed as completed playable files. `GET /api/v1/recordings/{recordingPath}/play` returns the completed media with a format-specific MIME type, `Content-Disposition: inline`, and standard HTTP Range behavior; active/not-ready recordings return 409. When the record module is absent, `GET /api/v1/recordings/status` returns 200 with `state=disabled`; item media routes still return 503. DVR exposes session and storage status and serves time-shift playlists/segments on its configured listener. The Storage Console previews completed recordings (native MP4/fMP4 or mpegts.js FLV/TS) and opens HLS playback for DVR sessions with segments. The DVR listener permits non-credentialed cross-origin HLS fetches for this split-port Console, while playlist and segment GETs still run synchronous subscribe authorization hooks only and do not emit asynchronous subscribe lifecycle work. Recording preview reuses the authenticated management session; the Console never persists or appends bearer tokens, so configure DVR subscribe authorization for the independent media listener. Use [docs/recipes/recording-dvr-management.md](docs/recipes/recording-dvr-management.md). +Recording supports FLV, FMP4, MP4, TS, and HLS plus authenticated listing, status, metadata, range download, inline range playback, and deletion. New recordings default to fMP4 and `.mp4`, with delayed media-track initialization so audio is retained when it arrives after video. Every rotated recording file restores the publisher's declared tracks and deep-copied latest audio/video sequence headers into its new container writer, then rebases each track to a zero-based file-local decode timeline, so FLV, fMP4, MP4, and TS files are independently initialized. TS emits PAT/PMT before the first media PES even when audio arrives first; classic MP4 owns independent previous-DTS state and timescales for audio and video sample durations, normalizes `mvhd`/`tkhd` durations to the movie timescale while retaining each `mdhd` media timescale, saturates out-of-range sample/version-0 timing fields instead of wrapping, emits `ctts` version 1 for any negative PTS-DTS offset and version 0 otherwise, and uses expandable AAC ESDS descriptor lengths. Deployments must retain rotation for files that could approach the version-0 duration limit. fMP4 Record declares only AAC directly; G.711, Opus, MP3, and other non-AAC source audio use the generation-bound AAC transform through the shared `audiocodec`/FFmpeg path when available. Without that optional dependency, unsupported audio is filtered and the result remains playable video-only. DVR TS applies the corresponding target-codec normalization. Sessions that end before a media frame arrives are preserved as failed and are never exposed as completed playable files. Plain GET and DELETE use the complete recording ID, including IDs ending in `/play` or `/download`; explicit `?action=play` and `?action=download` select inline range playback or range download for that full ID. Legacy suffix actions remain available only when no exact ID exists. Recording play/download acquires one global connection slot before opening media, releases it exactly once on every return path, and sets a 10-second write deadline immediately before `ServeContent`. Local deletion recognizes only exact TS segment/playlist names and their defined recovery variants as sidecars, removes cleanup artifacts before the primary, and leaves the primary retriable after any cleanup failure. When the record module is absent, `GET /api/v1/recordings/status` returns 200 with `state=disabled`; item media routes still return 503. DVR exposes session and storage status and serves time-shift playlists/segments on its configured listener. DVR `Close` captures one absolute drain deadline before waiting for admission/setup ownership; callers receive a timeout at that bound while already-started cleanup continues in the background. Finite DVR playlist and segment responses have a 10-second server write bound; admitted success, error, cancellation, and timeout paths each release exactly one global connection slot. Range handling and `ServeContent` metadata remain unchanged. The Storage Console previews completed recordings (native MP4/fMP4 or mpegts.js FLV/TS) and opens HLS playback for DVR sessions with segments. The DVR listener permits non-credentialed cross-origin HLS fetches for this split-port Console, while playlist and segment GETs still run synchronous subscribe authorization hooks only and do not emit asynchronous subscribe lifecycle work. Recording preview reuses the authenticated management session; the Console never persists or appends bearer tokens, so configure DVR subscribe authorization for the independent media listener. Use [docs/recipes/recording-dvr-management.md](docs/recipes/recording-dvr-management.md). -The SIP and GB28181 Console pages include local one-shot protocol labs that do not need a remote platform or device. SIP self-test runs an in-process fake-peer REGISTER/401/digest, INVITE/200/ACK/BYE, rejection/timeout, RTP media, and RTCP loop. GB28181 self-test runs an in-process fake-device REGISTER, Keepalive, Catalog, PS/90000 INVITE/SDP/ACK/BYE, rejection/timeout, PS-over-UDP media, and RTCP loop. Both providers additionally support transport-backed persistent fake-device sessions through `StartLabSession(ctx, LabSessionRequest)`, `ListLabSessions()`, and idempotent `StopLabSession(id)`. SIP publish negotiates separate H.264 video and PCMA/PCMU audio tracks into a gateway-created stream; the gateway binds and parses both RTCP receivers, and the Lab reports receiver-side packet counts. SIP receive accepts the gateway outbound INVITE, consumes the existing source without writing generated frames into it, counts each received track, and sends periodic per-track receiver reports; outbound sender reports use each RTP track's SSRC, RFC NTP timestamps, and per-track packet/octet counts. GB28181 publish starts a listening fake device, performs real REGISTER, Keepalive, and Catalog signaling, then invokes the normal server-initiated live-play path through the registered Contact; the fake device consumes INVITE/ACK/BYE and sends constrained-baseline H.264 plus 8 kHz mono G.711A as PS over RTP payload type 96 with RTCP into LiveForge's real RTP/RTCP receiver. GB28181 receive requires an existing source with both H.264 and G.711A, admits its stream subscriber synchronously before signaling activation, then uses a module-owned outbound media session to send PS/RTP/RTCP to the fake device; Lab code only receives and accounts the media. Subscriber-limit rejection fails `StartLabSession` without publishing an active Lab. A later outbound sender failure moves the Lab to `failed`, records a bounded redacted diagnostic, and releases its dialog, module session, subscriber, sockets, and ports. Both use the same native-dependency-free moving 160x90 sample at 25 fps with one IDR per second and audible 20 ms audio frames. Persistent GB28181 sessions continue sending Keepalive at roughly one-third of `gb28181.keepalive.timeout` with a bounded practical interval, so long-running previews do not expire the simulated device. When both modules share one SIP listener, dispatch identifies H.264 plus PCMA/PCMU RTP offers as SIP Gateway traffic and video RTP/AVP payload 96 with `PS/90000` as GB28181 traffic, so a SIP lab request such as `d1` / `s1` cannot be claimed by the GB28181 handler. Both labs validate identities, reject duplicates, bind simulator sockets to loopback, and release dialogs, UAs, sockets, ports, and goroutines on idempotent stop. Lab stream keys are printable ASCII up to 256 bytes and every slash-separated segment must be non-empty and neither `.` nor `..`; runtime validation, OpenAPI, and the Console enforce the same rule. Managers retain every active session plus at most 16 terminal records, pruning the oldest terminal records only. Failed sessions retain a bounded `last_error` redacted for SIP credentials and bearer tokens before truncation. An initialized SIP transport and enabled gateway or GB28181 module are required; standalone managers remain contract-only and report no active transport session. Session API responses include aggregate and per-track counters plus enabled cross-protocol playback paths; every accepted stream-key path segment is URL-escaped, DASH URL attributes are XML-escaped, and absolute RTMP/RTSP URLs use actual bound listener addresses with wildcard hosts normalized to the management request host. Console Lab Preview consumes the returned playback paths directly, while generic stream previews use the same segment-wise escaping as a fallback. The Streams API and Console expose a keyframe-driven GOP generation with interleaved video/audio frame counts and duration; audio-only streams report startup GOP as not applicable. A disabled module returns 503 and the Console renders that as unavailable. Use [docs/recipes/protocol-test-lab.md](docs/recipes/protocol-test-lab.md). +The SIP and GB28181 Console pages include local one-shot protocol labs that do not need a remote platform or device. SIP self-test runs an in-process fake-peer REGISTER/401/digest, INVITE/200/ACK/BYE, rejection/timeout, RTP media, and RTCP loop. GB28181 self-test runs an in-process fake-device REGISTER, Keepalive, Catalog, PS/90000 INVITE/SDP/ACK/BYE, rejection/timeout, PS-over-UDP media, and RTCP loop. Both providers additionally support transport-backed persistent fake-device sessions through `StartLabSession(ctx, LabSessionRequest)`, `ListLabSessions()`, and idempotent `StopLabSession(id)`. SIP publish negotiates separate H.264 video and PCMA/PCMU audio tracks into a gateway-created stream; the gateway binds and parses both RTCP receivers, and the Lab reports receiver-side packet counts. SIP receive accepts the gateway outbound INVITE, consumes the existing source without writing generated frames into it, counts each received track, and sends periodic per-track receiver reports; the requested PCMA/PCMU value is the actual outbound target. When source and target differ, an available generation-bound shared transcode reader supplies target audio while direct H.264 stays on the source live cursor; unavailable conversions fail before signaling. Outbound sender reports use each RTP track's SSRC, RFC NTP timestamps, and per-track packet/octet counts. GB28181 publish starts a listening fake device, performs real REGISTER, Keepalive, and Catalog signaling, then invokes the normal server-initiated live-play path through the registered Contact; the fake device consumes INVITE/ACK/BYE and sends constrained-baseline H.264 plus 8 kHz mono G.711A as PS over RTP payload type 96 with RTCP into LiveForge's real RTP/RTCP receiver. GB28181 receive requires H.264 plus direct G.711A or source audio that the tagged runtime can convert to G.711A, admits its stream subscriber synchronously before signaling activation, then uses a module-owned outbound media session to send PS/RTP/RTCP to the fake device; direct H.264 stays on the source live cursor while transformed audio uses an independent generation-bound reader, and unavailable conversion fails before signaling; Lab code only receives and accounts the media. Subscriber-limit rejection fails `StartLabSession` without publishing an active Lab. A later outbound sender failure moves the Lab to `failed`, records a bounded redacted diagnostic, and releases its dialog, module session, subscriber, sockets, and ports. Both use the same native-dependency-free moving 160x90 sample at 25 fps with one IDR per second and audible 20 ms audio frames. Persistent GB28181 sessions continue sending Keepalive at roughly one-third of `gb28181.keepalive.timeout` with a bounded practical interval, so long-running previews do not expire the simulated device. When both modules share one SIP listener, dispatch identifies H.264 plus PCMA/PCMU RTP offers as SIP Gateway traffic and video RTP/AVP payload 96 with `PS/90000` as GB28181 traffic, so a SIP lab request such as `d1` / `s1` cannot be claimed by the GB28181 handler. Both labs validate identities, reject duplicates, bind simulator sockets to loopback, and release dialogs, UAs, sockets, ports, and goroutines on idempotent stop. Lab stream keys are printable ASCII up to 256 bytes and every slash-separated segment must be non-empty and neither `.` nor `..`; runtime validation, OpenAPI, and the Console enforce the same rule. Managers retain every active session plus at most 16 terminal records, pruning the oldest terminal records only. Failed sessions retain a bounded `last_error` redacted for SIP credentials and bearer tokens before truncation. An initialized SIP transport and enabled gateway or GB28181 module are required; standalone managers remain contract-only and report no active transport session. Session API responses include aggregate and per-track counters plus enabled cross-protocol playback paths; every accepted stream-key path segment is URL-escaped, DASH URL attributes are XML-escaped, and absolute RTMP/RTSP URLs use actual bound listener addresses with wildcard hosts normalized to the management request host. Console Lab Preview consumes the returned playback paths directly, while generic stream previews use the same segment-wise escaping as a fallback. The Streams API and Console expose a keyframe-driven GOP generation with interleaved video/audio frame counts and duration; audio-only streams report startup GOP as not applicable. A disabled module returns 503 and the Console renders that as unavailable. Use [docs/recipes/protocol-test-lab.md](docs/recipes/protocol-test-lab.md). The fMP4 demuxer consumes every concatenated `moof`/`mdat` pair in a complete media segment, so earlier fragments are not dropped. GB28181 PS egress converts AVCC/HVCC H.264 or H.265 samples to Annex-B before muxing. Persistent SIP and GB28181 Lab cleanup releases completed-request and transport-idle references, closes the underlying Lab socket, waits for the sipgo reader to exit, and only then closes the fake UA; peer listeners also stop before their UA. Normal Lab stop therefore has neither negative UDP references nor closed-socket pool cleanup warnings. +Each persistent SIP and GB28181 protocol-lab provider has an independent active-session admission ceiling configured by `sip.gateway.max_lab_sessions` or `gb28181.max_lab_sessions`. The default is 16; `starting`, `active`, and SIP `contract` sessions count, terminal history does not, non-positive values use the default, and a full ceiling returns HTTP 429 before socket or media allocation. + RTP/GB28181 cluster signaling paths are configurable. Node clients load current atomic credentials for every request, preferring `api.auth.bearer_token`, then the first named admin token. Rotation is hot; when auth is configured without an admin credential, the request fails locally. Peer error bodies are bounded and redacted. Use [docs/recipes/cluster-relay-operations.md](docs/recipes/cluster-relay-operations.md). -High-concurrency cluster forwarding uses a reader-scoped `ReadContext` condition wait instead of consuming the RingBuffer's legacy shared `Signal()` channel, so independent relay targets cannot steal each other's wakeup. RTMP push connections reuse their FLV muxer and encoding buffer; RTSP TCP interleaving uses `net.Buffers` so writev-capable connections can send framing and payload together. Relay byte counters bind Prometheus labels once per operation, publish the first observation immediately, batch later bytes at 64 KiB, and flush on operation completion. WHEP source and target-audio readers also use independent condition-backed waiters, avoiding shared wakeup loss between concurrent browser feeds. The focused microbenchmarks are `go test -bench='BenchmarkRingReader|BenchmarkRTMPConn' -benchmem ./pkg/util ./module/cluster`; they are measurement aids, not capacity guarantees. +High-concurrency cluster forwarding uses a reader-scoped `ReadContext` condition wait instead of consuming the RingBuffer's legacy shared `Signal()` channel, so independent relay targets cannot steal each other's wakeup. RTMP push connections reuse their FLV muxer and encoding buffer; RTSP TCP interleaving uses `net.Buffers` so writev-capable connections can send framing and payload together. Relay byte counters bind Prometheus labels once per operation, publish the first observation immediately, batch later bytes at 64 KiB, and flush on operation completion. WHEP source and target-audio readers use independent condition-backed pumps; each pump serializes readiness and the following atomic read on its own cursor, avoiding both shared wakeup loss and false EOF from concurrent cursor consumption. The focused microbenchmarks are `go test -bench='BenchmarkRingReader|BenchmarkRTMPConn' -benchmem ./pkg/util ./module/cluster`. The production-path regression command is `go test -run '^$' -bench 'BenchmarkStreamIngressProduction|BenchmarkRTMPRelaySendMediaFrameProduction|BenchmarkRTSPRelaySendFrameProduction|BenchmarkRelayObservationAccounting' -benchmem -count=3 ./core ./module/cluster`; it exercises stable-publisher Stream admission plus ring/GOP writes, complete RTMP FLV/chunk framing with payload-scoped relay accounting, RTSP H.264 packetization/RTP/interleaved framing with framed-byte accounting, and isolated accounting states. On Apple M1 Pro with Go 1.26.0, the fixture measured 65.86-67.28 ns/op for stable Stream ingress, 155.1-155.6 ns/op for RTMP H.264, 73.60-73.76 ns/op for RTMP AAC, 1.825-1.833 us/op for RTSP single-NAL H.264, and 4.593-4.605 us/op for three-packet FU-A H.264. Stream uses shared immutable payloads in a preallocated monotonic 64-second H.264/G.711A frame pool, with no subscribers, bitrate limiting disabled, two retained GOPs, a 300-frame GOP bound, and a 4,096-entry ring. Both fixed-timestamp egress fixtures end at bounded in-memory writers and exclude socket writes, deadlines, TCP writev, and kernel/network syscalls. The isolated accounting ns/op figures exclude the production context lookup and are primarily allocation-regression evidence. These figures are not comparable to the older narrower `BenchmarkStreamWriteFrame` microbenchmark and are not throughput, concurrency, subscriber-count, or deployment-capacity guarantees. The old microbenchmark remains runnable with `go test -run '^$' -bench '^BenchmarkStreamWriteFrame$' -benchmem ./core`; SIP/GB28181 RTP output remains reproducible with `go test -run '^$' -bench 'BenchmarkGBOutboundSendFrame|BenchmarkSIPOutboundSendFrame' -benchmem ./module/gb28181 ./module/sipgateway`. + +Per-stream Prometheus series are disabled by default to prevent unbounded `stream_key` cardinality, while aggregate server metrics remain available. With `metrics.stream_detail=true` and no allowlist, one Collector admits active scalar stream keys in stable creation order until `metrics.stream_detail_limit` is full; admitted keys are retained for that Collector's lifetime and are never evicted or replaced after a stream disappears, so churn cannot create new label values. The Collector resolves active `*Stream` values on each gather and retains no stream objects or media buffers. Use the management API for current stream detail or configure an exact `metrics.stream_detail_allowlist` for selected labels. An allowlist is deduplicated and sorted once when the Collector is created, remains the authoritative eligible key universe, and is still subject to the per-gather limit. A configured limit of zero exports no per-stream labels; negative configured values are invalid and rejected. A directly constructed Collector defensively emits no per-stream labels for any non-positive limit. Measure first-Gather admission and steady-Gather costs with `go test ./module/metrics -run '^$' -bench '^BenchmarkCollectorGatherStreamDetails$' -benchmem`; this benchmark is a regression aid, not a capacity guarantee. RTP output benchmarks include `BenchmarkGBOutboundSendFrame` and `BenchmarkSIPOutboundSendFrame`; session-owned `MarshalTo` buffers remove one allocation per sent test frame, while packetizer allocations and UDP syscalls remain capacity risks. GB28181 Lab publish treats the requested `stream_key` as authoritative when it is printable ASCII, at most 256 bytes, and contains only non-empty slash-separated segments other than `.` or `..`. The simulator carries this override in a private SIP header accepted only from loopback, so ordinary network devices continue to publish to `{stream_prefix}/{channel_id}`. Receive mode reads the exact requested key. ## Runtime configuration -The bootstrap YAML file is read once during startup. The `config/runtime` manager then polls the selected source (`file`, `http`, `https`, `consul`, or `redis`) in a background goroutine. It parses, normalizes, validates, hashes, and atomically publishes immutable snapshots. Runtime reads are one atomic pointer load and do not perform file/network I/O, wait on channels, or contend with refresh locks. Source loads, Config Apply writes, and close are serialized with a cancellable source-I/O gate; Apply waits for the source write before returning 202 and schedules parse/application/publication asynchronously. +The bootstrap YAML file is read once during startup. The `config/runtime` manager then polls the selected source (`file`, `http`, `https`, `consul`, or `redis`) in a background goroutine. It parses, normalizes, validates, hashes, and atomically publishes immutable snapshots. Runtime reads are one atomic pointer load and do not perform file/network I/O, wait on channels, or contend with refresh locks. Source loads, Config Apply writes, and close are serialized with a cancellable source-I/O gate; Apply waits for the source write before returning 202 with `written_and_refresh_scheduled` and schedules parse/application/publication asynchronously. + +All file, HTTP/HTTPS, Consul, and Redis source reads have a 4 MiB complete-document/materialization limit by default. Configure the corresponding `runtime..max_bytes`; non-positive values select the same default. File and network responses are bounded before parsing. Redis prefix/hash values use length preflight and bounded batches so oversized data is rejected before materialization. + +For HTTP sources, `runtime.source: http` requires an `http://` URL and `runtime.source: https` requires an `https://` URL. Redirects are disabled, including same-origin redirects. `ETag` and `Last-Modified` conditional validators come only from the last accepted snapshot, so malformed, invalid, or unapplied responses cannot advance them. `X-Config-Version` remains source version metadata and is not treated as an ETag. Consul KV GET and PUT also reject redirects without dispatching to the target, so `X-Consul-Token` is never forwarded. -For HTTP sources, `runtime.source: http` requires an `http://` URL and `runtime.source: https` requires an `https://` URL. Redirects are disabled, including same-origin redirects. `ETag` and `Last-Modified` conditional validators come only from the last accepted snapshot, so malformed, invalid, or unapplied responses cannot advance them. `X-Config-Version` remains source version metadata and is not treated as an ETag. +Flattened Consul and Redis leaves conservatively infer case-insensitive booleans and null, canonical base-10 integers without leading zeroes, and finite decimal/exponent floats. Leading-zero identifiers, durations, non-finite or out-of-range numbers, and arbitrary YAML-looking strings remain strings; structured values must use a complete document entry. Dotted and slash-separated paths are canonicalized and sorted before materialization; duplicate paths and scalar/container prefix collisions fail closed with deterministic errors. Generated flattened documents retain deterministic serialization. `SIGHUP` and `POST /api/v1/server/config/refresh` only schedule asynchronous refresh. A source timeout, backend failure, malformed document, or validation error keeps the last valid snapshot active and updates manager status. Hot policy changes can be delivered to reloadable modules; listener addresses, module enablement, TLS files/mode, port ranges, and audio codec enablement are classified as `restart_required`. Status and Prometheus expose accepted/rejected/application-failed changes, callback failures, superseded callbacks, and pending restart paths. Simulcast configuration remains deferred because no layer selection runtime exists. Documented non-positive scalar sentinels retain their owning module defaults: SIP Gateway calls use 100, cluster eviction uses 3 failures, audit retains 1000 entries, and HTTP/Consul sources cap documents at 4 MiB. `http_stream.llhls.part_duration` controls low-latency parts while `http_stream.llhls.segment_duration` controls completed full segments; the latter defaults to 1.0 second and has a schema minimum of 0.1 second. Explicit empty RTSP, WebRTC, and GB28181 port ranges are valid and select their module fallback behavior; non-empty ranges require two ordered positive ports. Exact semantics are annotated in `docs/config/config.schema.json`. -Runnable source examples are in [docs/recipes/runtime-config-sources.md](docs/recipes/runtime-config-sources.md). The Config page can read every field from the redacted effective/desired document, retain raw source comments/unmapped fields, display the embedded versioned JSON Schema, show pending restart paths, validate a candidate, and apply it when the source is writable. The desired source document is editable; the effective applied document is shown separately. Source details identify file, HTTP/HTTPS, Consul, and Redis settings without credentials, and read-only sources keep the editor read-only and return 409 for Apply. File uses atomic replacement, HTTP/HTTPS use authenticated PUT, Consul writes `prefix/config.yaml`, and Redis writes `config.yaml` in hash/prefix mode and increments an optional version key. The deprecated `auth.api.bearer_token` migrates only when `api.auth.bearer_token` is empty; the current path wins if both exist. Credentials must come from environment expansion or an external secret store and are never logged. +Runnable source examples are in [docs/recipes/runtime-config-sources.md](docs/recipes/runtime-config-sources.md). The Config page can read every field from the redacted effective/desired document, retain raw source comments/unmapped fields, display the embedded versioned JSON Schema, show pending restart paths, validate a candidate, and apply it when the source is writable. Viewer Validate does not expand process environment variables: it treats references literally, accepts exactly one YAML/JSON document, and rejects unknown root or nested typed fields. Apply and trusted runtime source loading remain permissive for fields not mapped by the typed runtime struct, and trusted source loading retains environment expansion. Secret maps/sequences retain shape; reordered token, ICE, and endpoint collections restore placeholders by stable non-secret identity rather than index, insertion/deletion cannot transplant another item's secret, and ambiguous identity rejects Apply. The desired source document is editable; the effective applied document is shown separately. Source details identify file, HTTP/HTTPS, Consul, and Redis settings without credentials, and read-only sources keep the editor read-only and return 409 for Apply. File uses atomic replacement, HTTP/HTTPS use authenticated PUT, Consul writes `prefix/config.yaml`, and Redis writes `config.yaml` in hash/prefix mode and increments an optional version key. New file targets use mode `0600` and existing mode bits are preserved. Redis queues its document and optional version write in one `MULTI/EXEC` transaction; transaction/EXEC errors are returned instead of claiming a successful Apply, while command errors inside Redis EXEC are not rolled back. Apply returns `written_and_refresh_scheduled` only after the serialized write succeeds; refresh returns `scheduled`. The Console captures the submitted text and editor revision, so a newer local edit wins over a stale desired snapshot. The deprecated `auth.api.bearer_token` migrates only when `api.auth.bearer_token` is empty; the current path wins if both exist. Credentials for trusted loading must come from environment expansion or an external secret store and are never logged. + +Config document redaction covers every schema `x-liveforge-secret` field plus `api_key` and `tls.key_file`, recursively preserves collection shape, and retains only stable identity fields (`id`, `name`, `username`, `channel_id`, and `device_id`) inside sensitive containers; every other scalar descendant is redacted. Valid absolute hierarchical URL scalars are recognized by value even under unmapped non-URL-shaped keys and retain safe public scheme/host/port identity, but every non-root path is replaced by a stable opaque digest marker in documents/source details and by an opaque marker in errors; userinfo/query/fragment are removed. URL-shaped keys retain TURN/opaque, malformed/hostless fail-closed, and plain-address handling. Ordinary strings, durations, IDs, and bare host/address values remain unchanged outside that key policy. Apply restores matching digest markers from the current desired source document, matches reordered structured collections by stable public identity, and rejects missing, ambiguous, or marked shape-mismatched originals rather than transplanting a secret. One-element unknown sensitive sequences round-trip through this same fail-closed path. + +## Recording and DVR contracts + +Record accepts `flv`, `fmp4`, `mp4`, `ts`, and `hls`, where `hls` is a TS storage alias. `record.segment.max_size` accepts an empty/whitespace value or zero to disable size rotation, or a non-negative decimal byte count with `B`, `KB`, `MB`, or `GB`; fractions, negative values, unknown suffixes, and overflow are rejected. Only completed recordings are served by download and inline play. Active or failed recordings return HTTP 409 with the management JSON error envelope and never return media bytes. The same readiness contract applies to explicit `?action=download`/`?action=play` and the legacy suffix routes. + +DVR audio-only sessions rotate and publish media when audio DTS reaches `segment_duration` while the publisher remains online; video sessions continue to use a valid keyframe boundary. DVR media routes support nested stream keys while preserving slash hierarchy. Each key segment is escaped independently in HLS playlist URIs, so reserved `?`, `#`, and `%` characters remain data; encoded separators, backslashes, empty segments, and dot segments are rejected before authorization or storage lookup. `/api/v1/server/info` reports the bound non-zero DVR listener address after initialization and `endpoint_schemes.dvr` reports its actual `http` or `https` transport scheme, which the Console uses to construct DVR HLS URLs. Portable `!audiocodec` builds are covered by a H.264 plus G.711 DVR test that publishes demuxable video-only TS and no audio frames. ## Verification @@ -172,7 +222,7 @@ The first command skips FFmpeg-tagged transcoding integration tests. The tagged The `lf-test` tool emits human-readable or JSON reports and uses exit code 0 for success, 1 for assertion failure, and 2 for an execution error. -GitHub Actions maintenance uses Node 24-compatible action majors: checkout and setup-go v7, upload-artifact v7, Docker setup-buildx v4, login v4, build-push v7, golangci-lint v9, and action-gh-release v3. Do not reintroduce deprecated Node 20 action versions or the `ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION` workaround. +GitHub Actions maintenance uses Node 24-compatible action majors: checkout and setup-go v7, upload-artifact v7, Docker setup-buildx v4, login v4, build-push v7, golangci-lint v9, and action-gh-release v3. CI passes `--new-from-rev` against the fetched base revision (or `HEAD^` on main pushes), so the new-code lint gate does not depend on the GitHub PR diff API's 20,000-line limit. The Linux lint job remains authoritative for platform-specific `x/sys/unix` types that a Darwin lint run cannot reproduce. Do not reintroduce deprecated Node 20 action versions or the `ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION` workaround. ## Security boundaries diff --git a/llms.txt b/llms.txt index 9654d404..985d1949 100644 --- a/llms.txt +++ b/llms.txt @@ -43,22 +43,26 @@ This is the short Agent entrypoint. Use `agent-manifest.json` for structured fac - Treat release binaries as the portable default profile; use a source or Docker `audiocodec` build for audio transcoding. - Do not expose the sample configuration publicly: it disables auth and TLS and uses `admin/admin`. - Do not use `latest` when a versioned image or commit SHA is available. -- Runtime configuration uses a background poller with lock-free snapshot reads. Sources are selectable with `runtime.source`: `file`, `http`, `https`, `consul`, or `redis`; source loads, writes, and close are serialized, while Apply waits for the source write and schedules background publication. +- Runtime configuration uses a background poller with lock-free snapshot reads. Sources are selectable with `runtime.source`: `file`, `http`, `https`, `consul`, or `redis`; source loads, writes, and close are serialized, while Apply waits for the source write and schedules background publication. Every source defaults to a 4 MiB document/materialization limit via `runtime..max_bytes`; Redis hash reads prefer `HSCAN NOVALUES`, with bounded `HKEYS` fallback for older servers. +- File Apply creates a new target with mode `0600` and preserves existing permission bits. Consul/Redis flattened dotted or slash-separated keys are canonicalized and sorted; duplicate paths and scalar/container prefix collisions fail closed deterministically. +- Redis Apply queues the document write and optional version increment in one `MULTI/EXEC` transaction and returns transaction errors; Apply responds with `written_and_refresh_scheduled`, while refresh responds with `scheduled`. +- Console Apply captures a monotonic editor revision and never lets a stale desired snapshot overwrite newer local editor text. - HTTP source scheme must match `runtime.source`, redirects are disabled, and conditional validators come only from the last accepted snapshot. - Source failures keep the last valid snapshot. `SIGHUP` schedules an asynchronous refresh; listener/module/TLS/port changes remain restart-required. - The console tabs, in order, are Streams, GB28181, Config, Cluster, SIP Calls, Storage, and Security. Recent Audit is a surface inside Security, not a separate tab. Visual groups are Workspace (Streams, GB28181, SIP Calls, Storage), Operations (Cluster), and System (Config, Security); Config/Security are not peer video-stream tabs. - Console preview URLs use the active HTTP/WebRTC listener reported by `/api/v1/server/info`; if another process owns `127.0.0.1:8080`, browser HTTP-FLV/HLS/DASH/FMP4 requests can receive that process's response while RTMP and WHEP remain healthy. Check the response `Server` header and move the HTTP listener or release the conflicting port. -- Recording supports FLV, FMP4, MP4, TS, and HLS with authenticated metadata/download/inline-range-play/delete APIs and DVR status. New recordings default to fMP4 with a `.mp4` extension; absent recording modules report `state=disabled` instead of making the Storage page fail. -- fMP4 recording declares AAC directly, derives omitted AAC timing metadata from ASC, converts non-AAC source audio through the optional `audiocodec`/FFmpeg path, and otherwise filters audio for playable video-only output; explicit AAC timing arguments remain authoritative. +- Recording supports FLV, FMP4, MP4, TS, and HLS with authenticated metadata/download/inline-range-play/delete APIs and DVR status. New recordings default to fMP4 with a `.mp4` extension; `hls` is a TS storage alias; `record.segment.max_size` accepts empty/zero or decimal B/KB/MB/GB values only; absent recording modules report `state=disabled` instead of making the Storage page fail; active or failed recordings return JSON 409 from media actions without serving bytes. +- fMP4 recording declares AAC directly, derives omitted AAC timing metadata from ASC, converts non-AAC source audio through the optional `audiocodec`/FFmpeg path, and otherwise filters audio for playable video-only output; at publisher-generation end, fixed-size transforms flush retained resampler samples, silence-pad partial PCM, and emit delayed encoder packets exactly once before Record/DVR output closes; explicit AAC timing arguments remain authoritative. - Config exposes the complete redacted effective/desired YAML document and embedded versioned JSON Schema at `/api/v1/server/config/document` and `/schema`; raw desired source comments and unmapped fields are retained. `/validate` is read-only for viewers, while `/apply` and `/refresh` require operator/admin. File, HTTP/HTTPS, Consul, and Redis sources expose their documented writer behavior; read-only sources return 409 for apply. -- SIP and GB28181 pages expose one-shot local protocol labs at `/api/v1/sipgateway/test` and `/api/v1/gb28181/test`; both providers also support persistent loopback fake-device publish/receive sessions with cancellable stop/close cleanup and cross-protocol playback. SIP uses real per-track RTP/RTCP paths without mutating receive-mode source streams. Receive mode waits for the selected publisher generation's required sequence headers before signaling and rejects a known unsupported codec before waiting. GB28181 publish uses normal server-initiated live play through the registered device Contact and the real RTP/RTCP receiver; receive validates H.264 plus G.711A, admits its stream subscriber before activation, and uses module-owned PS/RTP/RTCP egress. Admission rejection is synchronous; later outbound media failures move the Lab to `failed` and release its signaling, session, sockets, and ports. The native-dependency-free sample is a moving 160x90 constrained-baseline pattern at 25 fps with one IDR per second and audible 20 ms audio frames. Managers retain active sessions plus 16 terminal records; failures remain visible with credential/token-redacted `last_error` diagnostics. Lab stream keys are printable ASCII up to 256 bytes with non-empty slash-separated segments excluding `.` and `..`. Playback paths escape stream-key segments and derive absolute RTMP/RTSP URLs from actual listeners; Console Lab Preview consumes those paths directly. HLS, LL-HLS, and DASH manifests escape every accepted stream-key segment, DASH XML-escapes URL attributes, and media routing preserves arbitrary valid depth. GB28181 Lab publish honors a validated requested `stream_key` only on loopback simulator INVITEs; ordinary devices retain `{stream_prefix}/{channel_id}`. See [docs/recipes/protocol-test-lab.md](docs/recipes/protocol-test-lab.md). -- The Storage Console can preview completed recordings in-browser and open DVR HLS playback when segments exist. Recording preview reuses the management session; DVR media stays on the separate `dvr.listen` listener with non-credentialed CORS, and the Console never persists or appends bearer tokens. -- DVR playlist and segment authorization runs only synchronous subscribe hooks and emits no asynchronous subscribe lifecycle work. +- SIP and GB28181 pages expose one-shot local protocol labs at `/api/v1/sipgateway/test` and `/api/v1/gb28181/test`; both providers also support persistent loopback fake-device publish/receive sessions with cancellable stop/close cleanup and cross-protocol playback. SIP uses real per-track RTP/RTCP paths without mutating receive-mode source streams. Receive mode waits for the selected publisher generation's required sequence headers before signaling and rejects a known unsupported codec before waiting. GB28181 publish uses normal server-initiated live play through the registered device Contact and the real RTP/RTCP receiver; receive requires H.264 plus direct G.711A or audio the tagged runtime can convert to G.711A, admits its stream subscriber before activation, and uses module-owned PS/RTP/RTCP egress with an independent generation-bound target-audio reader when conversion is needed. Admission rejection is synchronous; later outbound media failures move the Lab to `failed` and release its signaling, session, sockets, and ports. The native-dependency-free sample is a moving 160x90 constrained-baseline pattern at 25 fps with one IDR per second and audible 20 ms audio frames. Managers retain active sessions plus 16 terminal records; failures remain visible with credential/token-redacted `last_error` diagnostics. Lab stream keys are printable ASCII up to 256 bytes with non-empty slash-separated segments excluding `.` and `..`. Playback paths escape stream-key segments and derive absolute RTMP/RTSP URLs from actual listeners; Console Lab Preview consumes those paths directly. HLS, LL-HLS, and DASH manifests escape every accepted stream-key segment, DASH XML-escapes URL attributes, and media routing preserves arbitrary valid depth. GB28181 Lab publish honors a validated requested `stream_key` only on loopback simulator INVITEs; ordinary devices retain `{stream_prefix}/{channel_id}`. See [docs/recipes/protocol-test-lab.md](docs/recipes/protocol-test-lab.md). +- The Storage Console can preview completed recordings in-browser and open DVR HLS playback when segments exist. Recording preview reuses the management session; DVR media stays on the separate `dvr.listen` listener with non-credentialed CORS, and the Console never persists or appends bearer tokens. `/api/v1/server/info` reports the DVR listener's bound non-zero port and actual `http`/`https` scheme for URL construction. +- DVR playlist and segment authorization runs only synchronous subscribe hooks and emits no asynchronous subscribe lifecycle work. Audio-only DVR rotates and publishes at the audio DTS duration threshold while the publisher remains online; nested stream keys preserve slash hierarchy and reserved characters are escaped per path segment, while encoded separators/dot segments are rejected. - RTSP supports separate audio/video SETUP tracks for TCP-interleaved and UDP sessions; track IDs are validated for uniqueness, range, and session eligibility before transport allocation. - WHIP and WHEP accept SDP offer bodies up to 1 MiB; larger bodies return HTTP 413 without creating session or stream state. - Console WHIP publishing with H.265 + Opus has a browser verification path across HTTP-FLV, WS-FLV, HTTP-TS, FMP4, HLS, DASH, WHEP realtime, and WHEP Live. A visible `Playing` label is insufficient: require a decoded frame, non-zero dimensions, an advancing media clock, and no media error. -- Known regression with confirmed root cause: Console WHEP can report `No advancing media received (check codec support and keyframes)` because the default realtime path gates video on a post-snapshot keyframe and a long GOP can outlast the watchdog. H.264 `mode=live` browser playback and Pion/VP8 automated paths pass; default behavior, write-error diagnostics, and real GB28181/SIP H.264 browser coverage remain open. Use [the technical risk record](docs/TECHNICAL-RISKS.md) for evidence and required diagnostics. Do not close this issue based only on SDP success or an `ontrack` callback. +- WHEP omitted `mode` and the Console default now use the atomic `mode=live` GOP startup, while explicit `mode=realtime` still waits for a new keyframe. Each WHEP session exposes `GET /webrtc/session/{sessionId}/status` with generation, cursor, keyframe gate, media counters, and bounded sample-write errors. The tagged SIP/GB28181/WHIP cross-protocol Chromium matrix requires decoded dimensions, advancing media time, increasing audio/video RTP and decoded frames, and non-stalled server status; `LIVEFORGE_PROTOCOL_MATRIX_SOAK=60s` extends per-second checks. SDP success or an `ontrack` callback alone is not proof of playback. Use [the technical risk record](docs/TECHNICAL-RISKS.md) for the remaining coverage. - TLS API listeners issue the `lf_session` console cookie with `Secure`; plain HTTP development listeners do not. +- API, WebRTC signaling, and metrics HTTP servers share a 5-second `ReadHeaderTimeout` and 2-minute `IdleTimeout`; existing write-deadline behavior is unchanged. - RTP/GB cluster signaling reads current credentials per request, preferring `api.auth.bearer_token`, then the first named admin token; configured auth without an admin credential fails locally. - High-concurrency forwarding uses reader-scoped condition waits (including concurrent WHEP source/audio feeds), reusable RTMP FLV encoding state, vectored RTSP interleaved writes, and batched relay-byte metrics; benchmark with `go test -bench='BenchmarkRingReader|BenchmarkRTMPConn' -benchmem ./pkg/util ./module/cluster`. - Simulcast layer selection is deferred and is not implemented. diff --git a/module/api/config.go b/module/api/config.go index 2fa3c34d..2487aba9 100644 --- a/module/api/config.go +++ b/module/api/config.go @@ -2,12 +2,16 @@ package api import ( "bytes" + "crypto/sha256" _ "embed" + "encoding/hex" "encoding/json" "fmt" "io" + "net" "net/http" "net/url" + "strconv" "strings" "github.com/im-pingo/liveforge/config" @@ -90,9 +94,9 @@ func (h *Handlers) handleConfigValidate(w http.ResponseWriter, r *http.Request) writeError(w, http.StatusBadRequest, err.Error()) return } - cfg, err := configruntime.ValidateDocument(document) + cfg, err := configruntime.ValidateKnownDocument(document) if err != nil { - writeError(w, http.StatusBadRequest, err.Error()) + writeError(w, http.StatusBadRequest, configruntime.RedactError(err)) return } writeJSON(w, http.StatusOK, map[string]any{ @@ -113,14 +117,16 @@ func (h *Handlers) handleConfigApply(w http.ResponseWriter, r *http.Request) { return } secretSource := h.server.Config() + var sourceDocument []byte if snapshot := manager.Snapshot(); snapshot != nil { if snapshot.DesiredConfig != nil { secretSource = snapshot.DesiredConfig } else if snapshot.Config != nil { secretSource = snapshot.Config } + sourceDocument = append([]byte(nil), snapshot.DesiredDocument...) } - document, err = preserveRedactedSecrets(document, secretSource) + document, err = preserveRedactedSecretsWithDocument(document, secretSource, sourceDocument) if err != nil { writeError(w, http.StatusBadRequest, err.Error()) return @@ -132,7 +138,7 @@ func (h *Handlers) handleConfigApply(w http.ResponseWriter, r *http.Request) { } else if _, parseErr := configruntime.ValidateDocument(document); parseErr != nil { status = http.StatusBadRequest } - writeError(w, status, err.Error()) + writeError(w, status, configruntime.RedactError(err)) return } writeJSON(w, http.StatusAccepted, map[string]any{"status": "written_and_refresh_scheduled"}) @@ -213,6 +219,10 @@ func redactYAMLNode(node *yaml.Node) { return } switch node.Kind { + case yaml.ScalarNode: + if isHierarchicalConfigURLValue(node.Value) { + redactURLYAMLNode(node, false) + } case yaml.DocumentNode, yaml.SequenceNode: for _, child := range node.Content { redactYAMLNode(child) @@ -220,10 +230,15 @@ func redactYAMLNode(node *yaml.Node) { case yaml.MappingNode: for index := 0; index+1 < len(node.Content); index += 2 { key, value := node.Content[index], node.Content[index+1] - if key.Kind == yaml.ScalarNode && isSensitiveConfigKey(key.Value) && value.Kind == yaml.ScalarNode { - value.Tag = "!!str" - value.Style = yaml.DoubleQuotedStyle - value.Value = "[REDACTED]" + if key.Kind != yaml.ScalarNode { + redactYAMLNode(value) + continue + } + if isSensitiveConfigKey(key.Value) && redactSensitiveYAMLNode(value) { + continue + } + if isURLConfigKey(key.Value) { + redactURLYAMLNode(value, isAddressConfigKey(key.Value)) continue } redactYAMLNode(value) @@ -231,23 +246,94 @@ func redactYAMLNode(node *yaml.Node) { } } +func redactSensitiveYAMLNode(node *yaml.Node) bool { + if node == nil { + return false + } + redactOpaqueSensitiveYAMLNode(node) + return true +} + +func redactOpaqueSensitiveYAMLNode(node *yaml.Node) { + if node == nil { + return + } + switch node.Kind { + case yaml.ScalarNode: + node.Tag = "!!str" + node.Style = yaml.DoubleQuotedStyle + node.Value = "[REDACTED]" + case yaml.DocumentNode, yaml.SequenceNode: + for _, child := range node.Content { + redactOpaqueSensitiveYAMLNode(child) + } + case yaml.MappingNode: + for index := 0; index+1 < len(node.Content); index += 2 { + key, value := node.Content[index], node.Content[index+1] + if key.Kind == yaml.ScalarNode { + if isStableConfigIdentityKey(key.Value) && value.Kind == yaml.ScalarNode { + continue + } + if isURLConfigKey(key.Value) && isScalarURLYAMLNode(value) { + redactURLYAMLNode(value, isAddressConfigKey(key.Value)) + continue + } + } + redactOpaqueSensitiveYAMLNode(value) + } + } +} + +func isScalarURLYAMLNode(node *yaml.Node) bool { + if node == nil { + return false + } + if node.Kind == yaml.ScalarNode { + return true + } + if node.Kind != yaml.SequenceNode { + return false + } + for _, child := range node.Content { + if child.Kind != yaml.ScalarNode { + return false + } + } + return true +} + func preserveRedactedSecrets(document []byte, current *config.Config) ([]byte, error) { + return preserveRedactedSecretsWithDocument(document, current, nil) +} + +func preserveRedactedSecretsWithDocument(document []byte, current *config.Config, currentDocument []byte) ([]byte, error) { var root yaml.Node if err := yaml.Unmarshal(document, &root); err != nil { return nil, err } - restoreRedactedYAMLSecrets(&root, rawConfigMapFromConfig(current)) + currentValues := rawConfigMapFromConfig(current) + if len(currentDocument) > 0 { + var sourceValues map[string]any + if err := yaml.Unmarshal(currentDocument, &sourceValues); err == nil { + mergeConfigMaps(currentValues, sourceValues) + } + } + if err := restoreRedactedYAMLSecrets(&root, currentValues, nil); err != nil { + return nil, err + } return yaml.Marshal(&root) } -func restoreRedactedYAMLSecrets(node *yaml.Node, current any) { +func restoreRedactedYAMLSecrets(node *yaml.Node, current any, path []string) error { if node == nil { - return + return nil } switch node.Kind { case yaml.DocumentNode: for _, child := range node.Content { - restoreRedactedYAMLSecrets(child, current) + if err := restoreRedactedYAMLSecrets(child, current, path); err != nil { + return err + } } case yaml.MappingNode: original, _ := current.(map[string]any) @@ -261,18 +347,44 @@ func restoreRedactedYAMLSecrets(node *yaml.Node, current any) { replaceYAMLNode(value, replacement) continue } - restoreRedactedYAMLSecrets(value, replacement) + childPath := appendConfigPath(path, key.Value) + if value.Kind == yaml.ScalarNode && value.Value == "[REDACTED]" && configPathContainsSensitiveKey(childPath) { + if !ok || !isScalarConfigValue(replacement) { + return fmt.Errorf("cannot restore redacted configuration value at %s", formatConfigPath(childPath)) + } + replaceYAMLNode(value, replacement) + continue + } + if isSensitiveConfigKey(key.Value) && isRedactedYAMLNode(value) && !ok { + return fmt.Errorf("cannot restore redacted configuration value at %s", formatConfigPath(childPath)) + } + if isURLConfigKey(key.Value) { + if err := restoreRedactedURLYAMLNode(value, replacement, childPath); err != nil { + return err + } + continue + } + if err := restoreRedactedYAMLSecrets(value, replacement, childPath); err != nil { + return err + } } case yaml.SequenceNode: - original, _ := current.([]any) - for index, child := range node.Content { - var replacement any - if index < len(original) { - replacement = original[index] - } - restoreRedactedYAMLSecrets(child, replacement) + return restoreRedactedYAMLSequence(node, current, path) + case yaml.ScalarNode: + if !isRedactedConfigURL(node.Value) { + return nil + } + source, ok := current.(string) + if !ok { + return fmt.Errorf("cannot restore redacted URL at %s", formatConfigPath(path)) + } + restored, err := restoreRedactedConfigURL(node.Value, source) + if err != nil { + return fmt.Errorf("restore redacted URL at %s: %w", formatConfigPath(path), err) } + replaceYAMLNode(node, restored) } + return nil } func isRedactedYAMLNode(node *yaml.Node) bool { @@ -297,28 +409,359 @@ func replaceYAMLNode(node *yaml.Node, value any) { *node = *replacement.Content[0] } -func mergeRedactedSecrets(candidate, current any) { - switch value := candidate.(type) { +func redactURLYAMLNode(node *yaml.Node, allowPlainAddress bool) { + if node == nil { + return + } + switch node.Kind { + case yaml.ScalarNode: + if redacted := redactConfigURL(node.Value, allowPlainAddress); redacted != node.Value { + node.Tag = "!!str" + node.Style = yaml.DoubleQuotedStyle + node.Value = redacted + } + case yaml.SequenceNode: + for _, child := range node.Content { + redactURLYAMLNode(child, allowPlainAddress) + } + case yaml.DocumentNode, yaml.MappingNode: + redactYAMLNode(node) + } +} + +func restoreRedactedURLYAMLNode(node *yaml.Node, current any, path []string) error { + if node == nil { + return nil + } + containsRedaction := yamlNodeContainsRedaction(node) + switch value := current.(type) { + case string: + if containsRedaction && node.Kind != yaml.ScalarNode { + return fmt.Errorf("cannot restore redacted URL at %s: value shape changed", formatConfigPath(path)) + } + if node.Kind != yaml.ScalarNode { + return nil + } + if node.Value == "[REDACTED]" { + if value == "[REDACTED]" { + return fmt.Errorf("cannot restore redacted URL at %s: source URL is unavailable", formatConfigPath(path)) + } + replaceYAMLNode(node, value) + return nil + } + if isRedactedConfigURL(node.Value) { + restored, err := restoreRedactedConfigURL(node.Value, value) + if err != nil { + return fmt.Errorf("restore redacted URL at %s: %w", formatConfigPath(path), err) + } + replaceYAMLNode(node, restored) + } + case []any: + if containsRedaction && node.Kind != yaml.SequenceNode { + return fmt.Errorf("cannot restore redacted URL at %s: value shape changed", formatConfigPath(path)) + } + if node.Kind != yaml.SequenceNode { + return nil + } + return restoreRedactedYAMLSequence(node, value, path) case map[string]any: - original, _ := current.(map[string]any) - for key, child := range value { - if isSensitiveConfigKey(key) && child == "[REDACTED]" { - if replacement, ok := original[key]; ok { - value[key] = replacement - } + if containsRedaction && node.Kind != yaml.MappingNode { + return fmt.Errorf("cannot restore redacted URL at %s: value shape changed", formatConfigPath(path)) + } + return restoreRedactedYAMLSecrets(node, value, path) + default: + if containsRedaction { + return fmt.Errorf("cannot restore redacted URL at %s", formatConfigPath(path)) + } + } + return nil +} + +func restoreRedactedConfigURL(candidate, current string) (string, error) { + if !isRedactedConfigURL(candidate) { + return candidate, nil + } + candidateURL, err := url.Parse(strings.TrimSpace(candidate)) + if err != nil || !isValidConfigURL(candidateURL) { + return "", fmt.Errorf("candidate URL is invalid") + } + currentURL, err := url.Parse(strings.TrimSpace(current)) + if err != nil || !isValidConfigURL(currentURL) { + return "", fmt.Errorf("source URL is unavailable") + } + if isRedactedConfigURLPath(candidateURL.Path) { + expected := *currentURL + if !redactConfigURLPath(&expected) || candidateURL.Path != expected.Path { + return "", fmt.Errorf("source URL path identity does not match") + } + candidateURL.Path = currentURL.Path + candidateURL.RawPath = currentURL.RawPath + } + candidateURL.User = currentURL.User + candidateURL.RawQuery = currentURL.RawQuery + candidateURL.ForceQuery = currentURL.ForceQuery + candidateURL.Fragment = currentURL.Fragment + return candidateURL.String(), nil +} + +func restoreRedactedYAMLSequence(node *yaml.Node, current any, path []string) error { + original, _ := current.([]any) + redactedIndexes := make([]int, 0, len(node.Content)) + for index, child := range node.Content { + if yamlNodeContainsRedaction(child) { + redactedIndexes = append(redactedIndexes, index) + } + } + if len(redactedIndexes) == 0 { + return nil + } + if len(node.Content) == 1 && len(original) == 1 { + if child := node.Content[0]; child.Kind == yaml.ScalarNode && child.Value == "[REDACTED]" && + isImmediateSensitiveConfigPath(path) && isScalarConfigValue(original[0]) { + replaceYAMLNode(child, original[0]) + return nil + } + } + + candidateValues := make([]any, len(node.Content)) + for index, child := range node.Content { + if err := child.Decode(&candidateValues[index]); err != nil { + return fmt.Errorf("decode configuration collection at %s[%d]: %w", formatConfigPath(path), index, err) + } + } + candidateIdentities := make([][]string, len(candidateValues)) + for index, value := range candidateValues { + candidateIdentities[index] = configStableIdentities(value, true) + } + originalIdentities := make([][]string, len(original)) + for index, value := range original { + originalIdentities[index] = configStableIdentities(value, false) + } + used := make(map[int]struct{}, len(redactedIndexes)) + for _, candidateIndex := range redactedIndexes { + originalIndex := uniqueConfigIdentityMatch(candidateIndex, candidateIdentities, originalIdentities, used) + if originalIndex < 0 { + return fmt.Errorf("cannot uniquely restore redacted configuration item at %s[%d]", formatConfigPath(path), candidateIndex) + } + used[originalIndex] = struct{}{} + if err := restoreRedactedYAMLSecrets(node.Content[candidateIndex], original[originalIndex], appendConfigPath(path, fmt.Sprintf("[%d]", candidateIndex))); err != nil { + return err + } + } + return nil +} + +func isImmediateSensitiveConfigPath(path []string) bool { + if len(path) == 0 { + return false + } + return isSensitiveConfigKey(path[len(path)-1]) +} + +func configPathContainsSensitiveKey(path []string) bool { + for _, element := range path { + if isSensitiveConfigKey(element) { + return true + } + } + return false +} + +func isScalarConfigValue(value any) bool { + switch value.(type) { + case map[string]any, []any: + return false + default: + return true + } +} + +func uniqueConfigIdentityMatch(candidateIndex int, candidates, originals [][]string, used map[int]struct{}) int { + for _, identity := range candidates[candidateIndex] { + candidateCount := 0 + for _, identities := range candidates { + if containsConfigIdentity(identities, identity) { + candidateCount++ + } + } + if candidateCount != 1 { + continue + } + match := -1 + for originalIndex, identities := range originals { + if _, exists := used[originalIndex]; exists || !containsConfigIdentity(identities, identity) { continue } - mergeRedactedSecrets(child, original[key]) + if match >= 0 { + match = -1 + break + } + match = originalIndex + } + if match >= 0 { + return match + } + } + return -1 +} + +func configStableIdentities(value any, candidate bool) []string { + if text, ok := value.(string); ok { + if identity := publicConfigURLIdentity(text, candidate); identity != "" { + return []string{"url:" + identity} + } + return nil + } + mapping, ok := value.(map[string]any) + if !ok { + return nil + } + identities := make([]string, 0, 5) + for _, key := range []string{"id", "name", "username", "channel_id", "device_id"} { + if identity := scalarConfigIdentity(mapping[key]); identity != "" { + identities = append(identities, key+":"+identity) + } + } + nonSecret := make(map[string]any) + for key, child := range mapping { + if isSensitiveConfigKey(key) || isURLConfigKey(key) || configValueContainsRedaction(child) { + continue + } + nonSecret[key] = child + } + if len(nonSecret) > 0 { + if encoded, err := json.Marshal(nonSecret); err == nil { + identities = append(identities, "fields:"+string(encoded)) + } + } + for key, child := range mapping { + if !isURLConfigKey(key) { + continue + } + switch urls := child.(type) { + case string: + if identity := publicConfigURLIdentity(urls, candidate); identity != "" { + identities = append(identities, key+":"+identity) + } + case []any: + public := make([]string, 0, len(urls)) + for _, item := range urls { + if text, ok := item.(string); ok { + public = append(public, publicConfigURLIdentity(text, candidate)) + } + } + if encoded, err := json.Marshal(public); err == nil { + identities = append(identities, key+":"+string(encoded)) + } + } + } + return identities +} + +func isStableConfigIdentityKey(key string) bool { + switch strings.ToLower(strings.TrimSpace(key)) { + case "id", "name", "username", "channel_id", "device_id": + return true + default: + return false + } +} + +func publicConfigURLIdentity(raw string, candidate bool) string { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed.Scheme == "" { + return strings.TrimSpace(raw) + } + parsed.User = nil + parsed.RawQuery = "" + parsed.ForceQuery = false + parsed.Fragment = "" + if !candidate || !isRedactedConfigURLPath(parsed.Path) { + redactConfigURLPath(parsed) + } + return parsed.String() +} + +func scalarConfigIdentity(value any) string { + switch value := value.(type) { + case string: + return strings.TrimSpace(value) + case fmt.Stringer: + return value.String() + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + return fmt.Sprint(value) + default: + return "" + } +} + +func containsConfigIdentity(identities []string, identity string) bool { + for _, candidate := range identities { + if candidate == identity { + return true } + } + return false +} + +func yamlNodeContainsRedaction(node *yaml.Node) bool { + if node == nil { + return false + } + if node.Kind == yaml.ScalarNode && (node.Value == "[REDACTED]" || isRedactedConfigURL(node.Value)) { + return true + } + for _, child := range node.Content { + if yamlNodeContainsRedaction(child) { + return true + } + } + return false +} + +func configValueContainsRedaction(value any) bool { + switch value := value.(type) { + case string: + return value == "[REDACTED]" || isRedactedConfigURL(value) case []any: - original, _ := current.([]any) - for index, child := range value { - var replacement any - if index < len(original) { - replacement = original[index] + for _, child := range value { + if configValueContainsRedaction(child) { + return true } - mergeRedactedSecrets(child, replacement) } + case map[string]any: + for _, child := range value { + if configValueContainsRedaction(child) { + return true + } + } + } + return false +} + +func appendConfigPath(path []string, element string) []string { + appended := make([]string, len(path), len(path)+1) + copy(appended, path) + return append(appended, element) +} + +func formatConfigPath(path []string) string { + if len(path) == 0 { + return "configuration" + } + return strings.Join(path, ".") +} + +func mergeConfigMaps(dst, src map[string]any) { + for key, sourceValue := range src { + destinationValue := dst[key] + sourceMap, sourceIsMap := sourceValue.(map[string]any) + destinationMap, destinationIsMap := destinationValue.(map[string]any) + if sourceIsMap && destinationIsMap { + mergeConfigMaps(destinationMap, sourceMap) + continue + } + dst[key] = sourceValue } } @@ -326,27 +769,91 @@ func redactConfigValue(value any) { switch current := value.(type) { case map[string]any: for key, child := range current { - switch child.(type) { - case map[string]any, []any: - redactConfigValue(child) + if isSensitiveConfigKey(key) { + if redactSensitiveConfigValue(current, key, child) { + continue + } + } + if isURLConfigKey(key) { + redactConfigURLValue(current, key, child) continue } - if isSensitiveConfigKey(key) { - current[key] = "[REDACTED]" + if text, ok := child.(string); ok && isHierarchicalConfigURLValue(text) { + current[key] = redactConfigURL(text, false) continue } redactConfigValue(child) } case []any: - for _, child := range current { + for index, child := range current { + if text, ok := child.(string); ok && isHierarchicalConfigURLValue(text) { + current[index] = redactConfigURL(text, false) + continue + } redactConfigValue(child) } } } +func redactSensitiveConfigValue(parent map[string]any, key string, value any) bool { + switch value.(type) { + case map[string]any, []any: + redactOpaqueSensitiveConfigValue(value) + default: + parent[key] = "[REDACTED]" + } + return true +} + +func redactOpaqueSensitiveConfigValue(value any) { + switch current := value.(type) { + case map[string]any: + for key, child := range current { + if isStableConfigIdentityKey(key) && isScalarConfigValue(child) { + continue + } + if isURLConfigKey(key) && isScalarConfigURLValue(child) { + redactConfigURLValue(current, key, child) + continue + } + switch child.(type) { + case map[string]any, []any: + redactOpaqueSensitiveConfigValue(child) + default: + current[key] = "[REDACTED]" + } + } + case []any: + for index, child := range current { + switch child.(type) { + case map[string]any, []any: + redactOpaqueSensitiveConfigValue(child) + default: + current[index] = "[REDACTED]" + } + } + } +} + +func isScalarConfigURLValue(value any) bool { + switch value := value.(type) { + case string: + return true + case []any: + for _, child := range value { + if _, ok := child.(string); !ok { + return false + } + } + return true + default: + return false + } +} + func isSensitiveConfigKey(key string) bool { key = strings.ToLower(strings.TrimSpace(key)) - for _, marker := range []string{"token", "password", "secret", "credential", "passphrase", "private_key"} { + for _, marker := range []string{"token", "password", "secret", "credential", "passphrase", "private_key", "api_key", "key_file"} { if strings.Contains(key, marker) { return true } @@ -354,14 +861,145 @@ func isSensitiveConfigKey(key string) bool { return false } +const ( + redactedURLQuery = "__liveforge_redacted__=1" + redactedURLPathPrefix = "/__liveforge_redacted_path__/" +) + +func isURLConfigKey(key string) bool { + key = strings.ToLower(strings.TrimSpace(key)) + return key == "url" || key == "urls" || strings.Contains(key, "_url") || + strings.Contains(key, "uri") || strings.Contains(key, "endpoint") || + strings.Contains(key, "address") || key == "addr" +} + +func isAddressConfigKey(key string) bool { + key = strings.ToLower(strings.TrimSpace(key)) + return strings.Contains(key, "address") || key == "addr" +} + +func isHierarchicalConfigURLValue(raw string) bool { + parsed, err := url.Parse(strings.TrimSpace(raw)) + return err == nil && parsed.IsAbs() && parsed.Opaque == "" && parsed.Host != "" && parsed.Hostname() != "" +} + +func redactConfigURLValue(parent map[string]any, key string, value any) { + switch value := value.(type) { + case string: + parent[key] = redactConfigURL(value, isAddressConfigKey(key)) + case []any: + for index, child := range value { + if text, ok := child.(string); ok { + value[index] = redactConfigURL(text, isAddressConfigKey(key)) + continue + } + redactConfigValue(child) + } + case map[string]any: + redactConfigValue(value) + } +} + +func redactConfigURL(raw string, allowPlainAddress bool) string { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "" + } + parsed, err := url.Parse(trimmed) + if err != nil || !isValidConfigURL(parsed) { + if allowPlainAddress && (net.ParseIP(trimmed) != nil || isPlainHostPort(trimmed)) { + return trimmed + } + return "[REDACTED]" + } + pathRedacted := redactConfigURLPath(parsed) + if parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == "" && !pathRedacted { + return trimmed + } + if parsed.User != nil { + parsed.User = url.User("REDACTED") + } + if parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" { + parsed.RawQuery = redactedURLQuery + } + parsed.ForceQuery = false + parsed.Fragment = "" + return parsed.String() +} + +func isRedactedConfigURL(raw string) bool { + parsed, err := url.Parse(strings.TrimSpace(raw)) + return err == nil && isValidConfigURL(parsed) && + (parsed.RawQuery == redactedURLQuery || isRedactedConfigURLPath(parsed.Path) || + (parsed.User != nil && parsed.User.Username() == "REDACTED")) +} + +func redactConfigURLPath(parsed *url.URL) bool { + if parsed == nil || parsed.Path == "" || parsed.Path == "/" { + return false + } + digest := sha256.Sum256([]byte(parsed.EscapedPath())) + parsed.Path = redactedURLPathPrefix + hex.EncodeToString(digest[:16]) + parsed.RawPath = "" + return true +} + +func isRedactedConfigURLPath(path string) bool { + if !strings.HasPrefix(path, redactedURLPathPrefix) { + return false + } + digest := strings.TrimPrefix(path, redactedURLPathPrefix) + if len(digest) != 32 { + return false + } + _, err := hex.DecodeString(digest) + return err == nil +} + +func isValidConfigURL(parsed *url.URL) bool { + if parsed == nil || parsed.Scheme == "" { + return false + } + if parsed.Host != "" { + return true + } + switch strings.ToLower(parsed.Scheme) { + case "stun", "stuns", "turn", "turns": + return parsed.Opaque != "" && !strings.ContainsAny(parsed.Opaque, "@/\\\r\n\t ") + default: + return false + } +} + func redactedSourceDetails(runtimeConfig config.RuntimeConfig) map[string]any { return map[string]any{ "kind": runtimeConfig.Source, - "file": map[string]any{"path": runtimeConfig.File.Path}, + "file": map[string]any{"path": runtimeConfig.File.Path, "max_bytes": runtimeConfig.File.MaxBytes}, "http": map[string]any{"url": redactedSourceURL(runtimeConfig.HTTP.URL), "max_bytes": runtimeConfig.HTTP.MaxBytes}, "consul": map[string]any{"address": redactedSourceURL(runtimeConfig.Consul.Address), "prefix": runtimeConfig.Consul.Prefix, "max_bytes": runtimeConfig.Consul.MaxBytes}, - "redis": map[string]any{"addr": runtimeConfig.Redis.Addr, "username": runtimeConfig.Redis.Username, "db": runtimeConfig.Redis.DB, "prefix": runtimeConfig.Redis.Prefix, "hash": runtimeConfig.Redis.Hash, "version_key": runtimeConfig.Redis.VersionKey, "tls": runtimeConfig.Redis.TLS}, + "redis": map[string]any{"addr": redactedSourceAddress(runtimeConfig.Redis.Addr), "username": runtimeConfig.Redis.Username, "db": runtimeConfig.Redis.DB, "prefix": runtimeConfig.Redis.Prefix, "hash": runtimeConfig.Redis.Hash, "version_key": runtimeConfig.Redis.VersionKey, "tls": runtimeConfig.Redis.TLS, "max_bytes": runtimeConfig.Redis.MaxBytes}, + } +} + +func redactedSourceAddress(raw string) string { + trimmed := strings.TrimSpace(raw) + if trimmed == "" || isPlainHostPort(trimmed) { + return trimmed + } + return redactedSourceURL(trimmed) +} + +func isPlainHostPort(raw string) bool { + host, portText, err := net.SplitHostPort(raw) + if err != nil || strings.TrimSpace(host) == "" { + return false + } + parsed, err := url.Parse("//" + raw) + if err != nil || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.Path != "" || parsed.Hostname() == "" { + return false } + port, err := strconv.Atoi(portText) + return err == nil && port >= 1 && port <= 65535 } func redactedSourceURL(raw string) string { @@ -371,7 +1009,9 @@ func redactedSourceURL(raw string) string { } parsed.User = nil parsed.RawQuery = "" + parsed.ForceQuery = false parsed.Fragment = "" + redactConfigURLPath(parsed) return parsed.String() } diff --git a/module/api/config_api_test.go b/module/api/config_api_test.go index 13c636ee..0943dd5b 100644 --- a/module/api/config_api_test.go +++ b/module/api/config_api_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "os" @@ -83,7 +84,7 @@ func TestConfigAndProtocolReadAccessUsesViewerRBAC(t *testing.T) { } } -func TestHandleConfigApplyWritesFileAndPreservesRedactedSecrets(t *testing.T) { +func TestHandleConfigApplyWritesFileAndPreservesRedactedSecretsAndUnmappedFields(t *testing.T) { cfg := config.Defaults() cfg.API.Auth.BearerToken = "api-secret" cfg.API.Console.Username = "admin" @@ -93,6 +94,7 @@ func TestHandleConfigApplyWritesFileAndPreservesRedactedSecrets(t *testing.T) { if err != nil { t.Fatal(err) } + document = append(document, []byte("custom_runtime_field: retained\n")...) path := filepath.Join(t.TempDir(), "liveforge.yaml") if err := os.WriteFile(path, document, 0o600); err != nil { t.Fatal(err) @@ -110,8 +112,8 @@ func TestHandleConfigApplyWritesFileAndPreservesRedactedSecrets(t *testing.T) { t.Fatal(err) } defer manager.Close() - if err := manager.Start(context.Background()); err != nil { - t.Fatal(err) + if startErr := manager.Start(context.Background()); startErr != nil { + t.Fatal(startErr) } h, server := newTestHandlers(t) @@ -148,6 +150,9 @@ func TestHandleConfigApplyWritesFileAndPreservesRedactedSecrets(t *testing.T) { if parsed.Server.Name != "edited" || parsed.API.Auth.BearerToken != "api-secret" || parsed.API.Console.Password != "console-secret" { t.Fatalf("written config lost edits or secrets: server=%q bearer=%q password=%q", parsed.Server.Name, parsed.API.Auth.BearerToken, parsed.API.Console.Password) } + if !strings.Contains(string(written), "custom_runtime_field: retained") { + t.Fatalf("written config dropped unmapped desired-source field:\n%s", written) + } } func TestPreserveRedactedSecretsRestoresArrayValues(t *testing.T) { @@ -206,6 +211,48 @@ func TestPreserveRedactedSecretsKeepsSourceCommentsAndUnknownFields(t *testing.T } } +func TestPreserveRedactedSecretsRestoresUnknownOneElementSensitiveSequence(t *testing.T) { + const sourceDocument = "custom_private_keys: [secret-value]\n" + redacted, err := redactedConfigDocument([]byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(redacted), "secret-value") { + t.Fatalf("redacted document leaked the source secret: %s", redacted) + } + + restored, err := preserveRedactedSecretsWithDocument(redacted, config.Defaults(), []byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := yaml.Unmarshal(restored, &document); err != nil { + t.Fatal(err) + } + values, ok := document["custom_private_keys"].([]any) + if !ok || len(values) != 1 || values[0] != "secret-value" { + t.Fatalf("restored custom_private_keys = %#v, want original one-element sequence", document["custom_private_keys"]) + } +} + +func TestPreserveRedactedSecretsRejectsUnknownSensitiveSequenceWithoutUniqueOriginal(t *testing.T) { + tests := []struct { + name string + currentDocument string + }{ + {name: "missing original"}, + {name: "ambiguous original", currentDocument: "custom_private_keys: [first-secret, second-secret]\n"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + candidate := []byte("custom_private_keys: [\"[REDACTED]\"]\n") + if _, err := preserveRedactedSecretsWithDocument(candidate, config.Defaults(), []byte(test.currentDocument)); err == nil { + t.Fatal("redacted sensitive sequence was accepted without a unique original") + } + }) + } +} + func TestHandleConfigApplyRejectsInvalidDocument(t *testing.T) { h, server := newTestHandlers(t) manager, err := configruntime.NewManager(configruntime.Options{Source: testConfigSource{}, Initial: server.Config()}) @@ -247,6 +294,36 @@ func TestHandleConfigApplyRejectsReadOnlySource(t *testing.T) { } } +func TestHandleConfigApplyRedactsSourceURLFromWriteError(t *testing.T) { + h, server := newTestHandlers(t) + const sourceURL = "https://config-user:config-password@config.example.test/live.yaml?token=query-secret" //nolint:gosec // Synthetic value verifies redaction. + manager, err := configruntime.NewManager(configruntime.Options{ + Source: errorConfigWriterSource{err: errors.New("write " + sourceURL + ": connection refused")}, + Initial: server.Config(), + }) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + server.SetConfigManager(manager) + + request := httptest.NewRequest(http.MethodPost, "/api/v1/server/config/apply", strings.NewReader("server:\n name: edited\n")) + request.Header.Set("Content-Type", "application/yaml") + w := httptest.NewRecorder() + h.handleConfigApply(w, request) + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + for _, secret := range []string{"config-user", "config-password", "query-secret", "token="} { + if strings.Contains(w.Body.String(), secret) { + t.Fatalf("config apply error leaked %q: %s", secret, w.Body.String()) + } + } + if !strings.Contains(w.Body.String(), "config.example.test") { + t.Fatalf("redacted error lost useful endpoint identity: %s", w.Body.String()) + } +} + func TestConfigDocumentPreservesRawSourceFieldsAndComments(t *testing.T) { const sourceDocument = "# keep this source comment\nserver:\n name: liveforge\napi:\n auth:\n bearer_token: source-secret\ncustom_runtime_field: retained\n" h, server := newTestHandlers(t) @@ -258,8 +335,8 @@ func TestConfigDocumentPreservesRawSourceFieldsAndComments(t *testing.T) { t.Fatal(err) } defer manager.Close() - if err := manager.Start(context.Background()); err != nil { - t.Fatal(err) + if startErr := manager.Start(context.Background()); startErr != nil { + t.Fatal(startErr) } if snapshot := manager.Snapshot(); snapshot == nil || string(snapshot.DesiredDocument) != sourceDocument { t.Fatalf("manager did not retain source document: %+v", snapshot) @@ -277,8 +354,8 @@ func TestConfigDocumentPreservesRawSourceFieldsAndComments(t *testing.T) { DesiredText string `json:"desired_document"` Schema map[string]any `json:"schema"` } - if err := json.Unmarshal(data, &response); err != nil { - t.Fatal(err) + if unmarshalErr := json.Unmarshal(data, &response); unmarshalErr != nil { + t.Fatal(unmarshalErr) } if response.Desired["custom_runtime_field"] != "retained" { t.Fatalf("raw source field was dropped: %+v", response.Desired) @@ -294,6 +371,129 @@ func TestConfigDocumentPreservesRawSourceFieldsAndComments(t *testing.T) { } } +func TestHandleConfigDocumentRedactsUnmappedHierarchicalURLValues(t *testing.T) { + const sourceDocument = `server: + name: liveforge +primary: https://hooks.slack.com/services/T111/B111/scalar-path-token +mirrors: + - https://hooks.slack.com/services/T222/B222/sequence-path-token +ordinary: + - 30s + - camera-001 + - relay.example.test:443 + - 239.0.0.1 + - turn:relay.example.test:3478?transport=udp +` + h, server := newTestHandlers(t) + manager, err := configruntime.NewManager(configruntime.Options{ + Source: &rawDocumentSource{document: []byte(sourceDocument)}, + Initial: nil, + }) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + if startErr := manager.Start(context.Background()); startErr != nil { + t.Fatal(startErr) + } + server.SetConfigManager(manager) + + w := httptest.NewRecorder() + h.handleConfigDocument(w, httptest.NewRequest(http.MethodGet, "/api/v1/server/config/document", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + data := decodeAPIData(t, w.Body.Bytes()) + var response struct { + Desired map[string]any `json:"desired"` + DesiredText string `json:"desired_document"` + } + if unmarshalErr := json.Unmarshal(data, &response); unmarshalErr != nil { + t.Fatal(unmarshalErr) + } + desiredJSON, err := json.Marshal(response.Desired) + if err != nil { + t.Fatal(err) + } + for _, surface := range []struct { + name string + text string + }{ + {name: "decoded desired", text: string(desiredJSON)}, + {name: "desired_document", text: response.DesiredText}, + } { + t.Run(surface.name, func(t *testing.T) { + for _, secret := range []string{"T111", "B111", "scalar-path-token", "T222", "B222", "sequence-path-token"} { + if strings.Contains(surface.text, secret) { + t.Errorf("management response leaked unmapped URL path credential %q: %s", secret, surface.text) + } + } + if !strings.Contains(surface.text, "hooks.slack.com") || !strings.Contains(surface.text, redactedURLPathPrefix) { + t.Errorf("management response lost safe URL host or digest marker: %s", surface.text) + } + for _, ordinary := range []string{"30s", "camera-001", "relay.example.test:443", "239.0.0.1", "turn:relay.example.test:3478?transport=udp"} { + if !strings.Contains(surface.text, ordinary) { + t.Errorf("management response changed ordinary value %q: %s", ordinary, surface.text) + } + } + }) + } +} + +func TestHandleConfigDocumentRehashesLiteralRedactedPathMarker(t *testing.T) { + //nolint:gosec // Intentional fake URL credentials verify management-response redaction. + const sourceURL = "https://literal-user:literal-password@hooks.slack.com:8443/__liveforge_redacted_path__/0123456789abcdef0123456789abcdef?token=literal-query#literal-fragment" + const redactedURL = "https://REDACTED@hooks.slack.com:8443/__liveforge_redacted_path__/3b08eb10aa25a39ac0cf6bf776391a6b?__liveforge_redacted__=1" + const sourceDocument = "server:\n name: liveforge\nprimary: " + sourceURL + "\n" + h, server := newTestHandlers(t) + manager, err := configruntime.NewManager(configruntime.Options{ + Source: &rawDocumentSource{document: []byte(sourceDocument)}, + Initial: nil, + }) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + if err := manager.Start(context.Background()); err != nil { + t.Fatal(err) + } + server.SetConfigManager(manager) + + w := httptest.NewRecorder() + h.handleConfigDocument(w, httptest.NewRequest(http.MethodGet, "/api/v1/server/config/document", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + data := decodeAPIData(t, w.Body.Bytes()) + var response struct { + Desired map[string]any `json:"desired"` + DesiredText string `json:"desired_document"` + } + if err := json.Unmarshal(data, &response); err != nil { + t.Fatal(err) + } + var desiredDocument map[string]any + if err := yaml.Unmarshal([]byte(response.DesiredText), &desiredDocument); err != nil { + t.Fatal(err) + } + for _, surface := range []struct { + name string + value any + }{ + {name: "decoded desired", value: response.Desired["primary"]}, + {name: "desired_document", value: desiredDocument["primary"]}, + } { + t.Run(surface.name, func(t *testing.T) { + if surface.value != redactedURL { + t.Fatalf("management response URL = %q, want source-path digest %q", surface.value, redactedURL) + } + if strings.Contains(surface.value.(string), "/__liveforge_redacted_path__/0123456789abcdef0123456789abcdef") { + t.Fatalf("management response leaked literal marker-shaped source path: %q", surface.value) + } + }) + } +} + func TestHandleConfigValidateAcceptsDocumentContentTypes(t *testing.T) { h, _ := newTestHandlers(t) for name, contentType := range map[string]string{ @@ -316,6 +516,75 @@ func TestHandleConfigValidateAcceptsDocumentContentTypes(t *testing.T) { } } +func TestHandleConfigValidateDoesNotExpandProcessEnvironment(t *testing.T) { + const processSecret = "viewer-must-not-read-this-process-secret" + t.Setenv("LIVEFORGE_VALIDATE_PROCESS_SECRET", processSecret) + h, _ := newTestHandlers(t) + req := httptest.NewRequest(http.MethodPost, "/api/v1/server/config/validate", strings.NewReader("server:\n name: \"${LIVEFORGE_VALIDATE_PROCESS_SECRET}\"\n")) + req.Header.Set("Content-Type", "application/yaml") + w := httptest.NewRecorder() + + h.handleConfigValidate(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + if strings.Contains(w.Body.String(), processSecret) { + t.Fatalf("validate response disclosed process environment value: %s", w.Body.String()) + } + data := decodeAPIData(t, w.Body.Bytes()) + var response struct { + Config map[string]any `json:"config"` + } + if err := json.Unmarshal(data, &response); err != nil { + t.Fatal(err) + } + if got := response.Config["server"].(map[string]any)["name"]; got != "${LIVEFORGE_VALIDATE_PROCESS_SECRET}" { + t.Fatalf("validated server.name=%q, want literal environment reference", got) + } +} + +func TestHandleConfigValidateRejectsUnknownKeys(t *testing.T) { + h, _ := newTestHandlers(t) + tests := []struct { + name string + document string + field string + }{ + {name: "top level", document: "servre:\n name: typo\n", field: "servre"}, + {name: "nested", document: "server:\n naem: typo\n", field: "naem"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/api/v1/server/config/validate", strings.NewReader(test.document)) + req.Header.Set("Content-Type", "application/yaml") + w := httptest.NewRecorder() + + h.handleConfigValidate(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), test.field) { + t.Fatalf("unknown-field error did not identify %q: %s", test.field, w.Body.String()) + } + }) + } +} + +func TestHandleConfigValidateRejectsSecondYAMLDocument(t *testing.T) { + h, _ := newTestHandlers(t) + req := httptest.NewRequest(http.MethodPost, "/api/v1/server/config/validate", strings.NewReader("server:\n name: liveforge\n---\nmalicious_or_unknown:\n value: ignored\n")) + req.Header.Set("Content-Type", "application/yaml") + w := httptest.NewRecorder() + + h.handleConfigValidate(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } +} + func TestEmbeddedConfigSchemaMatchesRepository(t *testing.T) { _, filename, _, ok := runtime.Caller(0) if !ok { @@ -342,19 +611,1006 @@ func TestEmbeddedConfigSchemaMatchesRepository(t *testing.T) { func TestRedactedSourceDetailsRemoveURLCredentials(t *testing.T) { details := redactedSourceDetails(config.RuntimeConfig{ + File: config.RuntimeFileSourceConfig{MaxBytes: 11}, HTTP: config.RuntimeHTTPSourceConfig{URL: "https://user:password@config.example.test/live.yaml?token=secret"}, Consul: config.RuntimeConsulSourceConfig{Address: "http://token:secret@consul.example.test:8500?auth=secret"}, + Redis: config.RuntimeRedisSourceConfig{MaxBytes: 13}, }) httpDetails := details["http"].(map[string]any) consulDetails := details["consul"].(map[string]any) - if got := httpDetails["url"]; got != "https://config.example.test/live.yaml" { - t.Fatalf("redacted HTTP URL = %q", got) + if got := details["file"].(map[string]any)["max_bytes"]; got != int64(11) { + t.Fatalf("redacted file max_bytes = %v, want 11", got) + } + if got := details["redis"].(map[string]any)["max_bytes"]; got != int64(13) { + t.Fatalf("redacted Redis max_bytes = %v, want 13", got) + } + if got := httpDetails["url"].(string); !strings.Contains(got, "config.example.test") || + !strings.Contains(got, redactedURLPathPrefix) || strings.Contains(got, "live.yaml") { + t.Fatalf("redacted HTTP URL = %q, want visible host and opaque path", got) } if got := consulDetails["address"]; got != "http://consul.example.test:8500" { t.Fatalf("redacted Consul address = %q", got) } } +func TestConfigSecretRedactionCoversAPIKeyAndSchemaSecretFields(t *testing.T) { + var schema map[string]any + if err := json.Unmarshal(embeddedConfigSchema, &schema); err != nil { + t.Fatal(err) + } + secretProperties := make(map[string]struct{}) + var collect func(map[string]any) + collect = func(node map[string]any) { + if properties, ok := node["properties"].(map[string]any); ok { + for name, raw := range properties { + child, ok := raw.(map[string]any) + if !ok { + continue + } + if secret, _ := child["x-liveforge-secret"].(bool); secret { + secretProperties[name] = struct{}{} + } + collect(child) + } + } + if definitions, ok := node["$defs"].(map[string]any); ok { + for _, raw := range definitions { + if child, ok := raw.(map[string]any); ok { + collect(child) + } + } + } + if items, ok := node["items"].(map[string]any); ok { + collect(items) + } + } + collect(schema) + for key := range secretProperties { + if !isSensitiveConfigKey(key) { + t.Errorf("schema x-liveforge-secret property %q is not classified as sensitive", key) + } + } + if !isSensitiveConfigKey("api_key") { + t.Error("api_key is not classified as sensitive") + } + + const sourceDocument = `tls: + cert_file: /etc/liveforge/public-cert.pem + key_file: /etc/liveforge/private-key-material.pem +custom_service: + api_key: custom-api-key-material +` + redacted, err := redactedConfigDocument([]byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"private-key-material", "custom-api-key-material"} { + if strings.Contains(string(redacted), secret) { + t.Fatalf("redacted document leaked %q: %s", secret, redacted) + } + } + restored, err := preserveRedactedSecretsWithDocument(redacted, config.Defaults(), []byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"private-key-material", "custom-api-key-material"} { + if !strings.Contains(string(restored), secret) { + t.Fatalf("restored document lost %q: %s", secret, restored) + } + } +} + +func TestConfigURLPathCredentialsAreOpaqueAndRestoreByStableIdentity(t *testing.T) { + const sourceDocument = `custom_callback_urls: + - https://hooks.slack.com/services/T111/B111/first-path-token + - https://hooks.slack.com/services/T222/B222/second-path-token +` + redacted, err := redactedConfigDocument([]byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + redactedText := string(redacted) + for _, secret := range []string{"T111", "B111", "first-path-token", "T222", "B222", "second-path-token"} { + if strings.Contains(redactedText, secret) { + t.Fatalf("redacted document leaked URL path credential %q: %s", secret, redacted) + } + } + if !strings.Contains(redactedText, "hooks.slack.com") { + t.Fatalf("redacted document lost safe URL host: %s", redacted) + } + + var candidate map[string]any + if unmarshalErr := yaml.Unmarshal(redacted, &candidate); unmarshalErr != nil { + t.Fatal(unmarshalErr) + } + reverseConfigSequence(t, candidate, "custom_callback_urls") + candidateDocument, err := yaml.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + restored, err := preserveRedactedSecretsWithDocument(candidateDocument, config.Defaults(), []byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := yaml.Unmarshal(restored, &document); err != nil { + t.Fatal(err) + } + urls := document["custom_callback_urls"].([]any) + if len(urls) != 2 || urls[0] != "https://hooks.slack.com/services/T222/B222/second-path-token" || urls[1] != "https://hooks.slack.com/services/T111/B111/first-path-token" { + t.Fatalf("restored reordered path-token URLs = %#v", urls) + } +} + +func TestRedactedSourceDetailsFailClosedForMalformedAddressAndPreserveHostPort(t *testing.T) { + malformed := redactedSourceDetails(config.RuntimeConfig{ + HTTP: config.RuntimeHTTPSourceConfig{URL: "http-user:http-password@config.example.test/live?token=query-secret"}, + Redis: config.RuntimeRedisSourceConfig{Addr: "redis-user:redis-password@redis.example.test:6379?token=query-secret"}, + }) + if got := malformed["http"].(map[string]any)["url"]; got != "" { + t.Fatalf("malformed HTTP URL was returned as %q", got) + } + if got := malformed["redis"].(map[string]any)["addr"]; got != "" { + t.Fatalf("malformed Redis address was returned as %q", got) + } + userinfo := redactedSourceDetails(config.RuntimeConfig{ + Redis: config.RuntimeRedisSourceConfig{Addr: "redis-user@redis.example.test:6379"}, + }) + if got := userinfo["redis"].(map[string]any)["addr"]; got != "" { + t.Fatalf("credential-like Redis address was returned as %q", got) + } + + plain := redactedSourceDetails(config.RuntimeConfig{Redis: config.RuntimeRedisSourceConfig{Addr: "127.0.0.1:6379"}}) + if got := plain["redis"].(map[string]any)["addr"]; got != "127.0.0.1:6379" { + t.Fatalf("plain Redis host:port = %q, want preserved address", got) + } +} + +func TestHandleConfigDocumentRedactsRedisAddressCredentials(t *testing.T) { + h, server := newTestHandlers(t) + cfg := config.Defaults() + cfg.Runtime.Source = "redis" + //nolint:gosec // Synthetic URL credentials verify the management response boundary. + cfg.Runtime.Redis.Addr = "redis://redis-user:redis-password@redis.example.test:6379?token=redis-secret#fragment" + cfg.Runtime.Redis.Username = "liveforge" + server.UpdateConfig(cfg) + manager, err := configruntime.NewManager(configruntime.Options{Source: testConfigSource{}, Initial: cfg}) + if err != nil { + t.Fatal(err) + } + defer manager.Close() + server.SetConfigManager(manager) + + w := httptest.NewRecorder() + h.handleConfigDocument(w, httptest.NewRequest(http.MethodGet, "/api/v1/server/config/document", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + data := decodeAPIData(t, w.Body.Bytes()) + var response struct { + SourceDetails map[string]any `json:"source_details"` + } + if err := json.Unmarshal(data, &response); err != nil { + t.Fatal(err) + } + redisDetails := response.SourceDetails["redis"].(map[string]any) + if got := redisDetails["addr"]; got != "redis://redis.example.test:6379" { + t.Fatalf("redacted Redis address = %q", got) + } + if got := redisDetails["username"]; got != "liveforge" { + t.Fatalf("Redis ACL identity = %q", got) + } + for _, secret := range []string{"redis-user", "redis-password", "redis-secret", "token=", "fragment"} { + if strings.Contains(w.Body.String(), secret) { + t.Fatalf("config document response leaked %q: %s", secret, w.Body.String()) + } + } +} + +func TestRedactedConfigDocumentRemovesURLCredentialsAndRestoresOnApply(t *testing.T) { + //nolint:gosec // Intentional fake URL credentials verify redaction. + const sourceDocument = `runtime: + source: https + http: + url: https://user:password@config.example.test/live.yaml?token=source-secret +` + redacted, err := redactedConfigDocument([]byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + redactedText := string(redacted) + for _, secret := range []string{"user", "password", "source-secret", "token="} { + if strings.Contains(redactedText, secret) { + t.Fatalf("redacted document leaked %q: %s", secret, redactedText) + } + } + if !strings.Contains(redactedText, "REDACTED") { + t.Fatalf("redacted URL did not contain an explicit marker: %s", redactedText) + } + + current := config.Defaults() + current.Runtime.Source = "https" + current.Runtime.HTTP.URL = "https://user:password@config.example.test/live.yaml?token=source-secret" + restored, err := preserveRedactedSecrets(redacted, current) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(restored), "https://user:password@config.example.test/live.yaml?token=source-secret") { + t.Fatalf("restored document lost the original URL credentials: %q", restored) + } +} + +func TestRedactedConfigDocumentFailsClosedForHostlessURLAndPreservesPlainAddress(t *testing.T) { + const sourceDocument = `custom_callback_url: callback-user:callback-password@callback.example.test/hook?token=query-secret +runtime: + redis: + addr: 127.0.0.1:6379 +` + redacted, err := redactedConfigDocument([]byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"callback-user", "callback-password", "query-secret", "token="} { + if strings.Contains(string(redacted), secret) { + t.Fatalf("redacted document leaked %q: %s", secret, redacted) + } + } + var document map[string]any + if err := yaml.Unmarshal(redacted, &document); err != nil { + t.Fatal(err) + } + if document["custom_callback_url"] != "[REDACTED]" { + t.Fatalf("hostless callback URL = %#v, want opaque marker", document["custom_callback_url"]) + } + if got := document["runtime"].(map[string]any)["redis"].(map[string]any)["addr"]; got != "127.0.0.1:6379" { + t.Fatalf("plain Redis host:port = %q, want preserved address", got) + } +} + +func TestConfigMapRedactionFailsClosedForHostlessURLAndPreservesPlainAddress(t *testing.T) { + document := map[string]any{ + "custom_callback_url": "callback-user:callback-password@callback.example.test/hook?token=query-secret", + "runtime": map[string]any{ + "redis": map[string]any{"addr": "127.0.0.1:6379"}, + }, + } + redactConfigValue(document) + if document["custom_callback_url"] != "[REDACTED]" { + t.Fatalf("hostless callback URL = %#v, want opaque marker", document["custom_callback_url"]) + } + if got := document["runtime"].(map[string]any)["redis"].(map[string]any)["addr"]; got != "127.0.0.1:6379" { + t.Fatalf("plain Redis host:port = %q, want preserved address", got) + } +} + +func TestConfigRedactionPreservesOnlyBareIPOrValidatedHostPortAddresses(t *testing.T) { + const sourceDocument = `ipv4_address: 239.0.0.1 +ipv6_address: "ff15::1" +hostname_address: relay.example.test +credential_address: relay-user@relay.example.test +path_address: /var/run/relay.sock +query_address: 239.0.0.1?token=query-secret +fragment_address: 239.0.0.1#fragment-secret +malformed_address: "[invalid" +redis_address: 127.0.0.1:6379 +rtsp: + multicast: + address: 239.0.0.1 +` + + assertAddresses := func(t *testing.T, document map[string]any) { + t.Helper() + if document["ipv4_address"] != "239.0.0.1" { + t.Fatalf("bare IPv4 address = %#v, want preserved", document["ipv4_address"]) + } + if document["ipv6_address"] != "ff15::1" { + t.Fatalf("bare IPv6 address = %#v, want preserved", document["ipv6_address"]) + } + if document["redis_address"] != "127.0.0.1:6379" { + t.Fatalf("validated host:port = %#v, want preserved", document["redis_address"]) + } + multicast := document["rtsp"].(map[string]any)["multicast"].(map[string]any) + if multicast["address"] != "239.0.0.1" { + t.Fatalf("RTSP multicast address = %#v, want preserved", multicast["address"]) + } + for _, key := range []string{ + "hostname_address", "credential_address", "path_address", "query_address", + "fragment_address", "malformed_address", + } { + if document[key] != "[REDACTED]" { + t.Fatalf("unsafe %s = %#v, want opaque marker", key, document[key]) + } + } + } + + t.Run("YAML document", func(t *testing.T) { + redacted, err := redactedConfigDocument([]byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := yaml.Unmarshal(redacted, &document); err != nil { + t.Fatal(err) + } + assertAddresses(t, document) + }) + + t.Run("decoded map", func(t *testing.T) { + var document map[string]any + if err := yaml.Unmarshal([]byte(sourceDocument), &document); err != nil { + t.Fatal(err) + } + redactConfigValue(document) + assertAddresses(t, document) + }) +} + +func TestConfigRedactionPreservesSecretContainerShapeAndRedactsNestedValues(t *testing.T) { + //nolint:gosec // Intentional fake credentials verify the management redaction boundary. + const sourceDocument = `api: + auth: + tokens: + - name: viewer + token: viewer-secret + role: viewer +notify: + http: + endpoints: + - url: https://hook-user:hook-password@notify.example.test/live?token=query-secret#fragment-secret + events: [publish] + secret: webhook-secret + retry: 2 + timeout: 3s +` + redacted, err := redactedConfigDocument([]byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := yaml.Unmarshal(redacted, &document); err != nil { + t.Fatal(err) + } + tokens, ok := document["api"].(map[string]any)["auth"].(map[string]any)["tokens"].([]any) + if !ok || len(tokens) != 1 { + t.Fatalf("tokens structure = %#v, want one-item sequence", document["api"]) + } + token := tokens[0].(map[string]any) + if token["name"] != "viewer" || token["role"] != "[REDACTED]" || token["token"] != "[REDACTED]" { + t.Fatalf("redacted token = %#v", token) + } + endpoints, ok := document["notify"].(map[string]any)["http"].(map[string]any)["endpoints"].([]any) + if !ok || len(endpoints) != 1 { + t.Fatalf("endpoints structure = %#v, want one-item sequence", document["notify"]) + } + endpoint := endpoints[0].(map[string]any) + if endpoint["secret"] != "[REDACTED]" || endpoint["events"].([]any)[0] != "publish" { + t.Fatalf("redacted endpoint = %#v", endpoint) + } + for _, leaked := range []string{"hook-user", "hook-password", "query-secret", "fragment-secret", "webhook-secret", "viewer-secret"} { + if strings.Contains(string(redacted), leaked) { + t.Fatalf("redacted document leaked %q: %s", leaked, redacted) + } + } +} + +func TestRedactedConfigDocumentRedactsOpaqueSensitiveContainers(t *testing.T) { + // #nosec G101 -- synthetic credential-like URLs exercise redaction without real secrets. + const sourceDocument = `custom_credentials: + name: primary + value: mapping-secret + nested: + id: nested + material: nested-secret +custom_private_keys: + - name: first + material: item-secret + - raw-sequence-secret + - [nested-sequence-secret] +` + redacted, err := redactedConfigDocument([]byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"mapping-secret", "nested-secret", "item-secret", "raw-sequence-secret", "nested-sequence-secret"} { + if strings.Contains(string(redacted), secret) { + t.Fatalf("redacted YAML leaked %q: %s", secret, redacted) + } + } + var document map[string]any + if err := yaml.Unmarshal(redacted, &document); err != nil { + t.Fatal(err) + } + assertOpaqueSensitiveContainersRedacted(t, document) +} + +func TestConfigMapRedactionRedactsOpaqueSensitiveContainers(t *testing.T) { + const sourceDocument = `custom_credentials: + name: primary + value: mapping-secret + nested: + id: nested + material: nested-secret +custom_private_keys: + - name: first + material: item-secret + - raw-sequence-secret + - [nested-sequence-secret] +` + var document map[string]any + if err := yaml.Unmarshal([]byte(sourceDocument), &document); err != nil { + t.Fatal(err) + } + redactConfigValue(document) + assertOpaqueSensitiveContainersRedacted(t, document) +} + +func TestOpaqueSensitiveContainerRedactionKeepsStructuredURLValuesOpaque(t *testing.T) { + // #nosec G101 -- synthetic credential-like URLs exercise redaction without real secrets. + const sourceDocument = `custom_credentials: + name: primary + callback_url: + name: mapping + neutral: mapping-secret + address: + id: nested-address + host: address-secret + callback_urls: + - name: sequence-entry + neutral: sequence-secret + endpoint: + channel_id: nested-endpoint + payload: endpoint-secret + - scalar-sequence-secret + public_url: https://url-user:url-password@public.example.test/hook?token=url-secret + public_urls: + - https://list-user:list-password@list.example.test/hook?token=list-secret +` + + t.Run("YAML document", func(t *testing.T) { + redacted, err := redactedConfigDocument([]byte(sourceDocument)) + if err != nil { + t.Fatal(err) + } + assertNoStructuredURLSecretLeak(t, string(redacted)) + var document map[string]any + if err := yaml.Unmarshal(redacted, &document); err != nil { + t.Fatal(err) + } + assertStructuredURLValuesOpaque(t, document) + }) + + t.Run("decoded map", func(t *testing.T) { + var document map[string]any + if err := yaml.Unmarshal([]byte(sourceDocument), &document); err != nil { + t.Fatal(err) + } + redactConfigValue(document) + encoded, err := yaml.Marshal(document) + if err != nil { + t.Fatal(err) + } + assertNoStructuredURLSecretLeak(t, string(encoded)) + assertStructuredURLValuesOpaque(t, document) + }) +} + +func TestPreserveRedactedSecretsRestoresOpaqueSensitiveItemsByIdentity(t *testing.T) { + const currentDocument = `custom_private_keys: + - {name: first, material: first-secret} + - {name: second, material: second-secret} +` + const candidateDocument = `custom_private_keys: + - {name: second, material: "[REDACTED]"} + - {name: first, material: "[REDACTED]"} +` + restored, err := preserveRedactedSecretsWithDocument([]byte(candidateDocument), config.Defaults(), []byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + text := string(restored) + if !strings.Contains(text, "first-secret") || !strings.Contains(text, "second-secret") || strings.Contains(text, "[REDACTED]") { + t.Fatalf("opaque structured secrets were not restored by identity: %s", restored) + } +} + +func TestConfigMapRedactionPreservesNamedTokenAndEndpointCollections(t *testing.T) { + current := config.Defaults() + current.API.Auth.Tokens = []config.APIAuthToken{{Name: "viewer", Token: "viewer-secret", Role: "viewer"}} + current.Notify.HTTP.Endpoints = []config.NotifyEndpointConfig{{ + URL: "https://notify.example.test/live?token=query-secret", Events: []string{"publish"}, Secret: "webhook-secret", + }} + + redacted := configMapFromConfig(current) + tokens, ok := redacted["api"].(map[string]any)["auth"].(map[string]any)["tokens"].([]any) + if !ok || len(tokens) != 1 { + t.Fatalf("tokens structure = %#v", redacted["api"]) + } + if token := tokens[0].(map[string]any); token["name"] != "viewer" || token["token"] != "[REDACTED]" { + t.Fatalf("redacted token = %#v", token) + } + endpoints, ok := redacted["notify"].(map[string]any)["http"].(map[string]any)["endpoints"].([]any) + if !ok || len(endpoints) != 1 { + t.Fatalf("endpoints structure = %#v", redacted["notify"]) + } + if endpoint := endpoints[0].(map[string]any); endpoint["secret"] != "[REDACTED]" || strings.Contains(endpoint["url"].(string), "query-secret") { + t.Fatalf("redacted endpoint = %#v", endpoint) + } +} + +func TestPreserveRedactedSecretsMatchesReorderedCollectionsByStableIdentity(t *testing.T) { + //nolint:gosec // Intentional fake credentials verify identity-based restoration. + const currentDocument = `api: + auth: + tokens: + - {name: alpha, token: alpha-secret, role: viewer} + - {name: beta, token: beta-secret, role: operator} +webrtc: + ice_servers: + - {urls: ["turn:one.example.test"], username: one, credential: ice-one} + - {urls: ["turn:two.example.test"], username: two, credential: ice-two} +notify: + http: + endpoints: + - {url: "https://one.example.test/hook?token=one-query", events: [publish], secret: hook-one, retry: 1, timeout: 1s} + - {url: "https://two.example.test/hook?token=two-query", events: [unpublish], secret: hook-two, retry: 2, timeout: 2s} +` + redacted, err := redactedConfigDocument([]byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + var candidate map[string]any + if unmarshalErr := yaml.Unmarshal(redacted, &candidate); unmarshalErr != nil { + t.Fatal(unmarshalErr) + } + reverseConfigSequence(t, candidate, "api", "auth", "tokens") + reverseConfigSequence(t, candidate, "webrtc", "ice_servers") + reverseConfigSequence(t, candidate, "notify", "http", "endpoints") + candidateDocument, err := yaml.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + restored, err := preserveRedactedSecretsWithDocument(candidateDocument, config.Defaults(), []byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := yaml.Unmarshal(restored, &got); err != nil { + t.Fatal(err) + } + assertConfigSecretByIdentity(t, got, []string{"api", "auth", "tokens"}, "name", "beta", "token", "beta-secret") + assertConfigSecretByIdentity(t, got, []string{"webrtc", "ice_servers"}, "username", "two", "credential", "ice-two") + assertConfigSecretByIdentity(t, got, []string{"notify", "http", "endpoints"}, "events", "unpublish", "secret", "hook-two") +} + +func TestPreserveRedactedSecretsDoesNotTransplantDeletedSecretIntoInsertedItem(t *testing.T) { + const currentDocument = `api: + auth: + tokens: + - {name: keep, token: keep-secret, role: viewer} + - {name: delete, token: delete-secret, role: viewer} +` + const candidateDocument = `api: + auth: + tokens: + - {name: inserted, token: inserted-secret, role: operator} + - {name: keep, token: "[REDACTED]", role: viewer} +` + restored, err := preserveRedactedSecretsWithDocument([]byte(candidateDocument), config.Defaults(), []byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + text := string(restored) + if !strings.Contains(text, "inserted-secret") || !strings.Contains(text, "keep-secret") || strings.Contains(text, "delete-secret") { + t.Fatalf("insert/delete restoration crossed identities: %s", text) + } +} + +func TestPreserveRedactedSecretsRejectsRenamedSingletonStructuredItem(t *testing.T) { + const currentDocument = `api: + auth: + tokens: + - {name: original, token: original-secret, role: viewer} +` + const candidateDocument = `api: + auth: + tokens: + - {name: renamed, token: "[REDACTED]", role: viewer} +` + if _, err := preserveRedactedSecretsWithDocument([]byte(candidateDocument), config.Defaults(), []byte(currentDocument)); err == nil { + t.Fatal("renamed singleton token received the original item's secret") + } +} + +func TestPreserveRedactedURLKeepsEditedLocationAndRestoresOnlySecretComponents(t *testing.T) { + //nolint:gosec // Intentional fake URL credentials verify component restoration. + const currentDocument = `runtime: + source: https + http: + url: https://source-user:source-password@old.example.test/old.yaml?token=source-secret#source-fragment +` + redacted, err := redactedConfigDocument([]byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + var edited map[string]any + if unmarshalErr := yaml.Unmarshal(redacted, &edited); unmarshalErr != nil { + t.Fatal(unmarshalErr) + } + edited["runtime"].(map[string]any)["http"].(map[string]any)["url"] = "https://REDACTED@new.example.test/new.yaml?__liveforge_redacted__=1" + editedDocument, err := yaml.Marshal(edited) + if err != nil { + t.Fatal(err) + } + restored, err := preserveRedactedSecretsWithDocument(editedDocument, config.Defaults(), []byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + want := strings.Join([]string{ + "https://", "source-user", ":", "source-password", "@new.example.test/new.yaml", + "?token=", "source-secret", "#", "source-fragment", + }, "") + if !strings.Contains(string(restored), want) { + t.Fatalf("restored URL = %s, want edited location with original secret components %q", restored, want) + } +} + +func TestPreserveRedactedURLRestoresOpaqueScalarMarker(t *testing.T) { + const currentDocument = `custom_callback_url: callback-user:callback-password@callback.example.test/hook?token=query-secret +` + const candidateDocument = `custom_callback_url: "[REDACTED]" +` + restored, err := preserveRedactedSecretsWithDocument([]byte(candidateDocument), config.Defaults(), []byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(restored), "callback-user:callback-password@callback.example.test/hook?token=query-secret") { + t.Fatalf("opaque URL marker was persisted instead of restored: %s", restored) + } +} + +func TestPreserveRedactedURLRestoresOpaqueTURNURIQuery(t *testing.T) { + const currentDocument = `webrtc: + ice_servers: + - urls: ["turn:relay.example.test:3478?transport=udp&token=turn-secret"] + username: relay + credential: relay-secret +` + redacted, err := redactedConfigDocument([]byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(redacted), "turn-secret") { + t.Fatalf("redacted TURN URI leaked its query: %s", redacted) + } + restored, err := preserveRedactedSecretsWithDocument(redacted, config.Defaults(), []byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(restored), "turn:relay.example.test:3478?transport=udp&token=turn-secret") { + t.Fatalf("TURN URI query was not restored: %s", restored) + } +} + +func TestPreserveRedactedURLRejectsMarkedShapeMismatch(t *testing.T) { + tests := []struct { + name string + current string + candidate string + }{ + { + name: "scalar original and sequence candidate", + current: "custom_callback_url: https://user:password@scalar.example.test/hook?token=secret\n", + candidate: "custom_callback_url:\n - https://REDACTED@scalar.example.test/hook?__liveforge_redacted__=1\n", + }, + { + name: "scalar original and mapping candidate", + current: "custom_callback_url: https://user:password@scalar.example.test/hook?token=secret\n", + candidate: "custom_callback_url:\n primary_url: https://REDACTED@scalar.example.test/hook?__liveforge_redacted__=1\n", + }, + { + name: "sequence original and scalar candidate", + current: "custom_callback_url:\n - https://user:password@sequence.example.test/hook?token=secret\n", + candidate: "custom_callback_url: \"[REDACTED]\"\n", + }, + { + name: "sequence original and mapping candidate", + current: "custom_callback_url:\n - https://user:password@sequence.example.test/hook?token=secret\n", + candidate: "custom_callback_url:\n primary_url: https://REDACTED@sequence.example.test/hook?__liveforge_redacted__=1\n", + }, + { + name: "mapping original and scalar candidate", + current: "custom_callback_url:\n primary_url: https://user:password@mapping.example.test/hook?token=secret\n", + candidate: "custom_callback_url: \"[REDACTED]\"\n", + }, + { + name: "mapping original and sequence candidate", + current: "custom_callback_url:\n primary_url: https://user:password@mapping.example.test/hook?token=secret\n", + candidate: "custom_callback_url:\n - https://REDACTED@mapping.example.test/hook?__liveforge_redacted__=1\n", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + restored, err := preserveRedactedSecretsWithDocument([]byte(test.candidate), config.Defaults(), []byte(test.current)) + if err == nil { + t.Fatalf("marked URL shape mismatch was accepted and produced: %s", restored) + } + if restored != nil { + t.Fatalf("failed restoration returned a document containing placeholders: %s", restored) + } + }) + } +} + +func TestPreserveRedactedURLSequenceMatchesReorderedPublicIdentity(t *testing.T) { + const currentDocument = `custom_callback_urls: + - https://first-user:first-password@first.example.test/hook?token=first-secret + - https://second-user:second-password@second.example.test/hook?token=second-secret +` + redacted, err := redactedConfigDocument([]byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + var candidate map[string]any + if unmarshalErr := yaml.Unmarshal(redacted, &candidate); unmarshalErr != nil { + t.Fatal(unmarshalErr) + } + reverseConfigSequence(t, candidate, "custom_callback_urls") + candidateDocument, err := yaml.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + restored, err := preserveRedactedSecretsWithDocument(candidateDocument, config.Defaults(), []byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := yaml.Unmarshal(restored, &document); err != nil { + t.Fatal(err) + } + urls := document["custom_callback_urls"].([]any) + wantFirst := "https://second-user:second-password@second.example.test/hook?token=second-secret" // #nosec G101 -- synthetic redaction fixture. + wantSecond := "https://first-user:first-password@first.example.test/hook?token=first-secret" // #nosec G101 -- synthetic redaction fixture. + if len(urls) != 2 || urls[0] != wantFirst || urls[1] != wantSecond { + t.Fatalf("restored reordered URLs = %#v, want [%q %q]", urls, wantFirst, wantSecond) + } +} + +func TestPreserveRedactedUnmappedURLSequenceUsesStableValueIdentity(t *testing.T) { + // #nosec G101 -- synthetic credential-like URLs exercise redaction without real secrets. + const currentDocument = `mirrors: + - https://first-user:first-password@hooks.slack.com/services/T111/B111/first-path-token?token=first-query#first-fragment + - https://second-user:second-password@hooks.slack.com/services/T222/B222/second-path-token?token=second-query#second-fragment +` + redacted, err := redactedConfigDocument([]byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"first-user", "first-password", "T111", "B111", "first-path-token", "first-query", "second-user", "second-password", "T222", "B222", "second-path-token", "second-query"} { + if strings.Contains(string(redacted), secret) { + t.Errorf("redacted unmapped URL sequence leaked %q: %s", secret, redacted) + } + } + + t.Run("reordered", func(t *testing.T) { + var candidate map[string]any + if err := yaml.Unmarshal(redacted, &candidate); err != nil { + t.Fatal(err) + } + reverseConfigSequence(t, candidate, "mirrors") + candidateDocument, err := yaml.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + restored, err := preserveRedactedSecretsWithDocument(candidateDocument, config.Defaults(), []byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := yaml.Unmarshal(restored, &document); err != nil { + t.Fatal(err) + } + urls := document["mirrors"].([]any) + wantFirst := "https://second-user:second-password@hooks.slack.com/services/T222/B222/second-path-token?token=second-query#second-fragment" // #nosec G101 -- synthetic redaction fixture. + wantSecond := "https://first-user:first-password@hooks.slack.com/services/T111/B111/first-path-token?token=first-query#first-fragment" // #nosec G101 -- synthetic redaction fixture. + if len(urls) != 2 || urls[0] != wantFirst || urls[1] != wantSecond { + t.Fatalf("restored reordered unmapped URLs = %#v, want [%q %q]", urls, wantFirst, wantSecond) + } + }) + + t.Run("edited identity", func(t *testing.T) { + var candidate map[string]any + if err := yaml.Unmarshal(redacted, &candidate); err != nil { + t.Fatal(err) + } + urls := candidate["mirrors"].([]any) + urls[0] = strings.Replace(urls[0].(string), "hooks.slack.com", "edited.example.test", 1) + candidateDocument, err := yaml.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + if restored, err := preserveRedactedSecretsWithDocument(candidateDocument, config.Defaults(), []byte(currentDocument)); err == nil { + t.Fatalf("edited unmapped URL identity received a source credential: %s", restored) + } + }) + + t.Run("ambiguous identity", func(t *testing.T) { + // #nosec G101 -- synthetic credential-like URLs exercise ambiguous restoration handling. + const ambiguousDocument = `mirrors: + - https://first-user:first-password@hooks.slack.com/services/SHARED/PATH/token?token=first-query + - https://second-user:second-password@hooks.slack.com/services/SHARED/PATH/token?token=second-query +` + ambiguous, err := redactedConfigDocument([]byte(ambiguousDocument)) + if err != nil { + t.Fatal(err) + } + if restored, err := preserveRedactedSecretsWithDocument(ambiguous, config.Defaults(), []byte(ambiguousDocument)); err == nil { + t.Fatalf("ambiguous unmapped URL identities were restored by position: %s", restored) + } + }) +} + +func TestPreserveRedactedURLRoundTripsLiteralPathMarker(t *testing.T) { + //nolint:gosec // Intentional fake URL credentials verify placeholder restoration. + const sourceURL = "https://source-user:source-password@hooks.slack.com/__liveforge_redacted_path__/0123456789abcdef0123456789abcdef?token=source-query#source-fragment" + const redactedURL = "https://REDACTED@hooks.slack.com/__liveforge_redacted_path__/3b08eb10aa25a39ac0cf6bf776391a6b?__liveforge_redacted__=1" + const currentDocument = "primary: " + sourceURL + "\n" + redacted, err := redactedConfigDocument([]byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + var candidate map[string]any + if unmarshalErr := yaml.Unmarshal(redacted, &candidate); unmarshalErr != nil { + t.Fatal(unmarshalErr) + } + if candidate["primary"] != redactedURL { + t.Fatalf("redacted URL = %q, want source-path digest %q", candidate["primary"], redactedURL) + } + + restored, err := preserveRedactedSecretsWithDocument(redacted, config.Defaults(), []byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + var document map[string]any + if err := yaml.Unmarshal(restored, &document); err != nil { + t.Fatal(err) + } + if document["primary"] != sourceURL { + t.Fatalf("restored URL = %q, want exact source URL %q", document["primary"], sourceURL) + } +} + +func TestPreserveRedactedURLRejectsLiteralMarkerCandidateWithoutMatchingSourceDigest(t *testing.T) { + //nolint:gosec // Intentional fake URL credentials verify fail-closed restoration. + const currentDocument = "primary: https://source-user:source-password@hooks.slack.com/__liveforge_redacted_path__/0123456789abcdef0123456789abcdef?token=source-query#source-fragment\n" + const candidateDocument = "primary: https://hooks.slack.com/__liveforge_redacted_path__/0123456789abcdef0123456789abcdef\n" + restored, err := preserveRedactedSecretsWithDocument([]byte(candidateDocument), config.Defaults(), []byte(currentDocument)) + if err == nil { + t.Fatalf("marker-looking candidate bypassed source-path identity matching: %s", restored) + } + if restored != nil { + t.Fatalf("failed restoration returned a document containing source credentials: %s", restored) + } +} + +func TestPreserveRedactedSecretsRejectsAmbiguousCollectionIdentity(t *testing.T) { + const currentDocument = `notify: + http: + endpoints: + - {url: "https://one.example.test/hook?token=one", events: [publish], secret: one, retry: 1, timeout: 1s} + - {url: "https://two.example.test/hook?token=two", events: [publish], secret: two, retry: 1, timeout: 1s} +` + redacted, err := redactedConfigDocument([]byte(currentDocument)) + if err != nil { + t.Fatal(err) + } + edited := strings.ReplaceAll(string(redacted), "one.example.test", "edited-one.example.test") + edited = strings.ReplaceAll(edited, "two.example.test", "edited-two.example.test") + if _, err := preserveRedactedSecretsWithDocument([]byte(edited), config.Defaults(), []byte(currentDocument)); err == nil { + t.Fatal("ambiguous endpoint identity was accepted") + } +} + +func reverseConfigSequence(t *testing.T, document map[string]any, path ...string) { + t.Helper() + var current any = document + for _, key := range path { + current = current.(map[string]any)[key] + } + items := current.([]any) + for left, right := 0, len(items)-1; left < right; left, right = left+1, right-1 { + items[left], items[right] = items[right], items[left] + } +} + +func assertConfigSecretByIdentity(t *testing.T, document map[string]any, path []string, identityKey, identityValue, secretKey, secretValue string) { + t.Helper() + var current any = document + for _, key := range path { + current = current.(map[string]any)[key] + } + for _, item := range current.([]any) { + entry := item.(map[string]any) + matches := entry[identityKey] == identityValue + if values, ok := entry[identityKey].([]any); ok { + matches = len(values) == 1 && values[0] == identityValue + } + if matches { + if entry[secretKey] != secretValue { + t.Fatalf("%s=%v for %s=%v, want %q", secretKey, entry[secretKey], identityKey, entry[identityKey], secretValue) + } + return + } + } + t.Fatalf("identity %s=%q not found at %v", identityKey, identityValue, path) +} + +func assertOpaqueSensitiveContainersRedacted(t *testing.T, document map[string]any) { + t.Helper() + credentials := document["custom_credentials"].(map[string]any) + if credentials["name"] != "primary" || credentials["value"] != "[REDACTED]" { + t.Fatalf("redacted custom_credentials = %#v", credentials) + } + nested := credentials["nested"].(map[string]any) + if nested["id"] != "nested" || nested["material"] != "[REDACTED]" { + t.Fatalf("redacted nested credentials = %#v", nested) + } + keys := document["custom_private_keys"].([]any) + first := keys[0].(map[string]any) + if first["name"] != "first" || first["material"] != "[REDACTED]" { + t.Fatalf("redacted structured private key = %#v", first) + } + if keys[1] != "[REDACTED]" || keys[2].([]any)[0] != "[REDACTED]" { + t.Fatalf("redacted heterogeneous private keys = %#v", keys) + } +} + +func assertNoStructuredURLSecretLeak(t *testing.T, encoded string) { + t.Helper() + for _, secret := range []string{ + "mapping-secret", "address-secret", "sequence-secret", "endpoint-secret", + "scalar-sequence-secret", "url-user", "url-password", "url-secret", + "list-user", "list-password", "list-secret", + } { + if strings.Contains(encoded, secret) { + t.Fatalf("structured URL redaction leaked %q: %s", secret, encoded) + } + } +} + +func assertStructuredURLValuesOpaque(t *testing.T, document map[string]any) { + t.Helper() + credentials := document["custom_credentials"].(map[string]any) + callback := credentials["callback_url"].(map[string]any) + if callback["name"] != "mapping" || callback["neutral"] != "[REDACTED]" { + t.Fatalf("structured callback_url = %#v", callback) + } + address := callback["address"].(map[string]any) + if address["id"] != "nested-address" || address["host"] != "[REDACTED]" { + t.Fatalf("structured address = %#v", address) + } + callbacks := credentials["callback_urls"].([]any) + entry := callbacks[0].(map[string]any) + if entry["name"] != "sequence-entry" || entry["neutral"] != "[REDACTED]" { + t.Fatalf("structured callback_urls entry = %#v", entry) + } + endpoint := entry["endpoint"].(map[string]any) + if endpoint["channel_id"] != "nested-endpoint" || endpoint["payload"] != "[REDACTED]" { + t.Fatalf("structured endpoint = %#v", endpoint) + } + if callbacks[1] != "[REDACTED]" { + t.Fatalf("heterogeneous scalar URL value = %#v", callbacks[1]) + } + publicURL := credentials["public_url"].(string) + if !strings.Contains(publicURL, "public.example.test") || strings.Contains(publicURL, "/hook") || !isRedactedConfigURL(publicURL) { + t.Fatalf("scalar public_url lost safe identity: %q", publicURL) + } + publicURLs := credentials["public_urls"].([]any) + if len(publicURLs) != 1 || !strings.Contains(publicURLs[0].(string), "list.example.test") || strings.Contains(publicURLs[0].(string), "/hook") || !isRedactedConfigURL(publicURLs[0].(string)) { + t.Fatalf("scalar public_urls lost safe identity: %#v", publicURLs) + } +} + type rawDocumentSource struct { document []byte } @@ -364,3 +1620,12 @@ func (s *rawDocumentSource) Load(context.Context, configruntime.Version) (config } func (s *rawDocumentSource) Close() error { return nil } + +type errorConfigWriterSource struct{ err error } + +func (s errorConfigWriterSource) Load(context.Context, configruntime.Version) (configruntime.Snapshot, error) { + return configruntime.Snapshot{}, s.err +} + +func (s errorConfigWriterSource) Write(context.Context, []byte) error { return s.err } +func (s errorConfigWriterSource) Close() error { return nil } diff --git a/module/api/configschema/config.schema.json b/module/api/configschema/config.schema.json index a60b4f36..1b1e6889 100644 --- a/module/api/configschema/config.schema.json +++ b/module/api/configschema/config.schema.json @@ -74,7 +74,7 @@ "max_bitrate_per_stream": {"type": "integer", "minimum": 0}, "rate_limit": { "type": "object", "additionalProperties": false, - "properties": {"enabled": {"type": "boolean"}, "rate": {"type": "number", "minimum": 0}, "burst": {"type": "integer", "minimum": 0}} + "properties": {"enabled": {"type": "boolean"}, "rate": {"type": "number", "minimum": 0}, "burst": {"type": "integer", "minimum": 0}, "trusted_proxies": {"type": "array", "items": {"type": "string", "minLength": 1}, "uniqueItems": true, "default": [], "description": "IP addresses or CIDR networks allowed to supply X-Forwarded-For or X-Real-IP. Forwarded headers are ignored for every other direct peer."}} } } }, @@ -141,13 +141,13 @@ "properties": { "enabled": {"type": "boolean"}, "listen": {"type": "string"}, "transport": {"type": "array", "items": {"type": "string", "enum": ["udp", "tcp"]}}, "server_id": {"type": "string"}, "domain": {"type": "string"}, "auth": {"type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean"}, "password": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}}}, - "gateway": {"type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean"}, "stream_prefix": {"type": "string"}, "rtp_port_range": {"$ref": "#/$defs/port_range"}, "codecs": {"type": "array", "items": {"type": "string"}}, "max_calls": {"type": "integer", "description": "Maximum concurrent calls; any non-positive value selects the gateway default of 100."}}} + "gateway": {"type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean"}, "stream_prefix": {"type": "string"}, "rtp_port_range": {"$ref": "#/$defs/port_range"}, "codecs": {"type": "array", "items": {"type": "string"}}, "max_calls": {"type": "integer", "description": "Maximum concurrent calls; any non-positive value selects the gateway default of 100."}, "max_lab_sessions": {"type": "integer", "minimum": 1, "description": "Maximum active persistent local SIP protocol-lab sessions; non-positive values use the default of 16.", "x-liveforge-reload": "restart_required"}}} } }, "gb28181": { "type": "object", "additionalProperties": false, "properties": { - "enabled": {"type": "boolean"}, "stream_prefix": {"type": "string"}, "rtp_port_range": {"$ref": "#/$defs/defaultable_port_range", "description": "An empty list selects the GB28181 module fallback range 40000-50000."}, + "enabled": {"type": "boolean"}, "stream_prefix": {"type": "string"}, "rtp_port_range": {"$ref": "#/$defs/defaultable_port_range", "description": "An empty list selects the GB28181 module fallback range 40000-50000."}, "max_lab_sessions": {"type": "integer", "minimum": 1, "description": "Maximum active persistent local GB28181 protocol-lab sessions; non-positive values use the default of 16.", "x-liveforge-reload": "restart_required"}, "ssrc": {"type": "object", "additionalProperties": false, "properties": {"prefix": {"type": "string"}}}, "keepalive": {"type": "object", "additionalProperties": false, "properties": {"interval": {"$ref": "#/$defs/duration"}, "timeout": {"$ref": "#/$defs/duration"}}}, "auto_invite": {"type": "boolean"}, "catalog_interval": {"$ref": "#/$defs/duration"}, "dump_file": {"type": "string"} @@ -174,11 +174,12 @@ "stream": { "type": "object", "additionalProperties": false, "properties": { - "gop_cache": {"type": "boolean", "x-liveforge-reload": "hot_reload"}, "gop_cache_num": {"type": "integer", "minimum": 0, "x-liveforge-reload": "hot_reload"}, + "gop_cache": {"type": "boolean", "x-liveforge-reload": "hot_reload"}, "gop_cache_num": {"type": "integer", "minimum": 0, "x-liveforge-reload": "hot_reload"}, "gop_cache_max_frames": {"type": "integer", "minimum": 0, "description": "Maximum frames in one cached GOP; zero disables this bound. Hot reload recomputes the retained prefix; relaxation affects future frames only.", "x-liveforge-reload": "hot_reload"}, "gop_cache_max_duration": {"$ref": "#/$defs/duration", "description": "Maximum duration of one cached GOP; zero disables this bound. Hot reload recomputes the retained prefix; relaxation affects future frames only.", "x-liveforge-reload": "hot_reload"}, "gop_cache_max_bytes": {"type": "integer", "minimum": 0, "description": "Maximum payload bytes in one cached GOP; zero disables this bound. Hot reload recomputes the retained prefix; relaxation affects future frames only.", "x-liveforge-reload": "hot_reload"}, "ring_buffer_size": {"type": "integer", "minimum": 1, "x-liveforge-reload": "restart_required"}, "idle_timeout": {"$ref": "#/$defs/duration", "x-liveforge-reload": "hot_reload"}, "no_publisher_timeout": {"$ref": "#/$defs/duration", "x-liveforge-reload": "hot_reload"}, "slow_consumer": {"$ref": "#/$defs/slow_consumer", "x-liveforge-reload": "hot_reload"}, "feedback": {"$ref": "#/$defs/feedback", "x-liveforge-reload": "hot_reload"}, "simulcast": {"$ref": "#/$defs/simulcast", "x-liveforge-reload": "restart_required", "x-liveforge-support": "deferred"} - } + }, + "allOf": [{"if": {"properties": {"gop_cache": {"const": true}, "gop_cache_num": {"minimum": 1}}, "required": ["gop_cache", "gop_cache_num"]}, "then": {"anyOf": [{"required": ["gop_cache_max_frames"], "properties": {"gop_cache_max_frames": {"minimum": 1}}}, {"required": ["gop_cache_max_bytes"], "properties": {"gop_cache_max_bytes": {"minimum": 1}}}]}}] }, "auth_rule": { "type": "object", "additionalProperties": false, @@ -236,8 +237,8 @@ "record": { "type": "object", "additionalProperties": false, "properties": { - "enabled": {"type": "boolean", "x-liveforge-reload": "restart_required"}, "stream_pattern": {"type": "string", "x-liveforge-reload": "hot_reload"}, "format": {"type": "string", "enum": ["flv", "fmp4", "mp4", "ts", "hls"], "default": "fmp4", "description": "Recording container. The default is fMP4 and the default extension is .mp4; fMP4/MP4 are the browser-friendly unified recording formats.", "x-liveforge-reload": "hot_reload"}, "path": {"type": "string", "description": "Recording path template. Supported placeholders are {stream_key}, {date}, {time}, and {ext}.", "x-liveforge-reload": "restart_required"}, - "segment": {"type": "object", "additionalProperties": false, "x-liveforge-reload": "hot_reload", "properties": {"mode": {"type": "string", "description": "Operator label; segmentation is activated by positive duration and/or max_size values."}, "duration": {"$ref": "#/$defs/duration"}, "max_size": {"type": "string"}}}, + "enabled": {"type": "boolean", "x-liveforge-reload": "restart_required"}, "stream_pattern": {"type": "string", "x-liveforge-reload": "hot_reload"}, "format": {"type": "string", "enum": ["flv", "fmp4", "mp4", "ts", "hls"], "default": "fmp4", "description": "Recording container. The default is fMP4 and the default extension is .mp4; fMP4/MP4 are browser-friendly unified recording formats. hls is an alias for TS storage and uses a .ts extension.", "x-liveforge-reload": "hot_reload"}, "path": {"type": "string", "description": "Recording path template. Supported placeholders are {stream_key}, {date}, {time}, and {ext}.", "x-liveforge-reload": "restart_required"}, + "segment": {"type": "object", "additionalProperties": false, "x-liveforge-reload": "hot_reload", "properties": {"mode": {"type": "string", "description": "Operator label; segmentation is activated by positive duration and/or max_size values."}, "duration": {"$ref": "#/$defs/duration"}, "max_size": {"type": "string", "pattern": "^\\s*(?:[0-9]+(?:[bB]|[kK][bB]|[mM][bB]|[gG][bB])?)?\\s*$", "description": "Optional non-negative decimal byte count with suffix B, KB, MB, or GB. Empty or zero disables size rotation; fractional, negative, unknown-suffix, and overflow values are invalid."}}}, "on_file_complete": {"type": "object", "additionalProperties": false, "x-liveforge-reload": "hot_reload", "properties": {"url": {"type": "string"}}} } }, @@ -245,7 +246,7 @@ "type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean", "x-liveforge-reload": "restart_required"}, "listen": {"type": "string", "x-liveforge-reload": "restart_required"}, "stream_pattern": {"type": "string", "x-liveforge-reload": "hot_reload"}, "path": {"type": "string", "x-liveforge-reload": "restart_required"}, "window": {"$ref": "#/$defs/duration", "x-liveforge-reload": "hot_reload"}, "segment_duration": {"$ref": "#/$defs/duration", "x-liveforge-reload": "hot_reload"}, "cleanup_interval": {"$ref": "#/$defs/duration", "x-liveforge-reload": "hot_reload"}} }, - "metrics": {"type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean"}, "listen": {"type": "string"}, "path": {"type": "string"}}}, + "metrics": {"type": "object", "additionalProperties": false, "properties": {"enabled": {"type": "boolean"}, "listen": {"type": "string"}, "path": {"type": "string"}, "stream_detail": {"type": "boolean", "default": false, "description": "Opt in to per-stream Prometheus series carrying stream_key labels. Server-level aggregate metrics remain available when disabled."}, "stream_detail_limit": {"type": "integer", "minimum": 0, "default": 100, "description": "Without an allowlist, maximum distinct stream keys admitted for one Collector lifetime; admitted keys are not evicted or replaced after streams disappear. With an allowlist, maximum keys exported per scrape. Zero exports no per-stream series; negative values are invalid."}, "stream_detail_allowlist": {"type": "array", "items": {"type": "string", "minLength": 1}, "uniqueItems": true, "default": [], "description": "Optional authoritative exact stream-key universe, deduplicated and sorted when the Collector is created, then subject to stream_detail_limit per scrape. When empty, lifetime creation-order admission applies."}}}, "api": { "type": "object", "additionalProperties": false, "properties": { @@ -263,10 +264,10 @@ "description": "Bootstrap-controlled background source. Loads run on the manager worker; snapshot and typed-key reads are atomic and non-blocking. Config writes are serialized with loads and close, complete before Apply returns 202, and then schedule background parse/apply/publication. The Config API exposes the complete versioned JSON Schema, raw source document, redacted document, validation, and apply operations. Apply is writable for file, HTTP/HTTPS, Consul, and Redis sources when their backend accepts writes; other sources return a read-only conflict.", "properties": { "source": {"type": "string", "enum": ["file", "http", "https", "consul", "redis"]}, "poll_interval": {"$ref": "#/$defs/duration"}, "load_timeout": {"$ref": "#/$defs/duration"}, - "file": {"type": "object", "additionalProperties": false, "description": "Writable by atomic replacement of the configured path.", "properties": {"path": {"type": "string"}}}, + "file": {"type": "object", "additionalProperties": false, "description": "Writable by atomic replacement of the configured path. New targets use private mode 0600; an existing target's permission bits are preserved. Loading is capped by max_bytes.", "properties": {"path": {"type": "string"}, "max_bytes": {"type": "integer", "description": "Maximum file document bytes; any non-positive value selects the 4 MiB source default."}}}, "http": {"type": "object", "additionalProperties": false, "description": "HTTP configuration source. runtime.source=http requires an http:// URL and runtime.source=https requires an https:// URL. Scheme mismatches are rejected before dispatch, redirects are disabled, and ETag/Last-Modified validators advance only after a document is accepted. Apply uses authenticated PUT.", "properties": {"url": {"type": "string", "description": "Complete source URL whose scheme must exactly match the selected http or https runtime.source."}, "token": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}, "max_bytes": {"type": "integer", "description": "Maximum response bytes; any non-positive value selects the 4 MiB source default."}}}, - "consul": {"type": "object", "additionalProperties": false, "description": "Apply writes the complete document to prefix/config.yaml through the Consul KV API.", "properties": {"address": {"type": "string"}, "prefix": {"type": "string", "minLength": 1}, "token": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}, "max_bytes": {"type": "integer", "description": "Maximum response bytes; any non-positive value selects the 4 MiB source default."}}}, - "redis": {"type": "object", "additionalProperties": false, "description": "Apply writes config.yaml in hash or prefix mode and increments version_key when configured.", "properties": {"addr": {"type": "string"}, "username": {"type": "string"}, "password": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}, "db": {"type": "integer", "minimum": 0}, "prefix": {"type": "string"}, "hash": {"type": "string"}, "version_key": {"type": "string"}, "tls": {"type": "boolean"}}} + "consul": {"type": "object", "additionalProperties": false, "description": "Apply writes the complete document to prefix/config.yaml through the Consul KV API. Flattened dotted/slashed keys are canonicalized and any duplicate path or scalar/container prefix collision is rejected deterministically before materialization.", "properties": {"address": {"type": "string"}, "prefix": {"type": "string", "minLength": 1}, "token": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}, "max_bytes": {"type": "integer", "description": "Maximum response bytes; any non-positive value selects the 4 MiB source default."}}}, + "redis": {"type": "object", "additionalProperties": false, "description": "Apply writes config.yaml in hash or prefix mode and increments version_key when configured. The document write and optional version increment are queued in one MULTI/EXEC transaction; transaction and EXEC errors are returned to Apply. Flattened dotted/slashed keys reject duplicate paths and scalar/container prefix collisions deterministically. Hash field names prefer HSCAN NOVALUES and fall back to HKEYS only when that subcommand is unsupported; values are read in bounded HSTRLEN/HGET batches. Prefix keys use bounded SCAN batches and length preflight.", "properties": {"addr": {"type": "string"}, "username": {"type": "string"}, "password": {"type": "string", "writeOnly": true, "x-liveforge-secret": true}, "db": {"type": "integer", "minimum": 0}, "prefix": {"type": "string"}, "hash": {"type": "string"}, "version_key": {"type": "string"}, "tls": {"type": "boolean"}, "max_bytes": {"type": "integer", "description": "Maximum Redis configuration document/materialization bytes; any non-positive value selects the 4 MiB source default."}}} } } } diff --git a/module/api/console.html b/module/api/console.html index c19bb35d..2b87ea8d 100644 --- a/module/api/console.html +++ b/module/api/console.html @@ -1280,6 +1280,8 @@

Playback ` } type statsSnapshot struct { @@ -526,4 +700,3 @@ connect(); ` } - diff --git a/module/webrtc/whep_e2e_test.go b/module/webrtc/whep_e2e_test.go index 322887f5..c3d5dc73 100644 --- a/module/webrtc/whep_e2e_test.go +++ b/module/webrtc/whep_e2e_test.go @@ -837,6 +837,9 @@ func runJitterDiagnostic(t *testing.T, withAudio bool, streamPath string) { if withAudio { s.StreamHub().SetAudioCodecEnabled(true) } + streamConfig := s.Config().Stream + streamConfig.RingBufferSize = 4096 + s.StreamHub().UpdatePolicy(streamConfig, s.Config().Limits) stream, err := s.StreamHub().GetOrCreate(streamPath) if err != nil { @@ -1093,6 +1096,15 @@ func runJitterDiagnostic(t *testing.T, withAudio bool, streamPath string) { if len(frames) < 20 { t.Fatalf("too few video frames: %d", len(frames)) } + if minimum := totalFrames * 8 / 10; len(frames) < minimum { + t.Fatalf("video feed ended early: %d frames, want at least %d", len(frames), minimum) + } + if withAudio { + minimum := len(aacPayloads) * 8 / 10 + if len(aSamples) < minimum { + t.Fatalf("transformed audio ended early: %d packets, want at least %d", len(aSamples), minimum) + } + } // 1. Sequence number gap analysis (packet loss). seqGaps := 0 diff --git a/module/webrtc/whep_feed.go b/module/webrtc/whep_feed.go index 76017d6f..1f2df5f2 100644 --- a/module/webrtc/whep_feed.go +++ b/module/webrtc/whep_feed.go @@ -2,12 +2,15 @@ package webrtc import ( "context" + "errors" "log/slog" "sync" + "sync/atomic" "time" "github.com/im-pingo/liveforge/core" "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/im-pingo/liveforge/pkg/codec/h265" pkgrtp "github.com/im-pingo/liveforge/pkg/rtp" "github.com/im-pingo/liveforge/pkg/util" "github.com/pion/interceptor/pkg/cc" @@ -15,6 +18,455 @@ import ( "github.com/pion/webrtc/v4/pkg/media" ) +// WHEPFeedState describes the part of the playback startup lifecycle that is +// useful to a browser and to an operator diagnosing a stalled session. +type WHEPFeedState string + +const ( + WHEPFeedWaitingKeyframe WHEPFeedState = "waiting_keyframe" + WHEPFeedPlaying WHEPFeedState = "playing" + WHEPFeedNoMediaInput WHEPFeedState = "no_media_input" + WHEPFeedMediaStalled WHEPFeedState = "media_stalled" + WHEPFeedCodecMismatch WHEPFeedState = "codec_mismatch" + WHEPFeedSampleWriteFailed WHEPFeedState = "sample_write_failed" + WHEPFeedTargetAudioFailed WHEPFeedState = "target_audio_failed" + WHEPFeedGenerationEnded WHEPFeedState = "generation_ended" + WHEPFeedClosed WHEPFeedState = "closed" +) + +const whepNoMediaInputTimeout = 8 * time.Second + +// WHEPFeedStatus is a point-in-time diagnostic snapshot. Counters are +// intentionally session-local so one stalled subscriber can be diagnosed +// without turning stream metrics into high-cardinality labels. +type WHEPFeedStatus struct { + Generation uint64 `json:"generation"` + Cursor int64 `json:"cursor"` + Mode string `json:"mode"` + State WHEPFeedState `json:"state"` + FirstMediaAt time.Time `json:"first_media_at,omitempty"` + FirstMediaWaitMS int64 `json:"first_media_wait_ms"` + LastVideoAt time.Time `json:"last_video_at,omitempty"` + LastAudioAt time.Time `json:"last_audio_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` + ExpectedVideo bool `json:"expected_video"` + ExpectedAudio bool `json:"expected_audio"` + VideoFrames uint64 `json:"video_frames"` + AudioFrames uint64 `json:"audio_frames"` + DroppedVideo uint64 `json:"dropped_video"` + DroppedAudio uint64 `json:"dropped_audio"` + SourceOverwrites uint64 `json:"source_overwrites"` + RTPPacketsSent uint64 `json:"rtp_packets_sent"` + RTPBytesSent uint64 `json:"rtp_bytes_sent"` + RTCPPacketsReceived uint64 `json:"rtcp_packets_received"` + LastError string `json:"last_error,omitempty"` +} + +type whepFeedPhase struct { + state WHEPFeedState + changedAt int64 +} + +type whepFeedStatus struct { + generation uint64 + cursor int64 + mode string + + phase atomic.Pointer[whepFeedPhase] + updateMu sync.Mutex + terminal atomic.Bool + createdAt atomic.Int64 + firstMediaAt atomic.Int64 + lastVideoAt atomic.Int64 + lastAudioAt atomic.Int64 + updatedAt atomic.Int64 + expectedVideo atomic.Bool + expectedAudio atomic.Bool + videoFrames atomic.Uint64 + audioFrames atomic.Uint64 + droppedVideo atomic.Uint64 + droppedAudio atomic.Uint64 + sourceOverwrites atomic.Uint64 + rtpPackets atomic.Uint64 + rtpBytes atomic.Uint64 + rtcpPackets atomic.Uint64 + + errorMu sync.RWMutex + lastError string +} + +func newWHEPFeedStatus(generation uint64, cursor int64, mode string) *whepFeedStatus { + now := time.Now().UTC() + status := &whepFeedStatus{generation: generation, cursor: cursor, mode: mode} + status.phase.Store(&whepFeedPhase{state: WHEPFeedWaitingKeyframe, changedAt: now.UnixNano()}) + status.createdAt.Store(now.UnixNano()) + status.updatedAt.Store(now.UnixNano()) + return status +} + +func (s *whepFeedStatus) Snapshot() WHEPFeedStatus { + phase := s.phase.Load() + createdAt := s.createdAt.Load() + firstMediaAt := s.firstMediaAt.Load() + firstMediaWaitMS := int64(0) + if createdAt > 0 && firstMediaAt > createdAt { + firstMediaWaitMS = (firstMediaAt - createdAt) / int64(time.Millisecond) + } + s.errorMu.RLock() + lastError := s.lastError + s.errorMu.RUnlock() + return WHEPFeedStatus{ + Generation: s.generation, + Cursor: s.cursor, + Mode: s.mode, + State: phase.state, + FirstMediaAt: whepTimeFromUnixNano(firstMediaAt), + FirstMediaWaitMS: firstMediaWaitMS, + LastVideoAt: whepTimeFromUnixNano(s.lastVideoAt.Load()), + LastAudioAt: whepTimeFromUnixNano(s.lastAudioAt.Load()), + UpdatedAt: whepTimeFromUnixNano(s.updatedAt.Load()), + ExpectedVideo: s.expectedVideo.Load(), + ExpectedAudio: s.expectedAudio.Load(), + VideoFrames: s.videoFrames.Load(), + AudioFrames: s.audioFrames.Load(), + DroppedVideo: s.droppedVideo.Load(), + DroppedAudio: s.droppedAudio.Load(), + SourceOverwrites: s.sourceOverwrites.Load(), + RTPPacketsSent: s.rtpPackets.Load(), + RTPBytesSent: s.rtpBytes.Load(), + RTCPPacketsReceived: s.rtcpPackets.Load(), + LastError: lastError, + } +} + +func (s *whepFeedStatus) SetState(state WHEPFeedState) { + now := time.Now().UTC() + if isWHEPFeedTerminal(state) { + s.setTerminalAt(state, nil, now) + return + } + if !s.beginUpdate() { + return + } + defer s.endUpdate() + s.setNonterminalStateAt(state, now) + s.updatedAt.Store(now.UnixNano()) +} + +func (s *whepFeedStatus) SetError(state WHEPFeedState, err error) { + s.setTerminalAt(state, err, time.Now().UTC()) +} + +func (s *whepFeedStatus) RecordVideo(sent bool) { + s.recordVideoAt(sent, time.Now().UTC()) +} + +func (s *whepFeedStatus) recordVideoAt(sent bool, now time.Time) { + if !s.beginUpdate() { + return + } + defer s.endUpdate() + nowUnix := now.UnixNano() + if sent { + s.videoFrames.Add(1) + s.lastVideoAt.Store(nowUnix) + s.firstMediaAt.CompareAndSwap(0, nowUnix) + s.updatePlayingAt(now) + } else { + s.droppedVideo.Add(1) + if phase := s.phase.Load(); phase.state == WHEPFeedNoMediaInput && s.lastVideoAt.Load() == 0 { + s.setNonterminalStateAt(WHEPFeedWaitingKeyframe, now) + } + } + s.updatedAt.Store(nowUnix) +} + +func (s *whepFeedStatus) RecordAudio(sent bool) { + s.recordAudioAt(sent, time.Now().UTC()) +} + +func (s *whepFeedStatus) beginVideoRecovery() { + if !s.beginUpdate() { + return + } + defer s.endUpdate() + now := time.Now().UTC() + s.lastVideoAt.Store(0) + s.setNonterminalStateAt(WHEPFeedWaitingKeyframe, now) + s.updatedAt.Store(now.UnixNano()) +} + +func (s *whepFeedStatus) recordSourceOverwrite(dropped uint64) { + if dropped == 0 || !s.beginUpdate() { + return + } + defer s.endUpdate() + s.sourceOverwrites.Add(dropped) + s.updatedAt.Store(time.Now().UTC().UnixNano()) +} + +func (s *whepFeedStatus) recordDroppedAudio(dropped uint64) { + if dropped == 0 || !s.beginUpdate() { + return + } + defer s.endUpdate() + s.droppedAudio.Add(dropped) + s.updatedAt.Store(time.Now().UTC().UnixNano()) +} + +func (s *whepFeedStatus) recordAudioAt(sent bool, now time.Time) { + if !s.beginUpdate() { + return + } + defer s.endUpdate() + nowUnix := now.UnixNano() + if sent { + s.audioFrames.Add(1) + s.lastAudioAt.Store(nowUnix) + s.firstMediaAt.CompareAndSwap(0, nowUnix) + s.updatePlayingAt(now) + } else { + s.droppedAudio.Add(1) + } + s.updatedAt.Store(nowUnix) +} + +func (s *whepFeedStatus) MarkNoMediaInput() { + now := time.Now().UTC() + if s.videoFrames.Load()+s.audioFrames.Load()+s.droppedVideo.Load()+s.droppedAudio.Load() == 0 { + if !s.beginUpdate() { + return + } + defer s.endUpdate() + s.setNonterminalStateAt(WHEPFeedNoMediaInput, now) + s.updatedAt.Store(now.UnixNano()) + return + } + s.checkInactivityAt(now) +} + +func (s *whepFeedStatus) checkInactivityAt(now time.Time) { + if !s.beginUpdate() { + return + } + defer s.endUpdate() + if s.expectedVideo.Load() && s.lastVideoAt.Load() == 0 && s.droppedVideo.Load() > 0 { + s.setNonterminalStateAt(WHEPFeedWaitingKeyframe, now) + return + } + if s.videoFrames.Load()+s.audioFrames.Load() == 0 { + if now.UnixNano()-s.createdAt.Load() >= whepNoMediaInputTimeout.Nanoseconds() { + s.setNonterminalStateAt(WHEPFeedNoMediaInput, now) + s.updatedAt.Store(now.UnixNano()) + } + return + } + if s.missingExpectedMediaWithinStartupGraceAt(now) { + return + } + if s.mediaReadyAt(now) { + s.setNonterminalStateAt(WHEPFeedPlaying, now) + return + } + s.setNonterminalStateAt(WHEPFeedMediaStalled, now) + s.updatedAt.Store(now.UnixNano()) +} + +func (s *whepFeedStatus) missingExpectedMediaWithinStartupGraceAt(now time.Time) bool { + firstMediaAt := s.firstMediaAt.Load() + if firstMediaAt == 0 || now.UnixNano()-firstMediaAt >= whepNoMediaInputTimeout.Nanoseconds() { + return false + } + return s.expectedVideo.Load() && s.lastVideoAt.Load() == 0 || + s.expectedAudio.Load() && s.lastAudioAt.Load() == 0 +} + +func (s *whepFeedStatus) SetTransportStats(rtpPackets, rtpBytes, rtcpPackets uint64) { + if !s.beginUpdate() { + return + } + defer s.endUpdate() + changed := storeAtomicMaximum(&s.rtpPackets, rtpPackets) + changed = storeAtomicMaximum(&s.rtpBytes, rtpBytes) || changed + changed = storeAtomicMaximum(&s.rtcpPackets, rtcpPackets) || changed + if changed { + s.updatedAt.Store(time.Now().UTC().UnixNano()) + } +} + +func (s *whepFeedStatus) setFinalTransportStats(rtpPackets, rtpBytes, rtcpPackets uint64) { + storeAtomicMaximum(&s.rtpPackets, rtpPackets) + storeAtomicMaximum(&s.rtpBytes, rtpBytes) + storeAtomicMaximum(&s.rtcpPackets, rtcpPackets) +} + +func (s *whepFeedStatus) setExpectedMedia(video, audio bool) { + s.expectedVideo.Store(video) + s.expectedAudio.Store(audio) +} + +func (s *whepFeedStatus) watchInactivity(stop, generationDone <-chan struct{}, timeout time.Duration) { + if timeout <= 0 { + timeout = whepNoMediaInputTimeout + } + interval := timeout / 4 + if interval <= 0 { + interval = timeout + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case now := <-ticker.C: + s.checkInactivityAt(now.UTC()) + case <-stop: + return + case <-generationDone: + return + } + } +} + +func (s *whepFeedStatus) updatePlayingAt(now time.Time) { + if s.mediaReadyAt(now) { + s.setNonterminalStateAt(WHEPFeedPlaying, now) + } +} + +func (s *whepFeedStatus) mediaReadyAt(now time.Time) bool { + expectedVideo := s.expectedVideo.Load() + expectedAudio := s.expectedAudio.Load() + if !expectedVideo && !expectedAudio { + return s.videoFrames.Load()+s.audioFrames.Load() > 0 + } + deadline := now.UnixNano() - whepNoMediaInputTimeout.Nanoseconds() + if expectedVideo { + last := s.lastVideoAt.Load() + if last == 0 || last <= deadline { + return false + } + } + if expectedAudio { + last := s.lastAudioAt.Load() + if last == 0 || last <= deadline { + return false + } + } + return true +} + +func (s *whepFeedStatus) setNonterminalStateAt(state WHEPFeedState, now time.Time) { + nowUnix := now.UnixNano() + for { + current := s.phase.Load() + if isWHEPFeedTerminal(current.state) || nowUnix < current.changedAt || current.state == state { + return + } + next := &whepFeedPhase{state: state, changedAt: nowUnix} + if s.phase.CompareAndSwap(current, next) { + s.logStateTransition(current.state, state, "") + return + } + } +} + +func (s *whepFeedStatus) setTerminalAt(state WHEPFeedState, err error, now time.Time) { + if !s.closeUpdates() { + return + } + defer s.updateMu.Unlock() + previous := s.phase.Load().state + lastError := "" + if err != nil { + lastError = boundedWHEPError(err) + s.errorMu.Lock() + s.lastError = lastError + s.errorMu.Unlock() + } + nowUnix := now.UnixNano() + s.phase.Store(&whepFeedPhase{state: state, changedAt: nowUnix}) + s.updatedAt.Store(nowUnix) + s.logStateTransition(previous, state, lastError) +} + +func (s *whepFeedStatus) logStateTransition(previous, state WHEPFeedState, lastError string) { + attributes := []any{ + "module", "webrtc", + "generation", s.generation, + "cursor", s.cursor, + "mode", s.mode, + "previous", previous, + "state", state, + } + if lastError != "" { + attributes = append(attributes, "error", lastError) + } + slog.Info("WHEP feed state changed", attributes...) +} + +func (s *whepFeedStatus) beginUpdate() bool { + s.updateMu.Lock() + if s.terminal.Load() { + s.updateMu.Unlock() + return false + } + return true +} + +func (s *whepFeedStatus) endUpdate() { + s.updateMu.Unlock() +} + +func (s *whepFeedStatus) closeUpdates() bool { + s.updateMu.Lock() + if s.terminal.Load() { + s.updateMu.Unlock() + return false + } + s.terminal.Store(true) + return true +} + +func storeAtomicMaximum(destination *atomic.Uint64, value uint64) bool { + for { + current := destination.Load() + if value <= current { + return false + } + if destination.CompareAndSwap(current, value) { + return true + } + } +} + +func whepTimeFromUnixNano(value int64) time.Time { + if value == 0 { + return time.Time{} + } + return time.Unix(0, value).UTC() +} + +func isWHEPFeedTerminal(state WHEPFeedState) bool { + switch state { + case WHEPFeedCodecMismatch, WHEPFeedSampleWriteFailed, + WHEPFeedTargetAudioFailed, WHEPFeedGenerationEnded, WHEPFeedClosed: + return true + default: + return false + } +} + +func boundedWHEPError(err error) string { + if err == nil { + return "" + } + message := err.Error() + if len(message) > 512 { + return message[:512] + } + return message +} + // whepFeedLoop reads AVFrames from the stream's RingBuffer and writes them // to the WebRTC tracks via TrackSender. It waits for the peer connection to // be established before sending any data. @@ -25,7 +477,45 @@ import ( // mode controls startup behavior: // - "realtime": skip GOP cache, read live frames, discard until first keyframe. // - "live": send GOP cache (paced at 10x speed), then live frames. -func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video, audio *TrackSender, done <-chan struct{}, connected <-chan struct{}, mode string, targetAudioCodec avframe.CodecType, bwe cc.BandwidthEstimator) { +func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video, audio *TrackSender, done <-chan struct{}, connected <-chan struct{}, mode string, targetAudioCodec avframe.CodecType, bwe cc.BandwidthEstimator, status *whepFeedStatus, sendGates ...*whepSendGate) { + var sendGate *whepSendGate + if len(sendGates) > 0 { + sendGate = sendGates[0] + } + if sendGate == nil { + sendGate = newWHEPSendGate() + } + if status == nil { + status = newWHEPFeedStatus(startup.Generation, startup.LiveCursor, mode) + } + defer func() { + if !isWHEPFeedTerminal(status.Snapshot().State) { + select { + case <-startup.GenerationDone: + status.SetState(WHEPFeedGenerationEnded) + default: + status.SetState(WHEPFeedClosed) + } + } + }() + gateStop := make(chan struct{}) + gateDone := make(chan struct{}) + go func() { + defer close(gateDone) + select { + case <-startup.GenerationDone: + case <-done: + case <-gateStop: + return + } + sendGate.close() + }() + defer func() { + sendGate.close() + close(gateStop) + <-gateDone + }() + // Wait for ICE+DTLS to complete before sending media. select { case <-connected: @@ -35,6 +525,16 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video case <-startup.GenerationDone: return } + watchdogStop := make(chan struct{}) + watchdogDone := make(chan struct{}) + go func() { + defer close(watchdogDone) + status.watchInactivity(watchdogStop, startup.GenerationDone, whepNoMediaInputTimeout) + }() + defer func() { + close(watchdogStop) + <-watchdogDone + }() if bwe != nil { bwe.OnTargetBitrateChange(func(bitrate int) { @@ -60,8 +560,29 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video needsAnnexB := videoCodec == avframe.CodecH264 || videoCodec == avframe.CodecH265 if needsAnnexB { if sh := startup.VideoSequenceHeader; sh != nil { + if !whepParameterSetsReady(videoCodec, sh.Payload) { + status.SetError(WHEPFeedCodecMismatch, errors.New("invalid video sequence header: required parameter sets are missing")) + return + } + paramSetBuf = pkgrtp.VideoToAnnexB(videoCodec, sh.Payload, true) + } + } + refreshVideoParameterSets := func() bool { + if !needsAnnexB { + return true + } + current := stream.StartupSnapshot() + if current.StreamInstanceID != startup.StreamInstanceID || current.Generation != startup.Generation { + return false + } + if sh := current.VideoSequenceHeader; sh != nil { + if !whepParameterSetsReady(videoCodec, sh.Payload) { + status.SetError(WHEPFeedCodecMismatch, errors.New("invalid video sequence header: required parameter sets are missing")) + return false + } paramSetBuf = pkgrtp.VideoToAnnexB(videoCodec, sh.Payload, true) } + return true } // B-frame drop: Chrome's WebRTC H.264 decoder does not perform B-frame @@ -93,6 +614,10 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video // SequenceHeader: cache parameter sets, do not send as a sample. if frame.FrameType == avframe.FrameTypeSequenceHeader { if needsAnnexB { + if !whepParameterSetsReady(videoCodec, frame.Payload) { + status.SetError(WHEPFeedCodecMismatch, errors.New("invalid video sequence header: required parameter sets are missing")) + return false + } paramSetBuf = pkgrtp.VideoToAnnexB(videoCodec, frame.Payload, true) } return false @@ -103,6 +628,7 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video // gets a normal ~40ms duration instead of a multi-second gap. if video.NeedsKeyframe() { if frame.FrameType != avframe.FrameTypeKeyframe { + status.RecordVideo(false) if frame.DTS > 0 { lastVideoDTS = frame.DTS } @@ -117,6 +643,7 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video // reference frame. H.265 B-frames may themselves be references and // must stay in the decode-order stream. if shouldDropWHEPVideoFrame(videoCodec, frame, maxSentVideoPTS) { + status.RecordVideo(false) if frame.DTS > 0 { lastVideoDTS = frame.DTS } @@ -128,6 +655,7 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video // H264/H265: convert AVCC/HVCC length-prefixed NALs to Annex-B. payload = pkgrtp.VideoToAnnexB(videoCodec, frame.Payload, false) if len(payload) == 0 { + status.SetError(WHEPFeedCodecMismatch, errors.New("empty video access unit")) return false } // Prepend parameter sets to keyframes. @@ -141,6 +669,7 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video // VP8/VP9/AV1: raw frame data, no conversion needed. payload = frame.Payload if len(payload) == 0 { + status.RecordVideo(false) return false } } @@ -168,12 +697,15 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video maxSentVideoPTS = frame.PTS } - if err := video.WriteSample(media.Sample{ + if err := writeWHEPSample(sendGate, video, media.Sample{ Data: payload, Duration: duration, }); err != nil { + status.SetError(WHEPFeedSampleWriteFailed, err) + slog.Warn("whep: video sample write failed", "module", "webrtc", "generation", startup.Generation, "error", err) return false } + status.RecordVideo(true) return true } @@ -189,13 +721,14 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video // writeAudioSample writes only frames matching the negotiated track codec. // This prevents source AAC from being packetized on an Opus track if a // transcoder fails or a source-codec cache reaches this path. - writeAudioSample := func(frame *avframe.AVFrame) { + writeAudioSample := func(frame *avframe.AVFrame) bool { if audio == nil { - return + return true } if !whepAudioFrameAllowed(frame, targetAudioCodec) { - return + status.RecordAudio(false) + return true } payload := frame.Payload @@ -213,12 +746,16 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video lastAudioDTS = frame.DTS } - if err := audio.WriteSample(media.Sample{ + if err := writeWHEPSample(sendGate, audio, media.Sample{ Data: payload, Duration: duration, }); err != nil { - return + status.SetError(WHEPFeedSampleWriteFailed, err) + slog.Warn("whep: audio sample write failed", "module", "webrtc", "generation", startup.Generation, "error", err) + return false } + status.RecordAudio(true) + return true } var gopCache []*avframe.AVFrame @@ -244,7 +781,12 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video cacheKeyframeSent = true } } else if frame.MediaType.IsAudio() { - writeAudioSample(frame) + if !writeAudioSample(frame) { + return + } + } + if isWHEPFeedTerminal(status.Snapshot().State) { + return } if frame.DTS > 0 && prevDTS > 0 { dtMs := frame.DTS - prevDTS @@ -295,19 +837,74 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video // In realtime mode, skip all frames until the first video keyframe // arrives, then start sending from that keyframe onward. gotKeyframe := whepInitialMediaReady(mode, cacheKeyframeSent, video != nil) + readers.startWaiters(done, startup.GenerationDone) for { - if !readers.drainTargetAudio(stream, startup.Generation, gotKeyframe, targetAudioCodec, writeAudioSample) { + select { + case <-done: + return + case <-startup.GenerationDone: + return + default: + } + if readers.activeTargetAudioEOF(done, startup.GenerationDone) { + status.SetError(WHEPFeedTargetAudioFailed, errors.New("target audio reader closed during active WHEP feed")) + return + } + if !readers.drainTargetAudio(stream, startup.Generation, gotKeyframe, targetAudioCodec, writeAudioSample, status) { + if readers.activeTargetAudioEOF(done, startup.GenerationDone) { + status.SetError(WHEPFeedTargetAudioFailed, errors.New("target audio reader closed during active WHEP feed")) + } return } - frame, ok := readers.source.TryRead() - if ok { + readEvent, sourceReady := readers.tryReadSource() + if sourceReady { + read := readEvent.result if !stream.IsPublisherGeneration(startup.Generation) { return } + if read.Overwritten > 0 { + overwritten := read.Overwritten + if !stream.IsPublisherGeneration(startup.Generation) { + return + } + action := "continue_audio" + // Source overwrite resets direct audio pacing as well as video + // pacing. Transcoded audio has an independent target reader and + // must keep its own clock intact. + if !needsTranscode { + lastAudioDTS = 0 + } + status.recordSourceOverwrite(uint64(overwritten)) + if video != nil { + action = "wait_keyframe" + video.RequestKeyframe() + status.beginVideoRecovery() + lastVideoDTS = 0 + lastSentVideoDTS = -1 + maxSentVideoPTS = 0 + paceBaseWall = time.Time{} + paceBaseDTS = 0 + if !refreshVideoParameterSets() { + return + } + } else { + status.recordDroppedAudio(uint64(overwritten)) + } + slog.Warn("whep: ring overwritten", + "protocol", "whep", + "reader", "source", + "overwritten", read.Overwritten, + "action", action, + ) + continue + } + frame := read.Value if frame.MediaType.IsAudio() { if !needsTranscode && gotKeyframe { - writeAudioSample(frame) + if !writeAudioSample(frame) { + return + } } continue } @@ -319,11 +916,13 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video gotKeyframe = true slog.Info("whep: got first live keyframe", "module", "webrtc", "mode", mode) } else if !gotKeyframe { + status.RecordVideo(false) continue } // DTS-based pacing: sleep if we're sending video faster than real-time. - if frame.DTS > 0 { + paceVideo := video == nil || !video.NeedsKeyframe() || frame.FrameType == avframe.FrameTypeKeyframe + if paceVideo && frame.DTS > 0 { if paceBaseWall.IsZero() { paceBaseWall = time.Now() paceBaseDTS = frame.DTS @@ -353,10 +952,16 @@ func whepFeedLoop(stream *core.Stream, startup core.StreamStartupSnapshot, video } } writeVideoSample(frame) + if isWHEPFeedTerminal(status.Snapshot().State) { + return + } } continue } if !readers.wait(done, startup.GenerationDone) { + if readers.activeTargetAudioEOF(done, startup.GenerationDone) { + status.SetError(WHEPFeedTargetAudioFailed, errors.New("target audio reader closed during active WHEP feed")) + } return } } @@ -371,7 +976,7 @@ func whepLiveSnapshot(snapshot core.StreamStartupSnapshot, needsTranscode bool) return frames } - videoOnly := frames[:0] + videoOnly := make([]*avframe.AVFrame, 0, len(frames)) for _, frame := range frames { if frame.MediaType.IsVideo() { videoOnly = append(videoOnly, frame) @@ -381,16 +986,86 @@ func whepLiveSnapshot(snapshot core.StreamStartupSnapshot, needsTranscode bool) } type whepFeedReaders struct { - source *util.RingReader[*avframe.AVFrame] - targetAudio *util.RingReader[*avframe.AVFrame] - release func() - waitOnce sync.Once - waitCancel context.CancelFunc - sourceWake chan struct{} - audioWake chan struct{} - sourceClosed chan struct{} + source *util.RingReader[*avframe.AVFrame] + targetAudio *util.RingReader[*avframe.AVFrame] + release func() + waitOnce sync.Once + closeOnce sync.Once + lifecycleMu sync.Mutex + closed bool + waitContext context.Context + waitCancel func() + done <-chan struct{} + generationDone <-chan struct{} + waitGroup sync.WaitGroup + sourceEvents chan whepReaderEvent + audioEvents chan whepReaderEvent + sourceReady chan struct{} + audioReady chan struct{} + sourcePermit chan struct{} + audioPermit chan struct{} + sourceTerminalNotify chan struct{} + audioTerminalNotify chan struct{} + pendingSource *whepReaderEvent + pendingAudio *whepReaderEvent + sourceTerminal atomic.Uint32 + audioTerminal atomic.Uint32 +} + +type whepReaderKind uint8 + +const ( + whepReaderSource whepReaderKind = iota + 1 + whepReaderTargetAudio +) + +func (k whepReaderKind) String() string { + switch k { + case whepReaderSource: + return "source" + case whepReaderTargetAudio: + return "target_audio" + default: + return "unknown" + } +} + +type whepReaderTerminalCause uint8 + +const ( + whepReaderTerminalNone whepReaderTerminalCause = iota + whepReaderTerminalEOF + whepReaderTerminalCanceled + whepReaderTerminalGenerationEnded +) + +func (c whepReaderTerminalCause) String() string { + switch c { + case whepReaderTerminalNone: + return "none" + case whepReaderTerminalEOF: + return "eof" + case whepReaderTerminalCanceled: + return "canceled" + case whepReaderTerminalGenerationEnded: + return "generation_ended" + default: + return "unknown" + } +} + +type whepReaderEvent struct { + reader whepReaderKind + result util.RingReadResult[*avframe.AVFrame] + terminal whepReaderTerminalCause + ack func() } +var ( + errWHEPReaderCanceled = errors.New("whep reader canceled") + errWHEPReaderGenerationEnded = errors.New("whep reader generation ended") +) + func newWHEPFeedReaders(stream *core.Stream, snapshot core.StreamStartupSnapshot, needsTranscode bool, targetAudioCodec avframe.CodecType) *whepFeedReaders { readers := &whepFeedReaders{source: stream.RingBuffer().NewReaderAt(snapshot.LiveCursor)} if needsTranscode { @@ -408,86 +1083,446 @@ func newWHEPFeedReaders(stream *core.Stream, snapshot core.StreamStartupSnapshot } func (r *whepFeedReaders) Close() { - if r.waitCancel != nil { - r.waitCancel() - } - if r.release != nil { - r.release() - } + r.closeOnce.Do(func() { + r.lifecycleMu.Lock() + r.closed = true + if r.waitCancel != nil { + r.waitCancel() + } + if r.source != nil { + r.source.Close() + } + if r.targetAudio != nil { + r.targetAudio.Close() + } + r.lifecycleMu.Unlock() + r.waitGroup.Wait() + if r.release != nil { + r.release() + } + }) } -func (r *whepFeedReaders) drainTargetAudio(stream *core.Stream, generation uint64, ready bool, targetCodec avframe.CodecType, writeAudio func(*avframe.AVFrame)) bool { +func (r *whepFeedReaders) drainTargetAudio(stream *core.Stream, generation uint64, ready bool, targetCodec avframe.CodecType, writeAudio func(*avframe.AVFrame) bool, status *whepFeedStatus) bool { if r.targetAudio == nil { return stream.IsPublisherGeneration(generation) } + r.ensureWaiters() for { - frame, ok := r.targetAudio.TryRead() - if !ok { + readEvent, available := r.tryReadTargetAudio() + if !available { + if r.targetAudioTerminalCause() == whepReaderTerminalEOF { + return false + } return stream.IsPublisherGeneration(generation) } + read := readEvent.result if !stream.IsPublisherGeneration(generation) { return false } + if read.Overwritten > 0 { + overwritten := read.Overwritten + if status != nil { + status.recordDroppedAudio(uint64(overwritten)) + } + slog.Warn("whep: ring overwritten", + "protocol", "whep", + "reader", "target_audio", + "overwritten", read.Overwritten, + "action", "continue_audio", + ) + continue + } + frame := read.Value if ready && frame.MediaType.IsAudio() && whepAudioFrameAllowed(frame, targetCodec) { - writeAudio(frame) + if !writeAudio(frame) { + return false + } } } } func (r *whepFeedReaders) startWaiters(done, generationDone <-chan struct{}) { + r.lifecycleMu.Lock() + defer r.lifecycleMu.Unlock() + if r.closed { + return + } r.waitOnce.Do(func() { - ctx, cancel := context.WithCancel(context.Background()) - r.waitCancel = cancel - r.sourceWake = make(chan struct{}, 1) - r.sourceClosed = make(chan struct{}) - go watchWHEPReader(ctx, r.source, r.sourceWake, r.sourceClosed) + ctx, cancel := context.WithCancelCause(context.Background()) + r.waitContext = ctx + r.done = done + r.generationDone = generationDone + r.waitCancel = func() { cancel(errWHEPReaderCanceled) } + if r.source != nil { + r.sourceEvents = make(chan whepReaderEvent) + r.sourceReady = make(chan struct{}) + r.sourcePermit = make(chan struct{}) + r.sourceTerminalNotify = make(chan struct{}, 1) + r.waitGroup.Add(1) + go func() { + defer r.waitGroup.Done() + pumpWHEPReaderGated(ctx, whepReaderSource, r.source, r.sourceEvents, &r.sourceTerminal, r.sourceReady, r.sourcePermit, r.sourceTerminalNotify, generationDone) + }() + } if r.targetAudio != nil { - r.audioWake = make(chan struct{}, 1) - go watchWHEPReader(ctx, r.targetAudio, r.audioWake, nil) + r.audioEvents = make(chan whepReaderEvent) + r.audioReady = make(chan struct{}) + r.audioPermit = make(chan struct{}) + r.audioTerminalNotify = make(chan struct{}, 1) + r.waitGroup.Add(1) + go func() { + defer r.waitGroup.Done() + pumpWHEPReaderGated(ctx, whepReaderTargetAudio, r.targetAudio, r.audioEvents, &r.audioTerminal, r.audioReady, r.audioPermit, r.audioTerminalNotify, generationDone) + }() + } + if done != nil || generationDone != nil { + r.waitGroup.Add(1) + go func() { + defer r.waitGroup.Done() + if generationDone != nil { + select { + case <-generationDone: + cancel(errWHEPReaderGenerationEnded) + return + default: + } + } + select { + case <-done: + select { + case <-generationDone: + cancel(errWHEPReaderGenerationEnded) + default: + cancel(errWHEPReaderCanceled) + } + case <-generationDone: + cancel(errWHEPReaderGenerationEnded) + case <-ctx.Done(): + } + }() } - go func() { - select { - case <-done: - cancel() - case <-generationDone: - cancel() - case <-ctx.Done(): - } - }() }) } -func watchWHEPReader(ctx context.Context, reader *util.RingReader[*avframe.AVFrame], wake chan<- struct{}, closed chan<- struct{}) { - defer func() { - if closed != nil { - close(closed) +func (r *whepFeedReaders) ensureWaiters() { + r.startWaiters(nil, nil) +} + +func pumpWHEPReader(ctx context.Context, readerKind whepReaderKind, reader *util.RingReader[*avframe.AVFrame], events chan<- whepReaderEvent, terminal *atomic.Uint32) { + pumpWHEPReaderGated(ctx, readerKind, reader, events, terminal, nil, nil, nil) +} + +func pumpWHEPReaderGated(ctx context.Context, readerKind whepReaderKind, reader *util.RingReader[*avframe.AVFrame], events chan<- whepReaderEvent, terminal *atomic.Uint32, ready chan<- struct{}, permit <-chan struct{}, terminalNotify chan<- struct{}, generationDone ...<-chan struct{}) { + defer close(events) + var generationEnd <-chan struct{} + if len(generationDone) > 0 { + generationEnd = generationDone[0] + } + publishTerminal := func(cause whepReaderTerminalCause) { + terminal.Store(uint32(cause)) + if terminalNotify != nil { + select { + case terminalNotify <- struct{}{}: + default: + } + } + select { + case events <- whepReaderEvent{reader: readerKind, terminal: cause}: + default: + } + } + terminalCause := func() whepReaderTerminalCause { + select { + case <-generationEnd: + return whepReaderTerminalGenerationEnded + default: + } + if errors.Is(context.Cause(ctx), errWHEPReaderGenerationEnded) { + return whepReaderTerminalGenerationEnded + } + if ctx.Err() != nil { + return whepReaderTerminalCanceled + } + return whepReaderTerminalEOF + } + for { + var read util.RingReadResult[*avframe.AVFrame] + if ready != nil { + if !reader.WaitContext(ctx) { + publishTerminal(terminalCause()) + return + } + select { + case ready <- struct{}{}: + case <-ctx.Done(): + return + } + select { + case <-permit: + case <-ctx.Done(): + return + } + read = reader.TryReadResult() + } else { + read = reader.ReadResultContext(ctx) + } + if !read.OK { + publishTerminal(terminalCause()) + return + } + if ctx.Err() != nil { + return + } + event := whepReaderEvent{reader: readerKind, result: read} + if read.Overwritten > 0 { + reader.AdvanceToLive() + } + acknowledged := make(chan struct{}) + var acknowledgeOnce sync.Once + event.ack = func() { + acknowledgeOnce.Do(func() { close(acknowledged) }) + } + select { + case events <- event: + case <-ctx.Done(): + return } - }() - for reader.WaitContext(ctx) { select { - case wake <- struct{}{}: + case <-acknowledged: case <-ctx.Done(): return } } } +func acknowledgeWHEPReaderEvent(event *whepReaderEvent) { + if event == nil || event.ack == nil { + return + } + event.ack() + event.ack = nil +} + +func (r *whepFeedReaders) activeTargetAudioEOF(done, generationDone <-chan struct{}) bool { + r.startWaiters(done, generationDone) + if r.audioEvents == nil { + return false + } + if r.pendingAudio == nil && r.targetAudioTerminalCause() == whepReaderTerminalNone { + if event, ok := r.tryReadTargetAudio(); ok { + // EOF probing must not consume media. The feed loop owns the + // merge order, so preserve the event for drainTargetAudio. + r.pendingAudio = &event + } + } + if r.targetAudioTerminalCause() != whepReaderTerminalEOF { + return false + } + return whepReaderStopCause(done, generationDone) == whepReaderTerminalNone +} + func (r *whepFeedReaders) wait(done, generationDone <-chan struct{}) bool { r.startWaiters(done, generationDone) + if r.pendingSource != nil || r.pendingAudio != nil { + return true + } select { case <-done: return false case <-generationDone: return false - case <-r.sourceClosed: + case <-r.sourceTerminalNotify: return false - case <-r.sourceWake: + case <-r.audioTerminalNotify: + return false + case <-r.sourceReady: + return r.receiveSourceAfterReady(done, generationDone) + case <-r.audioReady: + return r.receiveAudioAfterReady(done, generationDone) + case event, ok := <-r.sourceEvents: + event, ok = r.acceptSourceEvent(event, ok) + if !ok { + return false + } + r.pendingSource = &event + return true + case event, ok := <-r.audioEvents: + event, ok = r.acceptTargetAudioEvent(event, ok) + if !ok { + return false + } + r.pendingAudio = &event return true - case <-r.audioWake: + } +} + +func (r *whepFeedReaders) receiveSourceAfterReady(done, generationDone <-chan struct{}) bool { + if !r.grantRead(r.sourcePermit, done, generationDone) { + return false + } + event, ok := <-r.sourceEvents + event, ok = r.acceptSourceEvent(event, ok) + if !ok { + return false + } + r.pendingSource = &event + return true +} + +func (r *whepFeedReaders) receiveAudioAfterReady(done, generationDone <-chan struct{}) bool { + if !r.grantRead(r.audioPermit, done, generationDone) { + return false + } + event, ok := <-r.audioEvents + event, ok = r.acceptTargetAudioEvent(event, ok) + if !ok { + return false + } + r.pendingAudio = &event + return true +} + +func (r *whepFeedReaders) grantRead(permit chan<- struct{}, done, generationDone <-chan struct{}) bool { + r.lifecycleMu.Lock() + if done == nil { + done = r.done + } + if generationDone == nil { + generationDone = r.generationDone + } + var lifecycleDone <-chan struct{} + if r.waitContext != nil { + lifecycleDone = r.waitContext.Done() + } + r.lifecycleMu.Unlock() + select { + case permit <- struct{}{}: return true + case <-done: + return false + case <-generationDone: + return false + case <-lifecycleDone: + return false + } +} + +func (r *whepFeedReaders) tryReadSource() (whepReaderEvent, bool) { + r.ensureWaiters() + if r.pendingSource != nil { + event := *r.pendingSource + r.pendingSource = nil + acknowledgeWHEPReaderEvent(&event) + return event, true + } + select { + case <-r.sourceReady: + if !r.grantRead(r.sourcePermit, nil, nil) { + return whepReaderEvent{}, false + } + event, ok := <-r.sourceEvents + event, ok = r.acceptSourceEvent(event, ok) + if ok { + acknowledgeWHEPReaderEvent(&event) + } + return event, ok + default: + } + select { + case event, ok := <-r.sourceEvents: + event, ok = r.acceptSourceEvent(event, ok) + if ok { + acknowledgeWHEPReaderEvent(&event) + } + return event, ok + default: + return whepReaderEvent{}, false + } +} + +func (r *whepFeedReaders) tryReadTargetAudio() (whepReaderEvent, bool) { + r.ensureWaiters() + if r.pendingAudio != nil { + event := *r.pendingAudio + r.pendingAudio = nil + acknowledgeWHEPReaderEvent(&event) + return event, true + } + select { + case <-r.audioReady: + if !r.grantRead(r.audioPermit, nil, nil) { + return whepReaderEvent{}, false + } + event, ok := <-r.audioEvents + event, ok = r.acceptTargetAudioEvent(event, ok) + if ok { + acknowledgeWHEPReaderEvent(&event) + } + return event, ok + default: + } + select { + case event, ok := <-r.audioEvents: + event, ok = r.acceptTargetAudioEvent(event, ok) + if ok { + acknowledgeWHEPReaderEvent(&event) + } + return event, ok + default: + return whepReaderEvent{}, false + } +} + +func (r *whepFeedReaders) acceptSourceEvent(event whepReaderEvent, ok bool) (whepReaderEvent, bool) { + if !ok { + return whepReaderEvent{}, false + } + if event.terminal != whepReaderTerminalNone { + r.sourceTerminal.Store(uint32(event.terminal)) + return whepReaderEvent{}, false + } + return event, true +} + +func (r *whepFeedReaders) acceptTargetAudioEvent(event whepReaderEvent, ok bool) (whepReaderEvent, bool) { + if !ok { + return whepReaderEvent{}, false + } + if event.terminal != whepReaderTerminalNone { + r.audioTerminal.Store(uint32(event.terminal)) + return whepReaderEvent{}, false + } + return event, true +} + +func (r *whepFeedReaders) targetAudioTerminalCause() whepReaderTerminalCause { + switch r.audioTerminal.Load() { + case uint32(whepReaderTerminalEOF): + return whepReaderTerminalEOF + case uint32(whepReaderTerminalCanceled): + return whepReaderTerminalCanceled + case uint32(whepReaderTerminalGenerationEnded): + return whepReaderTerminalGenerationEnded + default: + return whepReaderTerminalNone } } +func whepReaderStopCause(done, generationDone <-chan struct{}) whepReaderTerminalCause { + select { + case <-done: + return whepReaderTerminalCanceled + default: + } + select { + case <-generationDone: + return whepReaderTerminalGenerationEnded + default: + } + return whepReaderTerminalNone +} + func whepInitialKeyframeReady(mode string, cacheKeyframeSent bool) bool { return mode == "live" && cacheKeyframeSent } @@ -509,6 +1544,18 @@ func whepAudioFrameAllowed(frame *avframe.AVFrame, targetCodec avframe.CodecType return frame.Codec == targetCodec && frame.FrameType != avframe.FrameTypeSequenceHeader && len(frame.Payload) > 0 } +func whepParameterSetsReady(codec avframe.CodecType, configuration []byte) bool { + annexB := pkgrtp.VideoToAnnexB(codec, configuration, true) + switch codec { + case avframe.CodecH264: + return len(pkgrtp.BuildAVCDecoderConfig(annexB)) > 0 + case avframe.CodecH265: + return len(h265.BuildHVCCDecoderConfig(annexB)) > 0 + default: + return true + } +} + // dtsPaceAction returns the action the feed loop should take based on // how far ahead or behind the DTS pacer is relative to wall clock. // diff --git a/module/webrtc/whep_feed_bench_test.go b/module/webrtc/whep_feed_bench_test.go new file mode 100644 index 00000000..e5938e3f --- /dev/null +++ b/module/webrtc/whep_feed_bench_test.go @@ -0,0 +1,28 @@ +package webrtc + +import "testing" + +func BenchmarkWHEPFeedStatusRecordMedia(b *testing.B) { + status := newWHEPFeedStatus(1, 1, "live") + status.setExpectedMedia(true, false) + status.RecordVideo(true) + baseline := status.Snapshot().VideoFrames + + b.ReportAllocs() + b.ResetTimer() + for range b.N { + status.RecordVideo(true) + } + b.StopTimer() + + snapshot := status.Snapshot() + if snapshot.VideoFrames != baseline+uint64(b.N) { // #nosec G115 -- benchmark iteration count is bounded by testing.B. + b.Fatalf("video frames = %d, want %d", snapshot.VideoFrames, baseline+uint64(b.N)) // #nosec G115 -- benchmark iteration count is bounded by testing.B. + } + if snapshot.State != WHEPFeedPlaying || !snapshot.ExpectedVideo || snapshot.ExpectedAudio { + b.Fatalf("feed state = %+v, want playing video-only status", snapshot) + } + if snapshot.LastVideoAt.IsZero() || snapshot.UpdatedAt.IsZero() { + b.Fatalf("media timestamps were not recorded: %+v", snapshot) + } +} diff --git a/module/webrtc/whep_feed_overwrite_audiocodec_test.go b/module/webrtc/whep_feed_overwrite_audiocodec_test.go new file mode 100644 index 00000000..0390ed24 --- /dev/null +++ b/module/webrtc/whep_feed_overwrite_audiocodec_test.go @@ -0,0 +1,362 @@ +//go:build audiocodec + +package webrtc + +import ( + "bytes" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/im-pingo/liveforge/config" + "github.com/im-pingo/liveforge/core" + "github.com/im-pingo/liveforge/pkg/audiocodec" + "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/im-pingo/liveforge/pkg/util" + "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media" +) + +const ( + whepOverwriteSourceAudio avframe.CodecType = 240 + whepOverwriteTargetAudio avframe.CodecType = 241 +) + +type whepOverwriteDecoder struct { + blockMarker byte + entered chan struct{} + release chan struct{} + decoded chan byte + blockOnce sync.Once +} + +func (*whepOverwriteDecoder) SetExtradata([]byte) {} + +func (d *whepOverwriteDecoder) Decode(payload []byte) (*audiocodec.PCMFrame, error) { + marker := byte(0) + if len(payload) > 0 { + marker = payload[0] + } + if d.entered != nil && marker == d.blockMarker { + d.blockOnce.Do(func() { close(d.entered) }) + <-d.release + } + if d.decoded != nil { + d.decoded <- marker + } + return &audiocodec.PCMFrame{Samples: []int16{int16(marker)}, SampleRate: 48000, Channels: 1}, nil +} + +func (*whepOverwriteDecoder) SampleRate() int { return 48000 } +func (*whepOverwriteDecoder) Channels() int { return 1 } +func (*whepOverwriteDecoder) Close() {} + +type whepOverwriteEncoder struct{} + +func (*whepOverwriteEncoder) Encode(pcm *audiocodec.PCMFrame) ([]byte, error) { + if pcm == nil || len(pcm.Samples) == 0 { + return nil, nil + } + return []byte{byte(pcm.Samples[0])}, nil // #nosec G115 -- the encoder intentionally emits a one-byte test marker. +} + +func (*whepOverwriteEncoder) SampleRate() int { return 48000 } +func (*whepOverwriteEncoder) Channels() int { return 1 } +func (*whepOverwriteEncoder) FrameSize() int { return 1 } +func (*whepOverwriteEncoder) Close() {} + +func newWHEPOverwriteTranscodeManager(stream *core.Stream, bufferSize int, decoder *whepOverwriteDecoder) { + registry := audiocodec.Global() + registry.RegisterDecoder(whepOverwriteSourceAudio, func() audiocodec.Decoder { + if decoder != nil { + return decoder + } + return &whepOverwriteDecoder{} + }) + registry.RegisterEncoder(whepOverwriteTargetAudio, func() audiocodec.Encoder { return &whepOverwriteEncoder{} }) + manager := core.NewTranscodeManager(stream, registry, bufferSize) + core.SetTranscodeManagerForTest(stream, manager) +} + +func whepOverwriteTranscodeAudio(marker byte, dts int64) *avframe.AVFrame { + return avframe.NewAVFrame( + avframe.MediaTypeAudio, whepOverwriteSourceAudio, avframe.FrameTypeInterframe, + dts, dts, []byte{marker}, + ) +} + +func waitWHEPDecodedMarker(t *testing.T, decoded <-chan byte, target byte) { + t.Helper() + select { + case got := <-decoded: + if got != target { + t.Fatalf("decoded source-audio marker = %x, want %x", got, target) + } + case <-time.After(2 * time.Second): + t.Fatalf("target-audio producer did not decode source marker %x", target) + } +} + +func drainWHEPCapture(capture *whepRTPCapture) { + for { + select { + case <-capture.packets: + default: + return + } + } +} + +func TestWHEPTranscodedMixedSourceOverwriteKeepsTargetAudioAndRecoversVideo(t *testing.T) { + events := installWHEPOverwriteLogObserver(t) + stream := core.NewStream("whep/transcoded-source-overwrite", config.StreamConfig{ + RingBufferSize: 3, GOPCache: true, GOPCacheNum: 1, + }, config.LimitsConfig{}, core.NewEventBus()) + if err := stream.SetPublisher(&authorizationTestPublisher{id: "source", info: &avframe.MediaInfo{ + VideoCodec: avframe.CodecH264, AudioCodec: whepOverwriteSourceAudio, SampleRate: 48000, Channels: 1, + }}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stream.Close() }) + decoder := &whepOverwriteDecoder{decoded: make(chan byte, 16)} + newWHEPOverwriteTranscodeManager(stream, 64, decoder) + stream.WriteFrame(whepOverwriteHeader(0xa0, 900)) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeKeyframe, 0xa1, 1000)) + stream.WriteFrame(whepOverwriteTranscodeAudio(0xa2, 1020)) + startup := stream.StartupSnapshot() + + videoSender, videoCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000}, 96) + audioSender, audioCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 1}, 111) + status := newWHEPFeedStatus(startup.Generation, startup.LiveCursor, "live") + status.setExpectedMedia(true, true) + done, feedDone := startWHEPOverwriteFeed(t, stream, startup, videoSender, audioSender, "live", whepOverwriteTargetAudio, status) + t.Cleanup(func() { stopWHEPOverwriteFeed(t, done, feedDone) }) + startupVideo := videoCapture.readSample(t) + if payload := whepSamplePayload(audioCapture.readSample(t)); !bytes.Contains(payload, []byte{0xa2}) { + t.Fatalf("startup target-audio RTP payload = %x, want marker a2", payload) + } + waitWHEPDecodedMarker(t, decoder.decoded, 0xa2) + + pause := videoCapture.armPause() + releasePause := sync.OnceFunc(func() { close(pause.release) }) + t.Cleanup(releasePause) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb0, 1040)) + waitWHEPOverwriteSignal(t, pause.entered, "source video did not enter deterministic WHEP write barrier") + stream.WriteFrame(whepOverwriteTranscodeAudio(0xb1, 1060)) + waitWHEPDecodedMarker(t, decoder.decoded, 0xb1) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb2, 1080)) + stream.WriteFrame(whepOverwriteTranscodeAudio(0xb3, 1100)) + waitWHEPDecodedMarker(t, decoder.decoded, 0xb3) + for _, frame := range []*avframe.AVFrame{ + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb4, 1120), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb5, 1130), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb6, 1140), + } { + stream.WriteFrame(frame) + } + releasePause() + _ = videoCapture.readSample(t) + for _, marker := range []byte{0xb1, 0xb3} { + if payload := whepSamplePayload(audioCapture.readSample(t)); !bytes.Contains(payload, []byte{marker}) { + t.Fatalf("target audio during source recovery = %x, want marker %x", payload, marker) + } + } + if got := waitWHEPOverwriteEvent(t, events); got.protocol != "whep" || got.reader != "source" || got.action != "wait_keyframe" || got.overwritten <= 0 { + t.Fatalf("transcoded source overwrite event = %+v", got) + } + if got := status.Snapshot().State; got != WHEPFeedWaitingKeyframe { + t.Fatalf("transcoded source recovery state = %q, want waiting_keyframe", got) + } + + stream.WriteFrame(whepOverwriteTranscodeAudio(0xc0, 1140)) + waitWHEPDecodedMarker(t, decoder.decoded, 0xc0) + if payload := whepSamplePayload(audioCapture.readSample(t)); !bytes.Contains(payload, []byte{0xc0}) { + t.Fatalf("post-overwrite target audio RTP payload = %x, want marker c0", payload) + } + if got := status.Snapshot().State; got != WHEPFeedWaitingKeyframe { + t.Fatalf("target audio alone cleared video recovery state: %q", got) + } + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeInterframe, 0xc1, 1160)) + stream.WriteFrame(whepOverwriteHeader(0xc2, 1180)) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeKeyframe, 0xc3, 6000)) + recovered := videoCapture.readSample(t) + payload := whepSamplePayload(recovered) + if !bytes.Contains(payload, whepOverwriteSPS(0xc2)) || !bytes.Contains(payload, []byte{0x65, 0xc3}) || bytes.Contains(payload, []byte{0x41, 0xc1}) { + t.Fatalf("transcoded first recovered video RTP payload = %x", payload) + } + if delta := recovered[0].header.Timestamp - startupVideo[0].header.Timestamp; delta != 7200 { + // One clean interframe was admitted before the overwrite barrier; recovery + // itself must add one normal 40ms step rather than the source DTS gap. + t.Fatalf("transcoded recovered RTP timestamp delta = %d, want two 40ms steps/7200", delta) + } +} + +func TestWHEPTargetAudioOverwriteKeepsDirectVideoContinuous(t *testing.T) { + events := installWHEPOverwriteLogObserver(t) + stream := core.NewStream("whep/target-audio-overwrite", config.StreamConfig{RingBufferSize: 8}, config.LimitsConfig{}, core.NewEventBus()) + if err := stream.SetPublisher(&authorizationTestPublisher{id: "source", info: &avframe.MediaInfo{ + VideoCodec: avframe.CodecVP8, AudioCodec: whepOverwriteSourceAudio, + }}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stream.Close() }) + snapshot := stream.StartupSnapshot() + sourceRing := util.NewRingBuffer[*avframe.AVFrame](8) + targetRing := util.NewRingBuffer[*avframe.AVFrame](2) + sourceReader := sourceRing.NewReaderAt(0) + targetReader := targetRing.NewReaderAt(0) + readers := &whepFeedReaders{targetAudio: targetReader} + t.Cleanup(readers.Close) + for _, frame := range []*avframe.AVFrame{ + avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecVP8, avframe.FrameTypeInterframe, 20, 20, []byte{0xb0}), + avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecVP8, avframe.FrameTypeInterframe, 40, 40, []byte{0xb1}), + } { + sourceRing.Write(frame) + } + for _, marker := range []byte{0xa0, 0xa1, 0xa2, 0xa3} { + targetRing.Write(avframe.NewAVFrame(avframe.MediaTypeAudio, whepOverwriteTargetAudio, avframe.FrameTypeInterframe, int64(marker), int64(marker), []byte{marker})) + } + readers.startWaiters(nil, nil) + if !readers.wait(nil, nil) { + t.Fatal("target-audio pump did not publish the full-ring overwrite") + } + + audioSender, audioCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 1}, 111) + writeAudio := func(frame *avframe.AVFrame) bool { + return audioSender.WriteSample(mediaSample(frame.Payload, 20*time.Millisecond)) == nil + } + if !readers.drainTargetAudio(stream, snapshot.Generation, true, whepOverwriteTargetAudio, writeAudio, nil) { + t.Fatal("target-audio overwrite stopped the active generation") + } + if got := waitWHEPOverwriteEvent(t, events); got != (whepOverwriteLogEvent{protocol: "whep", reader: "target_audio", action: "continue_audio", overwritten: 2}) { + t.Fatalf("target-audio overwrite event = %+v", got) + } + if got := sourceReader.ReadCursor(); got != 0 { + t.Fatalf("target-audio overwrite advanced source reader to %d, want 0", got) + } + audioCapture.assertEmpty(t) + targetRing.Write(avframe.NewAVFrame(avframe.MediaTypeAudio, whepOverwriteTargetAudio, avframe.FrameTypeInterframe, 120, 120, []byte{0xc0})) + if !readers.wait(nil, nil) { + t.Fatal("target-audio pump did not publish the post-overwrite frame") + } + if !readers.drainTargetAudio(stream, snapshot.Generation, true, whepOverwriteTargetAudio, writeAudio, nil) { + t.Fatal("target audio did not continue from live") + } + if payload := whepSamplePayload(audioCapture.readSample(t)); !bytes.Contains(payload, []byte{0xc0}) { + t.Fatalf("target-audio recovered RTP payload = %x, want c0", payload) + } + + videoSender, videoCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000}, 96) + for _, marker := range []byte{0xb0, 0xb1} { + result := sourceReader.TryReadResult() + if !result.OK || result.Overwritten != 0 { + t.Fatalf("clean source read after target overwrite = %+v", result) + } + if err := videoSender.WriteSample(mediaSample(result.Value.Payload, 40*time.Millisecond)); err != nil { + t.Fatal(err) + } + if payload := whepSamplePayload(videoCapture.readSample(t)); !bytes.Contains(payload, []byte{marker}) { + t.Fatalf("continuous video RTP payload = %x, want marker %x", payload, marker) + } + } +} + +func TestWHEPTranscodeProducerSourceOverwriteEOFIsTerminalAndReleasesOnce(t *testing.T) { + decoder := &whepOverwriteDecoder{blockMarker: 0xe0, entered: make(chan struct{}), release: make(chan struct{})} + stream := core.NewStream("whep/transcode-producer-overwrite", config.StreamConfig{ + RingBufferSize: 4, GOPCache: true, GOPCacheNum: 1, + }, config.LimitsConfig{}, core.NewEventBus()) + if err := stream.SetPublisher(&authorizationTestPublisher{id: "source", info: &avframe.MediaInfo{ + VideoCodec: avframe.CodecH264, AudioCodec: whepOverwriteSourceAudio, SampleRate: 48000, Channels: 1, + }}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stream.Close() }) + newWHEPOverwriteTranscodeManager(stream, 16, decoder) + stream.WriteFrame(whepOverwriteHeader(0xa0, 0)) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeKeyframe, 0xa1, 40)) + stream.WriteFrame(whepOverwriteTranscodeAudio(0xa2, 60)) + startup := stream.StartupSnapshot() + videoSender, videoCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000}, 96) + audioSender, audioCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 1}, 111) + status := newWHEPFeedStatus(startup.Generation, startup.LiveCursor, "live") + status.setExpectedMedia(true, true) + done, feedDone := startWHEPOverwriteFeed(t, stream, startup, videoSender, audioSender, "live", whepOverwriteTargetAudio, status) + _ = videoCapture.readSample(t) + _ = audioCapture.readSample(t) + + stream.WriteFrame(whepOverwriteTranscodeAudio(0xe0, 80)) + waitWHEPOverwriteSignal(t, decoder.entered, "target-audio producer did not enter decoder barrier") + pumpStop := make(chan struct{}) + pumpDone := make(chan struct{}) + pumped := make(chan struct{}, 8) + stopPump := sync.OnceFunc(func() { close(pumpStop) }) + t.Cleanup(func() { + stopPump() + <-pumpDone + }) + go func() { + defer close(pumpDone) + nextIndex := 0 + writeNext := func() { + dts := int64(100 + nextIndex*20) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeInterframe, byte(nextIndex), dts)) // #nosec G115 -- overwrite fixture index is bounded. + nextIndex++ + } + writeNext() + writeNext() + for { + select { + case <-pumpStop: + return + case packet := <-videoCapture.packets: + if packet.header.Marker { + writeNext() + select { + case pumped <- struct{}{}: + default: + } + } + } + } + }() + for range 6 { + waitWHEPOverwriteSignal(t, pumped, "continuous source video did not advance while target producer was blocked") + } + close(decoder.release) + select { + case <-feedDone: + case <-time.After(2 * time.Second): + stopPump() + close(done) + t.Fatal("active target-audio EOF waited for the WHEP stall watchdog") + } + stopPump() + <-pumpDone + if snapshot := status.Snapshot(); snapshot.State != WHEPFeedTargetAudioFailed || !bytes.Contains([]byte(snapshot.LastError), []byte("target audio")) { + t.Fatalf("target-audio EOF terminal status = %+v", snapshot) + } + drainWHEPCapture(videoCapture) + drainWHEPCapture(audioCapture) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeKeyframe, 0xc0, 180)) + stream.WriteFrame(whepOverwriteTranscodeAudio(0xc1, 200)) + videoCapture.assertEmpty(t) + audioCapture.assertEmpty(t) + + var releases atomic.Int32 + owned := &whepFeedReaders{ + source: util.NewRingBuffer[*avframe.AVFrame](1).NewReader(), + targetAudio: util.NewRingBuffer[*avframe.AVFrame](1).NewReader(), + release: func() { releases.Add(1) }, + } + owned.Close() + owned.Close() + if got := releases.Load(); got != 1 { + t.Fatalf("target-audio release calls = %d, want 1", got) + } +} + +func mediaSample(payload []byte, duration time.Duration) media.Sample { + return media.Sample{Data: payload, Duration: duration} +} diff --git a/module/webrtc/whep_feed_overwrite_test.go b/module/webrtc/whep_feed_overwrite_test.go new file mode 100644 index 00000000..34f2132c --- /dev/null +++ b/module/webrtc/whep_feed_overwrite_test.go @@ -0,0 +1,608 @@ +package webrtc + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "sync" + "testing" + "time" + + "github.com/im-pingo/liveforge/config" + "github.com/im-pingo/liveforge/core" + "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/pion/interceptor" + pionrtp "github.com/pion/rtp" + "github.com/pion/webrtc/v4" +) + +type whepOverwriteLogEvent struct { + protocol string + reader string + action string + overwritten int64 +} + +type whepOverwriteLogHandler struct { + next slog.Handler + events chan<- whepOverwriteLogEvent +} + +func (h *whepOverwriteLogHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.next.Enabled(ctx, level) +} + +func (h *whepOverwriteLogHandler) Handle(ctx context.Context, record slog.Record) error { + if record.Message == "whep: ring overwritten" { + event := whepOverwriteLogEvent{} + record.Attrs(func(attr slog.Attr) bool { + switch attr.Key { + case "protocol": + event.protocol = attr.Value.String() + case "reader": + event.reader = attr.Value.String() + case "action": + event.action = attr.Value.String() + case "overwritten": + event.overwritten = attr.Value.Int64() + } + return true + }) + h.events <- event + } + return h.next.Handle(ctx, record) +} + +func (h *whepOverwriteLogHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return &whepOverwriteLogHandler{next: h.next.WithAttrs(attrs), events: h.events} +} + +func (h *whepOverwriteLogHandler) WithGroup(name string) slog.Handler { + return &whepOverwriteLogHandler{next: h.next.WithGroup(name), events: h.events} +} + +type whepCapturedRTP struct { + header pionrtp.Header + payload []byte +} + +type whepCapturePause struct { + entered chan struct{} + release chan struct{} +} + +type whepRTPCapture struct { + packets chan whepCapturedRTP + mu sync.Mutex + pause *whepCapturePause +} + +func newWHEPRTPCapture() *whepRTPCapture { + return &whepRTPCapture{packets: make(chan whepCapturedRTP, 128)} +} + +func (c *whepRTPCapture) WriteRTP(header *pionrtp.Header, payload []byte) (int, error) { + packet := whepCapturedRTP{header: *header, payload: bytes.Clone(payload)} + c.packets <- packet + c.mu.Lock() + pause := c.pause + c.pause = nil + c.mu.Unlock() + if pause != nil { + close(pause.entered) + <-pause.release + } + return len(payload), nil +} + +func (c *whepRTPCapture) Write(raw []byte) (int, error) { + var packet pionrtp.Packet + if err := packet.Unmarshal(raw); err != nil { + return 0, err + } + _, err := c.WriteRTP(&packet.Header, packet.Payload) + return len(raw), err +} + +func (c *whepRTPCapture) armPause() *whepCapturePause { + c.mu.Lock() + defer c.mu.Unlock() + pause := &whepCapturePause{entered: make(chan struct{}), release: make(chan struct{})} + c.pause = pause + return pause +} + +func (c *whepRTPCapture) readSample(t *testing.T) []whepCapturedRTP { + t.Helper() + var sample []whepCapturedRTP + deadline := time.NewTimer(2 * time.Second) + defer deadline.Stop() + for { + select { + case packet := <-c.packets: + sample = append(sample, packet) + if packet.header.Marker { + return sample + } + case <-deadline.C: + t.Fatalf("timed out waiting for RTP sample after %d packets", len(sample)) + return nil + } + } +} + +func (c *whepRTPCapture) assertEmpty(t *testing.T) { + t.Helper() + select { + case packet := <-c.packets: + t.Fatalf("unexpected RTP packet timestamp=%d payload=%x", packet.header.Timestamp, packet.payload) + default: + } +} + +type whepTrackLocalContext struct { + id string + codec webrtc.RTPCodecParameters + writer webrtc.TrackLocalWriter +} + +func (c whepTrackLocalContext) ID() string { return c.id } +func (c whepTrackLocalContext) SSRC() webrtc.SSRC { return 1234 } +func (c whepTrackLocalContext) SSRCRetransmission() webrtc.SSRC { return 0 } +func (c whepTrackLocalContext) SSRCForwardErrorCorrection() webrtc.SSRC { + return 0 +} +func (c whepTrackLocalContext) WriteStream() webrtc.TrackLocalWriter { return c.writer } +func (c whepTrackLocalContext) HeaderExtensions() []webrtc.RTPHeaderExtensionParameter { + return nil +} +func (c whepTrackLocalContext) RTCPReader() interceptor.RTCPReader { return nil } +func (c whepTrackLocalContext) CodecParameters() []webrtc.RTPCodecParameters { + return []webrtc.RTPCodecParameters{c.codec} +} + +func newWHEPOverwriteSender(t *testing.T, capability webrtc.RTPCodecCapability, payloadType webrtc.PayloadType) (*TrackSender, *whepRTPCapture) { + t.Helper() + track, err := webrtc.NewTrackLocalStaticSample(capability, "track", "whep-overwrite") + if err != nil { + t.Fatalf("NewTrackLocalStaticSample: %v", err) + } + capture := newWHEPRTPCapture() + bindContext := whepTrackLocalContext{ + id: "binding", + codec: webrtc.RTPCodecParameters{ + RTPCodecCapability: capability, + PayloadType: payloadType, + }, + writer: capture, + } + if _, err := track.Bind(bindContext); err != nil { + t.Fatalf("Bind track: %v", err) + } + t.Cleanup(func() { _ = track.Unbind(bindContext) }) + return NewTrackSender("overwrite", track, nil), capture +} + +func installWHEPOverwriteLogObserver(t *testing.T) <-chan whepOverwriteLogEvent { + t.Helper() + events := make(chan whepOverwriteLogEvent, 16) + previous := slog.Default() + handler := &whepOverwriteLogHandler{ + next: slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelWarn}), + events: events, + } + slog.SetDefault(slog.New(handler)) + t.Cleanup(func() { slog.SetDefault(previous) }) + return events +} + +func waitWHEPOverwriteEvent(t *testing.T, events <-chan whepOverwriteLogEvent) whepOverwriteLogEvent { + t.Helper() + select { + case event := <-events: + return event + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for WHEP overwrite event") + return whepOverwriteLogEvent{} + } +} + +func waitWHEPOverwriteSignal(t *testing.T, signal <-chan struct{}, message string) { + t.Helper() + select { + case <-signal: + case <-time.After(2 * time.Second): + t.Fatal(message) + } +} + +func whepOverwriteAVCC(nal []byte) []byte { + payload := make([]byte, 4+len(nal)) + payload[0] = byte(len(nal) >> 24) // #nosec G115 -- test NAL length is encoded as four explicit bytes. + payload[1] = byte(len(nal) >> 16) // #nosec G115 -- test NAL length is encoded as four explicit bytes. + payload[2] = byte(len(nal) >> 8) // #nosec G115 -- test NAL length is encoded as four explicit bytes. + payload[3] = byte(len(nal)) // #nosec G115 -- test NAL length is encoded as four explicit bytes. + copy(payload[4:], nal) + return payload +} + +func whepOverwriteHeader(marker byte, dts int64) *avframe.AVFrame { + sps := whepOverwriteSPS(marker) + pps := []byte{0x68, 0xce, 0x38, marker} + return avframe.NewAVFrame( + avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeSequenceHeader, + dts, dts, buildTestAVCConfigPayload(sps, pps), + ) +} + +func whepOverwriteSPS(marker byte) []byte { + return []byte{0x67, 0x42, 0x00, 0x1f, 0xe9, marker} +} + +func whepOverwriteVideo(frameType avframe.FrameType, marker byte, dts int64) *avframe.AVFrame { + nalType := byte(0x41) + if frameType == avframe.FrameTypeKeyframe { + nalType = 0x65 + } + return avframe.NewAVFrame( + avframe.MediaTypeVideo, avframe.CodecH264, frameType, + dts, dts, whepOverwriteAVCC([]byte{nalType, marker}), + ) +} + +func whepOverwriteAudio(marker byte, dts int64) *avframe.AVFrame { + return avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711A, avframe.FrameTypeInterframe, + dts, dts, []byte{marker}, + ) +} + +func whepSamplePayload(sample []whepCapturedRTP) []byte { + var payload []byte + for _, packet := range sample { + payload = append(payload, packet.payload...) + } + return payload +} + +func startWHEPOverwriteFeed( + t *testing.T, + stream *core.Stream, + startup core.StreamStartupSnapshot, + video, audio *TrackSender, + mode string, + targetAudioCodec avframe.CodecType, + status *whepFeedStatus, +) (chan struct{}, <-chan struct{}) { + t.Helper() + connected := make(chan struct{}) + close(connected) + done := make(chan struct{}) + feedDone := make(chan struct{}) + go func() { + defer close(feedDone) + whepFeedLoop(stream, startup, video, audio, done, connected, mode, targetAudioCodec, nil, status) + }() + return done, feedDone +} + +func stopWHEPOverwriteFeed(t *testing.T, done chan struct{}, feedDone <-chan struct{}) { + t.Helper() + select { + case <-done: + default: + close(done) + } + select { + case <-feedDone: + case <-time.After(2 * time.Second): + t.Fatal("WHEP overwrite feed did not stop") + } +} + +func TestWHEPEstablishedDirectMixedSourceOverwriteRecoversAtFreshKeyframe(t *testing.T) { + events := installWHEPOverwriteLogObserver(t) + stream := core.NewStream("whep/direct-mixed-overwrite", config.StreamConfig{RingBufferSize: 3, GOPCache: true, GOPCacheNum: 1}, config.LimitsConfig{}, core.NewEventBus()) + if err := stream.SetPublisher(&authorizationTestPublisher{id: "source", info: &avframe.MediaInfo{ + VideoCodec: avframe.CodecH264, AudioCodec: avframe.CodecG711A, SampleRate: 8000, Channels: 1, + }}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stream.Close() }) + + oldHeader := whepOverwriteHeader(0xa0, 900) + stream.WriteFrame(oldHeader) + startupKeyframe := whepOverwriteVideo(avframe.FrameTypeKeyframe, 0xa1, 1000) + startupKeyframe.PTS = 10000 + stream.WriteFrame(startupKeyframe) + stream.WriteFrame(whepOverwriteAudio(0xa2, 1020)) + startup := stream.StartupSnapshot() + for _, frame := range []*avframe.AVFrame{ + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb0, 1100), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb1, 1120), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb2, 1140), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb3, 1160), + whepOverwriteAudio(0xb4, 1180), + } { + stream.WriteFrame(frame) + } + + videoSender, videoCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000}, 96) + audioSender, audioCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMA, ClockRate: 8000, Channels: 1}, 8) + status := newWHEPFeedStatus(startup.Generation, startup.LiveCursor, "live") + status.setExpectedMedia(true, true) + done, feedDone := startWHEPOverwriteFeed(t, stream, startup, videoSender, audioSender, "live", avframe.CodecG711A, status) + t.Cleanup(func() { stopWHEPOverwriteFeed(t, done, feedDone) }) + + startupVideo := videoCapture.readSample(t) + if payload := whepSamplePayload(startupVideo); !bytes.Contains(payload, whepOverwriteSPS(0xa0)) || !bytes.Contains(payload, []byte{0x65, 0xa1}) { + t.Fatalf("startup video RTP payload = %x, want startup parameter sets and IDR", payload) + } + startupAudio := audioCapture.readSample(t) + if payload := whepSamplePayload(startupAudio); !bytes.Contains(payload, []byte{0xa2}) { + t.Fatalf("startup audio RTP payload = %x, want marker a2", payload) + } + + if got := waitWHEPOverwriteEvent(t, events); got != (whepOverwriteLogEvent{protocol: "whep", reader: "source", action: "wait_keyframe", overwritten: 2}) { + t.Fatalf("source overwrite event = %+v", got) + } + if snapshot := status.Snapshot(); snapshot.State != WHEPFeedWaitingKeyframe || snapshot.Generation != startup.Generation { + t.Fatalf("source recovery status = %+v", snapshot) + } + videoCapture.assertEmpty(t) + + stream.WriteFrame(whepOverwriteAudio(0xc0, 1200)) + recoveredAudio := audioCapture.readSample(t) + if payload := whepSamplePayload(recoveredAudio); !bytes.Contains(payload, []byte{0xc0}) { + t.Fatalf("recovery audio RTP payload = %x, want marker c0", payload) + } + if delta := recoveredAudio[0].header.Timestamp - startupAudio[0].header.Timestamp; delta != 160 { + t.Fatalf("recovered audio RTP timestamp delta = %d, want one 20ms/160 step", delta) + } + stream.WriteFrame(whepOverwriteAudio(0xc5, 1220)) + secondRecoveredAudio := audioCapture.readSample(t) + if delta := secondRecoveredAudio[0].header.Timestamp - recoveredAudio[0].header.Timestamp; delta != 160 { + t.Fatalf("post-recovery audio RTP timestamp delta = %d (%d -> %d), want one 20ms/160 step", delta, recoveredAudio[0].header.Timestamp, secondRecoveredAudio[0].header.Timestamp) + } + if got := status.Snapshot().State; got != WHEPFeedWaitingKeyframe { + t.Fatalf("state after current audio only = %q, want waiting_keyframe", got) + } + + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeInterframe, 0xc1, 1220)) + currentHeader := whepOverwriteHeader(0xc2, 1240) + stream.WriteFrame(currentHeader) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeKeyframe, 0xc3, 6000)) + recoveredVideo := videoCapture.readSample(t) + payload := whepSamplePayload(recoveredVideo) + if !bytes.Contains(payload, whepOverwriteSPS(0xc2)) || !bytes.Contains(payload, []byte{0x65, 0xc3}) { + t.Fatalf("first recovered video RTP payload = %x, want current parameter sets and IDR", payload) + } + if bytes.Contains(payload, []byte{0x41, 0xb2}) || bytes.Contains(payload, []byte{0x41, 0xc1}) { + t.Fatalf("first recovered video RTP payload retained an interframe: %x", payload) + } + if delta := recoveredVideo[0].header.Timestamp - startupVideo[0].header.Timestamp; delta != 3600 { + t.Fatalf("recovered video RTP timestamp delta = %d, want reset 40ms/3600", delta) + } + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeInterframe, 0xc4, 6040)) + if payload := whepSamplePayload(videoCapture.readSample(t)); !bytes.Contains(payload, []byte{0x41, 0xc4}) { + t.Fatalf("post-recovery P-frame RTP payload = %x, want marker c4", payload) + } + if snapshot := status.Snapshot(); snapshot.State != WHEPFeedPlaying || snapshot.VideoFrames != 3 || snapshot.AudioFrames < 2 || snapshot.SourceOverwrites != 2 || snapshot.DroppedVideo != 1 { + t.Fatalf("recovered mixed feed status = %+v", snapshot) + } +} + +func TestWHEPRepeatedSourceOverwriteBeforeKeyframeUsesNewestConfiguration(t *testing.T) { + events := installWHEPOverwriteLogObserver(t) + stream := core.NewStream("whep/repeated-source-overwrite", config.StreamConfig{RingBufferSize: 2, GOPCache: true, GOPCacheNum: 1}, config.LimitsConfig{}, core.NewEventBus()) + if err := stream.SetPublisher(&authorizationTestPublisher{id: "source", info: &avframe.MediaInfo{ + VideoCodec: avframe.CodecH264, AudioCodec: avframe.CodecG711A, SampleRate: 8000, Channels: 1, + }}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stream.Close() }) + startupHeader := whepOverwriteHeader(0xa0, 900) + stream.WriteFrame(startupHeader) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeKeyframe, 0xa1, 1000)) + stream.WriteFrame(whepOverwriteAudio(0xa2, 1020)) + startup := stream.StartupSnapshot() + for _, frame := range []*avframe.AVFrame{ + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb0, 1040), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb1, 1060), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb2, 1080), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb3, 1100), + } { + stream.WriteFrame(frame) + } + + videoSender, videoCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000}, 96) + audioSender, audioCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMA, ClockRate: 8000, Channels: 1}, 8) + status := newWHEPFeedStatus(startup.Generation, startup.LiveCursor, "live") + status.setExpectedMedia(true, true) + done, feedDone := startWHEPOverwriteFeed(t, stream, startup, videoSender, audioSender, "live", avframe.CodecG711A, status) + t.Cleanup(func() { stopWHEPOverwriteFeed(t, done, feedDone) }) + startupVideo := videoCapture.readSample(t) + _ = audioCapture.readSample(t) + if got := waitWHEPOverwriteEvent(t, events); got.reader != "source" || got.overwritten != 2 { + t.Fatalf("first overwrite event = %+v", got) + } + + pause := audioCapture.armPause() + stream.WriteFrame(whepOverwriteAudio(0xc0, 1120)) + waitWHEPOverwriteSignal(t, pause.entered, "established audio did not advance during first recovery") + newestHeader := whepOverwriteHeader(0xd2, 1180) + for _, frame := range []*avframe.AVFrame{ + whepOverwriteHeader(0xd0, 1140), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xd1, 1160), + newestHeader, + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xd3, 1200), + } { + stream.WriteFrame(frame) + } + close(pause.release) + _ = audioCapture.readSample(t) + if got := waitWHEPOverwriteEvent(t, events); got.reader != "source" || got.overwritten != 2 { + t.Fatalf("second overwrite event = %+v", got) + } + videoCapture.assertEmpty(t) + + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeInterframe, 0xe0, 1220)) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeKeyframe, 0xe1, 7000)) + recovered := videoCapture.readSample(t) + payload := whepSamplePayload(recovered) + if !bytes.Contains(payload, whepOverwriteSPS(0xd2)) || !bytes.Contains(payload, []byte{0x65, 0xe1}) { + t.Fatalf("repeated-overwrite recovery RTP payload = %x, want newest retained configuration and IDR", payload) + } + for _, stale := range [][]byte{{0xe9, 0xa0}, {0xe9, 0xd0}, {0x41, 0xd3}, {0x41, 0xe0}} { + if bytes.Contains(payload, stale) { + t.Fatalf("repeated-overwrite recovery RTP payload contains stale marker %x: %x", stale, payload) + } + } + if delta := recovered[0].header.Timestamp - startupVideo[0].header.Timestamp; delta != 3600 { + t.Fatalf("repeated-overwrite RTP timestamp delta = %d, want one reset epoch", delta) + } + videoCapture.assertEmpty(t) +} + +func TestWHEPAudioOnlyDirectSourceOverwriteContinuesAtLive(t *testing.T) { + events := installWHEPOverwriteLogObserver(t) + stream := core.NewStream("whep/audio-only-overwrite", config.StreamConfig{RingBufferSize: 2}, config.LimitsConfig{}, core.NewEventBus()) + if err := stream.SetPublisher(&authorizationTestPublisher{id: "source", info: &avframe.MediaInfo{ + AudioCodec: avframe.CodecG711A, SampleRate: 8000, Channels: 1, + }}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stream.Close() }) + startup := stream.StartupSnapshot() + audioSender, audioCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMA, ClockRate: 8000, Channels: 1}, 8) + status := newWHEPFeedStatus(startup.Generation, startup.LiveCursor, "realtime") + status.setExpectedMedia(false, true) + done, feedDone := startWHEPOverwriteFeed(t, stream, startup, nil, audioSender, "realtime", avframe.CodecG711A, status) + t.Cleanup(func() { stopWHEPOverwriteFeed(t, done, feedDone) }) + + stream.WriteFrame(whepOverwriteAudio(0xa0, 0)) + _ = audioCapture.readSample(t) + pause := audioCapture.armPause() + stream.WriteFrame(whepOverwriteAudio(0xa1, 20)) + waitWHEPOverwriteSignal(t, pause.entered, "audio-only feed did not enter the deterministic write barrier") + for _, frame := range []*avframe.AVFrame{ + whepOverwriteAudio(0xb0, 40), + whepOverwriteAudio(0xb1, 60), + whepOverwriteAudio(0xb2, 80), + whepOverwriteAudio(0xb3, 100), + } { + stream.WriteFrame(frame) + } + close(pause.release) + _ = audioCapture.readSample(t) + if got := waitWHEPOverwriteEvent(t, events); got != (whepOverwriteLogEvent{protocol: "whep", reader: "source", action: "continue_audio", overwritten: 2}) { + t.Fatalf("audio-only overwrite event = %+v", got) + } + if got := status.Snapshot().State; got == WHEPFeedWaitingKeyframe { + t.Fatal("audio-only source overwrite entered a video keyframe wait") + } + audioCapture.assertEmpty(t) + + stream.WriteFrame(whepOverwriteAudio(0xc0, 120)) + payload := whepSamplePayload(audioCapture.readSample(t)) + if !bytes.Contains(payload, []byte{0xc0}) || bytes.Contains(payload, []byte{0xb2}) { + t.Fatalf("audio-only recovered RTP payload = %x, want only next live marker c0", payload) + } + if got := status.Snapshot().State; got != WHEPFeedPlaying { + t.Fatalf("audio-only recovered state = %q, want playing", got) + } +} + +func TestWHEPOverwriteRecoveryStopsBeforeReplacementGeneration(t *testing.T) { + events := installWHEPOverwriteLogObserver(t) + stream := core.NewStream("whep/overwrite-generation", config.StreamConfig{RingBufferSize: 2, GOPCache: true, GOPCacheNum: 1}, config.LimitsConfig{}, core.NewEventBus()) + if err := stream.SetPublisher(&authorizationTestPublisher{id: "original", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stream.Close() }) + stream.WriteFrame(whepOverwriteHeader(0xa0, 0)) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeKeyframe, 0xa1, 40)) + startup := stream.StartupSnapshot() + for _, frame := range []*avframe.AVFrame{ + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb0, 80), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb1, 120), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb2, 160), + whepOverwriteVideo(avframe.FrameTypeInterframe, 0xb3, 200), + } { + stream.WriteFrame(frame) + } + videoSender, videoCapture := newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000}, 96) + status := newWHEPFeedStatus(startup.Generation, startup.LiveCursor, "live") + status.setExpectedMedia(true, false) + done, feedDone := startWHEPOverwriteFeed(t, stream, startup, videoSender, nil, "live", 0, status) + _ = videoCapture.readSample(t) + if got := waitWHEPOverwriteEvent(t, events); got.reader != "source" { + t.Fatalf("source overwrite event = %+v", got) + } + videoCapture.assertEmpty(t) + + stream.RemovePublisher() + if err := stream.SetPublisher(&authorizationTestPublisher{id: "replacement", info: &avframe.MediaInfo{VideoCodec: avframe.CodecH264}}); err != nil { + t.Fatal(err) + } + stream.WriteFrame(whepOverwriteHeader(0xc0, 240)) + stream.WriteFrame(whepOverwriteVideo(avframe.FrameTypeKeyframe, 0xc1, 280)) + select { + case <-feedDone: + case <-time.After(2 * time.Second): + select { + case packet := <-videoCapture.packets: + close(done) + t.Fatalf("replacement-generation RTP escaped cancellation: timestamp=%d payload=%x", packet.header.Timestamp, packet.payload) + default: + close(done) + t.Fatal("old-generation WHEP feed and reader waiters did not stop") + } + } + videoCapture.assertEmpty(t) + if snapshot := status.Snapshot(); snapshot.Generation != startup.Generation || snapshot.State != WHEPFeedGenerationEnded { + t.Fatalf("old-generation terminal status = %+v", snapshot) + } + select { + case <-done: + default: + close(done) + } +} + +func TestWHEPFeedClosesSendGateAtGenerationBoundary(t *testing.T) { + stream := core.NewStream("whep/send-gate-generation", config.StreamConfig{RingBufferSize: 4}, config.LimitsConfig{}, core.NewEventBus()) + if err := stream.SetPublisher(&authorizationTestPublisher{id: "source", info: &avframe.MediaInfo{VideoCodec: avframe.CodecVP8}}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { stream.Close() }) + startup := stream.StartupSnapshot() + status := newWHEPFeedStatus(startup.Generation, startup.LiveCursor, "realtime") + status.setExpectedMedia(true, false) + connected := make(chan struct{}) + close(connected) + done := make(chan struct{}) + feedDone := make(chan struct{}) + gate := newWHEPSendGate() + go func() { + defer close(feedDone) + whepFeedLoop(stream, startup, nil, nil, done, connected, "realtime", 0, nil, status, gate) + }() + + stream.RemovePublisher() + select { + case <-feedDone: + case <-time.After(time.Second): + close(done) + t.Fatal("WHEP feed did not stop after generation retirement") + } + if err := gate.write(func() error { return nil }); !errors.Is(err, errWHEPSendGateClosed) { + t.Fatalf("send after generation retirement = %v, want %v", err, errWHEPSendGateClosed) + } + if got := status.Snapshot().State; got != WHEPFeedGenerationEnded { + t.Fatalf("generation terminal state = %q, want generation ended", got) + } + close(done) +} diff --git a/module/webrtc/whep_feed_test.go b/module/webrtc/whep_feed_test.go index 64fa10eb..7e00fd78 100644 --- a/module/webrtc/whep_feed_test.go +++ b/module/webrtc/whep_feed_test.go @@ -1,6 +1,11 @@ package webrtc import ( + "bytes" + "errors" + "log/slog" + "strings" + "sync" "testing" "time" @@ -9,6 +14,7 @@ import ( "github.com/im-pingo/liveforge/pkg/audiocodec" "github.com/im-pingo/liveforge/pkg/avframe" "github.com/im-pingo/liveforge/pkg/util" + "github.com/pion/webrtc/v4" ) func TestWHEPStartupSnapshotKeepsFramesWrittenWhileCacheIsSent(t *testing.T) { @@ -42,12 +48,26 @@ func TestWHEPStartupSnapshotDropsSourceAudioWhenTranscoding(t *testing.T) { }, config.LimitsConfig{}, core.NewEventBus()) video := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH265, avframe.FrameTypeKeyframe, 1000, 1000, []byte{1}) aac := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 1020, 1020, []byte{2}) + interframe := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH265, avframe.FrameTypeInterframe, 1040, 1040, []byte{3}) stream.WriteFrame(video) stream.WriteFrame(aac) + stream.WriteFrame(interframe) - gopCache := whepLiveSnapshot(stream.StartupSnapshot(), true) - if len(gopCache) != 1 || gopCache[0] != video { - t.Fatalf("transcoded live GOP snapshot = %v, want cached video only", gopCache) + snapshot := stream.StartupSnapshot() + alias := snapshot.ReplayFrames + gopCache := whepLiveSnapshot(snapshot, true) + if len(gopCache) != 2 || gopCache[0] != video || gopCache[1] != interframe { + t.Fatalf("transcoded live GOP snapshot = %v, want cached video frames", gopCache) + } + if len(snapshot.ReplayFrames) != 3 || snapshot.ReplayFrames[0] != video || snapshot.ReplayFrames[1] != aac || snapshot.ReplayFrames[2] != interframe { + t.Fatalf("startup snapshot was mutated while filtering: %v", snapshot.ReplayFrames) + } + if len(alias) != 3 || alias[0] != video || alias[1] != aac || alias[2] != interframe { + t.Fatalf("startup snapshot alias was mutated while filtering: %v", alias) + } + gopCache[0] = nil + if snapshot.ReplayFrames[0] != video || alias[0] != video { + t.Fatal("filtered replay slice aliases the startup snapshot backing storage") } } @@ -71,26 +91,30 @@ func TestWHEPFeedReadersKeepAtomicSourceCursorWhenTranscoderUnavailable(t *testi stream.WriteFrame(betweenSnapshotAndReader) readers := newWHEPFeedReaders(stream, snapshot, true, avframe.CodecOpus) - defer readers.Close() - if got, ok := readers.source.TryRead(); !ok || got != betweenSnapshotAndReader { - t.Fatalf("source reader first frame = (%v, %v), want frame written after snapshot", got, ok) + done := make(chan struct{}) + readers.startWaiters(done, snapshot.GenerationDone) + defer func() { + close(done) + readers.Close() + }() + if !readers.wait(done, snapshot.GenerationDone) { + t.Fatal("source reader wait stopped before the frame written after the snapshot") } - select { - case <-readers.source.Signal(): // Consume the signal for the frame read above. - default: + read, ok := readers.tryReadSource() + if !ok || read.result.Value != betweenSnapshotAndReader { + t.Fatalf("source reader first frame = (%v, %v), want frame written after snapshot", read.result.Value, ok) } if readers.targetAudio == nil { t.Fatal("transcode reader missing") } - if _, ok := readers.targetAudio.TryRead(); ok { + if _, targetAudioOK := readers.tryReadTargetAudio(); targetAudioOK { t.Fatal("unavailable transcoder unexpectedly produced a frame") } woke := make(chan bool, 1) - waitDone := make(chan struct{}) go func() { - woke <- readers.wait(waitDone, snapshot.GenerationDone) + woke <- readers.wait(done, snapshot.GenerationDone) }() select { case <-woke: @@ -109,10 +133,10 @@ func TestWHEPFeedReadersKeepAtomicSourceCursorWhenTranscoderUnavailable(t *testi t.Fatal("reader wait stopped while the transcode epoch was unavailable") } case <-time.After(time.Second): - close(waitDone) t.Fatal("source video did not wake reader while the transcode epoch was unavailable") } - if got, ok := readers.source.TryRead(); !ok || got != afterUnavailableEpoch { + read, ok = readers.tryReadSource() + if got := read.result.Value; !ok || got != afterUnavailableEpoch { t.Fatalf("source reader during unavailable transcode epoch = (%v, %v), want uninterrupted video", got, ok) } } @@ -130,9 +154,10 @@ func TestWHEPFeedReadersWakeIndependently(t *testing.T) { done := make(chan struct{}) woke1 := make(chan bool, 1) woke2 := make(chan bool, 1) + r1.startWaiters(done, snapshot.GenerationDone) + r2.startWaiters(done, snapshot.GenerationDone) go func() { woke1 <- r1.wait(done, snapshot.GenerationDone) }() go func() { woke2 <- r2.wait(done, snapshot.GenerationDone) }() - time.Sleep(20 * time.Millisecond) stream.WriteFrame(avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 1, 1, []byte{1})) for i, woke := range []<-chan bool{woke1, woke2} { @@ -166,6 +191,32 @@ func TestWHEPFeedReadersStopOnGenerationEnd(t *testing.T) { } } +func TestWHEPTargetAudioWaiterCancellationIsNotEOF(t *testing.T) { + sourceRing := util.NewRingBuffer[*avframe.AVFrame](2) + targetRing := util.NewRingBuffer[*avframe.AVFrame](2) + readers := &whepFeedReaders{ + source: sourceRing.NewReader(), + targetAudio: targetRing.NewReader(), + } + defer readers.Close() + + done := make(chan struct{}) + generationDone := make(chan struct{}) + readers.startWaiters(done, generationDone) + readers.lifecycleMu.Lock() + cancel := readers.waitCancel + readers.lifecycleMu.Unlock() + if cancel == nil { + t.Fatal("WHEP reader waiter context was not initialized") + } + + cancel() + readers.waitGroup.Wait() + if readers.activeTargetAudioEOF(done, generationDone) { + t.Fatal("local waiter cancellation was misclassified as target-audio EOF") + } +} + func TestWHEPInitialKeyframeGateRequiresSentCachedKeyframe(t *testing.T) { if whepInitialKeyframeReady("live", false) { t.Fatal("live mode bypassed keyframe gate without sending a cached keyframe") @@ -190,70 +241,416 @@ func TestWHEPInitialMediaGateAllowsAudioOnlyStreams(t *testing.T) { } } -func TestWHEPFeedReadersDrainOnlyTargetAudioAfterKeyframe(t *testing.T) { - stream := core.NewStream("live/whep-target-audio", config.StreamConfig{RingBufferSize: 16}, config.LimitsConfig{}, core.NewEventBus()) - if err := stream.SetPublisher(&authorizationTestPublisher{ - id: "source", info: &avframe.MediaInfo{AudioCodec: avframe.CodecAAC}, - }); err != nil { - t.Fatal(err) +func TestWHEPModeDefaultsToLiveSnapshot(t *testing.T) { + if got := normalizeWHEPMode(""); got != "live" { + t.Fatalf("empty WHEP mode = %q, want live", got) } - snapshot := stream.StartupSnapshot() - targetRing := util.NewRingBuffer[*avframe.AVFrame](16) - readers := &whepFeedReaders{targetAudio: targetRing.NewReaderAt(0)} - video := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH265, avframe.FrameTypeInterframe, 0, 0, []byte{1}) - aac := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 0, 0, []byte{2}) - earlyOpus := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecOpus, avframe.FrameTypeInterframe, 0, 0, []byte{3}) - targetRing.Write(video) - targetRing.Write(aac) - targetRing.Write(earlyOpus) - - var delivered []*avframe.AVFrame - readers.drainTargetAudio(stream, snapshot.Generation, false, avframe.CodecOpus, func(frame *avframe.AVFrame) { - delivered = append(delivered, frame) - }) - if len(delivered) != 0 { - t.Fatalf("target audio delivered before keyframe: %v", delivered) - } - - lateOpus := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecOpus, avframe.FrameTypeInterframe, 20, 20, []byte{4}) - targetRing.Write(video) - targetRing.Write(aac) - targetRing.Write(lateOpus) - readers.drainTargetAudio(stream, snapshot.Generation, true, avframe.CodecOpus, func(frame *avframe.AVFrame) { - delivered = append(delivered, frame) - }) - if len(delivered) != 1 || delivered[0] != lateOpus { - t.Fatalf("delivered target frames = %v, want late Opus only", delivered) + if got := normalizeWHEPMode("invalid"); got != "live" { + t.Fatalf("invalid WHEP mode = %q, want live", got) + } + if got := normalizeWHEPMode("realtime"); got != "realtime" { + t.Fatalf("explicit realtime mode = %q, want realtime", got) } } -func TestWHEPTranscodeReaderStopsBeforeReplacementGenerationFrame(t *testing.T) { - stream := core.NewStream("live/whep-transcode-generation", config.StreamConfig{RingBufferSize: 16}, config.LimitsConfig{}, core.NewEventBus()) - if err := stream.SetPublisher(&authorizationTestPublisher{ - id: "old", info: &avframe.MediaInfo{AudioCodec: avframe.CodecAAC}, - }); err != nil { - t.Fatal(err) +func TestWHEPFeedStatusDistinguishesWaitingAndTerminalFailure(t *testing.T) { + status := newWHEPFeedStatus(7, 42, "realtime") + if got := status.Snapshot().State; got != WHEPFeedWaitingKeyframe { + t.Fatalf("initial feed state = %q, want %q", got, WHEPFeedWaitingKeyframe) } - snapshot := stream.StartupSnapshot() - targetRing := util.NewRingBuffer[*avframe.AVFrame](16) - readers := &whepFeedReaders{targetAudio: targetRing.NewReaderAt(0)} - oldFrame := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecOpus, avframe.FrameTypeInterframe, 0, 0, []byte{1}) - replacementFrame := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecOpus, avframe.FrameTypeInterframe, 20, 20, []byte{2}) - targetRing.Write(oldFrame) - - var delivered []*avframe.AVFrame - readers.drainTargetAudio(stream, snapshot.Generation, true, avframe.CodecOpus, func(frame *avframe.AVFrame) { - delivered = append(delivered, frame) - stream.RemovePublisher() - if err := stream.SetPublisher(&authorizationTestPublisher{ - id: "replacement", info: &avframe.MediaInfo{AudioCodec: avframe.CodecAAC}, - }); err != nil { - t.Fatal(err) + + status.SetError(WHEPFeedSampleWriteFailed, errors.New("track closed")) + snapshot := status.Snapshot() + if snapshot.State != WHEPFeedSampleWriteFailed { + t.Fatalf("terminal feed state = %q, want %q", snapshot.State, WHEPFeedSampleWriteFailed) + } + if snapshot.LastError != "track closed" { + t.Fatalf("feed error = %q, want track closed", snapshot.LastError) + } + if snapshot.Generation != 7 || snapshot.Cursor != 42 { + t.Fatalf("feed identity = generation %d cursor %d, want generation 7 cursor 42", snapshot.Generation, snapshot.Cursor) + } +} + +func TestWHEPFeedStatusSeparatesSourceOverwriteFromTrackDrops(t *testing.T) { + status := newWHEPFeedStatus(8, 11, "live") + status.setExpectedMedia(true, true) + status.recordSourceOverwrite(3) + + snapshot := status.Snapshot() + if snapshot.SourceOverwrites != 3 { + t.Fatalf("source overwrite count = %d, want 3", snapshot.SourceOverwrites) + } + if snapshot.DroppedVideo != 0 || snapshot.DroppedAudio != 0 { + t.Fatalf("source overwrite was misclassified as track drops: %+v", snapshot) + } +} + +func TestWHEPFeedStatusReportsStableFirstMediaWait(t *testing.T) { + status := newWHEPFeedStatus(17, 25, "live") + createdAt := time.Date(2026, time.August, 29, 12, 0, 0, 0, time.UTC) + status.createdAt.Store(createdAt.UnixNano()) + + if got := status.Snapshot().FirstMediaWaitMS; got != 0 { + t.Fatalf("first-media wait before media = %dms, want 0", got) + } + + status.recordVideoAt(true, createdAt.Add(1250*time.Millisecond)) + watchdogStop := make(chan struct{}) + close(watchdogStop) + status.watchInactivity(watchdogStop, make(chan struct{}), time.Second) + if got := status.Snapshot().FirstMediaWaitMS; got != 1250 { + t.Fatalf("first-media wait after watchdog start = %dms, want 1250", got) + } + + status.recordAudioAt(true, createdAt.Add(3*time.Second)) + if got := status.Snapshot().FirstMediaWaitMS; got != 1250 { + t.Fatalf("first-media wait after later audio = %dms, want stable 1250", got) + } +} + +func TestWHEPFeedStatusMarksNoInputAndRecoversOnMedia(t *testing.T) { + status := newWHEPFeedStatus(8, 11, "live") + status.MarkNoMediaInput() + if got := status.Snapshot().State; got != WHEPFeedNoMediaInput { + t.Fatalf("idle feed state = %q, want %q", got, WHEPFeedNoMediaInput) + } + + status.RecordAudio(true) + if got := status.Snapshot().State; got != WHEPFeedPlaying { + t.Fatalf("recovered feed state = %q, want %q", got, WHEPFeedPlaying) + } + + waiting := newWHEPFeedStatus(9, 12, "realtime") + waiting.RecordVideo(false) + waiting.MarkNoMediaInput() + if got := waiting.Snapshot().State; got != WHEPFeedWaitingKeyframe { + t.Fatalf("feed with dropped interframe state = %q, want %q", got, WHEPFeedWaitingKeyframe) + } +} + +func TestWHEPFeedStatusLogsStateTransitionsWithContext(t *testing.T) { + var logs bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + status := newWHEPFeedStatus(16, 24, "live") + status.setExpectedMedia(true, false) + status.RecordVideo(true) + status.RecordVideo(true) + status.SetError(WHEPFeedSampleWriteFailed, errors.New("closed track")) + + output := logs.String() + for _, expected := range []string{ + "msg=\"WHEP feed state changed\"", + "previous=waiting_keyframe", + "state=playing", + "previous=playing", + "state=sample_write_failed", + "generation=16", + "cursor=24", + "mode=live", + } { + if !strings.Contains(output, expected) { + t.Fatalf("state transition log missing %q:\n%s", expected, output) } - targetRing.Write(replacementFrame) - }) - if len(delivered) != 1 || delivered[0] != oldFrame { - t.Fatalf("delivered target frames = %v, want old-generation frame only", delivered) + } + if got := strings.Count(output, "state=playing"); got != 1 { + t.Fatalf("playing transition log count = %d, want 1:\n%s", got, output) + } +} + +func TestWHEPFeedStatusMarksPostMediaInactivityStalledAndRecovers(t *testing.T) { + type statusTimeline interface { + setExpectedMedia(video, audio bool) + recordVideoAt(sent bool, now time.Time) + checkInactivityAt(now time.Time) + } + + status := newWHEPFeedStatus(10, 13, "live") + timeline, ok := any(status).(statusTimeline) + if !ok { + t.Fatal("WHEP feed status does not support video inactivity tracking") + } + timeline.setExpectedMedia(true, false) + t0 := time.Now().UTC() + timeline.recordVideoAt(true, t0) + timeline.checkInactivityAt(t0.Add(whepNoMediaInputTimeout)) + if got := status.Snapshot().State; got != WHEPFeedState("media_stalled") { + t.Fatalf("inactive feed state = %q, want media_stalled", got) + } + + timeline.recordVideoAt(true, t0.Add(whepNoMediaInputTimeout+time.Second)) + if got := status.Snapshot().State; got != WHEPFeedPlaying { + t.Fatalf("recovered feed state = %q, want %q", got, WHEPFeedPlaying) + } +} + +func TestWHEPFeedStatusRequiresEveryExpectedMediaKindForPlayAndRecovery(t *testing.T) { + type statusTimeline interface { + setExpectedMedia(video, audio bool) + recordVideoAt(sent bool, now time.Time) + recordAudioAt(sent bool, now time.Time) + checkInactivityAt(now time.Time) + } + + status := newWHEPFeedStatus(12, 15, "live") + timeline, ok := any(status).(statusTimeline) + if !ok { + t.Fatal("WHEP feed status does not support expected-media timeline tracking") + } + timeline.setExpectedMedia(true, true) + t0 := time.Now().UTC() + timeline.recordVideoAt(true, t0.Add(time.Second)) + if got := status.Snapshot().State; got == WHEPFeedPlaying { + t.Fatal("video-only progress marked a mixed expected feed playing") + } + timeline.recordAudioAt(true, t0.Add(2*time.Second)) + if got := status.Snapshot().State; got != WHEPFeedPlaying { + t.Fatalf("both expected media kinds state = %q, want %q", got, WHEPFeedPlaying) + } + + timeline.checkInactivityAt(t0.Add(whepNoMediaInputTimeout + 3*time.Second)) + if got := status.Snapshot().State; got != WHEPFeedState("media_stalled") { + t.Fatalf("stalled mixed feed state = %q, want media_stalled", got) + } + timeline.recordVideoAt(true, t0.Add(whepNoMediaInputTimeout+4*time.Second)) + if got := status.Snapshot().State; got != WHEPFeedState("media_stalled") { + t.Fatalf("one-kind recovery state = %q, want media_stalled", got) + } + timeline.recordAudioAt(true, t0.Add(whepNoMediaInputTimeout+5*time.Second)) + if got := status.Snapshot().State; got != WHEPFeedPlaying { + t.Fatalf("full recovery state = %q, want %q", got, WHEPFeedPlaying) + } +} + +func TestWHEPFeedStatusGivesMissingExpectedKindFullStartupGrace(t *testing.T) { + for _, test := range []struct { + name string + record func(*whepFeedStatus, time.Time) + }{ + { + name: "audio arrives before video", + record: func(status *whepFeedStatus, now time.Time) { + status.recordAudioAt(true, now) + }, + }, + { + name: "video arrives before audio", + record: func(status *whepFeedStatus, now time.Time) { + status.recordVideoAt(true, now) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + status := newWHEPFeedStatus(18, 26, "live") + status.setExpectedMedia(true, true) + createdAt := time.Date(2026, time.August, 29, 12, 0, 0, 0, time.UTC) + status.createdAt.Store(createdAt.UnixNano()) + status.phase.Store(&whepFeedPhase{state: WHEPFeedWaitingKeyframe, changedAt: createdAt.UnixNano()}) + firstMediaAt := createdAt.Add(time.Second) + test.record(status, firstMediaAt) + + status.checkInactivityAt(firstMediaAt.Add(2 * time.Second)) + if got := status.Snapshot().State; got == WHEPFeedMediaStalled { + t.Fatalf("missing expected kind stalled after 2s, want full %s grace", whepNoMediaInputTimeout) + } + + status.checkInactivityAt(firstMediaAt.Add(whepNoMediaInputTimeout)) + if got := status.Snapshot().State; got != WHEPFeedMediaStalled { + t.Fatalf("missing expected kind state after grace = %q, want %q", got, WHEPFeedMediaStalled) + } + }) + } +} + +func TestWHEPFeedStatusKeepsWaitingForFirstExpectedVideoKeyframe(t *testing.T) { + status := newWHEPFeedStatus(15, 18, "realtime") + status.setExpectedMedia(true, true) + t0 := time.Now().UTC() + status.recordAudioAt(true, t0) + status.recordVideoAt(false, t0.Add(time.Second)) + status.checkInactivityAt(t0.Add(whepNoMediaInputTimeout + 2*time.Second)) + + if got := status.Snapshot().State; got != WHEPFeedWaitingKeyframe { + t.Fatalf("mixed feed before first video keyframe state = %q, want %q", got, WHEPFeedWaitingKeyframe) + } +} + +func TestWHEPFeedTerminalStateRejectsLateMediaAndTransportUpdates(t *testing.T) { + status := newWHEPFeedStatus(11, 14, "live") + status.SetError(WHEPFeedSampleWriteFailed, errors.New("track closed")) + want := status.Snapshot() + + status.RecordVideo(true) + status.RecordAudio(false) + status.MarkNoMediaInput() + status.SetTransportStats(10, 20, 30) + if got := status.Snapshot(); got != want { + t.Fatalf("terminal feed changed after late updates:\n got %+v\nwant %+v", got, want) + } +} + +func TestWHEPFeedTerminalStateWaitsForConcurrentUpdates(t *testing.T) { + status := newWHEPFeedStatus(14, 17, "live") + status.setExpectedMedia(true, true) + start := make(chan struct{}) + var workers sync.WaitGroup + for worker := 0; worker < 8; worker++ { + workers.Add(1) + go func(offset uint64) { + defer workers.Done() + <-start + for index := uint64(0); index < 1000; index++ { + status.RecordVideo(true) + status.RecordAudio(true) + status.SetTransportStats(index+offset, (index+offset)*100, index+offset) + status.checkInactivityAt(time.Now().UTC()) + } + }(uint64(worker) * 1000) + } + close(start) + status.SetError(WHEPFeedSampleWriteFailed, errors.New("terminal")) + workers.Wait() + want := status.Snapshot() + if want.State != WHEPFeedSampleWriteFailed || want.LastError != "terminal" { + t.Fatalf("terminal feed = %+v", want) + } + + status.RecordVideo(true) + status.RecordAudio(true) + status.SetTransportStats(^uint64(0), ^uint64(0), ^uint64(0)) + status.checkInactivityAt(time.Now().UTC().Add(whepNoMediaInputTimeout)) + if got := status.Snapshot(); got != want { + t.Fatalf("terminal feed changed after concurrent shutdown:\n got %+v\nwant %+v", got, want) + } +} + +func TestWHEPFeedWatchdogExitsWithSessionOrGeneration(t *testing.T) { + type statusWatchdog interface { + watchInactivity(stop, generationDone <-chan struct{}, timeout time.Duration) + } + + for _, terminal := range []string{"session", "generation"} { + t.Run(terminal, func(t *testing.T) { + status := newWHEPFeedStatus(13, 16, "live") + watchdog, ok := any(status).(statusWatchdog) + if !ok { + t.Fatal("WHEP feed status does not expose a lifecycle-bound inactivity watchdog") + } + stop := make(chan struct{}) + generationDone := make(chan struct{}) + exited := make(chan struct{}) + go func() { + watchdog.watchInactivity(stop, generationDone, 20*time.Millisecond) + close(exited) + }() + if terminal == "session" { + close(stop) + } else { + close(generationDone) + } + select { + case <-exited: + case <-time.After(time.Second): + t.Fatal("WHEP inactivity watchdog did not exit") + } + }) + } +} + +func TestWHEPFeedDoesNotCountUnrequestedMediaAsDropped(t *testing.T) { + tests := []struct { + name string + video bool + audio bool + wantVideoSent uint64 + wantAudioSent uint64 + }{ + {name: "audio only offer", audio: true, wantAudioSent: 1}, + {name: "video only offer", video: true, wantVideoSent: 1}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + stream := core.NewStream("live/whep-unrequested-"+test.name, config.StreamConfig{ + RingBufferSize: 16, + }, config.LimitsConfig{}, core.NewEventBus()) + if err := stream.SetPublisher(&authorizationTestPublisher{ + id: "source", + info: &avframe.MediaInfo{ + VideoCodec: avframe.CodecVP8, + AudioCodec: avframe.CodecG711A, + SampleRate: 8000, + Channels: 1, + }, + }); err != nil { + t.Fatal(err) + } + startup := stream.StartupSnapshot() + status := newWHEPFeedStatus(startup.Generation, startup.LiveCursor, "realtime") + status.setExpectedMedia(test.video, test.audio) + + var videoSender, audioSender *TrackSender + var videoCapture, audioCapture *whepRTPCapture + if test.video { + videoSender, videoCapture = newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000}, 96) + } + if test.audio { + audioSender, audioCapture = newWHEPOverwriteSender(t, webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMA, ClockRate: 8000, Channels: 1}, 8) + } + + connected := make(chan struct{}) + close(connected) + done := make(chan struct{}) + feedDone := make(chan struct{}) + go func() { + defer close(feedDone) + whepFeedLoop(stream, startup, videoSender, audioSender, done, connected, "realtime", avframe.CodecG711A, nil, status) + }() + + stream.WriteFrame(avframe.NewAVFrame( + avframe.MediaTypeVideo, avframe.CodecVP8, avframe.FrameTypeKeyframe, + 40, 40, []byte{0x01}, + )) + stream.WriteFrame(avframe.NewAVFrame( + avframe.MediaTypeAudio, avframe.CodecG711A, avframe.FrameTypeInterframe, + 60, 60, []byte{0xd5}, + )) + + if test.video { + _ = videoCapture.readSample(t) + } else { + _ = audioCapture.readSample(t) + } + close(done) + select { + case <-feedDone: + case <-time.After(time.Second): + t.Fatal("WHEP feed did not stop") + } + + snapshot := status.Snapshot() + if snapshot.VideoFrames != test.wantVideoSent || snapshot.AudioFrames != test.wantAudioSent { + t.Fatalf("sent media counters = %+v", snapshot) + } + if snapshot.DroppedVideo != 0 || snapshot.DroppedAudio != 0 { + t.Fatalf("unrequested media counted as dropped: %+v", snapshot) + } + }) + } +} + +func TestWHEPParameterSetsRejectInvalidH264Configuration(t *testing.T) { + valid := buildTestAVCConfigPayload( + []byte{0x67, 0x42, 0x00, 0x1f, 0xe9, 0x40}, + []byte{0x68, 0xce, 0x38, 0x80}, + ) + if !whepParameterSetsReady(avframe.CodecH264, valid) { + t.Fatal("valid H.264 SPS/PPS configuration was rejected") + } + if whepParameterSetsReady(avframe.CodecH264, []byte{0x01}) { + t.Fatal("H.264 configuration without SPS/PPS was accepted") } } diff --git a/module/webrtc/whep_reader_pump_test.go b/module/webrtc/whep_reader_pump_test.go new file mode 100644 index 00000000..28c2a740 --- /dev/null +++ b/module/webrtc/whep_reader_pump_test.go @@ -0,0 +1,313 @@ +package webrtc + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/im-pingo/liveforge/pkg/util" +) + +func receiveWHEPReaderEvent(t *testing.T, events <-chan whepReaderEvent) whepReaderEvent { + t.Helper() + select { + case event, ok := <-events: + if !ok { + t.Fatal("WHEP reader event stream closed before an event") + } + acknowledgeWHEPReaderEvent(&event) + return event + case <-time.After(time.Second): + t.Fatal("timed out waiting for WHEP reader event") + return whepReaderEvent{} + } +} + +func TestWHEPReaderPumpEmitsOneAtomicOverwriteEvent(t *testing.T) { + ring := util.NewRingBuffer[*avframe.AVFrame](2) + reader := ring.NewReaderAt(0) + ctx, cancel := context.WithCancel(context.Background()) + events := make(chan whepReaderEvent, 1) + var terminal atomic.Uint32 + + first := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecVP8, avframe.FrameTypeKeyframe, 0, 0, []byte{0xa0}) + retained := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecVP8, avframe.FrameTypeInterframe, 20, 20, []byte{0xa1}) + live := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecVP8, avframe.FrameTypeInterframe, 40, 40, []byte{0xa2}) + ring.Write(first) + ring.Write(retained) + ring.Write(live) + go pumpWHEPReader(ctx, whepReaderSource, reader, events, &terminal) + + event := receiveWHEPReaderEvent(t, events) + if event.reader != whepReaderSource { + t.Fatalf("overwrite event reader = %q, want source", event.reader) + } + if event.terminal != whepReaderTerminalNone { + t.Fatalf("overwrite event terminal cause = %q, want none", event.terminal) + } + if event.result.Value != retained || event.result.Overwritten != 1 || !event.result.OK { + t.Fatalf("atomic overwrite result = %+v, want retained frame with one overwrite", event.result) + } + if got := reader.ReadCursor(); got != ring.WriteCursor() { + t.Fatalf("source cursor after overwrite event = %d, want live cursor %d", got, ring.WriteCursor()) + } + + cancel() + terminalEvent := receiveWHEPReaderEvent(t, events) + if terminalEvent.terminal != whepReaderTerminalCanceled { + t.Fatalf("canceled pump cause = %q, want canceled", terminalEvent.terminal) + } +} + +func TestWHEPReaderPumpDistinguishesEOFFromGenerationEnd(t *testing.T) { + tests := []struct { + name string + setup func(*util.RingBuffer[*avframe.AVFrame], context.CancelCauseFunc) + want whepReaderTerminalCause + }{ + { + name: "ring eof", + setup: func(ring *util.RingBuffer[*avframe.AVFrame], _ context.CancelCauseFunc) { + ring.Close() + }, + want: whepReaderTerminalEOF, + }, + { + name: "generation end", + setup: func(_ *util.RingBuffer[*avframe.AVFrame], cancel context.CancelCauseFunc) { + cancel(errWHEPReaderGenerationEnded) + }, + want: whepReaderTerminalGenerationEnded, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ring := util.NewRingBuffer[*avframe.AVFrame](1) + ctx, cancel := context.WithCancelCause(context.Background()) + events := make(chan whepReaderEvent, 1) + var terminal atomic.Uint32 + go pumpWHEPReader(ctx, whepReaderTargetAudio, ring.NewReaderAt(0), events, &terminal) + test.setup(ring, cancel) + + event := receiveWHEPReaderEvent(t, events) + if event.reader != whepReaderTargetAudio { + t.Fatalf("terminal event reader = %q, want target audio", event.reader) + } + if event.terminal != test.want { + t.Fatalf("terminal cause = %q, want %q", event.terminal, test.want) + } + if event.result.OK { + t.Fatalf("terminal event retained a media result: %+v", event.result) + } + if test.want == whepReaderTerminalGenerationEnded && !errors.Is(context.Cause(ctx), errWHEPReaderGenerationEnded) { + t.Fatalf("context cause = %v, want generation end", context.Cause(ctx)) + } + }) + } +} + +func TestWHEPFeedReadersPreserveIndependentReaderIdentity(t *testing.T) { + sourceRing := util.NewRingBuffer[*avframe.AVFrame](2) + targetRing := util.NewRingBuffer[*avframe.AVFrame](2) + sourceReader := sourceRing.NewReaderAt(0) + targetReader := targetRing.NewReaderAt(0) + readers := &whepFeedReaders{source: sourceReader, targetAudio: targetReader} + done := make(chan struct{}) + generationDone := make(chan struct{}) + readers.startWaiters(done, generationDone) + t.Cleanup(readers.Close) + + sourceFrame := avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecVP8, avframe.FrameTypeKeyframe, 0, 0, []byte{0xb0}) + targetFrame := avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecOpus, avframe.FrameTypeInterframe, 0, 0, []byte{0xc0}) + sourceRing.Write(sourceFrame) + targetRing.Write(targetFrame) + + var sourceEvent, targetEvent whepReaderEvent + var sourceOK, targetOK bool + for !sourceOK || !targetOK { + if !readers.wait(done, generationDone) { + t.Fatal("reader wait stopped before both independent events arrived") + } + if event, ok := readers.tryReadSource(); ok { + sourceEvent, sourceOK = event, true + } + if event, ok := readers.tryReadTargetAudio(); ok { + targetEvent, targetOK = event, true + } + } + + if sourceEvent.reader != whepReaderSource || sourceEvent.result.Value != sourceFrame { + t.Fatalf("source event = %+v, want source frame identity", sourceEvent) + } + if targetEvent.reader != whepReaderTargetAudio || targetEvent.result.Value != targetFrame { + t.Fatalf("target-audio event = %+v, want target frame identity", targetEvent) + } + if got := sourceReader.ReadCursor(); got != 1 { + t.Fatalf("source reader cursor = %d, want 1", got) + } + if got := targetReader.ReadCursor(); got != 1 { + t.Fatalf("target-audio reader cursor = %d, want 1", got) + } +} + +func TestWHEPFeedReadersCloseJoinsPumpsAndReleasesOnce(t *testing.T) { + sourceRing := util.NewRingBuffer[*avframe.AVFrame](1) + targetRing := util.NewRingBuffer[*avframe.AVFrame](1) + var releases int + readers := &whepFeedReaders{ + source: sourceRing.NewReaderAt(0), + targetAudio: targetRing.NewReaderAt(0), + release: func() { releases++ }, + } + readers.startWaiters(make(chan struct{}), make(chan struct{})) + + readers.Close() + readers.Close() + if releases != 1 { + t.Fatalf("target-audio release calls = %d, want 1", releases) + } + for _, events := range map[string]<-chan whepReaderEvent{ + "source": readers.sourceEvents, + "target_audio": readers.audioEvents, + } { + for { + _, ok := <-events + if !ok { + break + } + } + } +} + +func TestWHEPFeedReadersDoNotClassifyCancellationOrGenerationEndAsAudioEOF(t *testing.T) { + for _, test := range []struct { + name string + stop func(chan struct{}, chan struct{}) + }{ + { + name: "session cancellation", + stop: func(done, _ chan struct{}) { close(done) }, + }, + { + name: "generation end", + stop: func(_, generationDone chan struct{}) { close(generationDone) }, + }, + } { + t.Run(test.name, func(t *testing.T) { + readers := &whepFeedReaders{ + source: util.NewRingBuffer[*avframe.AVFrame](1).NewReaderAt(0), + targetAudio: util.NewRingBuffer[*avframe.AVFrame](1).NewReaderAt(0), + } + done := make(chan struct{}) + generationDone := make(chan struct{}) + readers.startWaiters(done, generationDone) + test.stop(done, generationDone) + readers.waitGroup.Wait() + if readers.activeTargetAudioEOF(done, generationDone) { + t.Fatal("lifecycle terminal cause was classified as target-audio EOF") + } + readers.Close() + }) + } +} + +func TestWHEPFeedReadersCloseUnblocksPermitAfterReady(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + readers := &whepFeedReaders{waitContext: ctx} + permit := make(chan struct{}) + done := make(chan bool, 1) + go func() { + done <- readers.grantRead(permit, nil, nil) + }() + time.Sleep(20 * time.Millisecond) + cancel() + + select { + case ok := <-done: + if ok { + t.Fatal("permit delivery succeeded after lifecycle cancellation") + } + case <-time.After(time.Second): + t.Fatal("permit delivery remained blocked after lifecycle cancellation") + } +} + +func TestWHEPFeedReaderFastPathPermitObservesLifecycleCancellation(t *testing.T) { + for _, test := range []struct { + name string + ready func(*whepFeedReaders) chan struct{} + try func(*whepFeedReaders) (whepReaderEvent, bool) + }{ + { + name: "source", + ready: func(readers *whepFeedReaders) chan struct{} { + return readers.sourceReady + }, + try: func(readers *whepFeedReaders) (whepReaderEvent, bool) { + return readers.tryReadSource() + }, + }, + { + name: "target audio", + ready: func(readers *whepFeedReaders) chan struct{} { + return readers.audioReady + }, + try: func(readers *whepFeedReaders) (whepReaderEvent, bool) { + return readers.tryReadTargetAudio() + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + done := make(chan struct{}) + readers := &whepFeedReaders{ + closed: true, + waitContext: context.Background(), + done: done, + generationDone: make(chan struct{}), + } + readers.sourceReady = make(chan struct{}) + readers.sourcePermit = make(chan struct{}) + readers.sourceEvents = make(chan whepReaderEvent) + readers.audioReady = make(chan struct{}) + readers.audioPermit = make(chan struct{}) + readers.audioEvents = make(chan whepReaderEvent) + close(test.ready(readers)) + + result := make(chan bool, 1) + go func() { + _, ok := test.try(readers) + result <- ok + }() + close(done) + select { + case ok := <-result: + if ok { + t.Fatal("fast-path read succeeded after lifecycle cancellation") + } + case <-time.After(time.Second): + t.Fatal("fast-path permit remained blocked after lifecycle cancellation") + } + }) + } +} + +func TestWHEPReaderPumpPrefersGenerationEndWhenRingAlsoEOF(t *testing.T) { + ring := util.NewRingBuffer[*avframe.AVFrame](1) + reader := ring.NewReaderAt(0) + ring.Close() + generationDone := make(chan struct{}) + close(generationDone) + events := make(chan whepReaderEvent, 1) + var terminal atomic.Uint32 + ctx := context.Background() + + pumpWHEPReaderGated(ctx, whepReaderTargetAudio, reader, events, &terminal, nil, nil, nil, generationDone) + event := receiveWHEPReaderEvent(t, events) + if event.terminal != whepReaderTerminalGenerationEnded { + t.Fatalf("terminal cause = %q, want generation end", event.terminal) + } +} diff --git a/module/webrtc/whip.go b/module/webrtc/whip.go index 61dac723..54e6f003 100644 --- a/module/webrtc/whip.go +++ b/module/webrtc/whip.go @@ -73,9 +73,9 @@ func (m *Module) handleWHIP(w http.ResponseWriter, r *http.Request) { SDP: string(offerBytes), } - pc, err := m.api.NewPeerConnection(webrtc.Configuration{ + pc, _, _, err := m.newPeerConnection(webrtc.Configuration{ ICEServers: m.iceServersFromConfig(), - }) + }, 0) if err != nil { releaseConn() http.Error(w, "failed to create peer connection", http.StatusInternalServerError) @@ -102,21 +102,27 @@ func (m *Module) handleWHIP(w http.ResponseWriter, r *http.Request) { sess := newSession(sessionID, pc, streamKey, "whip", m) var ( - videoDetected bool - audioDetected bool - publisherSet bool - pubMu sync.Mutex + videoDetected bool + audioDetected bool + publisherSet bool + publisherInstanceID uint64 + publisherGeneration uint64 + pubMu sync.Mutex ) mediaClock := newWHIPMediaClock() sess.setCleanup(func() { pubMu.Lock() wasPublisher := publisherSet + instanceID := publisherInstanceID + generation := publisherGeneration pubMu.Unlock() if wasPublisher { stream.RemovePublisherIf(pub) } lifecycleCtx := *publishCtx lifecycleCtx.PublisherID = pub.ID() + lifecycleCtx.StreamInstanceID = instanceID + lifecycleCtx.PublisherGeneration = generation sess.stopLifecycle(m.server.GetEventBus(), core.EventPublishStop, &lifecycleCtx) releaseConn() }) @@ -128,18 +134,31 @@ func (m *Module) handleWHIP(w http.ResponseWriter, r *http.Request) { setPublisherOnce := func() { pubMu.Lock() - defer pubMu.Unlock() if publisherSet || (!videoDetected && !audioDetected) || sess.isClosed() { + pubMu.Unlock() return } if err := stream.SetPublisher(pub); err != nil { + pubMu.Unlock() slog.Error("WHIP set publisher failed", "module", "webrtc", "error", err) return } - publisherSet = true + startup := stream.StartupSnapshot() + publisherInstanceID = startup.StreamInstanceID + publisherGeneration = startup.Generation lifecycleCtx := *publishCtx lifecycleCtx.PublisherID = pub.ID() - sess.startLifecycle(m.server.GetEventBus(), core.EventPublish, &lifecycleCtx) + lifecycleCtx.StreamInstanceID = publisherInstanceID + lifecycleCtx.PublisherGeneration = publisherGeneration + if !sess.startLifecycle(m.server.GetEventBus(), core.EventPublish, &lifecycleCtx) { + stream.RemovePublisherIf(pub) + pubMu.Unlock() + slog.Error("WHIP publish lifecycle admission failed", "module", "webrtc", "stream", streamKey) + sess.Close() + return + } + publisherSet = true + pubMu.Unlock() } pc.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { @@ -418,10 +437,12 @@ func mimeToCodecType(mime string) avframe.CodecType { // MediaInfo is stored behind an atomic pointer: OnTrack callbacks publish // updated snapshots while subscriber goroutines read concurrently. type WHIPPublisher struct { - id string - info atomic.Pointer[avframe.MediaInfo] - pc *webrtc.PeerConnection - done chan struct{} + id string + info atomic.Pointer[avframe.MediaInfo] + pc *webrtc.PeerConnection + done chan struct{} + closeOnce sync.Once + closeErr error } var _ core.Publisher = (*WHIPPublisher)(nil) @@ -429,10 +450,11 @@ var _ core.Publisher = (*WHIPPublisher)(nil) func (p *WHIPPublisher) ID() string { return p.id } func (p *WHIPPublisher) MediaInfo() *avframe.MediaInfo { return p.info.Load() } func (p *WHIPPublisher) Close() error { - select { - case <-p.done: - default: + p.closeOnce.Do(func() { close(p.done) - } - return p.pc.Close() + if p.pc != nil { + p.closeErr = p.pc.Close() + } + }) + return p.closeErr } diff --git a/pkg/audiocodec/codec.go b/pkg/audiocodec/codec.go index a94b7a47..6cd63b50 100644 --- a/pkg/audiocodec/codec.go +++ b/pkg/audiocodec/codec.go @@ -1,5 +1,37 @@ package audiocodec +import "errors" + +// ErrInvalidSourceSpan reports an invalid half-open source cursor interval. +var ErrInvalidSourceSpan = errors.New("invalid source span") + +// SourceSpan is a half-open interval in source-ring cursor space. +// The zero value is invalid. +type SourceSpan struct { + Begin int64 + End int64 +} + +// Valid reports whether the span contains source media. +func (s SourceSpan) Valid() bool { + return s.Begin < s.End +} + +// Union returns the smallest span covering both inputs. If either input is +// invalid, the result is invalid rather than attributing unrelated media. +func (s SourceSpan) Union(other SourceSpan) SourceSpan { + if !s.Valid() || !other.Valid() { + return SourceSpan{} + } + if other.Begin < s.Begin { + s.Begin = other.Begin + } + if other.End > s.End { + s.End = other.End + } + return s +} + // PCMFrame is the universal exchange format between all audio codecs. type PCMFrame struct { Samples []int16 // interleaved samples (L,R,L,R... or mono) @@ -27,6 +59,34 @@ type Encoder interface { Close() } +// AttributedPacket adds by-value source provenance to a compressed payload. +// Payload retains the ownership and backing storage of the encoder result. +type AttributedPacket struct { + Payload []byte + SourceSpan SourceSpan +} + +// AttributedEncoder is an opt-in source provenance extension for Encoder. +// One input may produce zero, one, or multiple packets. +type AttributedEncoder interface { + Encoder + EncodeAttributed(pcm *PCMFrame, sourceSpan SourceSpan) ([]AttributedPacket, error) +} + +// DrainingEncoder exposes delayed packets held by an encoder at a finite +// stream boundary. Drain is idempotent and returns each delayed packet once. +type DrainingEncoder interface { + Encoder + Drain() ([][]byte, error) +} + +// AttributedDrainingEncoder exposes source provenance on delayed packets. +type AttributedDrainingEncoder interface { + AttributedEncoder + DrainingEncoder + DrainAttributed() ([]AttributedPacket, error) +} + // SequenceHeaderFunc returns an initial sequence header frame for the // target codec, or nil if the codec does not use sequence headers. type SequenceHeaderFunc func() []byte @@ -37,3 +97,139 @@ type Resampler interface { Resample(pcm *PCMFrame) *PCMFrame Close() } + +// AttributedPCMFrame adds by-value source provenance to resampled PCM. An +// empty frame has an invalid SourceSpan. +type AttributedPCMFrame struct { + PCMFrame + SourceSpan SourceSpan +} + +// AttributedResampler is an opt-in source provenance extension for Resampler. +type AttributedResampler interface { + Resampler + ResampleAttributed(pcm *PCMFrame, sourceSpan SourceSpan) (*AttributedPCMFrame, error) +} + +// DrainingResampler exposes samples retained by a streaming resampler at a +// finite input boundary. Drain is idempotent and returns the terminal samples +// exactly once. +type DrainingResampler interface { + Resampler + Drain() *PCMFrame +} + +// AttributedDrainingResampler exposes source provenance on terminal samples. +type AttributedDrainingResampler interface { + AttributedResampler + DrainingResampler + DrainAttributed() (*AttributedPCMFrame, error) +} + +type sourceSpanSegment struct { + samples int64 + span SourceSpan +} + +// sourceSpanQueue tracks input samples per channel that may still contribute +// to future output. +type sourceSpanQueue struct { + segments []sourceSpanSegment + samples int64 +} + +func (q *sourceSpanQueue) append(samples int64, span SourceSpan) bool { + if samples <= 0 || !span.Valid() { + return false + } + q.segments = append(q.segments, sourceSpanSegment{samples: samples, span: span}) + q.samples += samples + return true +} + +func (q *sourceSpanQueue) span() SourceSpan { + if len(q.segments) == 0 { + return SourceSpan{} + } + span := q.segments[0].span + for _, segment := range q.segments[1:] { + span = span.Union(segment.span) + if !span.Valid() { + return SourceSpan{} + } + } + return span +} + +func (q *sourceSpanQueue) retainTail(samples int64) { + if samples <= 0 { + q.clear() + return + } + if samples >= q.samples { + return + } + + drop := q.samples - samples + droppedSegments := 0 + for droppedSegments < len(q.segments) && drop >= q.segments[droppedSegments].samples { + drop -= q.segments[droppedSegments].samples + droppedSegments++ + } + if droppedSegments > 0 { + copy(q.segments, q.segments[droppedSegments:]) + q.segments = q.segments[:len(q.segments)-droppedSegments] + } + if drop > 0 { + q.segments[0].samples -= drop + } + q.samples = samples +} + +func (q *sourceSpanQueue) clear() { + q.segments = nil + q.samples = 0 +} + +// ceilRetainedInputSamples converts an exact resampler delay to the number of +// whole input samples whose provenance must be retained. exactBase must be a +// common multiple of the input and output sample rates. +func ceilRetainedInputSamples(delay, exactBase int64, inputRate int) int64 { + if delay <= 0 || exactBase <= 0 || inputRate <= 0 { + return 0 + } + ticksPerInputSample := exactBase / int64(inputRate) + if ticksPerInputSample <= 0 { + return 0 + } + return 1 + (delay-1)/ticksPerInputSample +} + +func attributePackets(payloads [][]byte, span SourceSpan) ([]AttributedPacket, error) { + if len(payloads) == 0 { + return nil, nil + } + if !span.Valid() { + return nil, ErrInvalidSourceSpan + } + packets := make([]AttributedPacket, len(payloads)) + for i, payload := range payloads { + packets[i] = AttributedPacket{Payload: payload, SourceSpan: span} + } + return packets, nil +} + +func attributePCMFrame(frame *PCMFrame, span SourceSpan) (*AttributedPCMFrame, error) { + attributed := &AttributedPCMFrame{} + if frame != nil { + attributed.PCMFrame = *frame + } + if len(attributed.Samples) == 0 { + return attributed, nil + } + if !span.Valid() { + return nil, ErrInvalidSourceSpan + } + attributed.SourceSpan = span + return attributed, nil +} diff --git a/pkg/audiocodec/ff_encoder.go b/pkg/audiocodec/ff_encoder.go index 84d36c65..a65cb354 100644 --- a/pkg/audiocodec/ff_encoder.go +++ b/pkg/audiocodec/ff_encoder.go @@ -163,21 +163,68 @@ static int ff_encode(AVCodecContext *ctx, static int ff_encoder_frame_size(AVCodecContext *ctx) { return ctx->frame_size; } + +static int ff_encoder_send_eof(AVCodecContext *ctx) { + return avcodec_send_frame(ctx, NULL); +} + +// ff_encoder_receive returns 1 with one caller-owned packet, 0 for EAGAIN, 2 +// for EOF, or a negative FFmpeg/allocation error. +static int ff_encoder_receive(AVCodecContext *ctx, uint8_t **out, int *out_size) { + AVPacket *pkt = av_packet_alloc(); + if (!pkt) return AVERROR(ENOMEM); + + int ret = avcodec_receive_packet(ctx, pkt); + if (ret == AVERROR(EAGAIN)) { + av_packet_free(&pkt); + return 0; + } + if (ret == AVERROR_EOF) { + av_packet_free(&pkt); + return 2; + } + if (ret < 0) { + av_packet_free(&pkt); + return ret; + } + + uint8_t *buf = (uint8_t *)malloc(pkt->size); + if (!buf) { + av_packet_free(&pkt); + return AVERROR(ENOMEM); + } + memcpy(buf, pkt->data, pkt->size); + *out = buf; + *out_size = pkt->size; + av_packet_free(&pkt); + return 1; +} + +static int ff_encoder_again(void) { + return AVERROR(EAGAIN); +} */ import "C" import ( + "errors" "fmt" + "io" "log/slog" "unsafe" ) +var errFFmpegDrainAgain = errors.New("ffmpeg encoder drain needs receive") + // FFmpegEncoder encodes PCM into compressed audio using FFmpeg's C API. type FFmpegEncoder struct { - ctx *C.AVCodecContext - codecName string - sampleRate int - channels int + ctx *C.AVCodecContext + codecName string + sampleRate int + channels int + drainSent bool + drained bool + pendingSourceSpan SourceSpan } // NewFFmpegEncoder creates an encoder for the given FFmpeg codec name @@ -202,14 +249,37 @@ func NewFFmpegEncoder(codecName string, sampleRate, channels int) *FFmpegEncoder } func (e *FFmpegEncoder) Encode(pcm *PCMFrame) ([]byte, error) { + return e.encode(pcm, SourceSpan{}) +} + +// EncodeAttributed encodes PCM and attaches its source interval without +// copying the Go-owned compressed payload. +func (e *FFmpegEncoder) EncodeAttributed(pcm *PCMFrame, sourceSpan SourceSpan) ([]AttributedPacket, error) { + if !sourceSpan.Valid() { + return nil, ErrInvalidSourceSpan + } + payload, err := e.encode(pcm, sourceSpan) + if err != nil { + return nil, err + } + return e.attributeImmediatePackets([][]byte{payload}) +} + +func (e *FFmpegEncoder) encode(pcm *PCMFrame, sourceSpan SourceSpan) ([]byte, error) { if e.ctx == nil { return nil, fmt.Errorf("ffmpeg encoder %q: context not initialised", e.codecName) } + if e.drainSent { + return nil, fmt.Errorf("ffmpeg encoder %q: already draining", e.codecName) + } if len(pcm.Samples) == 0 { return nil, fmt.Errorf("ffmpeg encoder %q: empty PCM frame", e.codecName) } nbSamples := len(pcm.Samples) / pcm.Channels + if sourceSpan.Valid() { + e.trackSourceSpan(sourceSpan) + } var ( out *C.uint8_t @@ -230,6 +300,21 @@ func (e *FFmpegEncoder) Encode(pcm *PCMFrame) ([]byte, error) { return result, nil } +func (e *FFmpegEncoder) trackSourceSpan(sourceSpan SourceSpan) { + if !sourceSpan.Valid() { + return + } + if !e.pendingSourceSpan.Valid() { + e.pendingSourceSpan = sourceSpan + return + } + e.pendingSourceSpan = e.pendingSourceSpan.Union(sourceSpan) +} + +func (e *FFmpegEncoder) attributeImmediatePackets(payloads [][]byte) ([]AttributedPacket, error) { + return attributePackets(payloads, e.pendingSourceSpan) +} + func (e *FFmpegEncoder) SampleRate() int { return e.sampleRate } func (e *FFmpegEncoder) Channels() int { return e.channels } @@ -240,6 +325,128 @@ func (e *FFmpegEncoder) FrameSize() int { return int(C.ff_encoder_frame_size(e.ctx)) } +// Drain sends the terminal nil frame once and copies every delayed packet into +// Go-owned memory. Subsequent calls return no packets. +func (e *FFmpegEncoder) Drain() ([][]byte, error) { + if e.drained { + return nil, nil + } + if e.ctx == nil { + return nil, fmt.Errorf("ffmpeg encoder %q: context not initialised", e.codecName) + } + return e.drainWith(e.sendDrainEOF, e.receiveDrainPacket) +} + +// DrainAttributed returns every delayed packet with the conservative union of +// all source spans submitted through EncodeAttributed. +func (e *FFmpegEncoder) DrainAttributed() ([]AttributedPacket, error) { + if e.drained { + return nil, nil + } + if e.ctx == nil { + return nil, fmt.Errorf("ffmpeg encoder %q: context not initialised", e.codecName) + } + if !e.pendingSourceSpan.Valid() { + return nil, ErrInvalidSourceSpan + } + return e.drainAttributedWith(e.sendDrainEOF, e.receiveDrainPacket) +} + +func (e *FFmpegEncoder) drainAttributedWith( + sendEOF func() error, + receive func() ([]byte, error), +) ([]AttributedPacket, error) { + payloads, err := e.drainWith(sendEOF, receive) + packets, attributionErr := attributePackets(payloads, e.pendingSourceSpan) + if attributionErr != nil { + return nil, attributionErr + } + return packets, err +} + +func (e *FFmpegEncoder) sendDrainEOF() error { + ret := C.ff_encoder_send_eof(e.ctx) + if ret == C.ff_encoder_again() { + return errFFmpegDrainAgain + } + if ret < 0 { + return fmt.Errorf("drain send error %d", int(ret)) + } + return nil +} + +func (e *FFmpegEncoder) receiveDrainPacket() ([]byte, error) { + var ( + out *C.uint8_t + outSize C.int + ) + ret := C.ff_encoder_receive(e.ctx, &out, &outSize) + switch { + case ret == 0: + return nil, errFFmpegDrainAgain + case ret == 2: + return nil, io.EOF + case ret < 0: + return nil, fmt.Errorf("drain receive error %d", int(ret)) + } + + packet := make([]byte, int(outSize)) + copy(packet, unsafe.Slice((*byte)(unsafe.Pointer(out)), int(outSize))) + C.free(unsafe.Pointer(out)) + return packet, nil +} + +func (e *FFmpegEncoder) drainWith( + sendEOF func() error, + receive func() ([]byte, error), +) ([][]byte, error) { + if e.drained { + return nil, nil + } + + var packets [][]byte + for !e.drainSent { + err := sendEOF() + if err == nil { + e.drainSent = true + break + } + if !errors.Is(err, errFFmpegDrainAgain) { + return packets, fmt.Errorf("ffmpeg encoder %q: %w", e.codecName, err) + } + + received := false + for { + packet, receiveErr := receive() + if receiveErr == nil { + received = true + packets = append(packets, packet) + continue + } + if errors.Is(receiveErr, errFFmpegDrainAgain) { + if !received { + return packets, fmt.Errorf("ffmpeg encoder %q: terminal send and receive both returned EAGAIN", e.codecName) + } + break + } + return packets, fmt.Errorf("ffmpeg encoder %q: %w", e.codecName, receiveErr) + } + } + + for { + packet, err := receive() + if err == nil { + packets = append(packets, packet) + continue + } + if errors.Is(err, io.EOF) || errors.Is(err, errFFmpegDrainAgain) { + e.drained = true + return packets, nil + } + return packets, fmt.Errorf("ffmpeg encoder %q: %w", e.codecName, err) + } +} + func (e *FFmpegEncoder) Close() { if e.ctx != nil { C.avcodec_free_context(&e.ctx) diff --git a/pkg/audiocodec/ff_encoder_test.go b/pkg/audiocodec/ff_encoder_test.go index 267abb18..e250a4bd 100644 --- a/pkg/audiocodec/ff_encoder_test.go +++ b/pkg/audiocodec/ff_encoder_test.go @@ -2,7 +2,15 @@ package audiocodec -import "testing" +import ( + "bytes" + "errors" + "io" + "testing" +) + +var _ DrainingEncoder = (*FFmpegEncoder)(nil) +var _ AttributedDrainingEncoder = (*FFmpegEncoder)(nil) func TestFFmpegEncoderPCMU(t *testing.T) { enc := NewFFmpegEncoder("pcm_mulaw", 8000, 1) @@ -51,3 +59,277 @@ func TestFFmpegEncoderDecodeRoundTrip(t *testing.T) { } } } + +func TestFFmpegEncoderDrainReturnsDelayedAACPacketsExactlyOnce(t *testing.T) { + enc := NewFFmpegEncoder("aac", 48000, 2) + defer enc.Close() + + frameSize := enc.FrameSize() + if frameSize <= 0 { + t.Fatalf("AAC encoder frame size = %d, want fixed frame size", frameSize) + } + pcm := &PCMFrame{ + Samples: make([]int16, frameSize*enc.Channels()), + SampleRate: enc.SampleRate(), + Channels: enc.Channels(), + } + for i := range pcm.Samples { + pcm.Samples[i] = int16((i%257 - 128) * 128) + } + + beforeDrain, err := enc.Encode(pcm) + if err != nil { + t.Fatalf("encode AAC frame: %v", err) + } + if len(beforeDrain) == 0 { + t.Fatal("encode returned no primed AAC packet before terminal drain") + } + + drainer, ok := any(enc).(DrainingEncoder) + if !ok { + t.Fatal("FFmpeg encoder does not expose terminal drain") + } + delayed, err := drainer.Drain() + if err != nil { + t.Fatalf("drain AAC encoder: %v", err) + } + if len(delayed) == 0 { + t.Fatal("drain returned no delayed AAC packets") + } + seen := map[string]int{string(beforeDrain): -1} + for i, packet := range delayed { + if len(packet) == 0 { + t.Fatalf("drained AAC packet %d is empty", i) + } + if previous, duplicate := seen[string(packet)]; duplicate { + t.Fatalf("drained AAC packet %d duplicates packet %d; a missing packet could be masked by repeated output", i, previous) + } + seen[string(packet)] = i + } + + again, err := drainer.Drain() + if err != nil { + t.Fatalf("second drain: %v", err) + } + if len(again) != 0 { + t.Fatalf("second drain returned %d duplicate AAC packets, want 0", len(again)) + } + + enc.Close() + for i, packet := range delayed { + if len(packet) == 0 { + t.Fatalf("drained AAC packet %d was not retained in Go memory after close", i) + } + } +} + +func TestFFmpegEncoderDrainRetriesTerminalSendAfterEAGAIN(t *testing.T) { + enc := &FFmpegEncoder{codecName: "scripted-aac"} + sendCalls := 0 + sendEOF := func() error { + sendCalls++ + if sendCalls == 1 { + return errFFmpegDrainAgain + } + return nil + } + + type receiveStep struct { + packet []byte + err error + } + steps := []receiveStep{ + {packet: []byte{0x10}}, + {packet: []byte{0x20}}, + {err: errFFmpegDrainAgain}, + {packet: []byte{0x30}}, + {err: io.EOF}, + } + receiveCalls := 0 + receive := func() ([]byte, error) { + if receiveCalls >= len(steps) { + t.Fatal("drain received beyond scripted EOF") + } + step := steps[receiveCalls] + receiveCalls++ + return append([]byte(nil), step.packet...), step.err + } + + packets, err := enc.drainWith(sendEOF, receive) + if err != nil { + t.Fatalf("drain after send-side EAGAIN: %v", err) + } + want := [][]byte{{0x10}, {0x20}, {0x30}} + if len(packets) != len(want) { + t.Fatalf("drain packets = %d, want %d", len(packets), len(want)) + } + for i := range want { + if !bytes.Equal(packets[i], want[i]) { + t.Fatalf("drain packet[%d] = %x, want %x", i, packets[i], want[i]) + } + } + if sendCalls != 2 { + t.Fatalf("terminal send calls = %d, want one EAGAIN plus one successful submission", sendCalls) + } + if receiveCalls != len(steps) { + t.Fatalf("receive calls = %d, want all %d scripted results", receiveCalls, len(steps)) + } + if !enc.drainSent || !enc.drained { + t.Fatalf("drain state sent/drained = %v/%v, want true/true", enc.drainSent, enc.drained) + } + + again, err := enc.drainWith( + func() error { return errors.New("second terminal send") }, + func() ([]byte, error) { return nil, errors.New("second receive") }, + ) + if err != nil { + t.Fatalf("idempotent second drain: %v", err) + } + if len(again) != 0 { + t.Fatalf("idempotent second drain returned %d packets, want 0", len(again)) + } +} + +// Mutation caught: assigning encoder drain packets only the newest submitted +// span, or omitting attribution from the second packet of a multi-packet drain. +func TestFFmpegEncoderAttributedScriptedDrainUsesOutstandingUnionForEveryPacket(t *testing.T) { + enc := &FFmpegEncoder{codecName: "scripted-aac"} + enc.trackSourceSpan(SourceSpan{Begin: 10, End: 20}) + enc.trackSourceSpan(SourceSpan{Begin: 30, End: 40}) + + type receiveStep struct { + packet []byte + err error + } + steps := []receiveStep{ + {packet: []byte{0x10}}, + {packet: []byte{0x20}}, + {err: io.EOF}, + } + receiveCalls := 0 + receive := func() ([]byte, error) { + step := steps[receiveCalls] + receiveCalls++ + return step.packet, step.err + } + + packets, err := enc.drainAttributedWith(func() error { return nil }, receive) + if err != nil { + t.Fatalf("attributed scripted drain: %v", err) + } + wantSpan := SourceSpan{Begin: 10, End: 40} + if len(packets) != 2 { + t.Fatalf("attributed drain packets = %d, want 2", len(packets)) + } + for i, packet := range packets { + if packet.SourceSpan != wantSpan { + t.Fatalf("attributed drain packet %d span = %+v, want outstanding union %+v", i, packet.SourceSpan, wantSpan) + } + if len(packet.Payload) != 1 || packet.Payload[0] != byte((i+1)*0x10) { + t.Fatalf("attributed drain packet %d payload = %x", i, packet.Payload) + } + } +} + +// Mutation caught: attributing an immediate encoder packet to only the newest +// submission after an older accepted submission produced no packet. +func TestFFmpegEncoderAttributedDelayedImmediateUsesOutstandingUnion(t *testing.T) { + enc := &FFmpegEncoder{codecName: "scripted-aac"} + enc.trackSourceSpan(SourceSpan{Begin: 10, End: 20}) + + delayed, err := enc.attributeImmediatePackets(nil) + if err != nil { + t.Fatalf("attribute accepted zero-output submission: %v", err) + } + if len(delayed) != 0 { + t.Fatalf("accepted zero-output submission returned %d packets, want 0", len(delayed)) + } + + enc.trackSourceSpan(SourceSpan{Begin: 30, End: 40}) + payload := []byte{0x12, 0x34} + packets, err := enc.attributeImmediatePackets([][]byte{payload}) + if err != nil { + t.Fatalf("attribute delayed immediate packet: %v", err) + } + if len(packets) != 1 { + t.Fatalf("delayed immediate packets = %d, want 1", len(packets)) + } + wantSpan := SourceSpan{Begin: 10, End: 40} + if packets[0].SourceSpan != wantSpan { + t.Fatalf("delayed immediate span = %+v, want outstanding union %+v", packets[0].SourceSpan, wantSpan) + } + if len(packets[0].Payload) != len(payload) || &packets[0].Payload[0] != &payload[0] { + t.Fatal("delayed immediate attribution copied or changed the scripted payload") + } +} + +// Mutation caught: changing packet bytes/order/idempotency in the attributed +// path, or assigning an immediate packet an invalid/unrelated interval. +func TestFFmpegEncoderAttributedMatchesLegacyMediaAndDrainsOnce(t *testing.T) { + legacy := NewFFmpegEncoder("aac", 48000, 2) + defer legacy.Close() + attributed := NewFFmpegEncoder("aac", 48000, 2) + defer attributed.Close() + + frameSize := legacy.FrameSize() + if frameSize <= 0 || attributed.FrameSize() != frameSize { + t.Fatalf("AAC frame sizes legacy/attributed = %d/%d", frameSize, attributed.FrameSize()) + } + for frame := 0; frame < 3; frame++ { + pcm := &PCMFrame{ + Samples: make([]int16, frameSize*legacy.Channels()), + SampleRate: legacy.SampleRate(), + Channels: legacy.Channels(), + } + for i := range pcm.Samples { + pcm.Samples[i] = int16(((i+frame*31)%257 - 128) * 128) + } + span := SourceSpan{Begin: int64(100 + frame), End: int64(101 + frame)} + legacyPayload, err := legacy.Encode(pcm) + if err != nil { + t.Fatalf("legacy encode frame %d: %v", frame, err) + } + packets, err := attributed.EncodeAttributed(pcm, span) + if err != nil { + t.Fatalf("attributed encode frame %d: %v", frame, err) + } + if len(packets) != 1 { + t.Fatalf("attributed encode frame %d packets = %d, want 1", frame, len(packets)) + } + if !bytes.Equal(packets[0].Payload, legacyPayload) { + t.Fatalf("attributed encode frame %d payload differs from legacy", frame) + } + wantSpan := SourceSpan{Begin: 100, End: span.End} + if packets[0].SourceSpan != wantSpan || !packets[0].SourceSpan.Valid() { + t.Fatalf("attributed encode frame %d span = %+v, want conservative union %+v", frame, packets[0].SourceSpan, wantSpan) + } + } + + legacyTail, err := legacy.Drain() + if err != nil { + t.Fatalf("legacy drain: %v", err) + } + attributedTail, err := attributed.DrainAttributed() + if err != nil { + t.Fatalf("attributed drain: %v", err) + } + if len(attributedTail) != len(legacyTail) { + t.Fatalf("attributed/legacy drain packet counts = %d/%d", len(attributedTail), len(legacyTail)) + } + wantTailSpan := SourceSpan{Begin: 100, End: 103} + for i := range legacyTail { + if !bytes.Equal(attributedTail[i].Payload, legacyTail[i]) { + t.Fatalf("attributed drain packet %d differs from legacy", i) + } + if attributedTail[i].SourceSpan != wantTailSpan { + t.Fatalf("attributed drain packet %d span = %+v, want %+v", i, attributedTail[i].SourceSpan, wantTailSpan) + } + } + again, err := attributed.DrainAttributed() + if err != nil { + t.Fatalf("second attributed drain: %v", err) + } + if len(again) != 0 { + t.Fatalf("second attributed drain returned %d packets, want 0", len(again)) + } +} diff --git a/pkg/audiocodec/ff_resampler.go b/pkg/audiocodec/ff_resampler.go index 606f2d78..7296735b 100644 --- a/pkg/audiocodec/ff_resampler.go +++ b/pkg/audiocodec/ff_resampler.go @@ -5,6 +5,7 @@ package audiocodec /* #include #include +#include #include #include @@ -46,7 +47,8 @@ static int ff_resampler_open(int in_rate, int in_channels, static int ff_resample(SwrContext *ctx, const int16_t *in, int in_count, int out_channels, int in_rate, int out_rate, - int16_t **out) { + int16_t **out, + int64_t *retained_delay, int64_t *delay_base) { // Upper bound: input samples scaled by rate ratio, plus any delay // already buffered, plus padding. int64_t delay = swr_get_delay(ctx, (int64_t)in_rate); @@ -64,6 +66,38 @@ static int ff_resample(SwrContext *ctx, return got; } + *out = buf; + int64_t gcd = av_gcd((int64_t)in_rate, (int64_t)out_rate); + if (gcd <= 0) { + free(buf); + *out = NULL; + return -1; + } + int64_t exact_base = ((int64_t)in_rate / gcd) * (int64_t)out_rate; + *delay_base = exact_base; + *retained_delay = swr_get_delay(ctx, exact_base); + return got; +} + +// ff_resampler_drain returns samples retained by the resampling filter after +// finite input ends. The caller repeats this until zero and frees *out. +static int ff_resampler_drain(SwrContext *ctx, + int out_channels, int out_rate, + int16_t **out) { + int64_t delay = swr_get_delay(ctx, (int64_t)out_rate); + int64_t out_max = delay + 32; + if (out_max < 32) out_max = 32; + + int16_t *buf = (int16_t *)malloc((size_t)(out_max * out_channels) * sizeof(int16_t)); + if (!buf) return -1; + + uint8_t *out_data[1] = { (uint8_t *)buf }; + int got = swr_convert(ctx, out_data, (int)out_max, NULL, 0); + if (got <= 0) { + free(buf); + return got; + } + *out = buf; return got; } @@ -82,6 +116,8 @@ type FFmpegResampler struct { inRate int outRate int outChannels int + drained bool + sourceSpans sourceSpanQueue } // NewFFmpegResampler creates a resampler that converts from @@ -106,22 +142,56 @@ func NewFFmpegResampler(inRate, inChannels, outRate, outChannels int) *FFmpegRes // Resample converts pcm to the target sample-rate and channel layout. // Returns a new PCMFrame; the input is not modified. func (r *FFmpegResampler) Resample(pcm *PCMFrame) *PCMFrame { - if r.ctx == nil || len(pcm.Samples) == 0 { - return &PCMFrame{SampleRate: r.outRate, Channels: r.outChannels} + frame, _ := r.resample(pcm) + return frame +} + +// ResampleAttributed converts PCM and attributes output to every queued input +// span that may have contributed before measured retained-delay aging. +func (r *FFmpegResampler) ResampleAttributed(pcm *PCMFrame, sourceSpan SourceSpan) (*AttributedPCMFrame, error) { + if !sourceSpan.Valid() { + return nil, ErrInvalidSourceSpan + } + if r.ctx == nil || r.drained || len(pcm.Samples) == 0 { + return attributePCMFrame(&PCMFrame{SampleRate: r.outRate, Channels: r.outChannels}, SourceSpan{}) + } + + inCount := len(pcm.Samples) / pcm.Channels + r.sourceSpans.append(int64(inCount), sourceSpan) + contributors := r.sourceSpans.span() + frame, retainedInputSamples := r.resample(pcm) + if retainedInputSamples >= 0 { + r.sourceSpans.retainTail(retainedInputSamples) + } + if len(frame.Samples) == 0 { + return attributePCMFrame(frame, SourceSpan{}) + } + return attributePCMFrame(frame, contributors) +} + +func (r *FFmpegResampler) resample(pcm *PCMFrame) (*PCMFrame, int64) { + if r.ctx == nil || r.drained || len(pcm.Samples) == 0 { + return &PCMFrame{SampleRate: r.outRate, Channels: r.outChannels}, 0 } inCount := len(pcm.Samples) / pcm.Channels - var out *C.int16_t + var ( + out *C.int16_t + retainedDelay C.int64_t + delayBase C.int64_t + ) ret := C.ff_resample(r.ctx, (*C.int16_t)(unsafe.Pointer(&pcm.Samples[0])), C.int(inCount), C.int(r.outChannels), C.int(r.inRate), C.int(r.outRate), - &out) + &out, + &retainedDelay, + &delayBase) if ret < 0 { - return &PCMFrame{SampleRate: r.outRate, Channels: r.outChannels} + return &PCMFrame{SampleRate: r.outRate, Channels: r.outChannels}, -1 } defer C.free(unsafe.Pointer(out)) @@ -134,6 +204,50 @@ func (r *FFmpegResampler) Resample(pcm *PCMFrame) *PCMFrame { Samples: samples, SampleRate: r.outRate, Channels: r.outChannels, + }, ceilRetainedInputSamples(int64(retainedDelay), int64(delayBase), r.inRate) +} + +// Drain flushes all samples retained by the resampling filter and returns +// them in Go-owned memory. Subsequent calls return an empty frame. +func (r *FFmpegResampler) Drain() *PCMFrame { + result := r.drain() + r.sourceSpans.clear() + return result +} + +// DrainAttributed flushes retained samples with the union of their remaining +// source contributors. Repeated calls return an empty frame with invalid span. +func (r *FFmpegResampler) DrainAttributed() (*AttributedPCMFrame, error) { + contributors := r.sourceSpans.span() + result := r.drain() + r.sourceSpans.clear() + return attributePCMFrame(result, contributors) +} + +func (r *FFmpegResampler) drain() *PCMFrame { + result := &PCMFrame{SampleRate: r.outRate, Channels: r.outChannels} + if r.ctx == nil || r.drained { + return result + } + r.drained = true + + for { + var out *C.int16_t + ret := C.ff_resampler_drain( + r.ctx, + C.int(r.outChannels), + C.int(r.outRate), + &out, + ) + if ret <= 0 { + return result + } + + total := int(ret) * r.outChannels + start := len(result.Samples) + result.Samples = append(result.Samples, make([]int16, total)...) + copy(result.Samples[start:], unsafe.Slice((*int16)(unsafe.Pointer(out)), total)) + C.free(unsafe.Pointer(out)) } } @@ -143,4 +257,5 @@ func (r *FFmpegResampler) Close() { C.swr_free(&r.ctx) r.ctx = nil } + r.sourceSpans.clear() } diff --git a/pkg/audiocodec/ff_resampler_test.go b/pkg/audiocodec/ff_resampler_test.go index 1f4b8cc7..6fac4184 100644 --- a/pkg/audiocodec/ff_resampler_test.go +++ b/pkg/audiocodec/ff_resampler_test.go @@ -4,9 +4,54 @@ package audiocodec import ( "math" + "slices" "testing" ) +var _ DrainingResampler = (*FFmpegResampler)(nil) +var _ AttributedDrainingResampler = (*FFmpegResampler)(nil) + +func TestFFmpegResamplerDrainReturnsTerminalSamplesExactlyOnce(t *testing.T) { + input := make([]int16, 160) + for i := range input { + input[i] = int16((i*197)%20000 - 10000) + } + pcm := &PCMFrame{Samples: input, SampleRate: 8000, Channels: 1} + + r := NewFFmpegResampler(8000, 1, 48000, 1) + defer r.Close() + beforeDrain := r.Resample(pcm) + if len(beforeDrain.Samples) >= len(input)*6 { + t.Fatalf("streaming resample returned %d samples before drain, want fewer than terminal count %d", len(beforeDrain.Samples), len(input)*6) + } + + drainer, ok := any(r).(interface{ Drain() *PCMFrame }) + if !ok { + t.Fatal("FFmpeg resampler does not expose terminal drain") + } + tail := drainer.Drain() + wantTailSamples := len(input)*6 - len(beforeDrain.Samples) + if len(tail.Samples) != wantTailSamples { + t.Fatalf("resampler tail samples = %d, want %d", len(tail.Samples), wantTailSamples) + } + + nonZero := false + for _, sample := range tail.Samples { + if sample != 0 { + nonZero = true + break + } + } + if !nonZero { + t.Fatal("terminal resampler tail was replaced entirely by silence") + } + + again := drainer.Drain() + if len(again.Samples) != 0 { + t.Fatalf("second resampler drain returned %d duplicate samples, want 0", len(again.Samples)) + } +} + func TestFFmpegResampler8kTo48k(t *testing.T) { r := NewFFmpegResampler(8000, 1, 48000, 1) defer r.Close() @@ -44,7 +89,7 @@ func TestFFmpegResampler48kTo44k(t *testing.T) { r := NewFFmpegResampler(48000, 2, 44100, 2) defer r.Close() - pcm := &PCMFrame{Samples: make([]int16, 960 * 2), SampleRate: 48000, Channels: 2} + pcm := &PCMFrame{Samples: make([]int16, 960*2), SampleRate: 48000, Channels: 2} out := r.Resample(pcm) if out.SampleRate != 44100 { t.Fatalf("expected 44100, got %d", out.SampleRate) @@ -82,3 +127,117 @@ func TestFFmpegResamplerMonoToStereo(t *testing.T) { t.Fatalf("expected 320 samples, got %d", len(out.Samples)) } } + +// Mutation caught: dropping a zero-output input span, using only the newest +// span for later output, never aging the first span, or re-emitting a drain. +func TestFFmpegResamplerAttributedStreamingAgesMeasuredContributors(t *testing.T) { + r := NewFFmpegResampler(8000, 1, 48000, 1) + defer r.Close() + + sawZeroOutput := false + sawUnionAfterZero := false + sawFirstSpanAgeOut := false + for i := 0; i < 80; i++ { + span := SourceSpan{Begin: int64(i), End: int64(i + 1)} + out, err := r.ResampleAttributed(&PCMFrame{ + Samples: []int16{int16(i*257 - 10000)}, + SampleRate: 8000, + Channels: 1, + }, span) + if err != nil { + t.Fatalf("attributed resample input %d: %v", i, err) + } + if len(out.Samples) == 0 { + sawZeroOutput = true + if out.SourceSpan.Valid() { + t.Fatalf("zero-output input %d span = %+v, want invalid result metadata", i, out.SourceSpan) + } + continue + } + if !out.SourceSpan.Valid() { + t.Fatalf("non-empty output %d has invalid source span %+v", i, out.SourceSpan) + } + if out.SourceSpan.Begin > span.Begin || out.SourceSpan.End < span.End { + t.Fatalf("output %d span %+v does not cover current input %+v", i, out.SourceSpan, span) + } + if sawZeroOutput && out.SourceSpan.Begin == 0 && out.SourceSpan.End == span.End { + sawUnionAfterZero = true + } + if out.SourceSpan.Begin > 0 { + sawFirstSpanAgeOut = true + } + } + if !sawZeroOutput { + t.Fatal("fixture produced no zero-output streaming call") + } + if !sawUnionAfterZero { + t.Fatal("first output did not conservatively union retained zero-output and current contributors") + } + if !sawFirstSpanAgeOut { + t.Fatal("first source span never aged out under measured streaming delay") + } + + tail, err := r.DrainAttributed() + if err != nil { + t.Fatalf("attributed resampler drain: %v", err) + } + if len(tail.Samples) == 0 { + t.Fatal("attributed resampler drain returned no terminal samples") + } + if !tail.SourceSpan.Valid() || tail.SourceSpan.Begin == 0 || tail.SourceSpan.End != 80 { + t.Fatalf("terminal source span = %+v, want valid remaining tail ending at 80 with first span aged out", tail.SourceSpan) + } + again, err := r.DrainAttributed() + if err != nil { + t.Fatalf("second attributed resampler drain: %v", err) + } + if len(again.Samples) != 0 || again.SourceSpan.Valid() { + t.Fatalf("second attributed drain samples/span = %d/%+v, want empty/invalid", len(again.Samples), again.SourceSpan) + } +} + +// Mutation caught: attribution changing streaming or terminal sample content, +// ordering, ownership, or legacy drain behavior. +func TestFFmpegResamplerAttributedMatchesLegacySamples(t *testing.T) { + legacy := NewFFmpegResampler(8000, 1, 48000, 2) + defer legacy.Close() + attributed := NewFFmpegResampler(8000, 1, 48000, 2) + defer attributed.Close() + + for frame := 0; frame < 4; frame++ { + pcm := &PCMFrame{ + Samples: make([]int16, 160), + SampleRate: 8000, + Channels: 1, + } + for i := range pcm.Samples { + pcm.Samples[i] = int16(((i+frame*17)%211 - 105) * 127) + } + legacyOut := legacy.Resample(pcm) + attributedOut, err := attributed.ResampleAttributed( + pcm, + SourceSpan{Begin: int64(frame + 1), End: int64(frame + 2)}, + ) + if err != nil { + t.Fatalf("attributed resample frame %d: %v", frame, err) + } + if !slices.Equal(attributedOut.Samples, legacyOut.Samples) { + t.Fatalf("attributed resample frame %d samples differ from legacy", frame) + } + if len(attributedOut.Samples) > 0 && !attributedOut.SourceSpan.Valid() { + t.Fatalf("attributed resample frame %d has invalid span %+v", frame, attributedOut.SourceSpan) + } + } + + legacyTail := legacy.Drain() + attributedTail, err := attributed.DrainAttributed() + if err != nil { + t.Fatalf("attributed drain: %v", err) + } + if !slices.Equal(attributedTail.Samples, legacyTail.Samples) { + t.Fatal("attributed resampler terminal samples differ from legacy") + } + if len(attributedTail.Samples) > 0 && !attributedTail.SourceSpan.Valid() { + t.Fatalf("attributed terminal samples have invalid span %+v", attributedTail.SourceSpan) + } +} diff --git a/pkg/audiocodec/source_span_test.go b/pkg/audiocodec/source_span_test.go new file mode 100644 index 00000000..4256f806 --- /dev/null +++ b/pkg/audiocodec/source_span_test.go @@ -0,0 +1,143 @@ +package audiocodec + +import ( + "testing" + "unsafe" +) + +// Mutation caught: accepting a default/reversed interval or allowing Union to +// bless one invalid operand as an unrelated valid span. +func TestSourceSpanValidationAndConservativeUnion(t *testing.T) { + tests := []struct { + name string + span SourceSpan + want bool + }{ + {name: "default", span: SourceSpan{}, want: false}, + {name: "empty", span: SourceSpan{Begin: 7, End: 7}, want: false}, + {name: "reversed", span: SourceSpan{Begin: 8, End: 7}, want: false}, + {name: "negative valid", span: SourceSpan{Begin: -2, End: -1}, want: true}, + {name: "valid", span: SourceSpan{Begin: 7, End: 8}, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.span.Valid(); got != tt.want { + t.Fatalf("SourceSpan.Valid() = %v, want %v", got, tt.want) + } + }) + } + + left := SourceSpan{Begin: 10, End: 20} + right := SourceSpan{Begin: 18, End: 30} + if got := left.Union(right); got != (SourceSpan{Begin: 10, End: 30}) { + t.Fatalf("overlapping union = %+v, want [10,30)", got) + } + if got := right.Union(left); got != (SourceSpan{Begin: 10, End: 30}) { + t.Fatalf("reverse-order union = %+v, want [10,30)", got) + } + if got := left.Union(SourceSpan{}); got.Valid() { + t.Fatalf("union with default span = %+v, want invalid", got) + } + if got := (SourceSpan{}).Union(right); got.Valid() { + t.Fatalf("default span union = %+v, want invalid", got) + } +} + +// Mutation caught: clearing pending resampler spans on a zero-output call or +// attributing the later output to only the newest input span. +func TestSourceSpanQueueRetainsZeroOutputContributors(t *testing.T) { + var queue sourceSpanQueue + first := SourceSpan{Begin: 100, End: 101} + second := SourceSpan{Begin: 200, End: 201} + if !queue.append(8, first) { + t.Fatal("append first source span failed") + } + queue.retainTail(8) // measured zero-output state retains all input samples + if got := queue.span(); got != first { + t.Fatalf("zero-output pending span = %+v, want %+v", got, first) + } + if !queue.append(8, second) { + t.Fatal("append second source span failed") + } + if got := queue.span(); got != (SourceSpan{Begin: 100, End: 201}) { + t.Fatalf("later output contributors = %+v, want union [100,201)", got) + } +} + +// Mutation caught: retaining the first source span forever instead of aging +// segments using the measured post-conversion input-sample tail. +func TestSourceSpanQueueAgesOutOldSpanUsingMeasuredTail(t *testing.T) { + var queue sourceSpanQueue + first := SourceSpan{Begin: 100, End: 101} + second := SourceSpan{Begin: 200, End: 201} + queue.append(8, first) + queue.append(8, second) + + queue.retainTail(12) + if got := queue.span(); got != (SourceSpan{Begin: 100, End: 201}) { + t.Fatalf("partially retained old contributor = %+v, want union [100,201)", got) + } + queue.retainTail(4) + if got := queue.span(); got != second { + t.Fatalf("measured newest tail span = %+v, want old span aged out and %+v retained", got, second) + } + queue.retainTail(0) + if got := queue.span(); got.Valid() { + t.Fatalf("empty measured tail span = %+v, want invalid", got) + } +} + +// Mutation caught: rounding an exact fractional resampler delay down or to +// nearest input samples and aging the older span one sample too early. +func TestSourceSpanQueueCeilsFractionalRetainedDelayBeforeAging(t *testing.T) { + var queue sourceSpanQueue + first := SourceSpan{Begin: 100, End: 101} + second := SourceSpan{Begin: 200, End: 201} + queue.append(1, first) + queue.append(1, second) + + // LCM(32000, 48000) is 96000: one input sample is three exact + // delay ticks, so four ticks retain one whole sample plus a fraction. + retained := ceilRetainedInputSamples(4, 96000, 32000) + if retained != 2 { + t.Fatalf("fractional retained samples = %d, want conservative ceil 2", retained) + } + queue.retainTail(retained) + if got := queue.span(); got != (SourceSpan{Begin: 100, End: 201}) { + t.Fatalf("fractional tail span = %+v, want older contributor retained", got) + } + + retained = ceilRetainedInputSamples(3, 96000, 32000) + if retained != 1 { + t.Fatalf("integral retained samples = %d, want 1", retained) + } + queue.retainTail(retained) + if got := queue.span(); got != second { + t.Fatalf("integral tail span = %+v, want older contributor aged out and %+v retained", got, second) + } +} + +// Mutation caught: copying an already Go-owned compressed payload merely to +// attach by-value source metadata. +func TestAttributePacketsPreservesPayloadBacking(t *testing.T) { + first := []byte{0x10, 0x11, 0x12} + second := []byte{0x20, 0x21} + span := SourceSpan{Begin: 5, End: 9} + + packets, err := attributePackets([][]byte{first, second}, span) + if err != nil { + t.Fatalf("attribute packets: %v", err) + } + if len(packets) != 2 { + t.Fatalf("attributed packets = %d, want 2", len(packets)) + } + if packets[0].SourceSpan != span || packets[1].SourceSpan != span { + t.Fatalf("packet spans = %+v/%+v, want %+v", packets[0].SourceSpan, packets[1].SourceSpan, span) + } + if unsafe.SliceData(packets[0].Payload) != unsafe.SliceData(first) { + t.Fatal("first attributed payload does not share its input backing array") + } + if unsafe.SliceData(packets[1].Payload) != unsafe.SliceData(second) { + t.Fatal("second attributed payload does not share its input backing array") + } +} diff --git a/pkg/muxer/fmp4/media_segment.go b/pkg/muxer/fmp4/media_segment.go index 0b79dbcc..3d54f3d4 100644 --- a/pkg/muxer/fmp4/media_segment.go +++ b/pkg/muxer/fmp4/media_segment.go @@ -12,10 +12,10 @@ import ( // (typically the audio sample rate, e.g. 44100). Pass 0 to fall back to raw ms values. // Returns the concatenated moof+mdat bytes. func BuildMediaSegment(frames []*avframe.AVFrame, sequenceNumber uint32, audioTimescale uint32) []byte { - return buildMediaSegment(frames, sequenceNumber, audioTimescale, 0) + return buildMediaSegment(frames, sequenceNumber, audioTimescale, 0, 0, 0) } -func buildMediaSegment(frames []*avframe.AVFrame, sequenceNumber uint32, audioTimescale uint32, videoEndDTS int64) []byte { +func buildMediaSegment(frames []*avframe.AVFrame, sequenceNumber uint32, audioTimescale uint32, videoEndDTS, videoBaseDTS, audioBaseDTS int64) []byte { if len(frames) == 0 { return nil } @@ -80,12 +80,12 @@ func buildMediaSegment(frames []*avframe.AVFrame, sequenceNumber uint32, audioTi // Video traf if len(videoFrames) > 0 { - writeTraf(&moof, videoTrackID, videoFrames, timescaleVideo, videoEndDTS) + writeTraf(&moof, videoTrackID, videoFrames, timescaleVideo, videoEndDTS, videoBaseDTS) } // Audio traf — timescale must match the audio mdhd timescale (sample rate). if len(audioFrames) > 0 { - writeTraf(&moof, audioTrackID, audioFrames, audioTimescale, 0) + writeTraf(&moof, audioTrackID, audioFrames, audioTimescale, 0, audioBaseDTS) } moofBytes := moof.Bytes() @@ -117,7 +117,7 @@ func buildMediaSegment(frames []*avframe.AVFrame, sequenceNumber uint32, audioTi return buf.Bytes() } -func writeTraf(w *bytes.Buffer, trackID uint32, frames []*avframe.AVFrame, timescale uint32, endDTS int64) { +func writeTraf(w *bytes.Buffer, trackID uint32, frames []*avframe.AVFrame, timescale uint32, endDTS, baseDTS int64) { var traf bytes.Buffer // tfhd: track ID + default flags @@ -128,11 +128,12 @@ func writeTraf(w *bytes.Buffer, trackID uint32, frames []*avframe.AVFrame, times // tfdt: base media decode time if len(frames) > 0 { - var dts int64 + dts := frames[0].DTS - baseDTS + if dts < 0 { + dts = 0 + } if timescale > 0 { - dts = frames[0].DTS * int64(timescale) / 1000 - } else { - dts = frames[0].DTS + dts = dts * int64(timescale) / 1000 } tfdt := make([]byte, 8) binary.BigEndian.PutUint64(tfdt, uint64(dts)) diff --git a/pkg/muxer/fmp4/muxer.go b/pkg/muxer/fmp4/muxer.go index bc717369..1caa2fb8 100644 --- a/pkg/muxer/fmp4/muxer.go +++ b/pkg/muxer/fmp4/muxer.go @@ -1,6 +1,8 @@ package fmp4 import ( + "math" + "github.com/im-pingo/liveforge/pkg/avframe" "github.com/im-pingo/liveforge/pkg/codec/aac" "github.com/im-pingo/liveforge/pkg/codec/h265" @@ -10,7 +12,7 @@ import ( type Muxer struct { videoCodec avframe.CodecType audioCodec avframe.CodecType - audioSampleRate int + audioSampleRate uint32 sequenceNumber uint32 } @@ -33,7 +35,8 @@ func (m *Muxer) Init(videoSeqHeader, audioSeqHeader *avframe.AVFrame, width, hei audioData = audioSeqHeader.Payload } sampleRate, channels = resolveAudioConfig(m.audioCodec, audioData, sampleRate, channels) - m.audioSampleRate = sampleRate + sampleRate = boundedAudioSampleRate(sampleRate) + m.audioSampleRate = uint32(sampleRate) //nolint:gosec // bounded by boundedAudioSampleRate if width <= 0 || height <= 0 { if derivedWidth, derivedHeight := ParseVideoDimensions(m.videoCodec, videoData); derivedWidth > 0 && derivedHeight > 0 { width, height = derivedWidth, derivedHeight @@ -71,6 +74,16 @@ func resolveAudioConfig(codec avframe.CodecType, audioData []byte, sampleRate, c return sampleRate, channels } +func boundedAudioSampleRate(sampleRate int) int { + if sampleRate <= 0 { + return timescaleAudio + } + if sampleRate > math.MaxInt32 { + return math.MaxInt32 + } + return sampleRate +} + // ParseVideoDimensions extracts display dimensions from a codec configuration record. func ParseVideoDimensions(codec avframe.CodecType, config []byte) (width, height int) { switch codec { @@ -86,11 +99,18 @@ func ParseVideoDimensions(codec avframe.CodecType, config []byte) (width, height // WriteSegment generates a moof+mdat segment from a GOP or group of frames. func (m *Muxer) WriteSegment(frames []*avframe.AVFrame) []byte { m.sequenceNumber++ - return BuildMediaSegment(frames, m.sequenceNumber, uint32(m.audioSampleRate)) + return buildMediaSegment(frames, m.sequenceNumber, m.audioSampleRate, 0, 0, 0) +} + +// WriteSegmentWithBaseDTS writes a segment with independent per-track decode +// timestamp origins. Sample durations and composition offsets are unchanged. +func (m *Muxer) WriteSegmentWithBaseDTS(frames []*avframe.AVFrame, videoBaseDTS, audioBaseDTS int64) []byte { + m.sequenceNumber++ + return buildMediaSegment(frames, m.sequenceNumber, m.audioSampleRate, 0, videoBaseDTS, audioBaseDTS) } // WriteSegmentUntil generates a segment whose final video sample ends at endDTS. func (m *Muxer) WriteSegmentUntil(frames []*avframe.AVFrame, endDTS int64) []byte { m.sequenceNumber++ - return buildMediaSegment(frames, m.sequenceNumber, uint32(m.audioSampleRate), endDTS) + return buildMediaSegment(frames, m.sequenceNumber, m.audioSampleRate, endDTS, 0, 0) } diff --git a/pkg/muxer/mp4/muxer.go b/pkg/muxer/mp4/muxer.go index b28017c6..cdcc895a 100644 --- a/pkg/muxer/mp4/muxer.go +++ b/pkg/muxer/mp4/muxer.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "io" + "math" "github.com/im-pingo/liveforge/pkg/avframe" "github.com/im-pingo/liveforge/pkg/codec/aac" @@ -29,6 +30,8 @@ type Muxer struct { audioSampleRate uint32 audioChannels uint16 + prevVideoDTS int64 + prevAudioDTS int64 } type sampleEntry struct { @@ -47,6 +50,8 @@ func NewMuxer(videoCodec, audioCodec avframe.CodecType) *Muxer { timescale: 90000, audioSampleRate: 44100, audioChannels: 2, + prevVideoDTS: -1, + prevAudioDTS: -1, } } @@ -63,8 +68,8 @@ func (m *Muxer) SetAudioParams(sampleRate uint32, channels uint16) { // WriteFtyp writes the ftyp box. func (m *Muxer) WriteFtyp(w io.Writer) error { var buf bytes.Buffer - buf.Write([]byte("isom")) // major brand - putU32Buf(&buf, 0x00000200) // minor version + buf.Write([]byte("isom")) // major brand + putU32Buf(&buf, 0x00000200) // minor version buf.Write([]byte("isomiso2")) // compatible brands if m.videoCodec == avframe.CodecH264 { buf.Write([]byte("avc1")) @@ -90,7 +95,7 @@ func (m *Muxer) WriteMdatHeader(w io.WriteSeeker) (int64, error) { // WriteFrame appends a frame to the mdat region and records sample metadata. // Returns the number of bytes written. -func (m *Muxer) WriteFrame(w io.WriteSeeker, frame *avframe.AVFrame, prevDTS int64) (int, error) { +func (m *Muxer) WriteFrame(w io.WriteSeeker, frame *avframe.AVFrame) (int, error) { if frame.FrameType == avframe.FrameTypeSequenceHeader { if frame.MediaType.IsVideo() { m.videoCodec = frame.Codec @@ -126,18 +131,22 @@ func (m *Muxer) WriteFrame(w io.WriteSeeker, frame *avframe.AVFrame, prevDTS int m.mdatSize += int64(n) duration := uint32(0) - if prevDTS >= 0 { - d := frame.DTS - prevDTS - if d > 0 { - duration = uint32(d * int64(m.timescale) / 1000) - } + previousDTS := &m.prevVideoDTS + timescale := m.timescale + if frame.MediaType.IsAudio() { + previousDTS = &m.prevAudioDTS + timescale = m.audioSampleRate + } + if *previousDTS >= 0 { + d := frame.DTS - *previousDTS + duration = scaleDurationMillis(d, timescale) } entry := sampleEntry{ size: uint32(n), offset: offset, isSync: frame.FrameType.IsKeyframe() || frame.FrameType == avframe.FrameTypeSequenceHeader, - cts: int32((frame.PTS - frame.DTS) * int64(m.timescale) / 1000), + cts: scaleCompositionOffsetMillis(frame.PTS-frame.DTS, timescale), } if frame.MediaType.IsVideo() { @@ -146,11 +155,13 @@ func (m *Muxer) WriteFrame(w io.WriteSeeker, frame *avframe.AVFrame, prevDTS int } entry.isSync = frame.FrameType.IsKeyframe() m.videoSamples = append(m.videoSamples, entry) + m.prevVideoDTS = frame.DTS } else if frame.MediaType.IsAudio() { if len(m.audioSamples) > 0 { m.audioSamples[len(m.audioSamples)-1].duration = duration } m.audioSamples = append(m.audioSamples, entry) + m.prevAudioDTS = frame.DTS } return n, nil @@ -218,22 +229,22 @@ func (m *Muxer) totalDuration(samples []sampleEntry) uint64 { } func (m *Muxer) buildMvhd() []byte { - d := m.totalDuration(m.videoSamples) + d := scaleDurationUnits(m.totalDuration(m.videoSamples), m.timescale, m.timescale) if len(m.audioSamples) > 0 { - ad := m.totalDuration(m.audioSamples) + ad := scaleDurationUnits(m.totalDuration(m.audioSamples), m.audioSampleRate, m.timescale) if ad > d { d = ad } } buf := make([]byte, 100) - putU32(buf[0:4], 0) // version + flags - putU32(buf[4:8], 0) // creation time - putU32(buf[8:12], 0) // modification time + putU32(buf[0:4], 0) // version + flags + putU32(buf[4:8], 0) // creation time + putU32(buf[8:12], 0) // modification time putU32(buf[12:16], m.timescale) // timescale - putU32(buf[16:20], uint32(d)) // duration - putU32(buf[20:24], 0x00010000) // rate 1.0 - putU16(buf[24:26], 0x0100) // volume 1.0 + putU32(buf[16:20], d) // duration + putU32(buf[20:24], 0x00010000) // rate 1.0 + putU16(buf[24:26], 0x0100) // volume 1.0 // reserved + matrix + predefined copy(buf[26:], make([]byte, 10+36+24)) @@ -267,7 +278,7 @@ func (m *Muxer) buildTrak(isVideo bool) []byte { dur := m.totalDuration(samples) - tkhd := m.buildTkhd(trackID, uint32(dur), isVideo) + tkhd := m.buildTkhd(trackID, scaleDurationUnits(dur, ts, m.timescale), isVideo) writeFullBox(&buf, [4]byte{'t', 'k', 'h', 'd'}, 0, 3, tkhd) mdia := m.buildMdia(isVideo, ts, samples) @@ -278,8 +289,8 @@ func (m *Muxer) buildTrak(isVideo bool) []byte { func (m *Muxer) buildTkhd(trackID, duration uint32, isVideo bool) []byte { buf := make([]byte, 80) - putU32(buf[0:4], 0) // creation time - putU32(buf[4:8], 0) // modification time + putU32(buf[0:4], 0) // creation time + putU32(buf[4:8], 0) // modification time putU32(buf[8:12], trackID) // reserved 4 bytes putU32(buf[16:20], duration) @@ -307,7 +318,7 @@ func (m *Muxer) buildMdia(isVideo bool, timescale uint32, samples []sampleEntry) dur := m.totalDuration(samples) mdhd := make([]byte, 24) putU32(mdhd[8:12], timescale) - putU32(mdhd[12:16], uint32(dur)) + putU32(mdhd[12:16], clampUint64ToUint32(dur)) putU32(mdhd[16:20], 0x55C40000) // und language writeFullBox(&buf, [4]byte{'m', 'd', 'h', 'd'}, 0, 0, mdhd) @@ -372,9 +383,9 @@ func (m *Muxer) buildStbl(isVideo bool, samples []sampleEntry) []byte { writeFullBox(&buf, [4]byte{'s', 't', 't', 's'}, 0, 0, stts) if isVideo { - ctts := buildCtts(samples) + ctts, cttsVersion := buildCtts(samples) if ctts != nil { - writeFullBox(&buf, [4]byte{'c', 't', 't', 's'}, 0, 0, ctts) + writeFullBox(&buf, [4]byte{'c', 't', 't', 's'}, cttsVersion, 0, ctts) } stss := buildStss(samples) @@ -477,26 +488,26 @@ func buildEsds(asc []byte, sampleRate uint32) []byte { // ES_Descriptor ascLen := len(asc) - decConfigLen := 13 + 2 + ascLen - esLen := 3 + 2 + decConfigLen + 2 + 1 + decConfigLen := 13 + 1 + descriptorLengthWidth(ascLen) + ascLen + esLen := 3 + 1 + descriptorLengthWidth(decConfigLen) + decConfigLen + 3 - buf.WriteByte(0x03) // ES_DescrTag - buf.WriteByte(byte(esLen)) // length - putU16Buf(&buf, 1) // ES_ID - buf.WriteByte(0) // flags + buf.WriteByte(0x03) // ES_DescrTag + writeDescriptorLength(&buf, esLen) + putU16Buf(&buf, 1) // ES_ID + buf.WriteByte(0) // flags // DecoderConfigDescriptor - buf.WriteByte(0x04) // DecoderConfigDescrTag - buf.WriteByte(byte(decConfigLen)) - buf.WriteByte(0x40) // objectTypeIndication (AAC) - buf.WriteByte(0x15) // streamType (audio) + buf.WriteByte(0x04) // DecoderConfigDescrTag + writeDescriptorLength(&buf, decConfigLen) + buf.WriteByte(0x40) // objectTypeIndication (AAC) + buf.WriteByte(0x15) // streamType (audio) buf.Write([]byte{0x00, 0x00, 0x00}) // bufferSizeDB - putU32Buf(&buf, 0) // maxBitrate - putU32Buf(&buf, 0) // avgBitrate + putU32Buf(&buf, 0) // maxBitrate + putU32Buf(&buf, 0) // avgBitrate // DecoderSpecificInfo buf.WriteByte(0x05) - buf.WriteByte(byte(ascLen)) + writeDescriptorLength(&buf, ascLen) buf.Write(asc) // SLConfigDescriptor @@ -507,6 +518,87 @@ func buildEsds(asc []byte, sampleRate uint32) []byte { return buf.Bytes() } +func scaleDurationMillis(milliseconds int64, timescale uint32) uint32 { + if milliseconds <= 0 || timescale == 0 { + return 0 + } + maxMillis := int64(math.MaxUint32) * 1000 / int64(timescale) + if milliseconds > maxMillis { + return math.MaxUint32 + } + return uint32(milliseconds * int64(timescale) / 1000) //nolint:gosec // bounded by maxMillis +} + +func scaleCompositionOffsetMillis(milliseconds int64, timescale uint32) int32 { + if timescale == 0 { + return clampInt64ToInt32(milliseconds) + } + maxMillis := int64(math.MaxInt32) * 1000 / int64(timescale) + minMillis := int64(math.MinInt32) * 1000 / int64(timescale) + if milliseconds > maxMillis { + return math.MaxInt32 + } + if milliseconds < minMillis { + return math.MinInt32 + } + return int32(milliseconds * int64(timescale) / 1000) //nolint:gosec // bounded above +} + +func clampInt64ToInt32(value int64) int32 { + if value > math.MaxInt32 { + return math.MaxInt32 + } + if value < math.MinInt32 { + return math.MinInt32 + } + return int32(value) //nolint:gosec // bounded above +} + +func clampUint64ToUint32(value uint64) uint32 { + if value > math.MaxUint32 { + return math.MaxUint32 + } + return uint32(value) //nolint:gosec // bounded above +} + +func scaleDurationUnits(value uint64, sourceTimescale, targetTimescale uint32) uint32 { + if value == 0 || sourceTimescale == 0 || targetTimescale == 0 { + return 0 + } + source := uint64(sourceTimescale) + target := uint64(targetTimescale) + whole := value / source + if whole > uint64(math.MaxUint32)/target { + return math.MaxUint32 + } + scaled := whole * target + fraction := (value % source) * target / source + if scaled > uint64(math.MaxUint32)-fraction { + return math.MaxUint32 + } + return uint32(scaled + fraction) //nolint:gosec // bounded above +} + +func descriptorLengthWidth(value int) int { + width := 1 + for value >= 1<<7 && width < 4 { + value >>= 7 + width++ + } + return width +} + +func writeDescriptorLength(buf *bytes.Buffer, value int) { + width := descriptorLengthWidth(value) + for shift := (width - 1) * 7; shift >= 0; shift -= 7 { + encoded := byte((value >> shift) & 0x7f) //nolint:gosec // masked to seven bits + if shift > 0 { + encoded |= 0x80 + } + buf.WriteByte(encoded) + } +} + func buildStts(samples []sampleEntry) []byte { if len(samples) == 0 { buf := make([]byte, 4) @@ -537,16 +629,19 @@ func buildStts(samples []sampleEntry) []byte { return buf } -func buildCtts(samples []sampleEntry) []byte { +func buildCtts(samples []sampleEntry) ([]byte, uint8) { hasCTS := false + version := uint8(0) for _, s := range samples { if s.cts != 0 { hasCTS = true - break + } + if s.cts < 0 { + version = 1 } } if !hasCTS { - return nil + return nil, 0 } type cttsEntry struct { @@ -570,7 +665,7 @@ func buildCtts(samples []sampleEntry) []byte { putU32(buf[off:off+4], e.count) putU32(buf[off+4:off+8], uint32(e.offset)) } - return buf + return buf, version } func buildStss(samples []sampleEntry) []byte { @@ -595,9 +690,9 @@ func buildStss(samples []sampleEntry) []byte { func buildStsc(sampleCount int) []byte { // One chunk per sample (simplest approach) buf := make([]byte, 4+12) - putU32(buf[0:4], 1) // entry count - putU32(buf[4:8], 1) // first chunk - putU32(buf[8:12], 1) // samples per chunk + putU32(buf[0:4], 1) // entry count + putU32(buf[4:8], 1) // first chunk + putU32(buf[8:12], 1) // samples per chunk putU32(buf[12:16], 1) // sample description index return buf } diff --git a/pkg/muxer/mp4/muxer_test.go b/pkg/muxer/mp4/muxer_test.go index 6e2e3de3..0e8b149f 100644 --- a/pkg/muxer/mp4/muxer_test.go +++ b/pkg/muxer/mp4/muxer_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "io" + "math" "testing" "github.com/im-pingo/liveforge/pkg/avframe" @@ -67,18 +68,23 @@ func TestMuxerWriteAndFinalize(t *testing.T) { m := NewMuxer(avframe.CodecH264, avframe.CodecAAC) w := &memSeeker{} - m.WriteFtyp(w) - m.WriteMdatHeader(w) + if err := m.WriteFtyp(w); err != nil { + t.Fatal(err) + } + if _, err := m.WriteMdatHeader(w); err != nil { + t.Fatal(err) + } // Write video sequence header seqFrame := avframe.NewAVFrame( avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeSequenceHeader, 0, 0, []byte{0x01, 0x64, 0x00, 0x28, 0xFF, 0xE1, 0x00, 0x04, 0x67, 0x64, 0x00, 0x28, 0x01, 0x00, 0x04, 0x68, 0xEE, 0x3C, 0x80}, ) - m.WriteFrame(w, seqFrame, -1) + if _, err := m.WriteFrame(w, seqFrame); err != nil { + t.Fatal(err) + } // Write some video frames - var prevDTS int64 = -1 for i := range 5 { ft := avframe.FrameTypeInterframe if i == 0 { @@ -90,8 +96,9 @@ func TestMuxerWriteAndFinalize(t *testing.T) { avframe.MediaTypeVideo, avframe.CodecH264, ft, pts, dts, []byte{0x00, 0x00, 0x00, byte(i + 1), 0x65, 0x88}, ) - m.WriteFrame(w, frame, prevDTS) - prevDTS = dts + if _, err := m.WriteFrame(w, frame); err != nil { + t.Fatal(err) + } } if err := m.Finalize(w); err != nil { @@ -145,10 +152,188 @@ func TestMuxerWriteAndFinalize(t *testing.T) { func TestMuxerEmptyFinalize(t *testing.T) { m := NewMuxer(avframe.CodecH264, avframe.CodecAAC) w := &memSeeker{} - m.WriteFtyp(w) - m.WriteMdatHeader(w) + if err := m.WriteFtyp(w); err != nil { + t.Fatal(err) + } + if _, err := m.WriteMdatHeader(w); err != nil { + t.Fatal(err) + } if err := m.Finalize(w); err != nil { t.Fatalf("Finalize on empty: %v", err) } } + +func TestMuxerInterleavedTracksUseIndependentTimelines(t *testing.T) { + m := NewMuxer(avframe.CodecH264, avframe.CodecAAC) + w := &memSeeker{} + if err := m.WriteFtyp(w); err != nil { + t.Fatal(err) + } + if _, err := m.WriteMdatHeader(w); err != nil { + t.Fatal(err) + } + for _, frame := range []*avframe.AVFrame{ + avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeSequenceHeader, 0, 0, []byte{0x12, 0x10}), + avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 0, 0, []byte{0x01}), + avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, 0, 0, []byte{0x00, 0x00, 0x00, 0x01}), + avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecAAC, avframe.FrameTypeInterframe, 23, 23, []byte{0x02}), + avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, 40, 40, []byte{0x00, 0x00, 0x00, 0x02}), + } { + if _, err := m.WriteFrame(w, frame); err != nil { + t.Fatal(err) + } + } + + if got, want := m.audioSamples[0].duration, uint32(23*44100/1000); got != want { + t.Fatalf("first audio duration = %d, want %d", got, want) + } + if got, want := m.videoSamples[0].duration, uint32(40*90000/1000); got != want { + t.Fatalf("first video duration = %d, want %d", got, want) + } +} + +func TestMuxerSaturatesOutOfRangeSampleTimings(t *testing.T) { + m := NewMuxer(avframe.CodecH264, 0) + w := &memSeeker{} + frames := []*avframe.AVFrame{ + avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeKeyframe, + 0, 1<<40, []byte{0x01}), + avframe.NewAVFrame(avframe.MediaTypeVideo, avframe.CodecH264, avframe.FrameTypeInterframe, + 1<<40, 0, []byte{0x02}), + } + for _, frame := range frames { + if _, err := m.WriteFrame(w, frame); err != nil { + t.Fatal(err) + } + } + + if got := m.videoSamples[0].duration; got != math.MaxUint32 { + t.Fatalf("large sample duration = %d, want %d", got, uint32(math.MaxUint32)) + } + if got := m.videoSamples[0].cts; got != math.MaxInt32 { + t.Fatalf("large positive CTS = %d, want %d", got, int32(math.MaxInt32)) + } + if got := m.videoSamples[1].cts; got != math.MinInt32 { + t.Fatalf("large negative CTS = %d, want %d", got, int32(math.MinInt32)) + } +} + +func TestMuxerSaturatesVersionZeroMovieDuration(t *testing.T) { + m := NewMuxer(avframe.CodecH264, 0) + m.videoSamples = []sampleEntry{{duration: math.MaxUint32}, {duration: 1}} + + mvhd := m.buildMvhd() + if got := binary.BigEndian.Uint32(mvhd[16:20]); got != math.MaxUint32 { + t.Fatalf("movie duration = %d, want %d", got, uint32(math.MaxUint32)) + } + + track := m.buildTrak(true) + if got := binary.BigEndian.Uint32(track[28:32]); got != math.MaxUint32 { + t.Fatalf("track duration = %d, want %d", got, uint32(math.MaxUint32)) + } + tkhdSize := int(binary.BigEndian.Uint32(track[0:4])) + if got := binary.BigEndian.Uint32(track[tkhdSize+32 : tkhdSize+36]); got != math.MaxUint32 { + t.Fatalf("media duration = %d, want %d", got, uint32(math.MaxUint32)) + } +} + +func TestMuxerNormalizesMovieAndTrackDurationsToMovieTimescale(t *testing.T) { + m := NewMuxer(avframe.CodecH264, avframe.CodecAAC) + m.audioSampleRate = 48000 + m.videoSamples = []sampleEntry{{duration: 90000}} + m.audioSamples = []sampleEntry{{duration: 96000}} + + mvhd := m.buildMvhd() + if got, want := binary.BigEndian.Uint32(mvhd[16:20]), uint32(180000); got != want { + t.Fatalf("movie duration = %d, want %d", got, want) + } + + track := m.buildTrak(false) + if got, want := binary.BigEndian.Uint32(track[28:32]), uint32(180000); got != want { + t.Fatalf("audio track duration = %d, want %d", got, want) + } + tkhdSize := int(binary.BigEndian.Uint32(track[0:4])) + if got, want := binary.BigEndian.Uint32(track[tkhdSize+32:tkhdSize+36]), uint32(96000); got != want { + t.Fatalf("audio media duration = %d, want %d", got, want) + } +} + +func TestBuildEsdsUsesExpandableDescriptorLengths(t *testing.T) { + esds := buildEsds(make([]byte, 128), 44100) + length, width, ok := decodeDescriptorLength(esds[1:]) + if !ok { + t.Fatal("ES descriptor length is not valid expandable-size encoding") + } + if want := len(esds) - 1 - width; length != want { + t.Fatalf("ES descriptor length = %d, want %d", length, want) + } +} + +func TestBuildStblUsesSignedCTTSVersionForNegativeCompositionOffsets(t *testing.T) { + m := NewMuxer(avframe.CodecH264, 0) + stbl := m.buildStbl(true, []sampleEntry{ + {cts: 900}, + {cts: -450}, + }) + + version, payload := findFullBox(t, stbl, "ctts") + if version != 1 { + t.Fatalf("ctts version = %d, want 1 for signed composition offsets", version) + } + if got, want := binary.BigEndian.Uint32(payload[:4]), uint32(2); got != want { + t.Fatalf("ctts entry count = %d, want %d", got, want) + } + var got int32 + if err := binary.Read(bytes.NewReader(payload[16:20]), binary.BigEndian, &got); err != nil { + t.Fatalf("decode signed ctts offset: %v", err) + } + if want := int32(-450); got != want { + t.Fatalf("second ctts offset = %d, want %d", got, want) + } +} + +func TestBuildStblKeepsUnsignedCTTSVersionForNonNegativeCompositionOffsets(t *testing.T) { + m := NewMuxer(avframe.CodecH264, 0) + stbl := m.buildStbl(true, []sampleEntry{ + {cts: 0}, + {cts: 450}, + }) + + version, payload := findFullBox(t, stbl, "ctts") + if version != 0 { + t.Fatalf("ctts version = %d, want 0 for non-negative composition offsets", version) + } + if got, want := binary.BigEndian.Uint32(payload[16:20]), uint32(450); got != want { + t.Fatalf("second ctts offset = %d, want %d", got, want) + } +} + +func findFullBox(t *testing.T, boxes []byte, wantType string) (byte, []byte) { + t.Helper() + for offset := 0; offset+12 <= len(boxes); { + size := int(binary.BigEndian.Uint32(boxes[offset : offset+4])) + if size < 12 || offset+size > len(boxes) { + t.Fatalf("invalid box at offset %d with size %d", offset, size) + } + if string(boxes[offset+4:offset+8]) == wantType { + return boxes[offset+8], boxes[offset+12 : offset+size] + } + offset += size + } + t.Fatalf("box %q not found", wantType) + return 0, nil +} + +func decodeDescriptorLength(data []byte) (value, width int, ok bool) { + for index, b := range data { + if index == 4 { + return 0, 0, false + } + value = value<<7 | int(b&0x7f) + if b&0x80 == 0 { + return value, index + 1, true + } + } + return 0, 0, false +} diff --git a/pkg/muxer/ts/muxer.go b/pkg/muxer/ts/muxer.go index 8f8e291b..8c8775db 100644 --- a/pkg/muxer/ts/muxer.go +++ b/pkg/muxer/ts/muxer.go @@ -23,15 +23,17 @@ type Muxer struct { pat []byte pmt []byte + + tablesSent bool } // NewMuxer creates a TS muxer. videoSeqHeader/audioSeqHeader are the raw codec config data // (e.g., AVCDecoderConfigurationRecord for H.264, AudioSpecificConfig for AAC). func NewMuxer(videoCodec, audioCodec avframe.CodecType, videoSeqHeader, audioSeqHeader []byte) *Muxer { m := &Muxer{ - videoCodec: videoCodec, - audioCodec: audioCodec, - lastPCR: -1, + videoCodec: videoCodec, + audioCodec: audioCodec, + lastPCR: -1, } // Parse video sequence header into Annex-B format for prepending on keyframes @@ -99,13 +101,21 @@ func (m *Muxer) WriteFrame(frame *avframe.AVFrame) []byte { return nil } + var data []byte if frame.MediaType.IsVideo() { - return m.writeVideoFrame(frame) + data = m.writeVideoFrame(frame) + } else if frame.MediaType.IsAudio() { + data = m.writeAudioFrame(frame) + } + if len(data) == 0 { + return nil } - if frame.MediaType.IsAudio() { - return m.writeAudioFrame(frame) + if !m.tablesSent || (frame.MediaType.IsVideo() && frame.FrameType.IsKeyframe()) { + tables := m.rebuildPATandPMT() + m.tablesSent = true + return append(tables, data...) } - return nil + return data } func (m *Muxer) writeVideoFrame(frame *avframe.AVFrame) []byte { @@ -113,11 +123,6 @@ func (m *Muxer) writeVideoFrame(frame *avframe.AVFrame) []byte { estSize := len(frame.Payload)*2 + 1024 result := make([]byte, 0, estSize) - // Prepend PAT+PMT before keyframes - if frame.FrameType.IsKeyframe() { - result = append(result, m.rebuildPATandPMT()...) - } - // Build video payload var payload []byte @@ -198,10 +203,10 @@ func (m *Muxer) shouldInsertPCR(dts int64) bool { return dts-m.lastPCR >= MaxPCRInterval } - // WritePATAndPMT generates fresh PAT and PMT packets. // Used by LL-HLS to insert PAT/PMT at partial segment boundaries. func (m *Muxer) WritePATAndPMT() []byte { + m.tablesSent = true return m.rebuildPATandPMT() } diff --git a/pkg/muxer/ts/ts_test.go b/pkg/muxer/ts/ts_test.go index c17c0630..6f67b847 100644 --- a/pkg/muxer/ts/ts_test.go +++ b/pkg/muxer/ts/ts_test.go @@ -221,10 +221,15 @@ func TestMuxerWriteAudioFrame(t *testing.T) { t.Fatalf("result length %d not multiple of %d", len(result), PacketSize) } - // Should be on audio PID - pid := uint16(result[1]&0x1F)<<8 | uint16(result[2]) - if pid != PIDAudio { - t.Errorf("audio packet PID = 0x%04X, want 0x%04X", pid, PIDAudio) + if len(result) < 3*PacketSize { + t.Fatalf("audio-first output has %d packets, want PAT, PMT, and audio PES", len(result)/PacketSize) + } + for packet, wantPID := range []uint16{PIDPat, PIDPmt, PIDAudio} { + offset := packet * PacketSize + pid := uint16(result[offset+1]&0x1F)<<8 | uint16(result[offset+2]) + if pid != wantPID { + t.Errorf("packet %d PID = 0x%04X, want 0x%04X", packet, pid, wantPID) + } } } diff --git a/pkg/portalloc/portalloc.go b/pkg/portalloc/portalloc.go index fbfa3f6a..b68bb3b9 100644 --- a/pkg/portalloc/portalloc.go +++ b/pkg/portalloc/portalloc.go @@ -3,6 +3,7 @@ package portalloc import ( "fmt" + "net" "sync" ) @@ -14,6 +15,16 @@ type PortAllocator struct { maxPort int } +// BoundUDPPair is an allocated RTP/RTCP pair with both UDP sockets already +// bound. The caller owns the sockets and must close them before freeing the +// ports in the allocator. +type BoundUDPPair struct { + RTPPort int + RTCPPort int + RTPConn *net.UDPConn + RTCPConn *net.UDPConn +} + // New creates a PortAllocator for the range [minPort, maxPort]. func New(minPort, maxPort int) (*PortAllocator, error) { if minPort < 1 || maxPort > 65535 { @@ -46,10 +57,7 @@ func (pa *PortAllocator) Allocate() (int, error) { func (pa *PortAllocator) AllocatePair() (rtpPort, rtcpPort int, err error) { pa.mu.Lock() defer pa.mu.Unlock() - for p := pa.minPort; p <= pa.maxPort-1; p += 2 { - if p%2 != 0 { - continue - } + for p := pa.firstEvenPort(); p <= pa.maxPort-1; p += 2 { if !pa.used[p] && !pa.used[p+1] { pa.used[p] = true pa.used[p+1] = true @@ -59,6 +67,50 @@ func (pa *PortAllocator) AllocatePair() (rtpPort, rtcpPort int, err error) { return 0, 0, fmt.Errorf("no available port pairs in range %d-%d", pa.minPort, pa.maxPort) } +// AllocateBoundUDPPair atomically reserves a pair in the allocator and binds +// both sockets. Ports already occupied outside the allocator are skipped. +func (pa *PortAllocator) AllocateBoundUDPPair(network string, ip net.IP) (*BoundUDPPair, error) { + pa.mu.Lock() + defer pa.mu.Unlock() + + var lastBindErr error + for p := pa.firstEvenPort(); p <= pa.maxPort-1; p += 2 { + if pa.used[p] || pa.used[p+1] { + continue + } + rtpConn, err := net.ListenUDP(network, &net.UDPAddr{IP: ip, Port: p}) + if err != nil { + lastBindErr = err + continue + } + rtcpConn, err := net.ListenUDP(network, &net.UDPAddr{IP: ip, Port: p + 1}) + if err != nil { + lastBindErr = err + _ = rtpConn.Close() + continue + } + pa.used[p] = true + pa.used[p+1] = true + return &BoundUDPPair{ + RTPPort: p, + RTCPPort: p + 1, + RTPConn: rtpConn, + RTCPConn: rtcpConn, + }, nil + } + if lastBindErr != nil { + return nil, fmt.Errorf("no bindable UDP port pairs in range %d-%d: %w", pa.minPort, pa.maxPort, lastBindErr) + } + return nil, fmt.Errorf("no available port pairs in range %d-%d", pa.minPort, pa.maxPort) +} + +func (pa *PortAllocator) firstEvenPort() int { + if pa.minPort%2 == 0 { + return pa.minPort + } + return pa.minPort + 1 +} + // Free returns one or more ports to the pool. func (pa *PortAllocator) Free(ports ...int) { pa.mu.Lock() diff --git a/pkg/portalloc/portalloc_test.go b/pkg/portalloc/portalloc_test.go index 1d38458c..1041659d 100644 --- a/pkg/portalloc/portalloc_test.go +++ b/pkg/portalloc/portalloc_test.go @@ -1,6 +1,7 @@ package portalloc import ( + "net" "testing" ) @@ -99,6 +100,103 @@ func TestAllocatePair(t *testing.T) { _ = rtcp2 } +func TestPairAllocatorsStartAtFirstEvenPortWhenMinimumIsOdd(t *testing.T) { + pa, err := New(10001, 10004) + if err != nil { + t.Fatal(err) + } + rtpPort, rtcpPort, err := pa.AllocatePair() + if err != nil { + t.Fatal(err) + } + if rtpPort != 10002 || rtcpPort != 10003 { + t.Fatalf("allocated pair = %d/%d, want 10002/10003", rtpPort, rtcpPort) + } + + start, occupiedRTP, occupiedRTCP := reserveFirstOfTwoUDPPairs(t) + defer occupiedRTP.Close() + defer occupiedRTCP.Close() + boundAllocator, err := New(start+1, start+4) + if err != nil { + t.Fatal(err) + } + pair, err := boundAllocator.AllocateBoundUDPPair("udp4", net.ParseIP("127.0.0.1")) + if err != nil { + t.Fatal(err) + } + defer pair.RTPConn.Close() + defer pair.RTCPConn.Close() + if pair.RTPPort != start+2 || pair.RTCPPort != start+3 { + t.Fatalf("bound pair = %d/%d, want %d/%d", pair.RTPPort, pair.RTCPPort, start+2, start+3) + } +} + +func TestAllocateBoundUDPPairSkipsPortsOccupiedOutsideAllocator(t *testing.T) { + start, occupiedRTP, occupiedRTCP := reserveFirstOfTwoUDPPairs(t) + defer occupiedRTP.Close() + defer occupiedRTCP.Close() + + pa, err := New(start, start+3) + if err != nil { + t.Fatalf("New: %v", err) + } + pair, err := pa.AllocateBoundUDPPair("udp4", net.ParseIP("127.0.0.1")) + if err != nil { + t.Fatalf("AllocateBoundUDPPair: %v", err) + } + defer pair.RTPConn.Close() + defer pair.RTCPConn.Close() + if pair.RTPPort != start+2 || pair.RTCPPort != start+3 { + t.Fatalf("bound pair = %d/%d, want unoccupied pair %d/%d", pair.RTPPort, pair.RTCPPort, start+2, start+3) + } + if got := pair.RTPConn.LocalAddr().(*net.UDPAddr).Port; got != pair.RTPPort { + t.Fatalf("RTP socket port = %d, want %d", got, pair.RTPPort) + } + if got := pair.RTCPConn.LocalAddr().(*net.UDPAddr).Port; got != pair.RTCPPort { + t.Fatalf("RTCP socket port = %d, want %d", got, pair.RTCPPort) + } +} + +func reserveFirstOfTwoUDPPairs(t *testing.T) (int, *net.UDPConn, *net.UDPConn) { + t.Helper() + loopback := net.ParseIP("127.0.0.1") + for attempt := 0; attempt < 128; attempt++ { + probe, err := net.ListenUDP("udp4", &net.UDPAddr{IP: loopback}) + if err != nil { + t.Fatalf("probe UDP range: %v", err) + } + start := probe.LocalAddr().(*net.UDPAddr).Port + _ = probe.Close() + if start%2 != 0 { + start-- + } + if start < 1024 || start+3 > 65535 { + continue + } + + conns := make([]*net.UDPConn, 0, 4) + for port := start; port <= start+3; port++ { + conn, listenErr := net.ListenUDP("udp4", &net.UDPAddr{IP: loopback, Port: port}) + if listenErr != nil { + for _, opened := range conns { + _ = opened.Close() + } + conns = nil + break + } + conns = append(conns, conn) + } + if len(conns) != 4 { + continue + } + _ = conns[2].Close() + _ = conns[3].Close() + return start, conns[0], conns[1] + } + t.Fatal("could not reserve two consecutive UDP pairs") + return 0, nil, nil +} + func TestFreeOutOfRange(t *testing.T) { pa, _ := New(10000, 10010) // Should not panic diff --git a/pkg/ratelimit/ratelimit.go b/pkg/ratelimit/ratelimit.go index 4e38d264..e599c5a1 100644 --- a/pkg/ratelimit/ratelimit.go +++ b/pkg/ratelimit/ratelimit.go @@ -3,6 +3,7 @@ package ratelimit import ( "net" "net/http" + "strings" "sync" "time" ) @@ -14,6 +15,8 @@ type Limiter struct { mu sync.Mutex visitors map[string]*bucket stopCh chan struct{} + stopOnce sync.Once + trusted []*net.IPNet } type bucket struct { @@ -24,11 +27,21 @@ type bucket struct { // New creates a Limiter that allows rate requests/sec with the given burst size. // Starts a background goroutine to clean up stale entries. func New(rate float64, burst int) *Limiter { + return NewWithTrustedProxies(rate, burst, nil) +} + +// NewWithTrustedProxies creates a limiter that accepts forwarded client +// headers only when the direct peer belongs to a configured proxy network. +func NewWithTrustedProxies(rate float64, burst int, trustedProxies []string) *Limiter { + if burst <= 0 { + burst = 1 + } l := &Limiter{ rate: rate, burst: burst, visitors: make(map[string]*bucket), stopCh: make(chan struct{}), + trusted: parseTrustedProxies(trustedProxies), } go l.cleanup() return l @@ -65,7 +78,7 @@ func (l *Limiter) Allow(ip string) bool { // Wrap returns an http.Handler middleware that rate limits by client IP. func (l *Limiter) Wrap(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ip := extractIP(r) + ip := extractIPWithTrustedProxies(r, l.trusted) if !l.Allow(ip) { http.Error(w, "rate limit exceeded", http.StatusTooManyRequests) return @@ -76,12 +89,12 @@ func (l *Limiter) Wrap(next http.Handler) http.Handler { // AllowRequest applies the limiter to the request's resolved client address. func (l *Limiter) AllowRequest(r *http.Request) bool { - return l.Allow(extractIP(r)) + return l.Allow(extractIPWithTrustedProxies(r, l.trusted)) } // Close stops the background cleanup goroutine. func (l *Limiter) Close() { - close(l.stopCh) + l.stopOnce.Do(func() { close(l.stopCh) }) } func (l *Limiter) cleanup() { @@ -104,28 +117,71 @@ func (l *Limiter) cleanup() { } } -func extractIP(r *http.Request) string { - // Check X-Forwarded-For first (first entry). - if xff := r.Header.Get("X-Forwarded-For"); xff != "" { - if i := 0; i < len(xff) { - for j := 0; j < len(xff); j++ { - if xff[j] == ',' { - return xff[:j] - } +func extractIPWithTrustedProxies(r *http.Request, trusted []*net.IPNet) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host = r.RemoteAddr + } + peer := net.ParseIP(host) + if peer != nil && isTrustedProxy(peer, trusted) { + if value := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); value != "" { + if xff, ok := forwardedClientIP(value, trusted); ok { + return xff } - return xff + return host + } + if xri := net.ParseIP(strings.TrimSpace(r.Header.Get("X-Real-IP"))); xri != nil { + return xri.String() + } + } + return host +} + +func forwardedClientIP(value string, trusted []*net.IPNet) (string, bool) { + hops := strings.Split(value, ",") + leftmost := "" + for index := len(hops) - 1; index >= 0; index-- { + ip := net.ParseIP(strings.TrimSpace(hops[index])) + if ip == nil { + return "", false + } + leftmost = ip.String() + if !isTrustedProxy(ip, trusted) { + return leftmost, true } } + return leftmost, leftmost != "" +} - // Check X-Real-IP. - if xri := r.Header.Get("X-Real-IP"); xri != "" { - return xri +func parseTrustedProxies(values []string) []*net.IPNet { + result := make([]*net.IPNet, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if strings.Contains(value, "/") { + if _, network, err := net.ParseCIDR(value); err == nil { + result = append(result, network) + } + continue + } + if ip := net.ParseIP(value); ip != nil { + bits := 128 + if ip.To4() != nil { + bits = 32 + } + result = append(result, &net.IPNet{IP: ip, Mask: net.CIDRMask(bits, bits)}) + } } + return result +} - // Fall back to RemoteAddr. - host, _, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - return r.RemoteAddr +func isTrustedProxy(ip net.IP, trusted []*net.IPNet) bool { + for _, network := range trusted { + if network.Contains(ip) { + return true + } } - return host + return false } diff --git a/pkg/ratelimit/ratelimit_test.go b/pkg/ratelimit/ratelimit_test.go index 4b12c6d3..85cf6d84 100644 --- a/pkg/ratelimit/ratelimit_test.go +++ b/pkg/ratelimit/ratelimit_test.go @@ -78,7 +78,7 @@ func TestExtractIP(t *testing.T) { }{ {"remote addr", "192.168.1.1:12345", "", "", "192.168.1.1"}, {"x-forwarded-for single", "10.0.0.1:80", "203.0.113.50", "", "203.0.113.50"}, - {"x-forwarded-for chain", "10.0.0.1:80", "203.0.113.50, 70.41.3.18", "", "203.0.113.50"}, + {"x-forwarded-for chain", "10.0.0.1:80", "203.0.113.50, 70.41.3.18", "", "70.41.3.18"}, {"x-real-ip", "10.0.0.1:80", "", "198.51.100.178", "198.51.100.178"}, {"xff takes priority over xri", "10.0.0.1:80", "203.0.113.50", "198.51.100.178", "203.0.113.50"}, } @@ -93,10 +93,54 @@ func TestExtractIP(t *testing.T) { if tt.xri != "" { r.Header.Set("X-Real-IP", tt.xri) } - got := extractIP(r) + got := extractIPWithTrustedProxies(r, parseTrustedProxies([]string{"10.0.0.1/32"})) if got != tt.want { t.Errorf("extractIP() = %q, want %q", got, tt.want) } }) } } + +func TestExtractIPStripsTrustedProxyChainFromRight(t *testing.T) { + r := httptest.NewRequest("GET", "/", nil) + r.RemoteAddr = "10.0.0.3:1234" + r.Header.Set("X-Forwarded-For", "203.0.113.50, 10.0.0.1, 10.0.0.2") + + trusted := parseTrustedProxies([]string{"10.0.0.0/8"}) + if got := extractIPWithTrustedProxies(r, trusted); got != "203.0.113.50" { + t.Fatalf("multi-proxy client IP = %q, want first untrusted hop from the right", got) + } +} + +func TestLimiterCannotBypassTrustedProxyBucketWithAttackerControlledXFFPrefix(t *testing.T) { + limiter := NewWithTrustedProxies(0, 1, []string{"10.0.0.0/8"}) + defer limiter.Close() + + request := func(attackerPrefix string) *http.Request { + r := httptest.NewRequest("GET", "/", nil) + r.RemoteAddr = "10.0.0.2:1234" + r.Header.Set("X-Forwarded-For", attackerPrefix+", 203.0.113.50") + return r + } + if !limiter.AllowRequest(request("198.51.100.1")) { + t.Fatal("first request was unexpectedly limited") + } + if limiter.AllowRequest(request("198.51.100.2")) { + t.Fatal("changed attacker-controlled XFF prefix bypassed the existing client bucket") + } +} + +func TestExtractIPIgnoresForwardedHeadersFromUntrustedPeer(t *testing.T) { + r := httptest.NewRequest("GET", "/", nil) + r.RemoteAddr = "10.0.0.1:1234" + r.Header.Set("X-Forwarded-For", "203.0.113.50") + if got := extractIPWithTrustedProxies(r, nil); got != "10.0.0.1" { + t.Fatalf("untrusted forwarded address = %q, want remote peer", got) + } +} + +func TestLimiterCloseIsIdempotent(t *testing.T) { + l := New(1, 1) + l.Close() + l.Close() +} diff --git a/pkg/util/ringbuffer.go b/pkg/util/ringbuffer.go index 0e859431..aa521cb5 100644 --- a/pkg/util/ringbuffer.go +++ b/pkg/util/ringbuffer.go @@ -17,10 +17,29 @@ type RingBuffer[T any] struct { mu sync.Mutex // protects cond for Read() blocking cond *sync.Cond // wakes blocked Read() callers on Write/Close dataMu sync.RWMutex // protects buf slot access against concurrent read/write + testHooks *ringBufferTestHooks +} + +type ringBufferTestHooks struct { + beforeReadSlotLock func() + afterAdvanceCapture func() + writeSlotLockAttempted func(bool) +} + +// RingReadResult binds overwrite metadata to the value returned by one read. +type RingReadResult[T any] struct { + Value T + OK bool + Overwritten int64 } // NewRingBuffer creates a new ring buffer with the given capacity. func NewRingBuffer[T any](size int) *RingBuffer[T] { + if size <= 0 { + // Keep the low-level container safe for direct callers. Configuration + // validation still rejects this value so production streams fail closed. + size = 1 + } rb := &RingBuffer[T]{ buf: make([]T, size), size: int64(size), @@ -41,7 +60,16 @@ func (rb *RingBuffer[T]) Write(val T) { // contents (otherwise a reader could fetch a just-overwritten slot // before the cursor reveals the overwrite, breaking frame ordering). pos := rb.writeCursor.Load() - rb.dataMu.Lock() + if hooks := rb.testHooks; hooks != nil && hooks.writeSlotLockAttempted != nil { + if rb.dataMu.TryLock() { + hooks.writeSlotLockAttempted(false) + } else { + hooks.writeSlotLockAttempted(true) + rb.dataMu.Lock() + } + } else { + rb.dataMu.Lock() + } rb.buf[pos%rb.size] = val rb.writeCursor.Store(pos + 1) rb.dataMu.Unlock() @@ -118,7 +146,7 @@ func newRingReader[T any](rb *RingBuffer[T], pos int64) *RingReader[T] { type RingReader[T any] struct { rb *RingBuffer[T] readCursor atomic.Int64 - lastSkipped int64 + lastSkipped atomic.Int64 closed atomic.Bool // per-reader close flag contextMu sync.Mutex contextDone <-chan struct{} @@ -128,33 +156,46 @@ type RingReader[T any] struct { // Read returns the next value, blocking until data is available. // Returns (value, true) on success, or (zero, false) if the buffer or reader is closed and no data remains. func (r *RingReader[T]) Read() (T, bool) { - return r.readContext(context.Background()) + result := r.ReadResult() + return result.Value, result.OK } // ReadContext returns the next value, blocking until data is available, the // reader or buffer is closed, or ctx is cancelled. Unlike Signal, the wait is // scoped to this reader and cannot be consumed by another consumer. func (r *RingReader[T]) ReadContext(ctx context.Context) (T, bool) { + result := r.ReadResultContext(ctx) + return result.Value, result.OK +} + +// ReadResult returns the next value and its overwrite metadata, blocking until +// data is available or the buffer or reader is closed. +func (r *RingReader[T]) ReadResult() RingReadResult[T] { + return r.readResultContext(context.Background()) +} + +// ReadResultContext returns the next value and its overwrite metadata, +// blocking until data is available, the reader or buffer is closed, or ctx is +// cancelled. +func (r *RingReader[T]) ReadResultContext(ctx context.Context) RingReadResult[T] { if ctx == nil { panic("nil context") } - return r.readContext(ctx) + return r.readResultContext(ctx) } -func (r *RingReader[T]) readContext(ctx context.Context) (T, bool) { +func (r *RingReader[T]) readResultContext(ctx context.Context) RingReadResult[T] { if r.closed.Load() || contextCanceled(ctx) { - var zero T - return zero, false + return RingReadResult[T]{} } - if val, ok := r.TryRead(); ok { - return val, true + if result := r.TryReadResult(); result.OK { + return result } r.ensureContextWake(ctx) for { if r.closed.Load() || contextCanceled(ctx) { - var zero T - return zero, false + return RingReadResult[T]{} } r.rb.mu.Lock() for r.readCursor.Load() >= r.rb.writeCursor.Load() && @@ -164,15 +205,13 @@ func (r *RingReader[T]) readContext(ctx context.Context) (T, bool) { r.rb.mu.Unlock() if contextCanceled(ctx) { - var zero T - return zero, false + return RingReadResult[T]{} } - if val, ok := r.TryRead(); ok { - return val, true + if result := r.TryReadResult(); result.OK { + return result } if r.rb.closed.Load() || r.closed.Load() { - var zero T - return zero, false + return RingReadResult[T]{} } } } @@ -259,24 +298,34 @@ func (r *RingReader[T]) Signal() <-chan struct{} { // TryRead attempts a non-blocking read. Returns (value, false) if no data available. func (r *RingReader[T]) TryRead() (T, bool) { - r.lastSkipped = 0 + result := r.TryReadResult() + return result.Value, result.OK +} + +// TryReadResult attempts a non-blocking read and returns overwrite metadata +// from the same operation as the value. +func (r *RingReader[T]) TryReadResult() RingReadResult[T] { + r.lastSkipped.Store(0) + var result RingReadResult[T] for { wc := r.rb.writeCursor.Load() readCursor := r.readCursor.Load() if readCursor >= wc { - var zero T - return zero, false + return result } // Check if our position was overwritten (reader too slow) oldest := wc - r.rb.size if readCursor < oldest { - r.lastSkipped += oldest - readCursor + result.Overwritten += oldest - readCursor readCursor = oldest r.readCursor.Store(readCursor) } + if hooks := r.rb.testHooks; hooks != nil && hooks.beforeReadSlotLock != nil { + hooks.beforeReadSlotLock() + } r.rb.dataMu.RLock() val := r.rb.buf[readCursor%r.rb.size] // Re-check under the lock: if the writer lapped us between loading @@ -290,14 +339,35 @@ func (r *RingReader[T]) TryRead() (T, bool) { } r.readCursor.Store(readCursor + 1) - return val, true + result.Value = val + result.OK = true + r.lastSkipped.Store(result.Overwritten) + return result } } // Skipped returns the number of frames skipped in the last TryRead call // due to the reader being too slow (ring buffer overwrite). func (r *RingReader[T]) Skipped() int64 { - return r.lastSkipped + return r.lastSkipped.Load() +} + +// AdvanceToLive discards unread positions through a captured write cursor. +// Values written after that cursor remain available to the reader. +func (r *RingReader[T]) AdvanceToLive() int64 { + r.rb.dataMu.RLock() + defer r.rb.dataMu.RUnlock() + + writeCursor := r.rb.writeCursor.Load() + if hooks := r.rb.testHooks; hooks != nil && hooks.afterAdvanceCapture != nil { + hooks.afterAdvanceCapture() + } + readCursor := r.readCursor.Load() + if readCursor >= writeCursor { + return 0 + } + r.readCursor.Store(writeCursor) + return writeCursor - readCursor } // Lag returns the fraction of the ring buffer capacity that the reader trails behind the writer. diff --git a/pkg/util/ringbuffer_test.go b/pkg/util/ringbuffer_test.go index 5b319b8b..592d89c7 100644 --- a/pkg/util/ringbuffer_test.go +++ b/pkg/util/ringbuffer_test.go @@ -2,6 +2,7 @@ package util import ( "context" + "strconv" "testing" "time" ) @@ -32,6 +33,22 @@ func TestRingBufferWriteRead(t *testing.T) { } } +func TestRingBufferInvalidCapacityFallsBackToOne(t *testing.T) { + for _, size := range []int{0, -1} { + t.Run(strconv.Itoa(size), func(t *testing.T) { + rb := NewRingBuffer[int](size) + rb.Write(42) + reader := rb.NewReader() + if got, ok := reader.TryRead(); !ok || got != 42 { + t.Fatalf("capacity %d read = (%d, %v), want (42, true)", size, got, ok) + } + if reader.Lag() != 0 { + t.Fatalf("capacity %d lag = %v, want zero", size, reader.Lag()) + } + }) + } +} + func TestRingBufferOverflow(t *testing.T) { rb := NewRingBuffer[int](4) // Write 6 items into size-4 buffer — oldest 2 should be overwritten @@ -46,6 +63,362 @@ func TestRingBufferOverflow(t *testing.T) { } } +func TestRingReaderReadReportsOverwriteAtomically(t *testing.T) { + t.Run("retry-accumulation", func(t *testing.T) { + rb := NewRingBuffer[int](2) + reader := rb.NewReaderAt(0) + for _, value := range []int{10, 20, 30, 40} { + rb.Write(value) + } + + firstSlotAttempt := make(chan struct{}) + resumeRead := make(chan struct{}) + hookCalls := 0 + rb.testHooks = &ringBufferTestHooks{ + beforeReadSlotLock: func() { + hookCalls++ + if hookCalls == 1 { + close(firstSlotAttempt) + <-resumeRead + } + }, + } + + result := make(chan RingReadResult[int], 1) + go func() { + result <- reader.TryReadResult() + }() + + select { + case <-firstSlotAttempt: + case <-time.After(time.Second): + t.Fatal("TryReadResult did not reach the first slot acquisition boundary") + } + rb.Write(50) + rb.Write(60) + close(resumeRead) + + var got RingReadResult[int] + select { + case got = <-result: + case <-time.After(time.Second): + t.Fatal("TryReadResult did not complete after retry release") + } + if hookCalls != 2 { + t.Fatalf("slot acquisition attempts = %d, want 2", hookCalls) + } + if !got.OK || got.Value != 50 || got.Overwritten != 4 { + t.Fatalf("TryReadResult = %+v, want {Value:50 OK:true Overwritten:4}", got) + } + + got = reader.TryReadResult() + if !got.OK || got.Value != 60 || got.Overwritten != 0 { + t.Fatalf("next TryReadResult = %+v, want {Value:60 OK:true Overwritten:0}", got) + } + }) + + t.Run("blocking", func(t *testing.T) { + rb := NewRingBuffer[int](2) + reader := rb.NewReaderAt(0) + for _, value := range []int{10, 20, 30, 40} { + rb.Write(value) + } + + got := reader.ReadResult() + if !got.OK || got.Value != 30 || got.Overwritten != 2 { + t.Fatalf("ReadResult = %+v, want {Value:30 OK:true Overwritten:2}", got) + } + + got = reader.ReadResult() + if !got.OK || got.Value != 40 || got.Overwritten != 0 { + t.Fatalf("next ReadResult = %+v, want {Value:40 OK:true Overwritten:0}", got) + } + }) + + t.Run("context", func(t *testing.T) { + rb := NewRingBuffer[int](2) + reader := rb.NewReaderAt(0) + for _, value := range []int{10, 20, 30, 40} { + rb.Write(value) + } + + got := reader.ReadResultContext(context.Background()) + if !got.OK || got.Value != 30 || got.Overwritten != 2 { + t.Fatalf("ReadResultContext = %+v, want {Value:30 OK:true Overwritten:2}", got) + } + + got = reader.ReadResultContext(context.Background()) + if !got.OK || got.Value != 40 || got.Overwritten != 0 { + t.Fatalf("next ReadResultContext = %+v, want {Value:40 OK:true Overwritten:0}", got) + } + }) +} + +func TestRingReaderLegacySkippedCompatibility(t *testing.T) { + legacyReads := []struct { + name string + read func(*RingReader[int]) (int, bool) + }{ + {name: "TryRead", read: func(reader *RingReader[int]) (int, bool) { + return reader.TryRead() + }}, + {name: "Read", read: func(reader *RingReader[int]) (int, bool) { + return reader.Read() + }}, + {name: "ReadContext", read: func(reader *RingReader[int]) (int, bool) { + return reader.ReadContext(context.Background()) + }}, + } + + for _, test := range legacyReads { + t.Run(test.name, func(t *testing.T) { + rb := NewRingBuffer[int](2) + reader := rb.NewReaderAt(0) + for _, value := range []int{10, 20, 30, 40} { + rb.Write(value) + } + + value, ok := test.read(reader) + if !ok || value != 30 { + t.Fatalf("first legacy read = (%d, %v), want (30, true)", value, ok) + } + if skipped := reader.Skipped(); skipped != 2 { + t.Fatalf("Skipped after overwrite = %d, want 2", skipped) + } + + value, ok = test.read(reader) + if !ok || value != 40 { + t.Fatalf("next legacy read = (%d, %v), want (40, true)", value, ok) + } + if skipped := reader.Skipped(); skipped != 0 { + t.Fatalf("Skipped after continuous read = %d, want 0", skipped) + } + }) + } + + t.Run("pre-cancel-preserves-nonzero", func(t *testing.T) { + rb := NewRingBuffer[int](2) + reader := rb.NewReaderAt(0) + for _, value := range []int{10, 20, 30, 40} { + rb.Write(value) + } + if value, ok := reader.TryRead(); !ok || value != 30 { + t.Fatalf("seed TryRead = (%d, %v), want (30, true)", value, ok) + } + if skipped := reader.Skipped(); skipped != 2 { + t.Fatalf("seed Skipped = %d, want 2", skipped) + } + cursor := reader.ReadCursor() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, ok := reader.ReadContext(ctx); ok { + t.Fatal("pre-cancelled ReadContext returned a value") + } + if skipped := reader.Skipped(); skipped != 2 { + t.Fatalf("Skipped after pre-cancelled read = %d, want preserved value 2", skipped) + } + if got := reader.ReadCursor(); got != cursor { + t.Fatalf("cursor after pre-cancelled read = %d, want %d", got, cursor) + } + }) +} + +func TestRingReaderSkippedConcurrentObserver(t *testing.T) { + rb := NewRingBuffer[int](1) + reader := rb.NewReaderAt(0) + rb.Write(0) + rb.Write(1) + if _, ok := reader.TryRead(); !ok || reader.Skipped() != 1 { + t.Fatal("failed to seed a nonzero compatibility skip") + } + + const iterations = 2000 + start := make(chan struct{}) + readDone := make(chan bool, 1) + observeDone := make(chan int64, 1) + go func() { + <-start + for i := range iterations { + rb.Write(i*2 + 2) + rb.Write(i*2 + 3) + if _, ok := reader.TryRead(); !ok { + readDone <- false + return + } + } + readDone <- true + }() + go func() { + <-start + var observed int64 + for range iterations { + observed += reader.Skipped() + } + observeDone <- observed + }() + close(start) + + select { + case ok := <-readDone: + if !ok { + t.Fatal("legacy reader unexpectedly ran out of data") + } + case <-time.After(time.Second): + t.Fatal("legacy reader did not complete") + } + select { + case observed := <-observeDone: + if observed <= 0 { + t.Fatalf("concurrent observer sum = %d, want positive", observed) + } + case <-time.After(time.Second): + t.Fatal("Skipped observer did not complete") + } +} + +func TestRingReaderAdvanceToLivePreservesLaterWrites(t *testing.T) { + rb := NewRingBuffer[int](4) + reader := rb.NewReaderAt(0) + for _, value := range []int{10, 20, 30} { + rb.Write(value) + } + + captured := make(chan struct{}) + resumeAdvance := make(chan struct{}) + writeSlotLockAttempted := make(chan bool, 1) + rb.testHooks = &ringBufferTestHooks{ + afterAdvanceCapture: func() { + close(captured) + <-resumeAdvance + }, + writeSlotLockAttempted: func(contended bool) { + writeSlotLockAttempted <- contended + }, + } + advanceDone := make(chan int64, 1) + go func() { + advanceDone <- reader.AdvanceToLive() + }() + + select { + case <-captured: + case <-time.After(time.Second): + t.Fatal("AdvanceToLive did not reach the capture boundary") + } + + writerDone := make(chan struct{}) + go func() { + rb.Write(40) + close(writerDone) + }() + + var writerContended bool + select { + case writerContended = <-writeSlotLockAttempted: + case <-time.After(time.Second): + close(resumeAdvance) + cleanupTimer := time.NewTimer(time.Second) + defer cleanupTimer.Stop() + for advanceDone != nil || writerDone != nil { + select { + case <-advanceDone: + advanceDone = nil + case <-writerDone: + writerDone = nil + case <-cleanupTimer.C: + t.Fatal("writer did not attempt the slot lock at the capture boundary; cleanup did not complete") + } + } + t.Fatal("writer did not attempt the slot lock at the capture boundary") + } + close(resumeAdvance) + + var discarded int64 + select { + case discarded = <-advanceDone: + case <-time.After(time.Second): + t.Fatal("AdvanceToLive did not complete after capture release") + } + if discarded != 3 { + t.Fatalf("AdvanceToLive discarded = %d, want captured count 3", discarded) + } + select { + case <-writerDone: + case <-time.After(time.Second): + t.Fatal("writer did not complete after AdvanceToLive released the slot lock") + } + if !writerContended { + t.Fatal("writer slot lock attempt did not contend with AdvanceToLive capture") + } + got := reader.TryReadResult() + if !got.OK || got.Value != 40 || got.Overwritten != 0 { + t.Fatalf("TryReadResult after later write = %+v, want {Value:40 OK:true Overwritten:0}", got) + } +} + +func TestRingReaderReadResultContextCancellationDoesNotAdvance(t *testing.T) { + rb := NewRingBuffer[int](2) + reader := rb.NewReaderAt(0) + for _, value := range []int{10, 20, 30} { + rb.Write(value) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + got := reader.ReadResultContext(ctx) + if got.OK || got.Value != 0 || got.Overwritten != 0 { + t.Fatalf("cancelled ReadResultContext = %+v, want zero unavailable result", got) + } + if cursor := reader.ReadCursor(); cursor != 0 { + t.Fatalf("reader cursor after cancellation = %d, want 0", cursor) + } + if skipped := reader.Skipped(); skipped != 0 { + t.Fatalf("Skipped after cancellation = %d, want 0", skipped) + } +} + +func TestRingReaderReadResultEmptyAndClosed(t *testing.T) { + rb := NewRingBuffer[int](2) + reader := rb.NewReader() + + if got := reader.TryReadResult(); got.OK || got.Value != 0 || got.Overwritten != 0 { + t.Fatalf("empty TryReadResult = %+v, want zero unavailable result", got) + } + rb.Close() + if got := reader.ReadResult(); got.OK || got.Value != 0 || got.Overwritten != 0 { + t.Fatalf("closed ReadResult = %+v, want zero unavailable result", got) + } +} + +func TestRingReaderReadResultAllocations(t *testing.T) { + rb := NewRingBuffer[int](8) + reader := rb.NewReader() + value := 0 + allocs := testing.AllocsPerRun(1000, func() { + value++ + rb.Write(value) + if got := reader.TryReadResult(); !got.OK { + t.Fatal("TryReadResult returned no value") + } + }) + if allocs != 0 { + t.Fatalf("TryReadResult allocations = %v, want 0", allocs) + } + + ctx := context.Background() + allocs = testing.AllocsPerRun(1000, func() { + value++ + rb.Write(value) + if got := reader.ReadResultContext(ctx); !got.OK { + t.Fatal("ReadResultContext returned no value") + } + }) + if allocs != 0 { + t.Fatalf("ReadResultContext immediate allocations = %v, want 0", allocs) + } +} + func TestRingBufferMultipleReaders(t *testing.T) { rb := NewRingBuffer[int](8) rb.Write(1) @@ -342,6 +715,19 @@ func BenchmarkRingReaderTryRead(b *testing.B) { } } +func BenchmarkRingReaderTryReadResult(b *testing.B) { + rb := NewRingBuffer[int](1024) + reader := rb.NewReader() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + rb.Write(i) + if result := reader.TryReadResult(); !result.OK { + b.Fatal("TryReadResult returned no frame") + } + } +} + func BenchmarkRingReaderReadContextImmediate(b *testing.B) { rb := NewRingBuffer[int](1024) reader := rb.NewReader() @@ -356,6 +742,20 @@ func BenchmarkRingReaderReadContextImmediate(b *testing.B) { } } +func BenchmarkRingReaderReadContextImmediateResult(b *testing.B) { + rb := NewRingBuffer[int](1024) + reader := rb.NewReader() + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + rb.Write(i) + if result := reader.ReadResultContext(ctx); !result.OK { + b.Fatal("ReadResultContext returned no frame") + } + } +} + // TestRingBufferReadBlocksAfterPublisherStops simulates the exact user scenario: // publisher writes frames, then stops. Reader goroutines should block (near-zero // CPU), NOT busy-spin. Before the sync.Cond fix, this consumed ~100% CPU per reader. diff --git a/test/integration/protocol_browser_matrix_test.go b/test/integration/protocol_browser_matrix_test.go new file mode 100644 index 00000000..4c986600 --- /dev/null +++ b/test/integration/protocol_browser_matrix_test.go @@ -0,0 +1,494 @@ +//go:build audiocodec + +package integration + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/chromedp/chromedp" + "github.com/im-pingo/liveforge/module/gb28181" + "github.com/im-pingo/liveforge/module/sipgateway" + webrtcmod "github.com/im-pingo/liveforge/module/webrtc" + "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/im-pingo/liveforge/tools/testkit/push" + "github.com/im-pingo/liveforge/tools/testkit/source" + "github.com/im-pingo/liveforge/tools/testkit/testutil" +) + +func TestSIPGB28181WHIPBrowserBridgeMatrix(t *testing.T) { + if testing.Short() { + t.Skip("skipping real browser protocol bridge matrix in short mode") + } + allocator, cancelAllocator := chromedp.NewExecAllocator(context.Background(), + append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("headless", true), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("no-sandbox", true), + chromedp.Flag("disable-dev-shm-usage", true), + chromedp.Flag("autoplay-policy", "no-user-gesture-required"), + )..., + ) + defer cancelAllocator() + chromium := &matrixChromiumAvailability{} + + t.Run("sip_publish_to_gb28181_and_whep", func(t *testing.T) { + srv, sipModule, gbModule := newProtocolMatrixServer(t) + streamKey := "matrix/sip-publish" + published, err := sipModule.StartLabSession(context.Background(), sipgateway.LabSessionRequest{ + Mode: sipgateway.LabModePublish, + DeviceID: "matrix-sip-publisher", + StreamKey: streamKey, + Codec: "PCMA", + }) + if err != nil { + t.Fatalf("start SIP publish lab: %v", err) + } + t.Cleanup(func() { _ = sipModule.StopLabSession(published.ID) }) + waitForMatrixStream(t, srv, streamKey, avframe.CodecG711A) + + received, err := gbModule.StartLabSession(context.Background(), gb28181.LabSessionRequest{ + Mode: gb28181.LabModeReceive, + DeviceID: "34020000001320000101", + ChannelID: "34020000001320000102", + StreamKey: streamKey, + }) + if err != nil { + t.Fatalf("start GB28181 receive lab: %v", err) + } + t.Cleanup(func() { _ = gbModule.StopLabSession(received.ID) }) + waitForGBMatrixReceive(t, gbModule, received.ID) + runMatrixWHEPBrowser(t, allocator, chromium, srv.WebRTCAddr(), streamKey, 160, 90) + }) + + t.Run("gb28181_publish_to_sip_and_whep", func(t *testing.T) { + srv, sipModule, gbModule := newProtocolMatrixServer(t) + streamKey := "matrix/gb-publish" + published, err := gbModule.StartLabSession(context.Background(), gb28181.LabSessionRequest{ + Mode: gb28181.LabModePublish, + DeviceID: "34020000001320000111", + ChannelID: "34020000001320000112", + StreamKey: streamKey, + }) + if err != nil { + t.Fatalf("start GB28181 publish lab: %v", err) + } + t.Cleanup(func() { _ = gbModule.StopLabSession(published.ID) }) + waitForMatrixStream(t, srv, streamKey, avframe.CodecG711A) + + received, err := sipModule.StartLabSession(context.Background(), sipgateway.LabSessionRequest{ + Mode: sipgateway.LabModeReceive, + DeviceID: "matrix-sip-receiver-gb", + StreamKey: streamKey, + Codec: "PCMU", + }) + if err != nil { + t.Fatalf("start SIP receive lab: %v", err) + } + t.Cleanup(func() { _ = sipModule.StopLabSession(received.ID) }) + waitForSIPMatrixReceive(t, sipModule, received.ID) + runMatrixWHEPBrowser(t, allocator, chromium, srv.WebRTCAddr(), streamKey, 160, 90) + }) + + t.Run("whip_publish_to_sip_gb28181_and_whep", func(t *testing.T) { + srv, sipModule, gbModule := newProtocolMatrixServer(t) + streamKey := "matrix/whip-publish" + ctx, cancel := context.WithCancel(context.Background()) + pushDone := make(chan error, 1) + pusher, err := push.NewPusher("whip") + if err != nil { + t.Fatalf("create WHIP pusher: %v", err) + } + go func() { + _, pushErr := pusher.Push(ctx, source.NewFLVSourceLoop(0), push.PushConfig{ + Protocol: "whip", + Target: fmt.Sprintf("http://%s/webrtc/whip/%s", srv.WebRTCAddr(), streamKey), + Duration: 0, + Realtime: true, + }) + pushDone <- pushErr + }() + t.Cleanup(func() { + cancel() + select { + case <-pushDone: + case <-time.After(3 * time.Second): + t.Error("WHIP pusher did not stop after cancellation") + } + }) + waitForMatrixStream(t, srv, streamKey, avframe.CodecOpus) + + sipReceived, err := sipModule.StartLabSession(context.Background(), sipgateway.LabSessionRequest{ + Mode: sipgateway.LabModeReceive, + DeviceID: "matrix-sip-receiver-whip", + StreamKey: streamKey, + Codec: "PCMA", + }) + if err != nil { + t.Fatalf("start SIP receive lab for WHIP: %v", err) + } + t.Cleanup(func() { _ = sipModule.StopLabSession(sipReceived.ID) }) + gbReceived, err := gbModule.StartLabSession(context.Background(), gb28181.LabSessionRequest{ + Mode: gb28181.LabModeReceive, + DeviceID: "34020000001320000121", + ChannelID: "34020000001320000122", + StreamKey: streamKey, + }) + if err != nil { + t.Fatalf("start GB28181 receive lab for WHIP: %v", err) + } + t.Cleanup(func() { _ = gbModule.StopLabSession(gbReceived.ID) }) + waitForSIPMatrixReceive(t, sipModule, sipReceived.ID) + waitForGBMatrixReceive(t, gbModule, gbReceived.ID) + runMatrixWHEPBrowser(t, allocator, chromium, srv.WebRTCAddr(), streamKey, 640, 360) + }) +} + +func newProtocolMatrixServer(t *testing.T) (*testutil.TestServer, *sipgateway.Module, *gb28181.Module) { + t.Helper() + srv := testutil.StartTestServer(t, + testutil.WithSIP(), + testutil.WithGB28181(), + testutil.WithSIPGateway(), + testutil.WithWebRTC(), + testutil.WithAudioCodec(), + ) + sipModule, ok := srv.ModuleByName("sipgateway").(*sipgateway.Module) + if !ok { + t.Fatal("test server did not expose SIP gateway module") + } + gbModule, ok := srv.ModuleByName("gb28181").(*gb28181.Module) + if !ok { + t.Fatal("test server did not expose GB28181 module") + } + return srv, sipModule, gbModule +} + +func waitForMatrixStream(t *testing.T, srv *testutil.TestServer, streamKey string, audio avframe.CodecType) { + t.Helper() + deadline := time.Now().Add(8 * time.Second) + for time.Now().Before(deadline) { + if srv.StreamHasVideoGOP(streamKey) && srv.StreamHasAudio(streamKey, audio) { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("stream %q did not expose H.264 GOP plus %s audio", streamKey, audio) +} + +func waitForSIPMatrixReceive(t *testing.T, module *sipgateway.Module, id string) { + t.Helper() + deadline := time.Now().Add(8 * time.Second) + for time.Now().Before(deadline) { + for _, snapshot := range module.ListLabSessions() { + if snapshot.ID != id { + continue + } + if snapshot.State == sipgateway.LabSessionStateFailed { + t.Fatalf("SIP receive lab failed: %s", snapshot.LastError) + } + if snapshot.State == sipgateway.LabSessionStateActive && + snapshot.AudioRTPPacketsRecv > 0 && snapshot.VideoRTPPacketsRecv > 0 && snapshot.RTCPPacketsRecv > 0 { + return + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("SIP receive lab %s did not receive audio, video, and RTCP", id) +} + +func waitForGBMatrixReceive(t *testing.T, module *gb28181.Module, id string) { + t.Helper() + deadline := time.Now().Add(8 * time.Second) + for time.Now().Before(deadline) { + for _, snapshot := range module.ListLabSessions() { + if snapshot.ID != id { + continue + } + if snapshot.State == gb28181.LabSessionStateFailed { + t.Fatalf("GB28181 receive lab failed: %s", snapshot.LastError) + } + if snapshot.State == gb28181.LabSessionStateActive && snapshot.RTPPacketsRecv > 0 && + snapshot.RTCPPacketsRecv > 0 && snapshot.PSFramesRecv > 0 && + snapshot.AudioFramesRecv > 0 && snapshot.VideoFramesRecv > 0 { + return + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("GB28181 receive lab %s did not receive RTP, RTCP, PS, audio, and video", id) +} + +type matrixBrowserProbe struct { + ReadyState int `json:"readyState"` + Width int `json:"width"` + Height int `json:"height"` + CurrentTime float64 `json:"currentTime"` + Error string `json:"error"` + ICE string `json:"ice"` + ConnectError string `json:"connectError"` + Stage string `json:"stage"` + H264Supported *bool `json:"h264Supported"` + SessionLocation string `json:"sessionLocation"` + VideoPackets uint64 `json:"videoPackets"` + AudioPackets uint64 `json:"audioPackets"` + FramesDecoded uint64 `json:"framesDecoded"` +} + +type matrixChromiumAvailability struct { + established bool +} + +func (a *matrixChromiumAvailability) canSkip(err error) bool { + if a == nil || a.established || err == nil { + return false + } + return strings.Contains(err.Error(), "websocket url timeout") || + strings.Contains(err.Error(), "executable file not found") +} + +func (a *matrixChromiumAvailability) markEstablished() { + if a != nil { + a.established = true + } +} + +func runMatrixWHEPBrowser(t *testing.T, allocator context.Context, chromium *matrixChromiumAvailability, whepAddr, streamKey string, width, height int) { + t.Helper() + page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(matrixWHEPPlayerHTML("http://"+whepAddr, streamKey))) + })) + defer page.Close() + browser, cancelBrowser := chromedp.NewContext(allocator, chromedp.WithLogf(t.Logf)) + defer cancelBrowser() + if err := chromedp.Run(browser, chromedp.Navigate(page.URL)); err != nil { + if chromium.canSkip(err) { + t.Skipf("headless Chrome unavailable: %v", err) + } + t.Fatalf("navigate WHEP matrix player: %v", err) + } + chromium.markEstablished() + if err := chromedp.Run(browser, chromedp.Evaluate(`void window.__connectMatrix(); true`, nil)); err != nil { + t.Fatalf("start WHEP matrix player: %v", err) + } + + first := waitForMatrixBrowserProbe(t, browser, func(probe matrixBrowserProbe) bool { + return probe.ReadyState >= 3 && probe.Width == width && probe.Height == height && + probe.CurrentTime > 0.2 && probe.VideoPackets > 0 && probe.AudioPackets > 0 && + probe.FramesDecoded > 0 && (probe.ICE == "connected" || probe.ICE == "completed") + }) + time.Sleep(1200 * time.Millisecond) + second := waitForMatrixBrowserProbe(t, browser, func(probe matrixBrowserProbe) bool { + return probe.CurrentTime > first.CurrentTime+0.3 && probe.VideoPackets > first.VideoPackets && + probe.AudioPackets > first.AudioPackets && probe.FramesDecoded > first.FramesDecoded + }) + if second.Error != "" { + t.Fatalf("browser media error after playback advance: %s", second.Error) + } + soakDeadline := time.Now().Add(protocolMatrixSoakDuration(t)) + for time.Now().Before(soakDeadline) { + previous := second + time.Sleep(time.Second) + second = waitForMatrixBrowserProbe(t, browser, func(probe matrixBrowserProbe) bool { + return probe.CurrentTime > previous.CurrentTime+0.3 && probe.VideoPackets > previous.VideoPackets && + probe.AudioPackets > previous.AudioPackets && probe.FramesDecoded > previous.FramesDecoded + }) + } + assertMatrixWHEPStatus(t, whepAddr, second.SessionLocation) +} + +func protocolMatrixSoakDuration(t *testing.T) time.Duration { + t.Helper() + value := strings.TrimSpace(os.Getenv("LIVEFORGE_PROTOCOL_MATRIX_SOAK")) + if value == "" { + return 0 + } + duration, err := time.ParseDuration(value) + if err != nil || duration < 0 { + t.Fatalf("LIVEFORGE_PROTOCOL_MATRIX_SOAK=%q is not a non-negative duration", value) + } + return duration +} + +func waitForMatrixBrowserProbe(t *testing.T, browser context.Context, accept func(matrixBrowserProbe) bool) matrixBrowserProbe { + t.Helper() + deadline := time.Now().Add(8 * time.Second) + var probe matrixBrowserProbe + for time.Now().Before(deadline) { + probeCtx, cancel := context.WithTimeout(browser, 2*time.Second) + err := chromedp.Run(probeCtx, chromedp.Evaluate(`window.__probeMatrix()`, &probe)) + cancel() + if err == nil { + if probe.H264Supported != nil && !*probe.H264Supported { + t.Skip("headless Chrome does not advertise H.264 WebRTC receive support") + } + if probe.ConnectError != "" { + t.Fatalf("WHEP browser connection failed at %s: %s", probe.Stage, probe.ConnectError) + } + if probe.Error != "" { + t.Fatalf("WHEP browser media error: %s", probe.Error) + } + if accept(probe) { + return probe + } + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("WHEP browser media did not advance: %+v", probe) + return matrixBrowserProbe{} +} + +func assertMatrixWHEPStatus(t *testing.T, whepAddr, location string) { + t.Helper() + if location == "" { + t.Fatal("WHEP response did not expose a session Location") + } + type response struct { + Feed webrtcmod.WHEPFeedStatus `json:"feed"` + } + deadline := time.Now().Add(5 * time.Second) + client := &http.Client{Timeout: 500 * time.Millisecond} + var status response + for time.Now().Before(deadline) { + requestCtx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + statusCode, err := requestMatrixWHEPStatus(requestCtx, client, "http://"+whepAddr+location+"/status", &status) + cancel() + if err == nil && statusCode == http.StatusOK { + if status.Feed.State == webrtcmod.WHEPFeedMediaStalled { + t.Fatalf("WHEP server status entered media_stalled: %+v", status.Feed) + } + if status.Feed.State == webrtcmod.WHEPFeedPlaying && status.Feed.ExpectedVideo && status.Feed.ExpectedAudio && + status.Feed.VideoFrames > 0 && status.Feed.AudioFrames > 0 && status.Feed.RTPPacketsSent > 0 && + status.Feed.RTCPPacketsReceived > 0 { + return + } + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("WHEP server status did not confirm advancing audio/video RTP/RTCP: %+v", status.Feed) +} + +func requestMatrixWHEPStatus(ctx context.Context, client *http.Client, url string, target any) (int, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, err + } + response, err := client.Do(request) + if err != nil { + return 0, err + } + defer response.Body.Close() + if err := json.NewDecoder(response.Body).Decode(target); err != nil { + return response.StatusCode, err + } + return response.StatusCode, nil +} + +func matrixWHEPPlayerHTML(whepBase, streamKey string) string { + return ` + +` +} + +func TestMatrixChromiumEnvironmentalSkipEndsAfterAvailabilityIsEstablished(t *testing.T) { + var availability matrixChromiumAvailability + startupFailure := errors.New("websocket url timeout") + if !availability.canSkip(startupFailure) { + t.Fatal("initial Chromium startup failure must remain an environmental skip") + } + availability.markEstablished() + if availability.canSkip(startupFailure) { + t.Fatal("Chromium startup failure was still skippable after a successful matrix launch") + } +} + +func TestMatrixWHEPStatusRequestIsBounded(t *testing.T) { + requestCanceled := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + close(requestCanceled) + })) + defer server.Close() + + client := &http.Client{Timeout: 50 * time.Millisecond} + requestCtx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + started := time.Now() + var status map[string]any + if _, err := requestMatrixWHEPStatus(requestCtx, client, server.URL, &status); err == nil { + t.Fatal("hanging WHEP status request returned no timeout error") + } + if elapsed := time.Since(started); elapsed > 500*time.Millisecond { + t.Fatalf("hanging WHEP status request returned after %s, want a bounded call", elapsed) + } + select { + case <-requestCanceled: + case <-time.After(time.Second): + t.Fatal("WHEP status request timeout did not cancel the HTTP request context") + } +} diff --git a/tools/testkit/play/player_test.go b/tools/testkit/play/player_test.go index e060c2ff..0fc872d8 100644 --- a/tools/testkit/play/player_test.go +++ b/tools/testkit/play/player_test.go @@ -14,6 +14,23 @@ import ( "github.com/im-pingo/liveforge/tools/testkit/testutil" ) +type videoOnlyTestSource struct { + source.Source +} + +func (s videoOnlyTestSource) NextFrame() (*avframe.AVFrame, error) { + for { + frame, err := s.Source.NextFrame() + if err != nil || frame.MediaType.IsVideo() { + return frame, err + } + } +} + +func (s videoOnlyTestSource) MediaInfo() *source.MediaInfo { + return &source.MediaInfo{VideoCodec: s.Source.MediaInfo().VideoCodec} +} + func TestNewPlayer_SRT(t *testing.T) { p, err := NewPlayer("srt") if err != nil { @@ -350,6 +367,7 @@ func TestSRTPlay(t *testing.T) { _, err := pusher.Push(pushCtx, src, push.PushConfig{ Protocol: "rtmp", Target: pushURL, + Realtime: true, }) pushDone <- err }() @@ -415,7 +433,7 @@ func TestWHEPPlay(t *testing.T) { srv := testutil.StartTestServer(t, testutil.WithRTMP(), testutil.WithWebRTC(), testutil.WithAPI()) // Push via RTMP in background so there is a stream for WHEP to subscribe to. - src := source.NewFLVSourceLoop(0) + src := videoOnlyTestSource{Source: source.NewFLVSourceLoop(0)} pusher, err := push.NewPusher("rtmp") if err != nil { t.Fatalf("NewPusher: %v", err) @@ -430,10 +448,12 @@ func TestWHEPPlay(t *testing.T) { _, err := pusher.Push(pushCtx, src, push.PushConfig{ Protocol: "rtmp", Target: pushURL, + Realtime: true, }) pushDone <- err }() + // Wait until the live subscriber can start from both required media kinds. readyTimer := time.NewTimer(10 * time.Second) defer readyTimer.Stop() readyTicker := time.NewTicker(10 * time.Millisecond) @@ -479,10 +499,8 @@ func TestWHEPPlay(t *testing.T) { if rpt.Video.FrameCount == 0 { t.Error("no video frames received") } - // Audio may not be present: server only supports Opus but source is AAC. - // Log the result instead of failing. - if rpt.Audio.FrameCount == 0 { - t.Log("note: no audio frames received (expected: server supports Opus but source is AAC)") + if rpt.Audio.FrameCount != 0 { + t.Errorf("audio frames received from video-only fixture: %d", rpt.Audio.FrameCount) } t.Logf("WHEP play report: video=%d frames, audio=%d frames, duration=%dms", @@ -528,12 +546,25 @@ func TestHTTPFLVPlay(t *testing.T) { _, err := pusher.Push(pushCtx, src, push.PushConfig{ Protocol: "rtmp", Target: pushURL, + Realtime: true, }) pushDone <- err }() - // Wait for stream to be established. - time.Sleep(1 * time.Second) + // Wait until the live subscriber can start from both required media kinds. + readyTimer := time.NewTimer(10 * time.Second) + defer readyTimer.Stop() + readyTicker := time.NewTicker(10 * time.Millisecond) + defer readyTicker.Stop() + for !srv.StreamHasVideoGOP("live/test") || !srv.StreamHasAudio("live/test", avframe.CodecAAC) { + select { + case err := <-pushDone: + t.Fatalf("RTMP pusher stopped before HTTP-FLV media was ready: %v", err) + case <-readyTimer.C: + t.Fatal("timed out waiting for RTMP audio/video before HTTP-FLV playback") + case <-readyTicker.C: + } + } // Play via HTTP-FLV. player, err := NewPlayer("httpflv") @@ -728,12 +759,24 @@ func TestWSFLVPlay(t *testing.T) { _, err := pusher.Push(pushCtx, src, push.PushConfig{ Protocol: "rtmp", Target: pushURL, + Realtime: true, }) pushDone <- err }() - // Wait for stream to be established. - time.Sleep(1 * time.Second) + readyTimer := time.NewTimer(10 * time.Second) + defer readyTimer.Stop() + readyTicker := time.NewTicker(10 * time.Millisecond) + defer readyTicker.Stop() + for !srv.StreamHasVideoGOP("live/test") || !srv.StreamHasAudio("live/test", avframe.CodecAAC) { + select { + case err := <-pushDone: + t.Fatalf("RTMP pusher stopped before WS-FLV media was ready: %v", err) + case <-readyTimer.C: + t.Fatal("timed out waiting for RTMP audio/video before WS-FLV playback") + case <-readyTicker.C: + } + } // Play via WS-FLV. player, err := NewPlayer("wsflv") diff --git a/tools/testkit/push/whip.go b/tools/testkit/push/whip.go index 4dfa6c00..adfdc002 100644 --- a/tools/testkit/push/whip.go +++ b/tools/testkit/push/whip.go @@ -13,6 +13,7 @@ import ( pkgrtp "github.com/im-pingo/liveforge/pkg/rtp" "github.com/im-pingo/liveforge/tools/testkit/report" "github.com/im-pingo/liveforge/tools/testkit/source" + pionrtp "github.com/pion/rtp" "github.com/pion/webrtc/v4" ) @@ -22,12 +23,11 @@ const ( // whipPusher implements Pusher for the WebRTC WHIP protocol. It creates a // PeerConnection, negotiates via HTTP POST to the WHIP endpoint, and sends -// RTP-packetized H.264 video frames over a WebRTC media track. +// RTP-packetized H.264 video and, when available, Opus audio tracks. type whipPusher struct{} // Push creates a WebRTC PeerConnection, performs WHIP signaling with the target -// endpoint, and sends H.264 video frames as RTP packets. Audio frames are -// skipped because the server only supports Opus while the source emits AAC. +// endpoint, and sends H.264 video plus an optional Opus audio track. func (p *whipPusher) Push(ctx context.Context, src source.Source, cfg PushConfig) (*report.PushReport, error) { start := time.Now() var framesSent int64 @@ -55,6 +55,31 @@ func (p *whipPusher) Push(ctx context.Context, src source.Source, cfg PushConfig fmt.Errorf("whip: add track: %w", err) } + var audioTrack *webrtc.TrackLocalStaticRTP + audioProcessor, err := newWHIPAudioProcessor(src.MediaInfo().AudioCodec) + if err != nil { + return buildPushReport(cfg, start, framesSent, bytesSent), err + } + if audioProcessor != nil { + defer audioProcessor.Close() + audioTrack, err = webrtc.NewTrackLocalStaticRTP( + webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeOpus, + ClockRate: 48000, + Channels: 2, + }, + "audio", "lf-test", + ) + if err != nil { + return buildPushReport(cfg, start, framesSent, bytesSent), + fmt.Errorf("whip: create audio track: %w", err) + } + if _, err := pc.AddTrack(audioTrack); err != nil { + return buildPushReport(cfg, start, framesSent, bytesSent), + fmt.Errorf("whip: add audio track: %w", err) + } + } + // Set up connection state callback before signaling. connected := make(chan struct{}) var connOnce sync.Once @@ -118,12 +143,17 @@ func (p *whipPusher) Push(ctx context.Context, src source.Source, cfg PushConfig fmt.Errorf("whip: create packetizer: %w", err) } session := pkgrtp.NewSession(106, 90000) // H264 PT=106, 90kHz clock + var audioSequence uint16 + var audioTimestamp uint32 + var audioBaseDTS int64 + var audioBaseSet bool // Determine deadline from cfg.Duration. var deadline time.Time if cfg.Duration > 0 { deadline = start.Add(cfg.Duration) } + pacer := whipRealtimePacer{enabled: cfg.Realtime} // Frame loop: read frames from source and send as RTP. for { @@ -145,9 +175,47 @@ func (p *whipPusher) Push(ctx context.Context, src source.Source, cfg PushConfig return buildPushReport(cfg, start, framesSent, bytesSent), fmt.Errorf("whip: read source frame: %w", err) } - - // Skip audio frames entirely (server only supports Opus, source has AAC). if frame.MediaType.IsAudio() { + if audioTrack == nil || audioProcessor == nil { + continue + } + if frame.FrameType != avframe.FrameTypeSequenceHeader && !audioBaseSet { + audioBaseDTS = frame.DTS + audioBaseSet = true + } + packets, processErr := audioProcessor.Process(frame) + if processErr != nil { + return buildPushReport(cfg, start, framesSent, bytesSent), + fmt.Errorf("whip: process audio: %w", processErr) + } + for _, payload := range packets { + durationSamples, ok := whipOpusPacketDurationSamples(payload) + if !ok { + return buildPushReport(cfg, start, framesSent, bytesSent), + fmt.Errorf("whip: invalid Opus packet duration") + } + mediaTime := time.Duration(audioBaseDTS)*time.Millisecond + + time.Duration(audioTimestamp)*time.Second/48000 + if err := pacer.Wait(ctx, mediaTime); err != nil { + return buildPushReport(cfg, start, framesSent, bytesSent), err + } + packet := &pionrtp.Packet{ + Header: pionrtp.Header{ + Version: 2, + SequenceNumber: audioSequence, + Timestamp: audioTimestamp, + }, + Payload: payload, + } + if err := audioTrack.WriteRTP(packet); err != nil { + return buildPushReport(cfg, start, framesSent, bytesSent), + fmt.Errorf("whip: write audio RTP: %w", err) + } + audioSequence++ + audioTimestamp += durationSamples + framesSent++ + bytesSent += int64(packet.MarshalSize()) + } continue } @@ -155,6 +223,11 @@ func (p *whipPusher) Push(ctx context.Context, src source.Source, cfg PushConfig if !frame.MediaType.IsVideo() { continue } + if frame.FrameType != avframe.FrameTypeSequenceHeader { + if err := pacer.Wait(ctx, time.Duration(frame.DTS)*time.Millisecond); err != nil { + return buildPushReport(cfg, start, framesSent, bytesSent), err + } + } // Packetize the video frame (including sequence headers with SPS/PPS). rtpPackets, err := packetizer.Packetize(frame, pkgrtp.DefaultMTU) @@ -190,6 +263,44 @@ func (p *whipPusher) Push(ctx context.Context, src source.Source, cfg PushConfig return buildPushReport(cfg, start, framesSent, bytesSent), nil } +type whipRealtimePacer struct { + enabled bool + baseWall time.Time + baseMedia time.Duration +} + +func (p *whipRealtimePacer) Wait(ctx context.Context, mediaTime time.Duration) error { + if !p.enabled { + return nil + } + now := time.Now() + if p.baseWall.IsZero() { + p.baseWall = now + p.baseMedia = mediaTime + return nil + } + target := p.baseWall.Add(mediaTime - p.baseMedia) + if !target.After(now) && mediaTime > p.baseMedia { + // Processing or transport can fall behind the source timeline. Rebase + // the current packet instead of sending a burst to catch up. + p.baseWall = now + p.baseMedia = mediaTime + return nil + } + wait := target.Sub(now) + if wait <= 0 { + return nil + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + return nil + } +} + // whipSignal sends the SDP offer to the WHIP endpoint via HTTP POST and returns // the SDP answer. Expects HTTP 201 with Content-Type application/sdp. func whipSignal(ctx context.Context, target, token, offerSDP string) (string, error) { diff --git a/tools/testkit/push/whip_audio.go b/tools/testkit/push/whip_audio.go new file mode 100644 index 00000000..2b510232 --- /dev/null +++ b/tools/testkit/push/whip_audio.go @@ -0,0 +1,70 @@ +package push + +import ( + "github.com/im-pingo/liveforge/pkg/avframe" +) + +type whipAudioProcessor interface { + Process(*avframe.AVFrame) ([][]byte, error) + Close() +} + +type whipOpusPassthrough struct{} + +func (*whipOpusPassthrough) Process(frame *avframe.AVFrame) ([][]byte, error) { + if frame == nil || frame.FrameType == avframe.FrameTypeSequenceHeader || len(frame.Payload) == 0 { + return nil, nil + } + return [][]byte{frame.Payload}, nil +} + +func (*whipOpusPassthrough) Close() {} + +func newWHIPAudioProcessor(codec avframe.CodecType) (whipAudioProcessor, error) { + if codec == 0 { + return nil, nil + } + if codec == avframe.CodecOpus { + return &whipOpusPassthrough{}, nil + } + return newWHIPAudioTranscoder(codec) +} + +func whipOpusPacketDurationSamples(payload []byte) (uint32, bool) { + if len(payload) == 0 { + return 0, false + } + + config := payload[0] >> 3 + var samplesPerFrame uint32 + switch { + case config < 12: + samplesPerFrame = [...]uint32{480, 960, 1920, 2880}[config&0x03] + case config < 16: + samplesPerFrame = 480 << (config & 0x01) + default: + samplesPerFrame = 120 << (config & 0x03) + } + + var frameCount uint32 + switch payload[0] & 0x03 { + case 0: + frameCount = 1 + case 1, 2: + frameCount = 2 + case 3: + if len(payload) < 2 { + return 0, false + } + frameCount = uint32(payload[1] & 0x3f) + if frameCount == 0 { + return 0, false + } + } + + duration := samplesPerFrame * frameCount + if duration == 0 || duration > 5760 { + return 0, false + } + return duration, true +} diff --git a/tools/testkit/push/whip_audio_audiocodec.go b/tools/testkit/push/whip_audio_audiocodec.go new file mode 100644 index 00000000..1a62dad2 --- /dev/null +++ b/tools/testkit/push/whip_audio_audiocodec.go @@ -0,0 +1,91 @@ +//go:build audiocodec + +package push + +import ( + "fmt" + + "github.com/im-pingo/liveforge/pkg/audiocodec" + "github.com/im-pingo/liveforge/pkg/avframe" +) + +type whipAudioTranscoder struct { + decoder audiocodec.Decoder + encoder audiocodec.Encoder + resampler audiocodec.Resampler + pcm []int16 +} + +func newWHIPAudioTranscoder(codec avframe.CodecType) (whipAudioProcessor, error) { + registry := audiocodec.Global() + decoder, err := registry.NewDecoder(codec) + if err != nil { + return nil, fmt.Errorf("whip: audio decoder for %s: %w", codec, err) + } + encoder, err := registry.NewEncoder(avframe.CodecOpus) + if err != nil { + decoder.Close() + return nil, fmt.Errorf("whip: Opus encoder: %w", err) + } + return &whipAudioTranscoder{decoder: decoder, encoder: encoder}, nil +} + +func (p *whipAudioTranscoder) Process(frame *avframe.AVFrame) ([][]byte, error) { + if frame == nil || !frame.MediaType.IsAudio() { + return nil, nil + } + if frame.FrameType == avframe.FrameTypeSequenceHeader { + p.decoder.SetExtradata(frame.Payload) + return nil, nil + } + pcm, err := p.decoder.Decode(frame.Payload) + if err != nil { + return nil, err + } + if p.resampler == nil { + p.resampler = audiocodec.Global().NewResampler(pcm.SampleRate, pcm.Channels, 48000, 2) + if p.resampler == nil { + return nil, fmt.Errorf("48 kHz stereo resampler unavailable") + } + } + converted := p.resampler.Resample(pcm) + p.pcm = append(p.pcm, converted.Samples...) + + frameSize := p.encoder.FrameSize() + if frameSize <= 0 { + return nil, fmt.Errorf("Opus encoder returned invalid frame size %d", frameSize) + } + needed := frameSize * 2 + packets := make([][]byte, 0, len(p.pcm)/needed) + for len(p.pcm) >= needed { + payload, encodeErr := p.encoder.Encode(&audiocodec.PCMFrame{ + Samples: p.pcm[:needed], + SampleRate: 48000, + Channels: 2, + }) + if encodeErr != nil { + return nil, encodeErr + } + copy(p.pcm, p.pcm[needed:]) + p.pcm = p.pcm[:len(p.pcm)-needed] + if len(payload) > 0 { + packets = append(packets, payload) + } + } + return packets, nil +} + +func (p *whipAudioTranscoder) Close() { + if p.resampler != nil { + p.resampler.Close() + } + if p.encoder != nil { + if drainer, ok := p.encoder.(audiocodec.DrainingEncoder); ok { + _, _ = drainer.Drain() + } + p.encoder.Close() + } + if p.decoder != nil { + p.decoder.Close() + } +} diff --git a/tools/testkit/push/whip_audio_stub.go b/tools/testkit/push/whip_audio_stub.go new file mode 100644 index 00000000..9d8aaafe --- /dev/null +++ b/tools/testkit/push/whip_audio_stub.go @@ -0,0 +1,9 @@ +//go:build !audiocodec + +package push + +import "github.com/im-pingo/liveforge/pkg/avframe" + +func newWHIPAudioTranscoder(avframe.CodecType) (whipAudioProcessor, error) { + return nil, nil +} diff --git a/tools/testkit/push/whip_audiocodec_test.go b/tools/testkit/push/whip_audiocodec_test.go new file mode 100644 index 00000000..a1e664d2 --- /dev/null +++ b/tools/testkit/push/whip_audiocodec_test.go @@ -0,0 +1,89 @@ +//go:build audiocodec + +package push + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/im-pingo/liveforge/tools/testkit/source" + "github.com/im-pingo/liveforge/tools/testkit/testutil" +) + +func TestWHIPPushPublishesOpusAudio(t *testing.T) { + srv := testutil.StartTestServer(t, testutil.WithWebRTC(), testutil.WithAudioCodec()) + pusher, err := NewPusher("whip") + if err != nil { + t.Fatalf("NewPusher: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { + _, pushErr := pusher.Push(ctx, source.NewFLVSourceLoop(0), PushConfig{ + Protocol: "whip", + Target: fmt.Sprintf("http://%s/webrtc/whip/live/whip-audio", srv.WebRTCAddr()), + Duration: 5 * time.Second, + Realtime: true, + }) + result <- pushErr + }() + + deadline := time.Now().Add(8 * time.Second) + for time.Now().Before(deadline) { + if srv.StreamHasAudio("live/whip-audio", avframe.CodecOpus) { + cancel() + if pushErr := <-result; pushErr != nil && pushErr != context.Canceled { + t.Fatalf("WHIP push after audio observed: %v", pushErr) + } + return + } + time.Sleep(20 * time.Millisecond) + } + cancel() + <-result + t.Fatal("WHIP publisher never delivered an Opus audio frame") +} + +func TestWHIPConvertedOpusPacketsAreIndividuallyPaced(t *testing.T) { + capture := newWHIPRTPCapture(t) + pusher, err := NewPusher("whip") + if err != nil { + t.Fatalf("NewPusher: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { + _, pushErr := pusher.Push(ctx, source.NewFLVSourceLoop(0), PushConfig{ + Protocol: "whip", + Target: capture.URL(), + Duration: 1200 * time.Millisecond, + Realtime: true, + }) + result <- pushErr + }() + + packets := capture.Wait(t, 20) + for i := 1; i < len(packets); i++ { + if gap := packets[i].Arrival.Sub(packets[i-1].Arrival); gap < 8*time.Millisecond { + t.Fatalf("converted Opus packets %d/%d arrived %s apart; timestamps=%d/%d durations=%v/%v; each emitted packet must be paced", i-1, i, gap, packets[i-1].Timestamp, packets[i].Timestamp, opusPacketDurationForTest(packets[i-1]), opusPacketDurationForTest(packets[i])) + } + } + if pushErr := <-result; pushErr != nil { + t.Fatalf("WHIP push: %v", pushErr) + } +} + +func opusPacketDurationForTest(packet whipCapturedRTP) time.Duration { + samples, ok := whipOpusPacketDurationSamples(packet.Payload) + if !ok { + return 0 + } + return time.Duration(samples) * time.Second / 48000 +} diff --git a/tools/testkit/push/whip_timing_test.go b/tools/testkit/push/whip_timing_test.go new file mode 100644 index 00000000..5296d67a --- /dev/null +++ b/tools/testkit/push/whip_timing_test.go @@ -0,0 +1,182 @@ +package push + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "slices" + "sync" + "testing" + "time" + + "github.com/im-pingo/liveforge/pkg/avframe" + "github.com/im-pingo/liveforge/tools/testkit/source" + "github.com/pion/webrtc/v4" +) + +func TestWHIPDirectOpusUsesPacketDurationsForRTPTimestamps(t *testing.T) { + capture := newWHIPRTPCapture(t) + src := &whipTimingSource{ + info: source.MediaInfo{AudioCodec: avframe.CodecOpus}, + frames: []*avframe.AVFrame{ + avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecOpus, avframe.FrameTypeInterframe, 0, 0, []byte{0x00, 0x01}), + avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecOpus, avframe.FrameTypeInterframe, 10, 10, []byte{0x10, 0x01}), + avframe.NewAVFrame(avframe.MediaTypeAudio, avframe.CodecOpus, avframe.FrameTypeInterframe, 50, 50, []byte{0x80, 0x01}), + }, + tailDelay: 100 * time.Millisecond, + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := (&whipPusher{}).Push(ctx, src, PushConfig{Protocol: "whip", Target: capture.URL()}); err != nil { + t.Fatalf("WHIP push: %v", err) + } + + packets := capture.Wait(t, 3) + timestamps := []uint32{packets[0].Timestamp, packets[1].Timestamp, packets[2].Timestamp} + want := []uint32{0, 480, 2400} + if !slices.Equal(timestamps, want) { + t.Fatalf("direct Opus RTP timestamps = %v, want packet-duration timeline %v", timestamps, want) + } +} + +func TestWHIPRealtimePacerRebasesAfterFallingBehind(t *testing.T) { + pacer := whipRealtimePacer{enabled: true} + ctx := context.Background() + if err := pacer.Wait(ctx, 0); err != nil { + t.Fatalf("initial pacer wait: %v", err) + } + time.Sleep(80 * time.Millisecond) + if err := pacer.Wait(ctx, 20*time.Millisecond); err != nil { + t.Fatalf("late packet pacer wait: %v", err) + } + + start := time.Now() + if err := pacer.Wait(ctx, 40*time.Millisecond); err != nil { + t.Fatalf("re-anchored packet pacer wait: %v", err) + } + if elapsed := time.Since(start); elapsed < 15*time.Millisecond { + t.Fatalf("re-anchored packet waited %s, want at least 15ms", elapsed) + } +} + +type whipCapturedRTP struct { + Timestamp uint32 + Arrival time.Time + Payload []byte +} + +type whipRTPCapture struct { + server *httptest.Server + packets chan whipCapturedRTP + mu sync.Mutex + peers []*webrtc.PeerConnection +} + +func newWHIPRTPCapture(t *testing.T) *whipRTPCapture { + t.Helper() + capture := &whipRTPCapture{packets: make(chan whipCapturedRTP, 256)} + capture.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + offer, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + capture.mu.Lock() + capture.peers = append(capture.peers, pc) + capture.mu.Unlock() + pc.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + if track.Kind() != webrtc.RTPCodecTypeAudio { + return + } + go func() { + for { + packet, _, readErr := track.ReadRTP() + if readErr != nil { + return + } + capture.packets <- whipCapturedRTP{Timestamp: packet.Timestamp, Arrival: time.Now(), Payload: append([]byte(nil), packet.Payload...)} + } + }() + }) + if err := pc.SetRemoteDescription(webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: string(offer)}); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + answer, err := pc.CreateAnswer(nil) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + gathered := webrtc.GatheringCompletePromise(pc) + if err := pc.SetLocalDescription(answer); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + <-gathered + w.Header().Set("Content-Type", "application/sdp") + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(pc.LocalDescription().SDP)) + })) + t.Cleanup(func() { + capture.server.Close() + capture.mu.Lock() + defer capture.mu.Unlock() + for _, pc := range capture.peers { + _ = pc.Close() + } + }) + return capture +} + +func (c *whipRTPCapture) URL() string { + return c.server.URL + "/webrtc/whip/live/timing" +} + +func (c *whipRTPCapture) Wait(t *testing.T, count int) []whipCapturedRTP { + t.Helper() + packets := make([]whipCapturedRTP, 0, count) + deadline := time.NewTimer(3 * time.Second) + defer deadline.Stop() + for len(packets) < count { + select { + case packet := <-c.packets: + packets = append(packets, packet) + case <-deadline.C: + t.Fatalf("captured %d Opus RTP packets, want %d", len(packets), count) + } + } + return packets +} + +type whipTimingSource struct { + info source.MediaInfo + frames []*avframe.AVFrame + index int + tailDelay time.Duration +} + +func (s *whipTimingSource) NextFrame() (*avframe.AVFrame, error) { + if s.index >= len(s.frames) { + if s.tailDelay > 0 { + time.Sleep(s.tailDelay) + s.tailDelay = 0 + } + return nil, io.EOF + } + frame := s.frames[s.index] + s.index++ + return frame, nil +} + +func (s *whipTimingSource) MediaInfo() *source.MediaInfo { return &s.info } + +func (s *whipTimingSource) Reset() { + s.index = 0 +} diff --git a/tools/testkit/testutil/server.go b/tools/testkit/testutil/server.go index 988adc4a..d2d103c5 100644 --- a/tools/testkit/testutil/server.go +++ b/tools/testkit/testutil/server.go @@ -4,8 +4,10 @@ package testutil import ( "net" + "strconv" "sync" "testing" + "time" "github.com/im-pingo/liveforge/config" "github.com/im-pingo/liveforge/core" @@ -16,8 +18,10 @@ import ( "github.com/im-pingo/liveforge/module/rtmp" "github.com/im-pingo/liveforge/module/rtsp" sipmod "github.com/im-pingo/liveforge/module/sip" + sipgwmod "github.com/im-pingo/liveforge/module/sipgateway" "github.com/im-pingo/liveforge/module/srt" "github.com/im-pingo/liveforge/module/webrtc" + "github.com/im-pingo/liveforge/pkg/avframe" ) // Option configures the test server's Config before startup. @@ -108,6 +112,13 @@ func WithAuth(secret string) Option { } } +// WithAudioCodec enables the optional audio transcoding path for test streams. +func WithAudioCodec() Option { + return func(c *config.Config) { + c.AudioCodec.Enabled = true + } +} + // WithSIP enables the SIP module on an auto-allocated UDP port. func WithSIP() Option { return func(c *config.Config) { @@ -125,9 +136,22 @@ func WithGB28181() Option { return func(c *config.Config) { c.GB28181.Enabled = true c.GB28181.StreamPrefix = "gb28181" - // Allocate a dynamic base port, clamped to avoid overflow. - base := allocUDPPortPair() - c.GB28181.RTPPortRange = []int{base, base + 100} + c.GB28181.Keepalive.Timeout = time.Minute + c.GB28181.RTPPortRange = allocUDPPortRange(8, + addressPortRange(c.SIP.Listen), c.SIP.Gateway.RTPPortRange) + } +} + +// WithSIPGateway enables the SIP gateway and its persistent loopback lab. +// Requires WithSIP() to be used as well. +func WithSIPGateway() Option { + return func(c *config.Config) { + c.SIP.Gateway.Enabled = true + c.SIP.Gateway.StreamPrefix = "sip" + c.SIP.Gateway.Codecs = []string{"PCMA", "PCMU"} + c.SIP.Gateway.MaxCalls = 8 + c.SIP.Gateway.RTPPortRange = allocUDPPortRange(8, + addressPortRange(c.SIP.Listen), c.GB28181.RTPPortRange) } } @@ -152,6 +176,9 @@ func StartTestServer(t *testing.T, opts ...Option) *TestServer { } s := core.NewServer(cfg) + if cfg.AudioCodec.Enabled { + s.StreamHub().SetAudioCodecEnabled(true) + } // Register modules based on what the options enabled. // Order matters: modules that register API handlers (e.g. GB28181) must @@ -188,6 +215,12 @@ func StartTestServer(t *testing.T, opts ...Option) *TestServer { } s.RegisterModule(gb28181mod.NewModule(sipModule.Service())) } + if cfg.SIP.Gateway.Enabled { + if sipModule == nil { + t.Fatal("WithSIPGateway requires WithSIP") + } + s.RegisterModule(sipgwmod.NewModule(sipModule.Service())) + } // API module must be registered last so cross-module handlers are available. if cfg.API.Enabled { @@ -267,6 +300,12 @@ func (ts *TestServer) Config() *config.Config { return ts.cfg } +// ModuleByName returns a registered module for integration tests that exercise +// the module's exported control-plane contract. +func (ts *TestServer) ModuleByName(name string) core.Module { + return ts.server.ModuleByName(name) +} + // StreamHasVideoGOP reports whether a published stream has a decodable video // start point available for playback integration tests. func (ts *TestServer) StreamHasVideoGOP(streamKey string) bool { @@ -274,6 +313,17 @@ func (ts *TestServer) StreamHasVideoGOP(streamKey string) bool { return ok && stream.Publisher() != nil && stream.GOPCacheDetail().VideoFrames > 0 } +// StreamHasAudio reports whether the active publisher declares codec and at +// least one audio frame has reached the shared stream hub. +func (ts *TestServer) StreamHasAudio(streamKey string, codec avframe.CodecType) bool { + stream, ok := ts.server.StreamHub().Find(streamKey) + if !ok || stream.Publisher() == nil { + return false + } + info := stream.Publisher().MediaInfo() + return info != nil && info.AudioCodec == codec && stream.Stats().AudioFrames > 0 +} + // Shutdown stops the server. It is safe to call multiple times; only the first // call performs the actual shutdown. func (ts *TestServer) Shutdown() { @@ -327,18 +377,54 @@ func allocUDPAddr() string { // allocUDPPortPair allocates a free even-numbered UDP port suitable as the // base of an RTP port range. The result is clamped so that base+100 stays // within the valid port space. -func allocUDPPortPair() int { - conn, err := net.ListenPacket("udp", "127.0.0.1:0") - if err != nil { - panic("allocUDPPortPair: " + err.Error()) +func allocUDPPortRange(pairCount int, excluded ...[]int) []int { + const ( + minimumPort = 35000 + maximumPort = 59999 + ) + portCount := pairCount * 2 + loopback := net.ParseIP("127.0.0.1") + for start := minimumPort; start+portCount-1 <= maximumPort; start += 2 { + end := start + portCount - 1 + overlaps := false + for _, other := range excluded { + if len(other) == 2 && start <= other[1] && other[0] <= end { + overlaps = true + break + } + } + if overlaps { + continue + } + + reservations := make([]*net.UDPConn, 0, portCount) + available := true + for port := start; port <= end; port++ { + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: loopback, Port: port}) + if err != nil { + available = false + break + } + reservations = append(reservations, conn) + } + for _, conn := range reservations { + _ = conn.Close() + } + if available { + return []int{start, end} + } } - port := conn.LocalAddr().(*net.UDPAddr).Port - conn.Close() - if port%2 != 0 { - port++ + panic("allocUDPPortRange: no contiguous loopback UDP range available") +} + +func addressPortRange(address string) []int { + _, portText, err := net.SplitHostPort(address) + if err != nil { + return nil } - if port+100 > 65535 { - port = 65400 // safe fallback + port, err := strconv.Atoi(portText) + if err != nil || port <= 0 { + return nil } - return port + return []int{port, port} } diff --git a/tools/testkit/testutil/server_test.go b/tools/testkit/testutil/server_test.go index b9904b4e..5fe689b6 100644 --- a/tools/testkit/testutil/server_test.go +++ b/tools/testkit/testutil/server_test.go @@ -3,6 +3,7 @@ package testutil import ( "net" "net/http" + "reflect" "testing" "time" ) @@ -162,3 +163,14 @@ func TestStartTestServer_ShutdownIdempotent(t *testing.T) { // Explicit shutdown before t.Cleanup fires should not panic. srv.Shutdown() } + +func TestStartTestServer_SIPGatewayAndGB28181Modules(t *testing.T) { + srv := StartTestServer(t, WithSIP(), WithGB28181(), WithSIPGateway()) + + if got, want := srv.server.ModuleNames(), []string{"sip", "gb28181", "sipgateway"}; !reflect.DeepEqual(got, want) { + t.Fatalf("module order = %v, want %v", got, want) + } + if !srv.Config().SIP.Gateway.Enabled { + t.Fatal("SIP gateway config was not enabled") + } +}