From 95ae1955dcff15c7cc4df77ebe66817dbc9b8094 Mon Sep 17 00:00:00 2001 From: Jacob Metoyer <150724801+Jacob-Met@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:50:31 -0700 Subject: [PATCH] Preserve recorded gaps in EMG transition features Use uninterrupted original sample pairs and triples for waveform length, zero crossings and slope-sign changes. Preserve amplitude features, window placement, sample validity and existing schema metadata. Add focused numerical regressions and real MCAP-to-Parquet receiving, consumer semantics and additive research progress. Refs #119. --- docs/EMG_GAP_FEATURES.md | 94 +++++++ docs/design/research/PROGRESS.md | 35 +++ .../capture_analysis/features/emg.py | 35 ++- tests/analysis/test_emg_gap_adjacency.py | 253 ++++++++++++++++++ 4 files changed, 412 insertions(+), 5 deletions(-) create mode 100644 docs/EMG_GAP_FEATURES.md create mode 100644 tests/analysis/test_emg_gap_adjacency.py diff --git a/docs/EMG_GAP_FEATURES.md b/docs/EMG_GAP_FEATURES.md new file mode 100644 index 00000000..543336bc --- /dev/null +++ b/docs/EMG_GAP_FEATURES.md @@ -0,0 +1,94 @@ +# EMG transition features across recorded gaps + +EMG waveform length, zero crossings and slope-sign changes use consecutive +observations within uninterrupted recorded spans. A disconnect must not create +a transition from the last sample before the gap to the first sample after it. +This applies even when no retained sample timestamp falls inside the gap. + +The correction is tracked in [issue #119](https://github.com/Jacob-Met/CaptureSuite/issues/119) +and implements the existing [analysis gap contract](design/ANALYSIS.md). + +## Feature definitions + +For each existing sample-count window, the extractor uses the recorded gap mask +and the original ordered sample timestamps: + +| Feature | Observations used | +| --- | --- | +| RMS and mean absolute value | All valid observed samples in the window, as before. | +| Waveform length (`*_wl`) | Absolute differences between originally adjacent valid samples whose time interval crosses no recorded gap. | +| Zero crossings (`*_zc`) | Those same adjacent pairs, counted when their product is strictly negative. | +| Slope-sign changes (`*_ssc`) | Originally consecutive triples for which both adjacent pairs are uninterrupted, counted when the two differences have a strictly negative product. | + +Gap endpoints follow the existing inclusive masking convention. An open gap ends +at the requested analysis window's end. Integer nanoseconds remain integers for +the gap intersection comparisons. A long timestamp interval by itself does not +create an inferred gap; only explicit recorded intervals and existing invalid +samples break adjacency. + +An isolated valid sample contributes to RMS and mean absolute value but has no +waveform-length, crossing or slope-sign-change contribution. Real changes +within each uninterrupted span remain in the result. + +### Small deterministic examples + +Each row describes valid observations separated by a recorded gap. + +| Valid observations within one window | Previous WL / ZC / SSC | Corrected WL / ZC / SSC | +| --- | --- | --- | +| `1, 1` then gap then `-1, -1` | `2 / 1 / 0` | `0 / 0 / 0` | +| `0, 1` then gap then `0, 1` | `3 / 0 / 2` | `2 / 0 / 0` | +| `0, 2, 1` then gap then `2, 1, 3` | `7 / 0 / 4` | `6 / 0 / 2` | + +The first example has RMS and mean absolute value of 1 on both versions. +If an additional retained sample is masked between the two spans, +`valid_fraction` is 4/5. If the recording contains only the four valid samples, +`valid_fraction` remains 1. This field describes the fraction of valid retained +observations; it is not elapsed-time coverage. Likewise, `t_mid_ns` remains the +median of valid sample timestamps and can lie within a gap. + +## Using corrected results + +Run a new feature job from a checkout containing the correction, using the +existing Python 3.12 analysis environment: + +```powershell +python tools/run_analysis.py features "C:\captures\example.mmsession" --gap-policy mask +``` + +The command prints the new job ID and directory. Inspect +`processing/jobs//features/emg//windows.csv` or the adjacent +`windows.parquet`. The accompanying `_schema.json` retains feature schema +version 1 and the established columns, units and calibration flags. + +Previous analysis jobs retain their original values. Keep them for comparison; +create a new job instead of passing an existing `--overwrite-job-id`. When +rebuilding an ML bundle, select the new feature job through the existing +`--features-job` argument so the downstream result records its actual input. +Raw MCAP data, sample timestamps and health records are not rewritten. + +The `mask` and `split` options retain the extractor's existing window placement +and amplitude aggregation. This fix prevents transition features from joining +separate spans; it does not introduce new output rows or redefine `split`. +The existing `fail` policy is unchanged. + +## Reproduce the regression + +```powershell +python -m pytest tests/analysis/test_emg_gap_adjacency.py -q +``` + +The regression file covers retained invalid samples, gaps between observations, +multiple channels, isolated samples, overlapping windows, open gaps, strict +crossing definitions and integer timestamps beyond floating-point precision. +Its full pipeline case writes two real protobuf MCAP batches at 40 Hz, records +a 50–100 ms disconnect, runs the public feature-job API, and reads both output +tables back. It verifies the corrected values, schema metadata and unchanged +raw-source hashes. The fixture is synthetic and contains no participant data. + +For the project verification gate, run the analysis and protocol suites with +the supported environment as described in the +[autonomous execution plan](design/research/AUTONOMOUS_EXECUTION_PLAN.md). +The numerical definitions remain provisional until qualified against tagged +real EMG data. These regressions establish offline analysis behavior; they do +not establish hardware or installed-desktop validation. diff --git a/docs/design/research/PROGRESS.md b/docs/design/research/PROGRESS.md index 3f8ab7ce..7f456be1 100644 --- a/docs/design/research/PROGRESS.md +++ b/docs/design/research/PROGRESS.md @@ -1,3 +1,38 @@ +## 2026-10-12 — Preserve recorded gaps in EMG transition features + +Contributor `chatgpt:7cb7e049e2c0/root`, issue #119, corrected artificial EMG +transitions caused by compacting valid samples before taking differences. +Waveform length and zero crossings now require originally adjacent valid +samples with no recorded gap between their timestamps; slope-sign changes +require two uninterrupted adjacent pairs. Explicit gaps also break adjacency +when no retained sample falls inside them. RMS, mean absolute value, window +placement, timestamps, valid-sample fractions and schema metadata retain their +definitions. Raw sources are preserved. + +The final 14 regression methods produce 10 failure records on the original +`3960c0c756cd4e9facbde0d76eaaa3c31ab7c167` source (including two integer-time +subcases). The candidate passes those methods and 39 inherited feature, stream +scope and numeric pipeline tests: 53 passes and six passing subtests on local +Python 3.12.14. A real protobuf-MCAP fixture with a recorded disconnect reaches +the public job API and both Parquet/CSV outputs with corrected transition +values, unchanged amplitude values and identical raw-source hashes. Touched +Python passes Ruff. The two emitted warnings are existing protobuf metaclass +deprecations concerning Python 3.14. + +A deterministic no-gap control (eight channels, 120,000 samples at 2 kHz, +1,199 windows) retains exactly equal table values and metadata. Ten alternating +local timings measured median wall time of 0.243 s before and 0.272 s after +(CPU: 0.243/0.272 s). The roughly 29 ms cost for this 60 s synthetic recording +is a documented correctness tradeoff, not a claim of unchanged throughput or +a hardware performance qualification. The continuous-input fast path and +per-window reuse of triple masks limit avoidable overhead. + +The change stays within the EMG extractor, its new regression file, this +additive progress entry and [consumer semantics](../../EMG_GAP_FEATURES.md). +Independent scientific review, supported Windows CI and an authorized native +receiving result are tracked separately on issue #119; local passing tests +do not imply installed-desktop or physical-acquisition validation. + ## 2026-10-08 — Preserve nearest feature pairs across retained row orders Contributor `chatgpt:/root/production_execution`, issue #86, repaired the diff --git a/libs/python/capture_analysis/capture_analysis/features/emg.py b/libs/python/capture_analysis/capture_analysis/features/emg.py index 7f753650..f16b6c6f 100644 --- a/libs/python/capture_analysis/capture_analysis/features/emg.py +++ b/libs/python/capture_analysis/capture_analysis/features/emg.py @@ -21,6 +21,17 @@ def _window_indices(n: int, win: int, hop: int) -> list[tuple[int, int]]: return out +def _uninterrupted_pairs(t: np.ndarray, valid: np.ndarray, gap_mask: GapMask) -> np.ndarray: + """Adjacent retained samples must not span a recorded gap, even an empty one.""" + pairs = valid[:-1] & valid[1:] + for gap in gap_mask.overlaps_window(): + end = gap.end_session_ns + if end is None: + end = gap_mask.window.end_session_ns + pairs &= ~((t[:-1] <= end) & (t[1:] >= gap.start_session_ns)) + return pairs + + def extract_emg_features( loaded: LoadedEmg, gap_mask: GapMask, @@ -41,12 +52,18 @@ def extract_emg_features( rows: list[dict] = [] x = loaded.X t = loaded.t_sample_ns + continuous = _uninterrupted_pairs(t, valid, gap_mask) + all_continuous = bool(np.all(continuous)) for i0, i1 in _window_indices(x.shape[1], win, hop): v = valid[i0:i1] if not np.any(v): continue seg = x[:, i0:i1][:, v] + pairs = continuous[i0:i1 - 1] + uninterrupted = all_continuous or bool(np.all(pairs)) + if not uninterrupted: + triples = pairs[:-1] & pairs[1:] t_mid = int(np.median(t[i0:i1][v])) row: dict = { "t_mid_ns": t_mid, @@ -56,11 +73,19 @@ def extract_emg_features( s = seg[c].astype(np.float64, copy=False) if s.size == 0: continue - diff = np.diff(s) - zc = int(np.sum((s[:-1] * s[1:]) < 0)) if s.size > 1 else 0 - ssc = ( - int(np.sum((diff[:-1] * diff[1:]) < 0)) if diff.size > 1 else 0 - ) + if uninterrupted: + diff = np.diff(s) + zc = int(np.sum((s[:-1] * s[1:]) < 0)) if s.size > 1 else 0 + ssc = int(np.sum((diff[:-1] * diff[1:]) < 0)) if diff.size > 1 else 0 + else: + original = x[c, i0:i1].astype(np.float64, copy=False) + left, right = original[:-1][pairs], original[1:][pairs] + diff = right - left + zc = int(np.sum((left * right) < 0)) + before = original[:-2][triples] + middle = original[1:-1][triples] + after = original[2:][triples] + ssc = int(np.sum(((middle - before) * (after - middle)) < 0)) row[f"{ch}_rms"] = float(np.sqrt(np.mean(s * s))) row[f"{ch}_mav"] = float(np.mean(np.abs(s))) row[f"{ch}_wl"] = float(np.sum(np.abs(diff))) if diff.size else 0.0 diff --git a/tests/analysis/test_emg_gap_adjacency.py b/tests/analysis/test_emg_gap_adjacency.py new file mode 100644 index 00000000..4ac8b997 --- /dev/null +++ b/tests/analysis/test_emg_gap_adjacency.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: GPL-3.0-only +"""EMG transition features need consecutive observations across recorded gaps.""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import tempfile +import unittest +from pathlib import Path + +import numpy as np +from capture_analysis.features.emg import extract_emg_features +from capture_analysis.types import GapInterval, GapMask, LoadedEmg, StreamRef, TimeWindow + + +def write_gapped_emg_package(destination: Path) -> Path: + """Two real MCAP batches with a recorded, unobserved disconnect between them.""" + from capture_protocol.generated.capture.v1.data import emg_batch_pb2 + from mcap.writer import Writer + + fixture = Path(__file__).resolve().parents[1] / "fixtures" / "mini_session" + shutil.copytree(fixture, destination) + shutil.rmtree(destination / "sources" / "sim.imu.upper") + manifest_path = destination / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["sourceIds"] = ["sim.emg.main"] + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + source = destination / "sources" / "sim.emg.main" + stream = source / "streams" / "sim.emg.main.batch" + stream_path = stream / "stream.json" + description = json.loads(stream_path.read_text(encoding="utf-8")) + description.update(nominalRateHz=40.0, dimensions=[1]) + stream_path.write_text(json.dumps(description), encoding="utf-8") + mcap_path = stream / "segments" / "000000.mcap" + with mcap_path.open("wb") as output: + writer = Writer(output) + writer.start(profile="", library="capturesuite-gap-regression") + schema = writer.register_schema(name="emg.batch/1", encoding="protobuf", data=b"") + channel = writer.register_channel( + topic="emg", message_encoding="protobuf", schema_id=schema + ) + for sequence, (start, level) in enumerate([(0, 1.0), (125_000_000, -1.0)]): + message = emg_batch_pb2.EmgBatch() + message.timing.session_time_ns = start + message.timing.sequence_number = sequence + message.first_sample_index = sequence * 5 + message.sample_count = 2 + message.channel_ids.append("ch0") + message.samples_f32_le = np.array([level, level], dtype="