From 72019700dd4f0d0b3fd47fd241abbd1398e407ec Mon Sep 17 00:00:00 2001 From: Brackyt <60280126+Brackyt@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:19:13 +0200 Subject: [PATCH 1/7] fix(gen5): lenient v18 decode for real WHOOP 5 hardware captures Protocol's strict gravity gate rejected every v18 record in a hardware export (fw 50.40.1.0) while v20/v26 deep buffers dominated offload. Recover HR/RR via a hardware fallback with alternate unix offset and honest absent accel when gravity fails validation. --- lib/ble/ble_engine.dart | 89 +++++++++++++++++++++++- test/gen5_v18_hardware_lenient_test.dart | 84 ++++++++++++++++++++++ 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 test/gen5_v18_hardware_lenient_test.dart diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index 774cb45..232597e 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -84,6 +84,82 @@ typedef ArchiveSink = Future Function(ArchiveRecord archive); /// trigger now that listening is continuous and there's no discrete sync end. typedef DataStoredSink = void Function(); +/// Plausible unix window for gen5 historical records (2020-01-01 .. 2030-01-01). +@visibleForTesting +bool gen5V18UnixPlausible(int unix) => + unix >= 1577836800 && unix < 1893456000; + +/// Read v18 unix from inner bytes. Protocol's shared header uses inner[7:11]; +/// real WHOOP 5 hardware exports (fw 50.40.x) occasionally land unix one byte +/// earlier when the record is padded to 112 B instead of the fixture's 116 B. +@visibleForTesting +int? gen5V18UnixFromInner(Uint8List inner) { + if (inner.length < 11) return null; + final view = inner.buffer.asByteData(inner.offsetInBytes, inner.lengthInBytes); + final at7 = view.getUint32(7, Endian.little); + if (gen5V18UnixPlausible(at7)) return at7; + if (inner.length >= 10) { + final at6 = view.getUint32(6, Endian.little); + if (gen5V18UnixPlausible(at6)) return at6; + } + return null; +} + +/// Hardware lenient v18 decode when [parseGen5Historical] returns null because +/// the protocol decoder's gravity/dynamic-accel gates reject real captures +/// (off-wrist / motion-heavy seconds still carry valid HR/RR). Never fabricates +/// gravity — ax/ay/az stay null when the vector fails the magnitude gate. +@visibleForTesting +Sample? sampleFromGen5V18Lenient(Uint8List inner) { + if (inner.length < kGen5V18MinInnerLen || inner[1] != 18) return null; + final unix = gen5V18UnixFromInner(inner); + if (unix == null) return null; + final view = inner.buffer.asByteData(inner.offsetInBytes, inner.lengthInBytes); + final counter = view.getUint32(3, Endian.little); + final hr = inner[14]; + if (hr != 0 && (hr < 25 || hr > 230)) return null; + + const minRrMs = 200; + const maxRrMs = 2500; + final declaredRr = inner[15]; + final rr = []; + if (declaredRr <= 4) { + for (int i = 0; i < declaredRr && 16 + 2 * i + 2 <= inner.length; i++) { + final val = view.getInt16(16 + 2 * i, Endian.little); + if (val >= minRrMs && val <= maxRrMs) rr.add(val); + } + } + + double? ax; + double? ay; + double? az; + if (inner.length >= 49) { + final gx = view.getFloat32(37, Endian.little); + final gy = view.getFloat32(41, Endian.little); + final gz = view.getFloat32(45, Endian.little); + if (gx.isFinite && gy.isFinite && gz.isFinite) { + final magSq = gx * gx + gy * gy + gz * gz; + // Same gate as protocol's Gen5V18Decoder — but abstain on ax/ay/az + // instead of rejecting the whole record (HR/RR are independent fields). + if (magSq >= 0.25 && magSq <= 2.25) { + ax = gx; + ay = gy; + az = gz; + } + } + } + + return Sample( + tsEpoch: unix, + counter: counter, + hr: hr, + rrIntervalsMs: rr, + ax: ax, + ay: ay, + az: az, + ); +} + /// Map a decoded gen5 historical record onto the band-agnostic `Sample` type, /// or null when this record kind has no `Sample` equivalent (yet). /// @@ -113,6 +189,17 @@ Sample? sampleFromGen5Historical(Gen5HistoricalRecord? g) { ); } +/// Decode a gen5 historical inner frame to a band-agnostic [Sample], or null. +@visibleForTesting +Sample? decodeGen5HistoricalSample(Uint8List inner) { + final strict = sampleFromGen5Historical(parseGen5Historical(inner)); + if (strict != null) return strict; + if (inner.length > 1 && inner[1] == 18) { + return sampleFromGen5V18Lenient(inner); + } + return null; +} + @visibleForTesting int countHistoricalBurstPackets({ required Map dataPacketCountsByRevision, @@ -1959,7 +2046,7 @@ class BleEngine { // own raw-buffer storage (a future db table), not a 1Hz Sample, so they // fall through to the undecodable archive below — that is honest // (correctly-identified-but-not-yet-stored), not a decode failure. - sample = sampleFromGen5Historical(parseGen5Historical(frame.inner)); + sample = decodeGen5HistoricalSample(frame.inner); } else if (recType == Record.r24 || recType == Record.r12) { // Legacy decoder first, firmware-fallback chain second, undecodable // archive last — see FirmwareAwareR24Decoder. diff --git a/test/gen5_v18_hardware_lenient_test.dart b/test/gen5_v18_hardware_lenient_test.dart new file mode 100644 index 0000000..de7daa0 --- /dev/null +++ b/test/gen5_v18_hardware_lenient_test.dart @@ -0,0 +1,84 @@ +// Pins the WHOOP 5 hardware v18 lenient decode path against real captures +// from openstrap_export_1785863370590.db (fw 50.40.1.0). Protocol's strict +// Gen5V18Decoder rejects every one of these on gravity/dyn gates; the lenient +// path must still recover HR + unix without fabricating accel. + +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/ble_engine.dart'; +import 'package:openstrap_protocol/openstrap_protocol.dart'; + +Uint8List hex(String s) { + final clean = s.replaceAll(' ', ''); + final out = Uint8List(clean.length ~/ 2); + for (int i = 0; i < out.length; i++) { + out[i] = int.parse(clean.substring(i * 2, i * 2 + 2), radix: 16); + } + return out; +} + +void main() { + group('gen5V18UnixFromInner — hardware export', () { + final inner = hex( + '2f1280540df701737c6a915c2f004d0000000000000000000021fb608c4d0000' + 'c330ebf63ecd4ad43f854b653e5298863da3017400000000000000000039014401' + '040d6003010c020c3100000000000000000000000000000000000000000000011f' + 'bed68080000000a80372c0000000', + ); + + test('strict protocol decode returns null (gravity gate)', () { + expect(parseGen5Historical(inner), isNull); + }); + + test('unix lands at inner[6] on 112-byte padded captures', () { + expect(gen5V18UnixFromInner(inner), 1786540801); + }); + + test('lenient path recovers HR without fabricating gravity', () { + final sample = sampleFromGen5V18Lenient(inner); + expect(sample, isNotNull); + expect(sample!.tsEpoch, 1786540801); + expect(sample.hr, 77); + expect(sample.ax, isNull); + expect(sample.ay, isNull); + expect(sample.az, isNull); + }); + + test('decodeGen5HistoricalSample prefers strict then lenient', () { + final sample = decodeGen5HistoricalSample(inner); + expect(sample, isNotNull); + expect(sample!.hr, 77); + }); + }); + + group('sampleFromGen5V18Lenient — all hardware-export v18 fixtures', () { + // One row per undecodable_rec_v18 in the export DB (12 total). + final fixtures = [ + '2f1280540df701737c6a915c2f004d0000000000000000000021fb608c4d0000c330ebf63ecd4ad43f854b653e5298863da3017400000000000000000039014401040d6003010c020c3100000000000000000000000000000000000000000000011fbed68080000000a80372c0000000', + '2f1280fa2af70142716af39939004700000000000000000000606f0781470000d820abee3e52c0ac3e00a0b93dcdcc9abe8702dd0000000000000000003e0149013b0d6003010c020c0000000000000000000000000000000000000000000000010081b780800000006c5c6fc0000000', + '2f12805a3ef701a2846af35158004100000000000000000000008e0788410000ff80ba953e5220863e9a59163e33bbbf3ec403d3000000000000000000490154018b0d6003010c020c00000000000000000000000000000000000000000000000100c2fa80800000006b34a0c0000000', + '2f1280494af80140b26a9d8f02003a00000000000000000000217c4a833c000077a2902d3f661e96be14ae103c3d0ad4bd001c6e0000000000000000004c015101760d6003010c020c21000000000000000000000000000000000000000000000100b6fc8080000000fd4a97c0000000', + '2f1280924ff80189b76a9deb11003d0000000000000000000020f6468e3f00005de3ddef3f5caf89bde1ba29bec345463e091c6e0000000000000000004c015401920d6003010c020c20000000000000000000000000000000000000000000000100b0d78080000000d4c765c0000000', + '2f12806d62f801aa826a64e17a004e0000000000000000000071f40f884e0000d72a29613e854b70beec91d5beec515b3d4d1da60000000000000000003b014601260d6003010c020c0100000000000000000000000000000000000000000000010092e38080000000612677c0000000', + '2f1280877ff801f0746aa9cc4c004f0000000000000000000270d808874f0000cbd67d02414879723f852ba83ec305013e8d1e84000000000000000000460151017b0d6003010c020c0000000000000000000000000000000000000000000000010098d08080000000091711c0000000', + '2f12809ca1f80114886a974721005b000000000000000000006176078d5b0000bff0d53b3e29dc2d3e9aa19c3e33f3b53e4c207b0000000000000000003e014901480d6003010c020c01000000000000000000000000000000000000000000000100a1f58080000000f9b43ac0000000', + '2f128070d2f801e8b86a97146e00750000000000000000000078520883750000d8e038b23e1fb541becd6c953dc3f5483e1e25ff00000001000000000057015701600d6004010c020c08000000000000000000000000000000000000000000000100617d8080000000bf4c66c0000000', + '2f128069d4f801e1ba6a97eb71005e00000000000000000000615403805e0000d1fc60963ee17a053e0ae7753e52b8213e2027bd00000001000000000036014101f70c6004010c020c01000000000000000000000000000000000000000000000100617d808000000090aaa9bf000000', + '2f12809bd9f80113c06a97997900580000000000000000000061e10b86580000dad8ea093f9a49d5be00000bbcec01623ef427620000000100000000003b014901340d6004010c020c01000000000000000000000000000000000000000000000100689380800000001e8753c0000000', + '2f128071dbf801e9c16a97287c00650000000000000000000071420e87650000daa86c493f52b83cbe9a693b3e858b7f3e6d28ff00000000000000000041014c01340d6004010c020c01000000000000000000000000000000000000000000000100ecf88080000000af6201c0000000', + ]; + + test('every export v18 decodes to a sample with plausible unix + hr', () { + for (final h in fixtures) { + final inner = hex(h); + expect(parseGen5Historical(inner), isNull, + reason: 'strict decode should still reject these fixtures'); + final sample = sampleFromGen5V18Lenient(inner); + expect(sample, isNotNull, reason: 'lenient decode failed for $h'); + expect(gen5V18UnixPlausible(sample!.tsEpoch), isTrue); + expect(sample.hr == 0 || (sample.hr >= 25 && sample.hr <= 230), isTrue); + } + }); + }); +} From 638c16d75d845e3cc113d6f3f565363a0e338b93 Mon Sep 17 00:00:00 2001 From: Brackyt <60280126+Brackyt@users.noreply.github.com> Date: Tue, 4 Aug 2026 20:15:02 +0200 Subject: [PATCH 2/7] fix(gen5): honest v18 unix + SET_CLOCK revision byte Drop the misaligned unix@6 fallback that could invent timestamps, and prepend revision1 on gen5 SET_CLOCK/GET_CLOCK only (WHOOP 4 path unchanged). Hardware showed Invalid revision when body[0] was the epoch low byte; a later connect with the 9-byte form correlated with drift=0. --- lib/ble/ble_engine.dart | 88 ++++++++++++------- test/gen5_v18_hardware_lenient_test.dart | 103 ++++++++++++----------- 2 files changed, 114 insertions(+), 77 deletions(-) diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index 232597e..0d99538 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -89,26 +89,26 @@ typedef DataStoredSink = void Function(); bool gen5V18UnixPlausible(int unix) => unix >= 1577836800 && unix < 1893456000; -/// Read v18 unix from inner bytes. Protocol's shared header uses inner[7:11]; -/// real WHOOP 5 hardware exports (fw 50.40.x) occasionally land unix one byte -/// earlier when the record is padded to 112 B instead of the fixture's 116 B. +/// Read v18 unix from the protocol shared header at inner[7:11]. +/// +/// Do **not** fall back to a misaligned offset-6 read: that overlaps the +/// record-index high byte and can invent a year-window-plausible timestamp +/// that is not monotonic with `recordIndex` (honesty violation — observed on +/// the fw 50.40.1.0 export where unix@7 was garbage because SET_CLOCK never +/// latched). Absent a plausible unix@7, abstain. @visibleForTesting int? gen5V18UnixFromInner(Uint8List inner) { if (inner.length < 11) return null; final view = inner.buffer.asByteData(inner.offsetInBytes, inner.lengthInBytes); final at7 = view.getUint32(7, Endian.little); - if (gen5V18UnixPlausible(at7)) return at7; - if (inner.length >= 10) { - final at6 = view.getUint32(6, Endian.little); - if (gen5V18UnixPlausible(at6)) return at6; - } - return null; + return gen5V18UnixPlausible(at7) ? at7 : null; } /// Hardware lenient v18 decode when [parseGen5Historical] returns null because /// the protocol decoder's gravity/dynamic-accel gates reject real captures /// (off-wrist / motion-heavy seconds still carry valid HR/RR). Never fabricates /// gravity — ax/ay/az stay null when the vector fails the magnitude gate. +/// Never fabricates time — requires a plausible unix@7. @visibleForTesting Sample? sampleFromGen5V18Lenient(Uint8List inner) { if (inner.length < kGen5V18MinInnerLen || inner[1] != 18) return null; @@ -160,6 +160,30 @@ Sample? sampleFromGen5V18Lenient(Uint8List inner) { ); } +/// Build gen5 SET_CLOCK_MAVERICK (0x92) / GET_CLOCK_GEN5 (0x93) payloads. +/// +/// Hardware evidence (fw 50.40.1.0 console): an empty GET_CLOCK body logs +/// `Invalid revision for get clock: 0`, and SET_CLOCK without a leading +/// form/revision byte logs `Invalid revision ` — i.e. the strap +/// treats body[0] as the command revision (same role as HELLO's `[0x01]` / +/// Alec's gen5 `b3`). Prepend [revision1] so the 8-byte time field starts at +/// body[1]. +@visibleForTesting +List gen5SetClockPayload({required int sec, required int subsec}) => [ + revision1, + sec & 0xff, + (sec >> 8) & 0xff, + (sec >> 16) & 0xff, + (sec >> 24) & 0xff, + subsec & 0xff, + (subsec >> 8) & 0xff, + 0, + 0, + ]; + +@visibleForTesting +List gen5GetClockPayload() => const [revision1]; + /// Map a decoded gen5 historical record onto the band-agnostic `Sample` type, /// or null when this record kind has no `Sample` equivalent (yet). /// @@ -2977,26 +3001,26 @@ class BleEngine { final ms = DateTime.now().millisecondsSinceEpoch; final sec = ms ~/ 1000; final subsec = ((ms % 1000) * 32768) ~/ 1000; // 0..32767, 1/32768 s units - final payload = [ - sec & 0xff, - (sec >> 8) & 0xff, - (sec >> 16) & 0xff, - (sec >> 24) & 0xff, - subsec & 0xff, - (subsec >> 8) & 0xff, - 0, - 0, - ]; - // gen5 ("Maverick") uses a DIFFERENT opcode for SET_CLOCK than gen4 — the - // 8-byte payload shape is unchanged, only the opcode value differs (see - // Cmd.setClockMaverick's doc in protocol/constants.dart). Sending gen4's - // opcode 0x0A to a gen5 strap here would silently fail to latch the RTC, - // which then refuses to serve type-47 history — the exact symptom fixed. + // gen5 ("Maverick") uses a DIFFERENT opcode for SET_CLOCK than gen4, and + // (per fw 50.40.1.0 console) also needs a leading revision/form byte — see + // [gen5SetClockPayload]. Gen4 keeps the hardware-verified 8-byte body. final isGen5 = _session?.band.isGen5 ?? false; + final payload = isGen5 + ? gen5SetClockPayload(sec: sec, subsec: subsec) + : [ + sec & 0xff, + (sec >> 8) & 0xff, + (sec >> 16) & 0xff, + (sec >> 24) & 0xff, + subsec & 0xff, + (subsec >> 8) & 0xff, + 0, + 0, + ]; final opcode = isGen5 ? Cmd.setClockMaverick : Cmd.setClock; await _send(opcode, payload); _log('SET_CLOCK${isGen5 ? " (gen5 Maverick)" : ""} → sec=$sec ' - 'subsec=$subsec (WHOOP-exact 8B).'); + 'subsec=$subsec (${payload.length}B${isGen5 ? ", rev=$revision1" : ""}).'); // Read the RTC back so the GET_CLOCK response handler can confirm it latched // (and re-issue SET_CLOCK if the strap clock is still off — see _onDecoded). await getClock(); @@ -3004,11 +3028,15 @@ class BleEngine { /// Read the strap RTC. The response carries `clock_epoch`, handled where we /// verify drift and re-correlate the strap-RTC ↔ wall clock. gen5 uses its - /// own GET_CLOCK opcode (147) — see [setClock]. - Future getClock() => _send( - (_session?.band.isGen5 ?? false) ? Cmd.getClockGen5 : Cmd.getClock, - const [], - ); + /// own GET_CLOCK opcode (147) and needs a leading revision byte — see + /// [gen5GetClockPayload] / [setClock]. + Future getClock() { + final isGen5 = _session?.band.isGen5 ?? false; + return _send( + isGen5 ? Cmd.getClockGen5 : Cmd.getClock, + isGen5 ? gen5GetClockPayload() : const [], + ); + } /// On-device wake alarm (SET_ALARM_TIME = 0x42) — the RICH 20-byte form that /// actually FIRES on WHOOP 4.0: diff --git a/test/gen5_v18_hardware_lenient_test.dart b/test/gen5_v18_hardware_lenient_test.dart index de7daa0..8959b97 100644 --- a/test/gen5_v18_hardware_lenient_test.dart +++ b/test/gen5_v18_hardware_lenient_test.dart @@ -1,7 +1,5 @@ -// Pins the WHOOP 5 hardware v18 lenient decode path against real captures -// from openstrap_export_1785863370590.db (fw 50.40.1.0). Protocol's strict -// Gen5V18Decoder rejects every one of these on gravity/dyn gates; the lenient -// path must still recover HR + unix without fabricating accel. +// Pins WHOOP 5 hardware v18 + clock payload behaviour against real evidence +// from openstrap_export_1785863370590.db / openstrap_sync.log (fw 50.40.1.0). import 'dart:typed_data'; @@ -19,7 +17,30 @@ Uint8List hex(String s) { } void main() { - group('gen5V18UnixFromInner — hardware export', () { + group('gen5 clock payloads — fw 50.40.1.0 Invalid revision evidence', () { + test('SET_CLOCK prepends revision1 before the 8-byte time', () { + final p = gen5SetClockPayload(sec: 1785710096, subsec: 16351); + expect(p, hasLength(9)); + expect(p[0], revision1); + // Seconds LE start at body[1] — not body[0] (that was read as "revision"). + expect(p[1], 1785710096 & 0xff); + expect(p[2], (1785710096 >> 8) & 0xff); + expect(p[3], (1785710096 >> 16) & 0xff); + expect(p[4], (1785710096 >> 24) & 0xff); + expect(p[5], 16351 & 0xff); + expect(p[6], (16351 >> 8) & 0xff); + expect(p[7], 0); + expect(p[8], 0); + }); + + test('GET_CLOCK sends revision1 (empty body logged revision 0)', () { + expect(gen5GetClockPayload(), [revision1]); + }); + }); + + group('gen5V18UnixFromInner — no fabricated misaligned unix', () { + // Real archive blob #1: unix@7 is garbage; offset-6 is a year-plausible + // false positive that is NOT monotonic with recordIndex. final inner = hex( '2f1280540df701737c6a915c2f004d0000000000000000000021fb608c4d0000' 'c330ebf63ecd4ad43f854b653e5298863da3017400000000000000000039014401' @@ -27,58 +48,46 @@ void main() { 'bed68080000000a80372c0000000', ); - test('strict protocol decode returns null (gravity gate)', () { + test('strict protocol decode returns null (gravity/dyn gate)', () { expect(parseGen5Historical(inner), isNull); }); - test('unix lands at inner[6] on 112-byte padded captures', () { - expect(gen5V18UnixFromInner(inner), 1786540801); + test('unix@7 garbage → abstain (do not invent offset-6 time)', () { + expect(gen5V18UnixFromInner(inner), isNull); + expect(sampleFromGen5V18Lenient(inner), isNull); + expect(decodeGen5HistoricalSample(inner), isNull); }); + }); - test('lenient path recovers HR without fabricating gravity', () { + group('sampleFromGen5V18Lenient — gravity-only abstention needs good unix', () { + // Synthetic: valid shared header unix@7 + HR, gravity out of gate range. + // Layout matches Gen5HistoricalHeader + HR @14. + test('recovers HR when unix@7 plausible and gravity fails gate', () { + final inner = Uint8List(112); + inner[0] = 0x2f; + inner[1] = 18; + inner[2] = 0x80; + // recordIndex = 1 + inner[3] = 1; + // unix = 1785801600 (2026-08-04 00:00:00 UTC) + const unix = 1785801600; + inner.buffer.asByteData().setUint32(7, unix, Endian.little); + inner[14] = 72; // HR + inner[15] = 0; // no RR + // dynAccel = 0.5 (ok), gravity magnitude ~0.1 (fails 0.5..1.5 gate) + inner.buffer.asByteData().setFloat32(33, 0.5, Endian.little); + inner.buffer.asByteData().setFloat32(37, 0.05, Endian.little); + inner.buffer.asByteData().setFloat32(41, 0.05, Endian.little); + inner.buffer.asByteData().setFloat32(45, 0.05, Endian.little); + + expect(parseGen5Historical(inner), isNull); final sample = sampleFromGen5V18Lenient(inner); expect(sample, isNotNull); - expect(sample!.tsEpoch, 1786540801); - expect(sample.hr, 77); + expect(sample!.tsEpoch, unix); + expect(sample.hr, 72); expect(sample.ax, isNull); expect(sample.ay, isNull); expect(sample.az, isNull); }); - - test('decodeGen5HistoricalSample prefers strict then lenient', () { - final sample = decodeGen5HistoricalSample(inner); - expect(sample, isNotNull); - expect(sample!.hr, 77); - }); - }); - - group('sampleFromGen5V18Lenient — all hardware-export v18 fixtures', () { - // One row per undecodable_rec_v18 in the export DB (12 total). - final fixtures = [ - '2f1280540df701737c6a915c2f004d0000000000000000000021fb608c4d0000c330ebf63ecd4ad43f854b653e5298863da3017400000000000000000039014401040d6003010c020c3100000000000000000000000000000000000000000000011fbed68080000000a80372c0000000', - '2f1280fa2af70142716af39939004700000000000000000000606f0781470000d820abee3e52c0ac3e00a0b93dcdcc9abe8702dd0000000000000000003e0149013b0d6003010c020c0000000000000000000000000000000000000000000000010081b780800000006c5c6fc0000000', - '2f12805a3ef701a2846af35158004100000000000000000000008e0788410000ff80ba953e5220863e9a59163e33bbbf3ec403d3000000000000000000490154018b0d6003010c020c00000000000000000000000000000000000000000000000100c2fa80800000006b34a0c0000000', - '2f1280494af80140b26a9d8f02003a00000000000000000000217c4a833c000077a2902d3f661e96be14ae103c3d0ad4bd001c6e0000000000000000004c015101760d6003010c020c21000000000000000000000000000000000000000000000100b6fc8080000000fd4a97c0000000', - '2f1280924ff80189b76a9deb11003d0000000000000000000020f6468e3f00005de3ddef3f5caf89bde1ba29bec345463e091c6e0000000000000000004c015401920d6003010c020c20000000000000000000000000000000000000000000000100b0d78080000000d4c765c0000000', - '2f12806d62f801aa826a64e17a004e0000000000000000000071f40f884e0000d72a29613e854b70beec91d5beec515b3d4d1da60000000000000000003b014601260d6003010c020c0100000000000000000000000000000000000000000000010092e38080000000612677c0000000', - '2f1280877ff801f0746aa9cc4c004f0000000000000000000270d808874f0000cbd67d02414879723f852ba83ec305013e8d1e84000000000000000000460151017b0d6003010c020c0000000000000000000000000000000000000000000000010098d08080000000091711c0000000', - '2f12809ca1f80114886a974721005b000000000000000000006176078d5b0000bff0d53b3e29dc2d3e9aa19c3e33f3b53e4c207b0000000000000000003e014901480d6003010c020c01000000000000000000000000000000000000000000000100a1f58080000000f9b43ac0000000', - '2f128070d2f801e8b86a97146e00750000000000000000000078520883750000d8e038b23e1fb541becd6c953dc3f5483e1e25ff00000001000000000057015701600d6004010c020c08000000000000000000000000000000000000000000000100617d8080000000bf4c66c0000000', - '2f128069d4f801e1ba6a97eb71005e00000000000000000000615403805e0000d1fc60963ee17a053e0ae7753e52b8213e2027bd00000001000000000036014101f70c6004010c020c01000000000000000000000000000000000000000000000100617d808000000090aaa9bf000000', - '2f12809bd9f80113c06a97997900580000000000000000000061e10b86580000dad8ea093f9a49d5be00000bbcec01623ef427620000000100000000003b014901340d6004010c020c01000000000000000000000000000000000000000000000100689380800000001e8753c0000000', - '2f128071dbf801e9c16a97287c00650000000000000000000071420e87650000daa86c493f52b83cbe9a693b3e858b7f3e6d28ff00000000000000000041014c01340d6004010c020c01000000000000000000000000000000000000000000000100ecf88080000000af6201c0000000', - ]; - - test('every export v18 decodes to a sample with plausible unix + hr', () { - for (final h in fixtures) { - final inner = hex(h); - expect(parseGen5Historical(inner), isNull, - reason: 'strict decode should still reject these fixtures'); - final sample = sampleFromGen5V18Lenient(inner); - expect(sample, isNotNull, reason: 'lenient decode failed for $h'); - expect(gen5V18UnixPlausible(sample!.tsEpoch), isTrue); - expect(sample.hr == 0 || (sample.hr >= 25 && sample.hr <= 230), isTrue); - } - }); }); } From 0ab4e4a09e032037a0ff90a74feb100c14264ded Mon Sep 17 00:00:00 2001 From: Brackyt <60280126+Brackyt@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:58:40 +0200 Subject: [PATCH 3/7] fix(gen5): persist v18 samples to decoded_onehz + align unix gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gen5 v18/lenient samples lack gen4 optics so R24 decode fails and decoded_onehz stayed empty — fall back to the BLE-preferred Sample when tsEpoch > 0, excluding gen4 R10-lite hr-only records. Align gen5V18UnixFromInner with RecordGate's isPlausibleUnix(wallNow) so implausible timestamps archive instead of silently dropping after decode. Restore git-sourced analytics lock (cbbe06a). --- lib/ble/ble_engine.dart | 22 +- lib/data/db.dart | 56 +++-- test/gen5_decoded_onehz_persistence_test.dart | 213 ++++++++++++++++++ test/gen5_v18_hardware_lenient_test.dart | 10 +- 4 files changed, 266 insertions(+), 35 deletions(-) create mode 100644 test/gen5_decoded_onehz_persistence_test.dart diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index 0d99538..ac37816 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -84,11 +84,6 @@ typedef ArchiveSink = Future Function(ArchiveRecord archive); /// trigger now that listening is continuous and there's no discrete sync end. typedef DataStoredSink = void Function(); -/// Plausible unix window for gen5 historical records (2020-01-01 .. 2030-01-01). -@visibleForTesting -bool gen5V18UnixPlausible(int unix) => - unix >= 1577836800 && unix < 1893456000; - /// Read v18 unix from the protocol shared header at inner[7:11]. /// /// Do **not** fall back to a misaligned offset-6 read: that overlaps the @@ -97,11 +92,11 @@ bool gen5V18UnixPlausible(int unix) => /// the fw 50.40.1.0 export where unix@7 was garbage because SET_CLOCK never /// latched). Absent a plausible unix@7, abstain. @visibleForTesting -int? gen5V18UnixFromInner(Uint8List inner) { +int? gen5V18UnixFromInner(Uint8List inner, int wallNow) { if (inner.length < 11) return null; final view = inner.buffer.asByteData(inner.offsetInBytes, inner.lengthInBytes); final at7 = view.getUint32(7, Endian.little); - return gen5V18UnixPlausible(at7) ? at7 : null; + return isPlausibleUnix(at7, wallNow) ? at7 : null; } /// Hardware lenient v18 decode when [parseGen5Historical] returns null because @@ -110,9 +105,9 @@ int? gen5V18UnixFromInner(Uint8List inner) { /// gravity — ax/ay/az stay null when the vector fails the magnitude gate. /// Never fabricates time — requires a plausible unix@7. @visibleForTesting -Sample? sampleFromGen5V18Lenient(Uint8List inner) { +Sample? sampleFromGen5V18Lenient(Uint8List inner, int wallNow) { if (inner.length < kGen5V18MinInnerLen || inner[1] != 18) return null; - final unix = gen5V18UnixFromInner(inner); + final unix = gen5V18UnixFromInner(inner, wallNow); if (unix == null) return null; final view = inner.buffer.asByteData(inner.offsetInBytes, inner.lengthInBytes); final counter = view.getUint32(3, Endian.little); @@ -215,11 +210,11 @@ Sample? sampleFromGen5Historical(Gen5HistoricalRecord? g) { /// Decode a gen5 historical inner frame to a band-agnostic [Sample], or null. @visibleForTesting -Sample? decodeGen5HistoricalSample(Uint8List inner) { +Sample? decodeGen5HistoricalSample(Uint8List inner, int wallNow) { final strict = sampleFromGen5Historical(parseGen5Historical(inner)); if (strict != null) return strict; if (inner.length > 1 && inner[1] == 18) { - return sampleFromGen5V18Lenient(inner); + return sampleFromGen5V18Lenient(inner, wallNow); } return null; } @@ -2061,6 +2056,7 @@ class BleEngine { // backfill (all received in one sync) splits into correct per-real-day // buckets instead of collapsing into one "today". Sample? sample; + final wallNow = DateTime.now().millisecondsSinceEpoch ~/ 1000; final isGen5 = _session?.band.isGen5 ?? false; if (isGen5) { // gen5 (WHOOP 5): `parseGen5Historical` dispatches across all four real @@ -2070,7 +2066,7 @@ class BleEngine { // own raw-buffer storage (a future db table), not a 1Hz Sample, so they // fall through to the undecodable archive below — that is honest // (correctly-identified-but-not-yet-stored), not a decode failure. - sample = decodeGen5HistoricalSample(frame.inner); + sample = decodeGen5HistoricalSample(frame.inner, wallNow); } else if (recType == Record.r24 || recType == Record.r12) { // Legacy decoder first, firmware-fallback chain second, undecodable // archive last — see FirmwareAwareR24Decoder. @@ -2134,7 +2130,7 @@ class BleEngine { // Past this point [sample] is non-null — undecodable records returned above. if (!_recordGate.admit( sample.tsEpoch, - wallNow: DateTime.now().millisecondsSinceEpoch ~/ 1000, + wallNow: wallNow, sessionOldestUnix: _sessionOldestUnix, sessionNewestUnix: _sessionNewestUnix, )) { diff --git a/lib/data/db.dart b/lib/data/db.dart index 33d11f7..efa76ca 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -11,6 +11,7 @@ import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; import 'package:openstrap_analytics/onehz.dart' as ana; import 'package:openstrap_protocol/openstrap_protocol.dart' as proto; @@ -1856,32 +1857,51 @@ class LocalDb { static String _localDayLabelFromEpoch(int epochSec) => _localDayLabel(DateTime.fromMillisecondsSinceEpoch(epochSec * 1000)); + /// Gen4 historical R10-lite (hr-only, no accel/optical) must stay out of + /// `decoded_onehz` — they belong in the legacy `samples` table only. + static bool _isGen4R10LiteHistorical(Uint8List inner) => + inner.isNotEmpty && + inner[0] == proto.PacketType.historicalData && + inner.length > 1 && + inner[1] == proto.Record.r10; + static Sample? _decodeOneHzSample(RawRecord raw, {Sample? preferred}) { if (preferred != null && preferred.hasDecodedOneHz) return preferred; + Uint8List bytes; + try { + bytes = proto.hexToBytes(raw.hex); + } catch (_) { + return null; + } try { // Legacy decoder first, firmware-fallback chain second — see // FirmwareAwareR24Decoder. This path only runs when no pre-decoded // `preferred` sample was supplied (e.g. a raw-hex import/merge), so a // fresh per-call instance is fine — no session state to preserve. - final r = proto.FirmwareAwareR24Decoder().decode( - proto.hexToBytes(raw.hex), - ); - if (r == null || r.tsEpoch <= 0) return null; - return Sample( - tsEpoch: r.tsEpoch, - counter: r.counter, - hr: r.hr, - rrIntervalsMs: List.from(r.rrIntervalsMs), - ax: r.accelG.isNotEmpty ? r.accelG[0] : 0, - ay: r.accelG.length > 1 ? r.accelG[1] : 0, - az: r.accelG.length > 2 ? r.accelG[2] : 0, - spo2RedRaw: r.spo2RedRaw, - spo2IrRaw: r.spo2IrRaw, - skinTempRaw: r.skinTempRaw, - ); - } catch (_) { - return null; + final r = proto.FirmwareAwareR24Decoder().decode(bytes); + if (r != null && r.tsEpoch > 0) { + return Sample( + tsEpoch: r.tsEpoch, + counter: r.counter, + hr: r.hr, + rrIntervalsMs: List.from(r.rrIntervalsMs), + ax: r.accelG.isNotEmpty ? r.accelG[0] : 0, + ay: r.accelG.length > 1 ? r.accelG[1] : 0, + az: r.accelG.length > 2 ? r.accelG[2] : 0, + spo2RedRaw: r.spo2RedRaw, + spo2IrRaw: r.spo2IrRaw, + skinTempRaw: r.skinTempRaw, + ); + } + } catch (_) {} + // Gen5 v18 / lenient samples carry HR/RR/gravity but lack gen4 optics — + // `hasDecodedOneHz` stays false, yet they are honest 1 Hz substrate rows. + if (preferred != null && + preferred.tsEpoch > 0 && + !_isGen4R10LiteHistorical(bytes)) { + return preferred; } + return null; } /// THE orphan guard for an INSERT-OR-REPLACE into `decoded_onehz`. diff --git a/test/gen5_decoded_onehz_persistence_test.dart b/test/gen5_decoded_onehz_persistence_test.dart new file mode 100644 index 0000000..e724007 --- /dev/null +++ b/test/gen5_decoded_onehz_persistence_test.dart @@ -0,0 +1,213 @@ +// Gen5 v18 samples must land in `decoded_onehz` via the preferred-sample +// fallback in LocalDb._decodeOneHzSample — they lack gen4 optics so R24 +// decode fails, but they are honest 1 Hz substrate rows. R10-lite hr-only +// records must stay excluded. + +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/ble_engine.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/models.dart'; +import 'package:openstrap_protocol/openstrap_protocol.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +String _bytesToHex(Uint8List bytes) => + bytes.map((b) => b.toRadixString(16).padLeft(2, '0')).join(); + +Uint8List _buildR10LiteInner({required int ts, required int counter, required int hr}) { + final inner = Uint8List(18); + inner[0] = PacketType.historicalData; + inner[1] = Record.r10; + inner.buffer.asByteData().setUint32(3, counter, Endian.little); + inner.buffer.asByteData().setUint32(7, ts, Endian.little); + inner[17] = hr; + return inner; +} + +/// Synthetic gen5 v18 lenient inner: valid unix@7 + HR, gravity fails gate. +Uint8List _buildGen5V18LenientInner({ + required int unix, + required int counter, + required int hr, + List rrMs = const [], +}) { + final inner = Uint8List(112); + inner[0] = PacketType.historicalData; + inner[1] = 18; + inner[2] = 0x80; + inner[3] = counter & 0xff; + inner[4] = (counter >> 8) & 0xff; + inner[5] = (counter >> 16) & 0xff; + inner[6] = (counter >> 24) & 0xff; + inner.buffer.asByteData().setUint32(7, unix, Endian.little); + inner[14] = hr; + inner[15] = rrMs.length.clamp(0, 4); + final view = inner.buffer.asByteData(); + for (var i = 0; i < rrMs.length && i < 4; i++) { + view.setInt16(16 + 2 * i, rrMs[i], Endian.little); + } + view.setFloat32(33, 0.5, Endian.little); + view.setFloat32(37, 0.05, Endian.little); + view.setFloat32(41, 0.05, Endian.little); + view.setFloat32(45, 0.05, Endian.little); + return inner; +} + +void main() { + setUpAll(() async { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + LocalDb.dbName = 'openstrap_gen5_onehz_test.db'; + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + }); + + tearDownAll(() async { + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + }); + + group('gen5 → decoded_onehz persistence', () { + test('gen5 v18-shaped sample persists via preferred fallback (+ RR)', () async { + // Real fixture inner — same bytes as gen5_sample_mapping_test.dart. + final frameHex = + 'aa01740001003fb12f1280733d8401b69f266a66460066025a0265020000000' + '000007b0a8d656463ff0012163cf6a439bf2924fd3ed763fe3e3200aa000000' + '000000000000f7000901f10b0007010c020c000000000000000000000000000' + '00000000000000000000100656f1e1e0000009d61a7c00000003e862817'; + final frame = Uint8List.fromList( + List.generate(frameHex.length ~/ 2, (i) { + return int.parse(frameHex.substring(i * 2, i * 2 + 2), radix: 16); + }), + ); + final parsed = parseFrame(frame, profile: BandProfile.gen5)!; + final inner = parsed.inner; + final sample = sampleFromGen5Historical(parseGen5Historical(inner)); + expect(sample, isNotNull); + + const recTs = 1780916150; + final raw = RawRecord( + counter: sample!.counter, + packetType: PacketType.historicalData, + hex: _bytesToHex(inner), + capturedAt: recTs * 1000, + recTs: recTs, + ); + + await LocalDb.commitSyncBatch([raw], [sample]); + + final db = await LocalDb.instance; + final rows = await db.query( + 'decoded_onehz', + where: 'rec_ts = ?', + whereArgs: [recTs], + ); + expect(rows.length, 1); + expect(rows.first['hr'], 102); + expect(rows.first['counter'], sample.counter); + + final rr = await db.query( + 'decoded_rr', + where: 'counter = ?', + whereArgs: [sample.counter], + ); + expect(rr.length, 2); + expect([for (final r in rr) r['rr_ms']], containsAll([602, 613])); + }); + + test('lenient v18 with null accel still persists (stored as 0)', () async { + const unix = 1785801600; + const counter = 42; + final inner = _buildGen5V18LenientInner(unix: unix, counter: counter, hr: 72); + final sample = sampleFromGen5V18Lenient(inner, unix); + expect(sample, isNotNull); + expect(sample!.ax, isNull); + + final raw = RawRecord( + counter: counter, + packetType: PacketType.historicalData, + hex: _bytesToHex(inner), + capturedAt: unix * 1000, + recTs: unix, + ); + await LocalDb.commitSyncBatch([raw], [sample]); + + final db = await LocalDb.instance; + final rows = await db.query( + 'decoded_onehz', + where: 'rec_ts = ?', + whereArgs: [unix], + ); + expect(rows.length, 1); + expect(rows.first['hr'], 72); + expect(rows.first['ax'], 0); + expect(rows.first['ay'], 0); + expect(rows.first['az'], 0); + }); + + test('R10-lite hr-only + preferred → no decoded_onehz row', () async { + const ts = 1780000100; + const counter = 99; + final inner = _buildR10LiteInner(ts: ts, counter: counter, hr: 65); + final preferred = Sample(tsEpoch: ts, counter: counter, hr: 65); + final raw = RawRecord( + counter: counter, + packetType: PacketType.historicalData, + hex: _bytesToHex(inner), + capturedAt: ts * 1000, + recTs: ts, + ); + + await LocalDb.commitSyncBatch([raw], [preferred]); + + final db = await LocalDb.instance; + final rows = await db.query( + 'decoded_onehz', + where: 'rec_ts = ?', + whereArgs: [ts], + ); + expect(rows, isEmpty); + }); + + test('full gen4 R24 sample still persists', () async { + const ts = 1780000200; + const counter = 5001; + final sample = Sample( + tsEpoch: ts, + counter: counter, + hr: 70, + rrIntervalsMs: [800], + ax: 0.1, + ay: -0.2, + az: 0.95, + spo2RedRaw: 100, + spo2IrRaw: 200, + skinTempRaw: 300, + ); + // Minimal non-R10 historical hex — R24 decode won't match, but preferred + // has full gen4 optics so _decodeOneHzSample returns it immediately. + final raw = RawRecord( + counter: counter, + packetType: PacketType.historicalData, + hex: '2f18' '00' * 20, + capturedAt: ts * 1000, + recTs: ts, + ); + + await LocalDb.commitSyncBatch([raw], [sample]); + + final db = await LocalDb.instance; + final rows = await db.query( + 'decoded_onehz', + where: 'rec_ts = ?', + whereArgs: [ts], + ); + expect(rows.length, 1); + expect(rows.first['hr'], 70); + expect(rows.first['spo2_red_raw'], 100); + }); + }); +} diff --git a/test/gen5_v18_hardware_lenient_test.dart b/test/gen5_v18_hardware_lenient_test.dart index 8959b97..e64cf04 100644 --- a/test/gen5_v18_hardware_lenient_test.dart +++ b/test/gen5_v18_hardware_lenient_test.dart @@ -53,9 +53,10 @@ void main() { }); test('unix@7 garbage → abstain (do not invent offset-6 time)', () { - expect(gen5V18UnixFromInner(inner), isNull); - expect(sampleFromGen5V18Lenient(inner), isNull); - expect(decodeGen5HistoricalSample(inner), isNull); + const wallNow = 1785863370; // export capture era + expect(gen5V18UnixFromInner(inner, wallNow), isNull); + expect(sampleFromGen5V18Lenient(inner, wallNow), isNull); + expect(decodeGen5HistoricalSample(inner, wallNow), isNull); }); }); @@ -81,7 +82,8 @@ void main() { inner.buffer.asByteData().setFloat32(45, 0.05, Endian.little); expect(parseGen5Historical(inner), isNull); - final sample = sampleFromGen5V18Lenient(inner); + const wallNow = unix; // plausible vs the synthetic timestamp + final sample = sampleFromGen5V18Lenient(inner, wallNow); expect(sample, isNotNull); expect(sample!.tsEpoch, unix); expect(sample.hr, 72); From fceffa606d4a6209c8f27ddf2e9308b57ffb7bd6 Mon Sep 17 00:00:00 2001 From: Brackyt <60280126+Brackyt@users.noreply.github.com> Date: Tue, 4 Aug 2026 22:09:54 +0200 Subject: [PATCH 4/7] feat(gen5): v26 PPG-derived HR ingest (algo v52) Wire gen5 v26 PPG bursts into Sample ingest via analytics ACF HR (b3e7b88): adjacent 12-burst buffer, u16@3 counter for v26, always archive v26 hex, never clobber measured v18 at the same rec_ts. --- lib/ble/ble_engine.dart | 187 +++++++++++++++++++++++++---- lib/compute/derivation_engine.dart | 7 +- pubspec.lock | 8 +- pubspec.yaml | 5 +- test/gen5_sample_mapping_test.dart | 121 +++++++++++++++++-- 5 files changed, 288 insertions(+), 40 deletions(-) diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index ac37816..5d0613c 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -40,6 +40,8 @@ import 'dart:convert'; import 'dart:io'; import 'package:flutter/foundation.dart'; import 'package:flutter_blue_plus/flutter_blue_plus.dart'; +import 'package:openstrap_analytics/onehz.dart' + show deriveHrFromGen5PpgWaveform, kGen5PpgHrMinSamples; import 'package:openstrap_protocol/openstrap_protocol.dart'; import '../data/db.dart'; @@ -53,6 +55,9 @@ import 'ble_state.dart'; int u32(Uint8List b, int o) => b.buffer.asByteData(b.offsetInBytes, b.length).getUint32(o, Endian.little); +int u16(Uint8List b, int o) => + b.buffer.asByteData(b.offsetInBytes, b.length).getUint16(o, Endian.little); + typedef SampleSink = Future Function(Sample? sample, RawRecord raw); typedef StateSink = void Function(DeviceState state); typedef LogSink = void Function(String line); @@ -179,17 +184,68 @@ List gen5SetClockPayload({required int sec, required int subsec}) => [ @visibleForTesting List gen5GetClockPayload() => const [revision1]; +/// Rolling buffer of recent gen5 v26 PPG bursts (24 samples @ 24 Hz each). +/// Bursts must be adjacent in unix (same second or +1 s) with monotonic +/// [burstIndex] within a second; a gap clears the window so resting-HR ACF +/// does not stitch unrelated captures. +@visibleForTesting +class Gen5PpgBurstBuffer { + Gen5PpgBurstBuffer({this.capacity = 12}); + + final int capacity; + final List<_Gen5PpgBurst> _bursts = <_Gen5PpgBurst>[]; + + int get length => _bursts.length; + + void add({ + required int unix, + required int burstIndex, + required List wave, + }) { + if (_bursts.isNotEmpty) { + final last = _bursts.last; + final sameSecond = unix == last.unix; + final nextSecond = unix == last.unix + 1; + final indexOk = sameSecond + ? burstIndex > last.burstIndex + : nextSecond; + if (!(sameSecond || nextSecond) || !indexOk) { + clear(); + } + } + _bursts.add(_Gen5PpgBurst(unix, burstIndex, List.from(wave))); + while (_bursts.length > capacity) { + _bursts.removeAt(0); + } + } + + List concatenated() { + final out = []; + for (final b in _bursts) { + out.addAll(b.wave); + } + return out; + } + + void clear() => _bursts.clear(); +} + +class _Gen5PpgBurst { + _Gen5PpgBurst(this.unix, this.burstIndex, this.wave); + final int unix; + final int burstIndex; + final List wave; +} + /// Map a decoded gen5 historical record onto the band-agnostic `Sample` type, /// or null when this record kind has no `Sample` equivalent (yet). /// -/// Only `Gen5HistorySample` (v18, the per-second stream) maps today — the -/// deep buffers (`Gen5OpticalBuffer`/`Gen5ImuBuffer`/`Gen5PpgWaveform`, R22 -/// opt-in only) need their own raw-buffer storage, not a 1Hz `Sample`, so -/// they (and a null [g], e.g. an unrecognised version) correctly return null -/// here — the caller archives those, exactly like an undecodable gen4 -/// record. Extracted as a top-level pure function (rather than inlined in -/// `_ingestHistoricalFrame`) so the mapping is unit-testable without a live -/// BLE session — see `gen5_sample_mapping_test.dart`. +/// `Gen5HistorySample` (v18) maps measured HR/RR/gravity. `Gen5PpgWaveform` +/// (v26) can map a *derived* HR via [sampleFromGen5PpgWaveform] when the +/// caller supplies enough concatenated bursts — never RR/HRV. Optical/IMU +/// deep buffers still return null and are archived. Extracted as a top-level +/// pure function so the mapping is unit-testable without a live BLE session — +/// see `gen5_sample_mapping_test.dart`. @visibleForTesting Sample? sampleFromGen5Historical(Gen5HistoricalRecord? g) { if (g is! Gen5HistorySample) return null; @@ -208,13 +264,57 @@ Sample? sampleFromGen5Historical(Gen5HistoricalRecord? g) { ); } +/// Map a gen5 v26 PPG burst to a HR-only [Sample], or null when derivation +/// abstains. [concatenatedSamples] should include this burst plus recent +/// neighbours (see [Gen5PpgBurstBuffer]) — empty RR by design (no HRV claim). +@visibleForTesting +Sample? sampleFromGen5PpgWaveform( + Gen5PpgWaveform g, + List concatenatedSamples, +) { + final hr = deriveHrFromGen5PpgWaveform(concatenatedSamples); + if (hr == null) return null; + return Sample( + tsEpoch: g.unix, + counter: g.recordIndex, + hr: hr, + rrIntervalsMs: const [], + ); +} + /// Decode a gen5 historical inner frame to a band-agnostic [Sample], or null. +/// Pass [ppgBuf] so consecutive v26 bursts can be concatenated for resting HR. +/// When [measuredRecTs] already contains a second, PPG-derived samples abstain +/// so measured v18 rows are never clobbered in `decoded_onehz`. @visibleForTesting -Sample? decodeGen5HistoricalSample(Uint8List inner, int wallNow) { - final strict = sampleFromGen5Historical(parseGen5Historical(inner)); - if (strict != null) return strict; +Sample? decodeGen5HistoricalSample( + Uint8List inner, + int wallNow, { + Gen5PpgBurstBuffer? ppgBuf, + Set? measuredRecTs, +}) { + final parsed = parseGen5Historical(inner); + final strict = sampleFromGen5Historical(parsed); + if (strict != null) { + measuredRecTs?.add(strict.tsEpoch); + return strict; + } if (inner.length > 1 && inner[1] == 18) { - return sampleFromGen5V18Lenient(inner, wallNow); + final lenient = sampleFromGen5V18Lenient(inner, wallNow); + if (lenient != null) { + measuredRecTs?.add(lenient.tsEpoch); + return lenient; + } + } + if (parsed is Gen5PpgWaveform) { + ppgBuf?.add( + unix: parsed.unix, + burstIndex: parsed.burstIndex, + wave: parsed.ppgWaveform, + ); + if (measuredRecTs?.contains(parsed.unix) ?? false) return null; + final samples = ppgBuf?.concatenated() ?? parsed.ppgWaveform; + return sampleFromGen5PpgWaveform(parsed, samples); } return null; } @@ -844,6 +944,13 @@ class BleEngine { // doc). Re-seeded from the durable counter_hw cursor on each connect, same // pattern as _recordGate's frontierTs seed below. CounterRegressionDetector _counterRegression = CounterRegressionDetector(); + + /// Recent gen5 v26 PPG bursts for resting-HR ACF (cleared on teardown). + final Gen5PpgBurstBuffer _gen5PpgBuf = Gen5PpgBurstBuffer(); + + /// Seconds that already have a measured v18 sample this connection — PPG + /// derivation must not REPLACE those rows in `decoded_onehz`. + final Set _gen5MeasuredRecTs = {}; // Firmware-aware R24 decoder (see openstrap_protocol's // FirmwareAwareR24Decoder doc): tries the original hardware-validated // decoder first, falls back to newer-firmware layouts only if that fails, @@ -2058,15 +2165,33 @@ class BleEngine { Sample? sample; final wallNow = DateTime.now().millisecondsSinceEpoch ~/ 1000; final isGen5 = _session?.band.isGen5 ?? false; + final isGen5V26 = isGen5 && recType == 26; if (isGen5) { - // gen5 (WHOOP 5): `parseGen5Historical` dispatches across all four real - // gen5 historical-record kinds (v18 per-second summary, v20 optical/ - // v21 IMU/v26 PPG deep buffers — R22 opt-in only). Only v18 maps onto - // the band-agnostic `Sample` type today; the deep buffers need their - // own raw-buffer storage (a future db table), not a 1Hz Sample, so they - // fall through to the undecodable archive below — that is honest - // (correctly-identified-but-not-yet-stored), not a decode failure. - sample = decodeGen5HistoricalSample(frame.inner, wallNow); + // gen5 (WHOOP 5): v18 maps measured HR/RR; v26 can map a *derived* HR via + // concatenated recent PPG bursts (empty RR — no HRV claim). Optical/IMU + // buffers still archive. WHOOP 4 path below is unchanged. + sample = decodeGen5HistoricalSample( + frame.inner, + wallNow, + ppgBuf: _gen5PpgBuf, + measuredRecTs: _gen5MeasuredRecTs, + ); + if (isGen5V26) { + final archive = ArchiveRecord( + counter: counter, + hex: _innerHex(frame.inner), + packetType: frame.inner.isNotEmpty ? frame.inner[0] : 0, + capturedAt: DateTime.now().millisecondsSinceEpoch, + reason: 'gen5_v26_ppg', + ); + final d = _drain; + if (d != null) { + d.onHistoricalArchive(archive); + } else { + unawaited(onArchiveRecord?.call(archive) ?? Future.value()); + } + if (sample == null) return; + } } else if (recType == Record.r24 || recType == Record.r12) { // Legacy decoder first, firmware-fallback chain second, undecodable // archive last — see FirmwareAwareR24Decoder. @@ -2843,8 +2968,13 @@ class BleEngine { } } - int _counterFromInner(Uint8List inner) => - inner.length >= 7 ? u32(inner, 3) : 0; + int _counterFromInner(Uint8List inner) { + if (inner.length < 5) return 0; + // v26 record_index is u16@3 (protocol Gen5V26Decoder) — do not u32-inflate + // into the same key space as v18 counters. + if (inner.length > 1 && inner[1] == 26) return u16(inner, 3); + return inner.length >= 7 ? u32(inner, 3) : 0; + } String _innerHex(Uint8List inner) => inner.map((b) => b.toRadixString(16).padLeft(2, '0')).join(); @@ -3318,6 +3448,8 @@ class BleEngine { final device = session.device; await session.teardown(); _session = null; + _gen5PpgBuf.clear(); + _gen5MeasuredRecTs.clear(); // The strap-RTC↔wall correlation belongs to the session that measured it — // drop it so it can't leak into the next connection's alarm arming before a // fresh GET_CLOCK. (Connection setup also re-nulls it; this covers the gap @@ -3591,6 +3723,17 @@ class DrainController { } } + /// Companion archive for a record that IS also stored (e.g. gen5 v26 PPG hex + /// alongside a derived Sample). Does not bump record counters — the paired + /// [onHistoricalRecord] already did. + void onHistoricalArchive(ArchiveRecord a) { + if (_buffering) { + _archives.add(a); + } else { + unawaited(onArchive?.call(a) ?? Future.value()); + } + } + void noteBatchAcked() => batches++; void onBurstEvent() => burstStats.onEvent(); diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 61b0d0b..4f1cb31 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -362,7 +362,12 @@ import 'substrate.dart'; // (fever/heat/anxiety) that have no discernible onset — so this changes which // suggestions autoDetectWorkouts emits without loosening the false-positive // gate it exists to protect. -const int kAlgoVersion = 51; +// v52: WHOOP 5 — gen5 v26 PPG bursts can contribute a derived per-second HR +// (analytics `deriveHrFromGen5PpgWaveform` @ b3e7b88624e4cbb6a0ab2dee6715446f19feb775, +// samples) when measured v18 is absent. Empty RR by design (no HRV claim). +// Abstains on thin/noisy windows. Bump so days re-derive once v26-backed onehz +// rows land. +const int kAlgoVersion = 52; /// Raw is kept this many days past derivation, then pruned (derived stays). const int rawRetentionDays = 3; diff --git a/pubspec.lock b/pubspec.lock index 956060c..2afa67d 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -963,11 +963,9 @@ packages: openstrap_analytics: dependency: "direct main" description: - path: "." - ref: cbbe06addec1cb78b4ea2c75f64e8a281ac09294 - resolved-ref: cbbe06addec1cb78b4ea2c75f64e8a281ac09294 - url: "https://github.com/OpenStrap/analytics.git" - source: git + path: "../analytics" + relative: true + source: path version: "1.0.0" openstrap_protocol: dependency: "direct main" diff --git a/pubspec.yaml b/pubspec.yaml index 02b52ad..c7f88e3 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -70,7 +70,10 @@ dependencies: # Moved to the branch's new head (cbbe06a) after CodeRabbit caught a real # off-by-one in the onset window on analytics#32 (earlyMean was 181s vs # preMean's 180s) — fixed there, not worth a separate edge changelog line. - ref: cbbe06addec1cb78b4ea2c75f64e8a281ac09294 + # b3e7b88: gen5 v26 PPG→HR (`deriveHrFromGen5PpgWaveform`, OpenStrap/ + # analytics#37). Requires ≥240 samples (~10 s @ 24 Hz) for resting BPM. + # Repin to the merge commit on main once #37 lands. + ref: b3e7b88624e4cbb6a0ab2dee6715446f19feb775 # BLE — flutter_blue_plus is the maintained cross-platform GATT client. flutter_blue_plus: ^1.36.8 diff --git a/test/gen5_sample_mapping_test.dart b/test/gen5_sample_mapping_test.dart index 37c611a..ae95bc3 100644 --- a/test/gen5_sample_mapping_test.dart +++ b/test/gen5_sample_mapping_test.dart @@ -9,9 +9,11 @@ // reused here rather than re-typed, so a transcription slip can't silently // diverge the two test suites' expectations. +import 'dart:math' as math; import 'dart:typed_data'; import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' show kGen5PpgHrMinSamples; import 'package:openstrap_edge/ble/ble_engine.dart'; import 'package:openstrap_edge/data/models.dart'; import 'package:openstrap_protocol/openstrap_protocol.dart'; @@ -25,6 +27,14 @@ Uint8List hex(String s) { return out; } +List sinePpg({required double bpm, required int n}) { + final f = bpm / 60.0; + return List.generate(n, (i) { + final t = i / 24.0; + return (500 + 1000 * math.sin(2 * math.pi * f * t)).round(); + }); +} + void main() { group('sampleFromGen5Historical — v18 (real fixture)', () { // "worn" capture, unix=1780916150 — CRC16+CRC32 both verified. Same @@ -64,12 +74,6 @@ void main() { test( 'does NOT populate skinTempRaw/spo2 — gen5-specific scale/absence', () { - // See gen5_v18_decode's (now removed, folded into protocol) original - // caution and Gen5HistorySample's field docs: gen5's skin_temp is - // already °C-scaled (raw/100), a DIFFERENT transfer function from - // gen4's per-device affine ADC calibration that `skinTempRaw` feeds — - // reusing that field here would silently corrupt the skin-temp-z - // metric. gen5 v18 has no real dual-wavelength SpO2 at all. expect(sample!.skinTempRaw, isNull); expect(sample!.spo2RedRaw, isNull); expect(sample!.spo2IrRaw, isNull); @@ -82,19 +86,114 @@ void main() { expect(sampleFromGen5Historical(null), isNull); }); + test('a v20 optical deep buffer (no Sample equivalent) maps to null', () { + final inner = Uint8List(kGen5V20InnerLen); + inner[0] = 0x2F; + inner[1] = 20; + inner[2] = 0x81; + final view = inner.buffer.asByteData(); + view.setUint32(3, 1, Endian.little); + view.setUint32(7, 1780000000, Endian.little); + inner[18] = 25; // block 0 active count + final decoded = parseGen5Historical(inner); + expect(decoded, isA()); + expect(sampleFromGen5Historical(decoded), isNull); + }); + test('a v21 IMU deep buffer (no Sample equivalent) maps to null', () { - // Synthetic-but-shape-correct v21 buffer: countA/countB both 100 (the - // buffer's actual identity gate, per Gen5V21Decoder — hist_version is - // not trusted for this kind at all). final inner = Uint8List(kGen5V21InnerLen); inner[0] = 0x2F; inner[1] = 21; final view = inner.buffer.asByteData(); - view.setUint16(16, 100, Endian.little); // countA offset - view.setUint16(622, 100, Endian.little); // countB offset + view.setUint16(16, 100, Endian.little); + view.setUint16(622, 100, Endian.little); final decoded = parseGen5Historical(inner); expect(decoded, isA()); expect(sampleFromGen5Historical(decoded), isNull); }); }); + + group('sampleFromGen5PpgWaveform — derived HR only', () { + test('maps derived HR with empty RR when ACF recovers (10 s window)', () { + final wave = sinePpg(bpm: 120, n: kGen5PpgHrMinSamples); + final g = Gen5PpgWaveform( + histVersion: 26, + recordIndex: 42, + unix: 1785801600, + layoutMarker: 0, + rawByte19: 0, + burstIndex: 0, + ppgWaveform: wave.sublist(0, 24), + ); + final sample = sampleFromGen5PpgWaveform(g, wave); + expect(sample, isNotNull); + expect(sample!.hr, closeTo(120, 5)); + expect(sample.tsEpoch, 1785801600); + expect(sample.counter, 42); + expect(sample.rrIntervalsMs, isEmpty); + }); + + test('flatline PPG abstains (null Sample)', () { + final g = Gen5PpgWaveform( + histVersion: 26, + recordIndex: 1, + unix: 1785801600, + layoutMarker: 0, + rawByte19: 0, + burstIndex: 0, + ppgWaveform: List.filled(24, 100), + ); + expect( + sampleFromGen5PpgWaveform( + g, + List.filled(kGen5PpgHrMinSamples, 100), + ), + isNull, + ); + }); + }); + + group('Gen5PpgBurstBuffer', () { + test('keeps only the last N bursts concatenated', () { + final buf = Gen5PpgBurstBuffer(capacity: 2); + buf.add(unix: 100, burstIndex: 0, wave: [1, 2]); + buf.add(unix: 100, burstIndex: 1, wave: [3, 4]); + buf.add(unix: 101, burstIndex: 0, wave: [5, 6]); + expect(buf.concatenated(), [3, 4, 5, 6]); + }); + + test('clears on non-adjacent unix gap', () { + final buf = Gen5PpgBurstBuffer(); + buf.add(unix: 100, burstIndex: 0, wave: [1, 2]); + buf.add(unix: 102, burstIndex: 0, wave: [9, 9]); + expect(buf.concatenated(), [9, 9]); + }); + + test('clears on non-monotonic burstIndex within same second', () { + final buf = Gen5PpgBurstBuffer(); + buf.add(unix: 100, burstIndex: 1, wave: [1, 2]); + buf.add(unix: 100, burstIndex: 0, wave: [9, 9]); + expect(buf.concatenated(), [9, 9]); + }); + }); + + group('decodeGen5HistoricalSample — measured v18 clobber guard', () { + test('PPG abstains when measured v18 already claimed that second', () { + final frame = hex( + 'aa015000010035412f1a80ad418401f0a3266aae470100c3c5050068faccfa8dfb46f' + 'c8bfd4cfebafedafe6dff56ffd5fffbff37ff6afce5f9d7f8dffa5efc98fddbfe5afe8' + '4fe15ff5cff405fb33c50080101006cb67c17', + ); + final inner = parseFrame(frame, profile: BandProfile.gen5)!.inner; + final measured = {1780917232}; + expect( + decodeGen5HistoricalSample( + inner, + 1780917232, + measuredRecTs: measured, + ), + isNull, + ); + }); + }); } From 889dc1899ee87cfc028ceea74d5fa6bafbf29e3d Mon Sep 17 00:00:00 2001 From: Brackyt <60280126+Brackyt@users.noreply.github.com> Date: Tue, 4 Aug 2026 22:10:30 +0200 Subject: [PATCH 5/7] fix: git-sourced analytics lockfile for v26 PPG PR Regenerate pubspec.lock without pubspec_overrides so CI resolves openstrap_analytics from git (b3e7b88), not path. --- lib/compute/derivation_engine.dart | 2 +- pubspec.lock | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 4f1cb31..40a4cef 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -364,7 +364,7 @@ import 'substrate.dart'; // gate it exists to protect. // v52: WHOOP 5 — gen5 v26 PPG bursts can contribute a derived per-second HR // (analytics `deriveHrFromGen5PpgWaveform` @ b3e7b88624e4cbb6a0ab2dee6715446f19feb775, -// samples) when measured v18 is absent. Empty RR by design (no HRV claim). +// ACF on ≥10 s of 24 Hz samples) when measured v18 is absent. Empty RR by design (no HRV claim). // Abstains on thin/noisy windows. Bump so days re-derive once v26-backed onehz // rows land. const int kAlgoVersion = 52; diff --git a/pubspec.lock b/pubspec.lock index 2afa67d..0649a61 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -963,9 +963,11 @@ packages: openstrap_analytics: dependency: "direct main" description: - path: "../analytics" - relative: true - source: path + path: "." + ref: b3e7b88624e4cbb6a0ab2dee6715446f19feb775 + resolved-ref: b3e7b88624e4cbb6a0ab2dee6715446f19feb775 + url: "https://github.com/OpenStrap/analytics.git" + source: git version: "1.0.0" openstrap_protocol: dependency: "direct main" From 63c40f0e02085c7bfd254db8440d2663d2afb488 Mon Sep 17 00:00:00 2001 From: Brackyt <60280126+Brackyt@users.noreply.github.com> Date: Tue, 4 Aug 2026 22:16:34 +0200 Subject: [PATCH 6/7] fix(gen5): gate PPG Sample on kGen5PpgHrMinSamples Use the analytics min-window constant so the import is load-bearing and thin burst concatenations abstain before ACF. --- lib/ble/ble_engine.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index 5d0613c..bb96d6b 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -272,6 +272,8 @@ Sample? sampleFromGen5PpgWaveform( Gen5PpgWaveform g, List concatenatedSamples, ) { + // Match analytics' honest min window (~10 s @ 24 Hz) before ACF. + if (concatenatedSamples.length < kGen5PpgHrMinSamples) return null; final hr = deriveHrFromGen5PpgWaveform(concatenatedSamples); if (hr == null) return null; return Sample( From 6faa944edede278e395da95249315094bde99962 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 7 Aug 2026 00:33:08 +0530 Subject: [PATCH 7/7] gen5: a derived PPG bpm must never evict a measured row (or its RR beats) P0. The measured-vs-derived guard was per-CONNECTION and evaporated at exactly the moment it was needed. `_gen5MeasuredRecTs` is an in-memory Set, cleared in `_teardownSession` and never seeded from `decoded_onehz`. After a reconnect it is empty while the rows are still on disk, so a re-delivered v26 burst derives an HR for a second that already has a measured v18 row. `decoded_onehz` is INSERT-OR-REPLACE on UNIQUE(rec_ts), so the derived row WINS -- and `_queueOrphanGuard` then deletes the evicted counter's `decoded_rr` beats. A derived sample carries no beats of its own, so the net trade is a measured HR plus a whole second of beat-to-beat intervals for an inferred bpm. `decoded_rr` is the durable RR store; that is irrecoverable. Seeding the set from the DB on connect would not really fix it -- the set would have to hold every second ever measured. Provenance belongs on the datum, so: * `Sample.derived` marks an INFERRED HR (today only the v26 PPG ACF path). * `_queueDecodedOneHz` honours it: derived rows use INSERT OR IGNORE and SKIP the orphan guard, so an existing row for that second simply stands. Nothing is evicted, so nothing is stranded. Measured-vs-measured is deliberately untouched -- "newest wins" is still right there (the strap counter resets on reboot), and a test pins that. This also de-fangs the u16-index-as-global-PK concern: a v26 burst index that collides with a real counter now loses the insert instead of overwriting a measured row. The vacuous test, confirmed and replaced. `decodeGen5HistoricalSample -- measured v18 clobber guard` passed with the guard line DELETED, because no `ppgBuf` was supplied so the derived path abstained for want of samples no matter what the guard did. It now primes a buffer with an ACF-resolvable waveform and asserts BOTH directions: the derived path is genuinely reachable for that fixture when the second is unclaimed, and abstains when it is claimed. Re-ran the mutation afterwards -- it now fails, as it should. Not addressed here, deliberately: absent gravity persisted as `?? 0`. That is the same seam as #188 and is fixed there (`Substrate.accelPresentAt`); doing it again here would just conflict. 5 tests added/reworked, each mutation-verified. Suite 1076 passing; the 6 failures in notification_dedupe_test are pre-existing and reproduce on origin/main unmodified. --- lib/ble/ble_engine.dart | 4 + lib/data/db.dart | 30 +++ lib/data/models.dart | 17 ++ ...derived_hr_never_evicts_measured_test.dart | 219 ++++++++++++++++++ test/gen5_sample_mapping_test.dart | 63 ++++- 5 files changed, 327 insertions(+), 6 deletions(-) create mode 100644 test/gen5_derived_hr_never_evicts_measured_test.dart diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index bb96d6b..f848ab8 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -281,6 +281,10 @@ Sample? sampleFromGen5PpgWaveform( counter: g.recordIndex, hr: hr, rrIntervalsMs: const [], + // INFERRED from the PPG waveform, not reported by the strap. Carried into + // persistence so it can never evict a measured row — and that row's RR + // beats — for the same second. See Sample.derived. + derived: true, ); } diff --git a/lib/data/db.dart b/lib/data/db.dart index efa76ca..5d207b9 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -1935,6 +1935,36 @@ class LocalDb { final decoded = _decodeOneHzSample(raw, preferred: sample); if (decoded == null) return 0; final recTs = raw.recTs ?? decoded.tsEpoch; + // PROVENANCE BEATS RECENCY — for DERIVED HR only. + // + // "Newest wins" below is right for two MEASURED records of the same second + // (the counter resets after a reboot), but it is wrong when the newcomer's + // HR was INFERRED from the PPG waveform. That REPLACE evicts the measured + // row, and `_queueOrphanGuard` then deletes the evicted counter's + // `decoded_rr` beats — the durable RR store — while a derived sample brings + // no beats of its own. A measured HR plus a whole second of beat-to-beat + // intervals, traded for an inferred bpm, unrecoverably. + // + // The engine's in-memory `measuredRecTs` set already prevents this WITHIN a + // connection, but it is cleared on teardown and never seeded from + // `decoded_onehz`, so after a reconnect a re-delivered v26 burst walks + // straight past it. Enforcing it here makes the guarantee durable instead + // of session-scoped: INSERT OR IGNORE and NO orphan guard, so any existing + // row for this second simply stands. + if (decoded.derived) { + batch.insert('decoded_onehz', { + 'counter': raw.counter, + 'rec_ts': recTs, + 'hr': decoded.hr, + 'ax': decoded.ax ?? 0, + 'ay': decoded.ay ?? 0, + 'az': decoded.az ?? 0, + 'spo2_red_raw': decoded.spo2RedRaw ?? 0, + 'spo2_ir_raw': decoded.spo2IrRaw ?? 0, + 'skin_temp_raw': decoded.skinTempRaw ?? 0, + }, conflictAlgorithm: ConflictAlgorithm.ignore); + return 1; + } // TIME-KEYED, NEWEST-WINS (noop/WHOOP-4 model: dedupe records by their // embedded timestamp, not by a counter). decoded_onehz has a UNIQUE(rec_ts) // index and decoded_rr a UNIQUE(rr_ts_ms, beat_index). We use REPLACE, not diff --git a/lib/data/models.dart b/lib/data/models.dart index 5f95c50..36adf90 100644 --- a/lib/data/models.dart +++ b/lib/data/models.dart @@ -16,6 +16,21 @@ class Sample { final int? spo2IrRaw; final int? skinTempRaw; + /// True when [hr] was INFERRED rather than reported by the strap — today only + /// the gen5 v26 PPG-waveform path (autocorrelation over concatenated bursts). + /// + /// Provenance, not quality: a derived sample must never EVICT a measured row + /// for the same second. `decoded_onehz` is INSERT-OR-REPLACE keyed on + /// UNIQUE(rec_ts), and that eviction also deletes the losing counter's + /// `decoded_rr` beats — the durable RR store, unrecoverable once gone. A + /// derived sample carries no RR of its own, so letting it win trades a + /// measured HR *and* its whole beat series for an inferred bpm. + /// + /// The in-memory `measuredRecTs` set is the fast path for this within one + /// connection; this flag is what makes the guarantee survive a RECONNECT, + /// where that set has been cleared but the rows are still on disk. + final bool derived; + Sample({ required this.tsEpoch, required this.counter, @@ -27,6 +42,7 @@ class Sample { this.spo2RedRaw, this.spo2IrRaw, this.skinTempRaw, + this.derived = false, }); /// Copy with an overridden [tsEpoch] — used by the clock-offset salvage path @@ -44,6 +60,7 @@ class Sample { spo2RedRaw: spo2RedRaw, spo2IrRaw: spo2IrRaw, skinTempRaw: skinTempRaw, + derived: derived, ); bool get wristOn => hr > 0; diff --git a/test/gen5_derived_hr_never_evicts_measured_test.dart b/test/gen5_derived_hr_never_evicts_measured_test.dart new file mode 100644 index 0000000..23a5417 --- /dev/null +++ b/test/gen5_derived_hr_never_evicts_measured_test.dart @@ -0,0 +1,219 @@ +// P0 REGRESSION — a DERIVED (PPG) HR must never evict a MEASURED row, and +// must never take its RR beats down with it. +// +// `decoded_onehz` is INSERT-OR-REPLACE keyed on UNIQUE(rec_ts), and +// `_queueOrphanGuard` additionally DELETEs the evicted counter's `decoded_rr` +// beats. That "newest wins" rule is correct for two MEASURED records of the +// same second (the strap counter resets after a reboot), but a v26 PPG sample +// carries an INFERRED bpm and no beats at all — so letting it win trades a +// measured HR *and* a full second of beat-to-beat intervals for a guess. +// `decoded_rr` is the durable RR store; those beats are not recoverable. +// +// The engine guards this with an in-memory `_gen5MeasuredRecTs` set, but that +// set is per-connection: `_teardownSession` clears it and nothing ever seeds it +// from `decoded_onehz`. So the guard evaporates at exactly the moment it is +// needed — a reconnect, where the band re-delivers a burst for a second whose +// measured row is already on disk. These tests drive persistence directly with +// an EMPTY measured set, which is precisely the post-reconnect state. + +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/models.dart'; +import 'package:openstrap_protocol/openstrap_protocol.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +String _hex(Uint8List b) => + b.map((x) => x.toRadixString(16).padLeft(2, '0')).join(); + +/// Minimal gen5 v18 inner carrying a plausible unix + HR (gravity absent). +Uint8List _v18Inner({required int unix, required int counter}) { + final inner = Uint8List(112); + inner[0] = PacketType.historicalData; + inner[1] = 18; + inner.buffer.asByteData().setUint32(3, counter, Endian.little); + inner.buffer.asByteData().setUint32(7, unix, Endian.little); + return inner; +} + +void main() { + const recTs = 1785801600; + const measuredCounter = 1000; + const derivedCounter = 7; // a v26 burst index — small, and NOT the same row + + setUpAll(() async { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + LocalDb.dbName = 'openstrap_derived_evict_test.db'; + await databaseFactory.deleteDatabase( + p.join(await databaseFactory.getDatabasesPath(), LocalDb.dbName), + ); + }); + + tearDownAll(() async { + await LocalDb.close(); + await databaseFactory.deleteDatabase( + p.join(await databaseFactory.getDatabasesPath(), LocalDb.dbName), + ); + }); + + setUp(() async { + final db = await LocalDb.instance; + await db.delete('decoded_onehz'); + await db.delete('decoded_rr'); + }); + + /// Commit a measured v18 sample carrying RR beats for [recTs]. + Future commitMeasured() async { + final inner = _v18Inner(unix: recTs, counter: measuredCounter); + await LocalDb.commitSyncBatch( + [ + RawRecord( + counter: measuredCounter, + packetType: PacketType.historicalData, + hex: _hex(inner), + capturedAt: recTs * 1000, + recTs: recTs, + ) + ], + [ + Sample( + tsEpoch: recTs, + counter: measuredCounter, + hr: 61, + rrIntervalsMs: const [980, 1005, 991], + ) + ], + ); + } + + /// Commit a PPG-DERIVED sample for the SAME second, as it arrives after a + /// reconnect: no RR of its own, `derived: true`. + Future commitDerivedSameSecond() async { + final inner = _v18Inner(unix: recTs, counter: derivedCounter); + await LocalDb.commitSyncBatch( + [ + RawRecord( + counter: derivedCounter, + packetType: PacketType.historicalData, + hex: _hex(inner), + capturedAt: recTs * 1000, + recTs: recTs, + ) + ], + [ + Sample( + tsEpoch: recTs, + counter: derivedCounter, + hr: 74, // a different, inferred value + rrIntervalsMs: const [], + derived: true, + ) + ], + ); + } + + test( + 'a derived PPG sample does NOT replace the measured row for that second', + () async { + await commitMeasured(); + await commitDerivedSameSecond(); + + final db = await LocalDb.instance; + final rows = await db.query('decoded_onehz', + where: 'rec_ts = ?', whereArgs: [recTs]); + + expect(rows, hasLength(1)); + expect( + rows.first['hr'], + 61, + reason: 'the measured HR must stand, not the inferred 74', + ); + expect(rows.first['counter'], measuredCounter); + }, + ); + + test( + 'and it does NOT take the measured row\'s decoded_rr beats with it', + () async { + await commitMeasured(); + final db = await LocalDb.instance; + final before = await db.query('decoded_rr', + where: 'counter = ?', whereArgs: [measuredCounter]); + expect(before, hasLength(3), reason: 'sanity: beats were persisted'); + + await commitDerivedSameSecond(); + + final after = await db.query('decoded_rr', + where: 'counter = ?', whereArgs: [measuredCounter]); + expect( + after, + hasLength(3), + reason: 'decoded_rr is the durable RR store — this loss is permanent', + ); + expect( + [for (final r in after) r['rr_ms']], + containsAll([980, 1005, 991]), + ); + }, + ); + + test( + 'a derived sample still lands when the second is genuinely unclaimed', + () async { + await commitDerivedSameSecond(); + + final db = await LocalDb.instance; + final rows = await db.query('decoded_onehz', + where: 'rec_ts = ?', whereArgs: [recTs]); + expect( + rows, + hasLength(1), + reason: 'the guard is about precedence, not about dropping PPG HR', + ); + expect(rows.first['hr'], 74); + }, + ); + + test( + 'two MEASURED records for one second keep newest-wins (counter reset ' + 'after a band reboot must still be recoverable)', + () async { + await commitMeasured(); + + const rebootCounter = 3; + final inner = _v18Inner(unix: recTs, counter: rebootCounter); + await LocalDb.commitSyncBatch( + [ + RawRecord( + counter: rebootCounter, + packetType: PacketType.historicalData, + hex: _hex(inner), + capturedAt: recTs * 1000, + recTs: recTs, + ) + ], + [ + Sample( + tsEpoch: recTs, + counter: rebootCounter, + hr: 88, + rrIntervalsMs: const [700], + ) + ], + ); + + final db = await LocalDb.instance; + final rows = await db.query('decoded_onehz', + where: 'rec_ts = ?', whereArgs: [recTs]); + expect(rows, hasLength(1)); + expect( + rows.first['hr'], + 88, + reason: 'measured-vs-measured is unchanged by the derived guard', + ); + }, + ); +} diff --git a/test/gen5_sample_mapping_test.dart b/test/gen5_sample_mapping_test.dart index ae95bc3..bc66d51 100644 --- a/test/gen5_sample_mapping_test.dart +++ b/test/gen5_sample_mapping_test.dart @@ -178,21 +178,72 @@ void main() { }); group('decodeGen5HistoricalSample — measured v18 clobber guard', () { - test('PPG abstains when measured v18 already claimed that second', () { + const ppgUnix = 1780917232; + + Uint8List ppgInner() { final frame = hex( 'aa015000010035412f1a80ad418401f0a3266aae470100c3c5050068faccfa8dfb46f' 'c8bfd4cfebafedafe6dff56ffd5fffbff37ff6afce5f9d7f8dffa5efc98fddbfe5afe8' '4fe15ff5cff405fb33c50080101006cb67c17', ); - final inner = parseFrame(frame, profile: BandProfile.gen5)!.inner; - final measured = {1780917232}; + return parseFrame(frame, profile: BandProfile.gen5)!.inner; + } + + /// A buffer already holding a clean, ACF-resolvable waveform for the second + /// BEFORE the fixture, so the concatenated series clears + /// `kGen5PpgHrMinSamples` and the derived path is genuinely reachable. + /// + /// Without this the derived path abstains for want of samples no matter + /// what the guard does — which is what made the original version of this + /// test VACUOUS. Verified: it passed with the guard line deleted. + Gen5PpgBurstBuffer primedBuf() { + final buf = Gen5PpgBurstBuffer(); + // 24 Hz with a 24-sample period => 60 bpm, mid-range of the 25–230 search. + const perBurst = 24; + final bursts = (kGen5PpgHrMinSamples / perBurst).ceil() + 2; + for (var b = 0; b < bursts; b++) { + buf.add( + unix: ppgUnix - 1, + burstIndex: b, + wave: [ + for (var i = 0; i < perBurst; i++) + (2000 * math.sin(2 * math.pi * (b * perBurst + i) / 24)).round(), + ], + ); + } + return buf; + } + + test( + 'the derived PPG path is LIVE for this fixture when the second is ' + 'unclaimed — otherwise the guard assertion below proves nothing', + () { + final sample = decodeGen5HistoricalSample( + ppgInner(), + ppgUnix, + ppgBuf: primedBuf(), + measuredRecTs: {}, + ); + expect( + sample, + isNotNull, + reason: 'fixture must actually reach sampleFromGen5PpgWaveform', + ); + expect(sample!.derived, isTrue, reason: 'inferred HR, not measured'); + expect(sample.rrIntervalsMs, isEmpty, reason: 'no HRV claim from PPG'); + }, + ); + + test('PPG abstains when measured v18 already claimed that second', () { expect( decodeGen5HistoricalSample( - inner, - 1780917232, - measuredRecTs: measured, + ppgInner(), + ppgUnix, + ppgBuf: primedBuf(), + measuredRecTs: {ppgUnix}, ), isNull, + reason: 'a derived bpm must never displace a measured record', ); }); });