diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml
index c0dd590c..22b93dbd 100644
--- a/android/app/src/main/AndroidManifest.xml
+++ b/android/app/src/main/AndroidManifest.xml
@@ -74,6 +74,37 @@
android:label="Edge"
android:name=".EdgeApplication"
android:icon="@mipmap/launcher_icon">
+
+
+
+
+
+
+
0, so an unknown need leaves it empty
+ // instead of filling against a fabricated 8h denominator.
+ val needMin = w.readInt(prefs, "sleep_need_min", -1)
val hrv = w.readInt(prefs, "hrv", -1)
val hrvBaseline = w.readInt(prefs, "hrv_baseline", -1)
diff --git a/ios/OpenStrapWatch Watch App/WatchMetrics.swift b/ios/OpenStrapWatch Watch App/WatchMetrics.swift
index 93f46414..60cccd63 100644
--- a/ios/OpenStrapWatch Watch App/WatchMetrics.swift
+++ b/ios/OpenStrapWatch Watch App/WatchMetrics.swift
@@ -19,7 +19,7 @@ struct WatchMetrics {
var readiness: Int // 0–100, -1 = none
var strain: Double // 0–21, -1 = none
var sleepMin: Int // minutes asleep, -1 = none
- var needMin: Int // sleep need (min)
+ var needMin: Int // sleep need (min); -1 = none — never fabricate 8h
var hrv: Int // RMSSD ms, -1 = none
var hrvBaseline: Int // baseline RMSSD ms, -1 = none
var rhr: Int // bpm, -1 = none
@@ -29,7 +29,7 @@ struct WatchMetrics {
var themeDark: Bool // mirror the app's Ember-on-Paper (false) / Char (true)
static let empty = WatchMetrics(
- hasData: false, readiness: -1, strain: -1, sleepMin: -1, needMin: 480,
+ hasData: false, readiness: -1, strain: -1, sleepMin: -1, needMin: -1,
hrv: -1, hrvBaseline: -1, rhr: -1, coachLine: "", battPct: -1, updatedAt: 0,
themeDark: true)
@@ -40,7 +40,7 @@ struct WatchMetrics {
readiness: d?.object(forKey: "readiness") as? Int ?? -1,
strain: d?.object(forKey: "strain") as? Double ?? -1,
sleepMin: d?.object(forKey: "sleep_min") as? Int ?? -1,
- needMin: d?.object(forKey: "sleep_need_min") as? Int ?? 480,
+ needMin: d?.object(forKey: "sleep_need_min") as? Int ?? -1,
hrv: d?.object(forKey: "hrv") as? Int ?? -1,
hrvBaseline: d?.object(forKey: "hrv_baseline") as? Int ?? -1,
rhr: d?.object(forKey: "rhr") as? Int ?? -1,
diff --git a/ios/OpenStrapWidget/OpenStrapWidget.swift b/ios/OpenStrapWidget/OpenStrapWidget.swift
index b24bfa33..99b57963 100644
--- a/ios/OpenStrapWidget/OpenStrapWidget.swift
+++ b/ios/OpenStrapWidget/OpenStrapWidget.swift
@@ -58,7 +58,7 @@ struct OpenStrapEntry: TimelineEntry {
let readiness: Int // -1 = none (composite 0..100) — the headline
let strain: Double // -1 = none
let sleepMin: Int // -1 = none
- let needMin: Int
+ let needMin: Int // -1 = none (sleep need, min) — never fabricate 8h
let hrv: Int // -1 = none (RMSSD, ms)
let hrvBaseline: Int // -1 = none (personal RMSSD baseline, ms)
let rhr: Int // -1 = none
@@ -106,7 +106,7 @@ private enum Store {
readiness: d?.object(forKey: "readiness") as? Int ?? -1,
strain: d?.object(forKey: "strain") as? Double ?? -1,
sleepMin: d?.object(forKey: "sleep_min") as? Int ?? -1,
- needMin: (d?.object(forKey: "sleep_need_min") as? Int) ?? 480,
+ needMin: (d?.object(forKey: "sleep_need_min") as? Int) ?? -1,
hrv: d?.object(forKey: "hrv") as? Int ?? -1,
hrvBaseline: d?.object(forKey: "hrv_baseline") as? Int ?? -1,
rhr: d?.object(forKey: "rhr") as? Int ?? -1,
@@ -171,7 +171,7 @@ private enum TodayAPI {
let strain = val(daily, "strain") ?? -1
let rhr = val(daily, "resting_hr").map { Int($0.rounded()) } ?? -1
let sleepMin = val(sleep, "duration_min").map { Int($0.rounded()) } ?? -1
- let needMin = val(sleep, "need_min").map { Int($0.rounded()) } ?? 480
+ let needMin = val(sleep, "need_min").map { Int($0.rounded()) } ?? -1
let hrv = (hrvObj?["rmssd"] as? NSNumber).map { Int($0.doubleValue.rounded()) } ?? -1
let hrvBase = (hrvObj?["baseline"] as? NSNumber).map { Int($0.doubleValue.rounded()) } ?? -1
diff --git a/ios/Runner/Info.plist b/ios/Runner/Info.plist
index e6ad6b44..3cfe2647 100644
--- a/ios/Runner/Info.plist
+++ b/ios/Runner/Info.plist
@@ -4,6 +4,31 @@
CADisableMinimumFrameDurationOnPhone
+
+ FIREBASE_ANALYTICS_COLLECTION_ENABLED
+
+ FirebaseCrashlyticsCollectionEnabled
+
+ firebase_performance_collection_enabled
+
+
+ GOOGLE_ANALYTICS_DEFAULT_ALLOW_AD_PERSONALIZATION_SIGNALS
+
+ GOOGLE_ANALYTICS_IDFV_COLLECTION_ENABLED
+
CFBundleDevelopmentRegion
$(DEVELOPMENT_LANGUAGE)
CFBundleDisplayName
diff --git a/lib/app.dart b/lib/app.dart
index f0b35985..45bba860 100644
--- a/lib/app.dart
+++ b/lib/app.dart
@@ -6,6 +6,7 @@ import 'package:provider/provider.dart';
import 'ai/briefing.dart';
import 'coach/coach_config.dart';
+import 'notify/notification_service.dart';
import 'notify/tap_router.dart';
import 'state/app_state.dart';
import 'state/prefs.dart';
@@ -67,6 +68,12 @@ class _OpenStrapAppState extends State with WidgetsBindingObserver
void didChangeAppLifecycleState(AppLifecycleState state) {
final app = context.read();
if (state == AppLifecycleState.resumed) {
+ // The user may have flipped our notification switch either way in OS
+ // Settings while we were backgrounded. Drop the cached authorization
+ // decision so the next present/schedule re-reads reality — a denial used
+ // to latch for the whole process, silencing every notification and
+ // scheduled reminder until a full app restart.
+ NotificationService.instance.invalidatePermissionCache();
app.maybeFinishFromLiveActivity();
unawaited(app.maybeStopBreathingFromLiveActivity());
app.refreshAppStatus(); // re-check OTA + admin banner on every foreground
diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart
index 9659e7d2..ed063a64 100644
--- a/lib/ble/ble_engine.dart
+++ b/lib/ble/ble_engine.dart
@@ -129,17 +129,6 @@ int countBurstTrafficPackets({
unknownCount;
}
-@visibleForTesting
-int nextBurstStablePollStreak({
- required bool queueEmpty,
- required int currentCount,
- required int previousCount,
- required int stableStreak,
-}) {
- if (!queueEmpty) return 0;
- return currentCount == previousCount ? (stableStreak + 1) : 0;
-}
-
@visibleForTesting
bool shouldPauseMaintenanceTraffic({required bool offloadActive}) =>
offloadActive;
@@ -427,30 +416,64 @@ class BleEngine {
/// harmless once we own the band).
Future _claimBand() async {
final other = _bandOwner;
- if (other != null && !identical(other, this)) {
- if (isBackgroundDrainer) {
+ final incumbentPresent = other != null && !identical(other, this);
+ final decision = BandClaimPolicy.decide(
+ incumbentPresent: incumbentPresent,
+ // LIVENESS, not just non-nullness: a claim held by an engine that has no
+ // session (its connect threw, or its link went down) is a STALE claim,
+ // and honouring it wedged every later background drain for the whole
+ // process lifetime ("strap not reachable this cycle", forever).
+ incumbentLive: incumbentPresent && other.holdsBandLink,
+ isBackgroundDrainer: isBackgroundDrainer,
+ );
+ switch (decision) {
+ case BandClaimDecision.yieldToOwner:
_log(
- 'band already owned by the foreground session — background drain '
+ 'band already owned by a live foreground session — background drain '
'yielding (avoids duplicate ACKs on the same offload).',
);
return false;
- }
- _log('preempting a background drain to take the foreground session.');
- try {
- await other
- .disconnect()
- .timeout(const Duration(seconds: 10));
- } on TimeoutException {
- _log('preempted engine teardown timed out after 10s — proceeding '
- 'with the foreground connect anyway.');
- } catch (e) {
- _log('preempted engine teardown failed ($e) — proceeding.');
- }
+ case BandClaimDecision.preemptThenClaim:
+ _log('preempting a background drain to take the foreground session.');
+ try {
+ await other!.disconnect().timeout(const Duration(seconds: 10));
+ } on TimeoutException {
+ _log('preempted engine teardown timed out after 10s — proceeding '
+ 'with the foreground connect anyway.');
+ } catch (e) {
+ _log('preempted engine teardown failed ($e) — proceeding.');
+ }
+ break;
+ case BandClaimDecision.claim:
+ if (incumbentPresent) {
+ _log('taking over a STALE band claim (the previous owner has no '
+ 'live link).');
+ }
+ break;
}
_bandOwner = this;
return true;
}
+ /// Whether this engine actually holds (or is actively bringing up) a BLE
+ /// link — the liveness test [BandClaimPolicy] uses on the incumbent owner.
+ /// A session object exists from the moment `_doConnect` starts, so a connect
+ /// still in flight correctly counts as live; every failure path nulls the
+ /// session and drops the phase to idle/error before returning.
+ bool get holdsBandLink =>
+ _session != null &&
+ _phase != BleConnState.idle &&
+ _phase != BleConnState.error;
+
+ /// Test-only view of the process-wide single-owner claim.
+ @visibleForTesting
+ static bool get bandClaimed => _bandOwner != null;
+
+ /// Test-only reset of the process-wide claim (static state otherwise leaks
+ /// across test cases).
+ @visibleForTesting
+ static void resetBandClaimForTest() => _bandOwner = null;
+
void _releaseBand() {
if (identical(_bandOwner, this)) _bandOwner = null;
}
@@ -567,7 +590,7 @@ class BleEngine {
// Historical-offload bookkeeping. A controller is live for the whole connection
// (we keep ACKing HISTORY_END markers as they arrive, even after the first
// HISTORY_COMPLETE — a later strap-triggered offload reuses it).
- _DrainController? _drain;
+ DrainController? _drain;
bool _liveEnabled = false;
// Background live downgrade: only the compact realtime-HR stream is armed
// (no high-rate R10/R11 + IMU + optical flood). Set by [enableHrOnlyLive].
@@ -639,6 +662,9 @@ class BleEngine {
// Lifetime count of GET_DATA_RANGE reads rejected by isCorruptFutureRtc —
// see the range_oldest/range_newest handler below.
int _corruptDataRangeCount = 0;
+ // Lifetime count of GET_CLOCK `clock_epoch` reads rejected by the same gate
+ // (ClockPolicy.acceptsClockRead) — see the clock_epoch handler below.
+ int _corruptClockReadCount = 0;
DateTime? _bondTime; // when the handshake completed (bond confirmed)
DateTime? _armTime; // when live (R10/R11) streams were last armed
int _autoContinueCount = 0; // consecutive auto-continues this connection
@@ -814,6 +840,10 @@ class BleEngine {
// recovery already happens automatically at the DB layer.
'counter_regressions_total': _counterRegression.regressions,
'corrupt_data_ranges_total': _corruptDataRangeCount,
+ 'corrupt_clock_reads_total': _corruptClockReadCount,
+ // Bursts whose open chunk was discarded un-committed and whose HISTORY_END
+ // token was therefore refused (see BurstTrimGuard).
+ 'poisoned_bursts_total': _drain?.poisonedBursts ?? 0,
'history_requests': _historyRequests,
'history_completions': _historyCompletions,
'successful_bursts': _successfulBursts,
@@ -922,9 +952,35 @@ class BleEngine {
if (!await _claimBand()) return false;
// Any prior session is dead to us now — tear it down before a new one.
await _teardownSession(intentional: true);
- return _doConnect(device);
+ try {
+ return await _doConnect(device);
+ } catch (e) {
+ // _doConnect guards its own known failure modes, but anything thrown
+ // OUTSIDE those guards (e.g. the connectionState subscription setup, which
+ // runs before the first try block) used to escape with the band claim
+ // still held and a half-built session left in `connecting` — which
+ // [BandClaimPolicy] would then read as a LIVE incumbent, permanently
+ // starving every later background drain. Route every escape through the
+ // one failure exit.
+ _log('connect failed (unhandled): $e');
+ await _failConnect();
+ return false;
+ }
});
+ /// Common exit for every failed-connect path: tear the half-built session
+ /// down, drop to `idle`, AND RELEASE THE BAND CLAIM.
+ ///
+ /// [_claimBand] runs BEFORE the link is up, so a connect that threw used to
+ /// leave `_bandOwner` pointing at an engine with no link — and only
+ /// `disconnect()` ever released it, which nothing calls on this path. Every
+ /// later background drain then saw a non-null owner and yielded forever.
+ Future _failConnect() async {
+ await _teardownSession(intentional: true);
+ _releaseBand();
+ _setPhase(BleConnState.idle);
+ }
+
Future _doConnect(BluetoothDevice device) async {
state.address = device.remoteId.str;
_setPhase(BleConnState.connecting);
@@ -958,8 +1014,7 @@ class BleEngine {
);
} catch (e) {
_log('connect failed: $e');
- await _teardownSession(intentional: true);
- _setPhase(BleConnState.idle);
+ await _failConnect();
return false;
}
@@ -1026,6 +1081,7 @@ class BleEngine {
if (!session.connected) {
_log('connect: link dropped during setup.');
+ await _failConnect();
return false;
}
@@ -1039,8 +1095,7 @@ class BleEngine {
}
if (svc == null) {
_log('Harvard service not found on device.');
- await _teardownSession(intentional: true);
- _setPhase(BleConnState.idle);
+ await _failConnect();
return false;
}
BluetoothCharacteristic? find(String prefix) {
@@ -1059,8 +1114,7 @@ class BleEngine {
events == null ||
data == null) {
_log('Missing one or more Harvard characteristics.');
- await _teardownSession(intentional: true);
- _setPhase(BleConnState.idle);
+ await _failConnect();
return false;
}
@@ -1144,7 +1198,7 @@ class BleEngine {
// fire INIT — which triggers the historical flood. Historical + live records
// then arrive on the same subscription; HISTORY_END markers are committed
// (raw+samples+cursor, atomically) BEFORE we ACK, so the offload is resumable.
- _drain = _DrainController(
+ _drain = DrainController(
onRecord: _storeRecord,
onRecordsBatch: onRecordsBatch == null ? null : _storeRecordsBatch,
onCommit: onCommitBatch == null ? null : _commitBatch,
@@ -1159,8 +1213,7 @@ class BleEngine {
return true;
} catch (e) {
_log('connect setup failed: $e');
- await _teardownSession(intentional: true);
- _setPhase(BleConnState.idle);
+ await _failConnect();
return false;
}
}
@@ -1313,7 +1366,7 @@ class BleEngine {
_lastRx = DateTime.now();
for (final frame in session.asm[role]!.feed(chunk)) {
if (frame.valid) {
- _onFrame(role, frame);
+ _onFrame(role, frame, session);
} else {
// Previously silent: a degrading radio corrupting frames looked
// identical to a healthy one everywhere. Now counted (surfaced in
@@ -1339,7 +1392,10 @@ class BleEngine {
// ── link-down handling (drives reconnect via the caller's contract) ─────────────
void _onLinkDown(_Session session) {
- if (_session != session) return; // a stale session's stream
+ if (LinkDownPolicy.evaluate(sessionIsCurrent: _session == session) ==
+ LinkDownAction.ignoreStaleSession) {
+ return; // a stale session's stream
+ }
final wasIntentional = session.intentionalClose;
session.connected = false;
// A drain in flight must complete (with linkDown) immediately, not run out
@@ -1359,6 +1415,26 @@ class BleEngine {
// through the same single-flight connect, so there's still exactly one path.
_setOffloadActive(false);
_setPhase(BleConnState.idle);
+ // TEAR THE SESSION DOWN NOW. This used to happen ONLY on the next
+ // connect()/disconnect() — which never comes when BondRefusalGiveUp pauses
+ // auto-reconnect (`state.autoReconnectPaused`), so the dead session's five
+ // timers (heartbeat 10 s, keep-alive 30 s, periodic backfill 900 s, idle
+ // watchdog, historical retry) kept firing into a dead characteristic
+ // forever and its four onValueReceived subscriptions stayed registered —
+ // one more full set leaked on every drop. Deferred off this notification
+ // callback (we are inside one of the very subscriptions being cancelled)
+ // and non-intentional, so no redundant device.disconnect() is issued.
+ unawaited(
+ Future(() async {
+ if (_session != session) return; // a connect already replaced us
+ await _teardownSession(intentional: false);
+ // A claim held with no link is a stale claim (see [BandClaimPolicy]);
+ // releasing it here is what stops a failed foreground link from
+ // wedging every later background drain. The caller's reconnect loop
+ // re-claims through connect(), where foreground still preempts.
+ _releaseBand();
+ }),
+ );
}
/// Feed an UNINTENTIONAL disconnect to the cross-reconnect detectors. A timeout
@@ -1416,7 +1492,12 @@ class BleEngine {
static const Duration _serviceDiscoveryTimeout = Duration(seconds: 15);
static const Duration _notifySetupTimeout = Duration(seconds: 15);
- Future _write(Uint8List raw) {
+ /// [owner] pins the write to ONE session. Without it a write queued by a
+ /// long-parked drain (a big commit, then up to ~25 s of ACK retries) lands on
+ /// whatever session happens to be current when the write chain reaches it —
+ /// i.e. an OLD connection's batch-ACK, with a re-used sync seq, written onto
+ /// a BRAND NEW link. Every offload write passes its owning session.
+ Future _write(Uint8List raw, {_Session? owner}) {
final session = _session;
final completer = Completer();
_writeChain = _writeChain.then((_) async {
@@ -1427,6 +1508,10 @@ class BleEngine {
_log('write skipped: link not ready.');
return;
}
+ if (owner != null && !identical(owner, session)) {
+ _log('write skipped: it belongs to a session that is no longer live.');
+ return;
+ }
// allowLongWrite: the rich SET_ALARM_TIME frame is 32B — the only write
// that exceeds the 20B ATT limit of a default (23B) MTU. Without a long
// write, flutter_blue_plus throws "value > mtu-3" if the negotiated MTU
@@ -1454,16 +1539,17 @@ class BleEngine {
/// false only after every attempt failed — the caller must then bounce the
/// link (the chunk is already durably committed; the band re-delivers it next
/// session and the decoded store dedups by REPLACE).
- Future _writeAckVerified(Uint8List ack) async {
+ Future _writeAckVerified(Uint8List ack, _Session session) async {
var failures = 0;
while (true) {
- if (await _write(ack)) return true;
+ if (_sessionIsStale(session)) return false;
+ if (await _write(ack, owner: session)) return true;
failures++;
if (!ackRetryPolicy.shouldRetry(failures)) return false;
_log('[SYNC] batch-ACK write failed (attempt $failures/'
'${ackRetryPolicy.maxAttempts}) — retrying.');
await Future.delayed(ackRetryPolicy.delayFor(failures));
- if (_session?.connected != true) return false;
+ if (_sessionIsStale(session)) return false;
}
}
@@ -1563,11 +1649,22 @@ class BleEngine {
}
// ── frame handling ─────────────────────────────────────────────────────────────
- void _onFrame(String role, Frame frame) {
+ void _onFrame(String role, Frame frame, _Session session) {
final pt = frame.packetType;
- if (role == 'data' &&
- (pt == PacketType.metadata || pt == PacketType.historicalData)) {
- _enqueueOffloadFrame(frame);
+ // Metadata ALWAYS takes the serialized queue, whatever characteristic it
+ // was reassembled on. It used to take the queue only on the `data` role;
+ // metadata off `cmd_from`/`events` was fired unawaited on the immediate
+ // path — the one route that could run a HISTORY_END handler CONCURRENTLY
+ // with the queued drain, i.e. two handlers on the same DrainController,
+ // where one snapshots an empty buffer and writes its ACK before the
+ // other's commit is durable. See [FrameRoutePolicy].
+ final route = FrameRoutePolicy.route(
+ isMetadata: pt == PacketType.metadata,
+ isHistorical: pt == PacketType.historicalData,
+ isDataRole: role == 'data',
+ );
+ if (route == FrameRoute.serializedQueue) {
+ _enqueueOffloadFrame(frame, session);
return;
}
_processImmediateFrame(frame);
@@ -1575,10 +1672,8 @@ class BleEngine {
void _processImmediateFrame(Frame frame) {
final pt = frame.packetType;
- if (pt == PacketType.metadata) {
- unawaited(_handleSyncMarker(frame));
- return;
- }
+ // NOTE: metadata never reaches here — [FrameRoutePolicy] routes every
+ // metadata frame to the serialized offload queue regardless of role.
// LIVE streams: realtime HR/RR (0x28), realtime R10 (0x2B), IMU (0x33).
// EPHEMERAL — these are the high-rate flood (~655 MB/day) and the daily
// metrics need ONLY the 1 Hz historical substrate (0x2F / R24). We do NOT
@@ -1639,17 +1734,28 @@ class BleEngine {
_absorbState(decoded);
}
- void _enqueueOffloadFrame(Frame frame) {
+ void _enqueueOffloadFrame(Frame frame, _Session session) {
+ if (_session != session || !session.connected) return; // stale session
_offloadFrames.add(frame);
if (_offloadActive || frame.packetType == PacketType.historicalData) {
_setOffloadActive(true);
}
if (_drainingOffloadFrames) return;
_drainingOffloadFrames = true;
- unawaited(_drainOffloadFrames());
- }
-
- Future _drainOffloadFrames() async {
+ unawaited(_drainOffloadFrames(session));
+ }
+
+ /// The ONE serialized offload-frame processor. [session] is the connection
+ /// that started this loop; it is re-checked around every await because a
+ /// drain can be parked for a long time (a multi-second large-batch commit,
+ /// then up to ~25 s of ACK retries) and `_teardownSession` clears
+ /// `_drainingOffloadFrames` underneath it. Without the guard a SECOND
+ /// drainer starts on the new session while this one is still alive — two
+ /// loops on the same DrainController, so one commit() snapshots an empty
+ /// buffer and its ACK can be written before the other's commit is durable —
+ /// and this stale loop would go on to write the OLD connection's token onto
+ /// the NEW link, tearing down a healthy session when that write failed.
+ Future _drainOffloadFrames(_Session session) async {
// this used to have no try/finally, so if anything inside the loop threw
// (a couple of the ledger writes in _handleSyncMarker weren't guarded),
// _drainingOffloadFrames never got reset back to false and every future
@@ -1657,6 +1763,7 @@ class BleEngine {
// sync progress until a full disconnect/reconnect.
try {
while (_offloadFrames.isNotEmpty) {
+ if (_sessionIsStale(session)) return;
final count = _offloadFrames.length > 64 ? 64 : _offloadFrames.length;
final batch = _offloadFrames.sublist(0, count);
_offloadFrames.removeRange(0, count);
@@ -1665,8 +1772,9 @@ class BleEngine {
// no Timer churn at flood rates. Markers re-arm it in _handleSyncMarker.
_armIdleWatchdog();
for (final frame in batch) {
+ if (_sessionIsStale(session)) return;
if (frame.packetType == PacketType.metadata) {
- await _handleSyncMarker(frame);
+ await _handleSyncMarker(frame, session);
} else {
_ingestHistoricalFrame(frame);
}
@@ -1676,10 +1784,18 @@ class BleEngine {
}
}
} finally {
- _drainingOffloadFrames = false;
+ // Only the CURRENT session's loop may clear the flag. A stale loop
+ // unwinding after the new session's drainer already started would
+ // otherwise re-open the door to a second concurrent drainer.
+ if (_session == session) _drainingOffloadFrames = false;
}
}
+ /// True once [session] is no longer the engine's live session — the guard
+ /// every long-parked offload callback shares.
+ bool _sessionIsStale(_Session session) =>
+ _session != session || !session.connected;
+
/// THE single historical-record processing path — used by BOTH the queued
/// offload drain (real traffic) and the immediate fallback. Decode → gate
/// (plausibility + frontier via [RecordGate]) → storage enqueue. Keeping one
@@ -1846,29 +1962,48 @@ class BleEngine {
if (f.containsKey('clock_epoch')) {
final dev = f['clock_epoch'] as int;
final wall = DateTime.now().millisecondsSinceEpoch ~/ 1000;
- _clockRef = ClockRef(device: dev, wall: wall);
- _log('Clock correlated: device=$dev wall=$wall (drift=${wall - dev}s).');
- // Re-issue SET_CLOCK if the strap RTC has drifted > 1 day or is unset —
- // but BOUND the retries: setClock() reads the clock back, so an unbounded
- // re-issue on a firmware that never latches either payload form would spin
- // SET_CLOCK/GET_CLOCK forever. Historical records carry their own embedded
- // unix time regardless, so giving up after a few tries is safe.
- if (ClockPolicy.shouldSetClock(dev, wall)) {
- if (_clockCorrectTries < 3) {
- _clockCorrectTries++;
- _log(
- 'Clock drift over policy — re-issuing SET_CLOCK '
- '(attempt $_clockCorrectTries/3).',
- );
- unawaited(setClock());
+ // SANITY GATE, mirroring the one `range_newest` gets below. An
+ // implausibly far-future `clock_epoch` yields a large NEGATIVE driftSec,
+ // and setAlarm arms at `when - driftSec` — years out, where the alarm
+ // silently never fires — while the bounded SET_CLOCK retry budget is
+ // spent chasing a value that was never real. Reject the read: with no
+ // correlation the alarm falls back to the raw wall epoch. connect()
+ // already issues an unconditional SET_CLOCK, and the periodic re-verify
+ // re-reads, so a genuinely-wrong RTC still gets corrected.
+ if (!ClockPolicy.acceptsClockRead(dev, wall)) {
+ _corruptClockReadCount++;
+ _log(
+ '[SYNC] GET_CLOCK clock_epoch=$dev is implausibly far in the future '
+ '— treating as a corrupt strap RTC read; NOT correlating the strap '
+ 'clock (alarms fall back to the raw wall epoch) '
+ '(corrupt_clock_reads_total=$_corruptClockReadCount).',
+ );
+ } else {
+ _clockRef = ClockRef(device: dev, wall: wall);
+ _log('Clock correlated: device=$dev wall=$wall (drift=${wall - dev}s).');
+ // Re-issue SET_CLOCK if the strap RTC has drifted > 1 day or is unset —
+ // but BOUND the retries: setClock() reads the clock back, so an
+ // unbounded re-issue on a firmware that never latches either payload
+ // form would spin SET_CLOCK/GET_CLOCK forever. Historical records carry
+ // their own embedded unix time regardless, so giving up after a few
+ // tries is safe.
+ if (ClockPolicy.shouldSetClock(dev, wall)) {
+ if (_clockCorrectTries < 3) {
+ _clockCorrectTries++;
+ _log(
+ 'Clock drift over policy — re-issuing SET_CLOCK '
+ '(attempt $_clockCorrectTries/3).',
+ );
+ unawaited(setClock());
+ } else {
+ _log(
+ 'Clock still off after 3 SET_CLOCK attempts — giving up; '
+ 'firmware may not accept our payload length.',
+ );
+ }
} else {
- _log(
- 'Clock still off after 3 SET_CLOCK attempts — giving up; '
- 'firmware may not accept our payload length.',
- );
+ _clockCorrectTries = 0; // latched — reset for the next drift episode
}
- } else {
- _clockCorrectTries = 0; // latched — reset for the next drift episode
}
}
if (f.containsKey('range_oldest') && f.containsKey('range_newest')) {
@@ -2007,7 +2142,84 @@ class BleEngine {
}
}
- Future _handleSyncMarker(Frame frame) async {
+ /// Refuse to echo a HISTORY_END token, for one of the [TrimAckVerdict]
+ /// blocked reasons. The band keeps the chunk and re-delivers it on the next
+ /// offload; re-delivery is dedup-safe (decoded rows REPLACE by rec_ts, raw
+ /// rows key on the record hex).
+ Future _refuseHistoryEndTrim(
+ TrimAckVerdict verdict, {
+ required DrainController d,
+ required _Session session,
+ required String tokenHex,
+ required int? batchId,
+ }) async {
+ switch (verdict) {
+ case TrimAckVerdict.send:
+ return;
+ case TrimAckVerdict.blockedStaleSession:
+ // Deliberately no ledger write and no teardown: we may be mid-teardown
+ // already, and every side effect here would land on a session that is
+ // not ours.
+ _log(
+ '[SYNC] HISTORY_END token=$tokenHex belongs to a session that is no '
+ 'longer live — NOT ACKing. Writing it would put an old connection\'s '
+ 'token (with a re-used sync seq) onto the new link. The band '
+ 're-delivers this chunk next offload.',
+ );
+ return;
+ case TrimAckVerdict.blockedDiscardedBurst:
+ // The idle watchdog abandoned this burst's buffered records. Persist
+ // anything that arrived since (dedup-safe) but WITHOUT the token, so
+ // the trim cursor never claims a chunk we threw away.
+ await d.commit(null);
+ _log(
+ '[SYNC] HISTORY_END token=$tokenHex terminates a DISCARDED burst '
+ '(its open chunk was abandoned un-committed) — NOT ACKing, so the '
+ 'band cannot trim the records we dropped. It re-delivers them next '
+ 'offload.',
+ );
+ await _bestEffortLedgerWrite(() => LocalDb.upsertSyncLedgerEntry(
+ chunkId: 'batch:$tokenHex',
+ kind: 'historical_batch',
+ status: 'trim_refused',
+ lastError: 'discarded_burst',
+ metaPatch: {'batch_id': batchId, 'records': d.records},
+ ));
+ return;
+ case TrimAckVerdict.blockedCommitFailed:
+ // THE safe-trim invariant. The transaction rolled back, so the cursor
+ // did not advance and the rows are not durable — commit() re-buffered
+ // them rather than losing them. Never ACK here: that is exactly the
+ // path where records existed nowhere, permanently and silently.
+ _log(
+ '[SYNC] DURABLE COMMIT FAILED for token=$tokenHex — NOT ACKing (the '
+ 'band must keep this chunk). Records were re-buffered; bouncing the '
+ 'link so the next session retries the commit from a clean batch.',
+ );
+ await _bestEffortLedgerWrite(() => LocalDb.upsertSyncLedgerEntry(
+ chunkId: 'batch:$tokenHex',
+ kind: 'historical_batch',
+ status: 'commit_failed',
+ lastError: 'durable_commit_failed',
+ metaPatch: {'batch_id': batchId, 'records': d.records},
+ ));
+ // Bounce rather than retry in place: a commit that failed on a large
+ // batch (the observed production OOM inside commitSyncBatch) only gets
+ // bigger if we keep appending to the same buffer. A reconnect drops
+ // the buffer, and the band re-delivers from its un-advanced cursor.
+ if (!_sessionIsStale(session)) {
+ unawaited(
+ _teardownSession(intentional: false).then((_) {
+ _setPhase(BleConnState.idle); // caller's reconnect loop takes over
+ }),
+ );
+ }
+ return;
+ }
+ }
+
+ Future _handleSyncMarker(Frame frame, _Session session) async {
+ if (_sessionIsStale(session)) return;
final m = parseMetadata(frame.inner);
if (m == null) return;
_armIdleWatchdog();
@@ -2033,6 +2245,9 @@ class BleEngine {
if (m.sub == SyncMeta.historyEnd && m.token != null) {
final d = _drain;
if (d == null) return;
+ final tokenHex = m.token!
+ .map((b) => b.toRadixString(16).padLeft(2, '0'))
+ .join();
if (!_offloadActive) {
_setHpsTerminal(
_HpsTerminalKind.metadataWhileNotSyncing,
@@ -2040,7 +2255,25 @@ class BleEngine {
drain: d,
);
}
- await _awaitBurstTrafficSettle(d);
+ // PRE-COMMIT GATE. Refuse the trim before anything else touches the link
+ // or the durable cursor: a stale session must not be written to at all,
+ // and a poisoned burst's records are already gone — there is nothing
+ // this token may legitimately trim.
+ final preVerdict = TrimAckPolicy.evaluate(
+ sessionCurrent: !_sessionIsStale(session),
+ burstDiscarded: d.burstDiscarded,
+ commitDurable: true,
+ );
+ if (preVerdict != TrimAckVerdict.send) {
+ await _refuseHistoryEndTrim(
+ preVerdict,
+ d: d,
+ session: session,
+ tokenHex: tokenHex,
+ batchId: m.batchId,
+ );
+ return;
+ }
final expected = m.expectedPacketCount;
// Records the plausibility gate silently rejected THIS burst (stale/
// wandering-clock block — by design, "neither stored nor counted",
@@ -2098,9 +2331,6 @@ class BleEngine {
}
_successfulBursts++;
_mergeValidatedBurst(d);
- final tokenHex = m.token!
- .map((b) => b.toRadixString(16).padLeft(2, '0'))
- .join();
final r = d.bufferedRecTsRange;
_log(
'[SYNC] HistoryEnd batch=${m.batchId} records=${d.records} '
@@ -2114,18 +2344,40 @@ class BleEngine {
// once the ACK is link-layer confirmed, so a crash before the ACK
// re-delivers the chunk. Echo the 8-byte slice the band acks verbatim —
// a mangled echo is the "Groundhog Day" re-flood bug.
- await d.commit(m.token); // raw + samples + strap_trim cursor, atomic
+ final durable = await d.commit(m.token); // raw+samples+cursor, atomic
+ // RE-GATE after the await. commit() can take seconds on a large batch —
+ // long enough for the link to die under us — and it now reports whether
+ // the transaction actually became durable instead of swallowing the
+ // exception. Either way the ACK is refused, which is the whole point:
+ // the ACK is what makes the band trim its flash.
+ final verdict = TrimAckPolicy.evaluate(
+ sessionCurrent: !_sessionIsStale(session),
+ burstDiscarded: d.burstDiscarded,
+ commitDurable: durable,
+ );
+ if (verdict != TrimAckVerdict.send) {
+ await _refuseHistoryEndTrim(
+ verdict,
+ d: d,
+ session: session,
+ tokenHex: tokenHex,
+ batchId: m.batchId,
+ );
+ return;
+ }
final ack = buildHistoryResultOk(_seq.nextSync(), m.token!);
_log(
'[SYNC] ACK frame='
'${ack.map((b) => b.toRadixString(16).padLeft(2, '0')).join()}',
);
- // VERIFIED ACK (retried): the cursor above is already durably committed,
- // so a silently-failed ACK write would leave the band never trimming and
- // re-flooding the same chunk forever. On persistent failure bounce the
- // link — the committed data is safe, and the next session's re-delivery
- // is dedup-safe (decoded rows REPLACE by rec_ts).
- if (!await _writeAckVerified(ack)) {
+ // VERIFIED ACK (retried). We only reach here when [TrimAckPolicy]
+ // returned `send` — i.e. the commit above REPORTED durable (it no longer
+ // swallows its exception) and the session is still ours — so the cursor
+ // genuinely is committed. A silently-failed ACK write would leave the
+ // band never trimming and re-flooding the same chunk forever. On
+ // persistent failure bounce the link — the committed data is safe, and
+ // the next session's re-delivery is dedup-safe (rows REPLACE by rec_ts).
+ if (!await _writeAckVerified(ack, session)) {
// Real per-chunk ledger row, keyed by the token itself — previously
// every ledger write here collapsed onto one shared 'capture' row,
// so a token that kept failing ACROSS reconnects (the "Groundhog
@@ -2168,11 +2420,16 @@ class BleEngine {
'${ackRetryPolicy.maxAttempts} attempts (token=$tokenHex, '
'failures_for_this_token=$failCount) — bouncing the link; data '
'is committed and the band will re-send.');
- unawaited(
- _teardownSession(intentional: false).then((_) {
- _setPhase(BleConnState.idle); // caller's reconnect loop takes over
- }),
- );
+ // ONLY bounce a session that is still OURS. _writeAckVerified also
+ // returns false when the session died under it, and tearing down then
+ // would kill the healthy session that replaced it.
+ if (!_sessionIsStale(session)) {
+ unawaited(
+ _teardownSession(intentional: false).then((_) {
+ _setPhase(BleConnState.idle); // caller's reconnect loop takes over
+ }),
+ );
+ }
return;
}
_chunkFailures.recordSuccess(tokenHex);
@@ -2215,7 +2472,24 @@ class BleEngine {
// Backlog fully handed over (cursor is now at the live edge). Commit the tail
// and KEEP LISTENING — live records continue on the same subscription. We do
// NOT ACK a HISTORY_COMPLETE and we do NOT switch modes.
- await d.commit(null); // tail (no new token) — persist anything buffered
+ final tailDurable = await d.commit(null); // tail — no new trim token
+ if (_sessionIsStale(session)) return; // link died under the tail commit
+ if (!tailDurable) {
+ // No ACK is written for a HISTORY_COMPLETE, so nothing was trimmed and
+ // no data is at risk — but the tail is NOT durable yet. commit() left
+ // the records buffered, so the next commit (the next burst's
+ // HISTORY_END, or the flush on teardown) re-attempts them.
+ _log(
+ '[SYNC] HistoryComplete tail commit FAILED — ${d.bufferedRecords} '
+ 'records stay buffered for the next commit. Nothing was trimmed '
+ '(HISTORY_COMPLETE is never ACKed), so no data is at risk.',
+ );
+ await _bestEffortLedgerWrite(() => LocalDb.upsertSyncLedgerEntry(
+ status: 'tail_commit_failed',
+ lastError: 'durable_commit_failed',
+ metaPatch: {'records_seen': d.records},
+ ));
+ }
d.onComplete();
_historyCompletions++;
_session?.idleWatchdog?.cancel();
@@ -2241,51 +2515,6 @@ class BleEngine {
}
}
- Future _awaitBurstTrafficSettle(_DrainController d) async {
- const poll = Duration(milliseconds: 60);
- const budget = Duration(milliseconds: 720);
- const requiredStablePolls = 3;
- final deadline = DateTime.now().add(budget);
- var previousCount = d.currentBurstPacketCount;
- var waitedMs = 0;
- var stablePolls = 0;
- while (DateTime.now().isBefore(deadline)) {
- if (_offloadFrames.isNotEmpty) {
- await Future.delayed(poll);
- waitedMs += poll.inMilliseconds;
- previousCount = d.currentBurstPacketCount;
- stablePolls = 0;
- continue;
- }
- await Future.delayed(poll);
- waitedMs += poll.inMilliseconds;
- final currentCount = d.currentBurstPacketCount;
- stablePolls = nextBurstStablePollStreak(
- queueEmpty: _offloadFrames.isEmpty,
- currentCount: currentCount,
- previousCount: previousCount,
- stableStreak: stablePolls,
- );
- if (_offloadFrames.isEmpty && stablePolls >= requiredStablePolls) {
- if (waitedMs > 0) {
- _log(
- '[SYNC] history-end settle: waited=${waitedMs}ms '
- 'traffic=$currentCount historical=${d.currentBurstHistoricalPacketCount} '
- 'stable_polls=$stablePolls',
- );
- }
- return;
- }
- previousCount = currentCount;
- }
- _log(
- '[SYNC] history-end settle timed out at ${waitedMs}ms '
- 'traffic=${d.currentBurstPacketCount} '
- 'historical=${d.currentBurstHistoricalPacketCount} '
- 'stable_polls=$stablePolls',
- );
- }
-
// ── post-offload policy: empty-sync, stuck-strap, auto-continue ──────────────
Future _onOffloadFinished({required bool complete}) async {
final d = _drain;
@@ -2735,7 +2964,7 @@ class BleEngine {
void _setHpsTerminal(
_HpsTerminalKind kind, {
String? reason,
- _DrainController? drain,
+ DrainController? drain,
}) {
final d = drain ?? _drain;
_lastHpsTerminal = _HpsTerminal(
@@ -2748,7 +2977,7 @@ class BleEngine {
);
}
- void _mergeValidatedBurst(_DrainController d) {
+ void _mergeValidatedBurst(DrainController d) {
final burstCounts = d.burstStats.dataPacketCountsByRevision;
final mergedCounts = {
..._sessionPacketCounts.dataPacketCountsByRevision,
@@ -2846,14 +3075,19 @@ class BleEngine {
/// [awaitComplete] future that resolves when the band signals HISTORY_COMPLETE (or
/// the link drops / a safety timeout elapses), so a caller can block until the
/// backlog is fully handed over without disturbing the continuous listen.
-class _DrainController {
+///
+/// ENGINE-INTERNAL. Public only so the safe-trim invariant it enforces (a
+/// commit that fails must re-buffer and must NOT let the caller ACK) can be
+/// regression-tested directly without a real band.
+@visibleForTesting
+class DrainController {
final SampleSink onRecord;
final BatchSink? onRecordsBatch;
final CommitSyncBatchSink? onCommit;
final ArchiveSink? onArchive;
final void Function(String) log;
- _DrainController({
+ DrainController({
required this.onRecord,
required this.onRecordsBatch,
required this.onCommit,
@@ -2869,7 +3103,7 @@ class _DrainController {
final List _archives = [];
// Per-burst packet accounting (per-revision counts + sequence gap detection),
// merged into the session totals when a burst validates.
- final _BurstStats burstStats = _BurstStats();
+ final BurstStats burstStats = BurstStats();
int records = 0; // total this connection
int recordsThisOffload = 0; // since the last HISTORY_COMPLETE / rearm
@@ -2908,6 +3142,17 @@ class _DrainController {
bool lastTrimAdvanced = false;
int consecutiveValidationFailures = 0;
+ // Poison latch for the burst currently open (see BurstTrimGuard): once its
+ // records have been discarded un-committed, its straggler HISTORY_END must
+ // never be echoed.
+ final BurstTrimGuard _trimGuard = BurstTrimGuard();
+
+ /// True while the open burst's terminal may NOT be ACKed.
+ bool get burstDiscarded => _trimGuard.discarded;
+
+ /// Bursts poisoned this connection (diagnostics).
+ int get poisonedBursts => _trimGuard.poisonedBursts;
+
bool get _buffering => onCommit != null || onRecordsBatch != null;
int get currentBurstPacketCount => burstStats.totalTrafficPacketCount;
int get currentBurstTrafficCount => burstStats.totalTrafficPacketCount;
@@ -2987,6 +3232,9 @@ class _DrainController {
_linkDown = false;
_lastProgressAt = DateTime.now();
burstStats.reset();
+ // A fresh burst starts un-poisoned: whatever was discarded belonged to the
+ // burst that just ended, and the band will re-deliver it.
+ _trimGuard.beginBurst();
}
void onLinkDown() => _linkDown = true;
@@ -2997,10 +3245,19 @@ class _DrainController {
/// Abandon the buffered-but-not-yet-committed chunk WITHOUT persisting (idle
/// watchdog). These records were never ACKed, so the band re-delivers them on the
/// next offload — dropping them here just avoids ACKing a partial.
+ ///
+ /// POISONS THE OPEN BURST. Discarding alone was not enough: the band had
+ /// already put that burst's HISTORY_END on the wire, and the terminal handler
+ /// went on to commit the (now empty) buffer and echo the token verbatim —
+ /// trimming exactly the records that were just dropped. The poison is
+ /// unconditional (even with an empty buffer, this burst was abandoned) and
+ /// is cleared only by [rearm] / a fresh HISTORY_START.
void discardOpenChunk() {
+ _trimGuard.discardOpenChunk();
if (_raws.isEmpty && _archives.isEmpty) return;
log('discarding ${_raws.length} un-ACKed buffered records + '
- '${_archives.length} archived (idle).');
+ '${_archives.length} archived (idle). This burst\'s HISTORY_END token '
+ 'is now un-ACKable — the band keeps the chunk.');
_raws.clear();
_samples.clear();
_archives.clear();
@@ -3010,10 +3267,24 @@ class _DrainController {
/// ATOMICALLY (via onCommit) and return only once durable — the caller writes
/// the ACK afterwards. Snapshots the buffer so records arriving during the await
/// land in the next commit. Updates [lastTrimAdvanced].
- Future commit(List? token) async {
+ ///
+ /// RETURNS WHETHER THE CHUNK IS ACTUALLY DURABLE. This used to be
+ /// `Future` with the exception swallowed into a log line, while the
+ /// buffer had ALREADY been snapshotted and cleared — so a failed transaction
+ /// left the rows nowhere (buffer cleared, transaction rolled back, cursor
+ /// unadvanced) and the caller went straight on to echo the HISTORY_END trim
+ /// token, telling the band to delete them from its flash. Those records
+ /// existed nowhere, permanently and silently. On failure the buffer is now
+ /// RESTORED (at the front, so arrival order is preserved) and the trim
+ /// bookkeeping rolled back, and the caller MUST NOT ACK — the band keeps the
+ /// chunk and re-delivers it, which is dedup-safe (decoded rows REPLACE by
+ /// rec_ts).
+ Future commit(List? token) async {
final tokenHex = token
?.map((b) => b.toRadixString(16).padLeft(2, '0'))
.join();
+ final previousAckedToken = _lastAckedToken;
+ final previousTrimAdvanced = lastTrimAdvanced;
lastTrimAdvanced = tokenHex != null && tokenHex != _lastAckedToken;
if (tokenHex != null) _lastAckedToken = tokenHex;
final raws = List.from(_raws);
@@ -3028,12 +3299,24 @@ class _DrainController {
} else if (onRecordsBatch != null && raws.isNotEmpty) {
await onRecordsBatch!(raws, samples);
}
+ return true;
} catch (e) {
- log('offload commit error: $e');
+ // Put the snapshot back at the FRONT: records that arrived during the
+ // await are already appended behind it, so this preserves arrival order.
+ _raws.insertAll(0, raws);
+ _samples.insertAll(0, samples);
+ _archives.insertAll(0, archives);
+ // Roll back the trim bookkeeping too — nothing advanced.
+ _lastAckedToken = previousAckedToken;
+ lastTrimAdvanced = previousTrimAdvanced;
+ log('offload commit FAILED ($e) — ${raws.length} records + '
+ '${archives.length} archived re-buffered; the caller MUST NOT ACK '
+ 'this chunk (the band still holds it).');
+ return false;
}
}
- Future flush() => commit(null);
+ Future flush() => commit(null);
/// Resolve once the current offload reaches HISTORY_COMPLETE, the link drops, or
/// [timeout] elapses. Pure waiting — NO abort is ever sent (cutting the offload
@@ -3077,7 +3360,8 @@ class _DrainController {
}
}
-class _BurstStats {
+@visibleForTesting
+class BurstStats {
static const Set _ordinaryHistoricalRevisions = {
7,
9,
@@ -3091,7 +3375,7 @@ class _BurstStats {
};
final Map _dataPacketCountsByRevision = {};
- final Map _sequenceByRevision = {};
+ final Map _sequenceByRevision = {};
int _eventCount = 0;
int _consoleCount = 0;
int _unknownCount = 0;
@@ -3103,8 +3387,8 @@ class _BurstStats {
Map get dataPacketCountsByRevision =>
Map.unmodifiable(_dataPacketCountsByRevision);
- Map get sequenceByRevision =>
- Map.unmodifiable(_sequenceByRevision);
+ Map get sequenceByRevision =>
+ Map.unmodifiable(_sequenceByRevision);
int get eventCount => _eventCount;
int get consoleCount => _consoleCount;
int get unknownCount => _unknownCount;
@@ -3194,7 +3478,7 @@ class _BurstStats {
(_dataPacketCountsByRevision[revision] ?? 0) + 1;
final seq = _sequenceByRevision.putIfAbsent(
revision,
- () => _SequenceState(firstSequence: counter),
+ () => SequenceState(firstSequence: counter),
);
seq.observe(counter);
return;
@@ -3241,8 +3525,9 @@ class _BurstStats {
}
}
-class _SequenceState {
- _SequenceState({required this.firstSequence});
+@visibleForTesting
+class SequenceState {
+ SequenceState({required this.firstSequence});
final int firstSequence;
int? lastSequence;
diff --git a/lib/ble/ble_state.dart b/lib/ble/ble_state.dart
index b55660bf..039dc37c 100644
--- a/lib/ble/ble_state.dart
+++ b/lib/ble/ble_state.dart
@@ -297,6 +297,199 @@ class AckRetryPolicy {
}
}
+/// Why an in-flight HISTORY_END token may (or may not) be echoed back to the
+/// band. See [TrimAckPolicy].
+enum TrimAckVerdict {
+ /// Every precondition holds — echo the verbatim 8-byte token.
+ send,
+
+ /// The session that received this HISTORY_END is no longer the engine's
+ /// current session (the link dropped / was replaced while the handler was
+ /// parked mid-await). Writing now would put an OLD connection's token, with
+ /// a re-used sync seq, onto a BRAND NEW link — and a failed write would tear
+ /// down the healthy new session.
+ blockedStaleSession,
+
+ /// The burst this token terminates had its buffered records DISCARDED
+ /// without ever being committed (idle watchdog / abort-and-retry). Its
+ /// straggler terminal is still in flight; echoing it would trim flash we
+ /// deliberately threw away.
+ blockedDiscardedBurst,
+
+ /// The durable commit did NOT complete. The records are not in the database,
+ /// so the band must keep them.
+ blockedCommitFailed,
+}
+
+/// THE gate on the one irreversible act in the whole offload protocol: echoing
+/// a HISTORY_END continuation token, which is what tells the band it may trim
+/// that chunk out of its flash.
+///
+/// The safe-trim invariant is "every row is durable BEFORE the caller echoes
+/// the HISTORY_END trim token, or none is". Historically only the HAPPY path
+/// honoured it: the commit's exception was swallowed and the caller ACKed
+/// regardless, so a failed transaction (a real production OOM inside
+/// `commitSyncBatch` — see `lib/data/db.dart`) meant the buffer had already
+/// been cleared, the transaction had rolled back, the cursor had not advanced,
+/// and the band was then told to trim — those records existed nowhere,
+/// permanently and silently.
+///
+/// Pure and total: the engine supplies three observations, this decides. The
+/// engine must call it AGAIN after the commit await (the commit can take
+/// seconds on a large batch — long enough for the session to die under it).
+class TrimAckPolicy {
+ const TrimAckPolicy._();
+
+ /// [sessionCurrent] — the receiving session is still the engine's session
+ /// AND still connected.
+ /// [burstDiscarded] — this burst's open chunk was discarded un-committed.
+ /// [commitDurable] — the atomic commit completed (pass `true` when asking
+ /// the pre-commit question "should I even commit this
+ /// token?").
+ static TrimAckVerdict evaluate({
+ required bool sessionCurrent,
+ required bool burstDiscarded,
+ required bool commitDurable,
+ }) {
+ // Order is deliberate: a stale session must be refused before anything
+ // else touches the (new) link, and a poisoned burst must be refused before
+ // the commit result is even considered — its records are already gone.
+ if (!sessionCurrent) return TrimAckVerdict.blockedStaleSession;
+ if (burstDiscarded) return TrimAckVerdict.blockedDiscardedBurst;
+ if (!commitDurable) return TrimAckVerdict.blockedCommitFailed;
+ return TrimAckVerdict.send;
+ }
+}
+
+/// Per-burst poison latch for [TrimAckVerdict.blockedDiscardedBurst].
+///
+/// The idle watchdog abandons the open chunk (N buffered, never-committed
+/// records) on the contract "the band re-delivers them next offload", and the
+/// abort path then sends ABORT_HISTORICAL. But that burst's HISTORY_END is
+/// ALREADY in flight and arrives anyway — and nothing stopped the handler from
+/// committing an empty buffer and echoing the token verbatim, which trims
+/// exactly the records that were just thrown away. The token is unknown at
+/// discard time (it only arrives with the terminal), so the poison is keyed to
+/// the BURST, not the token: a discard poisons the open burst, and only a
+/// fresh HISTORY_START / re-arm clears it.
+class BurstTrimGuard {
+ bool _discarded = false;
+
+ /// Bursts poisoned since construction (diagnostics — a rising count means
+ /// the drain keeps stalling mid-burst).
+ int poisonedBursts = 0;
+
+ /// True while the open burst may NOT be trimmed.
+ bool get discarded => _discarded;
+
+ /// A fresh burst begins (HISTORY_START / re-arm) — nothing lost yet.
+ void beginBurst() => _discarded = false;
+
+ /// The open chunk was abandoned without a durable commit.
+ void discardOpenChunk() {
+ if (_discarded) return;
+ _discarded = true;
+ poisonedBursts++;
+ }
+}
+
+/// What a link-down must do to the session that just died.
+enum LinkDownAction {
+ /// Cancel every timer + subscription the session owns, then surface `idle`.
+ tearDownSession,
+
+ /// The event belongs to a session we already replaced — ignore it entirely.
+ ignoreStaleSession,
+}
+
+/// A dropped link used to only flip a flag and surface the phase; the actual
+/// teardown happened solely on the NEXT connect()/disconnect(). When
+/// [BondRefusalGiveUp] trips, the app pauses auto-reconnect and never calls
+/// disconnect() — so the dead session's five timers (heartbeat, keep-alive,
+/// periodic backfill, idle watchdog, historical retry) kept firing forever and
+/// its four `onValueReceived` subscriptions stayed registered, one more full
+/// set leaked per drop.
+class LinkDownPolicy {
+ const LinkDownPolicy._();
+
+ static LinkDownAction evaluate({required bool sessionIsCurrent}) =>
+ sessionIsCurrent
+ ? LinkDownAction.tearDownSession
+ : LinkDownAction.ignoreStaleSession;
+}
+
+/// Outcome of a process-wide single-owner band claim.
+enum BandClaimDecision {
+ /// Take the claim (nobody holds it, or the incumbent's claim is stale).
+ claim,
+
+ /// A live foreground owner exists and we are the background drainer —
+ /// don't touch the band this cycle.
+ yieldToOwner,
+
+ /// A live background owner exists and we are foreground — drop its link
+ /// (awaited) and then take the claim.
+ preemptThenClaim,
+}
+
+/// Pure arbitration for the process-wide single-owner guard.
+///
+/// The claim used to be tested for NON-NULLNESS only, and was taken BEFORE the
+/// link was up and released only by an explicit `disconnect()`. So a connect
+/// that threw left the claim pointing at an engine with no link, forever: a
+/// later iOS restore wake spun up a background drainer, saw a non-null owner,
+/// yielded, and reported "strap not reachable this cycle" for the rest of the
+/// process lifetime. [incumbentLive] is the fix — a claim held by an engine
+/// with no session is not a claim.
+class BandClaimPolicy {
+ const BandClaimPolicy._();
+
+ static BandClaimDecision decide({
+ required bool incumbentPresent,
+ required bool incumbentLive,
+ required bool isBackgroundDrainer,
+ }) {
+ if (!incumbentPresent) return BandClaimDecision.claim;
+ if (!incumbentLive) return BandClaimDecision.claim;
+ if (isBackgroundDrainer) return BandClaimDecision.yieldToOwner;
+ return BandClaimDecision.preemptThenClaim;
+ }
+}
+
+/// Where an inbound reassembled frame is processed.
+enum FrameRoute {
+ /// The single serialized offload queue (`_offloadFrames`), so history
+ /// records and their terminals are handled strictly in arrival order by ONE
+ /// loop.
+ serializedQueue,
+
+ /// Handled inline (command responses, events, live high-rate frames).
+ immediate,
+}
+
+/// Pure routing decision for [FrameRoute].
+///
+/// Metadata (HISTORY_START / HISTORY_END / HISTORY_COMPLETE) used to reach the
+/// serialized queue only when it arrived on the `data` characteristic;
+/// metadata reassembled on `cmd_from`/`events` was fired unawaited on the
+/// immediate path instead — the one route that could run a HISTORY_END handler
+/// CONCURRENTLY with the queued drain, i.e. two handlers racing on the same
+/// drain controller (one snapshots an empty buffer and can ACK before the
+/// other's commit is durable). Metadata now always takes the queue.
+class FrameRoutePolicy {
+ const FrameRoutePolicy._();
+
+ static FrameRoute route({
+ required bool isMetadata,
+ required bool isHistorical,
+ required bool isDataRole,
+ }) {
+ if (isMetadata) return FrameRoute.serializedQueue;
+ if (isHistorical && isDataRole) return FrameRoute.serializedQueue;
+ return FrameRoute.immediate;
+ }
+}
+
/// Tracks ACK-write failures per historical-batch token ACROSS RECONNECTS —
/// a chunk whose ACK keeps failing for the SAME token (the "Groundhog Day"
/// re-flood signature: the band never trims, so it re-sends the identical
diff --git a/lib/cloud/cloud_import.dart b/lib/cloud/cloud_import.dart
index e0aa1eef..c6fd7ac6 100644
--- a/lib/cloud/cloud_import.dart
+++ b/lib/cloud/cloud_import.dart
@@ -17,6 +17,8 @@
import 'dart:convert';
+import 'package:flutter/foundation.dart' show visibleForTesting;
+
import '../compute/derivation_engine.dart' show kAlgoVersion;
import '../data/db.dart';
import 'backend_client.dart';
@@ -79,8 +81,7 @@ class CloudImporter {
var sessCount = 0;
for (final w in sessions) {
if (w is! Map) continue;
- await _writeSession(w.cast());
- sessCount++;
+ if (await _writeSession(w.cast())) sessCount++;
}
return CloudImportResult(dayCount, sessCount, _mapProfile(profileRaw));
@@ -250,14 +251,26 @@ class CloudImporter {
);
}
- static Future _writeSession(Map w) async {
+ /// Test seam for [_writeSession] — the malformed-row skip is a data-integrity
+ /// invariant, and `run()` needs a whole authenticated BackendClient to reach.
+ @visibleForTesting
+ static Future debugWriteSession(Map w) =>
+ _writeSession(w);
+
+ /// Returns true when a session row was actually written.
+ static Future _writeSession(Map w) async {
num? n(Object? v) => v is num ? v : null;
final start = n(w['start_ts'])?.toInt();
final end = n(w['end_ts'])?.toInt();
+ // A session with no usable start is not a workout — writing it as epoch 0
+ // filed a phantom workout on 1970-01-01 that then showed up in every query
+ // keyed on start_ts. Skip the row (same contract as WhoopImporter's
+ // _writeWorkout).
+ if (start == null) return false;
final zones = w['zones'];
await LocalDb.putSession({
'id': (w['id'] ?? 'cloud_$start').toString(),
- 'start_ts': start ?? 0,
+ 'start_ts': start,
'end_ts': end,
'type': (w['type'] ?? w['detected_type'] ?? 'other').toString(),
'status': (w['status'] ?? 'done').toString(),
@@ -266,10 +279,11 @@ class CloudImporter {
'strain': n(w['strain'])?.toDouble(),
'max_hr': n(w['max_hr'])?.toInt(),
'duration_min':
- (start != null && end != null) ? ((end - start) / 60).round() : null,
+ end != null ? ((end - start) / 60).round() : null,
'zone_min_json': zones == null ? null : jsonEncode(zones),
'created_at': DateTime.now().millisecondsSinceEpoch,
});
+ return true;
}
static String _ymd(DateTime d) => '${d.year.toString().padLeft(4, '0')}-'
diff --git a/lib/coach/coach_db.dart b/lib/coach/coach_db.dart
index 941f96d1..1d999bfb 100644
--- a/lib/coach/coach_db.dart
+++ b/lib/coach/coach_db.dart
@@ -4,15 +4,38 @@
// handle, over DERIVED-only views (v_metric/v_daily/v_series/v_hypnogram/
// v_sessions/v_baselines/v_insights — created by LocalDb._ensureCoachViews).
//
-// Two layers of safety, both fail-closed:
-// 1. guardAndPrepare() — a static validator: SELECT/WITH only, single
-// statement, no comments, no DML/DDL/PRAGMA, every FROM/JOIN target must be
-// an allow-listed view, no base/raw table name anywhere, auto-LIMIT.
-// 2. A read-only Database handle (openDatabase readOnly:true) — even if the
-// guard were bypassed, SQLite physically refuses writes/DDL, and raw byte
-// tables are blocked by the guard's identifier allow-list.
+// THREE layers of safety, all fail-closed:
+//
+// 1. guardAndPrepare() — a static validator built on a real (small) SQL
+// lexer + table-reference parser, NOT a single regex. It is an
+// ALLOW-LIST: SELECT/WITH only, single statement, no comments, no
+// DML/DDL/PRAGMA, and EVERY table position (including the extra members
+// of a comma-separated implicit cross-join, which the old FROM/JOIN
+// regex never even looked at) must resolve to an allow-listed `v_*` view
+// or a CTE declared in the same statement. Anything else — a base table,
+// a schema-qualified name, sqlite_master/sqlite_schema, a table-valued
+// function, a subquery in FROM — is rejected because it is not on the
+// list, so a table added to the schema tomorrow is closed by default
+// instead of open by default.
+//
+// 2. _assertAllowedBtrees() — a STRUCTURAL check that does not trust the
+// parser at all. sqflite exposes no sqlite3_set_authorizer() hook and a
+// read-only handle still sees every table in the file, so we ask SQLite
+// itself which btrees a statement would open: `EXPLAIN ` lists the
+// OpenRead/ReopenIdx opcodes with the ROOT PAGE of each btree, and
+// sqlite_master maps root pages back to the owning table. The allowed
+// root-page set is derived (once, at runtime) by EXPLAINing the allowed
+// views themselves, so it is exactly "the base tables the coach views
+// are made of" — every other btree in the database, including
+// workout_route (raw GPS) and sqlite_master, is unreachable no matter
+// what the text-level parser thinks. This is the authorizer we can't
+// install, implemented with the compiler's own answer.
+//
+// 3. A read-only Database handle (openDatabase readOnly:true) — even if
+// both layers above were bypassed, SQLite physically refuses writes/DDL.
import 'dart:convert';
+import 'package:flutter/foundation.dart' show visibleForTesting;
import 'package:path/path.dart' as p;
import 'package:sqflite/sqflite.dart';
@@ -30,7 +53,9 @@ class SqlGuardError implements Exception {
class CoachDb {
CoachDb._();
- /// The only tables the coach may read — derived, never raw.
+ /// The ONLY relations the coach may name in a table position — derived,
+ /// never raw. This is the whole allow-list: a CTE declared inside the same
+ /// statement is the only other thing that may appear after FROM/JOIN.
static const Set allowedViews = {
'v_metric',
'v_daily',
@@ -47,20 +72,42 @@ class CoachDb {
'insert', 'update', 'delete', 'drop', 'alter', 'create', 'replace',
'truncate', 'attach', 'detach', 'pragma', 'vacuum', 'reindex', 'analyze',
'begin', 'commit', 'rollback', 'savepoint', 'grant', 'revoke', 'trigger',
- 'into', 'load_extension',
+ 'into', 'load_extension', 'explain',
+ };
+
+ /// Real on-disk relations. This is NOT the security boundary any more (the
+ /// allow-list above is) — it is a redundant net whose only jobs are to give
+ /// the model a clearer rejection reason and to stop a CTE from SHADOWING a
+ /// real table name. Anything missing from it is still rejected by the
+ /// allow-list; nothing here is ever reachable.
+ static const Set reservedTableNames = {
+ // derived / user tables
+ 'day_result', 'metric_series', 'baselines', 'derived_day', 'sessions',
+ 'notifications', 'journal', 'cycle_log', 'cycle_symptom', 'notif_fired',
+ 'sleep_override', 'sleep_session_candidates', 'wake_day_features',
+ 'workout_suggestions', 'workout_route', 'live_coverage',
+ // raw / decoded substrate
+ 'raw_records', 'raw_archive', 'decoded_onehz', 'decoded_rr', 'samples',
+ 'events', 'band_events', 'band_battery',
+ // sync / compute bookkeeping
+ 'sync_ledger', 'sync_quarantine', 'sync_cursor', 'sync_ledger_legacy',
+ 'sync_quarantine_legacy', 'sync_cursor_legacy', 'compute_jobs',
+ 'compute_freshness',
+ // sqlite internals (the sqlite_* prefix is rejected wholesale as well)
+ 'sqlite_master', 'sqlite_schema', 'sqlite_temp_master',
+ 'sqlite_temp_schema', 'sqlite_sequence', 'sqlite_stat1', 'dbstat',
};
- // Base/raw table names that must NOT appear anywhere (force the views).
- static const Set _denyTables = {
- 'raw_records', 'decoded_onehz', 'decoded_rr', 'samples', 'sqlite_master',
- 'day_result', 'metric_series', 'baselines', 'sessions', 'notifications',
- 'derived_day', 'events', 'band_events', 'band_battery', 'live_coverage',
- 'sync_ledger', 'sync_quarantine', 'sync_cursor', 'compute_jobs',
- 'compute_freshness', 'journal', 'cycle_log', 'cycle_symptom',
- 'primitive_artifacts', 'wake_day_features', 'workout_suggestions',
+ /// Tokens that can follow a table reference without being an alias.
+ static const Set _clauseKeywords = {
+ 'where', 'group', 'order', 'limit', 'offset', 'having', 'window', 'union',
+ 'intersect', 'except', 'on', 'using', 'join', 'inner', 'left', 'right',
+ 'full', 'cross', 'natural', 'and', 'or', 'as', 'values', 'returning',
+ 'select', 'from', 'with',
};
static Database? _ro;
+ static Set? _allowedRoots;
/// Open (and cache) a read-only handle. Ensures the RW handle first so the
/// views exist (a read-only handle can't CREATE VIEW).
@@ -75,69 +122,344 @@ class CoachDb {
static Future close() async {
await _ro?.close();
_ro = null;
+ _allowedRoots = null;
}
+ // ── layer 1: lexer + table-reference parser ────────────────────────────────
+
/// Validate + normalize an LLM-supplied SELECT. Throws [SqlGuardError] on any
/// rejection. Returns the (possibly LIMIT-appended) query to execute.
static String guardAndPrepare(String raw, {int rowCap = 200}) {
var s = raw.trim();
if (s.isEmpty) throw SqlGuardError('Empty query.');
+ if (s.codeUnits.contains(0)) throw SqlGuardError('Illegal character.');
+
+ // (a) Strip one optional trailing ';' before anything else, so the
+ // single-statement check below sees only genuine inner separators.
+ var body = s.endsWith(';') ? s.substring(0, s.length - 1).trim() : s;
+ if (body.isEmpty) throw SqlGuardError('Empty query.');
+
+ // (b) Mask string literals to a neutral numeric token. Quoted IDENTIFIERS
+ // ("x", `x`, [x]) are rejected outright — the coach never needs them,
+ // and SQLite's double-quote fallback (a bare "foo" silently becomes a
+ // string when it doesn't resolve to a column) is a parser trap.
+ final masked = _maskStrings(body);
- // (a) No comments (they can hide payloads).
- if (s.contains('--') || s.contains('/*') || s.contains('*/')) {
+ // (c) No comments (they can hide payloads) — checked on the MASKED text so
+ // an inline '--' inside a legitimate string value isn't a false hit.
+ if (masked.contains('--') ||
+ masked.contains('/*') ||
+ masked.contains('*/')) {
throw SqlGuardError('Comments are not allowed.');
}
- // (b) Single statement — strip one optional trailing ';', reject any inner.
- var body = s.endsWith(';') ? s.substring(0, s.length - 1).trim() : s;
- if (body.contains(';')) {
+ // (d) Single statement.
+ if (masked.contains(';')) {
throw SqlGuardError('Only one statement is allowed.');
}
- // (c) Must start with SELECT or WITH.
- final lower = body.toLowerCase();
+ // (e) Must start with SELECT or WITH.
+ final lower = masked.toLowerCase();
if (!(lower.startsWith('select') || lower.startsWith('with'))) {
throw SqlGuardError('Query must start with SELECT or WITH.');
}
- // (d) Tokenize; reject any banned keyword + collect identifiers.
- final tokens = RegExp(r'[A-Za-z_][A-Za-z0-9_]*')
- .allMatches(lower)
- .map((m) => m.group(0)!)
- .toList();
- for (final t in tokens) {
- if (_banned.contains(t)) throw SqlGuardError('Disallowed keyword: $t');
- if (_denyTables.contains(t)) throw SqlGuardError('Table not allowed: $t');
- }
- // (e) CTE names declared via WITH … AS ( become valid FROM targets.
- final cte = {};
- for (final m in RegExp(r'(?:with|,)\s+([A-Za-z_][A-Za-z0-9_]*)\s+as\s*\(',
- caseSensitive: false)
- .allMatches(lower)) {
- cte.add(m.group(1)!);
- }
- // (f) Every FROM/JOIN target must be an allowed view (or a CTE name).
- final refRe = RegExp(r'\b(?:from|join)\s+("?)([A-Za-z_][A-Za-z0-9_.]*)\1',
- caseSensitive: false);
- final refs = [];
- for (final m in refRe.allMatches(lower)) {
- final id = m.group(2)!;
- if (id.contains('.')) {
- throw SqlGuardError('Schema-qualified names are not allowed: $id');
+
+ final toks = _lex(lower);
+
+ // (f) Banned keywords + reserved relation names, anywhere.
+ for (final t in toks) {
+ if (!t.ident) continue;
+ if (_banned.contains(t.text)) {
+ throw SqlGuardError('Disallowed keyword: ${t.text}');
+ }
+ if (t.text.startsWith('sqlite_') || t.text.startsWith('pragma_')) {
+ throw SqlGuardError('SQLite internals are not queryable: ${t.text}');
+ }
+ if (reservedTableNames.contains(t.text)) {
+ throw SqlGuardError('Table not allowed: ${t.text}. '
+ 'Allowed views: ${allowedViews.join(', ')}.');
}
- refs.add(id);
}
- if (refs.isEmpty) throw SqlGuardError('No table reference found.');
- for (final r in refs) {
- if (!allowedViews.contains(r) && !cte.contains(r)) {
- throw SqlGuardError(
- 'Table "$r" is not queryable. Allowed views: ${allowedViews.join(', ')}.');
+
+ // (g) CTE names declared via ` AS (` become valid table targets.
+ // A CTE may not shadow an allowed view (that would silently redefine
+ // the coach's data surface) — shadowing a REAL table is already
+ // impossible because (f) rejects every reserved name outright.
+ final cte = _cteNames(toks);
+ for (final c in cte) {
+ if (allowedViews.contains(c)) {
+ throw SqlGuardError('A CTE may not shadow the view "$c".');
}
}
- // (g) Auto-append a LIMIT to protect context.
+
+ // (h) THE allow-list gate: parse every table position and require it to be
+ // an allowed view or a declared CTE. Comma-separated cross-joins,
+ // subqueries in FROM, table-valued functions and schema-qualified
+ // names are all handled here (the old single-regex version only ever
+ // saw the identifier directly after FROM/JOIN).
+ final refs = _tableRefs(toks, allowed: {...allowedViews, ...cte});
+ if (refs.isEmpty) throw SqlGuardError('No table reference found.');
+
+ // (i) Auto-append a LIMIT to protect context.
if (!RegExp(r'\blimit\b', caseSensitive: false).hasMatch(lower)) {
body = '$body LIMIT $rowCap';
}
return body;
}
+ /// Replace every single-quoted string literal with the neutral token `0`.
+ /// Rejects unterminated literals and every quoted-identifier form.
+ static String _maskStrings(String src) {
+ final out = StringBuffer();
+ var i = 0;
+ while (i < src.length) {
+ final c = src[i];
+ if (c == "'") {
+ i++;
+ var closed = false;
+ while (i < src.length) {
+ if (src[i] == "'") {
+ if (i + 1 < src.length && src[i + 1] == "'") {
+ i += 2;
+ continue;
+ }
+ i++;
+ closed = true;
+ break;
+ }
+ i++;
+ }
+ if (!closed) throw SqlGuardError('Unterminated string literal.');
+ out.write('0');
+ continue;
+ }
+ if (c == '"' || c == '`' || c == '[' || c == ']') {
+ throw SqlGuardError(
+ 'Quoted identifiers are not allowed — use bare view names.');
+ }
+ out.write(c);
+ i++;
+ }
+ return out.toString();
+ }
+
+ static List<_Tok> _lex(String s) {
+ final out = <_Tok>[];
+ var i = 0;
+ bool identStart(String c) =>
+ (c.codeUnitAt(0) >= 97 && c.codeUnitAt(0) <= 122) || c == '_';
+ bool identPart(String c) =>
+ identStart(c) || (c.codeUnitAt(0) >= 48 && c.codeUnitAt(0) <= 57);
+ bool digit(String c) => c.codeUnitAt(0) >= 48 && c.codeUnitAt(0) <= 57;
+ while (i < s.length) {
+ final c = s[i];
+ if (c.trim().isEmpty) {
+ i++;
+ continue;
+ }
+ if (identStart(c)) {
+ final st = i;
+ while (i < s.length && identPart(s[i])) {
+ i++;
+ }
+ out.add(_Tok(s.substring(st, i), true));
+ continue;
+ }
+ if (digit(c)) {
+ final st = i;
+ while (i < s.length && (digit(s[i]) || s[i] == '.')) {
+ i++;
+ }
+ out.add(_Tok(s.substring(st, i), false));
+ continue;
+ }
+ out.add(_Tok(c, false));
+ i++;
+ }
+ return out;
+ }
+
+ /// CTE names: an identifier preceded by `with`/`recursive`/`,` and followed
+ /// (past an optional parenthesised column list) by `as (`.
+ static Set _cteNames(List<_Tok> toks) {
+ final names = {};
+ for (var i = 1; i < toks.length; i++) {
+ final prev = toks[i - 1];
+ final t = toks[i];
+ if (!t.ident) continue;
+ final opensCte = (prev.ident && (prev.text == 'with' || prev.text == 'recursive')) ||
+ (!prev.ident && prev.text == ',');
+ if (!opensCte) continue;
+ var j = i + 1;
+ if (j < toks.length && !toks[j].ident && toks[j].text == '(') {
+ j = _skipParens(toks, j);
+ }
+ if (j < toks.length && toks[j].ident && toks[j].text == 'as') {
+ var k = j + 1;
+ // AS [NOT] MATERIALIZED (
+ while (k < toks.length &&
+ toks[k].ident &&
+ (toks[k].text == 'not' || toks[k].text == 'materialized')) {
+ k++;
+ }
+ if (k < toks.length && !toks[k].ident && toks[k].text == '(') {
+ names.add(t.text);
+ }
+ }
+ }
+ return names;
+ }
+
+ /// Index just past the balanced ')' that matches the '(' at [open].
+ static int _skipParens(List<_Tok> toks, int open) {
+ var depth = 0;
+ for (var i = open; i < toks.length; i++) {
+ if (toks[i].ident) continue;
+ if (toks[i].text == '(') depth++;
+ if (toks[i].text == ')') {
+ depth--;
+ if (depth == 0) return i + 1;
+ }
+ }
+ return toks.length;
+ }
+
+ /// Every table position in the statement, validated against [allowed].
+ /// Walks the FULL table-reference list after each FROM/JOIN — including the
+ /// comma-separated members of an implicit cross-join.
+ static List _tableRefs(List<_Tok> toks, {required Set allowed}) {
+ final refs = [];
+ for (var i = 0; i < toks.length; i++) {
+ final t = toks[i];
+ if (!t.ident || (t.text != 'from' && t.text != 'join')) continue;
+ var j = i + 1;
+ while (true) {
+ if (j >= toks.length) {
+ throw SqlGuardError('Malformed FROM clause.');
+ }
+ final n = toks[j];
+ if (!n.ident) {
+ if (n.text == '(') {
+ throw SqlGuardError(
+ 'Subqueries in FROM are not allowed — query a view directly.');
+ }
+ throw SqlGuardError('Expected a view name after FROM/JOIN.');
+ }
+ final name = n.text;
+ j++;
+ // Schema-qualified (main.x / temp.x) or a table-valued function.
+ if (j < toks.length && !toks[j].ident && toks[j].text == '.') {
+ throw SqlGuardError('Schema-qualified names are not allowed: $name');
+ }
+ if (j < toks.length && !toks[j].ident && toks[j].text == '(') {
+ throw SqlGuardError('Functions are not allowed in FROM: $name');
+ }
+ if (!allowed.contains(name)) {
+ throw SqlGuardError(
+ 'Table "$name" is not queryable. Allowed views: '
+ '${allowedViews.join(', ')}.');
+ }
+ refs.add(name);
+ // Optional alias: `AS x` or a bare `x`.
+ if (j < toks.length && toks[j].ident && toks[j].text == 'as') {
+ j++;
+ if (j >= toks.length || !toks[j].ident) {
+ throw SqlGuardError('Malformed alias after "$name".');
+ }
+ j++;
+ } else if (j < toks.length &&
+ toks[j].ident &&
+ !_clauseKeywords.contains(toks[j].text)) {
+ j++;
+ }
+ // Another member of the same (comma-separated) table list?
+ if (j < toks.length && !toks[j].ident && toks[j].text == ',') {
+ j++;
+ continue;
+ }
+ break;
+ }
+ i = j - 1;
+ }
+ return refs;
+ }
+
+ // ── layer 2: structural btree gate (SQLite's own answer) ───────────────────
+
+ /// Root pages of every btree the ALLOWED VIEWS themselves read. Computed
+ /// once per read-only handle by EXPLAINing `SELECT * FROM `; SQLite
+ /// expands the view and reports exactly which base tables/indexes it opens.
+ static Future> _allowedRootPages(Database db) async {
+ final cached = _allowedRoots;
+ if (cached != null) return cached;
+ final roots = {};
+ for (final v in allowedViews) {
+ try {
+ roots.addAll(await _btreeRoots(db, 'SELECT * FROM $v'));
+ } catch (_) {/* a view that can't compile contributes nothing */}
+ }
+ // Indexes on an allowed base table are allowed too (the planner may pick
+ // one for a WHERE the plain scan above never used).
+ final schema = await db.rawQuery(
+ 'SELECT type, name, tbl_name, rootpage FROM sqlite_master');
+ final tableOfRoot = {};
+ for (final r in schema) {
+ final rp = (r['rootpage'] as num?)?.toInt();
+ final tbl = r['tbl_name']?.toString();
+ if (rp != null && rp > 0 && tbl != null) tableOfRoot[rp] = tbl;
+ }
+ final allowedTables = {
+ for (final rp in roots)
+ if (tableOfRoot[rp] != null) tableOfRoot[rp]!,
+ };
+ for (final e in tableOfRoot.entries) {
+ if (allowedTables.contains(e.value)) roots.add(e.key);
+ }
+ _allowedRoots = roots;
+ return roots;
+ }
+
+ /// Root pages of every btree a prepared [sql] opens, per SQLite's own
+ /// bytecode. Only cursor-opening opcodes carry a root page in P2 —
+ /// OpenEphemeral/SorterOpen/OpenPseudo reuse P2 for a column count and are
+ /// deliberately ignored.
+ static Future> _btreeRoots(Database db, String sql) async {
+ const opening = {'OpenRead', 'OpenWrite', 'ReopenIdx'};
+ final rows = await db.rawQuery('EXPLAIN $sql');
+ final out = {};
+ for (final r in rows) {
+ final op = r['opcode']?.toString();
+ if (op == null || !opening.contains(op)) continue;
+ final p2 = (r['p2'] as num?)?.toInt();
+ if (p2 != null && p2 > 0) out.add(p2);
+ }
+ return out;
+ }
+
+ /// Fail-closed structural gate: reject unless every btree the statement
+ /// opens belongs to a base table one of the allowed views is built from.
+ static Future _assertAllowedBtrees(Database db, String sql) async {
+ final allowed = await _allowedRootPages(db);
+ if (allowed.isEmpty) {
+ throw SqlGuardError('Query surface unavailable.');
+ }
+ final Set used;
+ try {
+ used = await _btreeRoots(db, sql);
+ } catch (e) {
+ throw SqlGuardError('Query could not be compiled: $e');
+ }
+ for (final r in used) {
+ if (!allowed.contains(r)) {
+ throw SqlGuardError('Query reaches storage outside the coach views. '
+ 'Allowed views: ${allowedViews.join(', ')}.');
+ }
+ }
+ }
+
+ /// Test seam for layer 2 on its own (layer 1 normally rejects these first).
+ @visibleForTesting
+ static Future debugAssertAllowedBtrees(String sql) async =>
+ _assertAllowedBtrees(await _readonly(), sql);
+
/// Run an LLM SELECT and return compact JSON for the tool result. On a guard
/// rejection, returns the reason (so the model fixes its query) — never throws.
static Future runCoachSql(String llmSql, {int rowCap = 200}) async {
@@ -149,6 +471,7 @@ class CoachDb {
}
try {
final db = await _readonly();
+ await _assertAllowedBtrees(db, sql);
final rows = await db.rawQuery(sql);
final shown = rows.take(rowCap).toList();
final out = {
@@ -164,8 +487,16 @@ class CoachDb {
var s = jsonEncode(out);
if (s.length > 16000) s = '${s.substring(0, 16000)}…(truncated)';
return s;
+ } on SqlGuardError catch (e) {
+ return jsonEncode({'error': e.reason});
} catch (e) {
return jsonEncode({'error': 'Query failed: $e'});
}
}
}
+
+class _Tok {
+ final String text;
+ final bool ident;
+ const _Tok(this.text, this.ident);
+}
diff --git a/lib/coach/coach_engine.dart b/lib/coach/coach_engine.dart
index 3cfade04..40a2d529 100644
--- a/lib/coach/coach_engine.dart
+++ b/lib/coach/coach_engine.dart
@@ -14,6 +14,7 @@
import 'dart:convert';
import 'dart:io';
import 'dart:math';
+import 'package:flutter/foundation.dart';
import 'package:http/http.dart' as http;
import 'package:path_provider/path_provider.dart';
@@ -170,6 +171,15 @@ class CoachEngine {
// Serializable display transcript (text bubbles + charts) shown in the UI.
final List transcript = [];
+ /// Test seams for the history-trimming invariant. `_history` is private and
+ /// the trim only runs deep inside the tool-calling loop, so there is no other
+ /// way to pin the orphaned-`tool` edge case without a live provider.
+ @visibleForTesting
+ List> get debugHistory => _history;
+
+ @visibleForTesting
+ void debugTrimHistory() => _trimHistory();
+
// Current session identity (sessions are persisted per-user, many per user).
String _sessionId = '';
String _title = '';
@@ -191,6 +201,60 @@ class CoachEngine {
CoachEngine({required this.config, required this.api, this.storageKey = 'anon'});
+ // ── prompt size ceilings ────────────────────────────────────────────────────
+ //
+ // The coach's tools read the on-device health database and every result is
+ // resent, verbatim, on EVERY subsequent turn. Without a ceiling a model that
+ // keeps widening its queries would eventually serialize the whole database
+ // into a request bound for a third-party endpoint. Three bounds, all
+ // independent of the provider's own context limit:
+ // • per tool result — one query can't dominate the window;
+ // • rolling history — the resent conversation is bounded in BYTES, not
+ // just in message count (60 × 16 KB was ~1 MB);
+ // • per request — a hard fail-closed ceiling in [postChat].
+
+ /// Max characters of any single tool result kept in the resent history.
+ static const int kMaxToolResultChars = 16000;
+
+ /// Max characters of running history resent on each turn.
+ static const int kMaxHistoryChars = 120000;
+
+ /// Hard ceiling on one serialized provider request body.
+ static const int kMaxRequestBytes = 400 * 1024;
+
+ static String _clipToolResult(String s) => s.length <= kMaxToolResultChars
+ ? s
+ : '${s.substring(0, kMaxToolResultChars)}…(truncated — narrow the query)';
+
+ int _historyChars() {
+ var n = 0;
+ for (final m in _history) {
+ n += jsonEncode(m).length;
+ }
+ return n;
+ }
+
+ /// Bound the resent history in bytes, dropping WHOLE turns from the oldest
+ /// end so a `tool` message never outlives the assistant turn whose
+ /// `tool_calls` it answers (providers 400 on an orphaned tool message).
+ void _trimHistory() {
+ while (_historyChars() > kMaxHistoryChars && _history.length > 1) {
+ _history.removeAt(0);
+ while (_history.length > 1 && _history.first['role'] != 'user') {
+ _history.removeAt(0);
+ }
+ }
+ // Both loops stop at `length > 1`, so one turn larger than the whole budget
+ // can strand a lone `tool` at the head: [assistant(tool_calls), tool] drops
+ // the assistant and then has nothing left to pair with. That orphan is the
+ // exact shape this method exists to prevent, and providers 400 on it, so
+ // enforce the invariant unconditionally rather than as a side effect of the
+ // loop bounds.
+ while (_history.isNotEmpty && _history.first['role'] == 'tool') {
+ _history.removeAt(0);
+ }
+ }
+
void reset() { _history.clear(); transcript.clear(); }
bool get hasHistory => _history.isNotEmpty;
@@ -392,6 +456,7 @@ class CoachEngine {
const maxIters = 10;
for (var i = 0; i < maxIters; i++) {
onStatus(_shenanigans[_rand.nextInt(_shenanigans.length)]);
+ _trimHistory();
final messages = >[
{'role': 'system', 'content': '$kCoachSystemPrompt\n\nToday is ${_today()} (device-local date; all day-keyed data uses these local dates).'},
..._history,
@@ -429,7 +494,12 @@ class CoachEngine {
onStatus(_statusFor(name, args));
final result = await _runTool(name, args, onItem: emit, confirm: confirm);
- _history.add({'role': 'tool', 'tool_call_id': id, 'name': name, 'content': result});
+ _history.add({
+ 'role': 'tool',
+ 'tool_call_id': id,
+ 'name': name,
+ 'content': _clipToolResult(result),
+ });
}
}
emit(CoachItem.assistant('I dug through several steps but couldn’t wrap that up — try narrowing the question.'));
@@ -533,6 +603,17 @@ class CoachEngine {
..remove('top_p')
..remove('top_k');
}
+ // FAIL-CLOSED size ceiling. Nothing leaves the device until this passes —
+ // the request is never truncated and silently sent, it is refused, so a
+ // runaway tool loop cannot ship the health database to a third party.
+ final payload = jsonEncode(body);
+ if (payload.length > kMaxRequestBytes) {
+ throw CoachException(
+ 'That request grew to ${payload.length ~/ 1024} KB, over the '
+ '${kMaxRequestBytes ~/ 1024} KB safety limit for data leaving this '
+ 'device. Start a new chat or ask a narrower question (aggregate with '
+ 'AVG/MIN/MAX/COUNT instead of selecting every row).');
+ }
try {
final resp = await c
.post(
@@ -541,17 +622,41 @@ class CoachEngine {
'Authorization': 'Bearer ${config.apiKey}',
'content-type': 'application/json',
},
- body: jsonEncode(body),
+ body: payload,
)
.timeout(const Duration(seconds: 120));
if (resp.statusCode != 200) {
throw CoachException(
'Provider error (${resp.statusCode}): ${_briefErr(resp.body)}');
}
- final j = jsonDecode(utf8.decode(resp.bodyBytes));
+ final Object? j;
+ try {
+ j = jsonDecode(utf8.decode(resp.bodyBytes));
+ } catch (_) {
+ throw CoachException(
+ 'Provider returned a non-JSON response. Check the API base URL — '
+ 'it must point at an OpenAI-compatible /chat/completions endpoint.');
+ }
+ if (j is! Map) throw CoachException('Unexpected response from provider.');
final choices = (j['choices'] as List?) ?? const [];
if (choices.isEmpty) throw CoachException('Empty response from provider.');
- return (choices.first as Map)['message'] as Map;
+ // Every shape below is a REAL thing OpenAI-compatible proxies return:
+ // a streaming chunk (`delta` instead of `message`), the legacy
+ // completions shape (`text`), or a bare string. Reaching for
+ // `choices.first['message'] as Map` blind surfaced a raw
+ // TypeError ("type 'Null' is not a subtype of type 'Map'") instead of the documented CoachException, so the UI showed
+ // a Dart type name to the user rather than an actionable message.
+ final first = choices.first;
+ if (first is! Map) throw CoachException('Unexpected response from provider.');
+ final msg = first['message'] ?? first['delta'];
+ if (msg is Map) return msg.cast();
+ final text = first['text'];
+ if (text is String) return {'content': text};
+ throw CoachException(
+ 'Provider returned an unsupported response shape (no message/delta). '
+ 'Streaming-only endpoints are not supported — use a standard '
+ 'OpenAI-compatible /chat/completions endpoint.');
} finally {
if (client == null) c.close();
}
diff --git a/lib/coach/coach_prompt.dart b/lib/coach/coach_prompt.dart
index 01940317..148413d7 100644
--- a/lib/coach/coach_prompt.dart
+++ b/lib/coach/coach_prompt.dart
@@ -57,10 +57,15 @@ the user's DERIVED data (raw signals are intentionally unavailable). Tables are
- v_baselines(key, value, mean, z, delta, ratio, n, updated_at) — rolling personal baselines.
- v_insights(id, kind, title, body, date, created_at, read) — the local insight/alert feed.
-Rules: SELECT only, derived views only (no raw/base tables), dates are 'YYYY-MM-DD', timestamps
-are epoch SECONDS, irregular_flag/irregular_rhythm_flag are 1/0. Prefer aggregates
-(AVG/MIN/MAX/COUNT, GROUP BY) over selecting many rows; results cap at 200 rows. If a query is
-rejected, read the error and fix it. Examples:
+Rules: SELECT (or WITH … SELECT) only, ONE statement, no comments. Every table position — after
+FROM, after JOIN, and every extra member of a comma-separated list — must be one of the v_* views
+above or a CTE you declared in the same statement; nothing else is queryable, including base
+tables, sqlite_master, schema-qualified names ("main.v_daily"), quoted identifiers, table
+functions, and subqueries in FROM (put the subquery in a WITH instead). Subqueries in
+SELECT/WHERE over the views are fine. Dates are 'YYYY-MM-DD', timestamps are epoch SECONDS,
+irregular_flag/irregular_rhythm_flag are 1/0. Prefer aggregates (AVG/MIN/MAX/COUNT, GROUP BY)
+over selecting many rows; results cap at 200 rows. If a query is rejected, read the error and fix
+it. Examples:
SELECT date, resting_hr, hrv, readiness FROM v_daily ORDER BY date DESC LIMIT 30
SELECT AVG(strain) FROM v_daily WHERE date >= '2026-06-01'
SELECT t, v FROM v_series WHERE date='2026-06-29' AND series='hr_curve' ORDER BY t
diff --git a/lib/compute/crossday_pipeline.dart b/lib/compute/crossday_pipeline.dart
index ba4e036a..c4b08c93 100644
--- a/lib/compute/crossday_pipeline.dart
+++ b/lib/compute/crossday_pipeline.dart
@@ -14,6 +14,8 @@
// which is a deterministic function of the input string. Safe for Isolate.run
// and directly unit-testable.
+import 'dart:math' as math;
+
import 'package:openstrap_analytics/onehz.dart' as ana;
/// Build the cross-day analytics bundle from a time-ordered (OLDEST FIRST) list
@@ -64,14 +66,18 @@ Map buildCrossDayBundle(
final anomaly = ana.multivariateAnomaly(dates, feats);
// ── CTL/ATL/TSB training load from the daily-TRIMP series ──────────────────
- // Only days that actually carry a TRIMP (time-ordered) contribute; a missing
- // TRIMP is NOT a 0-load impulse here — we simply omit it (the user may not
- // have worn / trained that day, and the EWMA in the package treats the gaps
- // it does see as decay over the series it receives).
- final dailyTrimp = [
- for (final d in days)
- if (d['trimp'] != null) (_numOrNull(d['trimp']) ?? 0.0),
- ];
+ // `ctlAtlTsb` is an EWMA that treats its input as ONE SAMPLE PER CALENDAR DAY
+ // (λ = 1 − e^(−1/τ) per element) and documents "a missing day contributes a
+ // 0-load impulse (rest day) — the EWMA decays". Filtering to only the days
+ // that carry a TRIMP handed it a COMPRESSED calendar instead: a user who
+ // trains sporadically (say 10 loaded days across 90) got 10 consecutive
+ // "days" of load with no decay between them, so fitness/fatigue never fell
+ // across the gaps and TSB was systematically wrong. Build a DENSE per-day
+ // series over the observed date span instead — a day with no TRIMP, and a
+ // calendar day with no derived row at all, are both 0-load impulses.
+ final dailyTrimp = _denseDailyTrimp(dates, [
+ for (final d in days) _numOrNull(d['trimp']),
+ ]);
final load = ana.ctlAtlTsb(dailyTrimp);
// ── skin-temp illness flag (Smarr, cycle-aware) ────────────────────────────
@@ -275,11 +281,19 @@ Map buildCrossDayBundle(
final latestAnomaly = anomaly.isEmpty ? null : anomaly.last;
final latestTemp = tempIllness.isEmpty ? null : tempIllness.last;
- // per-day flags (for notifications / trends): asleep/illness/anomaly/temp.
+ // per-day flags (for notifications / trends): asleep/illness/anomaly/temp,
+ // plus the nightly RESTING HR the "your resting HR trend shifted" CUSUM
+ // notification reads back off these rows. That consumer
+ // (`DerivationEngine._runNotifications`) needed `r['rhr']` and this builder
+ // never emitted it, so its series was always empty, its `length >= 10` gate
+ // never passed, and the notification was dead code. The value is already
+ // computed right here (`rhrList`, parallel to `dates`) — emit it rather than
+ // delete a wanted feature. Null stays null (the consumer filters on `is num`).
final recent = >[];
for (var i = 0; i < n; i++) {
recent.add({
'date': dates[i],
+ 'rhr': rhrList[i],
'illness': i < illness.length && illness[i].state == ana.IllnessState.red,
'anomaly': i < anomaly.length && anomaly[i].flagged,
'temp':
@@ -364,6 +378,58 @@ bool _isFreeDay(String? date) {
return dt.weekday == DateTime.saturday || dt.weekday == DateTime.sunday;
}
+/// Guard on how far [_denseDailyTrimp] will densify. Past this the input is not
+/// a plausible recent-history window (a corrupt/absurd date), so we fall back to
+/// the raw per-row series rather than allocating an unbounded list.
+const int _maxDenseTrimpDays = 400;
+
+/// A DENSE per-calendar-day TRIMP series (oldest→newest) over the span covered
+/// by [dates], with every unloaded or unobserved day contributing a 0 impulse.
+///
+/// [dates] and [trimps] are parallel (one entry per derived day, oldest first);
+/// [dates] may skip calendar days entirely (an underived day has no row).
+/// PURE: `DateTime.parse` / `DateTime(y, m, d + 1)` are deterministic functions
+/// of the input strings, and the day-step normalizes DST-length days correctly.
+List _denseDailyTrimp(List dates, List trimps) {
+ // NO observed load at all is not "a run of zero-load days" — it is no load
+ // history. Densifying it would hand the EWMA a synthetic all-zero series and
+ // turn "we have never seen a workout" into a confident CTL/ATL/TSB of 0,
+ // which is the fabrication this whole pass exists to remove. Abstain by
+ // handing back an empty series and let the load model decline.
+ if (!trimps.any((t) => t != null)) return const [];
+ final byDate = {};
+ String? first, last;
+ for (var i = 0; i < dates.length && i < trimps.length; i++) {
+ final date = dates[i];
+ if (date.isEmpty || DateTime.tryParse(date) == null) continue;
+ byDate[date] = trimps[i] ?? 0.0;
+ first ??= date;
+ if (last == null || date.compareTo(last) > 0) last = date;
+ if (date.compareTo(first) < 0) first = date;
+ }
+ if (first == null || last == null) return const [];
+ final start = DateTime.parse(first);
+ final end = DateTime.parse(last);
+ final out = [];
+ var cursor = DateTime(start.year, start.month, start.day);
+ final stop = DateTime(end.year, end.month, end.day);
+ while (!cursor.isAfter(stop)) {
+ if (out.length >= _maxDenseTrimpDays) {
+ // Implausible span — don't densify; the honest per-row series is better
+ // than an arbitrarily long zero-padded one.
+ return [for (final v in trimps) v ?? 0.0];
+ }
+ out.add(byDate[_dateKey(cursor)] ?? 0.0);
+ cursor = DateTime(cursor.year, cursor.month, cursor.day + 1);
+ }
+ return out;
+}
+
+String _dateKey(DateTime d) {
+ String two(int x) => x.toString().padLeft(2, '0');
+ return '${d.year.toString().padLeft(4, '0')}-${two(d.month)}-${two(d.day)}';
+}
+
/// Absolute value of each present element (used to orient skin-temp by |z|).
List _absList(List xs) => [for (final v in xs) v?.abs()];
@@ -436,11 +502,21 @@ ana.Metric _crossDaySri(List> days) {
// line ~100) already exists to avoid.
final startMin = _localTodMin(start.round()).floor();
// end is exclusive of the segment's trailing edge; cover [start,end).
- final endMin = _localTodMin(end.round()).ceil();
+ final endMinRaw = _localTodMin(end.round()).ceil();
+ // WRAP, don't clamp. Both bounds are clock-minutes-of-day in [0,1440),
+ // so a segment crossing local midnight reads e.g. start=1430,
+ // end=20 — and the old `for (m = startMin; m < endMin; m++)` simply
+ // never executed, silently DROPPING that segment (the comment claimed
+ // it "clamps into grid"; it didn't). Every night has exactly one such
+ // segment, so sleep-regularity was always computed with a hole at the
+ // boundary. Unwrap the end past 1440 and write back modulo the grid.
+ // `endMinRaw == startMin` stays a genuinely empty (sub-minute) segment.
+ final endMin =
+ endMinRaw < startMin ? endMinRaw + epochsPerDay : endMinRaw;
+ final span = math.min(endMin - startMin, epochsPerDay);
final asleepSeg = stage != null && stage != 'wake';
- for (var m = startMin; m < endMin; m++) {
- // Guard wrap: a segment that crosses midnight just clamps into grid.
- if (m < 0 || m >= epochsPerDay) continue;
+ for (var k = 0; k < span; k++) {
+ final m = (startMin + k) % epochsPerDay;
cov[m] = true;
if (asleepSeg) asleep[m] = true;
}
diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart
index f994a9e5..f6acb464 100644
--- a/lib/compute/derivation_engine.dart
+++ b/lib/compute/derivation_engine.dart
@@ -264,7 +264,92 @@ import 'substrate.dart';
// the real Baevsky SI was absent, violating the never-impute rule; the UI
// already correctly renders "—" when `score` is null. Bump so affected days
// re-derive without a same-day, no-sleep readiness/stress score.
-const int kAlgoVersion = 47;
+// v48: audit sweep across compute + the sibling analytics package.
+//
+// EDGE-LOCAL changes, which ship the moment this constant lands:
+// - The per-sweep baseline snapshot is genuinely frozen. `appendScalars` is
+// gone; the history is dated and loaded once, and `valuesBefore(key, date)`
+// excludes the target day from its own baseline. A re-derive sweep could
+// previously append each finished day back into the shared window and evict
+// a real old day, collapsing median/MAD toward duplicated recent values —
+// the same pollution shape edge#108 fixed on the load path.
+// - A day is no longer allowed to sit inside its own readiness baseline, and
+// lnRMSSD no longer double-appends today.
+// - `nocturnalRhr` is now fed the positionally-dense day series instead of a
+// compacted one, so its 30-minute window is real wall-clock again.
+// - Profile imputation (age 30 / 70 kg / sex m / RHR 60) no longer persists
+// strain, calories and zones as if they were measured.
+// - SRI no longer drops the one hypnogram segment per night that crosses
+// local midnight; CTL/ATL/TSB now sees a calendar-dense series so fitness
+// and fatigue decay across rest days.
+// - Historical days resolve their timezone offset at their own timestamp
+// rather than through today's offset.
+//
+// SIBLING-PACKAGE changes ride along with this bump: the analytics sweep from
+// the same review (sleep no longer reporting a no-data window as light sleep,
+// the Lipponen-Tarvainen threshold on the signed dRR series, abstention on
+// degenerate dispersion, the reconciled TRIMP stack, circular social jetlag)
+// and the protocol decode fixes. This was NOT true when v48 was first written —
+// pubspec.yaml still pointed at the pre-fix SHAs then, and this note said so.
+// The pins were moved as part of v49; see the pin-status note above it.
+// v49: steps/activity rebuilt on a calibration-invariant feature.
+//
+// Diagnosed on a real user database: the day reported 39,384 steps against a
+// true value of ~2,000. ENMO is `mean(max(0, |a| - gRef))` and gRef is
+// auto-calibrated per day from the stillest samples — which are the long sleep
+// block, where the wrist sits in a different orientation. gRef came out at
+// 0.9797 that day vs ~1.032 on every other, and since ENMO subtracts it from
+// every sample, a reference 0.05 g low adds 0.05 g to every minute: exactly the
+// 0.05 g walking floor. Sweeping gRef over the identical samples gave 42,155
+// steps at 0.97 and 0 at 1.02 — the signal and the calibration error are both
+// ~0.05 g, so no threshold could have fixed it.
+//
+// The analytics package now decides activity from the per-axis high-passed
+// dynamic amplitude (gravity is DC in the sensor frame; any per-axis offset or
+// gain error cancels exactly), anchored on a PERSONAL floor pooled from
+// trailing days rather than an absolute constant or a same-day baseline — both
+// of which were measured to fail, in opposite directions.
+//
+// Edge side of that change:
+// - `dyn_p90` joins the baseline series: each day persists its own high
+// quantile of the dynamic amplitude, and the next day's derive takes the
+// MEDIAN across trailing days (self-excluded, like every other baseline) as
+// its floor. Below the minimum history the estimator ABSTAINS — a day with
+// no personal baseline now reports the real 100 Hz count only, instead of a
+// fabricated 1 Hz number.
+// - `active_min` is persisted as a first-class series. Minutes are what 1 Hz
+// can resolve; steps are derived from them as a range.
+// - The steps bundle carries the range + the floor used, and says plainly
+// when the 1 Hz estimate is absent.
+//
+// This bump also re-derives days whose stored step figure came from the old
+// estimator.
+//
+// PIN STATUS: as of this bump pubspec.yaml points at the analytics and protocol
+// PR-branch commits, so everything described in v48 AND v49 is genuinely in the
+// build — the edge code physically cannot compile against the older analytics
+// pin, which is how we know. Those pins must move to the merge commits (and
+// this must bump again) when the sibling PRs land. Verify any analytics claim
+// made here with `git show :` before trusting it; a changelog
+// citing a change the pinned SHA never contained is how v43 documented a
+// readiness fix that stayed broken for three releases.
+
+// v50: the sibling PRs merged; pubspec.yaml now pins the resulting `main`
+// commits (analytics f5ccae6, protocol a98cd70) instead of the PR-branch heads
+// v49 briefly pointed at.
+//
+// A version bump is required even though no edge SOURCE line changed with it.
+// kAlgoVersion identifies the code that PRODUCED a day_result, and that code
+// includes the pinned siblings: a device holding v49 rows built against the
+// PR-branch SHAs must re-derive against the merged ones rather than serve them
+// as equivalent. Treating "same content, different commit" as not worth a bump
+// is the assumption that lets a stale bundle survive a dependency change.
+//
+// Verified at the merge commits themselves, not inferred from the PRs being
+// green: steps.dart carries the new step API, rr_correction.dart has the
+// signed-dRR `seg.add(x[k])`, advanced_stager.dart has maxAccelCarryForwardSec,
+// live.dart has kKnownRecordVersions.
+const int kAlgoVersion = 50;
/// Raw is kept this many days past derivation, then pruned (derived stays).
const int rawRetentionDays = 3;
@@ -326,6 +411,21 @@ const int _headlineFreezeMarginSec = 60 * 60;
Future> debugBaselineWindow(String key) async =>
(await _BaselineHistoryCache.load()).values(key);
+/// Test seam: the EXACT per-day baseline windows one derivation sweep would feed
+/// the readiness pass, in dispatch order (`orderedDays` is newest-first).
+///
+/// Pins the two properties the sweep path must have — the snapshot is loaded
+/// ONCE and never mutated as days complete, and each day's window self-excludes
+/// that day's own date. Both were violated by the old `appendScalars` sweep.
+@visibleForTesting
+Future>> debugSweepBaselineWindows(
+ String key,
+ List orderedDays,
+) async {
+ final history = await _BaselineHistoryCache.load();
+ return [for (final day in orderedDays) history.valuesBefore(key, day)];
+}
+
@visibleForTesting
({List days, String reason}) selectLightDeriveDays({
required Set rawDays,
@@ -350,10 +450,43 @@ class _DeriveScope {
});
}
+/// One (date, value) sample of a baseline series.
+typedef _DatedValue = ({String date, double value});
+
class _BaselineHistoryCache {
_BaselineHistoryCache(this._series);
- final Map> _series;
+ /// The baseline series this cache carries, keyed by `metric_series.key`.
+ static const List keys = [
+ 'ln_rmssd',
+ 'rmssd',
+ 'rhr',
+ 'resp_rate',
+ 'skin_temp_adc',
+ 'readiness',
+ // Per-day high quantile of the calibration-invariant dynamic accel
+ // amplitude. The 1 Hz activity estimator's floor is anchored on the MEDIAN
+ // of this series across trailing days, never on a same-day value: a
+ // single-day threshold collapses on a quiet day and passes everything,
+ // which is the mirror image of the absolute-constant failure it replaced.
+ 'dyn_p90',
+ ];
+
+ /// DATED baseline samples, ascending by date, one entry per day (metric_series
+ /// is keyed `(date, key)` with REPLACE, so it is structurally de-duplicated).
+ ///
+ /// IMMUTABLE for the lifetime of one derivation sweep. There is deliberately
+ /// no mutator: the previous `appendScalars` mutated this shared snapshot as
+ /// each day of a sweep finished, which re-introduced exactly the duplicate-day
+ /// pollution the load path was rewritten to prevent — `load()` had ALREADY
+ /// read the persisted values of the days about to be re-derived, so appending
+ /// each finished day again (and evicting a real old day to stay at 28) left
+ /// later days in the sweep reading a window with up to 21 duplicated recent
+ /// values in descending date order. Median/MAD then collapsed toward the
+ /// repeated value and readiness went blank/wrong — the load-path bug, moved
+ /// into the sweep path. A sweep now reads ONE frozen snapshot, and each day
+ /// derives its own window from it by date.
+ final Map> _series;
/// Load the rolling baseline window that feeds the readiness/illness
/// computations. This ALWAYS rebuilds from `metric_series` — the canonical
@@ -361,57 +494,73 @@ class _BaselineHistoryCache {
/// value per day.
///
/// We deliberately do NOT trust the persisted `rolling_artifact` for history.
- /// That artifact is written from an in-memory cache that [appendScalars] only
- /// appends to (no day identity), so repeated same-day re-derives could stack
- /// duplicate copies of today into the window; once enough slots matched, the
- /// readiness composite's robust z-score hit MAD=0 and went absent — the blank
- /// readiness ring. A polluted artifact is still valid JSON, so trusting it on
- /// read would let that pollution reach the computation on the first
- /// post-upgrade derive (and, when every day is finalized and `run()` does no
- /// work, forever). Rebuilding from the de-duplicated store on every load makes
- /// the read path immune and self-heals any already-polluted install.
+ /// That artifact was written from an in-memory cache with no day identity, so
+ /// repeated same-day re-derives could stack duplicate copies of today into the
+ /// window; once enough slots matched, the readiness composite's robust z-score
+ /// hit MAD=0 and went absent — the blank readiness ring. A polluted artifact
+ /// is still valid JSON, so trusting it on read would let that pollution reach
+ /// the computation on the first post-upgrade derive (and, when every day is
+ /// finalized and `run()` does no work, forever). Rebuilding from the
+ /// de-duplicated store on every load makes the read path immune and self-heals
+ /// any already-polluted install.
+ ///
+ /// NOTE ON THE QUERY: `LocalDb.metricSeries(key)` with NO `limit` is the whole
+ /// series, `date ASC` — the dates are what make a per-day `date < target`
+ /// window possible at all. It must NOT be given a `limit` (that is `date ASC
+ /// LIMIT n`, i.e. the OLDEST n — the opposite of a trailing window); the
+ /// trailing window is taken here, in Dart, per target day.
static Future<_BaselineHistoryCache> load() async {
- Future> hist(String key) =>
- LocalDb.trailingSeriesValues(key, _baselineWindowDays);
-
- final loaded = await Future.wait([
- hist('ln_rmssd'),
- hist('rmssd'),
- hist('rhr'),
- hist('resp_rate'),
- hist('skin_temp_adc'),
- hist('readiness'),
- ]);
+ Future> hist(String key) async {
+ final rows = await LocalDb.metricSeries(key);
+ final out = <_DatedValue>[];
+ for (final row in rows) {
+ final date = row['date'];
+ final value = row['value'];
+ if (date is! String || date.isEmpty || value is! num) continue;
+ out.add((date: date, value: value.toDouble()));
+ }
+ return out;
+ }
+
+ final loaded = await Future.wait([for (final k in keys) hist(k)]);
return _BaselineHistoryCache({
- 'ln_rmssd': loaded[0],
- 'rmssd': loaded[1],
- 'rhr': loaded[2],
- 'resp_rate': loaded[3],
- 'skin_temp_adc': loaded[4],
- 'readiness': loaded[5],
+ for (var i = 0; i < keys.length; i++) keys[i]: loaded[i],
});
}
- List values(String key) =>
- List.from(_series[key] ?? const []);
-
- void appendScalars(Map scalars) {
- void add(String seriesKey, String scalarKey) {
- final v = (scalars[scalarKey] as num?)?.toDouble();
- if (v == null) return;
- final list = _series.putIfAbsent(seriesKey, () => []);
- list.add(v);
- while (list.length > _baselineWindowDays) {
- list.removeAt(0);
- }
- }
-
- add('ln_rmssd', 'ln_rmssd');
- add('rmssd', 'rmssd');
- add('rhr', 'rhr');
- add('resp_rate', 'resp_rate');
- add('skin_temp_adc', 'skin_temp_adc');
- add('readiness', 'readiness');
+ /// The trailing [_baselineWindowDays] values for [key], oldest→newest.
+ ///
+ /// This is the WHOLE window including the newest day; it backs the persisted
+ /// rolling artifact + the rescan signature, which describe "the baseline as it
+ /// currently stands". Per-day derivation must use [valuesBefore] instead.
+ List values(String key) => _trailing(_series[key] ?? const []);
+
+ /// The trailing [_baselineWindowDays] values for [key] STRICTLY BEFORE
+ /// [beforeDate] (`date < ?`), oldest→newest — the baseline for deriving the
+ /// day labelled [beforeDate].
+ ///
+ /// SELF-EXCLUSION IS THE POINT. The previous derive of the same day has
+ /// already written its own row to `metric_series`, so an unfiltered trailing
+ /// window contained TODAY: every light pass after the first z-scored today's
+ /// RHR/HRV/temp against a baseline that already contained today (pulling the
+ /// baseline toward the value under test and understating a genuinely
+ /// off day), and the lnRMSSD stack — which is contractually handed
+ /// `[...history, today]` and takes all-but-last as its baseline — counted it
+ /// a second time. v38 fixed precisely this self-inclusion inside analytics;
+ /// this is the same defect at the edge layer that feeds it. Dates strictly
+ /// AFTER the target are excluded too: a baseline is prior days, and a backfill
+ /// sweep must not let later days leak into an older day's baseline (which
+ /// would also make the result depend on sweep order).
+ List valuesBefore(String key, String beforeDate) => _trailing([
+ for (final s in _series[key] ?? const <_DatedValue>[])
+ if (s.date.compareTo(beforeDate) < 0) s,
+ ]);
+
+ static List _trailing(List<_DatedValue> samples) {
+ final from = samples.length <= _baselineWindowDays
+ ? 0
+ : samples.length - _baselineWindowDays;
+ return [for (var i = from; i < samples.length; i++) samples[i].value];
}
Map toArtifactJson() {
@@ -935,7 +1084,12 @@ class DerivationEngine {
// The worker isolate dies after `Isolate.run`, so the recording flag can't
// leak into the next day's derivation — no try/finally reset needed.
final profileJson = await _loadSleepUserProfileJson();
- final (candidateJson, updatedProfileJson) = await Isolate.run(() {
+ // Cancellable + TIMED OUT. This site previously used a bare `Isolate.run`
+ // with no timeout at all, so a hung staging pass never completed its future
+ // — `_running` stayed true and `DeriveScheduler._drain` never returned, i.e.
+ // all derivation was dead until app restart.
+ final (candidateJson, updatedProfileJson) =
+ await _runIsolateCancellable(() {
try {
ana.cardioUserProfile = profileJson == null
? null
@@ -972,7 +1126,7 @@ class DerivationEngine {
}
}
return (jsonEncode(candidate.toJson()), foldedJson);
- });
+ }, _perDayTimeout, label: 'sleep-staging $dayId');
final candidate = SleepSessionCandidate.fromJson(
(jsonDecode(candidateJson) as Map).cast());
if (override == null) {
@@ -1015,13 +1169,35 @@ class DerivationEngine {
}) async {
if (toRecTs < fromRecTs) return Substrate.empty;
final port = ReceivePort();
- final isolate = await Isolate.spawn(derivationPrepareWorker, port.sendPort);
+ // onError/onExit are LOAD-BEARING. Without them, an uncaught throw inside
+ // the worker (a malformed SQLite row reaching one of the numeric reads in
+ // its 'page' handler — the worker only ever reported errors from its
+ // 'finish' branch) killed the isolate silently, and this side awaited
+ // `result.future` FOREVER with `_running == true`: DeriveScheduler._drain
+ // never returned and ALL derivation was dead until app restart. Now a
+ // worker death fails the future, and the timeout below bounds the wait
+ // even if no signal arrives at all.
+ final isolate = await Isolate.spawn(
+ derivationPrepareWorker,
+ port.sendPort,
+ onError: port.sendPort,
+ onExit: port.sendPort,
+ );
final ready = Completer();
final result = Completer();
+ // A failure completes BOTH completers, but we may bail out via `ready` and
+ // never await `result` — register a listener so that error is never an
+ // unobserved async error. (The real error still propagates via `ready`.)
+ unawaited(result.future.catchError((_) => Substrate.empty));
late final StreamSubscription sub;
- sub = port.listen((message) async {
+ void fail(Object error) {
+ if (!ready.isCompleted) ready.completeError(error);
+ if (!result.isCompleted) result.completeError(error);
+ }
+
+ sub = port.listen((message) {
if (message is SendPort) {
- ready.complete(message);
+ if (!ready.isCompleted) ready.complete(message);
return;
}
if (message is Map && message['type'] == 'result') {
@@ -1029,23 +1205,28 @@ class DerivationEngine {
if (kind == 'substrate') {
final payload = ((message['payload'] as Map?) ?? const {})
.cast();
- await sub.cancel();
- port.close();
- isolate.kill(priority: Isolate.immediate);
- result.complete(Substrate.fromJson(payload));
+ if (!result.isCompleted) result.complete(Substrate.fromJson(payload));
}
return;
}
if (message is Map && message['type'] == 'error') {
- await sub.cancel();
- port.close();
- isolate.kill(priority: Isolate.immediate);
- result.completeError(Exception(message['error']));
+ fail(Exception('prepare worker error: ${message['error']}'));
+ return;
+ }
+ if (message is List) {
+ // `onError` wire format ([error, stackTrace]) — an uncaught throw.
+ fail(Exception('prepare worker crashed: '
+ '${message.isNotEmpty ? message.first : "no detail"}'));
+ return;
+ }
+ if (message == null) {
+ // `onExit` — the isolate ended without ever sending a result.
+ fail(StateError('prepare worker exited without a result'));
}
});
- final worker = await ready.future;
- worker.send(const {'type': 'config', 'mode': 'substrate'});
try {
+ final worker = await ready.future;
+ worker.send(const {'type': 'config', 'mode': 'substrate'});
int? afterRecTs;
int? afterCursor;
var rangePages = 0;
@@ -1091,12 +1272,22 @@ class DerivationEngine {
break;
}
worker.send(const {'type': 'finish'});
- return result.future;
- } catch (_) {
+ // BOUNDED. `result.future` had no timeout at all, so any path that left
+ // the worker unable to answer hung this call — and with it the whole
+ // engine — permanently.
+ return await result.future.timeout(
+ _perDayTimeout,
+ onTimeout: () => throw TimeoutException(
+ 'substrate prepare for $dayId timed out after $_perDayTimeout',
+ ),
+ );
+ } finally {
+ // ALWAYS tear down: on success, on error, and on timeout. The isolate is
+ // killed rather than abandoned so a wedged worker can never outlive the
+ // call that spawned it.
await sub.cancel();
port.close();
isolate.kill(priority: Isolate.immediate);
- rethrow;
}
}
@@ -1467,9 +1658,13 @@ class DerivationEngine {
);
final withHistory = _attachHistory(input, history);
- final bundle = await Isolate.run(
+ // Cancellable: on timeout the isolate is KILLED, not merely abandoned to
+ // keep burning a core behind the worker pool's back.
+ final bundle = await _runIsolateCancellable(
() => deriveDayBundle(withHistory),
- ).timeout(_perDayTimeout);
+ _perDayTimeout,
+ label: 'day-bundle ${day.date}',
+ );
_logSpo2Diagnostics(day, input, bundle);
// Readiness came back absent for TODAY specifically (not a historical
// backfill day, which would just be noise) — log why. This ran inside
@@ -1547,6 +1742,33 @@ class DerivationEngine {
final scMap = (bundle['scalars'] as Map?)?.cast();
+ // ── NEVER WRITE NOTHING OVER SOMETHING ───────────────────────────────────
+ // Raw retention is 3 days, but derived history is forever — so a day older
+ // than retention has a good `day_result` and NO raw. Re-deriving it (which
+ // "Advanced data → Select all → Re-analyze" does for EVERY listed day, via
+ // runDays(force: true) → _prepareTargetDay, whose empty substrate yields an
+ // all-absent bundle) used to overwrite that good row: `putDayResult` is
+ // ConflictAlgorithm.replace on BOTH `day_result` and `metric_series`, so
+ // every scalar for the date was NULLed — and, because an empty bundle's
+ // endSec was 0, the blank was written FINALIZED and could never re-derive.
+ // Only `run()` had a pruned-raw guard, and only for user-override days.
+ //
+ // Detect it BEFORE the offloaded second half so its own writes
+ // (wake_day_features) can't clobber the early-read path either. With no day
+ // substrate and no sleep substrate the second half has nothing to add — its
+ // scalars are all derived from those two.
+ final producedNothing = daySub.isEmpty &&
+ sleepSub.isEmpty &&
+ (scMap == null || !scMap.values.any((v) => v != null));
+ if (producedNothing) {
+ final existing = await LocalDb.dayResult(day.date);
+ if (_isRealDayResult(existing)) {
+ _log('derive ${day.date}: no substrate (raw pruned) — kept the '
+ 'existing result rather than blanking it');
+ return;
+ }
+ }
+
// ── SECOND HALF — OFFLOADED to a background isolate ──────────────────────
// Everything that turns the isolate-1 bundle into the full day result (wake
// features, hybrid steps + TDEE, all-day HRV/RSA/skin-temp Timeline lines,
@@ -1581,6 +1803,15 @@ class DerivationEngine {
final stepCalib = await LocalDb.getStepCalibration();
final savedSessions = await LocalDb.sessionsInRange(dayLo, dayHi);
+ // PERSONAL ambulatory floor, from days STRICTLY BEFORE this one (the same
+ // self-exclusion every other baseline uses — a day must not help set the
+ // threshold it is then scored against). Anchoring on trailing days is the
+ // whole point: an absolute g constant is destroyed by a few-percent
+ // gravity-reference excursion, and a same-day floor collapses on a quiet
+ // day. Below the minimum history this is null and the estimator abstains.
+ final dynHistory = history.valuesBefore('dyn_p90', day.date);
+ final dynFloorG = ana.personalDynFloorFromDailySummaries(dynHistory);
+
// Built on THIS isolate so the Isolate.run closure captures only this plain
// sendable object (never `this`, `day`, or `bundle`).
final blocksInput = _DayBlocksInput(
@@ -1594,6 +1825,8 @@ class DerivationEngine {
coverageWindows: coverageWindows,
liveStepsReal: liveStepsReal,
stepCalib: stepCalib,
+ dynFloorG: dynFloorG,
+ dynHistoryDays: dynHistory.length,
savedSessions: savedSessions,
date: day.date,
dayEndSec: day.endSec,
@@ -1659,7 +1892,12 @@ class DerivationEngine {
// workouts/HRR/wear/curves would never get a chance to be filled in by a
// later retry.
final ageFinalized = (day.endSec + _finalizationSec) < dataNowSec;
- final finalized = forceFinalize || (ageFinalized && secondHalfOk);
+ // A result with NOTHING in it is never finalized — not even by
+ // forceFinalize. Locking an all-absent row is what made the destructive
+ // re-analyze permanent; leaving it unlocked means a later pass (or a
+ // restored/backfilled substrate) can still fill the day in.
+ final finalized =
+ !producedNothing && (forceFinalize || (ageFinalized && secondHalfOk));
final scalars =
(bundle['scalars'] as Map?)?.cast() ?? const {};
@@ -1703,6 +1941,14 @@ class DerivationEngine {
// Steps = real 100 Hz count + 1 Hz estimate over uncovered minutes
// (computed in _stepsAndEnergy; never double-counted).
'steps': sc('steps'),
+ // Ambulatory minutes — the quantity 1 Hz can actually resolve, and the
+ // unit public activity guidance uses. Steps are derived FROM this.
+ 'active_min': sc('active_min'),
+ // This day's high quantile of the calibration-invariant dynamic accel
+ // amplitude. Not a user-facing metric: it is the per-day summary the
+ // NEXT day's derive pools to anchor its personal ambulatory floor, so
+ // the threshold never depends on a single day (see _BaselineHistoryCache).
+ 'dyn_p90': sc('dyn_p90'),
'calories_total': sc('calories_total'),
// Daytime nap minutes (principled van Hees + HR-dip) → trend + Sleep Coach.
'nap_min': sc('nap_min'),
@@ -1722,7 +1968,10 @@ class DerivationEngine {
'hrr_bpm': sc('hrr_bpm'),
},
);
- history.appendScalars(scalars);
+ // NOTE: the sweep's `history` snapshot is deliberately NOT updated here.
+ // See _BaselineHistoryCache — mutating the shared snapshot mid-sweep is the
+ // duplicate-day pollution bug, and each day already derives its own
+ // date-bounded window from the frozen snapshot.
_log(
'derived ${day.date} v$kAlgoVersion '
'(sleep=${day.sleepOffsetSec > day.sleepOnsetSec}, final=$finalized)',
@@ -1844,7 +2093,48 @@ class DerivationEngine {
_log('[spo2-detect] ${jsonEncode(payload)}');
}
+ /// Skip reasons that describe a TRANSIENT failure of this particular pass
+ /// rather than a permanently pathological day. These must never finalize:
+ /// finalizing locks the day out of every future pass at this algo version.
+ static const Set _transientSkipReasons = {'timeout', 'error'};
+
+ /// Whether [row] is a REAL derived day result worth protecting — i.e. not a
+ /// skip marker and not an all-absent shell.
+ static bool _isRealDayResult(Map? row) {
+ if (row == null) return false;
+ if ((row['skipped'] as num?)?.toInt() == 1) return false;
+ final payload = _decodeBundle(row['payload_json']);
+ if (payload == null) return false;
+ if (payload['skipped'] == true) return false;
+ final scalars = payload['scalars'];
+ if (scalars is Map && scalars.values.any((v) => v != null)) return true;
+ return row['rhr'] != null || row['rmssd'] != null || row['readiness'] != null;
+ }
+
+ /// Test seam for [_markDaySkipped] — the "a skip marker must never destroy a
+ /// real result" guarantee is the whole point of the method, so it is pinned
+ /// directly rather than through a full derive pass.
+ @visibleForTesting
+ Future debugMarkDaySkipped(
+ String dayId,
+ int dayEndSec,
+ int dataNowSec, {
+ required String reason,
+ }) =>
+ _markDaySkipped(dayId, dayEndSec, dataNowSec, reason: reason);
+
/// Persist a minimal skip marker so a pathological day isn't retried forever.
+ ///
+ /// A SKIP MARKER MUST NEVER OVERWRITE A REAL RESULT. `putDayResult` is
+ /// ConflictAlgorithm.replace on both `day_result` AND `metric_series`, so this
+ /// used to blank a good day's every scalar on a single [_perDayTimeout]
+ /// overrun — and, once the day sat >48 h behind the data edge, wrote the blank
+ /// FINALIZED, making it permanent (raw is pruned 3 days later, so there is
+ /// nothing left to re-derive from). It hit TODAY too: a good 08:00 result
+ /// replaced by a skip marker after one transient 09:00 timeout on a loaded
+ /// phone. `rescanRecent` explicitly refuses to do this for exactly this
+ /// reason; `run()` did it anyway. Now: write the marker only when there is no
+ /// good row to lose, and never lock a transient failure.
Future _markDaySkipped(
String dayId,
int dayEndSec,
@@ -1852,12 +2142,23 @@ class DerivationEngine {
required String reason,
}) async {
try {
+ final existing = await LocalDb.dayResult(dayId);
+ if (_isRealDayResult(existing)) {
+ _log('derive $dayId $reason — existing result kept (not overwritten '
+ 'with a skip marker)');
+ return;
+ }
await LocalDb.putDayResult(
dayId: dayId,
algoVersion: kAlgoVersion,
payloadJson: jsonEncode({'skipped': true, 'reason': reason}),
windowJson: '{}',
- finalized: (dayEndSec + _finalizationSec) < dataNowSec,
+ // Structural failures (a day that can never be prepared / blows the
+ // prepare budget) still finalize once aged out, so they aren't retried
+ // forever. A timeout or a one-off error does not — that day gets
+ // another chance while it still has raw.
+ finalized: !_transientSkipReasons.contains(reason) &&
+ (dayEndSec + _finalizationSec) < dataNowSec,
skipped: true,
);
} catch (_) {
@@ -1865,23 +2166,30 @@ class DerivationEngine {
}
}
- /// Attach trailing personal history (from metric_series) for the readiness pass.
+ /// Attach trailing personal history (from metric_series) for the readiness
+ /// pass — the trailing window of days STRICTLY BEFORE the day being derived.
+ ///
+ /// The self-exclusion (`date < input.date`) is load-bearing, not cosmetic: see
+ /// [_BaselineHistoryCache.valuesBefore]. Every one of these series is a
+ /// BASELINE the day's own value is scored against, so the day's own row (which
+ /// a previous derive of the same day already persisted) must not be in it.
Map _attachHistory(
DayBundleInput input,
_BaselineHistoryCache history,
) {
final m = input.toJson();
- m['ln_rmssd_history'] = history.values('ln_rmssd');
- m['rhr_history'] = history.values('rhr');
- m['resp_history'] = history.values('resp_rate');
+ final date = input.date;
+ m['ln_rmssd_history'] = history.valuesBefore('ln_rmssd', date);
+ m['rhr_history'] = history.valuesBefore('rhr', date);
+ m['resp_history'] = history.valuesBefore('resp_rate', date);
// Robust nocturnal RMSSD history (the `rmssd` series) — feeds the EWMA hrv
// baseline so its center matches today's headline RMSSD (same metric).
- m['rmssd_history'] = history.values('rmssd');
+ m['rmssd_history'] = history.valuesBefore('rmssd', date);
// BASELINE for skin_temp_z is the RAW nightly ADC-mean series (`skin_temp_adc`),
// NOT the z-score series. Feeding z-scores back as the baseline was a unit
// mismatch that left z permanently null. The raw mean is stored every day so
// this series fills and z starts computing once ≥3 days exist.
- m['skin_temp_adc_history'] = history.values('skin_temp_adc');
+ m['skin_temp_adc_history'] = history.valuesBefore('skin_temp_adc', date);
return m;
}
@@ -1904,9 +2212,11 @@ class DerivationEngine {
// main isolate after Isolate.run returned it. Returning the already-
// encoded string avoids both the main-isolate encode cost AND transfers
// a flat string across the isolate boundary instead of a large nested Map.
- final bundleJson = await Isolate.run(
+ final bundleJson = await _runIsolateCancellable(
() => jsonEncode(buildCrossDayBundle(days, profileMap)),
- ).timeout(_crossDayTimeout);
+ _crossDayTimeout,
+ label: 'crossday',
+ );
await LocalDb.putBaseline('crossday', bundleJson);
_log('crossday: stored over ${days.length} day(s)');
} catch (e) {
@@ -1945,7 +2255,7 @@ class DerivationEngine {
// unconditionally on every heavy pass. _decodeBundle/_crossDayRecord are
// both static, so this whole transform+encode step is isolate-safe.
final rows = await LocalDb.recentDayResults(_crossDayWindow);
- final (days, json) = await Isolate.run(() {
+ final (days, json) = await _runIsolateCancellable(() {
final days = >[];
for (final row in rows.reversed) {
final payload = _decodeBundle(row['payload_json']);
@@ -1955,7 +2265,7 @@ class DerivationEngine {
if (rec != null) days.add(rec);
}
return (days, jsonEncode({'algo_version': kAlgoVersion, 'days': days}));
- });
+ }, _crossDayTimeout, label: 'crossday-input');
await LocalDb.putBaseline('crossday_input', json);
return days;
}
@@ -2289,6 +2599,8 @@ class DerivationEngine {
List> coverageWindows,
int liveStepsReal,
ana.StepCalibration? stepCalib,
+ double? dynFloorG,
+ int dynHistoryDays,
) {
try {
if (daySub.length < 60) return;
@@ -2296,6 +2608,12 @@ class DerivationEngine {
if (motion.isEmpty) return;
final hrPerMin = _hrPerMinuteAligned(motion, daySub);
+ // This day's own contribution to the personal floor, persisted to
+ // metric_series so tomorrow's derive can anchor on it. Null for a day too
+ // thin to summarise — we store nothing rather than a fabricated level.
+ final dynSummary = ana.dailyDynSummary(motion);
+ if (dynSummary != null) scMap?['dyn_p90'] = dynSummary;
+
// STEPS — hybrid, no double-count. Drop any minute already covered by a
// 100 Hz window (real count wins), estimate steps for the rest from 1 Hz.
bool covered(double tsMinStartMs) {
@@ -2317,29 +2635,52 @@ class DerivationEngine {
final rhr = (scMap?['rhr'] as num?)?.toDouble();
final est = ana.dailyStepEstimate(
motionUn,
+ personalDynFloorG: dynFloorG,
hrPerMin: hrUn,
restingHr: rhr,
calib: stepCalib,
+ pooledMinutesAvailable: dynHistoryDays,
);
- final estSteps = est.present ? est.value!.steps : 0;
+ final v = est.present ? est.value : null;
+ final estSteps = v?.steps ?? 0;
final daySteps = liveStepsReal + estSteps;
scMap?['steps'] = daySteps.toDouble();
+ // ACTIVE MINUTES is the primary, honest quantity here: 1 Hz cannot count
+ // steps (gait is 1.4-2.5 Hz and 120 spm aliases to DC at this rate), but
+ // it can resolve ambulatory MINUTES, which is also the unit public
+ // activity guidance is written in. The step figures are a RANGE over the
+ // free-living cadence band, and are absent entirely when the personal
+ // floor has not been established yet.
bundle['steps'] = {
'value': daySteps,
'real_100hz': liveStepsReal, // AN-2554 over live windows (real count)
- 'estimated_1hz':
- estSteps, // walking-min × cadence for uncovered minutes
- 'ambulatory_min': est.present ? est.value!.ambulatoryMinutes : 0,
- 'cadence_used_spm': est.present ? est.value!.cadenceUsed : 0,
+ 'estimated_1hz': estSteps, // midpoint of the 1 Hz range
+ 'estimated_1hz_low': v?.stepsLow,
+ 'estimated_1hz_high': v?.stepsHigh,
+ 'active_min': v?.activeMinutes ?? 0,
+ 'cadence_low_spm': v?.cadenceLowSpm,
+ 'cadence_high_spm': v?.cadenceHighSpm,
+ 'dyn_floor_g': v?.dynFloorG,
+ 'estimate_present': v != null,
'confidence': liveStepsReal > 0
? 0.7
: (est.present ? est.confidence : 0.2),
'tier': liveStepsReal > 0 && estSteps == 0 ? 'HIGH' : 'ESTIMATE',
- 'inputs_used': const ['live_100hz_pedometer', 'enmo_1hz', 'hr_1hz'],
- 'note':
- 'real 100 Hz count for streamed time + 1 Hz walking estimate for '
- 'the rest (1 Hz cannot count steps directly)',
+ 'inputs_used': const [
+ 'live_100hz_pedometer',
+ 'dyn_amp_1hz',
+ 'hr_1hz',
+ 'personal_dyn_floor',
+ ],
+ 'note': v == null
+ ? 'real 100 Hz count only — the 1 Hz activity estimate needs a '
+ 'personal movement baseline from several days of wear '
+ '(${est.note ?? 'need_baseline'})'
+ : 'real 100 Hz count for streamed time + ${v.activeMinutes} active '
+ 'minutes estimated from 1 Hz for the rest (1 Hz cannot count '
+ 'steps directly, so the step figure is a range)',
};
+ if (v != null) scMap?['active_min'] = v.activeMinutes.toDouble();
if (profile.isComplete) {
final perMinFull = [
for (final h in hrPerMin)
@@ -2396,6 +2737,7 @@ class DerivationEngine {
required int sleepOnsetSec,
required int sleepOffsetSec,
double? restingHr,
+ double? dynFloorG,
}) {
final activeMin = _activeMinutes(daySub, sleepOnsetSec, sleepOffsetSec);
final wear = _wearBlock(daySub);
@@ -2406,19 +2748,29 @@ class DerivationEngine {
for (final h in daySub.hr)
if (h > 0) h.toDouble(),
];
- final age = profile.ageYears?.toDouble() ?? 30.0; // fallback age
- final weightKg = profile.weightKg ?? 70.0; // fallback weight
- final sex = profile.sex?.toLowerCase() ?? 'm'; // fallback sex
- final hrMax = 208 - 0.7 * age;
- // Fallback to 60.0 so new users (no baseline yet, no manual RHR) still get Strain
- final rhrForTrimp = restingHr ?? profile.restingHrManual?.toDouble() ?? 60.0;
+ // NEVER IMPUTE A PROFILE. These used to default to age 30 / 70 kg / sex 'm'
+ // / RHR 60 "so new users still get Strain" — and the results were then
+ // persisted as REAL scalars (strain, calories, calories_total, steps) into
+ // day_result AND metric_series for someone who never entered a profile.
+ // That is a fabricated number wearing a real number's clothes, and it
+ // contradicts the never-impute contract the rest of this layer (and
+ // `Profile`'s own doc, and the pure `onehz_pipeline` which already gates on
+ // exactly these fields) enforces. A missing input now makes the DEPENDENT
+ // metric absent — the UI already renders "—" correctly.
+ final age = profile.ageYears?.toDouble();
+ final weightKg = profile.weightKg;
+ final sex = profile.sex?.toLowerCase();
+ final hrMax = profile.hrMaxTanaka; // null when age is unknown
+ final rhrForTrimp = restingHr ?? profile.restingHrManual?.toDouble();
double? strain;
double? calories;
double? steps;
double? caloriesTotal;
Map zones = const {};
- if (perMin.isNotEmpty) {
- if (dayHrValid.isNotEmpty) {
+ if (perMin.isNotEmpty && hrMax != null) {
+ // TRIMP needs a real resting HR (nightly or user-supplied) and a real sex
+ // constant — both are in the Banister formula itself.
+ if (dayHrValid.isNotEmpty && rhrForTrimp != null && sex != null) {
final trimp = ana.banisterTrimp(
perMin,
restingHr: rhrForTrimp,
@@ -2430,30 +2782,37 @@ class DerivationEngine {
if (score.present) strain = score.value;
}
}
+ // Zones are pure %HRmax bands — real as soon as HRmax is real.
zones = _wakeZoneMinutes(daySub, sleepOnsetSec, sleepOffsetSec, hrMax);
- // age/weightKg always have a value by this point (defaulted above), so
- // this used to be a dead "if (age != null && weightKg != null && ...)"
- // that flutter analyze flagged - there was never actually a gate here.
- calories = _keytelCaloriesWake(
- perMin,
- age,
- weightKg,
- hrMax,
- sex == 'f',
- );
+ // Keytel takes age, weight and sex directly.
+ if (age != null && weightKg != null && sex != null) {
+ calories = _keytelCaloriesWake(perMin, age, weightKg, hrMax, sex == 'f');
+ }
}
if (motion.isNotEmpty) {
+ // Steps do NOT need a profile: `dailyStepEstimate` falls back to the day's
+ // own 10th-percentile HR when `restingHr` is null, which is data-derived,
+ // not imputed. Pass the real value or nothing — never the old 60.0.
+ //
+ // This is the EARLY-READ path (what Today shows before the full day result
+ // exists); `_stepsAndEnergy` recomputes and overwrites it with the hybrid
+ // real-100 Hz + 1 Hz figure moments later. Without a personal floor the
+ // estimator abstains and `steps` stays null here, which is correct — the
+ // early read then shows no step figure rather than a fabricated one.
final stepMetric = ana.dailyStepEstimate(
motion,
+ personalDynFloorG: dynFloorG,
hrPerMin: hrPerMinAll,
restingHr: rhrForTrimp,
);
if (stepMetric.present && stepMetric.value != null) {
steps = stepMetric.value!.steps.toDouble();
}
- // same story - age/weightKg can't be null here, heightCm is the only
- // field that actually still needs a null check.
- if (profile.heightCm != null) {
+ // TDEE needs the full anthropometric set (Mifflin BMR + Keytel surplus).
+ if (age != null &&
+ weightKg != null &&
+ sex != null &&
+ profile.heightCm != null) {
final energy = ana.Calories.dailyEnergy(
hrPerMinAll,
profile: ana.WorkoutUserProfile(
@@ -3068,6 +3427,85 @@ class DerivationEngine {
}
}
+ /// Run [compute] in an explicitly spawned isolate and enforce [timeout] ON THE
+ /// ISOLATE — the general-purpose form of [_runDayBlocksCancellable].
+ ///
+ /// `Isolate.run(...).timeout(...)` only stops the CALLER awaiting; the spawned
+ /// isolate keeps burning CPU to completion in the background. With a bounded
+ /// per-day worker pool that silently blows the concurrency budget during a
+ /// backlog sweep — which is exactly why [_runDayBlocksCancellable] exists, and
+ /// it had been applied to only one of the file's isolate sites. Worse, some
+ /// sites (the sleep-staging pass) had NO timeout at all, so a hung isolate
+ /// wedged the engine with `_running == true` forever.
+ ///
+ /// Also wires `onError`/`onExit` so an uncaught throw or a silent death
+ /// FAILS the future instead of hanging it.
+ static Future _runIsolateCancellable(
+ FutureOr Function() compute,
+ Duration timeout, {
+ required String label,
+ }) async {
+ final port = ReceivePort();
+ final (SendPort, FutureOr Function()) message =
+ (port.sendPort, compute);
+ final isolate = await Isolate.spawn(
+ _cancellableIsolateEntry,
+ message,
+ onError: port.sendPort,
+ onExit: port.sendPort,
+ );
+ final completer = Completer();
+ late final StreamSubscription sub;
+ sub = port.listen((msg) {
+ if (completer.isCompleted) return;
+ if (msg is _IsolateValue) {
+ completer.complete(msg.value as R);
+ } else if (msg is List) {
+ // Our caught-exception report or the `onError` port's uncaught-error
+ // format — both 2-element lists of strings.
+ completer.completeError(
+ StateError(
+ msg.isNotEmpty
+ ? '$label isolate failed: ${msg.first}'
+ : '$label isolate failed with no error detail',
+ ),
+ );
+ } else if (msg == null) {
+ // `onExit` — the isolate ended without ever sending a result.
+ completer.completeError(
+ StateError('$label isolate exited without a result'),
+ );
+ }
+ });
+ try {
+ return await completer.future.timeout(
+ timeout,
+ onTimeout: () {
+ isolate.kill(priority: Isolate.immediate);
+ throw TimeoutException('$label timed out after $timeout');
+ },
+ );
+ } finally {
+ await sub.cancel();
+ port.close();
+ // No-op if it already exited; guarantees a hung isolate never outlives
+ // this call.
+ isolate.kill(priority: Isolate.immediate);
+ }
+ }
+
+ /// `Isolate.spawn` entry point for [_runIsolateCancellable].
+ static Future _cancellableIsolateEntry(
+ (SendPort, FutureOr Function()) args,
+ ) async {
+ final (sendPort, compute) = args;
+ try {
+ sendPort.send(_IsolateValue(await compute()));
+ } catch (e, st) {
+ sendPort.send([e.toString(), st.toString()]);
+ }
+ }
+
/// `Isolate.spawn` entry point for [_runDayBlocksCancellable]. Must be a
/// static/top-level function taking exactly one (sendable) argument.
static void _dayBlocksIsolateEntry((SendPort, _DayBlocksInput) args) {
@@ -3108,6 +3546,7 @@ class DerivationEngine {
sleepOnsetSec: onset,
sleepOffsetSec: offset,
restingHr: inp.rhr,
+ dynFloorG: inp.dynFloorG,
);
_applyWakeDayFeatures(bundlePatch, scMap, wake);
_stepsAndEnergy(
@@ -3118,6 +3557,8 @@ class DerivationEngine {
inp.coverageWindows,
inp.liveStepsReal,
inp.stepCalib,
+ inp.dynFloorG,
+ inp.dynHistoryDays,
);
// _stepsAndEnergy just corrected `steps`/`calories_total` in bundlePatch +
// scMap using the hybrid real-100Hz + 1Hz-estimate count, but `wake` (built
@@ -3451,18 +3892,55 @@ class DerivationEngine {
return 'error';
}
+ /// The substrate range to LOAD so [calendarDays] can actually run its
+ /// documented nocturnal search for [dayId].
+ ///
+ /// `calendarDays` searches from the previous local NOON
+ /// (`dayStart − kNocturnalSearchLookbackSec`), and its comment records that
+ /// widening from the old prev-18:00 window as deliberate — "the old
+ /// prev-18:00 → noon window missed late wakes and forced the detector to act
+ /// like there was only one candidate sleep". But this loader only fetched
+ /// `dayStart − 6 h` (= 18:00), and `searchStart = math.max(dataStart, …)`
+ /// clipped the search right back to the slice start, so the widening was a
+ /// no-op and any sleep onset before 18:00 was truncated. Load the whole
+ /// window the day model asks for, from the one shared constant.
(int, int) _targetDayWindow(String dayId) {
final startSec = _localDayLabelToSec(dayId);
final endSec = _localNextDayLabelToSec(dayId);
- return (math.max(0, startSec - 6 * 3600), endSec - 1);
+ return (math.max(0, startSec - kNocturnalSearchLookbackSec), endSec - 1);
}
+ /// Test seam for [_targetDayWindow] — the bug was that this loader and
+ /// [calendarDays]' search window silently disagreed, so the agreement is
+ /// pinned directly.
+ @visibleForTesting
+ (int, int) debugTargetDayWindow(String dayId) => _targetDayWindow(dayId);
+
void _log(String m) {
if (kDebugMode) debugPrint('[derive] $m');
log?.call('[derive] $m');
}
}
+/// Wrapper for a cancellable-isolate result, so a computation whose OWN result
+/// happens to be a `List` (the uncaught-error wire format) or `null` (the
+/// `onExit` signal) can never be misread as a failure.
+class _IsolateValue {
+ final Object? value;
+ const _IsolateValue(this.value);
+}
+
+/// Test seam for [DerivationEngine._runIsolateCancellable] — the isolate
+/// lifecycle guarantees (value / error / killed-on-timeout) are what the engine
+/// depends on to never hang, so they're pinned directly.
+@visibleForTesting
+Future runCancellableIsolate(
+ FutureOr Function() compute,
+ Duration timeout, {
+ String label = 'test',
+}) =>
+ DerivationEngine._runIsolateCancellable(compute, timeout, label: label);
+
/// Sendable input for [DerivationEngine._computeDayBlocks] — crosses the
/// `Isolate.run` boundary, so every field is plain data (Substrate is int/double
/// lists; Profile/StepCalibration are primitive data classes). DB reads that the
@@ -3478,6 +3956,15 @@ class _DayBlocksInput {
final List> coverageWindows;
final int liveStepsReal;
final ana.StepCalibration? stepCalib;
+
+ /// PERSONAL ambulatory floor (g, dynAmp units) from trailing days, or null
+ /// when there isn't enough history yet — in which case the 1 Hz estimator
+ /// abstains rather than falling back to a constant. Computed on the main
+ /// isolate (it needs metric_series) and carried in, like the other history.
+ final double? dynFloorG;
+
+ /// How many trailing days backed [dynFloorG] — only for the cold-start note.
+ final int dynHistoryDays;
final List> savedSessions;
final String date;
final int dayEndSec;
@@ -3493,6 +3980,8 @@ class _DayBlocksInput {
required this.coverageWindows,
required this.liveStepsReal,
required this.stepCalib,
+ required this.dynFloorG,
+ required this.dynHistoryDays,
required this.savedSessions,
required this.date,
required this.dayEndSec,
diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart
index 9923af24..d172128a 100644
--- a/lib/compute/derive_prepare.dart
+++ b/lib/compute/derive_prepare.dart
@@ -163,7 +163,16 @@ class SleepSessionCandidate {
required Substrate sleepSub,
}) => PreparedDerivationDay(
date: dayId,
- endSec: daySub.lastTs == null ? 0 : daySub.lastTs! + 1,
+ // `endSec` is what the engine anchors FINALIZATION on
+ // (`endSec + 48 h < dataNowSec` ⇒ lock). An empty substrate used to yield
+ // endSec = 0, which makes that comparison unconditionally true — so a day
+ // whose raw has been pruned (retention is 3 days) derived an all-absent
+ // bundle and wrote it FINALIZED over the good historical row, permanently.
+ // With no data, fall back to the day's real calendar end so an empty
+ // result is aged exactly like a real one.
+ endSec: daySub.lastTs != null
+ ? daySub.lastTs! + 1
+ : localNextMidnightSecForDayLabel(dayId),
confidence: confidence,
flags: flags,
sleepJson: sleepJson,
@@ -176,13 +185,25 @@ class SleepSessionCandidate {
);
}
+/// Local midnight at the START OF THE NEXT day for a `YYYY-MM-DD` day label.
+///
+/// `DateTime(y, m, d + 1)` normalizes the overflow itself and
+/// `millisecondsSinceEpoch` respects local DST rules, so this is correct on the
+/// two 23 h/25 h days a year — unlike `startOfDay + 86400`.
+int localNextMidnightSecForDayLabel(String dayId) {
+ final d = DateTime.tryParse(dayId);
+ if (d == null) return 0;
+ return DateTime(d.year, d.month, d.day + 1).millisecondsSinceEpoch ~/ 1000;
+}
+
void derivationPrepareWorker(SendPort mainSendPort) {
final port = ReceivePort();
final state = _PrepareAccumulator();
String? targetDay;
var mode = 'prepared_day';
mainSendPort.send(port.sendPort);
- port.listen((message) {
+
+ void handle(Object? message) {
if (message is! Map) return;
final type = message['type'];
if (type == 'page') {
@@ -230,12 +251,26 @@ void derivationPrepareWorker(SendPort mainSendPort) {
'payload': payload.toJson(),
});
}
- } catch (e, st) {
- mainSendPort.send({'type': 'error', 'error': '$e\n$st'});
} finally {
port.close();
}
}
+ }
+
+ // EVERY branch is guarded, not just 'finish'. A throw inside the 'page'
+ // handler (a malformed SQLite row reaching one of the numeric reads) used to
+ // kill this worker silently: the only error report lived in the 'finish'
+ // branch, so the main side awaited a Completer that could never complete —
+ // `_running` stayed true and DerivationEngine's scheduler never drained
+ // again, i.e. ALL derivation was dead until app restart. Now any failure is
+ // reported back and the port is closed so the isolate exits.
+ port.listen((message) {
+ try {
+ handle(message);
+ } catch (e, st) {
+ mainSendPort.send({'type': 'error', 'error': '$e\n$st'});
+ port.close();
+ }
});
}
@@ -325,6 +360,14 @@ class _PrepareAccumulator {
final List spo2Red = [];
final List spo2Ir = [];
final List skinTemp = [];
+ final List skinContact = [];
+
+ /// Defensive numeric read. The decoded-page rows come straight out of SQLite,
+ /// where a column's storage class is per-VALUE, not per-column — a row written
+ /// by an older/importing path can hand back a String or null where an INTEGER
+ /// is declared. A bare `row['hr'] as num?` throws on that, and a throw in this
+ /// worker used to hang the whole engine (see [derivationPrepareWorker]).
+ static num? _num(Object? v) => v is num ? v : null;
void addRawPage(List hexes) {
if (hexes.isEmpty) return;
@@ -340,6 +383,7 @@ class _PrepareAccumulator {
spo2Red.addAll(sub.spo2Red);
spo2Ir.addAll(sub.spo2Ir);
skinTemp.addAll(sub.skinTemp);
+ skinContact.addAll(sub.skinContact);
}
void addDecodedPage(
@@ -349,29 +393,35 @@ class _PrepareAccumulator {
if (frames.isEmpty) return;
final rrByCounter = >>{};
for (final row in rrRows) {
- final counter = (row['counter'] as num?)?.toInt();
+ final counter = _num(row['counter'])?.toInt();
if (counter == null) continue;
rrByCounter.putIfAbsent(counter, () => >[]).add(row);
}
for (final row in frames) {
- final recTs = (row['rec_ts'] as num?)?.toInt();
+ final recTs = _num(row['rec_ts'])?.toInt();
if (recTs == null || recTs <= 0) continue;
tsSec.add(recTs);
- hr.add((row['hr'] as num?)?.toInt() ?? 0);
- ax.add((row['ax'] as num?)?.toDouble() ?? 0);
- ay.add((row['ay'] as num?)?.toDouble() ?? 0);
- az.add((row['az'] as num?)?.toDouble() ?? 0);
- spo2Red.add((row['spo2_red_raw'] as num?)?.toInt() ?? 0);
- spo2Ir.add((row['spo2_ir_raw'] as num?)?.toInt() ?? 0);
- skinTemp.add((row['skin_temp_raw'] as num?)?.toInt() ?? 0);
- final counter = (row['counter'] as num?)?.toInt();
+ hr.add(_num(row['hr'])?.toInt() ?? 0);
+ ax.add(_num(row['ax'])?.toDouble() ?? 0);
+ ay.add(_num(row['ay'])?.toDouble() ?? 0);
+ az.add(_num(row['az'])?.toDouble() ?? 0);
+ spo2Red.add(_num(row['spo2_red_raw'])?.toInt() ?? 0);
+ spo2Ir.add(_num(row['spo2_ir_raw'])?.toInt() ?? 0);
+ skinTemp.add(_num(row['skin_temp_raw'])?.toInt() ?? 0);
+ // `decoded_onehz` has no skin-contact column, so the live decoded path
+ // genuinely has no contact signal to offer; a page that DOES carry one
+ // (the raw-decode fallback) is honoured. Keeping the array 1:1 with
+ // tsSec is what lets `Substrate.fromJson` tell "absent" (empty ⇒
+ // zero-filled) from "present but zero".
+ skinContact.add(_num(row['skin_contact'])?.toInt() ?? 0);
+ final counter = _num(row['counter'])?.toInt();
if (counter == null) continue;
final beats = rrByCounter[counter];
if (beats == null) continue;
for (final beat in beats) {
- final rr = (beat['rr_ms'] as num?)?.toDouble();
+ final rr = _num(beat['rr_ms'])?.toDouble();
if (rr == null || rr <= 0) continue;
- rrTsMs.add((beat['rr_ts_ms'] as num?)?.toDouble() ?? recTs * 1000.0);
+ rrTsMs.add(_num(beat['rr_ts_ms'])?.toDouble() ?? recTs * 1000.0);
rrMs.add(rr);
}
}
@@ -388,6 +438,6 @@ class _PrepareAccumulator {
spo2Red: spo2Red,
spo2Ir: spo2Ir,
skinTemp: skinTemp,
- skinContact: const [],
+ skinContact: skinContact,
);
}
diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart
index 5ce9240d..5c1afb51 100644
--- a/lib/compute/onehz_pipeline.dart
+++ b/lib/compute/onehz_pipeline.dart
@@ -294,8 +294,19 @@ Map deriveDayBundle(Map inputJson) {
tier: Tier.high,
inputs_used: ['rr_cleaned'],
);
- // Nocturnal RHR over the SLEEP HR (fallback to day-valid only if no sleep HR).
- final rhr = nocturnalRhr(sleepHr.isNotEmpty ? sleepHr : dayHrValid);
+ // Nocturnal RHR over the SLEEP HR (fallback to the DAY series only if there
+ // is no sleep HR at all).
+ //
+ // Both arguments must be POSITIONALLY DENSE 1 Hz series where 0 means
+ // off-skin — `nocturnalRhr` slides its 30-minute window over wall-clock
+ // POSITIONS and enforces a minimum on-skin coverage per window. Passing the
+ // compacted `dayHrValid` here defeated that: with gaps squeezed out, 1800
+ // consecutive entries could span many hours, so "lowest 30-minute mean"
+ // silently became "lowest mean over whatever 1800 samples happened to
+ // survive". `dayHr` keeps its zeros, so a day too sparse to contain a real
+ // contiguous window now abstains instead of reporting a stitched-together
+ // trough.
+ final rhr = nocturnalRhr(sleepHr.isNotEmpty ? sleepHr : dayHr);
// HR dip: day-side = waking HR outside the sleep window; night-side = sleep HR.
final dayOnly = _dayHrOutsideSleep(d);
final dip = hrDip(dayOnly, sleepHr);
@@ -384,7 +395,7 @@ Map deriveDayBundle(Map inputJson) {
? math.log(sleepSessionRmssd)
: null;
// Readiness's RHR input must come from an ACTUAL detected sleep session.
- // `rhr` above intentionally falls back to daytime HR (`dayHrValid`) for the
+ // `rhr` above intentionally falls back to daytime HR (`dayHr`) for the
// general-purpose "resting HR" display card, but feeding that fallback into
// readiness let a handful of minutes of live daytime HR masquerade as an
// overnight resting rate — the sole reason a same-day score of 100 could
@@ -444,7 +455,16 @@ Map deriveDayBundle(Map inputJson) {
'note': composite.note,
};
}
- // Plews lnRMSSD readiness over the trailing history INCLUDING today.
+ // Plews lnRMSSD readiness. `readinessLnRmssd` is contractually handed the
+ // trailing history with TONIGHT AS THE LAST ELEMENT and takes strictly the
+ // prior elements as its baseline (analytics v38). So appending `lnToday`
+ // exactly once is right — PROVIDED `d.lnRmssdHistory` holds only days BEFORE
+ // this one. It didn't: the engine filled it from an unfiltered trailing
+ // `metric_series` window that already contained the row a previous derive of
+ // THIS day wrote, so today was in its own baseline AND counted a second time
+ // by this append. The engine now self-excludes the target date
+ // (`_attachHistory` → `_BaselineHistoryCache.valuesBefore`), which is what
+ // makes this single append the correct, non-duplicating one.
final lnHist = [...d.lnRmssdHistory, ?lnToday];
final lnReadiness = lnHist.length >= 4
? readinessLnRmssd(lnHist)
diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart
index c497bd80..40df5050 100644
--- a/lib/compute/substrate.dart
+++ b/lib/compute/substrate.dart
@@ -388,6 +388,17 @@ class PhysioDay {
bool get hasSleep => sleep.present;
}
+/// How far BEFORE a calendar day's local midnight [calendarDays] searches for
+/// the main sleep that ends in that day — the previous local NOON.
+///
+/// Exported because the coordinator has to LOAD at least this much substrate
+/// before the day start, or `searchStart = math.max(dataStart, …)` silently
+/// clips the window back to wherever the loaded slice happens to begin and any
+/// sleep onset before that instant is truncated (see
+/// `DerivationEngine._targetDayWindow`). One constant, two call sites — they
+/// cannot drift apart again.
+const int kNocturnalSearchLookbackSec = 12 * 3600;
+
/// A user-asserted sleep window for one day — manual entry (Approach 1) or a
/// confirmation of the HR-led fallback (Approach 2). Passed into [calendarDays]
/// so it overrides auto detection for the matching [dayId].
@@ -426,7 +437,18 @@ String localDateLabel(int epochSec) {
/// A sleep that crosses midnight is attributed to the day it ENDS; its window
/// indices (sleepLoIdx/Hi) point into the full substrate, so the coordinator
/// still slices the whole window for HRV/RHR/recovery regardless of the boundary.
-List calendarDays(Substrate sub, {SleepWindowOverride? override}) {
+///
+/// [tzOffsetAt] resolves the local UTC offset in effect at an epoch second;
+/// it defaults to [tzOffsetSecondsAt] and exists so tests can pin that the
+/// habitual-midsleep prior is resolved AT THE DAY BEING SEGMENTED rather than
+/// at "now" (a zone-independent way to test the DST/travel fix, since the
+/// machine running the test may sit in a zone that never changes offset).
+List calendarDays(
+ Substrate sub, {
+ SleepWindowOverride? override,
+ int Function(int epochSec)? tzOffsetAt,
+}) {
+ final tzOffset = tzOffsetAt ?? tzOffsetSecondsAt;
if (sub.isEmpty) return const [];
final accel = sub.accelSamples();
final hr = sub.hr1hz();
@@ -452,7 +474,7 @@ List calendarDays(Substrate sub, {SleepWindowOverride? override}) {
// prev-18:00 → noon window missed late wakes and forced the detector to act
// like there was only one candidate sleep. The richer selector needs the
// full set of sessions that can legitimately end today.
- final searchStart = math.max(dataStart, dayStart - 12 * 3600);
+ final searchStart = math.max(dataStart, dayStart - kNocturnalSearchLookbackSec);
final searchEnd = math.min(dataEnd, dayEnd);
final loS = _lowerBound(sub.tsSec, searchStart);
final hiS = _lowerBound(sub.tsSec, searchEnd);
@@ -465,9 +487,17 @@ List calendarDays(Substrate sub, {SleepWindowOverride? override}) {
final ov =
(override != null && override.dayId == dayLabel) ? override : null;
if (hiS - loS >= 600 || ov != null) {
+ // The habitual-midsleep prior converts each HISTORICAL sleep block's epoch
+ // seconds to a local time-of-day. Using `DateTime.now()`'s offset applied
+ // the CURRENT UTC offset to those historical instants, so re-deriving days
+ // from the other side of a DST transition (or a trip) shifted every
+ // historical midsleep by an hour — which can change which candidate sleep
+ // the selector's alignment bonus picks. Resolve the offset AT THE DAY
+ // BEING SEGMENTED instead of "whenever this code happens to run", so a
+ // re-derive of an old day is reproducible regardless of today's zone.
final habitualMidsleepSec = ana.habitualMidsleepSecFromHistory(
sleepHistory,
- tzOffsetSeconds: DateTime.now().timeZoneOffset.inSeconds,
+ tzOffsetSeconds: tzOffset(dayStart),
);
// Daytime HR baseline = valid HR before the nocturnal search window.
final base = [for (var i = 0; i < loS; i++) if (hr[i] > 0) hr[i]];
@@ -581,6 +611,19 @@ List calendarDays(Substrate sub, {SleepWindowOverride? override}) {
return days;
}
+/// The LOCAL UTC offset in effect AT [epochSec] — not the offset in effect now.
+///
+/// `DateTime.now().timeZoneOffset` answers "what is the offset today", which is
+/// the wrong question for any historical instant: a day derived from the other
+/// side of a DST transition (or from before a trip) sits at a different offset,
+/// and using today's would shift that day's local clock-times by up to an hour.
+/// `DateTime.fromMillisecondsSinceEpoch(..., isUtc: false).timeZoneOffset` asks
+/// the platform zone database for the offset that actually applied then.
+int tzOffsetSecondsAt(int epochSec) =>
+ DateTime.fromMillisecondsSinceEpoch(epochSec * 1000, isUtc: false)
+ .timeZoneOffset
+ .inSeconds;
+
/// Local midnight (epoch sec) at/before [epochSec].
int _localMidnight(int epochSec) {
final d = DateTime.fromMillisecondsSinceEpoch(epochSec * 1000, isUtc: false);
diff --git a/lib/data/day_label.dart b/lib/data/day_label.dart
index 115156cc..5695c2b5 100644
--- a/lib/data/day_label.dart
+++ b/lib/data/day_label.dart
@@ -22,3 +22,49 @@ String dayLabelOf(DateTime dt) {
/// Today's LOCAL day label — the key `LocalDb`/the derivation engine file days
/// under. [now] is injectable for tests (defaults to the real clock).
String todayLabel([DateTime? now]) => dayLabelOf(now ?? DateTime.now());
+
+/// Parse a 'YYYY-MM-DD' label into its (year, month, day) parts, or null if it
+/// isn't one. Deliberately strict — a malformed label must not silently become
+/// epoch 0 and delete/export the wrong window.
+List? _partsOf(String dayId) {
+ final p = dayId.split('-');
+ if (p.length != 3) return null;
+ final y = int.tryParse(p[0]);
+ final m = int.tryParse(p[1]);
+ final d = int.tryParse(p[2]);
+ if (y == null || m == null || d == null) return null;
+ return [y, m, d];
+}
+
+/// Epoch SECONDS of the LOCAL midnight that STARTS day [dayId] ('YYYY-MM-DD').
+/// Returns null for a malformed label.
+int? localDayStartSec(String dayId) {
+ final p = _partsOf(dayId);
+ if (p == null) return null;
+ return DateTime(p[0], p[1], p[2]).millisecondsSinceEpoch ~/ 1000;
+}
+
+/// Epoch SECONDS of the LOCAL midnight that ENDS day [dayId] — i.e. the start
+/// of the NEXT local calendar day. Returns null for a malformed label.
+///
+/// This is deliberately NOT `localDayStartSec(dayId) + 86400`. A local calendar
+/// day is only 86 400 s when there is no DST transition inside it: a
+/// spring-forward day is 23 h and a fall-back day is 25 h. Using `+86400` made
+/// day-window deletes/exports overrun into the NEXT day's first hour on
+/// spring-forward, and leave the last hour behind on fall-back. `DateTime`'s
+/// local constructor normalizes day/month/year rollover AND the wall-clock
+/// offset, so "midnight of the next calendar date" is the only correct end.
+int? localDayEndSec(String dayId) {
+ final p = _partsOf(dayId);
+ if (p == null) return null;
+ return DateTime(p[0], p[1], p[2] + 1).millisecondsSinceEpoch ~/ 1000;
+}
+
+/// True local length of day [dayId] in seconds (86400 / 82800 / 90000 …), or
+/// null for a malformed label.
+int? localDayLengthSec(String dayId) {
+ final lo = localDayStartSec(dayId);
+ final hi = localDayEndSec(dayId);
+ if (lo == null || hi == null) return null;
+ return hi - lo;
+}
diff --git a/lib/data/db.dart b/lib/data/db.dart
index 0c4d71b5..e00b63c1 100644
--- a/lib/data/db.dart
+++ b/lib/data/db.dart
@@ -18,6 +18,7 @@ import 'package:path/path.dart' as p;
import 'package:path_provider/path_provider.dart';
import 'package:sqflite/sqflite.dart';
import 'day_label.dart';
+import 'live_coverage_policy.dart';
import 'models.dart';
class LocalDb {
@@ -85,7 +86,28 @@ class LocalDb {
return null;
}
- static const int _daySec = 86400;
+ /// The live schema version — the ONE place it is declared. Every
+ /// `openDatabase` this class performs (the app DB and the day-export DB) must
+ /// pass it: sqflite throws `ArgumentError('onCreate must be null if no
+ /// version is specified')` BEFORE opening anything when `onCreate` is given
+ /// without `version` (sqflite_common database_mixin.dart).
+ static const int schemaVersion = 26;
+
+ /// SQLite caps host parameters per statement (`SQLITE_MAX_VARIABLE_NUMBER` —
+ /// only 999 on the builds shipped with older Android/iOS). Any `IN (?, ?, …)`
+ /// built from row data MUST be chunked below this; a single day of
+ /// `decoded_onehz` is 86 400 counters. Same reason `commitSyncBatch` chunks.
+ static const int _maxSqlVars = 500;
+
+ /// Split [items] into `_maxSqlVars`-sized chunks for `IN (…)` binding.
+ static Iterable> _sqlVarChunks(List items) sync* {
+ for (var i = 0; i < items.length; i += _maxSqlVars) {
+ yield items.sublist(
+ i,
+ i + _maxSqlVars > items.length ? items.length : i + _maxSqlVars,
+ );
+ }
+ }
static Future _open() async {
final dir = await getDatabasesPath();
@@ -178,6 +200,12 @@ class LocalDb {
// rec_ts (not captured_at), so a multi-day flash backfill received in one
// sync no longer collapses into a single "today" bucket. Additive + safe
// on a populated DB.
+ // GUARDED add: an oldV <= 2 DB already had raw_records DROPped and
+ // re-created by the step-3 block above using the CURRENT `_createRaw`
+ // DDL — which already carries rec_ts. A bare ALTER … ADD COLUMN then
+ // threw "duplicate column name: rec_ts", and because onUpgrade runs
+ // inside ONE exclusive transaction the whole ladder rolled back and
+ // openDatabase rethrew → app stuck on the loading screen, forever.
await _addRecTsColumn(db);
await _backfillRecTs(db);
await db.execute(
@@ -244,7 +272,14 @@ class LocalDb {
await _backfillDecodedStore(db);
// Live workout steps (Tier-A pedometer over the session's 100 Hz
// R10 accel). Additive nullable column — old rows read null.
- await db.execute('ALTER TABLE sessions ADD COLUMN steps INTEGER');
+ //
+ // GUARDED add: an oldV <= 6 DB gets `sessions` from the step-7
+ // `_createUserTables` block above, which uses the CURRENT DDL — and
+ // that already declares `steps`. A bare ALTER … ADD COLUMN then threw
+ // "duplicate column name: steps", rolling back the whole (single,
+ // exclusive) onUpgrade transaction so openDatabase rethrew → app
+ // permanently stuck on the loading screen.
+ await _addColumnIfMissing(db, 'sessions', 'steps', 'INTEGER');
}
if (oldV < 12) {
// PURGE the old 1 Hz step ESTIMATE. 1 Hz can't count steps (Nyquist),
@@ -416,34 +451,56 @@ class LocalDb {
await _dropRawStore(db);
}
- static Future _ensureDayResultSkippedColumn(Database db) async {
- final info = await db.rawQuery('PRAGMA table_info(day_result)');
- final has = info.any((c) => c['name'] == 'skipped');
- if (!has) {
- try {
- await db.execute(
- 'ALTER TABLE day_result ADD COLUMN skipped INTEGER NOT NULL DEFAULT 0',
- );
- } catch (_) {
- /* another opener won the race — column now exists */
- }
- }
+ /// The column names [table] currently has (empty if the table is absent).
+ static Future> _columnsOf(Database db, String table) async {
+ final info = await db.rawQuery('PRAGMA table_info($table)');
+ return {
+ for (final c in info)
+ if (c['name'] is String) c['name'] as String,
+ };
}
- static Future _ensureDayResultPartialColumn(Database db) async {
- final info = await db.rawQuery('PRAGMA table_info(day_result)');
- final has = info.any((c) => c['name'] == 'partial');
- if (!has) {
- try {
- await db.execute(
- 'ALTER TABLE day_result ADD COLUMN partial INTEGER NOT NULL DEFAULT 0',
- );
- } catch (_) {
- /* another opener won the race — column now exists */
- }
+ /// THE ONLY sanctioned way to add a column in a migration step.
+ ///
+ /// A bare `ALTER TABLE … ADD COLUMN` in the ladder is a latent brick: the
+ /// `_create*` helpers are MODERNIZED IN PLACE (they always emit the current
+ /// DDL), so any DB old enough to have a table created by a LATER-numbered
+ /// step already has the column an EARLIER-numbered ALTER tries to add. That
+ /// throws "duplicate column name", and since `onUpgrade` runs inside ONE
+ /// exclusive transaction the entire ladder rolls back and `openDatabase`
+ /// rethrows — the app is stuck on the loading screen on EVERY launch, with no
+ /// way out. Check first; swallow a lost race (SQLite does statement-level
+ /// rollback, so a caught failure never poisons the surrounding transaction).
+ static Future _addColumnIfMissing(
+ Database db,
+ String table,
+ String column,
+ String ddlType,
+ ) async {
+ if ((await _columnsOf(db, table)).contains(column)) return;
+ try {
+ await db.execute('ALTER TABLE $table ADD COLUMN $column $ddlType');
+ } catch (_) {
+ /* another opener won the race — the column exists now */
}
}
+ static Future _ensureDayResultSkippedColumn(Database db) =>
+ _addColumnIfMissing(
+ db,
+ 'day_result',
+ 'skipped',
+ 'INTEGER NOT NULL DEFAULT 0',
+ );
+
+ static Future _ensureDayResultPartialColumn(Database db) =>
+ _addColumnIfMissing(
+ db,
+ 'day_result',
+ 'partial',
+ 'INTEGER NOT NULL DEFAULT 0',
+ );
+
// ── MENSTRUAL SYMPTOM LOG ──────────────────────────────────────────────────
static Future _createCycleSymptom(Database db) async {
await db.execute('''
@@ -491,7 +548,7 @@ class LocalDb {
// The "safe-trim invariant" is: persist decoded+raw → persist this cursor →
// ACK with-response. The band only trims its flash once the ACK is link-layer
// confirmed, so a crash anywhere before the ACK re-delivers the batch.
- static Future _createSyncCursor(Database db) async {
+ static Future _createSyncCursor(DatabaseExecutor db) async {
await db.execute('''
CREATE TABLE IF NOT EXISTS sync_cursor (
name TEXT PRIMARY KEY,
@@ -680,6 +737,17 @@ class LocalDb {
// Times are device epoch SECONDS (same clock as raw_records.rec_ts, since the
// band's RTC is SET_CLOCK'd to phone time on connect). `day` = local date label
// of the window start (for per-day step attribution).
+ //
+ // HISTORICAL ROWS. Databases written before the window derivation was fixed
+ // contain ZERO-WIDTH rows (`end_ts == start_ts`) — the old writer took both
+ // ends from a band record timestamp that does not advance during a live
+ // session. They are left as they are: their real durations were never
+ // recorded, and widening them after the fact would replace one wrong extent
+ // with another while silently changing already-derived days. Readers must
+ // tolerate them — `coverageWindowsOverlapping` matches them (`end_ts >= lo`)
+ // and the derivation's minute test (`s + 60 > start && s < end`) still
+ // excludes the minute containing the row, so such a row under-excludes but
+ // never crashes or double-adds its steps.
static Future _createLiveCoverage(Database db) async {
await db.execute('''
CREATE TABLE IF NOT EXISTS live_coverage (
@@ -696,17 +764,27 @@ class LocalDb {
}
/// Record a real 100 Hz step window (device-time seconds) + its step count.
+ ///
+ /// The window is normalised by [sanitizeCoverageWindow] first: a zero-width
+ /// window that claims steps is REPAIRED (widened to the duration those steps
+ /// physically imply) rather than dropped, because dropping it would lose a
+ /// real 100 Hz count; an inverted window is rejected. See that function for
+ /// the reasoning. This is a guard, not the derivation — the caller is
+ /// expected to have measured a real window (see
+ /// `deriveLiveCoverageWindow`); it exists so an upstream regression cannot
+ /// silently reintroduce degenerate rows.
static Future addLiveCoverage(
int startTs,
int endTs,
int steps,
String day,
) async {
- if (steps <= 0 || endTs < startTs) return;
+ final w = sanitizeCoverageWindow(startTs, endTs, steps);
+ if (w == null) return;
final db = await instance;
await db.insert('live_coverage', {
- 'start_ts': startTs,
- 'end_ts': endTs,
+ 'start_ts': w.startTs,
+ 'end_ts': w.endTs,
'steps': steps,
'day': day,
});
@@ -1066,7 +1144,7 @@ class LocalDb {
''');
}
- static Future _createSyncState(Database db) async {
+ static Future _createSyncState(DatabaseExecutor db) async {
await db.execute('''
CREATE TABLE IF NOT EXISTS sync_ledger (
chunk_id TEXT PRIMARY KEY,
@@ -1156,17 +1234,8 @@ class LocalDb {
}
static Future _ensureSessionSchema(Database db) async {
- final cols = await db.rawQuery("PRAGMA table_info(sessions)");
- final names = {
- for (final c in cols)
- if (c['name'] is String) c['name'] as String,
- };
- if (!names.contains('steps')) {
- await db.execute('ALTER TABLE sessions ADD COLUMN steps INTEGER');
- }
- if (!names.contains('hrr_bpm')) {
- await db.execute('ALTER TABLE sessions ADD COLUMN hrr_bpm REAL');
- }
+ await _addColumnIfMissing(db, 'sessions', 'steps', 'INTEGER');
+ await _addColumnIfMissing(db, 'sessions', 'hrr_bpm', 'REAL');
}
// ── WORKOUT SUGGESTIONS (opt-in auto-detect) ───────────────────────────────
@@ -1346,125 +1415,171 @@ class LocalDb {
''');
}
- static Future _ensureSyncCursorSchema(Database db) async {
- final tables = await db.rawQuery(
- "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'sync_cursor'",
- );
- if (tables.isEmpty) {
- await _createSyncCursor(db);
- return;
- }
- final cols = await db.rawQuery("PRAGMA table_info(sync_cursor)");
- final names = {
- for (final c in cols)
- if (c['name'] is String) c['name'] as String,
- };
- if (names.contains('name') &&
- names.contains('value') &&
- names.contains('updated_at')) {
- return;
- }
-
- await db.execute('ALTER TABLE sync_cursor RENAME TO sync_cursor_legacy');
- await _createSyncCursor(db);
- final legacyRows = await db.query('sync_cursor_legacy');
- final now = DateTime.now().millisecondsSinceEpoch;
- for (final row in legacyRows) {
- final name = row['name'] as String?;
- if (name == null || name.isEmpty) continue;
- await db.insert('sync_cursor', {
- 'name': name,
- 'value': row['value']?.toString(),
- 'updated_at': (row['updated_at'] as num?)?.toInt() ?? now,
- }, conflictAlgorithm: ConflictAlgorithm.replace);
- }
- await db.execute('DROP TABLE sync_cursor_legacy');
- }
-
- static Future _ensureSyncLedgerSchema(Database db) async {
- final tables = await db.rawQuery(
- "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'sync_ledger'",
- );
- if (tables.isEmpty) {
- await _createSyncState(db);
- return;
- }
- final cols = await db.rawQuery("PRAGMA table_info(sync_ledger)");
- final names = {
- for (final c in cols)
- if (c['name'] is String) c['name'] as String,
- };
- if (names.contains('chunk_id')) return;
+ /// Run a rename → recreate → copy legacy-shape migration ATOMICALLY and
+ /// IDEMPOTENTLY.
+ ///
+ /// The three callers below used to do `ALTER … RENAME`, then `CREATE`, then a
+ /// row-by-row copy, then `DROP` — all OUTSIDE any transaction. That is fine
+ /// under `onUpgrade` (sqflite wraps the whole ladder in one exclusive txn) but
+ /// these also run from `_repairOpenSchema` in `onOpen`, which is NOT wrapped.
+ /// A crash / OS kill between the rename and the end of the copy left
+ /// `` present AND correctly shaped, so the next open hit the
+ /// "already current" early-return and `_legacy` sat there orphaned with
+ /// its rows never migrated — losing `strap_trim` / `counter_hw` / `rec_ts_hw`,
+ /// i.e. the whole resumable-sync cursor and the safe-trim high-water.
+ ///
+ /// Now: one transaction — which sqflite JOINS to the already-open `onUpgrade`
+ /// transaction when called from the ladder, and opens for real from `onOpen`
+ /// — plus an explicit RESUME of an orphan left behind by any older build.
+ ///
+ /// [isCurrent] decides whether an existing `` is already the new shape.
+ /// [copy] must be idempotent (INSERT OR REPLACE on a natural key); set
+ /// [copyOnlyIntoEmpty] for a destination with no natural key (autoincrement
+ /// id), where re-running a resume would otherwise duplicate rows.
+ static Future _migrateLegacyTable(
+ Database db, {
+ required String table,
+ required bool Function(Set columns) isCurrent,
+ required Future Function(DatabaseExecutor ex) create,
+ required Future Function(
+ DatabaseExecutor ex,
+ List> legacyRows,
+ int nowMs,
+ )
+ copy,
+ bool copyOnlyIntoEmpty = false,
+ }) async {
+ final legacy = '${table}_legacy';
+ await db.transaction((txn) async {
+ Future exists(String t) async =>
+ (await txn.rawQuery(
+ "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
+ [t],
+ )).isNotEmpty;
+ Future> columnsOf(String t) async {
+ final info = await txn.rawQuery('PRAGMA table_info($t)');
+ return {
+ for (final c in info)
+ if (c['name'] is String) c['name'] as String,
+ };
+ }
- await db.execute('ALTER TABLE sync_ledger RENAME TO sync_ledger_legacy');
- await _createSyncState(db);
- final legacyRows = await db.query('sync_ledger_legacy');
- final now = DateTime.now().millisecondsSinceEpoch;
- for (final row in legacyRows) {
- final meta = {
- 'last_batch_token': row['last_batch_token'],
- 'last_batch_id': row['last_batch_id'],
- 'last_batch_records': row['last_batch_records'],
- 'last_history_complete_at': row['last_history_complete_at'],
- 'last_trim_cutoff_ms': row['last_trim_cutoff_ms'],
- 'last_trimmed_at': row['last_trimmed_at'],
- if (row['note'] != null) 'legacy_note': row['note'],
- };
- await db.insert('sync_ledger', {
- 'chunk_id': (row['id'] as String?) ?? 'capture',
- 'kind': 'historical',
- 'status': row['last_history_complete_at'] != null
- ? 'complete'
- : row['last_batch_acked_at'] != null
- ? 'acknowledged'
- : 'legacy',
- 'created_at': (row['updated_at'] as num?)?.toInt() ?? now,
- 'updated_at': (row['updated_at'] as num?)?.toInt() ?? now,
- 'acked_at': (row['last_batch_acked_at'] as num?)?.toInt(),
- 'last_error': null,
- 'meta_json': jsonEncode(meta),
- }, conflictAlgorithm: ConflictAlgorithm.replace);
- }
- await db.execute('DROP TABLE sync_ledger_legacy');
+ if (!await exists(legacy)) {
+ // Normal path.
+ if (!await exists(table)) {
+ await create(txn);
+ return;
+ }
+ if (isCurrent(await columnsOf(table))) return;
+ await txn.execute('ALTER TABLE $table RENAME TO $legacy');
+ }
+ // From here on `_legacy` holds the rows of record. `` is
+ // either absent (crash between RENAME and CREATE) or the new shape
+ // (possibly half-copied, or fully copied by an older build that then
+ // died before the DROP) — create it if needed and re-copy; the copy is
+ // idempotent, so a repeat is a no-op rather than a duplication.
+ if (!await exists(table)) await create(txn);
+ final skipCopy =
+ copyOnlyIntoEmpty &&
+ (Sqflite.firstIntValue(
+ await txn.rawQuery('SELECT COUNT(*) FROM $table'),
+ ) ??
+ 0) >
+ 0;
+ if (!skipCopy) {
+ await copy(
+ txn,
+ await txn.query(legacy),
+ DateTime.now().millisecondsSinceEpoch,
+ );
+ }
+ await txn.execute('DROP TABLE $legacy');
+ });
}
- static Future _ensureSyncQuarantineSchema(Database db) async {
- final tables = await db.rawQuery(
- "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'sync_quarantine'",
- );
- if (tables.isEmpty) {
- await _createSyncState(db);
- return;
- }
- final cols = await db.rawQuery("PRAGMA table_info(sync_quarantine)");
- final names = {
- for (final c in cols)
- if (c['name'] is String) c['name'] as String,
- };
- if (names.contains('payload_json')) return;
+ static Future _ensureSyncCursorSchema(Database db) =>
+ _migrateLegacyTable(
+ db,
+ table: 'sync_cursor',
+ isCurrent: (c) =>
+ c.contains('name') &&
+ c.contains('value') &&
+ c.contains('updated_at'),
+ create: _createSyncCursor,
+ copy: (ex, legacyRows, now) async {
+ for (final row in legacyRows) {
+ final name = row['name'] as String?;
+ if (name == null || name.isEmpty) continue;
+ await ex.insert('sync_cursor', {
+ 'name': name,
+ 'value': row['value']?.toString(),
+ 'updated_at': (row['updated_at'] as num?)?.toInt() ?? now,
+ }, conflictAlgorithm: ConflictAlgorithm.replace);
+ }
+ },
+ );
- await db.execute(
- 'ALTER TABLE sync_quarantine RENAME TO sync_quarantine_legacy',
- );
- await _createSyncState(db);
- final legacyRows = await db.query('sync_quarantine_legacy');
- final now = DateTime.now().millisecondsSinceEpoch;
- for (final row in legacyRows) {
- await db.insert('sync_quarantine', {
- 'kind': (row['source_role'] as String?) ?? 'legacy',
- 'payload_json': jsonEncode({
- 'fingerprint': row['fingerprint'],
- 'packet_type': row['packet_type'],
- 'hex': row['hex'],
- 'counter': row['counter'],
- 'captured_at': row['captured_at'],
- }),
- 'reason': (row['reason'] as String?) ?? 'legacy_migrated',
- 'created_at': (row['created_at'] as num?)?.toInt() ?? now,
- });
- }
- await db.execute('DROP TABLE sync_quarantine_legacy');
- }
+ static Future _ensureSyncLedgerSchema(Database db) => _migrateLegacyTable(
+ db,
+ table: 'sync_ledger',
+ isCurrent: (c) => c.contains('chunk_id'),
+ create: _createSyncState,
+ copy: (ex, legacyRows, now) async {
+ for (final row in legacyRows) {
+ final meta = {
+ 'last_batch_token': row['last_batch_token'],
+ 'last_batch_id': row['last_batch_id'],
+ 'last_batch_records': row['last_batch_records'],
+ 'last_history_complete_at': row['last_history_complete_at'],
+ 'last_trim_cutoff_ms': row['last_trim_cutoff_ms'],
+ 'last_trimmed_at': row['last_trimmed_at'],
+ if (row['note'] != null) 'legacy_note': row['note'],
+ };
+ await ex.insert('sync_ledger', {
+ 'chunk_id': (row['id'] as String?) ?? 'capture',
+ 'kind': 'historical',
+ 'status': row['last_history_complete_at'] != null
+ ? 'complete'
+ : row['last_batch_acked_at'] != null
+ ? 'acknowledged'
+ : 'legacy',
+ 'created_at': (row['updated_at'] as num?)?.toInt() ?? now,
+ 'updated_at': (row['updated_at'] as num?)?.toInt() ?? now,
+ 'acked_at': (row['last_batch_acked_at'] as num?)?.toInt(),
+ 'last_error': null,
+ 'meta_json': jsonEncode(meta),
+ }, conflictAlgorithm: ConflictAlgorithm.replace);
+ }
+ },
+ );
+
+ static Future _ensureSyncQuarantineSchema(Database db) =>
+ _migrateLegacyTable(
+ db,
+ table: 'sync_quarantine',
+ isCurrent: (c) => c.contains('payload_json'),
+ create: _createSyncState,
+ // `id INTEGER PRIMARY KEY AUTOINCREMENT` — no natural key to REPLACE
+ // on, so a resume must not re-copy into a destination that already has
+ // rows or the quarantine log would double on every retry.
+ copyOnlyIntoEmpty: true,
+ copy: (ex, legacyRows, now) async {
+ for (final row in legacyRows) {
+ await ex.insert('sync_quarantine', {
+ 'kind': (row['source_role'] as String?) ?? 'legacy',
+ 'payload_json': jsonEncode({
+ 'fingerprint': row['fingerprint'],
+ 'packet_type': row['packet_type'],
+ 'hex': row['hex'],
+ 'counter': row['counter'],
+ 'captured_at': row['captured_at'],
+ }),
+ 'reason': (row['reason'] as String?) ?? 'legacy_migrated',
+ 'created_at': (row['created_at'] as num?)?.toInt() ?? now,
+ });
+ }
+ },
+ );
// samples — LEGACY header-only record index (counter, ts, hr). Retained only
// so pre-v11 databases stay readable if decoded_onehz backfill was partial.
@@ -1648,11 +1763,15 @@ class LocalDb {
/// Add the additive `rec_ts` column to an EXISTING raw_records table (upgrade
/// path only). NOT NULL with a DEFAULT 0 so legacy rows are well-formed until
/// the backfill rewrites them.
- static Future _addRecTsColumn(Database db) async {
- await db.execute(
- 'ALTER TABLE raw_records ADD COLUMN rec_ts INTEGER NOT NULL DEFAULT 0',
- );
- }
+ /// GUARDED (see [_addColumnIfMissing]): the step-3 rebuild already creates
+ /// raw_records from the CURRENT `_createRaw` DDL, which carries rec_ts — so on
+ /// an oldV <= 2 upgrade this column is already there.
+ static Future _addRecTsColumn(Database db) => _addColumnIfMissing(
+ db,
+ 'raw_records',
+ 'rec_ts',
+ 'INTEGER NOT NULL DEFAULT 0',
+ );
/// Backfill `rec_ts` for every existing raw row by decoding its hex once. Runs
/// inside the migration on a populated DB. Falls back to captured_at/1000 when a
@@ -1763,6 +1882,29 @@ class LocalDb {
}
}
+ /// THE orphan guard for an INSERT-OR-REPLACE into `decoded_onehz`.
+ ///
+ /// Queue this onto [batch] IMMEDIATELY BEFORE writing the row for [counter] @
+ /// [recTs] — every write path into `decoded_onehz` must go through it, or it
+ /// strands `decoded_rr` beats (see [_queueDecodedOneHz] for the full
+ /// derivation of both eviction cases). Returns the number of ops queued.
+ static int _queueOrphanGuard(
+ Batch batch, {
+ required int counter,
+ required int recTs,
+ }) {
+ batch.rawDelete(
+ 'DELETE FROM decoded_rr WHERE '
+ // (a) UNIQUE(rec_ts) eviction — the LOSER counter's beats.
+ 'counter IN '
+ '(SELECT counter FROM decoded_onehz WHERE rec_ts = ? AND counter != ?) '
+ // (b) counter-PK eviction — stale-timestamped beats under OUR counter.
+ 'OR (counter = ? AND rr_ts_ms != ?)',
+ [recTs, counter, counter, recTs * 1000],
+ );
+ return 1;
+ }
+
/// Queues the decoded_onehz + decoded_rr (+ orphan-guard delete) writes for
/// one raw onto [batch]. Returns the number of batch operations added, so a
/// caller committing a large offload can chunk the batch to bound the native
@@ -1788,11 +1930,20 @@ class LocalDb {
// REPLACE on UNIQUE(rr_ts_ms, beat_index) only overwrites overlapping beat
// indexes, so delete the evicted counter's beats explicitly, in the same
// batch/transaction (mirrors the v17 rebuild's decoded_onehz join).
- batch.rawDelete(
- 'DELETE FROM decoded_rr WHERE counter IN '
- '(SELECT counter FROM decoded_onehz WHERE rec_ts = ? AND counter != ?)',
- [recTs, raw.counter],
- );
+ //
+ // …AND the COUNTER-PK eviction, which the guard used to miss entirely.
+ // `decoded_onehz` is `counter INTEGER PRIMARY KEY` as well as
+ // UNIQUE(rec_ts), and (per the comment above) the strap's counter RESETS to
+ // ~0 on every reboot — so this same REPLACE also silently DELETES the row
+ // of an OLDER SECOND that happened to reuse this counter. That older
+ // second's beats live under OUR counter carrying ITS rr_ts_ms, and only the
+ // overlapping beat_indexes get overwritten below: any beat at an index past
+ // the new record's beat count SURVIVES, still stamped days earlier. Neither
+ // prune path can ever see it (the counter-join finds a fresh rec_ts; the
+ // orphan sweep finds the counter present), so a later page's RR series was
+ // polluted with beats from another day — silently wrecking RMSSD/HRV.
+ // Drop every beat under this counter that is not stamped with THIS second.
+ var ops = _queueOrphanGuard(batch, counter: raw.counter, recTs: recTs);
batch.insert('decoded_onehz', {
'counter': raw.counter,
'rec_ts': recTs,
@@ -1804,7 +1955,7 @@ class LocalDb {
'spo2_ir_raw': decoded.spo2IrRaw ?? 0,
'skin_temp_raw': decoded.skinTempRaw ?? 0,
}, conflictAlgorithm: ConflictAlgorithm.replace);
- var ops = 2; // rawDelete + decoded_onehz insert
+ ops++; // the decoded_onehz insert
for (var i = 0; i < decoded.rrIntervalsMs.length; i++) {
final rr = decoded.rrIntervalsMs[i];
if (rr <= 0) continue;
@@ -1861,6 +2012,12 @@ class LocalDb {
captured_at INTEGER NOT NULL
)
''');
+ // The PK is the frame hex, so a `ts` window (the timeline's day query, and
+ // the retention prune) was a full table scan. Cheap to build — `events` is
+ // pruned to the retention window.
+ await db.execute(
+ 'CREATE INDEX IF NOT EXISTS idx_events_ts ON events(ts)',
+ );
}
// band_events / band_battery — structured local history for device-state
@@ -1899,19 +2056,8 @@ class LocalDb {
/// Additive: add the `millivolts` column to an existing band_battery table.
/// Guarded — the column already exists on fresh installs (see _createBandSignals)
/// and ALTER … ADD COLUMN throws if it's already there.
- static Future _ensureBandBatteryMillivolts(Database db) async {
- final info = await db.rawQuery('PRAGMA table_info(band_battery)');
- final has = info.any((c) => c['name'] == 'millivolts');
- if (!has) {
- try {
- await db.execute(
- 'ALTER TABLE band_battery ADD COLUMN millivolts INTEGER',
- );
- } catch (_) {
- /* another opener won the race — column now exists */
- }
- }
- }
+ static Future _ensureBandBatteryMillivolts(Database db) =>
+ _addColumnIfMissing(db, 'band_battery', 'millivolts', 'INTEGER');
/// Durable archive for historical records we received but could not decode
/// (unknown/unsupported version). NEVER pruned — the whole point is that a
@@ -2119,6 +2265,11 @@ class LocalDb {
};
}
+ /// The OLDEST [limit] queued events — an upload-queue drain head, and ONLY
+ /// that. Never use it to answer "what happened on day X": once `events` holds
+ /// more than [limit] rows the page can't reach recent days at all (the same
+ /// oldest-N-vs-trailing-N shape as the `metricSeries(limit:)` outage). Use
+ /// [eventsInRange] for a day/window query.
static Future>> unuploadedEvents({
int limit = 500,
}) async {
@@ -2126,6 +2277,25 @@ class LocalDb {
return db.query('events', orderBy: 'ts ASC', limit: limit);
}
+ /// Events whose `ts` (epoch SECONDS) is in the half-open window
+ /// `[fromTs, toTs)`, oldest first. Bounded BY THE WINDOW, not by an unrelated
+ /// global page, so a day's markers can never be crowded out by older rows.
+ /// [limit] is a defensive cap on a pathological window only.
+ static Future>> eventsInRange(
+ int fromTs,
+ int toTs, {
+ int limit = 5000,
+ }) async {
+ final db = await instance;
+ return db.query(
+ 'events',
+ where: 'ts >= ? AND ts < ?',
+ whereArgs: [fromTs, toTs],
+ orderBy: 'ts ASC',
+ limit: limit,
+ );
+ }
+
static Future deleteEvents(List hexes) async {
if (hexes.isEmpty) return;
final db = await instance;
@@ -2410,18 +2580,56 @@ class LocalDb {
);
}
- /// Sparse RR beats for a contiguous decoded 1 Hz page, keyed by the owning
- /// frame counter and ordered by that frame.
+ /// Sparse RR beats for one contiguous decoded 1 Hz page.
+ ///
+ /// [fromCounter] / [toCounter] are the page's FIRST and LAST row counters, as
+ /// returned by [decodedOneHzBatchByRecTsRange] (which orders `rec_ts ASC,
+ /// counter ASC`). They are page ENDPOINTS, **not** a monotonic counter span:
+ /// the strap's counter resets to ~0 on every reboot, so a page straddling a
+ /// reboot has first = a pre-reboot high and last = a post-reboot low. The old
+ /// `WHERE counter >= ? AND counter <= ?` then read `>= 1200000 AND <= 5` and
+ /// returned ZERO rows — the entire page's RR beats vanished with no error, so
+ /// that window silently produced no RMSSD/HRV at all.
+ ///
+ /// Selection is therefore by the page's real TIME window, resolved from those
+ /// two endpoint counters. `decoded_onehz` is UNIQUE(rec_ts), so
+ /// `[first.rec_ts, last.rec_ts]` contains exactly the page's rows — no
+ /// over-fetch — and the join to `decoded_onehz` additionally keeps orphaned
+ /// beats (whose owning row was evicted) out of the read path.
+ ///
+ /// When the endpoints are NOT real rows the caller is asking for a plain
+ /// counter span (e.g. `0 .. 1<<30` = "everything"); that falls back to a
+ /// NORMALIZED counter range so an inverted pair still can't return nothing.
static Future>> decodedRrByCounterRange({
required int fromCounter,
required int toCounter,
}) async {
final db = await instance;
+ final bounds = (await db.rawQuery(
+ 'SELECT COUNT(*) AS n, MIN(rec_ts) AS lo, MAX(rec_ts) AS hi '
+ 'FROM decoded_onehz WHERE counter IN (?, ?)',
+ [fromCounter, toCounter],
+ )).first;
+ final n = (bounds['n'] as num?)?.toInt() ?? 0;
+ final want = fromCounter == toCounter ? 1 : 2;
+ if (n == want) {
+ return db.rawQuery(
+ 'SELECT rr.counter AS counter, rr.beat_index AS beat_index, '
+ ' rr.rr_ts_ms AS rr_ts_ms, rr.rr_ms AS rr_ms '
+ 'FROM decoded_rr rr '
+ 'JOIN decoded_onehz d ON d.counter = rr.counter '
+ 'WHERE d.rec_ts >= ? AND d.rec_ts <= ? '
+ 'ORDER BY d.rec_ts ASC, rr.beat_index ASC',
+ [bounds['lo'], bounds['hi']],
+ );
+ }
+ final lo = fromCounter <= toCounter ? fromCounter : toCounter;
+ final hi = fromCounter <= toCounter ? toCounter : fromCounter;
return db.query(
'decoded_rr',
columns: ['counter', 'beat_index', 'rr_ts_ms', 'rr_ms'],
where: 'counter >= ? AND counter <= ?',
- whereArgs: [fromCounter, toCounter],
+ whereArgs: [lo, hi],
orderBy: 'counter ASC, beat_index ASC',
);
}
@@ -2512,6 +2720,46 @@ class LocalDb {
return [for (final r in rows) _withDate(r)];
}
+ /// Every day_id that has a `day_result` row at its LATEST algo_version, newest
+ /// first — WITHOUT touching `payload_json`.
+ ///
+ /// `recentDayResults()` does `SELECT r.*`, which drags the whole bundle
+ /// (hr_curve / hypnogram / HRV series, tens of KB a day) across the isolate
+ /// boundary. Screens that only need "which days exist" must use this instead:
+ /// over a multi-year history the payload variant is hundreds of MB.
+ static Future> dayResultDayIdsDesc() async {
+ final db = await instance;
+ final rows = await db.rawQuery(
+ 'SELECT day_id FROM day_result GROUP BY day_id ORDER BY day_id DESC',
+ );
+ return [
+ for (final r in rows)
+ if (r['day_id'] is String) r['day_id'] as String,
+ ];
+ }
+
+ /// Day labels whose LATEST-version bundle records a real sleep total
+ /// (`sleep.accounting.value.tst_sec` present). Extracted IN SQLite via
+ /// json_extract, so only the scalar crosses the boundary — never the payload.
+ static Future> daysWithSleepTst() async {
+ final db = await instance;
+ final rows = await db.rawQuery(
+ 'SELECT r.day_id FROM day_result r '
+ 'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result GROUP BY day_id) m '
+ ' ON r.day_id = m.day_id AND r.algo_version = m.v '
+ // json_valid() first: json_extract() ERRORS on a malformed payload, and a
+ // corrupt bundle must degrade to "no sleep that day", never take out the
+ // whole Records screen.
+ 'WHERE json_valid(r.payload_json) '
+ "AND json_extract(r.payload_json, '\$.sleep.accounting.value.tst_sec') "
+ 'IS NOT NULL',
+ );
+ return {
+ for (final r in rows)
+ if (r['day_id'] is String) r['day_id'] as String,
+ };
+ }
+
/// Every day label ('YYYY-MM-DD') the lookback screen can actually RENDER —
/// exactly the days [dayResult]/`_bundleForDate` would return a real bundle
/// for, newest first. That is: the LATEST-`algo_version` `day_result` row per
@@ -2677,8 +2925,23 @@ class LocalDb {
return out;
}
- static int _localDayStartSec(String dayId) =>
- DateTime.parse(dayId).millisecondsSinceEpoch ~/ 1000;
+ /// The half-open LOCAL window `[startSec, endSec)` covering day [dayId].
+ ///
+ /// `endSec` is the NEXT local midnight, NOT `startSec + 86400`: a local
+ /// calendar day is 23 h on spring-forward and 25 h on fall-back. With the
+ /// flat +86400 this returned a window that overran into the next day's first
+ /// hour (deleteDays silently deleted the following day's first hour of
+ /// decoded_onehz / sessions / band_* / events) or fell an hour short
+ /// (fall-back left the last hour behind, and the export dropped it). Shared
+ /// with the UI/coach via day_label.dart so every layer agrees.
+ static (int, int) _localDayWindow(String dayId) {
+ final lo = localDayStartSec(dayId);
+ final hi = localDayEndSec(dayId);
+ if (lo == null || hi == null) {
+ throw ArgumentError.value(dayId, 'dayId', 'not a YYYY-MM-DD day label');
+ }
+ return (lo, hi);
+ }
static Future exportDaysDb(Set dayIds) async {
final sorted = dayIds.toList()..sort();
@@ -2692,6 +2955,11 @@ class LocalDb {
await deleteDatabase(dest);
final out = await openDatabase(
dest,
+ // `version:` is MANDATORY here. Without it sqflite throws
+ // ArgumentError('onCreate must be null if no version is specified')
+ // before opening anything — so this whole export path (Profile → Data
+ // history → Export) had never once produced a file.
+ version: schemaVersion,
onCreate: (db, _) async {
await _createSamples(db);
await _createDecodedStore(db);
@@ -2751,25 +3019,28 @@ class LocalDb {
for (final row in decoded)
if (row['counter'] != null) row['counter'],
];
- if (counters.isNotEmpty) {
- final placeholders = List.filled(counters.length, '?').join(',');
+ // CHUNKED `IN (…)`: a full day is 86 400 counters, two orders of
+ // magnitude past SQLITE_MAX_VARIABLE_NUMBER — one giant statement can
+ // never bind. (This never surfaced only because the missing `version:`
+ // above aborted the export earlier.)
+ for (final chunk in _sqlVarChunks(counters)) {
+ final placeholders = List.filled(chunk.length, '?').join(',');
final rr = await src.rawQuery(
'SELECT * FROM decoded_rr WHERE counter IN ($placeholders)',
- counters,
+ chunk,
);
- if (rr.isNotEmpty) {
- await out.transaction((txn) async {
- final batch = txn.batch();
- for (final row in rr) {
- batch.insert(
- 'decoded_rr',
- Map.from(row),
- conflictAlgorithm: ConflictAlgorithm.replace,
- );
- }
- await batch.commit(noResult: true);
- });
- }
+ if (rr.isEmpty) continue;
+ await out.transaction((txn) async {
+ final batch = txn.batch();
+ for (final row in rr) {
+ batch.insert(
+ 'decoded_rr',
+ Map.from(row),
+ conflictAlgorithm: ConflictAlgorithm.replace,
+ );
+ }
+ await batch.commit(noResult: true);
+ });
}
}
await copyRows(
@@ -2805,8 +3076,7 @@ class LocalDb {
}
for (final dayId in sorted) {
- final startSec = _localDayStartSec(dayId);
- final endSec = startSec + _daySec;
+ final (startSec, endSec) = _localDayWindow(dayId);
await copyRawRange(startSec, endSec);
await copyRows('day_result', where: 'day_id = ?', whereArgs: [dayId]);
await copyRows('metric_series', where: 'date = ?', whereArgs: [dayId]);
@@ -2839,17 +3109,20 @@ class LocalDb {
String column,
List values,
) async {
- final placeholders = List.filled(values.length, '?').join(',');
- deleted += await txn.rawDelete(
- 'DELETE FROM $table WHERE $column IN ($placeholders)',
- values,
- );
+ // Chunked: "select all" on a multi-year history binds one day label per
+ // parameter, which would blow SQLITE_MAX_VARIABLE_NUMBER.
+ for (final chunk in _sqlVarChunks(values)) {
+ final placeholders = List.filled(chunk.length, '?').join(',');
+ deleted += await txn.rawDelete(
+ 'DELETE FROM $table WHERE $column IN ($placeholders)',
+ chunk,
+ );
+ }
}
await db.transaction((txn) async {
for (final dayId in sorted) {
- final startSec = _localDayStartSec(dayId);
- final endSec = startSec + _daySec;
+ final (startSec, endSec) = _localDayWindow(dayId);
deleted += await txn.delete(
'decoded_rr',
where:
@@ -2881,6 +3154,16 @@ class LocalDb {
where: 'ts >= ? AND ts < ?',
whereArgs: [startSec, endSec],
);
+ // CASCADE the GPS route BEFORE its session row disappears — otherwise
+ // the join key is gone and every lat/lng point of a deleted run stays
+ // on disk forever (deleteSession cascades explicitly; this path did
+ // not). Must run first: once `sessions` is deleted the subquery is
+ // empty and the route is unreachable.
+ deleted += await txn.rawDelete(
+ 'DELETE FROM workout_route WHERE session_id IN '
+ '(SELECT id FROM sessions WHERE start_ts >= ? AND start_ts < ?)',
+ [startSec, endSec],
+ );
deleted += await txn.delete(
'sessions',
where: 'start_ts >= ? AND start_ts < ?',
@@ -2899,6 +3182,11 @@ class LocalDb {
await deleteByIn(txn, 'notifications', 'date', sorted);
await deleteByIn(txn, 'sleep_session_candidates', 'day_id', sorted);
await deleteByIn(txn, 'wake_day_features', 'day_id', sorted);
+ // Day-keyed USER rows. Same class of leak as workout_route: "delete this
+ // day" must not leave the user's own logged health data behind.
+ await deleteByIn(txn, 'cycle_symptom', 'date', sorted);
+ await deleteByIn(txn, 'workout_suggestions', 'date', sorted);
+ await deleteByIn(txn, 'sleep_override', 'day_id', sorted);
});
return deleted;
}
@@ -2976,7 +3264,20 @@ class LocalDb {
}
var copied = 0;
await db.transaction((txn) async {
- final batch = txn.batch();
+ // CHUNKED, for the same reason commitSyncBatch chunks: sqflite
+ // serialises a whole batch's args into ONE platform message. A
+ // full-history import is hundreds of thousands of rows, and the
+ // orphan guard below adds an op per decoded_onehz row on top.
+ const chunkOps = 4000;
+ var batch = txn.batch();
+ var ops = 0;
+ Future flush() async {
+ if (ops == 0) return;
+ await batch.commit(noResult: true);
+ batch = txn.batch();
+ ops = 0;
+ }
+
for (final r in rows) {
final row = {
for (final e in r.entries)
@@ -2989,10 +3290,24 @@ class LocalDb {
)) {
continue; // locally finalized — never overwritten by an import
}
+ // ORPHAN GUARD ON THE IMPORT PATH. A plain replace-insert into
+ // decoded_onehz bypasses _queueDecodedOneHz entirely, so a foreign
+ // row colliding on UNIQUE(rec_ts) (different counter) or on the
+ // `counter` PRIMARY KEY (different second) evicted a local row and
+ // stranded its decoded_rr beats — the exact leak the ingest path is
+ // guarded against, wide open here. Queue the SAME guard, in the
+ // same batch/transaction, right before the row.
+ if (t == 'decoded_onehz') {
+ final counter = (row['counter'] as num?)?.toInt();
+ final recTs = (row['rec_ts'] as num?)?.toInt();
+ if (counter == null || recTs == null) continue;
+ ops += _queueOrphanGuard(batch, counter: counter, recTs: recTs);
+ }
batch.insert(t, row, conflictAlgorithm: ConflictAlgorithm.replace);
copied++;
+ if (++ops >= chunkOps) await flush();
}
- await batch.commit(noResult: true);
+ await flush();
});
counts[t] = copied;
}
@@ -3862,17 +4177,8 @@ class LocalDb {
/// existing workout_route table. Guarded — fresh installs get it from
/// _createWorkoutRoute directly once that's updated; ALTER … ADD COLUMN
/// throws if it's already there.
- static Future _ensureWorkoutRouteSpeed(Database db) async {
- final info = await db.rawQuery('PRAGMA table_info(workout_route)');
- final has = info.any((c) => c['name'] == 'speed');
- if (!has) {
- try {
- await db.execute('ALTER TABLE workout_route ADD COLUMN speed REAL');
- } catch (_) {
- /* another opener won the race — column now exists */
- }
- }
- }
+ static Future _ensureWorkoutRouteSpeed(Database db) =>
+ _addColumnIfMissing(db, 'workout_route', 'speed', 'REAL');
/// Append a batch of route rows (INSERT OR REPLACE — idempotent on
/// (session_id, seq)). Each row is a [RoutePoint.toRow] map.
diff --git a/lib/data/live_coverage_policy.dart b/lib/data/live_coverage_policy.dart
new file mode 100644
index 00000000..6c90ab34
--- /dev/null
+++ b/lib/data/live_coverage_policy.dart
@@ -0,0 +1,164 @@
+// live_coverage_policy.dart — pure, I/O-free policy for the 100 Hz step
+// COVERAGE WINDOW. Nothing here touches BLE, the DB, Flutter or the clock:
+// callers hand in the facts they observed for one live-pedometer session and
+// read back the window to persist. Every branch is unit-testable.
+//
+// WHY THE WINDOW EXISTS
+// `live_coverage` rows say "over THIS wall period the live 100 Hz pedometer
+// counted real steps". The derivation pass adds those real steps to the day and
+// EXCLUDES the minutes the window covers from the 1 Hz estimate, so a minute is
+// counted by 100 Hz or estimated by 1 Hz — never both. A window is therefore a
+// measurement, not a label: get its extent wrong and either minutes get counted
+// twice (window too short) or real minutes lose their estimate (too long).
+//
+// WHICH FAILURE WE PREFER
+// Too short fabricates steps that never happened (the same minute counted by
+// both paths). Too long drops an estimate for minutes we may not have covered —
+// an UNDER-report of something we genuinely did not measure at 100 Hz. Under
+// the honesty contract the under-report wins, so when the evidence is
+// ambiguous these rules lean toward the WIDER window.
+
+import 'dart:math' as math;
+
+/// Sample rate of the live accel stream the pedometer runs on.
+const double kLiveSampleRateHz = 100.0;
+
+/// Fraction of the session's wall-clock hull that must actually be sampled
+/// before we treat the hull as the covered period.
+///
+/// The streamed time is a UNION of intervals; a `live_coverage` row can only
+/// store its convex HULL. At/above this duty cycle the union dominates its own
+/// hull (dropouts are the exception), so the hull is the better model of the
+/// counting period and, per the preference above, the safer one. Below it the
+/// session was mostly NOT streaming — a link that woke for a few seconds an
+/// hour — and claiming the hull would silently delete hours of 1 Hz estimate
+/// for a handful of real steps. There we fall back to the only quantity we
+/// actually measured: the sampled duration.
+const double kCoverageDutyFloor = 0.5;
+
+/// Cadence ceiling used to derive a LOWER BOUND on elapsed time from a step
+/// count. Elite sprint cadence tops out near 220 spm; 240 leaves headroom so
+/// the bound is never tighter than physiology allows. This is a bound, never an
+/// estimate: N steps cannot have happened in less than N / (240/60) seconds.
+const double kMaxPlausibleCadenceSpm = 240.0;
+
+/// A persisted 100 Hz coverage window, in the SAME epoch-second base as
+/// `decoded_onehz.rec_ts` (see [deriveLiveCoverageWindow]).
+class LiveCoverageWindow {
+ const LiveCoverageWindow(this.startTs, this.endTs);
+ final int startTs;
+ final int endTs;
+ int get seconds => endTs - startTs;
+
+ @override
+ String toString() => 'LiveCoverageWindow($startTs..$endTs, ${seconds}s)';
+}
+
+/// The shortest elapsed time [steps] could physically span, in whole seconds.
+/// 0 for a non-positive count. See [kMaxPlausibleCadenceSpm].
+int minCoverageSecondsForSteps(int steps) =>
+ steps <= 0 ? 0 : (steps * 60 / kMaxPlausibleCadenceSpm).ceil();
+
+/// Decide the coverage window for one live-pedometer session, or null when
+/// there is nothing defensible to record.
+///
+/// TIME BASE. The window is compared against `decoded_onehz.rec_ts` by
+/// `LocalDb.coverageWindowsOverlapping`, so it must stay in the BAND's record
+/// time base. That is why the window is ANCHORED on [bandStartTs] — the record
+/// timestamp carried by the first live frame — whenever the band supplied one,
+/// and only falls back to the phone clock ([firstIngestMs]) when it never did.
+/// The engine SET_CLOCKs the band to phone time on connect, so the two bases
+/// agree to within the drift it already corrects, but we do not rely on that:
+/// the ANCHOR comes from the band and only the DURATION comes from measurement,
+/// and a duration is the same number in either base.
+///
+/// DURATION. Three observations bound the covered period:
+/// * [samples100Hz] / [kLiveSampleRateHz] — the time we can PROVE we sampled
+/// (each 100 Hz sample is 10 ms of covered signal). A lower bound.
+/// * the wall-clock hull [firstIngestMs] … [lastIngestMs] — the outer extent
+/// of the counting period, dropouts included. An upper bound.
+/// * [bandStartTs] … [bandEndTs] — the same hull in band time, used only when
+/// the phone timestamps are missing, because in practice the band repeats
+/// one record timestamp for a whole live session (span 0) and cannot be
+/// trusted to carry the duration.
+/// The duty cycle between the first two picks which one models the session —
+/// see [kCoverageDutyFloor]. The result is then raised to
+/// [minCoverageSecondsForSteps] if the claimed [steps] could not physically fit
+/// in it, and is never zero.
+LiveCoverageWindow? deriveLiveCoverageWindow({
+ required int steps,
+ required int samples100Hz,
+ int? bandStartTs,
+ int bandEndTs = 0,
+ int? firstIngestMs,
+ int? lastIngestMs,
+}) {
+ // No real steps → nothing to exclude from the 1 Hz estimate, nothing to store.
+ if (steps <= 0) return null;
+
+ final int? anchor = (bandStartTs != null && bandStartTs > 0)
+ ? bandStartTs
+ : (firstIngestMs != null && firstIngestMs > 0
+ ? firstIngestMs ~/ 1000
+ : null);
+ // Steps with no timestamp at all from either clock: we cannot place the
+ // window on any timeline, and a misplaced window would exclude the WRONG
+ // minutes. Drop it rather than invent a position.
+ if (anchor == null) return null;
+
+ final sampledS = samples100Hz <= 0 ? 0.0 : samples100Hz / kLiveSampleRateHz;
+
+ double hullS = 0;
+ if (firstIngestMs != null &&
+ lastIngestMs != null &&
+ lastIngestMs > firstIngestMs) {
+ hullS = (lastIngestMs - firstIngestMs) / 1000.0;
+ } else if (bandStartTs != null && bandEndTs > bandStartTs) {
+ hullS = (bandEndTs - bandStartTs).toDouble();
+ }
+
+ double coveredS;
+ if (hullS <= 0) {
+ coveredS = sampledS; // no hull observed — the sampled time is all we have
+ } else if (sampledS <= 0) {
+ coveredS = hullS; // steps without sample accounting — hull is the evidence
+ } else {
+ final duty = sampledS / hullS;
+ coveredS = duty >= kCoverageDutyFloor ? hullS : sampledS;
+ // The sampled time can exceed the hull (duplicate/backlogged frames). The
+ // hull is wall truth, so it also acts as the ceiling.
+ if (coveredS > hullS) coveredS = hullS;
+ }
+
+ // Physiological floor — a true lower bound, applied last so a broken duration
+ // measurement can never produce a window the steps could not fit into.
+ var secs = coveredS.ceil();
+ secs = math.max(secs, minCoverageSecondsForSteps(steps));
+ // A window claiming steps is never zero-width: that silently disables the
+ // no-double-count exclusion it exists to drive.
+ if (secs < 1) secs = 1;
+ return LiveCoverageWindow(anchor, anchor + secs);
+}
+
+/// Persistence-boundary guard: normalise a window before it reaches the
+/// `live_coverage` table, or null when it must not be stored at all.
+///
+/// This is defence in depth behind [deriveLiveCoverageWindow] — an upstream
+/// regression that stops advancing its clock must not be able to write a
+/// zero-width window again without being repaired here.
+///
+/// REPAIR, NOT REJECT. Dropping the row would throw away a REAL 100 Hz step
+/// count (the one measurement on the day that is not an estimate) and hand
+/// those minutes back to the 1 Hz estimator. Widening the window to the
+/// physically implied minimum ([minCoverageSecondsForSteps]) keeps the steps,
+/// restores a non-degenerate exclusion, and claims only what a bound supports.
+/// The one case that IS rejected is an inverted window (`endTs < startTs`):
+/// that is incoherent rather than merely imprecise, and there is no defensible
+/// way to decide which end was meant.
+LiveCoverageWindow? sanitizeCoverageWindow(int startTs, int endTs, int steps) {
+ if (steps <= 0) return null;
+ if (endTs < startTs) return null;
+ final minS = math.max(1, minCoverageSecondsForSteps(steps));
+ if (endTs - startTs < minS) return LiveCoverageWindow(startTs, startTs + minS);
+ return LiveCoverageWindow(startTs, endTs);
+}
diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart
index 5397c0d8..b1b3e21e 100644
--- a/lib/data/local_repository_impl.dart
+++ b/lib/data/local_repository_impl.dart
@@ -679,18 +679,29 @@ class LocalRepositoryImpl extends LocalRepository {
final b = await _bundleForDate(date);
if (b == null) return const {};
final cov = _sub(b, 'coverage');
- final total = (cov?['hr_samples'] as num?)?.toInt() ?? 0;
+ final hrSamples = (cov?['hr_samples'] as num?)?.toInt();
// Wear block (on/off segments, first/last on, longest off) computed in the
// engine; fall back to the coverage counts when absent.
final w = b['wear'] is Map
? (b['wear'] as Map).cast()
: null;
+ // MISSING IS NOT ZERO. This used to collapse "we never measured wear" into
+ // `worn_min: 0` via `hr_samples ?? 0`, which an imported day (no `wear`
+ // block, no `coverage` block) hits every time — making it byte-identical to
+ // a day the strap genuinely sat in a drawer, so the screen asserted "Not
+ // worn on this day — no wrist contact was recorded" about data it simply
+ // never had. Resolution order is now: engine wear block, then the day's
+ // own `worn_min` scalar, then the coverage record count, then absent.
+ final scalarWornMin = (_sub(b, 'scalars')?['worn_min'] as num?)?.toInt();
+ final wornMin = (w?['worn_min'] as num?)?.toInt() ??
+ scalarWornMin ??
+ (hrSamples == null ? null : (hrSamples / 60).round());
return {
// Wear = RECORD presence, not valid HR (HR drops out during daytime
// motion). Fall back to the total record count, never hr_valid.
- 'worn_min': (w?['worn_min'] as num?)?.toInt() ?? (total / 60).round(),
- 'coverage_pct':
- (w?['coverage_pct'] as num?)?.toInt() ?? (total > 0 ? 100 : 0),
+ 'worn_min': wornMin,
+ 'coverage_pct': (w?['coverage_pct'] as num?)?.toInt() ??
+ (hrSamples == null ? null : (hrSamples > 0 ? 100 : 0)),
'segments': w?['segments'] ?? const [],
'first_on': w?['first_on'],
'last_on': w?['last_on'],
@@ -860,7 +871,7 @@ class LocalRepositoryImpl extends LocalRepository {
// complete day.
final bundleDate = (b['date'] as String?) ?? date;
final dayStart = _localMidnightSec(bundleDate);
- final dayEnd = dayStart + 86400;
+ final dayEnd = _localDayEndSec(bundleDate);
// Sleep span (onset/wake) for the context band + sleep symbol.
final sw = _sub(b, 'sleep.window.value');
@@ -876,15 +887,19 @@ class LocalRepositoryImpl extends LocalRepository {
// Workouts + device events for that calendar day.
final sess = await LocalDb.sessionsInRange(dayStart, dayEnd);
- final allEvents = await LocalDb.unuploadedEvents(limit: 2000);
+ // Bounded BY THE DAY, in SQL. This used to pull `unuploadedEvents(limit:
+ // 2000)` — `ORDER BY ts ASC LIMIT 2000`, i.e. the OLDEST 2000 rows — and
+ // then filter that page down to this day. Once `events` held more than 2000
+ // rows the page could no longer reach recent days at all, so their markers
+ // silently vanished from the timeline (the same oldest-N-vs-trailing-N
+ // shape as the metricSeries(limit:) outage).
+ final dayEvents = await LocalDb.eventsInRange(dayStart, dayEnd);
final events = >[
- for (final e in allEvents)
- if (((e['ts'] as num?)?.toInt() ?? -1) >= dayStart &&
- ((e['ts'] as num?)?.toInt() ?? -1) < dayEnd)
- {
- 'event_id': (e['event_id'] as num?)?.toInt(),
- 'ts': (e['ts'] as num?)?.toInt(),
- },
+ for (final e in dayEvents)
+ {
+ 'event_id': (e['event_id'] as num?)?.toInt(),
+ 'ts': (e['ts'] as num?)?.toInt(),
+ },
];
// Daytime naps (principled detectNaps) as their own bands on the timeline.
@@ -969,15 +984,15 @@ class LocalRepositoryImpl extends LocalRepository {
}
/// Local midnight (epoch sec) of a 'YYYY-MM-DD' date string.
- int _localMidnightSec(String ymd) {
- final p = ymd.split('-');
- if (p.length != 3) return 0;
- final y = int.tryParse(p[0]),
- m = int.tryParse(p[1]),
- d = int.tryParse(p[2]);
- if (y == null || m == null || d == null) return 0;
- return DateTime(y, m, d).millisecondsSinceEpoch ~/ 1000;
- }
+ int _localMidnightSec(String ymd) => localDayStartSec(ymd) ?? 0;
+
+ /// End of that local day (epoch sec) — the NEXT local midnight.
+ ///
+ /// NOT `_localMidnightSec(ymd) + 86400`: a spring-forward day is 23 h local
+ /// and a fall-back day is 25 h, so the flat +86400 window pulled in an hour
+ /// of the next day (or dropped the last hour) on exactly those two days a
+ /// year. See day_label.dart.
+ int _localDayEndSec(String ymd) => localDayEndSec(ymd) ?? 0;
// ── lists / summaries ─────────────────────────────────────────────────────
@@ -1478,7 +1493,7 @@ class LocalRepositoryImpl extends LocalRepository {
final b = await _bundleForDate(today);
final curve = (_sub(b, 'series')?['hr_curve'] as List?) ?? const [];
final dayStart = _localMidnightSec(today);
- final dayEnd = dayStart + 86400;
+ final dayEnd = _localDayEndSec(today);
return {
'points': [
for (final e in curve)
@@ -1505,18 +1520,17 @@ class LocalRepositoryImpl extends LocalRepository {
@override
Future> getRecords() async {
- final rows = await LocalDb.recentDayResults(3650);
- final days = rows.length;
- int nights = 0;
- final sleepDays = {};
- for (final r in rows) {
- final b = _decode(r['payload_json']);
- if (_sub(b, 'sleep.accounting.value')?['tst_sec'] != null) {
- nights++;
- final d = r['date'] as String?;
- if (d != null) sleepDays.add(d);
- }
- }
+ // PAYLOAD-FREE. This used to be `recentDayResults(3650)` — `SELECT r.*`
+ // over TEN YEARS of day_result, dragging every bundle's hr_curve /
+ // hypnogram / HRV series across and `jsonDecode`ing each on the main
+ // isolate, for a screen that only ever needs scalar extremes. At ~2 years
+ // of history that is hundreds of MB decoded to compute two counts. Both
+ // are now answered in SQLite: day labels from an index-only GROUP BY, the
+ // sleep count via json_extract (only the scalar crosses the boundary).
+ final dayLabelList = await LocalDb.dayResultDayIdsDesc();
+ final days = dayLabelList.length;
+ final sleepDays = await LocalDb.daysWithSleepTst();
+ final nights = sleepDays.length;
// Personal records from the day scalars (metric_series) + the sessions
// table — computed locally with the record's own date attached.
@@ -1598,10 +1612,7 @@ class LocalRepositoryImpl extends LocalRepository {
}
// Streaks: consecutive most-recent days with derived data / with sleep.
- final dayLabels = {
- for (final r in rows)
- if (r['date'] is String) r['date'] as String,
- };
+ final dayLabels = dayLabelList.toSet();
int streakOf(Set have) {
var streak = 0;
var d = DateTime.now();
@@ -2261,8 +2272,18 @@ class LocalRepositoryImpl extends LocalRepository {
// Phase + fertile window — only when meanLength is known (else honest unknown).
String phase = 'unknown';
String? fertileStart, fertileEnd;
- if (meanLength != null && cycleDay != null && lastStart != null) {
- final ovDay = (meanLength - 14).round().clamp(10, meanLength.round());
+ // A mean cycle shorter than the 10-day ovulation floor makes the clamp
+ // bounds cross — `clamp(10, 8)` THROWS ArgumentError (lowerLimit >
+ // upperLimit), and it threw straight out of getCycle() so the entire cycle
+ // screen errored instead of degrading. Two logged `start` markers 8 days
+ // apart is enough: a mis-tap the user then corrected, or a genuinely short
+ // cycle. Below the floor there is no defensible ovulation day to place, so
+ // be honest — leave `phase: 'unknown'` and publish no fertile window
+ // (predictedNext / cycleDay / the biometric overlay still render).
+ final ovDay = (meanLength == null || meanLength.round() < 10)
+ ? null
+ : (meanLength - 14).round().clamp(10, meanLength.round());
+ if (ovDay != null && cycleDay != null && lastStart != null) {
if (cycleDay <= 5) {
phase = 'menstrual';
} else if (cycleDay < ovDay) {
diff --git a/lib/gps/route_math.dart b/lib/gps/route_math.dart
index b4d5f6e1..8de0af88 100644
--- a/lib/gps/route_math.dart
+++ b/lib/gps/route_math.dart
@@ -203,10 +203,16 @@ List computeSplits(
for (var i = 1; i < pts.length; i++) {
final prev = pts[i - 1];
final cur = pts[i];
- var segLen =
- haversineMeters(prev.lat, prev.lng, cur.lat, cur.lng);
final segStartTs = prev.tsMs;
final segEndTs = cur.tsMs;
+ var segLen = haversineMeters(prev.lat, prev.lng, cur.lat, cur.lng);
+ // SAME filter as [totalDistanceMeters]: a teleport across a recording gap
+ // (tunnel, screen-off, signal loss) is a SEGMENT BREAK, not distance.
+ // Without this, a 55 km jump made the headline `distanceMeters` read ~5 km
+ // while `splitsKm` emitted ~60 mostly-phantom splits on the same screen.
+ // Time still elapses across the break, so the split it lands in keeps its
+ // real duration — only the bogus distance is dropped.
+ if (isImplausibleSegment(segLen, segEndTs - segStartTs)) segLen = 0.0;
// A single segment may cross one or more split boundaries. Walk the
// boundaries, interpolating the crossing time linearly along the segment.
diff --git a/lib/gps/route_tracker.dart b/lib/gps/route_tracker.dart
index 7cc15f29..82e40951 100644
--- a/lib/gps/route_tracker.dart
+++ b/lib/gps/route_tracker.dart
@@ -147,6 +147,9 @@ class RouteTracker {
// Surface — a dead location service otherwise looks like eternal
// "Waiting for GPS…". The subscription stays up (cancelOnError: false)
// so fixes resume seamlessly if the service comes back.
+ // `_stopped` guard: an event already in flight when stop()/dispose()
+ // cancelled the subscription must not write to a disposed notifier.
+ if (_stopped) return;
error.value = e.toString();
},
cancelOnError: false,
@@ -250,11 +253,21 @@ class RouteTracker {
}
}
- /// Stop tracking and flush any buffered tail. Idempotent. Retries the final
- /// flush once — after stop() no later flush ever runs, so a single transient
- /// sink failure here used to silently drop the route's tail.
+ /// Stop tracking and flush any buffered tail, then release the tracker.
+ /// Idempotent. Retries the final flush once — after stop() no later flush
+ /// ever runs, so a single transient sink failure here used to silently drop
+ /// the route's tail.
+ ///
+ /// TERMINAL: a tracker is single-use (`start` is a no-op after the first
+ /// call), and both owners in AppState drop their reference the moment they
+ /// call this, so stop() also runs [dispose]. Previously nothing ever called
+ /// dispose(), which leaked the six ValueNotifiers (and their listeners) once
+ /// per route workout. Read the notifiers BEFORE awaiting stop().
Future stop() async {
- if (_stopped) return;
+ if (_stopped) {
+ dispose();
+ return;
+ }
_stopped = true;
_watchdog?.cancel();
_watchdog = null;
@@ -262,10 +275,24 @@ class RouteTracker {
_sub = null;
await _flush();
if (_buffer.isNotEmpty) await _flush(); // one retry for the tail
+ dispose();
}
+ bool _disposed = false;
+
+ /// Release every resource the tracker holds. Idempotent, and safe to call
+ /// WITHOUT stop() — it cancels the GPS subscription too, which the previous
+ /// implementation did not: it only cancelled the watchdog, so a dispose()
+ /// without a stop() left the location stream (and the platform's GPS
+ /// hardware session behind it) running for the life of the process.
void dispose() {
+ if (_disposed) return;
+ _disposed = true;
+ _stopped = true;
_watchdog?.cancel();
+ _watchdog = null;
+ unawaited(_sub?.cancel());
+ _sub = null;
path.dispose();
current.dispose();
distanceMeters.dispose();
diff --git a/lib/import/noop_import.dart b/lib/import/noop_import.dart
index e6daa1b7..245b9bde 100644
--- a/lib/import/noop_import.dart
+++ b/lib/import/noop_import.dart
@@ -26,9 +26,21 @@ import '../compute/substrate.dart';
class NoopImportResult {
final int days;
final int rows;
- NoopImportResult(this.days, this.rows);
+
+ /// Rows whose local date had ALREADY been derived and pruned by the time
+ /// they appeared in the file (a genuinely out-of-order export). They cannot
+ /// be folded back in, so they are counted here rather than silently dropped
+ /// — a non-zero value means the export was not fully time-ordered.
+ final int lateRows;
+ NoopImportResult(this.days, this.rows, [this.lateRows = 0]);
}
+/// What to do with an incoming row given the import's high-water date.
+/// [advance] closes out the previous date; [buffer] folds the row into the
+/// current rolling window; [late] means its day was already derived + pruned,
+/// so the row cannot be used (counted in [NoopImportResult.lateRows]).
+enum RowOrder { advance, buffer, late }
+
/// One second's worth of the 1 Hz channels (sparse — only set streams present).
class _Sec {
int? hr;
@@ -63,7 +75,8 @@ class NoopImporter {
final rrTs = []; // beat end time (epoch ms)
final rrMs = [];
String? curDate;
- var totalRows = 0, daysDone = 0;
+ final derived = {}; // dates already derived + pruned out of `secs`
+ var totalRows = 0, daysDone = 0, lateRows = 0;
Future deriveAndPrune(String date) async {
// Build a Substrate from everything buffered (prev + current date) and
@@ -113,11 +126,34 @@ class NoopImporter {
final stream = at(f, 'stream');
final date = localDateLabel(ts);
- // Date advanced → the previous date is complete; derive it from the window.
- if (curDate != null && date != curDate && _after(date, curDate)) {
- await deriveAndPrune(curDate);
+ //
+ // OUT-OF-ORDER ROWS. `curDate` is a HIGH-WATER mark and must only ever
+ // move forward. It used to be assigned unconditionally, so one backwards
+ // timestamp rewound it; the next forward row then called
+ // deriveAndPrune(), whose `secs.removeWhere(label != date)`
+ // discarded every buffered sample for the newer day — silent, unbounded
+ // loss with nothing surfaced.
+ //
+ // Now: a forward row closes out the previous date (derive + prune); a
+ // backwards row for a day we have NOT derived yet is simply folded into
+ // the buffer at its own timestamp (the Substrate is rebuilt sorted by ts,
+ // so arrival order never mattered); and a row for a day already derived
+ // is genuinely too late to use — counted in
+ // [NoopImportResult.lateRows] and reported instead of being allowed to
+ // wipe the current day.
+ switch (decideRow(date, curDate, derived)) {
+ case RowOrder.advance:
+ if (curDate != null) {
+ await deriveAndPrune(curDate);
+ derived.add(curDate);
+ }
+ curDate = date;
+ case RowOrder.buffer:
+ break;
+ case RowOrder.late:
+ lateRows++;
+ continue;
}
- curDate = date;
totalRows++;
switch (stream) {
@@ -161,7 +197,7 @@ class NoopImporter {
}
await engine.finalizeImport(profile);
- return NoopImportResult(daysDone, totalRows);
+ return NoopImportResult(daysDone, totalRows, lateRows);
}
/// Build a Substrate from the buffered seconds + RR beats. Gravity / SpO₂ /
@@ -224,4 +260,22 @@ class NoopImporter {
/// String date compare 'YYYY-MM-DD' — true when [a] is strictly after [b].
static bool _after(String a, String b) => a.compareTo(b) > 0;
+
+ /// Where a row belongs given the import's high-water date [curDate] and the
+ /// set of dates already [derived] (and therefore already pruned out of the
+ /// rolling buffer).
+ ///
+ /// This is the whole out-of-order contract, isolated so it can be tested
+ /// without a database: `curDate` only ever moves FORWARD. It used to be
+ /// assigned unconditionally, so one backwards timestamp rewound it and the
+ /// next forward row called `deriveAndPrune()` — whose
+ /// `secs.removeWhere(label != date)` threw away every buffered sample of the
+ /// NEWER day, silently and with no error surfaced.
+ static RowOrder decideRow(String date, String? curDate, Set derived) {
+ if (curDate == null || _after(date, curDate)) return RowOrder.advance;
+ if (date == curDate) return RowOrder.buffer;
+ // Older than the high-water day. Still usable as prior-evening context
+ // unless its day has already been derived and pruned.
+ return derived.contains(date) ? RowOrder.late : RowOrder.buffer;
+ }
}
diff --git a/lib/import/whoop_import.dart b/lib/import/whoop_import.dart
index a2d2f9d3..eb8288ff 100644
--- a/lib/import/whoop_import.dart
+++ b/lib/import/whoop_import.dart
@@ -22,10 +22,49 @@ import '../data/db.dart';
class WhoopImportResult {
final int days;
final int workouts;
- WhoopImportResult(this.days, this.workouts);
+
+ /// Days present in the export that were NOT written because the device
+ /// already holds a REAL (1 Hz-derived) day for that date. Vendor snapshots
+ /// never replace measured data — see [WhoopImporter._writeDay].
+ final int skippedExistingDays;
+ WhoopImportResult(this.days, this.workouts, [this.skippedExistingDays = 0]);
+}
+
+/// One CSV row, addressed BY HEADER NAME. Also exposes WHICH header matched, so
+/// unit-bearing columns ("… (cal)" vs "… (kJ)") can be interpreted from their
+/// declared unit instead of guessed from the magnitude of the number.
+class _Row {
+ final Map col;
+ final List f;
+ const _Row(this.col, this.f);
+
+ String get(List names) => _pick(names).$2;
+
+ /// The header name that matched (lower-cased), or '' if none did.
+ String header(List names) => _pick(names).$1;
+
+ (String, String) _pick(List names) {
+ for (final n in names) {
+ final i = col[n];
+ if (i != null && i < f.length) return (n, f[i].trim());
+ }
+ return ('', '');
+ }
}
class WhoopImporter {
+ /// Column aliases for the energy field. The unit is read from whichever of
+ /// these actually matched — never inferred from the value (see [_kcal]).
+ static const List _energyCols = [
+ 'energy burned (cal)',
+ 'energy burned (kcal)',
+ 'energy burned (kilocalories)',
+ 'energy burned (kj)',
+ 'energy burned (kilojoules)',
+ 'energy burned (kilojoule)',
+ 'energy burned',
+ ];
+
/// Import one or more WHOOP export CSVs. Derived snapshots only. Pass [engine]
/// + [profile] to run the cross-day rollup / baseline refresh once at the end.
static Future