Port upstream 0.67.0: retain inherited counter origin for direct Codex fork chains (stacked on #610)

CHANGELOG.md

@@ -16,6 +16,8 @@ menu-bar layout.
 
 ### Fixed
 - Claude: when Hide Personal Info is enabled, keep saved account rows distinguishable with stable localized `Account N` labels and matching redacted tooltips.
+- Codex: a fork of a fork whose parent has no token snapshot before the child forked now inherits the whole chain's cumulative counter origin instead of staying unresolved; ancestor chains deeper than 64 stay unresolved, and the Codex cost cache is rebuilt once (cache schema 5).
+- Codex: a fork larger than the per-refresh scan budget continues where the last refresh stopped instead of restarting from the top and never finishing, and a finished fork that grows is no longer billed for usage it inherited from its parent.
 
 ---
 

rust/src/core/jsonl_scanner.rs

@@ -396,12 +396,35 @@ pub struct CodexForkAccountingState {
     pub history_base_thread_id: Option<String>,
     pub fork_timestamp: Option<String>,
     pub inherited_totals: Option<CodexTotals>,
+    /// Timestamp of the first token event in this fork's log. A descendant
+    /// forked before it uses this fork's inherited counter origin.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub first_token_timestamp: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub remaining_inherited_totals: Option<CodexTotals>,
     /// True when the child log itself supplied enough copied-prefix history to
     /// establish the inherited baseline without consulting a parent cache row.
     #[serde(default, skip_serializing_if = "std::ops::Not::not")]
     pub locally_resolved: bool,
+    /// Parser state of an unfinished bounded parse, so the next pass continues
+    /// at `parsed_bytes` instead of rereading the fork from byte zero. Absent
+    /// once the parse reaches its target, and in caches written before it.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub resume: Option<CodexForkResumeState>,
+}
+
+/// Fork parser state that the per-file cache fields do not already carry.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct CodexForkResumeState {
+    /// Parent baseline the interrupted parse started from. A validated parent
+    /// that now supplies a different baseline restarts the parse.
+    pub parse_baseline: CodexTotals,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub totals_watermark: Option<CodexTotals>,
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+    pub saw_interleaved_totals: bool,
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+    pub paginated_baseline_checked: bool,
 }
 
 /// Snapshot of the last validated cost report, persisted so spend surfaces keep
@@ -449,6 +472,10 @@ pub struct CodexParseResult {
     pub token_timestamps_monotonic: Option<bool>,
     /// Last token timestamp observed by the parser.
     pub last_token_timestamp: Option<String>,
+    /// First token timestamp of the parsed history. A resumed fork parse
+    /// carries its saved value forward; resumed standard parses see only their
+    /// suffix, so fork accounting never reads it from them.
+    pub first_token_timestamp: Option<String>,
     /// Number of timestamp comparisons performed while validating this parse.
     pub token_timestamp_comparisons: u64,
     /// Newly consumed bytes in this parse pass.
@@ -463,6 +490,9 @@ pub struct CodexParseResult {
     /// Remaining inherited counters used when a fork emits last-only rows.
     pub remaining_inherited_totals: Option<CodexTotals>,
     pub fork_baseline_locally_resolved: bool,
+    /// State to continue an unfinished parent-baseline fork parse. `None` once
+    /// the parse is complete and for every other parse mode.
+    pub fork_resume_state: Option<CodexForkResumeState>,
 }
 
 /// A billable Codex token-count delta.
@@ -514,6 +544,7 @@ impl CostUsageDayRange {
 /// JSONL Scanner for cost/usage logs
 pub struct JsonlScanner;
 pub(crate) mod codex;
+pub(crate) use codex::CodexForkParseResume;
 pub(crate) use codex::source_rows::{
     read_source_rows, recover_rows, row_cache, row_cache_matches, row_cache_needs_recovery,
 };

rust/src/core/jsonl_scanner/codex.rs

@@ -10,12 +10,32 @@ use helpers::{
 };
 use parser::{CodexParseMode, CodexParserState};
 
+/// Saved cursor and parser state of an unfinished parent-baseline fork parse.
+/// Restoring all of it accounts the remaining bytes exactly as one
+/// uninterrupted parse would.
+#[derive(Debug, Clone)]
+pub(crate) struct CodexForkParseResume {
+    pub start_offset: i64,
+    pub paginated_continuation: bool,
+    /// Effective inherited baseline, after any paginated-continuation raise.
+    pub inherited_totals: CodexTotals,
+    pub remaining_inherited_totals: Option<CodexTotals>,
+    pub last_model: Option<String>,
+    pub last_totals: CodexTotals,
+    pub last_token_timestamp: Option<String>,
+    pub first_token_timestamp: Option<String>,
+    pub token_timestamps_monotonic: Option<bool>,
+    pub state: CodexForkResumeState,
+}
+
 /// Persisted Codex cache schema version. Version 0 predates 64-bit totals;
 /// version 1 can retain a terminal pause after treating a paginated v2
 /// subagent's independent counters as an inherited fork. Version 3 adds
 /// persisted paginated-fork accounting state. Version 4 reparses copied-prefix
-/// subagents with locally inferred component baselines. Rebuild older artifacts.
-pub(crate) const CODEX_CACHE_SCHEMA_VERSION: u32 = 4;
+/// subagents with locally inferred component baselines. Version 5 records a
+/// fork's first token timestamp so direct-fork chains resolve through parents
+/// with no token snapshot at the descendant's fork time. Rebuild older artifacts.
+pub(crate) const CODEX_CACHE_SCHEMA_VERSION: u32 = 5;
 
 /// Whether a persisted Codex cache artifact matches the current schema.
 /// A mismatched artifact (e.g. a pre-64-bit cache from an older release) is
@@ -293,6 +313,18 @@ impl JsonlScanner {
         }
     }
 
+    /// Strict counterpart of [`Self::codex_timestamp_at_or_before`]: `earlier`
+    /// is before `later`. Malformed timestamps fail closed.
+    pub(crate) fn codex_timestamp_before(earlier: &str, later: &str) -> bool {
+        match (
+            parse_rfc3339_timestamp(earlier),
+            parse_rfc3339_timestamp(later),
+        ) {
+            (Some(earlier), Some(later)) => earlier < later,
+            _ => false,
+        }
+    }
+
     /// Parse a Codex JSONL file
     pub fn parse_codex_file(
         file_path: &Path,
@@ -514,6 +546,26 @@ impl JsonlScanner {
         )
     }
 
+    /// Continue an unfinished fork parse at its saved cursor. Upstream 0.67.0
+    /// resumes a resolved fork the same way instead of rereading its prefix.
+    pub(crate) fn parse_codex_fork_resume(
+        file_path: &Path,
+        range: &CostUsageDayRange,
+        resume: CodexForkParseResume,
+        cancel: Option<&AtomicBool>,
+        scan_target_size: Option<i64>,
+        max_bytes_to_read: Option<i64>,
+    ) -> std::io::Result<CodexParseResult> {
+        Self::parse_codex_file_with_state_bounded_internal(
+            file_path,
+            range,
+            cancel,
+            scan_target_size,
+            max_bytes_to_read,
+            CodexParseMode::ResumeParentBaseline(resume),
+        )
+    }
+
     fn parse_codex_file_with_state_bounded_internal(
         file_path: &Path,
         range: &CostUsageDayRange,
@@ -631,6 +683,11 @@ impl JsonlScanner {
         let is_complete = !cancelled && !budget_exhausted && parsed_bytes >= effective_target_size;
         let bytes_read = parsed_bytes.saturating_sub(safe_start_offset).max(0);
         let fork_baseline_locally_resolved = parser.fork_baseline_locally_resolved();
+        let fork_resume_state = if is_complete {
+            None
+        } else {
+            parser.fork_resume_state()
+        };
         Ok(CodexParseResult {
             records: parser.records,
             parsed_bytes,
@@ -643,13 +700,15 @@ impl JsonlScanner {
             last_totals: parser.previous_totals,
             token_timestamps_monotonic: parser.token_timestamps_monotonic,
             last_token_timestamp: parser.previous_token_timestamp,
+            first_token_timestamp: parser.first_token_timestamp,
             token_timestamp_comparisons: parser.token_timestamp_comparisons,
             bytes_read,
             is_complete,
             fork_baseline_ambiguous: parser.fork_baseline_ambiguous,
             fork_baseline: parser.fork_baseline,
             remaining_inherited_totals: parser.remaining_inherited_totals,
             fork_baseline_locally_resolved,
+            fork_resume_state,
         })
     }
 

rust/src/core/jsonl_scanner/codex/parser.rs

@@ -1,5 +1,7 @@
 use super::helpers::*;
-use super::{CodexTotals, CodexUsageRecord, CostUsageDayRange};
+use super::{
+    CodexForkParseResume, CodexForkResumeState, CodexTotals, CodexUsageRecord, CostUsageDayRange,
+};
 use crate::core::CostUsagePricing;
 use chrono::{DateTime, Utc};
 use serde_json::Value;
@@ -14,6 +16,10 @@ pub(super) struct CodexParserState {
     saw_interleaved_totals: bool,
     pub(super) records: Vec<(CodexUsageRecord, i64)>,
     pub(super) previous_token_timestamp: Option<String>,
+    /// First token timestamp seen by this parse. A fork uses it to prove that
+    /// its inherited origin is still the counter state at a descendant's
+    /// cutoff. Resumed non-fork parses do not observe the whole prefix.
+    pub(super) first_token_timestamp: Option<String>,
     previous_token_timestamp_parsed: Option<DateTime<chrono::FixedOffset>>,
     pub(super) token_timestamps_monotonic: Option<bool>,
     pub(super) token_timestamp_comparisons: u64,
@@ -23,6 +29,8 @@ pub(super) struct CodexParserState {
     paginated_baseline_checked: bool,
     pub(super) fork_baseline_ambiguous: bool,
     fork_baseline_inference: Option<ForkBaselineInference>,
+    /// Parent baseline a parent-baseline fork parse started from.
+    fork_parse_baseline: Option<CodexTotals>,
 }
 
 pub(super) enum CodexParseMode {
@@ -38,6 +46,8 @@ pub(super) enum CodexParseMode {
         paginated_continuation: bool,
         remaining_inherited_totals: Option<CodexTotals>,
     },
+    /// Continue an unfinished `ParentBaseline` parse at its saved cursor.
+    ResumeParentBaseline(CodexForkParseResume),
     InferSubagent {
         start_ordinal: Option<i64>,
     },
@@ -47,6 +57,7 @@ impl CodexParseMode {
     pub(super) fn start_offset(&self) -> i64 {
         match self {
             Self::Standard { start_offset, .. } => *start_offset,
+            Self::ResumeParentBaseline(resume) => resume.start_offset,
             Self::ParentBaseline { .. } | Self::InferSubagent { .. } => 0,
         }
     }
@@ -213,6 +224,10 @@ impl CodexParserState {
     }
 
     pub(super) fn from_mode(mode: CodexParseMode) -> Self {
+        let fork_parse_baseline = match &mode {
+            CodexParseMode::ParentBaseline { baseline, .. } => Some(baseline.clone()),
+            _ => None,
+        };
         let (
             initial_model,
             initial_totals,
@@ -257,6 +272,7 @@ impl CodexParserState {
                     None,
                 )
             }
+            CodexParseMode::ResumeParentBaseline(resume) => return Self::resume_fork(resume),
             CodexParseMode::InferSubagent { start_ordinal } => (
                 None,
                 None,
@@ -278,6 +294,7 @@ impl CodexParserState {
             saw_interleaved_totals: false,
             records: Vec::new(),
             previous_token_timestamp,
+            first_token_timestamp: None,
             previous_token_timestamp_parsed,
             // A parser always validates a fresh prefix.  `None` is only an
             // input marker for the legacy-cache path, not an output state.
@@ -289,9 +306,54 @@ impl CodexParserState {
             paginated_baseline_checked: false,
             fork_baseline_ambiguous: false,
             fork_baseline_inference,
+            fork_parse_baseline,
         }
     }
 
+    /// Restore an unfinished fork parse at its saved cursor. A resumable parse
+    /// never latched an ambiguous baseline: that result is cached unresolved.
+    fn resume_fork(resume: CodexForkParseResume) -> Self {
+        let CodexForkParseResume {
+            start_offset,
+            paginated_continuation,
+            inherited_totals,
+            remaining_inherited_totals,
+            last_model,
+            last_totals,
+            last_token_timestamp,
+            first_token_timestamp,
+            token_timestamps_monotonic,
+            state,
+        } = resume;
+        let mut parser = Self::from_mode(CodexParseMode::Standard {
+            start_offset,
+            initial_model: last_model,
+            initial_totals: Some(last_totals),
+            previous_token_timestamp: last_token_timestamp,
+            token_timestamps_monotonic,
+        });
+        parser.totals_watermark = state.totals_watermark;
+        parser.saw_interleaved_totals = state.saw_interleaved_totals;
+        parser.first_token_timestamp = first_token_timestamp;
+        parser.fork_baseline = Some(inherited_totals);
+        // Kept verbatim: `None` means the inherited counters are used up.
+        parser.remaining_inherited_totals = remaining_inherited_totals;
+        parser.paginated_continuation = paginated_continuation;
+        parser.paginated_baseline_checked = state.paginated_baseline_checked;
+        parser.fork_parse_baseline = Some(state.parse_baseline);
+        parser
+    }
+
+    /// State a later pass needs to continue this parent-baseline fork parse.
+    pub(super) fn fork_resume_state(&self) -> Option<CodexForkResumeState> {
+        Some(CodexForkResumeState {
+            parse_baseline: self.fork_parse_baseline.clone()?,
+            totals_watermark: self.totals_watermark.clone(),
+            saw_interleaved_totals: self.saw_interleaved_totals,
+            paginated_baseline_checked: self.paginated_baseline_checked,
+        })
+    }
+
     pub(super) fn fork_baseline_locally_resolved(&self) -> bool {
         self.fork_baseline_inference
             .as_ref()
@@ -846,6 +908,9 @@ impl CodexParserState {
                 self.token_timestamps_monotonic = Some(false);
             }
         }
+        if self.first_token_timestamp.is_none() {
+            self.first_token_timestamp = Some(timestamp.to_string());
+        }
         self.previous_token_timestamp = Some(timestamp.to_string());
         self.previous_token_timestamp_parsed = current_parsed;
     }