delete buildTranscript

Author: phantom5099

packages/codingcode/src/context/compressor.ts

@@ -1,19 +1,17 @@
 import { randomUUID } from 'crypto';
-import { resolveSessionDir, appendLine } from '../session/io.js';
+import { resolveSessionJsonlPath, appendLine } from '../session/io.js';
 import {
   estimateTokens,
   estimateMessageTokens,
-  estimateTokensForContent,
 } from './util.js';
-import { applyVisibilityEvents } from '../session/messages.js';
+import { buildMessagesFromEvents } from '../session/messages.js';
 import { resolveLLM } from '../llm/llm-resolver.js';
 import { COMPACTION_SYSTEM_PROMPT } from './compaction-prompt.js';
 import type { ContextConfig } from './config.js';
 import type { Message } from '../core/types.js';
 import type { SessionEvent, SummaryEvent } from '../session/types.js';
 import type { LLMClient } from '../llm/client.js';
 import { assemblePayload } from './organizer.js';
-import { join } from 'path';
 
 export interface CompressResult {
   didCompress: boolean;
@@ -85,8 +83,8 @@ export async function compactWithLLM(
   usage?: number,
   modelMaxTokens?: number
 ): Promise<CompressResult> {
+  const payload = assemblePayload(sessionId, encodedProjectPath, config, modelMaxTokens);
   if (!compactedEvents || currentTurnId === undefined) {
-    const payload = assemblePayload(sessionId, encodedProjectPath, config, modelMaxTokens);
     compactedEvents = payload.compactedEvents;
     currentTurnId = payload.currentTurnId;
   }
@@ -95,38 +93,33 @@ export async function compactWithLLM(
 
   const threshold = modelMaxTokens ? modelMaxTokens * config.compactionThreshold : Infinity;
   if (usage === undefined || usage - released > threshold) {
-    released += await tryCompaction(sessionId, config, llm, compactedEvents, currentTurnId);
+    released += await tryCompaction(sessionId, config, llm, compactedEvents, currentTurnId, payload.compactedTurnIds);
   }
 
-  const payload = assemblePayload(sessionId, encodedProjectPath, config, modelMaxTokens);
+  const postPayload = assemblePayload(sessionId, encodedProjectPath, config, modelMaxTokens);
   return {
     didCompress: released > 0,
     released,
-    promptEstimate: estimateTokens(payload.messages),
+    promptEstimate: estimateTokens(postPayload.messages),
   };
 }
 
 
 // ---------- LLM Compaction ----------
 
-const ESTIMATED_SUMMARY_TOKENS = 5000;
-const MAX_TOOL_RESULT_TOKENS = 30000;
-
 async function tryCompaction(
   sessionId: string,
   config: ContextConfig,
   llm: LLMClient | null,
   compactedEvents: SessionEvent[],
-  currentTurnId: number
+  currentTurnId: number,
+  compactedTurnIds: Set<number>,
 ): Promise<number> {
   const endTurn = currentTurnId - config.keepRecentTurns - 1;
   if (endTurn < 1) return 0;
 
-  const { hidden } = applyVisibilityEvents(compactedEvents);
-
   const inRange = compactedEvents.filter((ev) => {
     if (ev.type === 'session_meta') return false;
-    if ('uuid' in ev && hidden.has((ev as any).uuid)) return false;
     if ('turnId' in ev && (ev as any).turnId >= 1 && (ev as any).turnId <= endTurn) return true;
     return false;
   });
@@ -135,15 +128,15 @@ async function tryCompaction(
   const targetEvents = getIncrementalEvents(inRange);
   if (targetEvents.length === 0) return 0;
 
-  const totalTokens = targetEvents.reduce((sum, e) => sum + estimateEventTokens(e), 0);
+  const msgs = buildMessagesFromEvents(targetEvents, compactedTurnIds);
+  const totalTokens = estimateTokens(msgs);
 
   let compactionLlm = await resolveLLM(config.compactionModel, llm);
   if (compactionLlm && compactionLlm.modelInfo.maxTokens < totalTokens + 25000) {
     compactionLlm = llm;
   }
 
-  const transcript = buildTranscript(targetEvents);
-  const summary = await callLLMForCompaction(transcript, compactionLlm, config);
+  const summary = await callLLMForCompaction(msgs, compactionLlm, config);
   if (!summary) return 0;
 
   const replacedUuids: string[] = [];
@@ -164,10 +157,7 @@ async function tryCompaction(
     lastSummarizedTurnId: lastTurnId,
     timestamp: new Date().toISOString(),
   };
-  const dir = resolveSessionDir(sessionId);
-  if (!dir) throw new Error(`Session ${sessionId} not found`);
-  appendLine(join(dir, `${sessionId}.jsonl`), event);
-  for (const u of replacedUuids) hidden.add(u);
+  appendLine(resolveSessionJsonlPath(sessionId), event);
 
   const summaryMsg: Message = { role: 'system', name: 'compacted_history', content: summary };
   return Math.max(0, totalTokens - estimateMessageTokens(summaryMsg));
@@ -184,64 +174,6 @@ function getIncrementalEvents(inRange: SessionEvent[]): SessionEvent[] {
   return inRange.filter((e) => 'turnId' in e && (e as any).turnId > lastTurn);
 }
 
-function buildTranscript(events: SessionEvent[]): Message[] {
-  const transcript: Message[] = [];
-  for (const ev of events) {
-    switch (ev.type) {
-      case 'user':
-        transcript.push({ role: 'user', content: ev.content });
-        break;
-      case 'assistant':
-        transcript.push({ role: 'assistant', content: ev.content });
-        break;
-      case 'tool_result': {
-        let content = ev.output;
-        const tokens = estimateTokensForContent(content);
-        if (tokens > MAX_TOOL_RESULT_TOKENS) {
-          const ratio = MAX_TOOL_RESULT_TOKENS / tokens;
-          const keepChars = Math.floor(content.length * ratio);
-          content =
-            content.slice(0, keepChars) +
-            `\n\n[...truncated: ${tokens} tokens total, showing first ${MAX_TOOL_RESULT_TOKENS}]`;
-        }
-        transcript.push({
-          role: 'tool',
-          content,
-          tool_call_id: ev.toolCallId,
-          tool_name: ev.toolName,
-        } as any);
-        break;
-      }
-      case 'summary':
-        transcript.push({ role: 'system', name: 'compacted_history', content: ev.summaryText });
-        break;
-    }
-  }
-  return transcript;
-}
-
-function estimateEventTokens(e: SessionEvent): number {
-  if (e.type === 'user') return estimateMessageTokens({ role: 'user', content: e.content });
-  if (e.type === 'assistant')
-    return estimateMessageTokens({ role: 'assistant', content: e.content });
-  if (e.type === 'tool_result') {
-    return estimateMessageTokens({
-      role: 'tool',
-      content: e.output,
-      tool_call_id: e.toolCallId,
-      tool_name: e.toolName,
-    } as any);
-  }
-  if (e.type === 'summary') {
-    return estimateMessageTokens({
-      role: 'system',
-      name: 'compacted_history',
-      content: e.summaryText,
-    });
-  }
-  return 0;
-}
-
 async function callLLMForCompaction(
   transcript: Message[],
   fallbackLlm: LLMClient | null,
@@ -251,7 +183,7 @@ async function callLLMForCompaction(
   if (!llm) return null;
 
   const transcriptText = transcript
-    .map((m) => `[${m.role}${m.tool_name ? ':' + m.tool_name : ''}]\n${m.content}`)
+    .map((m) => `[${m.role}${(m as any).tool_name ? ':' + (m as any).tool_name : ''}]\n${m.content}`)
     .join('\n\n');
 
   const system = COMPACTION_SYSTEM_PROMPT;

packages/codingcode/src/context/organizer.ts

@@ -1,9 +1,8 @@
 import type { ContextConfig } from './config.js';
 import type { Message } from '../core/types.js';
-import { findSessionIndex, resolveSessionDir, readHistory, appendLine } from '../session/io.js';
+import { findSessionIndex, resolveSessionJsonlPath, readHistory, appendLine } from '../session/io.js';
 import { applyVisibilityEvents, buildMessagesFromEvents } from '../session/messages.js';
 import { estimateTokens } from './util.js';
-import { join } from 'path';
 import { randomUUID } from 'crypto';
 import type { SessionEvent, ToolResultEvent, CompactEvent } from '../session/types.js';
 
@@ -23,6 +22,7 @@ export interface BuildResult {
   compactedEvents: SessionEvent[];
   promptEstimate: number;
   currentTurnId: number;
+  compactedTurnIds: Set<number>;
 }
 
 export function assemblePayload(
@@ -31,16 +31,15 @@ export function assemblePayload(
   config: ContextConfig,
   contextWindow: number = 128000
 ): BuildResult {
-  const dir = resolveSessionDir(sessionId);
-  if (!dir) throw new Error(`Session ${sessionId} not found`);
-  const jsonlPath = join(dir, `${sessionId}.jsonl`);
+  const jsonlPath = resolveSessionJsonlPath(sessionId);
   let events = readHistory(jsonlPath);
 
   const idx = findSessionIndex(sessionId);
   const currentTurnId = idx?.currentTurnId ?? 0;
 
-  const { hidden } = applyVisibilityEvents(events);
+  const { hidden, compactedTurnIds: initialCompactedTurnIds } = applyVisibilityEvents(events);
   let visible = filterVisible(events, hidden);
+  let compactedTurnIds = initialCompactedTurnIds;
 
   const preEstimate = estimateTokensFromEvents(visible);
 
@@ -57,6 +56,7 @@ export function assemblePayload(
     events = readHistory(jsonlPath);
     const updated = applyVisibilityEvents(events);
     visible = filterVisible(events, updated.hidden);
+    compactedTurnIds = updated.compactedTurnIds;
   }
 
   const messages = buildMessagesFromEvents(visible);
@@ -65,6 +65,7 @@ export function assemblePayload(
     compactedEvents: visible,
     promptEstimate: estimateTokens(messages),
     currentTurnId,
+    compactedTurnIds,
   };
 }
 

packages/codingcode/src/session/io.ts

@@ -59,6 +59,12 @@ export function resolveSessionDir(sessionId: string): string | null {
   return null;
 }
 
+export function resolveSessionJsonlPath(sessionId: string): string {
+  const dir = resolveSessionDir(sessionId);
+  if (!dir) throw new Error(`Session ${sessionId} not found`);
+  return join(dir, `${sessionId}.jsonl`);
+}
+
 export function ensureDirs(transcriptPath: string): void {
   if (!existsSync(CODINGCODE_DIR)) mkdirSync(CODINGCODE_DIR, { recursive: true });
   const dir = dirname(transcriptPath);

packages/codingcode/src/session/messages.ts

@@ -69,8 +69,12 @@ export function applyVisibilityEvents(events: SessionEvent[]): VisibilityResult
   return { hidden, compactedTurnIds };
 }
 
-export function buildMessagesFromEvents(events: SessionEvent[]): Message[] {
-  const { hidden, compactedTurnIds } = applyVisibilityEvents(events);
+export function buildMessagesFromEvents(
+  events: SessionEvent[],
+  externalCompactedTurnIds?: Set<number>,
+): Message[] {
+  const { hidden, compactedTurnIds: derivedIds } = applyVisibilityEvents(events);
+  const compactedTurnIds = externalCompactedTurnIds ?? derivedIds;
 
   const visible: SessionEvent[] = [];
   for (const ev of events) {

packages/codingcode/test/context/compressor/compact-if-needed.test.ts

@@ -28,10 +28,16 @@ const { mockCompactWithLLM, mockLLM } = vi.hoisted(() => ({
 
 vi.mock('../../../src/session/io.js', async (importOriginal) => {
   const actual = await importOriginal();
+  const mockResolveSessionDir = vi.fn(() => '/tmp/sessions');
   return {
     ...(actual as any),
     findSessionIndex: vi.fn(() => ({ currentTurnId: 10 })),
-    resolveSessionDir: vi.fn(() => '/tmp/sessions'),
+    resolveSessionDir: mockResolveSessionDir,
+    resolveSessionJsonlPath: vi.fn((sessionId: string) => {
+      const dir = mockResolveSessionDir(sessionId);
+      if (!dir) throw new Error(`Session ${sessionId} not found`);
+      return `${dir}/${sessionId}.jsonl`;
+    }),
     readHistory: vi.fn(() => [
       { type: 'user', content: 'a'.repeat(200), uuid: 'u1', turnId: 1 },
       { type: 'assistant', content: 'b'.repeat(200), uuid: 'a1', turnId: 1 },