Skip to content

Commit 3c4c4de

Browse files
authored
fix(search): check the last shortened read window against its budget (#9987)
1 parent 66baa21 commit 3c4c4de

2 files changed

Lines changed: 21 additions & 1 deletion

File tree

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/**
2+
* @vitest-environment node
3+
*/
4+
import { describe, expect, it } from 'vitest'
5+
import { encodedWindowEnd } from '@/lib/sim-search/live/read-window'
6+
7+
const encoded = (text: string) => JSON.stringify(text).length - 2
8+
9+
describe('encodedWindowEnd', () => {
10+
it('returns the longest window it found to fit when escapes lead the text', () => {
11+
const content = `${'"'.repeat(1_000)}${'x'.repeat(20_000)}`
12+
const end = encodedWindowEnd(content, 0, 20_000, 8_000)
13+
expect(end).toBe(19_000)
14+
expect(encoded(content.slice(0, end))).toBeLessThanOrEqual(20_000)
15+
})
16+
})

‎apps/sim/lib/sim-search/live/read-window.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
/** Characters one standard read returns, the page budget indexed document reads use. */
22
export const READ_WINDOW_CHARACTERS = 8000
33

4+
/** Times a window is shortened to fit its budget before the minimum is used instead. */
5+
const MAX_SHRINKS = 3
6+
47
/**
58
* Where a window of `content` from `start` ends so its text, as JSON encodes it, fits `budget`
69
* characters. JSON escapes quotes, backslashes and line breaks, so a window is shortened until its
@@ -13,9 +16,10 @@ export function encodedWindowEnd(
1316
minimumEnd: number
1417
): number {
1518
let end = Math.min(content.length, start + budget)
16-
for (let attempt = 0; attempt < 3 && end > minimumEnd; attempt++) {
19+
for (let shrinks = 0; end > minimumEnd; shrinks++) {
1720
const encoded = JSON.stringify(content.slice(start, end)).length - 2
1821
if (encoded <= budget) return end
22+
if (shrinks === MAX_SHRINKS) break
1923
end = start + Math.floor(((end - start) * budget) / encoded)
2024
}
2125
return minimumEnd

0 commit comments

Comments
 (0)