-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathdecoding_lab.html
More file actions
473 lines (441 loc) · 28.3 KB
/
Copy pathdecoding_lab.html
File metadata and controls
473 lines (441 loc) · 28.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Decoding Lab — temperature, top-k & top-p, generating live</title>
<link rel="stylesheet" href="lesson-components.css">
<link rel="stylesheet" href="site-polish.css">
<script src="catalog.js"></script>
<script src="theme.js" defer></script>
<script src="search.js" defer></script>
<script src="nav.js" defer></script>
<script src="glossary.js" defer></script>
<style>
:root{
--bg:#0d1117; --surface:#161b22; --surface2:#1c2128; --border:#30363d;
--text:#e6edf3; --muted:#8b949e; --accent:#7c83ff; --accent2:#58d6ff;
--green:#39d353; --red:#ff5c5c; --yellow:#ffdb5c; --blue:#58a6ff;
}
*{box-sizing:border-box;margin:0;padding:0;}
body{background:var(--bg);color:var(--text);font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;font-size:14px;line-height:1.55;min-height:100vh;}
.wrap{max-width:1060px;margin:0 auto;padding:18px 18px 60px;}
h1{font-size:22px;font-weight:800;}
h1 .grad{background:linear-gradient(90deg,var(--accent),var(--accent2));-webkit-background-clip:text;background-clip:text;color:transparent;}
.sub{color:var(--muted);font-size:14px;margin:8px 0 4px;max-width:820px;}
.kicker{color:var(--accent2);font-size:11px;font-weight:700;text-transform:uppercase;letter-spacing:.09em;margin-bottom:6px;}
.backlink{display:inline-block;margin-bottom:10px;color:var(--accent2);font-size:13px;text-decoration:none;}
.backlink:hover{text-decoration:underline;}
.callout{background:var(--surface);border:1px solid var(--border);border-left:3px solid var(--accent);border-radius:8px;padding:11px 14px;margin:14px 0;font-size:13.5px;}
.callout .lbl{display:block;font-weight:700;color:var(--accent2);font-size:12px;margin-bottom:4px;}
.callout.warn{border-left-color:var(--yellow);} .callout.warn .lbl{color:var(--yellow);}
.callout a{color:var(--accent2);font-weight:700;}
/* ── controls ── */
.controls{display:grid;grid-template-columns:1fr 1fr 1fr;gap:14px;margin:12px 0;}
@media(max-width:820px){.controls{grid-template-columns:1fr;}}
.ctl{background:var(--surface);border:1px solid var(--border);border-radius:10px;padding:13px 15px;}
.ctl label{font-size:11px;color:var(--muted);font-weight:700;text-transform:uppercase;letter-spacing:.05em;display:block;}
.ctl input[type=range]{width:100%;margin-top:9px;accent-color:var(--accent);}
.ctl .v{font-family:'SF Mono',monospace;color:var(--accent2);font-weight:800;font-size:17px;}
.ctl .hint{color:var(--muted);font-size:11px;margin-top:5px;}
.ctl input[type=number]{width:88px;background:var(--bg);border:1px solid var(--border);color:var(--text);border-radius:7px;padding:6px 9px;font-family:'SF Mono',monospace;font-size:14px;margin-top:6px;}
.row{display:flex;gap:9px;flex-wrap:wrap;align-items:center;margin:10px 0;}
button.act{background:var(--accent);color:#fff;border:0;border-radius:8px;padding:10px 18px;font-weight:700;cursor:pointer;font-size:14px;}
button.act:hover{filter:brightness(1.1);} button.act:disabled{opacity:.45;cursor:not-allowed;}
button.ghost{background:var(--surface2);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:10px 16px;font-weight:700;cursor:pointer;font-size:13.5px;}
button.ghost:hover{border-color:var(--accent);}
.presets{display:flex;gap:8px;flex-wrap:wrap;margin:6px 0 2px;}
.presets button{background:var(--surface);color:var(--muted);border:1px solid var(--border);border-radius:9px;padding:8px 13px;font-size:13px;font-weight:700;cursor:pointer;}
.presets button:hover{color:var(--text);border-color:var(--accent);}
.pl{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;font-weight:700;margin-right:2px;align-self:center;}
/* ── generated text strip ── */
.gen{background:var(--surface);border:1px solid var(--border);border-radius:10px;padding:16px;margin:12px 0;min-height:58px;font-size:19px;line-height:1.7;}
.gen .tok{display:inline-block;padding:1px 4px;border-radius:5px;margin:1px 1px;}
.gen .tok.new{background:rgba(57,211,83,.22);color:var(--green);font-weight:700;animation:pop .35s ease;}
.gen .cursor{display:inline-block;width:9px;color:var(--accent2);animation:blink 1s steps(2,start) infinite;}
@keyframes pop{from{transform:scale(.6);opacity:0;}to{transform:scale(1);opacity:1;}}
@keyframes blink{to{visibility:hidden;}}
/* ── per-step distribution panel ── */
.panel{background:var(--surface);border:1px solid var(--border);border-radius:10px;padding:16px;margin:12px 0;}
.panel h3{font-size:13px;color:var(--accent2);text-transform:uppercase;letter-spacing:.05em;margin-bottom:4px;}
.panel .psub{color:var(--muted);font-size:12.5px;margin-bottom:12px;}
.panel .psub b{color:var(--text);}
.bars{display:flex;flex-direction:column;gap:4px;}
.brow{display:grid;grid-template-columns:96px 1fr 64px;align-items:center;gap:9px;font-size:12.5px;}
.brow .lab{font-family:'SF Mono',monospace;text-align:right;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:var(--text);}
.brow .track{height:18px;background:var(--surface2);border-radius:5px;overflow:hidden;position:relative;}
.brow .track > i{display:block;height:100%;background:linear-gradient(90deg,var(--accent),var(--accent2));border-radius:5px;transition:width .25s;}
.brow .pv{font-family:'SF Mono',monospace;color:var(--muted);font-size:11.5px;text-align:left;}
/* masked / picked states */
.brow.masked .lab{color:#5a626d;text-decoration:line-through;}
.brow.masked .track > i{background:#2b313a;}
.brow.masked .pv{color:#5a626d;}
.brow.pick .lab{color:var(--green);font-weight:800;}
.brow.pick .track > i{background:linear-gradient(90deg,var(--green),#7af0a0);}
.brow.pick .pv{color:var(--green);font-weight:700;}
.brow .tag{font-size:9.5px;font-weight:800;text-transform:uppercase;letter-spacing:.04em;padding:1px 5px;border-radius:4px;margin-left:6px;vertical-align:middle;}
.brow.pick .tag{background:rgba(57,211,83,.18);color:var(--green);}
.legend{display:flex;gap:16px;flex-wrap:wrap;font-size:11.5px;color:var(--muted);margin-top:12px;}
.legend .sw{display:inline-block;width:11px;height:11px;border-radius:3px;vertical-align:middle;margin-right:5px;}
.stepmeta{font-family:'SF Mono',monospace;font-size:11.5px;color:var(--muted);margin-top:10px;line-height:1.7;}
.stepmeta b{color:var(--accent2);}
/* ── 3-sample compare ── */
.samples{display:grid;grid-template-columns:1fr 1fr 1fr;gap:12px;margin-top:8px;}
@media(max-width:820px){.samples{grid-template-columns:1fr;}}
.scard{background:var(--bg);border:1px solid var(--border);border-radius:9px;padding:12px 14px;}
.scard .sh{font-size:11px;color:var(--muted);font-weight:700;text-transform:uppercase;letter-spacing:.05em;margin-bottom:7px;}
.scard .stxt{font-size:14.5px;line-height:1.6;color:var(--text);}
.mono{font-family:'SF Mono','Fira Code',monospace;}
details.note{margin:10px 0;}
details.note summary{cursor:pointer;color:var(--accent2);font-weight:700;font-size:13px;}
details.note .body{font-size:13px;color:var(--muted);margin-top:8px;line-height:1.6;}
details.note .body b{color:var(--text);}
</style>
</head>
<body>
<div class="wrap">
<a class="backlink" href="index.html">← All lessons</a>
<div class="kicker">Interactive lab · the generation loop</div>
<h1>Decoding Lab — <span class="grad">temperature, top-k & top-p, generating live</span></h1>
<p class="sub">A language model does not "write" a sentence — it repeatedly predicts a <b>probability distribution over the next token</b>, then <em>samples</em> one, appends it, and predicts again. The knobs you set on that sampling step — <b>temperature</b>, <b>top-k</b>, <b>top-p</b> — decide whether the text comes out crisp, creative, or complete nonsense. Below is a tiny hand-built next-token model (a Markov table over a small vocabulary). Set the knobs, hit <b style="color:var(--accent2)">▶ Generate</b>, and watch it sample one word at a time: the candidate distribution, what gets <span style="color:#5a626d;text-decoration:line-through">masked out</span>, and the <b style="color:var(--green)">token it picks</b>.</p>
<div class="callout"><span class="lbl">🧠 The one idea</span>
Generation = <b>repeatedly sampling from a distribution</b>. <b>Temperature</b> reshapes that distribution (low = sharp/confident, high = flat/random). <b>Top-k</b> and <b>top-p</b> throw away the unlikely tail <em>before</em> sampling, so the model can be random among <em>good</em> options without ever picking absurd ones. <b>Greedy</b> (always take the single most-likely token) is just the extreme T→0 case — and it gets stuck in loops.</p></div>
<!-- presets -->
<div class="presets">
<span class="pl">Try a preset:</span>
<button onclick="preset('greedy')">🧊 Greedy (T→0) → loops</button>
<button onclick="preset('balanced')">✅ Balanced (T=0.8, top-p .9)</button>
<button onclick="preset('hot')">🔥 High temperature → word-salad</button>
<button onclick="preset('nucleus')">🎯 Top-p clips the tail</button>
</div>
<!-- controls -->
<div class="controls">
<div class="ctl">
<label>Temperature — <span class="v" id="tv">0.80</span></label>
<input type="range" id="temp" min="0" max="2" step="0.05" value="0.8" oninput="syncControls()">
<div class="hint" id="th">low = sharp & repetitive · high = flat & random</div>
</div>
<div class="ctl">
<label>Top-k — keep the <span class="v" id="kv">8</span> most-likely</label>
<input type="range" id="topk" min="1" max="1" step="1" value="8" oninput="syncControls()">
<div class="hint" id="kh">cap the candidate set to the k best tokens</div>
</div>
<div class="ctl">
<label>Top-p (nucleus) — <span class="v" id="pv">0.90</span></label>
<input type="range" id="topp" min="0.05" max="1" step="0.05" value="0.9" oninput="syncControls()">
<div class="hint" id="ph">keep the smallest set whose prob mass ≥ p</div>
</div>
</div>
<div class="row">
<button class="act" id="gen" onclick="generate()">▶ Generate</button>
<button class="ghost" id="step" onclick="stepOnce()">Step ▸</button>
<button class="ghost" onclick="resetGen()">↺ Reset</button>
<button class="ghost" onclick="genThree()">⫶ Generate 3 samples (see the variance)</button>
<span style="margin-left:auto;display:flex;align-items:center;gap:7px;">
<label class="pl" style="margin:0;">Seed</label>
<input type="number" id="seed" value="42" min="0" max="999999" style="width:88px;background:var(--bg);border:1px solid var(--border);color:var(--text);border-radius:7px;padding:6px 9px;" class="mono" oninput="resetGen()">
<label class="pl" style="margin:0 0 0 8px;">Max len</label>
<input type="number" id="maxlen" value="14" min="3" max="24" style="width:64px;background:var(--bg);border:1px solid var(--border);color:var(--text);border-radius:7px;padding:6px 9px;" class="mono">
</span>
</div>
<!-- generated text -->
<div class="gen" id="genout"><span class="mono" style="color:var(--muted);font-size:14px;">Press ▶ Generate (or Step ▸) — text appears here, one sampled token at a time…</span></div>
<!-- per-step distribution -->
<div class="panel">
<h3>This step’s next-token distribution</h3>
<div class="psub" id="psub">After picking a token, the model conditions on it and produces a fresh distribution for the <b>next</b> token. The bars are probabilities <em>after</em> temperature; greyed bars were clipped by top-k / top-p and cannot be sampled.</div>
<div class="bars" id="bars"></div>
<div class="legend">
<span><span class="sw" style="background:linear-gradient(90deg,var(--accent),var(--accent2));"></span>candidate (eligible)</span>
<span><span class="sw" style="background:#2b313a;"></span>masked by top-k / top-p</span>
<span><span class="sw" style="background:linear-gradient(90deg,var(--green),#7af0a0);"></span>sampled pick</span>
</div>
<div class="stepmeta" id="stepmeta"></div>
</div>
<details class="note">
<summary>Show the renormalization math (4 steps, done exactly in this widget)</summary>
<div class="body">
Given raw model scores (logits) <span class="mono">z₁…z_V</span> for the current token, each step does:
<br>1. <b>Temperature</b>: divide logits by T, then softmax → <span class="mono">p_i = exp(z_i/T) / Σ exp(z_j/T)</span>. Small T sharpens (the top token dominates); large T flattens toward uniform. <span class="mono">T=0</span> is treated as pure greedy (argmax).
<br>2. <b>Top-k filter</b>: keep only the k tokens with the highest probability; set the rest to 0.
<br>3. <b>Top-p (nucleus) filter</b>: sort by probability descending, keep tokens until their cumulative mass first reaches p; drop the long tail. (The single most-likely token is always kept.)
<br>4. <b>Renormalize & sample</b>: divide the surviving probabilities by their new sum so they total 1, then draw one token with a seeded RNG. Greedy just takes the max instead of drawing.
</div>
</details>
<!-- 3 samples -->
<div class="panel">
<h3>Three samples, same knobs, three seeds</h3>
<div class="psub">Greedy gives the <b>same</b> sentence every time (no randomness). Turn temperature up and each draw diverges — that variance is exactly what sampling buys you, and why "the model said X" is never the whole story.</div>
<div class="samples" id="samples">
<div class="scard"><div class="sh">seed 42</div><div class="stxt mono" style="color:var(--muted)">— press “Generate 3 samples” —</div></div>
<div class="scard"><div class="sh">seed 7</div><div class="stxt mono" style="color:var(--muted)">—</div></div>
<div class="scard"><div class="sh">seed 123</div><div class="stxt mono" style="color:var(--muted)">—</div></div>
</div>
</div>
<div class="callout warn"><span class="lbl">⚖ Greedy ≠ sampled</span>
Greedy looks "safe" but it is the move that <b>repeats itself to death</b> — once it lands on a high-probability token, the same context keeps re-picking it (try the 🧊 preset). Real systems sample with a little temperature plus top-k/top-p so output is varied but stays on-distribution. Crank temperature with <em>no</em> top-p (the 🔥 preset) and you get the opposite failure: grammatical glue words collapse and the long improbable tail leaks in as word-salad.</p></div>
<div class="callout"><span class="lbl">📚 Where this fits</span>
This is the live <b>generation loop</b> that sits on top of the model’s output layer. See it inside a full forward pass in
<a href="llm_training_explainer.html">The LLM Training Explainer</a> (Step 9, Output → Logits → Softmax — that page shows one static temperature row; this lab is the autoregressive loop it feeds). To force the sampled token to obey a JSON schema or grammar — by <em>masking illegal tokens</em>, the same mechanism as top-k/top-p — see
<a href="topic_structured_generation.html">Structured Generation (constrained decoding)</a>.</p></div>
</div>
<script>
/* ───────────────────────── seeded RNG (LCG) ───────────────────────── */
function lcg(seed){ let s=(seed>>>0)||1; return ()=>{ s=(Math.imul(1103515245,s)+12345)>>>0; return s/4294967296; }; }
/* ───────────────────────── toy next-token model ─────────────────────────
A tiny hand-authored Markov table over a small vocabulary. Each entry maps the
CURRENT token to a list of [nextToken, weight] pairs. Weights act as un-normalized
logits-ish scores; we convert to a probability vector over the WHOLE vocab so the
bar chart always shows every token (most at ~0). Authored so that at low temperature
the chain produces vaguely coherent little sentences, and so a couple of tokens have
self-ish / high-probability loops that make greedy decoding visibly repeat. */
const VOCAB = ["<start>","the","a","cat","dog","robot","quietly","slowly","sat","ran","slept",
"watched","chased","near","on","the_mat","the_door","and","then","it","purred","barked",".",
"happy","tired","very","small","big","old","garden"];
const VI = {}; VOCAB.forEach((w,i)=>VI[w]=i);
const V = VOCAB.length;
// transitions: current token -> [[next, weight], ...]. Unlisted = tiny floor prob.
const T = {
"<start>": [["the",6],["a",4],["it",2]],
"the": [["cat",5],["dog",4],["robot",3],["garden",2],["the_mat",1],["the_door",1],["small",2],["big",2],["old",1]],
"a": [["cat",4],["dog",4],["robot",2],["small",3],["big",2],["happy",1]],
"small": [["cat",4],["dog",4],["robot",2],["garden",2]],
"big": [["dog",4],["cat",3],["robot",2],["garden",2]],
"old": [["dog",3],["cat",3],["robot",2],["garden",2]],
"cat": [["sat",5],["ran",4],["slept",3],["watched",3],["purred",2],["quietly",2],["slowly",1],["very",1]],
"dog": [["ran",5],["barked",4],["chased",4],["sat",3],["slept",2],["slowly",2],["very",1]],
"robot": [["watched",5],["slowly",4],["ran",3],["sat",3],["quietly",2],["very",1]],
"garden": [[".",4],["and",3],["near",2]],
"quietly": [["sat",4],["slept",4],["watched",3],["purred",2]],
"slowly": [["ran",4],["watched",3],["sat",3],["chased",2]],
"very": [["happy",4],["tired",4],["small",2],["big",2],["old",2]],
"sat": [["near",4],["on",4],["quietly",3],["and",3],[".",2],["very",1]],
"ran": [["near",4],["and",4],["quietly",2],[".",3],["slowly",2]],
"slept": [["quietly",3],["near",3],["and",3],[".",4]],
"watched": [["the",5],["it",3],["quietly",2],["and",2]],
"chased": [["the",6],["it",3],["a",2]],
"purred": [["and",3],["quietly",3],[".",4]],
"barked": [["and",3],["loudly_x",0],["at_x",0],[".",4],["then",2]],
"near": [["the",6],["it",2],["the_door",2],["the_mat",2]],
"on": [["the_mat",6],["the_door",3],["the",2]],
"the_mat": [[".",5],["and",3],["then",2]],
"the_door":[[".",5],["and",3],["then",2]],
"and": [["then",4],["the",3],["it",3],["slept",2],["sat",2],["purred",1],["barked",1]],
"then": [["it",5],["the",4],["a",2]],
"it": [["sat",4],["ran",3],["slept",3],["purred",2],["barked",2],["watched",2],["quietly",1]],
"happy": [[".",5],["and",3]],
"tired": [[".",5],["and",3]],
".": [["<end>",1]] // a period strongly tends to end the sentence
};
const FLOOR = 0.02; // tiny mass most tokens get, so nothing is exactly impossible pre-filter
// Sentence-control tokens are NOT given floor mass: otherwise a flat (high-T) distribution
// would keep sampling "." at random and end every sentence after two words, hiding the
// word-salad. They only appear where the transition table explicitly places them.
const NO_FLOOR = new Set([VI["."], VI["<end>"], VI["<start>"]]);
// Build a full-length probability vector (length V) for the given current token.
function baseProbs(curWord){
const w = new Array(V); for(let i=0;i<V;i++) w[i] = NO_FLOOR.has(i)? 0 : FLOOR;
const row = T[curWord];
if(row){ row.forEach(([nx,wt])=>{ if(VI[nx]!==undefined) w[VI[nx]] += wt; }); }
const s = w.reduce((a,b)=>a+b,0) || 1;
return w.map(x=>x/s);
}
/* ───────────────────────── decoding pipeline ─────────────────────────
probs(base) -> temperature -> top-k -> top-p -> renormalize -> sample.
Returns rich per-token info so the UI can draw masks + pick. */
function applyTemperature(p, temp){
// operate in log-space so this is exactly softmax(logits/T) with logits=log p.
if(temp <= 0.0001){ // greedy: all mass on argmax
let mi=0; for(let i=1;i<p.length;i++) if(p[i]>p[mi]) mi=i;
return p.map((_,i)=> i===mi?1:0);
}
const logits = p.map(x=>Math.log(x + 1e-12));
const scaled = logits.map(z=> z/temp);
const m = Math.max(...scaled);
const ex = scaled.map(z=>Math.exp(z-m));
const s = ex.reduce((a,b)=>a+b,0);
return ex.map(x=>x/s);
}
// returns {probs:[...adjusted], keep:[bool...], picked:idx, order:[idx sorted desc]}
function decodeStep(curWord, temp, k, p, rng){
const base = baseProbs(curWord);
const tprobs = applyTemperature(base, temp); // after temperature
const order = tprobs.map((_,i)=>i).sort((a,b)=>tprobs[b]-tprobs[a]);
const keep = new Array(V).fill(false);
const isGreedy = temp <= 0.0001;
if(isGreedy){
keep[order[0]] = true; // greedy keeps only the argmax
} else {
// top-k: the k highest-prob tokens
const kk = Math.max(1, Math.min(k, V));
const inK = new Set(order.slice(0, kk));
// top-p: smallest prefix (by desc prob) whose cumulative mass >= p; always keep #1
let cum=0; const inP = new Set();
for(let r=0;r<order.length;r++){
inP.add(order[r]); cum += tprobs[order[r]];
if(cum >= p) break;
}
for(let i=0;i<V;i++){ if(inK.has(i) && inP.has(i)) keep[i]=true; }
if(!keep.some(Boolean)) keep[order[0]]=true; // safety
}
// renormalize over kept
let z=0; for(let i=0;i<V;i++) if(keep[i]) z+=tprobs[i];
const finalP = tprobs.map((x,i)=> keep[i]? x/z : 0);
// sample
let picked;
if(isGreedy){ picked = order[0]; }
else {
const r = rng(); let acc=0; picked=order.find(i=>keep[i]); // fallback = best kept
for(let i=0;i<V;i++){ if(!keep[i]) continue; acc+=finalP[i]; if(r<=acc){ picked=i; break; } }
}
return {tprobs, finalP, keep, picked, order, isGreedy};
}
/* ───────────────────────── state + UI ───────────────────────── */
let seq, rng, finished, lastStep;
function curWord(){ return seq.length? seq[seq.length-1] : "<start>"; }
function getTemp(){ return parseFloat(document.getElementById('temp').value); }
function getK(){ return parseInt(document.getElementById('topk').value); }
function getP(){ return parseFloat(document.getElementById('topp').value); }
function getSeed(){ return parseInt(document.getElementById('seed').value)||0; }
function getMax(){ return parseInt(document.getElementById('maxlen').value)||14; }
function syncControls(){
document.getElementById('tv').textContent = getTemp().toFixed(2);
document.getElementById('kv').textContent = getK();
document.getElementById('pv').textContent = getP().toFixed(2);
const t=getTemp();
document.getElementById('th').textContent = t<=0.05?'≈ greedy: always the single most-likely token':(t<0.7?'sharp — confident, can repeat':(t<=1.05?'balanced':'flat — random, risks word-salad'));
}
function resetGen(){
seq = [];
rng = lcg(getSeed());
finished = false; lastStep = null;
renderText(-1);
document.getElementById('bars').innerHTML='';
document.getElementById('stepmeta').textContent='';
document.getElementById('gen').disabled=false;
document.getElementById('step').disabled=false;
}
function pretty(word){
if(word==="<start>") return "";
if(word==="<end>") return "";
if(word===".") return ".";
if(word==="the_mat") return "the mat";
if(word==="the_door") return "the door";
return word;
}
function renderText(newIdx){
const out=document.getElementById('genout');
if(!seq.length){ out.innerHTML='<span class="mono" style="color:var(--muted);font-size:14px;">Press ▶ Generate (or Step ▸) — text appears here, one sampled token at a time…</span>'; return; }
let html='';
seq.forEach((w,i)=>{ const txt=pretty(w); if(txt==="") return;
const sp = (txt==="."?'':' ');
html += (i===0?'':sp)+'<span class="tok'+(i===newIdx?' new':'')+'">'+txt+'</span>';
});
if(!finished) html += '<span class="cursor">▌</span>';
out.innerHTML = html;
}
function renderDist(step, curW){
const bars=document.getElementById('bars'); bars.innerHTML='';
// show tokens sorted by probability desc, top 14 rows (rest folded)
const SHOWN=14;
const rows = step.order.slice(0, SHOWN);
const maxP = Math.max(...step.tprobs);
rows.forEach(i=>{
const kept = step.keep[i], isPick = (i===step.picked);
const div=document.createElement('div');
div.className='brow'+(kept? (isPick?' pick':'') : ' masked');
const w = (maxP>0? step.tprobs[i]/maxP : 0)*100;
const tag = isPick? '<span class="tag">sampled</span>' : '';
div.innerHTML =
'<div class="lab">'+pretty(VOCAB[i]).replace(/^$/, VOCAB[i])+'</div>'+
'<div class="track"><i style="width:'+w.toFixed(1)+'%"></i></div>'+
'<div class="pv">'+(step.tprobs[i]*100).toFixed(1)+'%'+tag+'</div>';
bars.appendChild(div);
});
const hiddenCount = V - rows.length;
const keptN = step.keep.filter(Boolean).length;
const maskedShown = rows.filter(i=>!step.keep[i]).length;
document.getElementById('psub').innerHTML = 'Conditioned on <b class="mono">'+(pretty(curW)||curW)+'</b> → distribution over the next token. '+
'<b>'+keptN+'</b> token'+(keptN===1?'':'s')+' survived the filters; the rest are <span style="color:#5a626d">masked</span> and cannot be sampled.';
document.getElementById('stepmeta').innerHTML =
'temperature <b>'+getTemp().toFixed(2)+'</b> · top-k <b>'+(step.isGreedy?'—':getK())+'</b> · top-p <b>'+(step.isGreedy?'—':getP().toFixed(2))+'</b>'+
(step.isGreedy? ' · <b>greedy</b> (argmax, no sampling)' : '')+
' | eligible candidates: <b>'+keptN+'</b> · masked (shown): <b>'+maskedShown+'</b>'+(hiddenCount>0? ' · '+hiddenCount+' more near-zero tokens hidden':'');
}
function stepOnce(){
if(finished){ return; }
const curW = curWord();
const step = decodeStep(curW, getTemp(), getK(), getP(), rng);
lastStep = step;
const nextWord = VOCAB[step.picked];
// render the distribution that LED to this pick (conditioned on curW)
renderDist(step, curW);
if(nextWord==="<end>"){ finished=true; renderText(-1);
document.getElementById('gen').disabled=true; document.getElementById('step').disabled=true; return; }
seq.push(nextWord);
renderText(seq.length-1);
if(nextWord==="." || seq.length>=getMax()){ finished=true;
setTimeout(()=>renderText(-1),50);
document.getElementById('gen').disabled=true; document.getElementById('step').disabled=true; }
}
let animTimer=null;
function generate(){
if(animTimer){ clearInterval(animTimer); animTimer=null; }
resetGen();
document.getElementById('gen').disabled=true;
animTimer=setInterval(()=>{
stepOnce();
if(finished){ clearInterval(animTimer); animTimer=null; }
}, 520);
}
/* run a full generation silently for a given seed; return the joined sentence */
function runSilent(seed){
let s=[], r=lcg(seed), fin=false, guard=0;
while(!fin && guard<40){ guard++;
const cw = s.length? s[s.length-1]:"<start>";
const st = decodeStep(cw, getTemp(), getK(), getP(), r);
const nw = VOCAB[st.picked];
if(nw==="<end>"){ break; }
s.push(nw);
if(nw==="." || s.length>=getMax()) break;
}
return s.map(pretty).filter(x=>x!=="").join(' ').replace(/ \./g,'.');
}
function genThree(){
const seeds=[42,7,123];
const cards=document.querySelectorAll('#samples .scard');
seeds.forEach((sd,i)=>{
const txt=runSilent(sd);
cards[i].querySelector('.sh').textContent='seed '+sd;
const el=cards[i].querySelector('.stxt');
el.style.color='var(--text)'; el.textContent = txt || '(empty)';
});
}
/* ───────────────────────── presets ───────────────────────── */
function setControls(temp,k,p){
document.getElementById('temp').value=temp;
document.getElementById('topk').value=k;
document.getElementById('topp').value=p;
syncControls(); resetGen();
}
function setSeedLen(seed,len){
document.getElementById('seed').value=seed;
document.getElementById('maxlen').value=len;
}
function preset(name){
if(name==='greedy'){ setSeedLen(42,16); setControls(0, V, 1); } // greedy → repetition loop (longer so the loop is obvious)
if(name==='balanced'){ setSeedLen(42,14); setControls(0.8, 8, 0.9); }
if(name==='hot'){ setSeedLen(9,18); setControls(2.0, V, 1); } // max T, no clipping, long → word-salad
if(name==='nucleus'){ setSeedLen(7,16); setControls(1.0, V, 0.5); } // top-p truncates the tail hard
setTimeout(generate, 60);
}
/* ───────────────────────── init ───────────────────────── */
(function(){
// top-k slider max = vocab size
const ks=document.getElementById('topk'); ks.max=V; ks.value=8;
syncControls();
resetGen();
})();
</script>
</body>
</html>