diff --git a/.research/events.jsonl b/.research/events.jsonl
index a8ab511..7f15bd9 100644
--- a/.research/events.jsonl
+++ b/.research/events.jsonl
@@ -7,3 +7,4 @@
 {"id": "5e9641ed983f4c599d8629bf643ee27b", "ts": "2026-10-16T15:52:32+09:00", "task": "NPUTRAIN-26", "result": "민감정보 제거·ai-rm.yaml·README ai-rm 절 추가, checker clean, push ad02402. examples/cluster_facts.py IP 3개(NAS·게이트웨이·Prometheus)를 MODEL_STORE_NAS_HOST/LLMD_GATEWAY_HOST/PROMETHEUS_HOST/PORT 환경변수(cluster.local.env)로, cli.py serve 예시 IP를 <vllm-host>로 교체. 생성 데이터는 이전과 동일 확인", "next": "lmd-top 내 vendored 사본도 같은 수정 필요(다른 agent). 정적 k8s 매니페스트·전용 이미지 없음", "state": "idle", "done": true, "blocker": "", "metrics": {}, "params": {}, "artifacts": [], "experiments": [], "labels": ["infra"], "review": false, "todos": ["deploy/k8s 정적 Job 매니페스트(kustomize) 추가"], "note": "", "agent": "claude", "host": "control-node", "branch": "main", "commit": "ad02402-dirty"}
 {"id": "46f3b37355d0471bb25b6f222771b396", "ts": "2026-10-16T17:02:12+09:00", "task": "NPUTRAIN-19", "result": "8B full FT가 RNGD 1장에서 돈다: S=512 accum1 10.33 s/step(49.6 tok/s). 계산 ~1.5 s, 나머지는 기울기 14 GB D2H(3.1 GB/s)+가중치 H2D 전송 대기 — PCIe 병목", "next": "accum 4 그래프 컴파일(npu-node-1, ~2h) 후 step 측정; 2카드 레이어 분할로 링크 2개 활용", "state": "active", "done": false, "blocker": "", "metrics": {"step_time_s": "10.33"}, "params": {"model": "Llama-3.1-8B-Instruct", "mode": "full", "seq_len": "512", "accum": "1", "device": "rngd", "node": "e11", "offload": "pipeline4"}, "artifacts": [{"path": "experiments/data/pipe8b-rngd.jsonl", "size": 17870, "sha256": "d3d4d7a91e5e7d48"}], "experiments": ["experiments/2026-10-16-full8b-offload-step.md"], "labels": ["benchmark", "fine-tuning", "rngd"], "review": false, "todos": [], "note": "", "agent": "claude", "host": "control-node", "branch": "main", "commit": "5de85aa-dirty"}
 {"id": "1f544207d4084d928f2bad7d2d298be7", "ts": "2026-10-16T18:50:23+09:00", "task": "NPUTRAIN-19", "result": "8B full FT RNGD 1장 accum 4: 13.13 s/step(2048 tok) = 156 tok/s, accum 1(49.6 tok/s)의 3.1배. 전송 고정비 ~8.6 s는 그대로", "next": "A100 8B full FT 대조(e11 GPU 비면); 카드 2장 레이어 분할(NPUTRAIN-18)", "state": "active", "done": false, "blocker": "", "metrics": {"step_time_s": "13.13"}, "params": {"model": "Llama-3.1-8B-Instruct", "mode": "full", "seq_len": "512", "accum": "4", "device": "rngd", "node": "e11", "offload": "pipeline4"}, "artifacts": [{"path": "experiments/data/pipe8b4-rngd.jsonl", "size": 20585, "sha256": "0426259b65b231a8"}], "experiments": ["experiments/2026-10-16-full8b-offload-step.md"], "labels": ["benchmark", "fine-tuning", "rngd"], "review": false, "todos": [], "note": "", "agent": "claude", "host": "control-node", "branch": "main", "commit": "5de85aa-dirty"}
+{"id": "9f702fdb114e4b25b23508bb2bab2d14", "ts": "2026-10-21T15:37:39+09:00", "task": "#34 resident device AdamW + host-throughput step model", "result": "compile --adamw-only 150 s (was >2.5 h); 2 RNGD x 8 resident layers 4.57 s/step vs 1-card offload 5.22 s (+14% tok/s, steps 3-7, intermittent OOM #40); --update-workers 8 -8%; host-throughput model within 15% on 6 configs", "next": "fix resident OOM (#40), then 200-step run", "state": "active", "done": false, "blocker": "", "metrics": {"tok_s": "448"}, "params": {"seq_len": "512", "accum": "4", "resident_layers": "8", "cards": "2", "sdk": "furiosa-torch-2026.2.0"}, "artifacts": [], "experiments": ["experiments/2026-10-21-resident-adamw.md"], "labels": ["benchmark"], "review": true, "todos": [], "note": "", "agent": "claude", "host": "control-node", "branch": "perf/34-resident-adamw-compile", "commit": "8539644-dirty"}
diff --git a/README.ko.md b/README.ko.md
index d8bee0b..fb34f3b 100644
--- a/README.ko.md
+++ b/README.ko.md
@@ -137,6 +137,8 @@ npu-lora eval  --endpoint http://vllm:8000 --model my-adapter --data eval.jsonl
 첫 실행은 학습 그래프를 조각 단위로 컴파일하고(Llama-8B: 64코어 호스트에서 약 1.5시간) `~/.cache/npu-lora`에
 캐시한다. 이후 실행은 바로 시작한다. `npu-lora compile`은 장치를 열지 않고 캐시를 채우므로 아무 호스트에서나
 컴파일한 뒤 캐시를 복사할 수 있다.
+`npu-lora compile --adamw-only --full --model M`은 `--resident-layers N` 전체 파인튜닝이 쓰는 장치 AdamW 그래프만
+컴파일한다(Llama-8B: 그래프 4개, 150초). 가중치와 학습 그래프는 읽지 않는다.
 
 <details>
 <summary><b>전체 파인튜닝</b></summary>
diff --git a/README.md b/README.md
index 0c455cf..1ef09e7 100644
--- a/README.md
+++ b/README.md
@@ -138,6 +138,8 @@ npu-lora eval  --endpoint http://vllm:8000 --model my-adapter --data eval.jsonl
 The first run compiles the training graphs part by part (Llama-8B: ~1.5 h on a 64-core host) and
 caches them under `~/.cache/npu-lora`; later runs start immediately. `npu-lora compile` fills the
 cache without opening a device, so you can compile on any host and copy the cache.
+`npu-lora compile --adamw-only --full --model M` compiles only the device AdamW graphs that full fine-tuning
+with `--resident-layers N` uses (Llama-8B: 4 graphs, 150 s), without loading weights or the training graphs.
 
 <details>
 <summary><b>Full fine-tuning</b></summary>
diff --git a/STATUS.md b/STATUS.md
index f315d17..d83d00e 100644
--- a/STATUS.md
+++ b/STATUS.md
@@ -1,12 +1,12 @@
 # STATUS — RNGD NPU 파인튜닝 학습
 
 <!-- research-hub:header:begin (maintained by .research/report.py; keep the "- **Key**: value" format) -->
-- **State**: idle
-- **Updated**: 2026-10-19 09:53+09:00
+- **State**: active
+- **Updated**: 2026-10-21 15:37+09:00
 - **Agent**: claude@control-node
-- **Task**: NPUTRAIN-32 (done)
-- **Last result**: PR #1 갱신(931a189, airm/v0.1): npu_lora_tokens_per_second를 airm_tok_s와 함께 프로토콜 0.2 전까지 재출력(README deprecated 표시), rm-core v0.1.1. 검증: conformance PASS 3/FAIL 0, test_airm 4/4, test_cpu 11/11, helm template OK, check_sensitive clean. airm.workloads conformance는 0.1.1에서도 KeyError(NPUTRAIN-31)
-- **Next**: PR 머지 후 lmd-top을 airm_tok_s 우선 읽기로 전환(NPUTRAIN-30), 프로토콜 0.2에서 예전 시리즈 제거
+- **Task**: #34 resident device AdamW + host-throughput step model
+- **Last result**: compile --adamw-only 150 s (was >2.5 h); 2 RNGD x 8 resident layers 4.57 s/step vs 1-card offload 5.22 s (+14% tok/s, steps 3-7, intermittent OOM #40); --update-workers 8 -8%; host-throughput model within 15% on 6 configs
+- **Next**: fix resident OOM (#40), then 200-step run
 - **Blockers**: none
 <!-- research-hub:header:end -->
 
@@ -42,6 +42,15 @@
 
 <!-- research-hub:log (report.py inserts new entries below this line, newest first) -->
 
+### 2026-10-21 15:37+09:00 — #34 resident device AdamW + host-throughput step model
+
+- Result: compile --adamw-only 150 s (was >2.5 h); 2 RNGD x 8 resident layers 4.57 s/step vs 1-card offload 5.22 s (+14% tok/s, steps 3-7, intermittent OOM #40); --update-workers 8 -8%; host-throughput model within 15% on 6 configs
+- Next: fix resident OOM (#40), then 200-step run
+- Metrics: tok_s=448
+- Params: seq_len=512, accum=4, resident_layers=8, cards=2, sdk=furiosa-torch-2026.2.0
+- Experiments: experiments/2026-10-21-resident-adamw.md
+- By: claude@control-node · perf/34-resident-adamw-compile@8539644-dirty
+
 ### 2026-10-19 09:53+09:00 — NPUTRAIN-32 ✅
 
 - Result: PR #1 갱신(931a189, airm/v0.1): npu_lora_tokens_per_second를 airm_tok_s와 함께 프로토콜 0.2 전까지 재출력(README deprecated 표시), rm-core v0.1.1. 검증: conformance PASS 3/FAIL 0, test_airm 4/4, test_cpu 11/11, helm template OK, check_sensitive clean. airm.workloads conformance는 0.1.1에서도 KeyError(NPUTRAIN-31)
diff --git a/docs/design.md b/docs/design.md
index 9a9e9ec..2fb456d 100644
--- a/docs/design.md
+++ b/docs/design.md
@@ -66,6 +66,51 @@ Llama-3.1-8B full fine-tuning, S=512, one card (tokens/s):
 
 The remaining ~3 s per step is shared between host AdamW over 7B parameters and the transfers.
 
+## Optimizer placement: step-time model
+
+With resident host buffers the offload path is bound by the host, not by the PCIe links: splitting the
+layers over two cards (two links) made the step slower on a 24-core host (6.36 → 9.26 s), where a model
+that charges transfer bytes / link bandwidth predicted 5.6 s
+([two-card record](../experiments/2026-10-21-two-card-split.md)). The model below charges host time instead.
+
+Layers are either *offloaded* (fp32 master and AdamW state on the host) or *resident* (`--resident-layers N`:
+the first N layers of each card keep fp32 master, m and v on the card and update with the `ResidentAdamW` graph).
+
+```
+T_step ≈ T_dev + T_res + T_wait
+T_res  = N_res · t_dev                      device AdamW, main thread (8B layer: 28–33 ms)
+T_wait = max(0, N_off · (t_a + t_u) / W  −  T_res  −  N_res,first · t_fwd)
+```
+
+- `T_dev`: forward + backward of the step on the device(s), including the hidden-state hops between cards.
+  Measure it once as step − `wait_weights_s` of an offload run on the same cards.
+- `t_a`, `t_u`: host AdamW and host→device upload time per offloaded layer, measured with W worker threads
+  (`--update-workers`, default 4) as `prev_adamw_s / 32` and `prev_upload_s / 32` of the offload run. They are
+  host throughput terms: on a contended or small host they grow, and adding cards does not shrink them.
+- Gradient downloads overlap the backward pass and are not on the critical path. The layer jobs start
+  when the global clip is known, and the pool runs them in backward order (layer 31 first), so the next
+  forward waits for nearly all offloaded layers. Resident layers sit at the front of each card's range,
+  so the next forward runs through them (`t_fwd` each) while the host jobs finish.
+
+Check on RNGD, Llama-3.1-8B, S=512, accum 4 (host terms from the one-card offload run of the same host;
+records: [resident AdamW](../experiments/2026-10-21-resident-adamw.md), [two cards](../experiments/2026-10-21-two-card-split.md)):
+
+| host, configuration | predicted | measured | error |
+| --- | ---: | ---: | ---: |
+| 96 cores, 1 card, offload, W=8 (per-layer terms of the W=4 run) | 4.53 s | 4.82 s | −6 % |
+| 96 cores, 1 card, 4 resident layers (steps 6–12; stopped at 12, #40) | 5.10 s | 5.12 s | −0.4 % |
+| 96 cores, 2 cards, 8 resident layers per card (steps 3–7; the run stopped at 7, #40) | 4.33 s | 4.57 s | −5 % |
+| 24 cores, 2 cards, offload | 8.33 s | 9.26 s | −10 % |
+
+The link-bandwidth model predicted the last row as 5.6 s (−40 %).
+
+What the model says: once half of the layers are resident, `T_wait` is a few hundred ms and `T_dev`
+(3.6 s on two cards) is the step. More resident layers only help until `T_wait` reaches zero; after that the
+device compute path, not the optimizer, is the next limit. A card holds about 9 resident 8B layers next to
+its share of bf16 weights and gradients, so all 32 layers resident needs 4 cards.
+Resident layers currently hit an intermittent device out-of-memory error after 3–13 steps
+([#40](https://github.com/ai-computing/npu-train/issues/40)).
+
 ## Job layout on Kubernetes
 
 furiosa-torch opens a card only from a privileged container, which then sees every RNGD on the node.
diff --git a/experiments/2026-10-21-resident-adamw.md b/experiments/2026-10-21-resident-adamw.md
new file mode 100644
index 0000000..0adc7bd
--- /dev/null
+++ b/experiments/2026-10-21-resident-adamw.md
@@ -0,0 +1,156 @@
+# 8B 전체 파인튜닝: 카드 상주 AdamW 컴파일·측정과 host 처리량 step 모델
+
+- **Date**: 2026-10-21
+- **Task**: #34
+- **Host / device**: npu-node-1 RNGD 2장(한 pod 가 `furiosa.ai/rngd: 2` 로 배정받은 두 장; 1장 실행은 그중 한 장), 노드 CPU 96코어,
+  `furiosaai/furiosa-llm:latest`(furiosa-torch 2026.2.0, torch 2.10.0+cpu, transformers 5.1.0), 드라이버·펌웨어 2026.4.0.
+  실행 전 `/proc/*/fd` 에 `/dev/rngd` 를 연 프로세스가 없음을 확인했다. 컴파일은 장치 없는 CPU 노드(요청 16코어).
+  npu-node-2 는 쓰지 않았다.
+- **Commit**: 브랜치 `perf/34-resident-adamw-compile`(main `833fef7` 위). pod 에는 작업 트리의 `npu_lora/` 를 ConfigMap 으로 올렸다.
+- **주의 — host 경합**: 14:15 KST 무렵부터 같은 노드에서 다른 사용자의 CPU 작업(nice, load ≈ 100/96코어)이 돌았다.
+  그 전 실행(아래 "경합 없음")과 그 뒤 실행("경합")은 직접 비교하지 않는다. load 는 `data/resident-adamw/load-npu-node-1.txt`(60 s 간격).
+
+## Question
+
+1. 상주 AdamW 그래프(`--resident-layers N`)가 2.5 시간 안에 컴파일되지 않은 이유는? 고칠 수 있는가?
+2. 카드당 상주 레이어 약 8개로 1장·2장 step 시간은? loss 는 오프로드 실행과 같은가?
+3. 링크 대역 대신 host 복사·AdamW 처리량으로 쓴 step 모델이 측정을 15 % 안으로 예측하는가?
+4. (부가) host AdamW 작업 스레드를 늘리면 빨라지는가?
+
+## Setup
+
+- **컴파일이 느렸던 이유**: `npu-lora compile --resident-layers 1` 은 상주 AdamW 그래프 전에 학습 그래프 경로 전체를 다시 돈다 —
+  8B 가중치 읽기, fp32 master 32 GB, 레이어 그래프 3종 추적(`graph.build` 가 CPU 에서 변환 전후를 실행해 비교), 그리고
+  `compile_parts` 가 조각마다 CPU 에서 샘플을 실행해 입력 모양을 얻는다. 조각이 모두 캐시에 있어도 이 CPU 실행은 매번 한다.
+  10-21 로그(`c-res8b`)에서 순전파 11조각은 캐시에서 0 s 에 읽혔고, 그다음 역전파 조각 실행에서 2시간 넘게 출력이 없었다
+  (공유 CPU 노드, 요청 8코어, 같은 노드에 컴파일 작업 여러 개). AdamW 그래프 자체는 가중치 모양에만 의존한다.
+- **고침**: `npu-lora compile --adamw-only` — 가중치를 읽지 않고 safetensors 헤더에서 레이어의 2차원 가중치 모양만 얻어
+  (`Weights.shape`, `optim_device.resident_shapes`) 모양마다 `ResidentAdamW` 그래프만 컴파일해 캐시에 넣는다. trainer 도 같은 함수로 모양을 정한다.
+- **카드 배정**: 카드 3장 노드에서 2장 pod 는 `--backend furiosa:0,furiosa:1` 이 배정받지 않은 카드를 열 수 있었다(서수 0·1 = 노드의 첫 두 장).
+  `--backend furiosa:auto,furiosa:auto` 가 init 컨테이너가 적은 배정 카드 목록(`npu1,npu3` 같은)에서 차례로 고르게 했다. 예전 코드는
+  이 목록을 숫자 하나로 읽었다(`npu1,npu3` → 13).
+- **메모리**: 상주 갱신이 새 bf16 가중치를 만들기 전에 옛 장치 가중치를 놓고(`_layer_job` 과 같게), 끝난 레이어의 누적 기울기를 바로 놓는다.
+- `--update-workers N`(기본 4): 레이어별 host 작업(기울기 D2H → AdamW → bf16 H2D) 스레드 수.
+- 측정: Llama-3.1-8B-Instruct, S=512, accum 4, lr 1e-5, nsmc_mix, seed 0 — two-card 기록과 같은 레시피. 25 step, 6–25 step 중앙값
+  (OOM 으로 일찍 멈춘 실행은 남은 구간, 표에 적었다).
+
+## Command
+
+```bash
+# 상주 AdamW 그래프만 (장치 없는 CPU 노드)
+python3 -m npu_lora.cli compile --adamw-only --full --model meta-llama/Llama-3.1-8B-Instruct --seq-len 512 --cache-dir /cache/furiosa
+# 학습 (RNGD pod; 2장은 furiosa.ai/rngd: 2 와 --backend furiosa:auto,furiosa:auto)
+python3 -m npu_lora.cli train --full --model meta-llama/Llama-3.1-8B-Instruct --data /mnt/store/datasets/quality/nsmc_mix_train.jsonl \
+  --cache-dir /cache/furiosa --seq-len 512 --steps 25 --accum 4 --lr 1e-5 --seed 0 --run-name ntb-<run> --out /tmp/out \
+  --backend furiosa:auto [--resident-layers N] [--update-workers 8]
+```
+
+Job 생성기는 `experiments/data/w7/jobgen.py` 와 같다(라벨·로그 경로만 바꿈).
+
+## Results
+
+### 1. 컴파일 — 2.5 h 이상 → 150 s
+
+| 그래프 | 모양 | 컴파일 |
+|---|---|---:|
+| `adamw-res-fp32` | 1024×4096 (k, v) | 4 s |
+| | 4096×4096 (q, o) | 15 s |
+| | 4096×14336 (down) | 58 s |
+| | 14336×4096 (gate, up) | 73 s |
+| 합 | | **150 s** |
+
+학습 실행은 이 캐시를 그대로 읽는다(`[furiosa] adamw-res-fp32-…: cache 0s`).
+
+### 2. step 시간 (경합 없음, 13:15–14:11 KST)
+
+| 실행 | 카드 | 상주/카드 | 구간 | step | tok/s | 대기 | D2H 합 | AdamW 합 | H2D 합 | 상주 갱신 |
+|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|
+| 오프로드 | 1 | 0 | 6–25 | **5.22 s** | 392 | 1.50 | 2.22 | 4.18 | 2.27 | — |
+| 오프로드, `--update-workers 8` | 1 | 0 | 6–25 | **4.82 s** | 425 | 1.23 | 3.21 | 8.05 | 3.62 | — |
+| 오프로드 | 2 | 0 | 6–25 | **5.09 s** | 402 | 1.50 | 2.21 | 4.21 | 2.30 | — |
+| 상주 | 2 | 8 | 3–7 (7 에서 OOM) | **4.57 s** | 448 | 0.15 | 1.12 | 4.19 | 1.19 | 0.52 |
+| 상주 | 1 | 4 | 6–12 (12 에서 OOM) | **5.12 s** | 400 | 1.04 | 1.93 | 4.22 | 1.96 | 0.18 |
+| 상주 | 1 | 8 | — | 첫 step 에서 OOM(카드 1장에 약 50 GB, 용량 초과) | | | | | | |
+
+(합은 레이어 작업 시간을 스레드에 걸쳐 더한 값. 상주 레이어는 host 작업이 없어서 2장 상주 실행의 합은 오프로드 16층 몫이다.)
+
+- **2장 + 카드당 8층 상주: 4.57 s/step, 1장 오프로드(5.22 s) 대비 +14 % tok/s.** 대기가 1.50 → 0.15 s 로 거의 사라졌고,
+  step 은 장치 계산(약 3.6 s)과 상주 갱신(0.52 s, 16층 × 33 ms)이 정한다. 단 OOM 으로 5 step 만 쟀다(아래).
+- 이 96코어 host 에서는 2장 오프로드가 1장과 같다(5.09 대 5.22 s). 24코어 npu-node-2 에서 2장이 31 % 느렸던 것(two-card 기록)과 다르다 —
+  host 처리량이 충분하면 분할 자체의 손해는 거의 없다.
+- **host 개선: 작업 스레드 4 → 8 로 −8 %(5.22 → 4.82 s).** 레이어당 AdamW 는 경합으로 0.131 → 0.252 s 로 늘지만 병렬도가 두 배라 대기가 줄었다.
+
+### 3. loss — 오프로드와 같은가
+
+| step | 1 | 2 | 3 | 4 | 5 |
+|---|---:|---:|---:|---:|---:|
+| 오프로드(1장·2장, 스레드 4·8 모두 같음) | 1.7157 | 1.5626 | 1.0879 | 1.1060 | 1.3361 |
+| 2장 상주 8 | 1.7157 | 1.5649 | 1.0858 | 1.1035 | 1.3369 |
+| 1장 상주 4 | 1.7157 | 1.5645 | 1.0821 | 1.1063 | 1.3412 |
+
+오프로드 실행끼리는 비트 단위로 같고, 상주 실행은 step 2 부터 최대 0.5 %(step 5, 상주 4) 다르다. 장치 AdamW 는 host fused AdamW 와 갱신량 기준
+1e-5 수준으로 같지만(10-19 기록), warmup 초반 lr 1e-6 에서 갱신이 bf16 가중치 한 ulp 보다 작은 원소가 대부분이라 fp32 차이가 bf16 반올림을
+바꾼 원소들이 loss 를 움직인 것으로 본다(검증은 하지 않았다). 같은 코드의 CPU 테스트(`test_two_device_split_and_resident_adamw_equal_one_device`)는
+갱신량 상대차 < 5e-2 로 통과한다.
+
+### 4. 상주 실행의 장치 OOM (#40)
+
+상주 실행은 계획 메모리가 용량(47.5 GiB) 아래인데도 몇 step 뒤 `failed to allocate … bytes on DRAM_SHARED` 로 멈췄다. 실패 위치는 레이어 작업 스레드의
+bf16 업로드(`to_device`)다.
+
+| 실행 | 계획 메모리/카드 | 멈춘 step |
+|---|---:|---:|
+| 2장 상주 8 (옛 가중치 놓기 전) | ~36 GB | 3 |
+| 2장 상주 8 | ~36 GB | 7, 7(경합), 13(경합), 완주 1회(경합) |
+| 2장 상주 6 (경합) | ~31 GB | 7 |
+| 1장 상주 4 | ~40 GB | 12 |
+| 1장 상주 3 (경합) | ~37 GB | 완주 |
+| 2장 오프로드 | ~15 GB | 완주(2회) |
+
+- 각 조각을 따로 반복하면 늘지 않는다: [14336, 4096] 상주 AdamW 그래프 400회(상태 교체), 같은 모양 bf16 업로드 400회, [1,1] 스칼라 업로드 2만 회 모두 OOM 없음.
+- `furiosa-smi status` 는 실행 내내 0.00/47.50 GiB 로 보여서 밖에서 메모리를 볼 수 없었다.
+- 상주 갱신을 오프로드 업로드보다 먼저 돌리는 순서(조각화 가설)도 시험했지만 결과가 섞여(2장 상주 8: 13, 완주; 2장 상주 6: 7) 되돌렸다.
+  `r2-res8-try3`, `r2-res8-mem`, `r2-res6`, `r1-res3` 이 그 변형이고, 나머지와 PR 코드는 오프로드 업로드와 상주 갱신이 겹치는 원래 순서다.
+  원인은 모른다 — #40 에서 따로 본다.
+
+### 5. 경합 host 에서 같은 조건 비교 (15:00–15:25 KST)
+
+host load 가 약 100(96코어)인 동안 PR 코드로 2장 상주 8 과 2장 오프로드를 이어서 돌렸다.
+
+| 실행 | 구간 | step | tok/s | 대기 | AdamW 합 | H2D 합 | 상주 갱신 |
+|---|---|---:|---:|---:|---:|---:|---:|
+| 2장 오프로드 | 6–25 | 10.96 s | 187 | 2.66 | 9.27 | 3.39 | — |
+| 2장 상주 8 | 6–7 (7 에서 OOM) | 8.83 s | 232 | 0.27 | 8.67 | 1.54 | 0.84 |
+
+경합 host 에서도 상주가 대기를 2.66 → 0.27 s 로 없애고 step −19 %(+24 % tok/s)다. 레이어당 host AdamW 가 경합 없을 때의 2–4배(0.29 s)라
+host 항을 없애는 이득이 더 크다. 이 조건의 항(T_dev 8.30 s, t_a + t_u 0.396 s)으로 모델은 9.71 s 를 예측한다(측정 8.83 s, +10 %).
+같은 경합 조건에서 이보다 앞서 다른 순서(상주 갱신 먼저) 변형이 25 step 을 완주했다: 9.48 s/step, 216 tok/s(`r2-res8-mem.jsonl`).
+
+### 6. step 모델 (docs/design.md 「Optimizer placement: step-time model」)
+
+`T_step ≈ T_dev + T_res + max(0, N_off·(t_a + t_u)/W − T_res − N_res,first·t_fwd)`.
+`T_dev` 는 같은 카드 구성의 오프로드 실행에서 step − 대기, `t_a`·`t_u` 는 같은 host 1장 오프로드 실행의 레이어당 host AdamW·업로드
+(0.131 + 0.071 s, W=4), 장치 AdamW 는 10-19 측정 28 ms/층, `t_fwd` = T_dev / (32 × accum × 3).
+기울기 D2H 는 역전파와 겹쳐 임계 경로에 없다.
+
+| 구성 | 예측 | 측정 | 오차 |
+|---|---:|---:|---:|
+| 96코어, 1장 오프로드, W=8 (W=4 의 레이어당 값) | 4.53 s | 4.82 s | −6 % |
+| 96코어, 1장 상주 4 | 5.10 s | 5.12 s | −0.4 % |
+| 96코어, 2장 상주 8 | 4.33 s | 4.57 s | −5 % |
+| 24코어(npu-node-2), 1장 오프로드 (대기 항) | 5.89 s | 6.36 s | −7 % |
+| 24코어(npu-node-2), 2장 오프로드 | 8.33 s | 9.26 s | −10 % |
+| 96코어 경합, 2장 상주 8 (같은 조건 2장 오프로드의 항) | 9.71 s | 8.83 s | +10 % |
+
+모두 15 % 안이다(경합 줄은 2 step 구간이라 잡음이 크다). 링크 대역 모델은 마지막 줄을 5.6 s(−40 %)로 예측했다.
+예: 2장 상주 8 은 T_dev 3.60 s(2장 오프로드 5.09 − 1.50) + T_res 16 × 0.028 = 0.45 s + 대기 max(0, 16 × 0.202 / 4 − 0.45 − 8 × 0.0094) = 0.28 s = 4.33 s.
+
+Raw: `experiments/data/resident-adamw/*.jsonl`(step 기록; `*-try*`·`*-oom` 은 OOM 으로 멈춘 실행, `.err` 는 그 오류 줄),
+`load-npu-node-1.txt`(host load).
+
+## Conclusion
+
+상주 AdamW 그래프는 학습 그래프 경로를 건너뛰는 `compile --adamw-only` 로 150 s 에 컴파일된다(2.5 h 이상에서). 2장에 카드당 8층을 상주시키면
+오프로드 대기가 1.5 → 0.15 s 로 사라져 4.57 s/step(+14 %)이지만, 남은 step 은 장치 계산 경로(3.6 s)가 정하고, 상주 실행은 아직 몇 step 뒤
+장치 OOM 이 간헐적으로 난다(#40) — 학습에 쓰려면 그것부터 고쳐야 한다. host 쪽에서는 작업 스레드를 8 로 늘리는 것만으로 −8 % 를 얻었다.
+host 복사·AdamW 처리량으로 쓴 step 모델은 측정한 여섯 구성을 −10 ~ +10 % 로 예측한다.
diff --git a/experiments/2026-10-21-two-card-split.md b/experiments/2026-10-21-two-card-split.md
index 8db662d..35a1de1 100644
--- a/experiments/2026-10-21-two-card-split.md
+++ b/experiments/2026-10-21-two-card-split.md
@@ -56,6 +56,10 @@ python3 -m npu_lora.cli compile --full --model meta-llama/Llama-3.1-8B-Instruct
   3. 장치 계산 경로가 3.3 → 5.8 s 로 느려졌다. 한 sequence 를 두 카드가 차례로 처리하므로 계산이 겹치지 않고, 한 프로세스가 동기 실행(2026.2)으로
      두 장치를 번갈아 부르는 비용과 host 경합이 더해진 것으로 보인다(원인 분해는 하지 않았다).
 - 결론적으로 모델에서 바꿔야 할 항은 "링크 대역" → "host 복사·AdamW 처리량"이다. 장치 상주 AdamW 는 바로 이 host 항을 없애므로 여전히 맞는 방향이다.
+- **고친 모델(#34, docs/design.md 「Optimizer placement: step-time model」)**: `T_step ≈ T_dev + T_res + max(0, N_off·(t_a + t_u)/W − T_res − …)`,
+  t_a·t_u 는 1장 오프로드 실행의 레이어당 host AdamW·업로드(7.07/32, 3.19/32 s), W=4 작업 스레드. 이 기록의 2장은 T_dev 5.76 + 2.57 = **8.33 s
+  예측, 측정 9.26 s(−10 %)**, 1장은 3.32 + 2.57 = 5.89 대 6.36 s(−7 %). 96코어 host 의 1·2장·상주 실행 검증은
+  [2026-10-21-resident-adamw.md](2026-10-21-resident-adamw.md).
 
 ### 상주 AdamW (질문 3) — 측정하지 못함
 
@@ -65,6 +69,9 @@ python3 -m npu_lora.cli compile --full --model meta-llama/Llama-3.1-8B-Instruct
   8B S=512 학습 그래프 전체를 다시 빌드·검증하느라 CPU 노드(요청 8코어, 공유)에서 2.5시간 넘게 상주 그래프 단계에 이르지 못해 중단했다.
   그래서 `--resident-layers` 의 RNGD 실행 시간은 없다. 기능은 CPU 테스트로만 검증됐다.
 - 예측(미측정): 카드당 8층 상주 시 host 레이어 작업이 절반(≈7 s)으로 줄고, 장치 AdamW 는 10-19 측정 28 ms/층 × 16층 ≈ 0.45 s 를 주 스레드에 더한다.
+- **후속(#34)**: 컴파일이 느린 원인은 `compile` 이 학습 그래프 경로 전체(CPU 추적·조각 실행)를 먼저 도는 것이었고, `compile --adamw-only` 로
+  상주 그래프 4종이 150 s 에 컴파일된다. 96코어 npu-node-1 에서 2장 + 카드당 8층 상주는 4.57 s/step(1장 오프로드 5.22 s 대비 +14 %)이지만
+  간헐적 장치 OOM(#40)으로 5 step 만 쟀다 — [2026-10-21-resident-adamw.md](2026-10-21-resident-adamw.md).
 
 Raw: `experiments/data/w7/split8b-rngd-2card.jsonl`(2장), `experiments/data/w7/energy-full8b-rngd.jsonl`(1장),
 `experiments/data/w7/split8b-rngd-2card-err-before-fix.txt`(첫 실행: 경계 레이어 입력이 카드 0 에 남아 역전파가 실패 — 9fbb411 에서 수정).
diff --git a/experiments/data/resident-adamw/load-npu-node-1.txt b/experiments/data/resident-adamw/load-npu-node-1.txt
new file mode 100644
index 0000000..d5fd1c8
--- /dev/null
+++ b/experiments/data/resident-adamw/load-npu-node-1.txt
@@ -0,0 +1,62 @@
+1792560790 92.19 85.73 56.79
+1792560850 87.73 85.40 58.48
+1792560910 35.85 71.07 55.25
+1792560970 26.72 61.26 52.82
+1792561030 84.15 70.85 56.60
+1792561090 101.62 78.13 59.99
+1792561150 74.14 74.75 59.98
+1792561210 87.65 77.81 61.92
+1792561270 96.55 82.33 64.49
+1792561330 98.81 85.53 66.70
+1792561390 99.60 88.17 68.79
+1792561450 99.41 90.22 70.71
+1792561510 114.37 95.35 73.65
+1792561570 113.68 98.74 76.18
+1792561630 110.96 100.77 78.30
+1792561690 102.87 100.31 79.55
+1792561750 100.55 100.12 80.79
+1792561810 100.09 100.06 81.98
+1792561870 100.01 100.04 83.11
+1792561930 99.73 99.95 84.14
+1792561990 109.54 102.31 85.92
+1792562050 113.56 104.92 87.85
+1792562110 107.17 104.56 88.79
+1792562170 106.08 104.87 89.90
+1792562230 109.53 106.36 91.37
+1792562290 103.40 105.18 91.91
+1792562350 99.80 103.83 92.28
+1792562410 86.84 99.94 91.70
+1792562470 91.69 98.50 91.70
+1792562530 96.46 98.61 92.16
+1792562590 98.21 98.72 92.60
+1792562650 98.80 98.82 93.03
+1792562710 93.71 99.48 93.74
+1792562770 76.95 93.46 92.02
+1792562830 96.31 96.27 93.09
+1792562890 115.61 101.48 95.06
+1792562950 114.51 103.48 96.14
+1792563010 74.10 94.40 93.52
+1792563070 89.01 95.01 93.79
+1792563130 94.46 95.50 94.03
+1792563190 96.70 95.99 94.30
+1792563250 100.52 97.06 94.76
+1792563310 114.57 101.22 96.31
+1792563370 108.19 101.50 96.70
+1792563430 147.84 113.73 101.18
+1792563490 161.19 123.22 105.20
+1792563550 193.97 139.29 111.84
+1792563610 209.14 153.24 118.33
+1792563670 240.82 171.79 126.85
+1792563730 242.68 186.54 134.83
+1792563790 240.33 196.40 141.48
+1792563850 242.38 204.98 147.87
+1792563910 243.86 212.20 153.93
+1792563970 244.04 218.02 159.58
+1792564030 243.28 222.65 164.83
+1792564090 234.96 223.88 168.87
+1792564150 237.17 226.27 173.12
+1792564210 214.80 223.21 175.47
+1792564270 179.09 213.36 175.19
+1792564330 154.82 199.43 172.75
+1792564390 147.45 189.96 171.20
+1792564450 141.45 179.63 168.78
diff --git a/experiments/data/resident-adamw/r1-off-w8.jsonl b/experiments/data/resident-adamw/r1-off-w8.jsonl
new file mode 100644
index 0000000..3bceeba
--- /dev/null
+++ b/experiments/data/resident-adamw/r1-off-w8.jsonl
@@ -0,0 +1,27 @@
+{"ts": 1792557839.679, "run": "ntb-r1-off-w8", "event": "start", "examples": 8000}
+{"ts": 1792558348.412, "run": "ntb-r1-off-w8", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 50.884, "tokens_per_s": 40.2, "wait_weights_s": 0.0, "norm_s": 0.032, "prev_d2h_s": 0.0, "prev_adamw_s": 0.0, "prev_upload_s": 0.0}
+{"ts": 1792558358.051, "run": "ntb-r1-off-w8", "event": "step", "step": 2, "loss": 1.5625602304935455, "grad_norm": 45.83976931927431, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 9.627, "tokens_per_s": 212.7, "wait_weights_s": 3.736, "norm_s": 0.029, "prev_d2h_s": 14.769, "prev_adamw_s": 24.377, "prev_upload_s": 3.375}
+{"ts": 1792558363.214, "run": "ntb-r1-off-w8", "event": "step", "step": 3, "loss": 1.0878866240382195, "grad_norm": 83.62784643840048, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 5.144, "tokens_per_s": 398.1, "wait_weights_s": 1.255, "norm_s": 0.033, "prev_d2h_s": 3.158, "prev_adamw_s": 10.19, "prev_upload_s": 3.457}
+{"ts": 1792558368.36, "run": "ntb-r1-off-w8", "event": "step", "step": 4, "loss": 1.105992036871612, "grad_norm": 20.339171194820047, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 5.136, "tokens_per_s": 398.8, "wait_weights_s": 1.313, "norm_s": 0.036, "prev_d2h_s": 3.045, "prev_adamw_s": 9.171, "prev_upload_s": 3.47}
+{"ts": 1792558373.629, "run": "ntb-r1-off-w8", "event": "step", "step": 5, "loss": 1.3360941708087921, "grad_norm": 13.385490951262877, "lr": 5e-06, "tokens": 2048, "seconds": 5.259, "tokens_per_s": 389.4, "wait_weights_s": 1.273, "norm_s": 0.033, "prev_d2h_s": 3.11, "prev_adamw_s": 8.43, "prev_upload_s": 3.632}
+{"ts": 1792558378.849, "run": "ntb-r1-off-w8", "event": "step", "step": 6, "loss": 1.706906497478485, "grad_norm": 16.335248507856008, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 5.21, "tokens_per_s": 393.1, "wait_weights_s": 1.3, "norm_s": 0.036, "prev_d2h_s": 3.167, "prev_adamw_s": 9.149, "prev_upload_s": 3.363}
+{"ts": 1792558383.712, "run": "ntb-r1-off-w8", "event": "step", "step": 7, "loss": 1.7494316697120667, "grad_norm": 16.13586935662257, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 4.852, "tokens_per_s": 422.1, "wait_weights_s": 1.236, "norm_s": 0.024, "prev_d2h_s": 3.179, "prev_adamw_s": 8.276, "prev_upload_s": 3.595}
+{"ts": 1792558388.537, "run": "ntb-r1-off-w8", "event": "step", "step": 8, "loss": 1.1023952513933182, "grad_norm": 12.115460431474599, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 4.806, "tokens_per_s": 426.2, "wait_weights_s": 1.282, "norm_s": 0.029, "prev_d2h_s": 3.188, "prev_adamw_s": 8.528, "prev_upload_s": 3.688}
+{"ts": 1792558393.38, "run": "ntb-r1-off-w8", "event": "step", "step": 9, "loss": 0.8882534727454185, "grad_norm": 11.315825465381323, "lr": 9e-06, "tokens": 2048, "seconds": 4.833, "tokens_per_s": 423.8, "wait_weights_s": 1.225, "norm_s": 0.027, "prev_d2h_s": 3.031, "prev_adamw_s": 8.359, "prev_upload_s": 3.506}
+{"ts": 1792558398.058, "run": "ntb-r1-off-w8", "event": "step", "step": 10, "loss": 1.1120138672413304, "grad_norm": 9.943527357181816, "lr": 1e-05, "tokens": 2048, "seconds": 4.669, "tokens_per_s": 438.6, "wait_weights_s": 1.245, "norm_s": 0.024, "prev_d2h_s": 3.214, "prev_adamw_s": 8.412, "prev_upload_s": 3.59}
+{"ts": 1792558402.884, "run": "ntb-r1-off-w8", "event": "step", "step": 11, "loss": 1.1846136450767517, "grad_norm": 10.343349532186359, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 4.814, "tokens_per_s": 425.4, "wait_weights_s": 1.241, "norm_s": 0.023, "prev_d2h_s": 3.493, "prev_adamw_s": 7.721, "prev_upload_s": 3.657}
+{"ts": 1792558407.511, "run": "ntb-r1-off-w8", "event": "step", "step": 12, "loss": 1.149821549654007, "grad_norm": 15.667075483844068, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 4.621, "tokens_per_s": 443.2, "wait_weights_s": 1.203, "norm_s": 0.035, "prev_d2h_s": 3.159, "prev_adamw_s": 7.761, "prev_upload_s": 3.762}
+{"ts": 1792558412.343, "run": "ntb-r1-off-w8", "event": "step", "step": 13, "loss": 1.3779243230819702, "grad_norm": 9.736832260042876, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 4.826, "tokens_per_s": 424.3, "wait_weights_s": 1.022, "norm_s": 0.023, "prev_d2h_s": 3.211, "prev_adamw_s": 7.926, "prev_upload_s": 3.659}
+{"ts": 1792558417.23, "run": "ntb-r1-off-w8", "event": "step", "step": 14, "loss": 1.1682318598031998, "grad_norm": 28.767873129222597, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 4.876, "tokens_per_s": 420.0, "wait_weights_s": 1.214, "norm_s": 0.024, "prev_d2h_s": 3.292, "prev_adamw_s": 7.562, "prev_upload_s": 3.722}
+{"ts": 1792558422.139, "run": "ntb-r1-off-w8", "event": "step", "step": 15, "loss": 1.0693852007389069, "grad_norm": 9.241371995864162, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 4.896, "tokens_per_s": 418.3, "wait_weights_s": 1.209, "norm_s": 0.03, "prev_d2h_s": 3.352, "prev_adamw_s": 7.903, "prev_upload_s": 3.691}
+{"ts": 1792558426.773, "run": "ntb-r1-off-w8", "event": "step", "step": 16, "loss": 1.2823715209960938, "grad_norm": 10.081775683695783, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 4.627, "tokens_per_s": 442.7, "wait_weights_s": 1.219, "norm_s": 0.027, "prev_d2h_s": 3.205, "prev_adamw_s": 7.753, "prev_upload_s": 3.782}
+{"ts": 1792558431.496, "run": "ntb-r1-off-w8", "event": "step", "step": 17, "loss": 1.1385412216186523, "grad_norm": 13.139631625067793, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 4.715, "tokens_per_s": 434.4, "wait_weights_s": 1.234, "norm_s": 0.033, "prev_d2h_s": 3.225, "prev_adamw_s": 7.684, "prev_upload_s": 3.826}
+{"ts": 1792558436.255, "run": "ntb-r1-off-w8", "event": "step", "step": 18, "loss": 1.4344590604305267, "grad_norm": 9.256871006458512, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 4.747, "tokens_per_s": 431.4, "wait_weights_s": 1.225, "norm_s": 0.036, "prev_d2h_s": 3.304, "prev_adamw_s": 7.958, "prev_upload_s": 3.684}
+{"ts": 1792558440.908, "run": "ntb-r1-off-w8", "event": "step", "step": 19, "loss": 0.8083997741341591, "grad_norm": 8.44625464614702, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 4.636, "tokens_per_s": 441.8, "wait_weights_s": 1.197, "norm_s": 0.026, "prev_d2h_s": 3.049, "prev_adamw_s": 8.694, "prev_upload_s": 3.382}
+{"ts": 1792558445.88, "run": "ntb-r1-off-w8", "event": "step", "step": 20, "loss": 1.5581613630056381, "grad_norm": 8.408779834946662, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 4.956, "tokens_per_s": 413.2, "wait_weights_s": 1.26, "norm_s": 0.041, "prev_d2h_s": 3.068, "prev_adamw_s": 8.356, "prev_upload_s": 3.472}
+{"ts": 1792558450.709, "run": "ntb-r1-off-w8", "event": "step", "step": 21, "loss": 1.272824227809906, "grad_norm": 11.512978955760328, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 4.805, "tokens_per_s": 426.2, "wait_weights_s": 1.29, "norm_s": 0.042, "prev_d2h_s": 3.145, "prev_adamw_s": 9.424, "prev_upload_s": 3.498}
+{"ts": 1792558455.981, "run": "ntb-r1-off-w8", "event": "step", "step": 22, "loss": 1.3625558465719223, "grad_norm": 7.841532302364816, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 5.26, "tokens_per_s": 389.3, "wait_weights_s": 1.414, "norm_s": 0.029, "prev_d2h_s": 3.464, "prev_adamw_s": 8.763, "prev_upload_s": 3.64}
+{"ts": 1792558460.722, "run": "ntb-r1-off-w8", "event": "step", "step": 23, "loss": 0.9013836085796356, "grad_norm": 7.894083656715112, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 4.731, "tokens_per_s": 432.9, "wait_weights_s": 1.226, "norm_s": 0.032, "prev_d2h_s": 3.481, "prev_adamw_s": 8.006, "prev_upload_s": 3.498}
+{"ts": 1792558465.631, "run": "ntb-r1-off-w8", "event": "step", "step": 24, "loss": 0.8878279626369476, "grad_norm": 10.04998445436565, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 4.897, "tokens_per_s": 418.2, "wait_weights_s": 1.392, "norm_s": 0.028, "prev_d2h_s": 4.203, "prev_adamw_s": 7.69, "prev_upload_s": 3.358}
+{"ts": 1792558470.48, "run": "ntb-r1-off-w8", "event": "step", "step": 25, "loss": 1.3050681483000517, "grad_norm": 12.53712000739592, "lr": 0.0, "tokens": 2048, "seconds": 4.828, "tokens_per_s": 424.2, "wait_weights_s": 1.204, "norm_s": 0.027, "prev_d2h_s": 3.359, "prev_adamw_s": 8.091, "prev_upload_s": 3.474}
+{"ts": 1792558484.624, "run": "ntb-r1-off-w8", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8083997741341591, "mode": "full"}
diff --git a/experiments/data/resident-adamw/r1-off.jsonl b/experiments/data/resident-adamw/r1-off.jsonl
new file mode 100644
index 0000000..6ddb36d
--- /dev/null
+++ b/experiments/data/resident-adamw/r1-off.jsonl
@@ -0,0 +1,27 @@
+{"ts": 1792555999.81, "run": "ntb-r1-off", "event": "start", "examples": 8000}
+{"ts": 1792556183.916, "run": "ntb-r1-off", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 13.369, "tokens_per_s": 153.2, "wait_weights_s": 0.0, "norm_s": 0.031, "prev_d2h_s": 0.0, "prev_adamw_s": 0.0, "prev_upload_s": 0.0}
+{"ts": 1792556194.13, "run": "ntb-r1-off", "event": "step", "step": 2, "loss": 1.5625602304935455, "grad_norm": 45.83976931927431, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 10.212, "tokens_per_s": 200.6, "wait_weights_s": 5.059, "norm_s": 0.041, "prev_d2h_s": 10.435, "prev_adamw_s": 10.306, "prev_upload_s": 2.364}
+{"ts": 1792556199.805, "run": "ntb-r1-off", "event": "step", "step": 3, "loss": 1.0878866240382195, "grad_norm": 83.62784643840048, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 5.671, "tokens_per_s": 361.1, "wait_weights_s": 1.481, "norm_s": 0.028, "prev_d2h_s": 2.113, "prev_adamw_s": 4.689, "prev_upload_s": 2.176}
+{"ts": 1792556205.628, "run": "ntb-r1-off", "event": "step", "step": 4, "loss": 1.105992036871612, "grad_norm": 20.339171194820047, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 5.819, "tokens_per_s": 351.9, "wait_weights_s": 1.549, "norm_s": 0.035, "prev_d2h_s": 2.128, "prev_adamw_s": 4.659, "prev_upload_s": 2.242}
+{"ts": 1792556211.436, "run": "ntb-r1-off", "event": "step", "step": 5, "loss": 1.3360941708087921, "grad_norm": 13.385490951262877, "lr": 5e-06, "tokens": 2048, "seconds": 5.805, "tokens_per_s": 352.8, "wait_weights_s": 1.474, "norm_s": 0.029, "prev_d2h_s": 2.122, "prev_adamw_s": 4.165, "prev_upload_s": 2.249}
+{"ts": 1792556216.905, "run": "ntb-r1-off", "event": "step", "step": 6, "loss": 1.706906497478485, "grad_norm": 16.335248507856008, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 5.465, "tokens_per_s": 374.7, "wait_weights_s": 1.468, "norm_s": 0.032, "prev_d2h_s": 2.14, "prev_adamw_s": 4.151, "prev_upload_s": 2.272}
+{"ts": 1792556222.513, "run": "ntb-r1-off", "event": "step", "step": 7, "loss": 1.7494316697120667, "grad_norm": 16.13586935662257, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 5.605, "tokens_per_s": 365.4, "wait_weights_s": 1.456, "norm_s": 0.03, "prev_d2h_s": 2.187, "prev_adamw_s": 3.958, "prev_upload_s": 2.355}
+{"ts": 1792556227.597, "run": "ntb-r1-off", "event": "step", "step": 8, "loss": 1.1023952513933182, "grad_norm": 12.115460431474599, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 5.08, "tokens_per_s": 403.1, "wait_weights_s": 1.504, "norm_s": 0.028, "prev_d2h_s": 2.235, "prev_adamw_s": 4.198, "prev_upload_s": 2.202}
+{"ts": 1792556233.026, "run": "ntb-r1-off", "event": "step", "step": 9, "loss": 0.8882534727454185, "grad_norm": 11.315825465381323, "lr": 9e-06, "tokens": 2048, "seconds": 5.424, "tokens_per_s": 377.5, "wait_weights_s": 1.578, "norm_s": 0.034, "prev_d2h_s": 2.313, "prev_adamw_s": 4.519, "prev_upload_s": 2.174}
+{"ts": 1792556238.436, "run": "ntb-r1-off", "event": "step", "step": 10, "loss": 1.1120138672413304, "grad_norm": 9.943527357181816, "lr": 1e-05, "tokens": 2048, "seconds": 5.407, "tokens_per_s": 378.8, "wait_weights_s": 1.487, "norm_s": 0.031, "prev_d2h_s": 2.202, "prev_adamw_s": 4.251, "prev_upload_s": 2.302}
+{"ts": 1792556244.041, "run": "ntb-r1-off", "event": "step", "step": 11, "loss": 1.1846136450767517, "grad_norm": 10.343349532186359, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 5.601, "tokens_per_s": 365.7, "wait_weights_s": 1.508, "norm_s": 0.033, "prev_d2h_s": 2.292, "prev_adamw_s": 3.926, "prev_upload_s": 2.362}
+{"ts": 1792556248.979, "run": "ntb-r1-off", "event": "step", "step": 12, "loss": 1.149821549654007, "grad_norm": 15.667075483844068, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 4.935, "tokens_per_s": 415.0, "wait_weights_s": 1.461, "norm_s": 0.034, "prev_d2h_s": 2.14, "prev_adamw_s": 4.043, "prev_upload_s": 2.272}
+{"ts": 1792556254.312, "run": "ntb-r1-off", "event": "step", "step": 13, "loss": 1.3779243230819702, "grad_norm": 9.736832260042876, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 5.325, "tokens_per_s": 384.6, "wait_weights_s": 1.465, "norm_s": 0.03, "prev_d2h_s": 2.224, "prev_adamw_s": 4.337, "prev_upload_s": 2.261}
+{"ts": 1792556259.961, "run": "ntb-r1-off", "event": "step", "step": 14, "loss": 1.1682318598031998, "grad_norm": 28.767873129222597, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 5.646, "tokens_per_s": 362.7, "wait_weights_s": 1.58, "norm_s": 0.029, "prev_d2h_s": 2.26, "prev_adamw_s": 4.558, "prev_upload_s": 2.267}
+{"ts": 1792556265.166, "run": "ntb-r1-off", "event": "step", "step": 15, "loss": 1.0693852007389069, "grad_norm": 9.241371995864162, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 5.202, "tokens_per_s": 393.7, "wait_weights_s": 1.486, "norm_s": 0.03, "prev_d2h_s": 2.344, "prev_adamw_s": 3.916, "prev_upload_s": 2.282}
+{"ts": 1792556270.412, "run": "ntb-r1-off", "event": "step", "step": 16, "loss": 1.2823715209960938, "grad_norm": 10.081775683695783, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 5.242, "tokens_per_s": 390.7, "wait_weights_s": 1.53, "norm_s": 0.03, "prev_d2h_s": 2.362, "prev_adamw_s": 4.127, "prev_upload_s": 2.313}
+{"ts": 1792556275.776, "run": "ntb-r1-off", "event": "step", "step": 17, "loss": 1.1385412216186523, "grad_norm": 13.139631625067793, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 5.359, "tokens_per_s": 382.1, "wait_weights_s": 1.508, "norm_s": 0.029, "prev_d2h_s": 2.25, "prev_adamw_s": 4.204, "prev_upload_s": 2.233}
+{"ts": 1792556281.384, "run": "ntb-r1-off", "event": "step", "step": 18, "loss": 1.4344590604305267, "grad_norm": 9.256871006458512, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 5.605, "tokens_per_s": 365.4, "wait_weights_s": 1.466, "norm_s": 0.034, "prev_d2h_s": 2.26, "prev_adamw_s": 3.904, "prev_upload_s": 2.333}
+{"ts": 1792556286.477, "run": "ntb-r1-off", "event": "step", "step": 19, "loss": 0.8083997741341591, "grad_norm": 8.44625464614702, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 5.09, "tokens_per_s": 402.3, "wait_weights_s": 1.47, "norm_s": 0.024, "prev_d2h_s": 2.159, "prev_adamw_s": 4.099, "prev_upload_s": 2.324}
+{"ts": 1792556291.596, "run": "ntb-r1-off", "event": "step", "step": 20, "loss": 1.5581613630056381, "grad_norm": 8.408779834946662, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 5.117, "tokens_per_s": 400.2, "wait_weights_s": 1.51, "norm_s": 0.027, "prev_d2h_s": 2.207, "prev_adamw_s": 4.207, "prev_upload_s": 2.265}
+{"ts": 1792556296.582, "run": "ntb-r1-off", "event": "step", "step": 21, "loss": 1.272824227809906, "grad_norm": 11.512978955760328, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 4.982, "tokens_per_s": 411.0, "wait_weights_s": 1.531, "norm_s": 0.032, "prev_d2h_s": 2.22, "prev_adamw_s": 4.41, "prev_upload_s": 2.17}
+{"ts": 1792556301.566, "run": "ntb-r1-off", "event": "step", "step": 22, "loss": 1.3625558465719223, "grad_norm": 7.841532302364816, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 4.982, "tokens_per_s": 411.1, "wait_weights_s": 1.512, "norm_s": 0.032, "prev_d2h_s": 2.143, "prev_adamw_s": 4.267, "prev_upload_s": 2.324}
+{"ts": 1792556306.484, "run": "ntb-r1-off", "event": "step", "step": 23, "loss": 0.9013836085796356, "grad_norm": 7.894083656715112, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 4.915, "tokens_per_s": 416.7, "wait_weights_s": 1.493, "norm_s": 0.028, "prev_d2h_s": 2.208, "prev_adamw_s": 4.142, "prev_upload_s": 2.268}
+{"ts": 1792556311.658, "run": "ntb-r1-off", "event": "step", "step": 24, "loss": 0.8878279626369476, "grad_norm": 10.04998445436565, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 5.171, "tokens_per_s": 396.1, "wait_weights_s": 1.491, "norm_s": 0.024, "prev_d2h_s": 2.206, "prev_adamw_s": 4.168, "prev_upload_s": 2.294}
+{"ts": 1792556316.725, "run": "ntb-r1-off", "event": "step", "step": 25, "loss": 1.3050681483000517, "grad_norm": 12.53712000739592, "lr": 0.0, "tokens": 2048, "seconds": 5.063, "tokens_per_s": 404.5, "wait_weights_s": 1.53, "norm_s": 0.025, "prev_d2h_s": 2.229, "prev_adamw_s": 4.279, "prev_upload_s": 2.147}
+{"ts": 1792556329.629, "run": "ntb-r1-off", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8083997741341591, "mode": "full"}
diff --git a/experiments/data/resident-adamw/r1-res3.jsonl b/experiments/data/resident-adamw/r1-res3.jsonl
new file mode 100644
index 0000000..389423c
--- /dev/null
+++ b/experiments/data/resident-adamw/r1-res3.jsonl
@@ -0,0 +1,27 @@
+{"ts": 1792561687.969, "run": "ntb-r1-res3", "event": "start", "examples": 8000}
+{"ts": 1792562000.042, "run": "ntb-r1-res3", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 19.556, "tokens_per_s": 104.7, "wait_weights_s": 0.001, "norm_s": 0.112, "prev_d2h_s": 0.0, "prev_adamw_s": 0.063, "prev_upload_s": 0.0, "resident_update_s": 0.063}
+{"ts": 1792562018.806, "run": "ntb-r1-res3", "event": "step", "step": 2, "loss": 1.5645148754119873, "grad_norm": 45.85897911729124, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 18.76, "tokens_per_s": 109.2, "wait_weights_s": 9.828, "norm_s": 0.144, "prev_d2h_s": 16.575, "prev_adamw_s": 25.522, "prev_upload_s": 3.906, "resident_update_s": 0.076}
+{"ts": 1792562028.212, "run": "ntb-r1-res3", "event": "step", "step": 3, "loss": 1.0930061228573322, "grad_norm": 85.88833651593623, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 9.401, "tokens_per_s": 217.9, "wait_weights_s": 2.503, "norm_s": 0.115, "prev_d2h_s": 2.923, "prev_adamw_s": 12.315, "prev_upload_s": 2.595, "resident_update_s": 0.06}
+{"ts": 1792562038.107, "run": "ntb-r1-res3", "event": "step", "step": 4, "loss": 1.104983190074563, "grad_norm": 20.595105664922393, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 9.888, "tokens_per_s": 207.1, "wait_weights_s": 1.865, "norm_s": 0.114, "prev_d2h_s": 2.919, "prev_adamw_s": 7.979, "prev_upload_s": 2.5, "resident_update_s": 0.06}
+{"ts": 1792562047.225, "run": "ntb-r1-res3", "event": "step", "step": 5, "loss": 1.3346887826919556, "grad_norm": 13.297706051177865, "lr": 5e-06, "tokens": 2048, "seconds": 9.112, "tokens_per_s": 224.8, "wait_weights_s": 2.269, "norm_s": 0.149, "prev_d2h_s": 3.268, "prev_adamw_s": 8.574, "prev_upload_s": 2.69, "resident_update_s": 0.065}
+{"ts": 1792562057.077, "run": "ntb-r1-res3", "event": "step", "step": 6, "loss": 1.701994776725769, "grad_norm": 16.38770979951688, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 9.847, "tokens_per_s": 208.0, "wait_weights_s": 2.136, "norm_s": 0.15, "prev_d2h_s": 2.963, "prev_adamw_s": 8.93, "prev_upload_s": 2.598, "resident_update_s": 0.056}
+{"ts": 1792562067.66, "run": "ntb-r1-res3", "event": "step", "step": 7, "loss": 1.745564192533493, "grad_norm": 16.473752173823065, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 10.576, "tokens_per_s": 193.6, "wait_weights_s": 2.003, "norm_s": 0.137, "prev_d2h_s": 3.018, "prev_adamw_s": 9.653, "prev_upload_s": 2.59, "resident_update_s": 0.084}
+{"ts": 1792562076.409, "run": "ntb-r1-res3", "event": "step", "step": 8, "loss": 1.1049477607011795, "grad_norm": 12.647991777244439, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 8.743, "tokens_per_s": 234.2, "wait_weights_s": 2.069, "norm_s": 0.159, "prev_d2h_s": 3.077, "prev_adamw_s": 10.031, "prev_upload_s": 2.697, "resident_update_s": 0.064}
+{"ts": 1792562086.111, "run": "ntb-r1-res3", "event": "step", "step": 9, "loss": 0.879561647772789, "grad_norm": 11.105616547190161, "lr": 9e-06, "tokens": 2048, "seconds": 9.697, "tokens_per_s": 211.2, "wait_weights_s": 2.192, "norm_s": 0.12, "prev_d2h_s": 3.133, "prev_adamw_s": 9.372, "prev_upload_s": 2.686, "resident_update_s": 0.059}
+{"ts": 1792562095.289, "run": "ntb-r1-res3", "event": "step", "step": 10, "loss": 1.1060402577277273, "grad_norm": 9.78830511867085, "lr": 1e-05, "tokens": 2048, "seconds": 9.173, "tokens_per_s": 223.3, "wait_weights_s": 2.029, "norm_s": 0.12, "prev_d2h_s": 2.98, "prev_adamw_s": 9.623, "prev_upload_s": 2.564, "resident_update_s": 0.066}
+{"ts": 1792562106.443, "run": "ntb-r1-res3", "event": "step", "step": 11, "loss": 1.1929064095020294, "grad_norm": 10.589193303649976, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 11.149, "tokens_per_s": 183.7, "wait_weights_s": 2.25, "norm_s": 0.157, "prev_d2h_s": 3.469, "prev_adamw_s": 9.216, "prev_upload_s": 2.8, "resident_update_s": 0.064}
+{"ts": 1792562117.367, "run": "ntb-r1-res3", "event": "step", "step": 12, "loss": 1.162868708372116, "grad_norm": 16.870592533985086, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 10.919, "tokens_per_s": 187.6, "wait_weights_s": 2.316, "norm_s": 0.146, "prev_d2h_s": 3.34, "prev_adamw_s": 10.194, "prev_upload_s": 2.835, "resident_update_s": 0.07}
+{"ts": 1792562127.412, "run": "ntb-r1-res3", "event": "step", "step": 13, "loss": 1.380837082862854, "grad_norm": 9.9298779775605, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 10.038, "tokens_per_s": 204.0, "wait_weights_s": 2.119, "norm_s": 0.116, "prev_d2h_s": 2.975, "prev_adamw_s": 10.268, "prev_upload_s": 2.598, "resident_update_s": 0.063}
+{"ts": 1792562137.477, "run": "ntb-r1-res3", "event": "step", "step": 14, "loss": 1.1859616339206696, "grad_norm": 24.446325860768738, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 10.061, "tokens_per_s": 203.6, "wait_weights_s": 2.513, "norm_s": 0.105, "prev_d2h_s": 4.336, "prev_adamw_s": 9.478, "prev_upload_s": 2.843, "resident_update_s": 0.058}
+{"ts": 1792562147.538, "run": "ntb-r1-res3", "event": "step", "step": 15, "loss": 1.0579295679926872, "grad_norm": 9.150650990234338, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 10.056, "tokens_per_s": 203.7, "wait_weights_s": 2.487, "norm_s": 0.127, "prev_d2h_s": 3.608, "prev_adamw_s": 11.849, "prev_upload_s": 2.716, "resident_update_s": 0.064}
+{"ts": 1792562156.092, "run": "ntb-r1-res3", "event": "step", "step": 16, "loss": 1.2859830558300018, "grad_norm": 9.89379016440003, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 8.549, "tokens_per_s": 239.6, "wait_weights_s": 2.111, "norm_s": 0.125, "prev_d2h_s": 3.417, "prev_adamw_s": 9.112, "prev_upload_s": 2.627, "resident_update_s": 0.071}
+{"ts": 1792562165.503, "run": "ntb-r1-res3", "event": "step", "step": 17, "loss": 1.146461695432663, "grad_norm": 12.566750767402239, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 9.402, "tokens_per_s": 217.8, "wait_weights_s": 1.922, "norm_s": 0.116, "prev_d2h_s": 2.949, "prev_adamw_s": 8.825, "prev_upload_s": 2.54, "resident_update_s": 0.054}
+{"ts": 1792562174.32, "run": "ntb-r1-res3", "event": "step", "step": 18, "loss": 1.4367725849151611, "grad_norm": 8.299777350638328, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 8.811, "tokens_per_s": 232.4, "wait_weights_s": 2.478, "norm_s": 0.109, "prev_d2h_s": 4.283, "prev_adamw_s": 8.402, "prev_upload_s": 2.946, "resident_update_s": 0.061}
+{"ts": 1792562184.858, "run": "ntb-r1-res3", "event": "step", "step": 19, "loss": 0.8071708828210831, "grad_norm": 8.553112275617082, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 10.535, "tokens_per_s": 194.4, "wait_weights_s": 2.89, "norm_s": 0.12, "prev_d2h_s": 4.358, "prev_adamw_s": 9.782, "prev_upload_s": 2.999, "resident_update_s": 0.065}
+{"ts": 1792562195.914, "run": "ntb-r1-res3", "event": "step", "step": 20, "loss": 1.5591884553432465, "grad_norm": 8.321620541085988, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 11.051, "tokens_per_s": 185.3, "wait_weights_s": 2.501, "norm_s": 0.141, "prev_d2h_s": 4.717, "prev_adamw_s": 8.453, "prev_upload_s": 2.93, "resident_update_s": 0.083}
+{"ts": 1792562205.966, "run": "ntb-r1-res3", "event": "step", "step": 21, "loss": 1.2711932510137558, "grad_norm": 11.429805773077193, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 10.048, "tokens_per_s": 203.8, "wait_weights_s": 1.947, "norm_s": 0.152, "prev_d2h_s": 3.254, "prev_adamw_s": 10.09, "prev_upload_s": 2.723, "resident_update_s": 0.075}
+{"ts": 1792562215.405, "run": "ntb-r1-res3", "event": "step", "step": 22, "loss": 1.360827460885048, "grad_norm": 7.709555299904752, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 9.435, "tokens_per_s": 217.1, "wait_weights_s": 2.47, "norm_s": 0.149, "prev_d2h_s": 3.151, "prev_adamw_s": 8.976, "prev_upload_s": 2.5, "resident_update_s": 0.067}
+{"ts": 1792562225.49, "run": "ntb-r1-res3", "event": "step", "step": 23, "loss": 0.8931946381926537, "grad_norm": 7.809231489498888, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 10.082, "tokens_per_s": 203.1, "wait_weights_s": 2.617, "norm_s": 0.15, "prev_d2h_s": 3.829, "prev_adamw_s": 10.269, "prev_upload_s": 2.734, "resident_update_s": 0.086}
+{"ts": 1792562235.537, "run": "ntb-r1-res3", "event": "step", "step": 24, "loss": 0.8854059875011444, "grad_norm": 10.101767395016207, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 10.038, "tokens_per_s": 204.0, "wait_weights_s": 2.733, "norm_s": 0.121, "prev_d2h_s": 4.475, "prev_adamw_s": 10.523, "prev_upload_s": 2.688, "resident_update_s": 0.063}
+{"ts": 1792562244.988, "run": "ntb-r1-res3", "event": "step", "step": 25, "loss": 1.2858505565673113, "grad_norm": 11.768050621239224, "lr": 0.0, "tokens": 2048, "seconds": 9.446, "tokens_per_s": 216.8, "wait_weights_s": 2.467, "norm_s": 0.128, "prev_d2h_s": 4.274, "prev_adamw_s": 9.423, "prev_upload_s": 2.754, "resident_update_s": 0.069}
+{"ts": 1792562271.208, "run": "ntb-r1-res3", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8071708828210831, "mode": "full"}
diff --git a/experiments/data/resident-adamw/r1-res4-try1.err b/experiments/data/resident-adamw/r1-res4-try1.err
new file mode 100644
index 0000000..f261cf1
--- /dev/null
+++ b/experiments/data/resident-adamw/r1-res4-try1.err
@@ -0,0 +1 @@
+torch.OutOfMemoryError: failed to allocate 117440512 bytes on DRAM_SHARED
diff --git a/experiments/data/resident-adamw/r1-res4-try1.jsonl b/experiments/data/resident-adamw/r1-res4-try1.jsonl
new file mode 100644
index 0000000..ac2c11d
--- /dev/null
+++ b/experiments/data/resident-adamw/r1-res4-try1.jsonl
@@ -0,0 +1,13 @@
+{"ts": 1792559513.831, "run": "ntb-r1-res4", "event": "start", "examples": 8000}
+{"ts": 1792559932.208, "run": "ntb-r1-res4", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 24.242, "tokens_per_s": 84.5, "wait_weights_s": 0.001, "norm_s": 0.377, "prev_d2h_s": 0.0, "prev_adamw_s": 0.332, "prev_upload_s": 0.0, "resident_update_s": 0.332}
+{"ts": 1792559945.106, "run": "ntb-r1-res4", "event": "step", "step": 2, "loss": 1.5645148754119873, "grad_norm": 45.859045193746276, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 12.895, "tokens_per_s": 158.8, "wait_weights_s": 5.207, "norm_s": 0.2, "prev_d2h_s": 12.899, "prev_adamw_s": 14.604, "prev_upload_s": 2.465, "resident_update_s": 0.168}
+{"ts": 1792559949.859, "run": "ntb-r1-res4", "event": "step", "step": 3, "loss": 1.0821138471364975, "grad_norm": 80.97597200661254, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 4.751, "tokens_per_s": 431.1, "wait_weights_s": 0.96, "norm_s": 0.234, "prev_d2h_s": 1.956, "prev_adamw_s": 4.226, "prev_upload_s": 2.057, "resident_update_s": 0.2}
+{"ts": 1792559958.26, "run": "ntb-r1-res4", "event": "step", "step": 4, "loss": 1.1063324678689241, "grad_norm": 20.461211574277034, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 8.399, "tokens_per_s": 243.8, "wait_weights_s": 1.68, "norm_s": 0.136, "prev_d2h_s": 2.607, "prev_adamw_s": 5.371, "prev_upload_s": 2.404, "resident_update_s": 0.088}
+{"ts": 1792559966.891, "run": "ntb-r1-res4", "event": "step", "step": 5, "loss": 1.3412490785121918, "grad_norm": 13.312377605287216, "lr": 5e-06, "tokens": 2048, "seconds": 8.628, "tokens_per_s": 237.4, "wait_weights_s": 2.275, "norm_s": 0.195, "prev_d2h_s": 3.219, "prev_adamw_s": 9.642, "prev_upload_s": 2.56, "resident_update_s": 0.169}
+{"ts": 1792559971.852, "run": "ntb-r1-res4", "event": "step", "step": 6, "loss": 1.6975219249725342, "grad_norm": 16.00722035826498, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 4.958, "tokens_per_s": 413.0, "wait_weights_s": 1.082, "norm_s": 0.243, "prev_d2h_s": 2.052, "prev_adamw_s": 4.181, "prev_upload_s": 1.88, "resident_update_s": 0.206}
+{"ts": 1792559976.538, "run": "ntb-r1-res4", "event": "step", "step": 7, "loss": 1.7448283731937408, "grad_norm": 16.504575148400374, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 4.682, "tokens_per_s": 437.4, "wait_weights_s": 1.011, "norm_s": 0.205, "prev_d2h_s": 1.931, "prev_adamw_s": 4.224, "prev_upload_s": 1.963, "resident_update_s": 0.179}
+{"ts": 1792559981.66, "run": "ntb-r1-res4", "event": "step", "step": 8, "loss": 1.1026900857686996, "grad_norm": 12.248325808936602, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 5.118, "tokens_per_s": 400.2, "wait_weights_s": 1.079, "norm_s": 0.226, "prev_d2h_s": 1.934, "prev_adamw_s": 4.215, "prev_upload_s": 1.978, "resident_update_s": 0.196}
+{"ts": 1792559986.799, "run": "ntb-r1-res4", "event": "step", "step": 9, "loss": 0.8907594978809357, "grad_norm": 11.50092219153214, "lr": 9e-06, "tokens": 2048, "seconds": 5.131, "tokens_per_s": 399.2, "wait_weights_s": 1.004, "norm_s": 0.206, "prev_d2h_s": 1.919, "prev_adamw_s": 4.325, "prev_upload_s": 1.96, "resident_update_s": 0.174}
+{"ts": 1792559992.111, "run": "ntb-r1-res4", "event": "step", "step": 10, "loss": 1.1144881709478796, "grad_norm": 9.905762855896496, "lr": 1e-05, "tokens": 2048, "seconds": 5.305, "tokens_per_s": 386.1, "wait_weights_s": 1.015, "norm_s": 0.203, "prev_d2h_s": 1.897, "prev_adamw_s": 4.13, "prev_upload_s": 1.979, "resident_update_s": 0.172}
+{"ts": 1792559997.213, "run": "ntb-r1-res4", "event": "step", "step": 11, "loss": 1.1876610815525055, "grad_norm": 10.395274096641511, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 5.098, "tokens_per_s": 401.7, "wait_weights_s": 1.036, "norm_s": 0.2, "prev_d2h_s": 1.892, "prev_adamw_s": 4.249, "prev_upload_s": 1.951, "resident_update_s": 0.172}
+{"ts": 1792560002.615, "run": "ntb-r1-res4", "event": "step", "step": 12, "loss": 1.1581125259399414, "grad_norm": 15.290384430447183, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 5.401, "tokens_per_s": 379.2, "wait_weights_s": 1.053, "norm_s": 0.203, "prev_d2h_s": 1.958, "prev_adamw_s": 4.26, "prev_upload_s": 1.964, "resident_update_s": 0.176}
diff --git a/experiments/data/resident-adamw/r1-res8-oom.err b/experiments/data/resident-adamw/r1-res8-oom.err
new file mode 100644
index 0000000..b9c0222
--- /dev/null
+++ b/experiments/data/resident-adamw/r1-res8-oom.err
@@ -0,0 +1 @@
+torch.OutOfMemoryError: failed to allocate 14680064 bytes on DRAM
diff --git a/experiments/data/resident-adamw/r1-res8-oom.jsonl b/experiments/data/resident-adamw/r1-res8-oom.jsonl
new file mode 100644
index 0000000..95ee487
--- /dev/null
+++ b/experiments/data/resident-adamw/r1-res8-oom.jsonl
@@ -0,0 +1 @@
+{"ts": 1792556378.444, "run": "ntb-r1-res8", "event": "start", "examples": 8000}
diff --git a/experiments/data/resident-adamw/r2-off-b.jsonl b/experiments/data/resident-adamw/r2-off-b.jsonl
new file mode 100644
index 0000000..14131b2
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-off-b.jsonl
@@ -0,0 +1,27 @@
+{"ts": 1792560824.925, "run": "ntb-r2-off-b", "event": "start", "examples": 8000}
+{"ts": 1792561016.236, "run": "ntb-r2-off-b", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 16.953, "tokens_per_s": 120.8, "wait_weights_s": 0.001, "norm_s": 0.061, "prev_d2h_s": 0.0, "prev_adamw_s": 0.0, "prev_upload_s": 0.0, "devices": 2}
+{"ts": 1792561035.303, "run": "ntb-r2-off-b", "event": "step", "step": 2, "loss": 1.5625602304935455, "grad_norm": 45.83976931927431, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 19.063, "tokens_per_s": 107.4, "wait_weights_s": 10.681, "norm_s": 0.075, "prev_d2h_s": 19.978, "prev_adamw_s": 25.326, "prev_upload_s": 4.615, "devices": 2}
+{"ts": 1792561044.666, "run": "ntb-r2-off-b", "event": "step", "step": 3, "loss": 1.0878866240382195, "grad_norm": 83.62784643840048, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 9.358, "tokens_per_s": 218.8, "wait_weights_s": 2.23, "norm_s": 0.085, "prev_d2h_s": 3.304, "prev_adamw_s": 8.659, "prev_upload_s": 2.981, "devices": 2}
+{"ts": 1792561054.422, "run": "ntb-r2-off-b", "event": "step", "step": 4, "loss": 1.105992036871612, "grad_norm": 20.339171194820047, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 9.743, "tokens_per_s": 210.2, "wait_weights_s": 2.442, "norm_s": 0.071, "prev_d2h_s": 3.405, "prev_adamw_s": 8.319, "prev_upload_s": 3.093, "devices": 2}
+{"ts": 1792561063.563, "run": "ntb-r2-off-b", "event": "step", "step": 5, "loss": 1.3360941708087921, "grad_norm": 13.385490951262877, "lr": 5e-06, "tokens": 2048, "seconds": 9.135, "tokens_per_s": 224.2, "wait_weights_s": 2.574, "norm_s": 0.068, "prev_d2h_s": 3.774, "prev_adamw_s": 9.018, "prev_upload_s": 3.227, "devices": 2}
+{"ts": 1792561073.278, "run": "ntb-r2-off-b", "event": "step", "step": 6, "loss": 1.706906497478485, "grad_norm": 16.335248507856008, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 9.705, "tokens_per_s": 211.0, "wait_weights_s": 2.472, "norm_s": 0.068, "prev_d2h_s": 3.485, "prev_adamw_s": 8.658, "prev_upload_s": 3.017, "devices": 2}
+{"ts": 1792561082.907, "run": "ntb-r2-off-b", "event": "step", "step": 7, "loss": 1.7494316697120667, "grad_norm": 16.13586935662257, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 9.62, "tokens_per_s": 212.9, "wait_weights_s": 2.463, "norm_s": 0.069, "prev_d2h_s": 3.675, "prev_adamw_s": 8.649, "prev_upload_s": 3.021, "devices": 2}
+{"ts": 1792561092.656, "run": "ntb-r2-off-b", "event": "step", "step": 8, "loss": 1.1023952513933182, "grad_norm": 12.115460431474599, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 9.74, "tokens_per_s": 210.3, "wait_weights_s": 2.816, "norm_s": 0.067, "prev_d2h_s": 3.476, "prev_adamw_s": 10.207, "prev_upload_s": 2.981, "devices": 2}
+{"ts": 1792561098.935, "run": "ntb-r2-off-b", "event": "step", "step": 9, "loss": 0.8882534727454185, "grad_norm": 11.315825465381323, "lr": 9e-06, "tokens": 2048, "seconds": 6.272, "tokens_per_s": 326.5, "wait_weights_s": 1.67, "norm_s": 0.047, "prev_d2h_s": 2.415, "prev_adamw_s": 7.333, "prev_upload_s": 2.461, "devices": 2}
+{"ts": 1792561105.076, "run": "ntb-r2-off-b", "event": "step", "step": 10, "loss": 1.1120138672413304, "grad_norm": 9.943527357181816, "lr": 1e-05, "tokens": 2048, "seconds": 6.136, "tokens_per_s": 333.8, "wait_weights_s": 1.86, "norm_s": 0.032, "prev_d2h_s": 2.82, "prev_adamw_s": 4.872, "prev_upload_s": 2.576, "devices": 2}
+{"ts": 1792561111.953, "run": "ntb-r2-off-b", "event": "step", "step": 11, "loss": 1.1846136450767517, "grad_norm": 10.343349532186359, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 6.872, "tokens_per_s": 298.0, "wait_weights_s": 1.585, "norm_s": 0.054, "prev_d2h_s": 2.248, "prev_adamw_s": 4.488, "prev_upload_s": 2.316, "devices": 2}
+{"ts": 1792561119.801, "run": "ntb-r2-off-b", "event": "step", "step": 12, "loss": 1.149821549654007, "grad_norm": 15.667075483844068, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 7.844, "tokens_per_s": 261.1, "wait_weights_s": 2.352, "norm_s": 0.025, "prev_d2h_s": 3.295, "prev_adamw_s": 7.533, "prev_upload_s": 2.965, "devices": 2}
+{"ts": 1792561126.753, "run": "ntb-r2-off-b", "event": "step", "step": 13, "loss": 1.3779243230819702, "grad_norm": 9.736832260042876, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 6.949, "tokens_per_s": 294.7, "wait_weights_s": 1.505, "norm_s": 0.035, "prev_d2h_s": 2.528, "prev_adamw_s": 5.123, "prev_upload_s": 2.259, "devices": 2}
+{"ts": 1792561132.475, "run": "ntb-r2-off-b", "event": "step", "step": 14, "loss": 1.1682318598031998, "grad_norm": 28.767873129222597, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 5.718, "tokens_per_s": 358.2, "wait_weights_s": 1.847, "norm_s": 0.043, "prev_d2h_s": 2.498, "prev_adamw_s": 5.449, "prev_upload_s": 2.385, "devices": 2}
+{"ts": 1792561138.023, "run": "ntb-r2-off-b", "event": "step", "step": 15, "loss": 1.0693852007389069, "grad_norm": 9.241371995864162, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 5.543, "tokens_per_s": 369.5, "wait_weights_s": 1.839, "norm_s": 0.031, "prev_d2h_s": 2.952, "prev_adamw_s": 4.352, "prev_upload_s": 2.23, "devices": 2}
+{"ts": 1792561143.767, "run": "ntb-r2-off-b", "event": "step", "step": 16, "loss": 1.2823715209960938, "grad_norm": 10.081775683695783, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 5.742, "tokens_per_s": 356.7, "wait_weights_s": 1.646, "norm_s": 0.032, "prev_d2h_s": 2.33, "prev_adamw_s": 4.519, "prev_upload_s": 2.268, "devices": 2}
+{"ts": 1792561149.632, "run": "ntb-r2-off-b", "event": "step", "step": 17, "loss": 1.1385412216186523, "grad_norm": 13.139631625067793, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 5.861, "tokens_per_s": 349.4, "wait_weights_s": 1.725, "norm_s": 0.035, "prev_d2h_s": 2.661, "prev_adamw_s": 4.577, "prev_upload_s": 2.351, "devices": 2}
+{"ts": 1792561158.339, "run": "ntb-r2-off-b", "event": "step", "step": 18, "loss": 1.4344590604305267, "grad_norm": 9.256871006458512, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 8.702, "tokens_per_s": 235.3, "wait_weights_s": 1.785, "norm_s": 0.046, "prev_d2h_s": 2.816, "prev_adamw_s": 5.051, "prev_upload_s": 2.272, "devices": 2}
+{"ts": 1792561165.435, "run": "ntb-r2-off-b", "event": "step", "step": 19, "loss": 0.8083997741341591, "grad_norm": 8.44625464614702, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 7.09, "tokens_per_s": 288.9, "wait_weights_s": 1.853, "norm_s": 0.032, "prev_d2h_s": 2.792, "prev_adamw_s": 7.094, "prev_upload_s": 2.573, "devices": 2}
+{"ts": 1792561172.738, "run": "ntb-r2-off-b", "event": "step", "step": 20, "loss": 1.5581613630056381, "grad_norm": 8.408779834946662, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 7.299, "tokens_per_s": 280.6, "wait_weights_s": 2.127, "norm_s": 0.029, "prev_d2h_s": 3.236, "prev_adamw_s": 6.946, "prev_upload_s": 2.697, "devices": 2}
+{"ts": 1792561180.048, "run": "ntb-r2-off-b", "event": "step", "step": 21, "loss": 1.272824227809906, "grad_norm": 11.512978955760328, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 7.307, "tokens_per_s": 280.3, "wait_weights_s": 1.868, "norm_s": 0.031, "prev_d2h_s": 2.615, "prev_adamw_s": 6.54, "prev_upload_s": 2.423, "devices": 2}
+{"ts": 1792561186.589, "run": "ntb-r2-off-b", "event": "step", "step": 22, "loss": 1.3625558465719223, "grad_norm": 7.841532302364816, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 6.538, "tokens_per_s": 313.3, "wait_weights_s": 2.359, "norm_s": 0.032, "prev_d2h_s": 4.074, "prev_adamw_s": 4.757, "prev_upload_s": 3.142, "devices": 2}
+{"ts": 1792561195.461, "run": "ntb-r2-off-b", "event": "step", "step": 23, "loss": 0.9013836085796356, "grad_norm": 7.894083656715112, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 8.869, "tokens_per_s": 230.9, "wait_weights_s": 1.631, "norm_s": 0.053, "prev_d2h_s": 2.299, "prev_adamw_s": 4.415, "prev_upload_s": 2.288, "devices": 2}
+{"ts": 1792561206.146, "run": "ntb-r2-off-b", "event": "step", "step": 24, "loss": 0.8878279626369476, "grad_norm": 10.04998445436565, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 10.681, "tokens_per_s": 191.8, "wait_weights_s": 2.905, "norm_s": 0.052, "prev_d2h_s": 4.201, "prev_adamw_s": 10.603, "prev_upload_s": 3.318, "devices": 2}
+{"ts": 1792561216.877, "run": "ntb-r2-off-b", "event": "step", "step": 25, "loss": 1.3050681483000517, "grad_norm": 12.53712000739592, "lr": 0.0, "tokens": 2048, "seconds": 10.726, "tokens_per_s": 190.9, "wait_weights_s": 3.256, "norm_s": 0.078, "prev_d2h_s": 5.07, "prev_adamw_s": 9.882, "prev_upload_s": 3.609, "devices": 2}
+{"ts": 1792561242.055, "run": "ntb-r2-off-b", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8083997741341591, "mode": "full"}
diff --git a/experiments/data/resident-adamw/r2-off-c.jsonl b/experiments/data/resident-adamw/r2-off-c.jsonl
new file mode 100644
index 0000000..985707f
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-off-c.jsonl
@@ -0,0 +1,27 @@
+{"ts": 1792563872.021, "run": "ntb-r2-off-c", "event": "start", "examples": 8000}
+{"ts": 1792564128.469, "run": "ntb-r2-off-c", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 17.976, "tokens_per_s": 113.9, "wait_weights_s": 0.001, "norm_s": 0.067, "prev_d2h_s": 0.0, "prev_adamw_s": 0.0, "prev_upload_s": 0.0, "devices": 2}
+{"ts": 1792564149.199, "run": "ntb-r2-off-c", "event": "step", "step": 2, "loss": 1.5625602304935455, "grad_norm": 45.83976931927431, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 20.725, "tokens_per_s": 98.8, "wait_weights_s": 12.675, "norm_s": 0.064, "prev_d2h_s": 25.565, "prev_adamw_s": 27.521, "prev_upload_s": 5.278, "devices": 2}
+{"ts": 1792564159.581, "run": "ntb-r2-off-c", "event": "step", "step": 3, "loss": 1.0878866240382195, "grad_norm": 83.62784643840048, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 10.379, "tokens_per_s": 197.3, "wait_weights_s": 2.535, "norm_s": 0.091, "prev_d2h_s": 3.952, "prev_adamw_s": 8.253, "prev_upload_s": 3.555, "devices": 2}
+{"ts": 1792564169.59, "run": "ntb-r2-off-c", "event": "step", "step": 4, "loss": 1.105992036871612, "grad_norm": 20.339171194820047, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 10.005, "tokens_per_s": 204.7, "wait_weights_s": 2.691, "norm_s": 0.071, "prev_d2h_s": 3.854, "prev_adamw_s": 9.167, "prev_upload_s": 3.53, "devices": 2}
+{"ts": 1792564180.248, "run": "ntb-r2-off-c", "event": "step", "step": 5, "loss": 1.3360941708087921, "grad_norm": 13.385490951262877, "lr": 5e-06, "tokens": 2048, "seconds": 10.653, "tokens_per_s": 192.3, "wait_weights_s": 2.766, "norm_s": 0.072, "prev_d2h_s": 3.799, "prev_adamw_s": 9.581, "prev_upload_s": 3.719, "devices": 2}
+{"ts": 1792564189.764, "run": "ntb-r2-off-c", "event": "step", "step": 6, "loss": 1.706906497478485, "grad_norm": 16.335248507856008, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 9.509, "tokens_per_s": 215.4, "wait_weights_s": 2.557, "norm_s": 0.061, "prev_d2h_s": 3.639, "prev_adamw_s": 9.009, "prev_upload_s": 3.419, "devices": 2}
+{"ts": 1792564200.102, "run": "ntb-r2-off-c", "event": "step", "step": 7, "loss": 1.7494316697120667, "grad_norm": 16.13586935662257, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 10.333, "tokens_per_s": 198.2, "wait_weights_s": 2.809, "norm_s": 0.069, "prev_d2h_s": 3.763, "prev_adamw_s": 9.784, "prev_upload_s": 3.52, "devices": 2}
+{"ts": 1792564210.296, "run": "ntb-r2-off-c", "event": "step", "step": 8, "loss": 1.1023952513933182, "grad_norm": 12.115460431474599, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 10.187, "tokens_per_s": 201.0, "wait_weights_s": 2.541, "norm_s": 0.106, "prev_d2h_s": 3.588, "prev_adamw_s": 8.501, "prev_upload_s": 3.379, "devices": 2}
+{"ts": 1792564220.173, "run": "ntb-r2-off-c", "event": "step", "step": 9, "loss": 0.8882534727454185, "grad_norm": 11.315825465381323, "lr": 9e-06, "tokens": 2048, "seconds": 9.864, "tokens_per_s": 207.6, "wait_weights_s": 2.372, "norm_s": 0.049, "prev_d2h_s": 3.351, "prev_adamw_s": 10.536, "prev_upload_s": 3.252, "devices": 2}
+{"ts": 1792564231.268, "run": "ntb-r2-off-c", "event": "step", "step": 10, "loss": 1.1120138672413304, "grad_norm": 9.943527357181816, "lr": 1e-05, "tokens": 2048, "seconds": 11.088, "tokens_per_s": 184.7, "wait_weights_s": 2.747, "norm_s": 0.067, "prev_d2h_s": 3.703, "prev_adamw_s": 9.038, "prev_upload_s": 3.262, "devices": 2}
+{"ts": 1792564241.35, "run": "ntb-r2-off-c", "event": "step", "step": 11, "loss": 1.1846136450767517, "grad_norm": 10.343349532186359, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 10.073, "tokens_per_s": 203.3, "wait_weights_s": 2.727, "norm_s": 0.061, "prev_d2h_s": 4.194, "prev_adamw_s": 8.541, "prev_upload_s": 3.194, "devices": 2}
+{"ts": 1792564254.833, "run": "ntb-r2-off-c", "event": "step", "step": 12, "loss": 1.149821549654007, "grad_norm": 15.667075483844068, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 13.479, "tokens_per_s": 151.9, "wait_weights_s": 2.554, "norm_s": 0.076, "prev_d2h_s": 4.128, "prev_adamw_s": 10.499, "prev_upload_s": 3.146, "devices": 2}
+{"ts": 1792564268.594, "run": "ntb-r2-off-c", "event": "step", "step": 13, "loss": 1.3779243230819702, "grad_norm": 9.736832260042876, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 13.756, "tokens_per_s": 148.9, "wait_weights_s": 2.507, "norm_s": 0.113, "prev_d2h_s": 4.334, "prev_adamw_s": 10.439, "prev_upload_s": 3.317, "devices": 2}
+{"ts": 1792564277.827, "run": "ntb-r2-off-c", "event": "step", "step": 14, "loss": 1.1682318598031998, "grad_norm": 28.767873129222597, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 9.227, "tokens_per_s": 222.0, "wait_weights_s": 2.579, "norm_s": 0.046, "prev_d2h_s": 4.054, "prev_adamw_s": 11.81, "prev_upload_s": 3.297, "devices": 2}
+{"ts": 1792564291.172, "run": "ntb-r2-off-c", "event": "step", "step": 15, "loss": 1.0693852007389069, "grad_norm": 9.241371995864162, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 13.34, "tokens_per_s": 153.5, "wait_weights_s": 2.492, "norm_s": 0.063, "prev_d2h_s": 3.915, "prev_adamw_s": 6.938, "prev_upload_s": 3.146, "devices": 2}
+{"ts": 1792564304.145, "run": "ntb-r2-off-c", "event": "step", "step": 16, "loss": 1.2823715209960938, "grad_norm": 10.081775683695783, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 12.968, "tokens_per_s": 157.9, "wait_weights_s": 2.622, "norm_s": 0.118, "prev_d2h_s": 4.394, "prev_adamw_s": 9.196, "prev_upload_s": 3.2, "devices": 2}
+{"ts": 1792564316.567, "run": "ntb-r2-off-c", "event": "step", "step": 17, "loss": 1.1385412216186523, "grad_norm": 13.139631625067793, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 12.414, "tokens_per_s": 165.0, "wait_weights_s": 2.782, "norm_s": 0.068, "prev_d2h_s": 4.062, "prev_adamw_s": 11.445, "prev_upload_s": 3.392, "devices": 2}
+{"ts": 1792564327.226, "run": "ntb-r2-off-c", "event": "step", "step": 18, "loss": 1.4344590604305267, "grad_norm": 9.256871006458512, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 10.654, "tokens_per_s": 192.2, "wait_weights_s": 2.707, "norm_s": 0.052, "prev_d2h_s": 4.268, "prev_adamw_s": 10.054, "prev_upload_s": 3.381, "devices": 2}
+{"ts": 1792564338.312, "run": "ntb-r2-off-c", "event": "step", "step": 19, "loss": 0.8083997741341591, "grad_norm": 8.44625464614702, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 11.083, "tokens_per_s": 184.8, "wait_weights_s": 2.692, "norm_s": 0.058, "prev_d2h_s": 4.466, "prev_adamw_s": 8.607, "prev_upload_s": 3.435, "devices": 2}
+{"ts": 1792564349.091, "run": "ntb-r2-off-c", "event": "step", "step": 20, "loss": 1.5581613630056381, "grad_norm": 8.408779834946662, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 10.775, "tokens_per_s": 190.1, "wait_weights_s": 2.708, "norm_s": 0.07, "prev_d2h_s": 3.717, "prev_adamw_s": 10.323, "prev_upload_s": 3.464, "devices": 2}
+{"ts": 1792564360.08, "run": "ntb-r2-off-c", "event": "step", "step": 21, "loss": 1.272824227809906, "grad_norm": 11.512978955760328, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 10.985, "tokens_per_s": 186.4, "wait_weights_s": 2.872, "norm_s": 0.059, "prev_d2h_s": 4.569, "prev_adamw_s": 9.035, "prev_upload_s": 3.594, "devices": 2}
+{"ts": 1792564371.027, "run": "ntb-r2-off-c", "event": "step", "step": 22, "loss": 1.3625558465719223, "grad_norm": 7.841532302364816, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 10.94, "tokens_per_s": 187.2, "wait_weights_s": 2.777, "norm_s": 0.078, "prev_d2h_s": 3.996, "prev_adamw_s": 9.34, "prev_upload_s": 3.407, "devices": 2}
+{"ts": 1792564383.051, "run": "ntb-r2-off-c", "event": "step", "step": 23, "loss": 0.9013836085796356, "grad_norm": 7.894083656715112, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 12.02, "tokens_per_s": 170.4, "wait_weights_s": 2.631, "norm_s": 0.056, "prev_d2h_s": 5.304, "prev_adamw_s": 9.146, "prev_upload_s": 3.55, "devices": 2}
+{"ts": 1792564394.095, "run": "ntb-r2-off-c", "event": "step", "step": 24, "loss": 0.8878279626369476, "grad_norm": 10.04998445436565, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 11.039, "tokens_per_s": 185.5, "wait_weights_s": 2.778, "norm_s": 0.072, "prev_d2h_s": 4.274, "prev_adamw_s": 10.384, "prev_upload_s": 3.657, "devices": 2}
+{"ts": 1792564404.581, "run": "ntb-r2-off-c", "event": "step", "step": 25, "loss": 1.3050681483000517, "grad_norm": 12.53712000739592, "lr": 0.0, "tokens": 2048, "seconds": 10.478, "tokens_per_s": 195.5, "wait_weights_s": 2.634, "norm_s": 0.069, "prev_d2h_s": 3.836, "prev_adamw_s": 9.201, "prev_upload_s": 3.472, "devices": 2}
+{"ts": 1792564428.389, "run": "ntb-r2-off-c", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8083997741341591, "mode": "full"}
diff --git a/experiments/data/resident-adamw/r2-off.jsonl b/experiments/data/resident-adamw/r2-off.jsonl
new file mode 100644
index 0000000..4f9d718
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-off.jsonl
@@ -0,0 +1,27 @@
+{"ts": 1792556866.79, "run": "ntb-r2-off", "event": "start", "examples": 8000}
+{"ts": 1792557519.905, "run": "ntb-r2-off", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 54.301, "tokens_per_s": 37.7, "wait_weights_s": 0.0, "norm_s": 0.034, "prev_d2h_s": 0.0, "prev_adamw_s": 0.0, "prev_upload_s": 0.0, "devices": 2}
+{"ts": 1792557531.274, "run": "ntb-r2-off", "event": "step", "step": 2, "loss": 1.5625602304935455, "grad_norm": 45.83976931927431, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 11.366, "tokens_per_s": 180.2, "wait_weights_s": 4.008, "norm_s": 0.031, "prev_d2h_s": 10.375, "prev_adamw_s": 9.459, "prev_upload_s": 2.452, "devices": 2}
+{"ts": 1792557536.475, "run": "ntb-r2-off", "event": "step", "step": 3, "loss": 1.0878866240382195, "grad_norm": 83.62784643840048, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 5.197, "tokens_per_s": 394.1, "wait_weights_s": 1.387, "norm_s": 0.03, "prev_d2h_s": 2.243, "prev_adamw_s": 4.374, "prev_upload_s": 2.28, "devices": 2}
+{"ts": 1792557542.063, "run": "ntb-r2-off", "event": "step", "step": 4, "loss": 1.105992036871612, "grad_norm": 20.339171194820047, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 5.585, "tokens_per_s": 366.7, "wait_weights_s": 1.554, "norm_s": 0.032, "prev_d2h_s": 2.138, "prev_adamw_s": 4.611, "prev_upload_s": 2.25, "devices": 2}
+{"ts": 1792557547.084, "run": "ntb-r2-off", "event": "step", "step": 5, "loss": 1.3360941708087921, "grad_norm": 13.385490951262877, "lr": 5e-06, "tokens": 2048, "seconds": 5.017, "tokens_per_s": 408.2, "wait_weights_s": 1.582, "norm_s": 0.029, "prev_d2h_s": 2.106, "prev_adamw_s": 5.067, "prev_upload_s": 2.211, "devices": 2}
+{"ts": 1792557553.385, "run": "ntb-r2-off", "event": "step", "step": 6, "loss": 1.706906497478485, "grad_norm": 16.335248507856008, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 6.297, "tokens_per_s": 325.2, "wait_weights_s": 1.679, "norm_s": 0.03, "prev_d2h_s": 2.227, "prev_adamw_s": 4.858, "prev_upload_s": 2.206, "devices": 2}
+{"ts": 1792557558.448, "run": "ntb-r2-off", "event": "step", "step": 7, "loss": 1.7494316697120667, "grad_norm": 16.13586935662257, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 5.058, "tokens_per_s": 404.9, "wait_weights_s": 1.575, "norm_s": 0.028, "prev_d2h_s": 2.161, "prev_adamw_s": 4.414, "prev_upload_s": 2.304, "devices": 2}
+{"ts": 1792557563.369, "run": "ntb-r2-off", "event": "step", "step": 8, "loss": 1.1023952513933182, "grad_norm": 12.115460431474599, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 4.916, "tokens_per_s": 416.6, "wait_weights_s": 1.466, "norm_s": 0.028, "prev_d2h_s": 2.216, "prev_adamw_s": 4.129, "prev_upload_s": 2.294, "devices": 2}
+{"ts": 1792557568.298, "run": "ntb-r2-off", "event": "step", "step": 9, "loss": 0.8882534727454185, "grad_norm": 11.315825465381323, "lr": 9e-06, "tokens": 2048, "seconds": 4.926, "tokens_per_s": 415.8, "wait_weights_s": 1.497, "norm_s": 0.03, "prev_d2h_s": 2.182, "prev_adamw_s": 4.147, "prev_upload_s": 2.338, "devices": 2}
+{"ts": 1792557573.599, "run": "ntb-r2-off", "event": "step", "step": 10, "loss": 1.1120138672413304, "grad_norm": 9.943527357181816, "lr": 1e-05, "tokens": 2048, "seconds": 5.297, "tokens_per_s": 386.6, "wait_weights_s": 1.526, "norm_s": 0.036, "prev_d2h_s": 2.249, "prev_adamw_s": 4.2, "prev_upload_s": 2.325, "devices": 2}
+{"ts": 1792557578.524, "run": "ntb-r2-off", "event": "step", "step": 11, "loss": 1.1846136450767517, "grad_norm": 10.343349532186359, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 4.92, "tokens_per_s": 416.3, "wait_weights_s": 1.507, "norm_s": 0.035, "prev_d2h_s": 2.204, "prev_adamw_s": 4.264, "prev_upload_s": 2.286, "devices": 2}
+{"ts": 1792557583.507, "run": "ntb-r2-off", "event": "step", "step": 12, "loss": 1.149821549654007, "grad_norm": 15.667075483844068, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 4.979, "tokens_per_s": 411.4, "wait_weights_s": 1.493, "norm_s": 0.03, "prev_d2h_s": 2.186, "prev_adamw_s": 4.245, "prev_upload_s": 2.228, "devices": 2}
+{"ts": 1792557588.558, "run": "ntb-r2-off", "event": "step", "step": 13, "loss": 1.3779243230819702, "grad_norm": 9.736832260042876, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 5.045, "tokens_per_s": 405.9, "wait_weights_s": 1.418, "norm_s": 0.033, "prev_d2h_s": 2.167, "prev_adamw_s": 4.039, "prev_upload_s": 2.255, "devices": 2}
+{"ts": 1792557593.737, "run": "ntb-r2-off", "event": "step", "step": 14, "loss": 1.1682318598031998, "grad_norm": 28.767873129222597, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 5.174, "tokens_per_s": 395.8, "wait_weights_s": 1.491, "norm_s": 0.024, "prev_d2h_s": 2.244, "prev_adamw_s": 4.084, "prev_upload_s": 2.317, "devices": 2}
+{"ts": 1792557598.622, "run": "ntb-r2-off", "event": "step", "step": 15, "loss": 1.0693852007389069, "grad_norm": 9.241371995864162, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 4.882, "tokens_per_s": 419.5, "wait_weights_s": 1.498, "norm_s": 0.031, "prev_d2h_s": 2.243, "prev_adamw_s": 4.071, "prev_upload_s": 2.242, "devices": 2}
+{"ts": 1792557603.745, "run": "ntb-r2-off", "event": "step", "step": 16, "loss": 1.2823715209960938, "grad_norm": 10.081775683695783, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 5.12, "tokens_per_s": 400.0, "wait_weights_s": 1.467, "norm_s": 0.028, "prev_d2h_s": 2.183, "prev_adamw_s": 4.093, "prev_upload_s": 2.182, "devices": 2}
+{"ts": 1792557608.712, "run": "ntb-r2-off", "event": "step", "step": 17, "loss": 1.1385412216186523, "grad_norm": 13.139631625067793, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 4.962, "tokens_per_s": 412.7, "wait_weights_s": 1.474, "norm_s": 0.032, "prev_d2h_s": 2.276, "prev_adamw_s": 4.074, "prev_upload_s": 2.294, "devices": 2}
+{"ts": 1792557614.035, "run": "ntb-r2-off", "event": "step", "step": 18, "loss": 1.4344590604305267, "grad_norm": 9.256871006458512, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 5.319, "tokens_per_s": 385.1, "wait_weights_s": 1.469, "norm_s": 0.037, "prev_d2h_s": 2.118, "prev_adamw_s": 4.214, "prev_upload_s": 2.277, "devices": 2}
+{"ts": 1792557619.275, "run": "ntb-r2-off", "event": "step", "step": 19, "loss": 0.8083997741341591, "grad_norm": 8.44625464614702, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 5.237, "tokens_per_s": 391.1, "wait_weights_s": 1.468, "norm_s": 0.031, "prev_d2h_s": 2.184, "prev_adamw_s": 3.985, "prev_upload_s": 2.392, "devices": 2}
+{"ts": 1792557624.482, "run": "ntb-r2-off", "event": "step", "step": 20, "loss": 1.5581613630056381, "grad_norm": 8.408779834946662, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 5.203, "tokens_per_s": 393.6, "wait_weights_s": 1.418, "norm_s": 0.031, "prev_d2h_s": 2.083, "prev_adamw_s": 4.448, "prev_upload_s": 2.308, "devices": 2}
+{"ts": 1792557629.483, "run": "ntb-r2-off", "event": "step", "step": 21, "loss": 1.272824227809906, "grad_norm": 11.512978955760328, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 4.997, "tokens_per_s": 409.8, "wait_weights_s": 1.538, "norm_s": 0.028, "prev_d2h_s": 2.126, "prev_adamw_s": 4.672, "prev_upload_s": 2.191, "devices": 2}
+{"ts": 1792557634.611, "run": "ntb-r2-off", "event": "step", "step": 22, "loss": 1.3625558465719223, "grad_norm": 7.841532302364816, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 5.125, "tokens_per_s": 399.6, "wait_weights_s": 1.651, "norm_s": 0.026, "prev_d2h_s": 2.662, "prev_adamw_s": 4.066, "prev_upload_s": 2.343, "devices": 2}
+{"ts": 1792557639.885, "run": "ntb-r2-off", "event": "step", "step": 23, "loss": 0.9013836085796356, "grad_norm": 7.894083656715112, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 5.271, "tokens_per_s": 388.5, "wait_weights_s": 1.629, "norm_s": 0.027, "prev_d2h_s": 2.463, "prev_adamw_s": 4.291, "prev_upload_s": 2.308, "devices": 2}
+{"ts": 1792557645.243, "run": "ntb-r2-off", "event": "step", "step": 24, "loss": 0.8878279626369476, "grad_norm": 10.04998445436565, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 5.356, "tokens_per_s": 382.4, "wait_weights_s": 1.677, "norm_s": 0.03, "prev_d2h_s": 2.431, "prev_adamw_s": 4.394, "prev_upload_s": 2.321, "devices": 2}
+{"ts": 1792557650.315, "run": "ntb-r2-off", "event": "step", "step": 25, "loss": 1.3050681483000517, "grad_norm": 12.53712000739592, "lr": 0.0, "tokens": 2048, "seconds": 5.068, "tokens_per_s": 404.1, "wait_weights_s": 1.635, "norm_s": 0.032, "prev_d2h_s": 2.703, "prev_adamw_s": 4.282, "prev_upload_s": 2.317, "devices": 2}
+{"ts": 1792557664.535, "run": "ntb-r2-off", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8083997741341591, "mode": "full"}
diff --git a/experiments/data/resident-adamw/r2-res6.err b/experiments/data/resident-adamw/r2-res6.err
new file mode 100644
index 0000000..82c69bd
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res6.err
@@ -0,0 +1 @@
+torch.OutOfMemoryError: failed to allocate 8388608 bytes on DRAM_SHARED
diff --git a/experiments/data/resident-adamw/r2-res6.jsonl b/experiments/data/resident-adamw/r2-res6.jsonl
new file mode 100644
index 0000000..cf6185c
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res6.jsonl
@@ -0,0 +1,8 @@
+{"ts": 1792561326.194, "run": "ntb-r2-res6", "event": "start", "examples": 8000}
+{"ts": 1792561562.696, "run": "ntb-r2-res6", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 16.348, "tokens_per_s": 125.3, "wait_weights_s": 0.001, "norm_s": 0.31, "prev_d2h_s": 0.0, "prev_adamw_s": 0.248, "prev_upload_s": 0.0, "resident_update_s": 0.249, "devices": 2}
+{"ts": 1792561575.246, "run": "ntb-r2-res6", "event": "step", "step": 2, "loss": 1.5652326047420502, "grad_norm": 45.87542198210046, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 12.546, "tokens_per_s": 163.2, "wait_weights_s": 5.215, "norm_s": 0.293, "prev_d2h_s": 10.528, "prev_adamw_s": 15.048, "prev_upload_s": 2.499, "resident_update_s": 0.232, "devices": 2}
+{"ts": 1792561583.333, "run": "ntb-r2-res6", "event": "step", "step": 3, "loss": 1.0845456086099148, "grad_norm": 85.64511264224448, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 8.081, "tokens_per_s": 253.4, "wait_weights_s": 1.186, "norm_s": 0.32, "prev_d2h_s": 2.037, "prev_adamw_s": 7.745, "prev_upload_s": 1.692, "resident_update_s": 0.256, "devices": 2}
+{"ts": 1792561592.434, "run": "ntb-r2-res6", "event": "step", "step": 4, "loss": 1.1031237998977304, "grad_norm": 20.50673949214962, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 9.094, "tokens_per_s": 225.2, "wait_weights_s": 1.354, "norm_s": 0.291, "prev_d2h_s": 2.169, "prev_adamw_s": 8.247, "prev_upload_s": 1.774, "resident_update_s": 0.24, "devices": 2}
+{"ts": 1792561601.192, "run": "ntb-r2-res6", "event": "step", "step": 5, "loss": 1.3389634490013123, "grad_norm": 13.446247013615157, "lr": 5e-06, "tokens": 2048, "seconds": 8.754, "tokens_per_s": 234.0, "wait_weights_s": 1.277, "norm_s": 0.297, "prev_d2h_s": 2.03, "prev_adamw_s": 8.797, "prev_upload_s": 1.818, "resident_update_s": 0.25, "devices": 2}
+{"ts": 1792561610.609, "run": "ntb-r2-res6", "event": "step", "step": 6, "loss": 1.7053483426570892, "grad_norm": 16.582995350622884, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 9.412, "tokens_per_s": 217.6, "wait_weights_s": 1.394, "norm_s": 0.313, "prev_d2h_s": 2.259, "prev_adamw_s": 8.919, "prev_upload_s": 1.814, "resident_update_s": 0.252, "devices": 2}
+{"ts": 1792561619.508, "run": "ntb-r2-res6", "event": "step", "step": 7, "loss": 1.7470318973064423, "grad_norm": 16.630915940777875, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 8.894, "tokens_per_s": 230.3, "wait_weights_s": 1.361, "norm_s": 0.316, "prev_d2h_s": 1.992, "prev_adamw_s": 8.886, "prev_upload_s": 1.767, "resident_update_s": 0.251, "devices": 2}
diff --git a/experiments/data/resident-adamw/r2-res8-c.err b/experiments/data/resident-adamw/r2-res8-c.err
new file mode 100644
index 0000000..82c69bd
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res8-c.err
@@ -0,0 +1 @@
+torch.OutOfMemoryError: failed to allocate 8388608 bytes on DRAM_SHARED
diff --git a/experiments/data/resident-adamw/r2-res8-c.jsonl b/experiments/data/resident-adamw/r2-res8-c.jsonl
new file mode 100644
index 0000000..3b06cef
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res8-c.jsonl
@@ -0,0 +1,8 @@
+{"ts": 1792563442.169, "run": "ntb-r2-res8-c", "event": "start", "examples": 8000}
+{"ts": 1792563732.082, "run": "ntb-r2-res8-c", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 23.745, "tokens_per_s": 86.2, "wait_weights_s": 0.001, "norm_s": 0.886, "prev_d2h_s": 0.0, "prev_adamw_s": 0.709, "prev_upload_s": 0.0, "resident_update_s": 0.71, "devices": 2}
+{"ts": 1792563743.817, "run": "ntb-r2-res8-c", "event": "step", "step": 2, "loss": 1.564907282590866, "grad_norm": 45.87919322328091, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 11.732, "tokens_per_s": 174.6, "wait_weights_s": 2.418, "norm_s": 0.981, "prev_d2h_s": 7.343, "prev_adamw_s": 16.095, "prev_upload_s": 2.206, "resident_update_s": 0.92, "devices": 2}
+{"ts": 1792563752.786, "run": "ntb-r2-res8-c", "event": "step", "step": 3, "loss": 1.0858428925275803, "grad_norm": 83.79944461492215, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 8.965, "tokens_per_s": 228.4, "wait_weights_s": 0.297, "norm_s": 1.028, "prev_d2h_s": 2.067, "prev_adamw_s": 9.319, "prev_upload_s": 1.69, "resident_update_s": 0.963, "devices": 2}
+{"ts": 1792563761.435, "run": "ntb-r2-res8-c", "event": "step", "step": 4, "loss": 1.1035496043041348, "grad_norm": 20.500255990601204, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 8.641, "tokens_per_s": 237.0, "wait_weights_s": 0.119, "norm_s": 0.829, "prev_d2h_s": 2.022, "prev_adamw_s": 9.315, "prev_upload_s": 1.64, "resident_update_s": 0.769, "devices": 2}
+{"ts": 1792563770.186, "run": "ntb-r2-res8-c", "event": "step", "step": 5, "loss": 1.336936205625534, "grad_norm": 13.448047566857905, "lr": 5e-06, "tokens": 2048, "seconds": 8.747, "tokens_per_s": 234.1, "wait_weights_s": 0.269, "norm_s": 0.827, "prev_d2h_s": 1.963, "prev_adamw_s": 8.593, "prev_upload_s": 1.67, "resident_update_s": 0.771, "devices": 2}
+{"ts": 1792563778.877, "run": "ntb-r2-res8-c", "event": "step", "step": 6, "loss": 1.6996679306030273, "grad_norm": 16.169723071831996, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 8.688, "tokens_per_s": 235.7, "wait_weights_s": 0.321, "norm_s": 0.88, "prev_d2h_s": 1.993, "prev_adamw_s": 8.637, "prev_upload_s": 1.551, "resident_update_s": 0.812, "devices": 2}
+{"ts": 1792563787.848, "run": "ntb-r2-res8-c", "event": "step", "step": 7, "loss": 1.744823396205902, "grad_norm": 16.281523243942537, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 8.966, "tokens_per_s": 228.4, "wait_weights_s": 0.213, "norm_s": 0.934, "prev_d2h_s": 2.063, "prev_adamw_s": 8.693, "prev_upload_s": 1.521, "resident_update_s": 0.86, "devices": 2}
diff --git a/experiments/data/resident-adamw/r2-res8-mem.jsonl b/experiments/data/resident-adamw/r2-res8-mem.jsonl
new file mode 100644
index 0000000..9ce40c3
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res8-mem.jsonl
@@ -0,0 +1,27 @@
+{"ts": 1792562400.627, "run": "ntb-mem", "event": "start", "examples": 8000}
+{"ts": 1792562744.923, "run": "ntb-mem", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 31.906, "tokens_per_s": 64.2, "wait_weights_s": 0.0, "norm_s": 0.273, "prev_d2h_s": 0.0, "prev_adamw_s": 0.224, "prev_upload_s": 0.0, "resident_update_s": 0.225, "devices": 2}
+{"ts": 1792562755.252, "run": "ntb-mem", "event": "step", "step": 2, "loss": 1.564907282590866, "grad_norm": 45.87919322328091, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 10.325, "tokens_per_s": 198.4, "wait_weights_s": 1.769, "norm_s": 0.208, "prev_d2h_s": 6.358, "prev_adamw_s": 8.646, "prev_upload_s": 1.39, "resident_update_s": 0.178, "devices": 2}
+{"ts": 1792562760.545, "run": "ntb-mem", "event": "step", "step": 3, "loss": 1.0858428925275803, "grad_norm": 83.79944461492215, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 5.28, "tokens_per_s": 387.9, "wait_weights_s": 0.603, "norm_s": 0.248, "prev_d2h_s": 1.435, "prev_adamw_s": 4.897, "prev_upload_s": 1.246, "resident_update_s": 0.21, "devices": 2}
+{"ts": 1792562769.258, "run": "ntb-mem", "event": "step", "step": 4, "loss": 1.1035496043041348, "grad_norm": 20.500255990601204, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 8.71, "tokens_per_s": 235.1, "wait_weights_s": 0.771, "norm_s": 0.42, "prev_d2h_s": 1.431, "prev_adamw_s": 4.852, "prev_upload_s": 1.175, "resident_update_s": 0.349, "devices": 2}
+{"ts": 1792562778.985, "run": "ntb-mem", "event": "step", "step": 5, "loss": 1.336936205625534, "grad_norm": 13.448047566857905, "lr": 5e-06, "tokens": 2048, "seconds": 9.719, "tokens_per_s": 210.7, "wait_weights_s": 1.118, "norm_s": 0.482, "prev_d2h_s": 2.358, "prev_adamw_s": 9.158, "prev_upload_s": 1.698, "resident_update_s": 0.409, "devices": 2}
+{"ts": 1792562788.48, "run": "ntb-mem", "event": "step", "step": 6, "loss": 1.6996679306030273, "grad_norm": 16.169723071831996, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 9.488, "tokens_per_s": 215.8, "wait_weights_s": 1.032, "norm_s": 0.45, "prev_d2h_s": 2.062, "prev_adamw_s": 10.331, "prev_upload_s": 1.581, "resident_update_s": 0.368, "devices": 2}
+{"ts": 1792562797.941, "run": "ntb-mem", "event": "step", "step": 7, "loss": 1.744823396205902, "grad_norm": 16.281523243942537, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 9.457, "tokens_per_s": 216.6, "wait_weights_s": 1.309, "norm_s": 0.448, "prev_d2h_s": 2.216, "prev_adamw_s": 9.812, "prev_upload_s": 1.72, "resident_update_s": 0.378, "devices": 2}
+{"ts": 1792562807.573, "run": "ntb-mem", "event": "step", "step": 8, "loss": 1.1050789058208466, "grad_norm": 12.173483887642671, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 9.628, "tokens_per_s": 212.7, "wait_weights_s": 1.158, "norm_s": 0.469, "prev_d2h_s": 2.203, "prev_adamw_s": 9.555, "prev_upload_s": 1.597, "resident_update_s": 0.386, "devices": 2}
+{"ts": 1792562817.014, "run": "ntb-mem", "event": "step", "step": 9, "loss": 0.8817741200327873, "grad_norm": 11.427005350223444, "lr": 9e-06, "tokens": 2048, "seconds": 9.433, "tokens_per_s": 217.1, "wait_weights_s": 1.261, "norm_s": 0.438, "prev_d2h_s": 2.156, "prev_adamw_s": 10.012, "prev_upload_s": 1.639, "resident_update_s": 0.375, "devices": 2}
+{"ts": 1792562826.224, "run": "ntb-mem", "event": "step", "step": 10, "loss": 1.1135812494903803, "grad_norm": 9.879150640935412, "lr": 1e-05, "tokens": 2048, "seconds": 9.206, "tokens_per_s": 222.5, "wait_weights_s": 1.359, "norm_s": 0.386, "prev_d2h_s": 2.271, "prev_adamw_s": 10.068, "prev_upload_s": 1.762, "resident_update_s": 0.326, "devices": 2}
+{"ts": 1792562835.731, "run": "ntb-mem", "event": "step", "step": 11, "loss": 1.1879100799560547, "grad_norm": 10.501148626361774, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 9.499, "tokens_per_s": 215.6, "wait_weights_s": 0.975, "norm_s": 0.448, "prev_d2h_s": 2.194, "prev_adamw_s": 9.129, "prev_upload_s": 1.644, "resident_update_s": 0.374, "devices": 2}
+{"ts": 1792562845.177, "run": "ntb-mem", "event": "step", "step": 12, "loss": 1.152946263551712, "grad_norm": 15.871865774465505, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 9.441, "tokens_per_s": 216.9, "wait_weights_s": 1.451, "norm_s": 0.46, "prev_d2h_s": 2.628, "prev_adamw_s": 10.208, "prev_upload_s": 1.763, "resident_update_s": 0.384, "devices": 2}
+{"ts": 1792562854.976, "run": "ntb-mem", "event": "step", "step": 13, "loss": 1.3831146657466888, "grad_norm": 9.823916977098824, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 9.794, "tokens_per_s": 209.1, "wait_weights_s": 1.351, "norm_s": 0.48, "prev_d2h_s": 3.344, "prev_adamw_s": 10.377, "prev_upload_s": 1.672, "resident_update_s": 0.412, "devices": 2}
+{"ts": 1792562864.402, "run": "ntb-mem", "event": "step", "step": 14, "loss": 1.1788776516914368, "grad_norm": 25.40595380374666, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 9.419, "tokens_per_s": 217.4, "wait_weights_s": 1.238, "norm_s": 0.399, "prev_d2h_s": 2.23, "prev_adamw_s": 10.632, "prev_upload_s": 1.678, "resident_update_s": 0.334, "devices": 2}
+{"ts": 1792562873.951, "run": "ntb-mem", "event": "step", "step": 15, "loss": 1.0667250454425812, "grad_norm": 9.45781636499448, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 9.541, "tokens_per_s": 214.7, "wait_weights_s": 1.488, "norm_s": 0.386, "prev_d2h_s": 3.612, "prev_adamw_s": 9.278, "prev_upload_s": 1.563, "resident_update_s": 0.318, "devices": 2}
+{"ts": 1792562883.275, "run": "ntb-mem", "event": "step", "step": 16, "loss": 1.2814564108848572, "grad_norm": 9.82564657798959, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 9.321, "tokens_per_s": 219.7, "wait_weights_s": 1.258, "norm_s": 0.446, "prev_d2h_s": 2.048, "prev_adamw_s": 9.486, "prev_upload_s": 1.53, "resident_update_s": 0.373, "devices": 2}
+{"ts": 1792562893.302, "run": "ntb-mem", "event": "step", "step": 17, "loss": 1.1525242775678635, "grad_norm": 12.536942518503388, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 10.022, "tokens_per_s": 204.4, "wait_weights_s": 1.944, "norm_s": 0.435, "prev_d2h_s": 3.794, "prev_adamw_s": 10.258, "prev_upload_s": 1.904, "resident_update_s": 0.353, "devices": 2}
+{"ts": 1792562902.664, "run": "ntb-mem", "event": "step", "step": 18, "loss": 1.4375255107879639, "grad_norm": 8.443515455504027, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 9.355, "tokens_per_s": 218.9, "wait_weights_s": 1.412, "norm_s": 0.446, "prev_d2h_s": 2.334, "prev_adamw_s": 10.202, "prev_upload_s": 1.474, "resident_update_s": 0.364, "devices": 2}
+{"ts": 1792562911.801, "run": "ntb-mem", "event": "step", "step": 19, "loss": 0.8065920546650887, "grad_norm": 8.344874941911247, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 9.126, "tokens_per_s": 224.4, "wait_weights_s": 1.167, "norm_s": 0.456, "prev_d2h_s": 2.329, "prev_adamw_s": 9.702, "prev_upload_s": 1.55, "resident_update_s": 0.389, "devices": 2}
+{"ts": 1792562921.573, "run": "ntb-mem", "event": "step", "step": 20, "loss": 1.5583547055721283, "grad_norm": 8.224272573940292, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 9.761, "tokens_per_s": 209.8, "wait_weights_s": 1.488, "norm_s": 0.437, "prev_d2h_s": 4.274, "prev_adamw_s": 7.878, "prev_upload_s": 1.78, "resident_update_s": 0.37, "devices": 2}
+{"ts": 1792562931.043, "run": "ntb-mem", "event": "step", "step": 21, "loss": 1.2753856480121613, "grad_norm": 11.440582162042189, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 9.464, "tokens_per_s": 216.4, "wait_weights_s": 1.03, "norm_s": 0.445, "prev_d2h_s": 2.357, "prev_adamw_s": 8.752, "prev_upload_s": 1.562, "resident_update_s": 0.379, "devices": 2}
+{"ts": 1792562939.948, "run": "ntb-mem", "event": "step", "step": 22, "loss": 1.357256829738617, "grad_norm": 7.770549618130633, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 8.9, "tokens_per_s": 230.1, "wait_weights_s": 1.051, "norm_s": 0.458, "prev_d2h_s": 2.317, "prev_adamw_s": 9.026, "prev_upload_s": 1.494, "resident_update_s": 0.383, "devices": 2}
+{"ts": 1792562950.305, "run": "ntb-mem", "event": "step", "step": 23, "loss": 0.8956179767847061, "grad_norm": 7.827806084439568, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 10.352, "tokens_per_s": 197.8, "wait_weights_s": 1.126, "norm_s": 0.534, "prev_d2h_s": 1.943, "prev_adamw_s": 9.62, "prev_upload_s": 1.517, "resident_update_s": 0.437, "devices": 2}
+{"ts": 1792562960.272, "run": "ntb-mem", "event": "step", "step": 24, "loss": 0.8916763514280319, "grad_norm": 10.107818403310903, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 9.96, "tokens_per_s": 205.6, "wait_weights_s": 1.027, "norm_s": 0.407, "prev_d2h_s": 1.957, "prev_adamw_s": 11.123, "prev_upload_s": 1.645, "resident_update_s": 0.342, "devices": 2}
+{"ts": 1792562970.557, "run": "ntb-mem", "event": "step", "step": 25, "loss": 1.2988715544342995, "grad_norm": 12.731063043562827, "lr": 0.0, "tokens": 2048, "seconds": 10.28, "tokens_per_s": 199.2, "wait_weights_s": 1.199, "norm_s": 0.459, "prev_d2h_s": 2.085, "prev_adamw_s": 9.538, "prev_upload_s": 1.705, "resident_update_s": 0.385, "devices": 2}
+{"ts": 1792562992.952, "run": "ntb-mem", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8065920546650887, "mode": "full"}
diff --git a/experiments/data/resident-adamw/r2-res8-try1.err b/experiments/data/resident-adamw/r2-res8-try1.err
new file mode 100644
index 0000000..c453c27
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res8-try1.err
@@ -0,0 +1 @@
+torch.OutOfMemoryError: failed to allocate 33554432 bytes on DRAM_SHARED
diff --git a/experiments/data/resident-adamw/r2-res8-try1.jsonl b/experiments/data/resident-adamw/r2-res8-try1.jsonl
new file mode 100644
index 0000000..c33a928
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res8-try1.jsonl
@@ -0,0 +1,4 @@
+{"ts": 1792556605.668, "run": "ntb-r2-res8", "event": "start", "examples": 8000}
+{"ts": 1792556807.481, "run": "ntb-r2-res8", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 13.159, "tokens_per_s": 155.6, "wait_weights_s": 0.0, "norm_s": 0.431, "prev_d2h_s": 0.0, "prev_adamw_s": 0.398, "prev_upload_s": 0.0, "resident_update_s": 0.4, "devices": 2}
+{"ts": 1792556813.691, "run": "ntb-r2-res8", "event": "step", "step": 2, "loss": 1.564907282590866, "grad_norm": 45.87919322328091, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 6.207, "tokens_per_s": 330.0, "wait_weights_s": 1.549, "norm_s": 0.466, "prev_d2h_s": 5.086, "prev_adamw_s": 6.43, "prev_upload_s": 1.221, "resident_update_s": 0.435, "devices": 2}
+{"ts": 1792556817.902, "run": "ntb-r2-res8", "event": "step", "step": 3, "loss": 1.0858428925275803, "grad_norm": 83.79944461492215, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 4.207, "tokens_per_s": 486.8, "wait_weights_s": 0.198, "norm_s": 0.479, "prev_d2h_s": 1.12, "prev_adamw_s": 3.901, "prev_upload_s": 1.21, "resident_update_s": 0.449, "devices": 2}
diff --git a/experiments/data/resident-adamw/r2-res8-try2.err b/experiments/data/resident-adamw/r2-res8-try2.err
new file mode 100644
index 0000000..82c69bd
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res8-try2.err
@@ -0,0 +1 @@
+torch.OutOfMemoryError: failed to allocate 8388608 bytes on DRAM_SHARED
diff --git a/experiments/data/resident-adamw/r2-res8-try2.jsonl b/experiments/data/resident-adamw/r2-res8-try2.jsonl
new file mode 100644
index 0000000..9abcf08
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res8-try2.jsonl
@@ -0,0 +1,8 @@
+{"ts": 1792558542.821, "run": "ntb-r2-res8", "event": "start", "examples": 8000}
+{"ts": 1792559437.888, "run": "ntb-r2-res8", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 70.741, "tokens_per_s": 29.0, "wait_weights_s": 0.0, "norm_s": 0.463, "prev_d2h_s": 0.0, "prev_adamw_s": 0.435, "prev_upload_s": 0.0, "resident_update_s": 0.435, "devices": 2}
+{"ts": 1792559447.647, "run": "ntb-r2-res8", "event": "step", "step": 2, "loss": 1.564907282590866, "grad_norm": 45.87919322328091, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 9.758, "tokens_per_s": 209.9, "wait_weights_s": 0.952, "norm_s": 0.491, "prev_d2h_s": 5.927, "prev_adamw_s": 7.96, "prev_upload_s": 1.253, "resident_update_s": 0.459, "devices": 2}
+{"ts": 1792559451.849, "run": "ntb-r2-res8", "event": "step", "step": 3, "loss": 1.0858428925275803, "grad_norm": 83.79944461492215, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 4.198, "tokens_per_s": 487.9, "wait_weights_s": 0.002, "norm_s": 0.519, "prev_d2h_s": 1.133, "prev_adamw_s": 4.242, "prev_upload_s": 1.165, "resident_update_s": 0.487, "devices": 2}
+{"ts": 1792559456.948, "run": "ntb-r2-res8", "event": "step", "step": 4, "loss": 1.1035496043041348, "grad_norm": 20.500255990601204, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 5.094, "tokens_per_s": 402.0, "wait_weights_s": 0.149, "norm_s": 0.582, "prev_d2h_s": 1.173, "prev_adamw_s": 3.908, "prev_upload_s": 1.278, "resident_update_s": 0.551, "devices": 2}
+{"ts": 1792559461.19, "run": "ntb-r2-res8", "event": "step", "step": 5, "loss": 1.336936205625534, "grad_norm": 13.448047566857905, "lr": 5e-06, "tokens": 2048, "seconds": 4.239, "tokens_per_s": 483.1, "wait_weights_s": 0.084, "norm_s": 0.544, "prev_d2h_s": 1.124, "prev_adamw_s": 4.1, "prev_upload_s": 1.211, "resident_update_s": 0.516, "devices": 2}
+{"ts": 1792559466.11, "run": "ntb-r2-res8", "event": "step", "step": 6, "loss": 1.6996679306030273, "grad_norm": 16.169723071831996, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 4.916, "tokens_per_s": 416.6, "wait_weights_s": 0.18, "norm_s": 0.554, "prev_d2h_s": 1.11, "prev_adamw_s": 4.194, "prev_upload_s": 1.155, "resident_update_s": 0.519, "devices": 2}
+{"ts": 1792559470.685, "run": "ntb-r2-res8", "event": "step", "step": 7, "loss": 1.744823396205902, "grad_norm": 16.281523243942537, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 4.57, "tokens_per_s": 448.1, "wait_weights_s": 0.163, "norm_s": 0.574, "prev_d2h_s": 1.124, "prev_adamw_s": 4.235, "prev_upload_s": 1.187, "resident_update_s": 0.539, "devices": 2}
diff --git a/experiments/data/resident-adamw/r2-res8-try3.err b/experiments/data/resident-adamw/r2-res8-try3.err
new file mode 100644
index 0000000..82c69bd
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res8-try3.err
@@ -0,0 +1 @@
+torch.OutOfMemoryError: failed to allocate 8388608 bytes on DRAM_SHARED
diff --git a/experiments/data/resident-adamw/r2-res8-try3.jsonl b/experiments/data/resident-adamw/r2-res8-try3.jsonl
new file mode 100644
index 0000000..0cc62dd
--- /dev/null
+++ b/experiments/data/resident-adamw/r2-res8-try3.jsonl
@@ -0,0 +1,14 @@
+{"ts": 1792560332.515, "run": "ntb-r2-res8", "event": "start", "examples": 8000}
+{"ts": 1792560560.212, "run": "ntb-r2-res8", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 18.934, "tokens_per_s": 108.2, "wait_weights_s": 0.001, "norm_s": 0.463, "prev_d2h_s": 0.0, "prev_adamw_s": 0.378, "prev_upload_s": 0.0, "resident_update_s": 0.379, "devices": 2}
+{"ts": 1792560572.186, "run": "ntb-r2-res8", "event": "step", "step": 2, "loss": 1.564907282590866, "grad_norm": 45.87919322328091, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 11.97, "tokens_per_s": 171.1, "wait_weights_s": 3.028, "norm_s": 0.336, "prev_d2h_s": 7.562, "prev_adamw_s": 12.943, "prev_upload_s": 1.954, "resident_update_s": 0.25, "devices": 2}
+{"ts": 1792560580.344, "run": "ntb-r2-res8", "event": "step", "step": 3, "loss": 1.0858428925275803, "grad_norm": 83.79944461492215, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 8.155, "tokens_per_s": 251.1, "wait_weights_s": 0.595, "norm_s": 0.441, "prev_d2h_s": 1.108, "prev_adamw_s": 9.021, "prev_upload_s": 1.066, "resident_update_s": 0.357, "devices": 2}
+{"ts": 1792560589.499, "run": "ntb-r2-res8", "event": "step", "step": 4, "loss": 1.1035496043041348, "grad_norm": 20.500255990601204, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 9.151, "tokens_per_s": 223.8, "wait_weights_s": 0.809, "norm_s": 0.409, "prev_d2h_s": 1.45, "prev_adamw_s": 8.978, "prev_upload_s": 1.419, "resident_update_s": 0.341, "devices": 2}
+{"ts": 1792560598.256, "run": "ntb-r2-res8", "event": "step", "step": 5, "loss": 1.336936205625534, "grad_norm": 13.448047566857905, "lr": 5e-06, "tokens": 2048, "seconds": 8.751, "tokens_per_s": 234.0, "wait_weights_s": 0.86, "norm_s": 0.374, "prev_d2h_s": 1.57, "prev_adamw_s": 9.172, "prev_upload_s": 1.364, "resident_update_s": 0.307, "devices": 2}
+{"ts": 1792560607.254, "run": "ntb-r2-res8", "event": "step", "step": 6, "loss": 1.6996679306030273, "grad_norm": 16.169723071831996, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 8.994, "tokens_per_s": 227.7, "wait_weights_s": 0.939, "norm_s": 0.421, "prev_d2h_s": 1.301, "prev_adamw_s": 8.464, "prev_upload_s": 1.358, "resident_update_s": 0.35, "devices": 2}
+{"ts": 1792560615.606, "run": "ntb-r2-res8", "event": "step", "step": 7, "loss": 1.744823396205902, "grad_norm": 16.281523243942537, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 8.349, "tokens_per_s": 245.3, "wait_weights_s": 0.779, "norm_s": 0.415, "prev_d2h_s": 1.393, "prev_adamw_s": 9.17, "prev_upload_s": 1.344, "resident_update_s": 0.344, "devices": 2}
+{"ts": 1792560624.288, "run": "ntb-r2-res8", "event": "step", "step": 8, "loss": 1.1050789058208466, "grad_norm": 12.173483887642671, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 8.677, "tokens_per_s": 236.0, "wait_weights_s": 0.74, "norm_s": 0.436, "prev_d2h_s": 1.467, "prev_adamw_s": 9.171, "prev_upload_s": 1.317, "resident_update_s": 0.349, "devices": 2}
+{"ts": 1792560632.913, "run": "ntb-r2-res8", "event": "step", "step": 9, "loss": 0.8817741200327873, "grad_norm": 11.427005350223444, "lr": 9e-06, "tokens": 2048, "seconds": 8.621, "tokens_per_s": 237.6, "wait_weights_s": 0.733, "norm_s": 0.28, "prev_d2h_s": 1.502, "prev_adamw_s": 9.376, "prev_upload_s": 1.367, "resident_update_s": 0.239, "devices": 2}
+{"ts": 1792560641.494, "run": "ntb-r2-res8", "event": "step", "step": 10, "loss": 1.1135812494903803, "grad_norm": 9.879150640935412, "lr": 1e-05, "tokens": 2048, "seconds": 8.575, "tokens_per_s": 238.8, "wait_weights_s": 0.658, "norm_s": 0.414, "prev_d2h_s": 1.342, "prev_adamw_s": 7.67, "prev_upload_s": 1.315, "resident_update_s": 0.337, "devices": 2}
+{"ts": 1792560650.446, "run": "ntb-r2-res8", "event": "step", "step": 11, "loss": 1.1879100799560547, "grad_norm": 10.501148626361774, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 8.945, "tokens_per_s": 229.0, "wait_weights_s": 0.81, "norm_s": 0.421, "prev_d2h_s": 1.531, "prev_adamw_s": 9.183, "prev_upload_s": 1.338, "resident_update_s": 0.346, "devices": 2}
+{"ts": 1792560659.552, "run": "ntb-r2-res8", "event": "step", "step": 12, "loss": 1.152946263551712, "grad_norm": 15.871865774465505, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 9.104, "tokens_per_s": 225.0, "wait_weights_s": 0.796, "norm_s": 0.392, "prev_d2h_s": 1.541, "prev_adamw_s": 9.051, "prev_upload_s": 1.312, "resident_update_s": 0.319, "devices": 2}
+{"ts": 1792560668.223, "run": "ntb-r2-res8", "event": "step", "step": 13, "loss": 1.3831146657466888, "grad_norm": 9.823916977098824, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 8.667, "tokens_per_s": 236.3, "wait_weights_s": 0.675, "norm_s": 0.46, "prev_d2h_s": 1.318, "prev_adamw_s": 9.116, "prev_upload_s": 1.269, "resident_update_s": 0.393, "devices": 2}
diff --git a/npu_lora/backends.py b/npu_lora/backends.py
index 6bbd2e8..ad86c48 100644
--- a/npu_lora/backends.py
+++ b/npu_lora/backends.py
@@ -114,7 +114,7 @@ def __init__(self, chip=0, cache_dir=None, tactic="ForMiscModel", fusion=8, mesh
         if compile_only:
             chip = 0
         elif chip < 0:
-            chip = visible_chip()
+            chip = visible_chip(-1 - chip)          # -1: first assigned card, -2: second, ...
         self.sdk = _version("furiosa-torch")
         self.new = tuple(int(x) for x in self.sdk.split(".")[:2] if x.isdigit()) >= (2026, 3)
         if self.new:
@@ -216,14 +216,14 @@ def describe(self):
                 "max_shapes": self.max_shapes}
 
 
-def visible_chip():
+def visible_chip(nth=0):
     """쓸 RNGD 의 furiosa-torch 번호.
 
     furiosa-torch 의 `rngd:N` 은 /dev/rngd 에 보이는 카드 중 N 번째(서수)다 — npu-node-1 처럼 npu0·npu1·npu3
     이 보이면 rngd:2 가 npu3 이다. 또 furiosa-torch 는 privileged 가 아니면 장치를 열지 못한다(device plugin
     이 노출한 카드만으로는 `device index out of range`). 그래서 쿠버네티스에서는 privileged 가 아닌 init
     컨테이너가 배정받은 카드 이름(npuN)을 NPU_LORA_RNGD_FILE 에 적고, 여기서 그 카드의 서수를 찾는다.
-    둘 다 없으면 첫 카드.
+    둘 다 없으면 첫 카드. A multi-card pod lists several names ("npu1,npu3"); nth picks one of them in order.
     """
     import re
     try:
@@ -237,11 +237,14 @@ def visible_chip():
     if not want and path and os.path.exists(path):
         want = open(path).read().strip()
     if want:
-        n = int(re.sub(r"\D", "", want))
+        names = [x for x in re.split(r"[,\s]+", want) if x]
+        if nth >= len(names):
+            raise RuntimeError(f"card {nth} requested but only {names} are assigned")
+        n = int(re.sub(r"\D", "", names[nth]))
         if n not in ids:
-            raise RuntimeError(f"assigned RNGD {want} is not visible ({ids}); is the container privileged?")
+            raise RuntimeError(f"assigned RNGD {names[nth]} is not visible ({ids}); is the container privileged?")
         return ids.index(n)
-    return 0
+    return nth
 
 
 def _version(pkg):
@@ -251,11 +254,12 @@ def _version(pkg):
         return "unknown"
 
 
-def make(spec, **kw):
-    """'furiosa:0' / 'furiosa:auto'(컨테이너에 보이는 카드) / 'cuda:0' / 'cpu' → Backend"""
+def make(spec, nth=0, **kw):
+    """'furiosa:0' / 'furiosa:auto'(컨테이너에 보이는 카드) / 'cuda:0' / 'cpu' → Backend.
+    nth: for 'furiosa:auto', which of the pod's assigned cards (`--backend furiosa:auto,furiosa:auto` = both)."""
     kind, _, idx = spec.partition(":")
     if kind in ("furiosa", "rngd"):
-        return FuriosaBackend(chip=-1 if idx in ("", "auto") else int(idx), **kw)
+        return FuriosaBackend(chip=-1 - nth if idx in ("", "auto") else int(idx), **kw)
     if kind == "cuda":
         return CUDABackend(int(idx or 0))
     if kind == "cpu":
diff --git a/npu_lora/cli.py b/npu_lora/cli.py
index be44957..c46c63a 100644
--- a/npu_lora/cli.py
+++ b/npu_lora/cli.py
@@ -31,7 +31,7 @@ def _backend(a):
     from . import backends
     kw = {"tactic": a.tactic, "cache_dir": a.cache_dir, "max_shapes": a.max_shapes} if a.backend.startswith(("furiosa", "rngd")) else {}
     specs = a.backend.split(",")
-    bes = [backends.make(x, **kw) for x in specs]
+    bes = [backends.make(x, nth=i, **kw) for i, x in enumerate(specs)]   # furiosa:auto,furiosa:auto = the pod's two cards
     return bes if len(bes) > 1 else bes[0]
 
 
@@ -59,7 +59,8 @@ def cmd_train(a):
                    airm_tags={"component": "npu-lora", "workload": "train", "model": model,
                               "vendor": vendor(a.backend), "run_id": run})
     tc = TrainConfig(seq_len=a.seq_len, lr=a.lr, steps=a.steps, accum=a.accum, warmup=a.warmup,
-                     grad_clip=a.grad_clip, seed=a.seed, full=a.full, resident_layers=a.resident_layers)
+                     grad_clip=a.grad_clip, seed=a.seed, full=a.full, resident_layers=a.resident_layers,
+                     update_workers=a.update_workers)
     tr = LoRATrainer(a.model, _backend(a), _lora(a), tc, log=lambda s: rep.event("log", **json.loads(s)),
                      max_mm=a.max_mm)
     tok = AutoTokenizer.from_pretrained(tr.path)
@@ -137,6 +138,9 @@ def cmd_compile(a):
     from .trainer import LoRATrainer, TrainConfig
     be = backends.FuriosaBackend(chip=0, cache_dir=a.cache_dir, tactic=a.tactic, max_shapes=a.max_shapes,
                                  compile_only=True)
+    if a.adamw_only:
+        _compile_adamw(a, be)
+        return
     tc = TrainConfig(seq_len=a.seq_len, accum=a.accum, full=a.full, resident_layers=a.resident_layers)
     tr = LoRATrainer(a.model, be, _lora(a), tc, log=lambda s: print(s, flush=True), max_mm=a.max_mm)
     t0 = time.time()
@@ -144,6 +148,26 @@ def cmd_compile(a):
     print(json.dumps({"event": "compiled", "seconds": round(time.time() - t0, 1), "cache": be.cache_dir}))
 
 
+def _compile_adamw(a, be):
+    """Only the resident device AdamW graphs (`train --full --resident-layers N`): one per 2-D weight shape.
+
+    They depend on the weight shapes alone, so this reads no weights and builds no training graph. The full
+    `compile` path first re-traces and re-runs every training graph part on the CPU (bf16, even when every
+    part is already cached); on a shared CPU node that did not reach the AdamW graphs in 2.5 h (#34).
+    """
+    from . import llama as L
+    from .optim_device import ResidentAdamW, resident_shapes
+    path = L.resolve_model_path(a.model)
+    c = L.LlamaConfig.from_hf(path)
+    ra = ResidentAdamW(be, log=lambda m: print(m, flush=True))
+    t0 = time.time()
+    for shape in resident_shapes(L.Weights(path), c):
+        t = time.time()
+        ra.compile(shape)
+        print(json.dumps({"event": "compiled_adamw", "shape": list(shape), "seconds": round(time.time() - t, 1)}), flush=True)
+    print(json.dumps({"event": "compiled", "seconds": round(time.time() - t0, 1), "cache": be.cache_dir}))
+
+
 def cmd_merge(a):
     from . import export
     export.save_merged(a.model, a.adapter, a.out)
@@ -221,6 +245,8 @@ def parser():
                    help="전체 파인튜닝: 레이어 가중치 전부 학습(--out 에 HF 전체 모델). 임베딩·lm_head 는 고정")
     t.add_argument("--resident-layers", type=int, default=0,
                    help="--full: per device, the first N layers keep fp32 master + AdamW state on the device")
+    t.add_argument("--update-workers", type=int, default=4,
+                   help="--full: host threads for the per-layer offload jobs (gradient D2H, AdamW, H2D)")
     t.add_argument("--run-name", default=None)
     t.add_argument("--log-file", default=None)
     t.add_argument("--metrics-port", type=int, default=None, help="Prometheus /metrics 포트")
@@ -237,6 +263,8 @@ def parser():
     cc.add_argument("--full", action="store_true")
     cc.add_argument("--accum", type=int, default=1)
     cc.add_argument("--resident-layers", type=int, default=0, help="also compile the resident device AdamW graphs")
+    cc.add_argument("--adamw-only", action="store_true",
+                    help="compile only the resident device AdamW graphs (no weights loaded, no training graphs)")
     cc.set_defaults(fn=cmd_compile)
 
     m = sub.add_parser("merge", help="어댑터를 base 가중치에 합친다")
diff --git a/npu_lora/llama.py b/npu_lora/llama.py
index 14e2279..6eb8556 100644
--- a/npu_lora/llama.py
+++ b/npu_lora/llama.py
@@ -365,6 +365,14 @@ def get(self, name, dtype=BF):
             self._open[fn] = safe_open(os.path.join(self.path, fn), "pt")
         return self._open[fn].get_tensor(name).to(dtype)
 
+    def shape(self, name):
+        """the tensor's shape from the safetensors header (reads no data)"""
+        from safetensors import safe_open
+        fn = self.where[name]
+        if fn not in self._open:
+            self._open[fn] = safe_open(os.path.join(self.path, fn), "pt")
+        return tuple(self._open[fn].get_slice(name).get_shape())
+
     def layer(self, i, c=None):
         keys = layer_weights(c) if c is not None else LAYER_WEIGHTS
         out = {}
diff --git a/npu_lora/optim_device.py b/npu_lora/optim_device.py
index 5397a05..27dff98 100644
--- a/npu_lora/optim_device.py
+++ b/npu_lora/optim_device.py
@@ -100,6 +100,12 @@ def adamw_scaled_fn(w, m, v, g, gscale, lr, b1, b2, eps, wd, bc1, bc2):
     return adamw_fn(w, m, v, g.float() * gscale, lr, b1, b2, eps, wd, bc1, bc2)
 
 
+def resident_shapes(W, c):
+    """The distinct 2-D weight shapes of a decoder layer: one ResidentAdamW graph each (1-D norms stay on the host).
+    Read from the safetensors header, so no weights are loaded (Llama-3.1-8B: 4 shapes)."""
+    return sorted({s for s in (W.shape(f"model.layers.0.{L.hf_name(k)}") for k in L.layer_weights(c)) if len(s) >= 2})
+
+
 class ResidentAdamW:
     """AdamW for weights whose fp32 master and moments stay on the device (trainer `resident_layers`).
 
diff --git a/npu_lora/trainer.py b/npu_lora/trainer.py
index 6e8e727..6e02a64 100644
--- a/npu_lora/trainer.py
+++ b/npu_lora/trainer.py
@@ -52,6 +52,7 @@ class TrainConfig:
     moe_replan_every: int = 1      #   re-choose them from routing counts every N steps
     resident_layers: int = 0       # full FT, per device: the first N layers of each device's range keep fp32
                                    #   master + AdamW state on the device and update there (no offload)
+    update_workers: int = 4        # full FT: host threads running the per-layer offload jobs (D2H, AdamW, H2D)
 
 
 class LoRATrainer:
@@ -184,7 +185,7 @@ def mk(ps):
             self.opts = [mk(per) for per in self.master]
             self._ready = [None] * c.n_layers           # 레이어별 업로드 future (다음 순전파가 기다린다)
             # 레이어마다 [내리기 → (clip 확정 대기) → AdamW → 올리기] 를 한 작업으로, 여러 레이어를 병렬로.
-            self._layer_pool = ThreadPoolExecutor(4, thread_name_prefix="npu-lora-layer")
+            self._layer_pool = ThreadPoolExecutor(self.tc.update_workers, thread_name_prefix="npu-lora-layer")
             self._step_ev = None
             self._bufs = [None] * c.n_layers            # 레이어별 host 버퍼 (bf16 기울기, fp32 기울기, bf16 업로드)
             self._rstate = {}                           # resident layer -> [(w32, m, v) on the device | None (1-D)]
@@ -253,11 +254,10 @@ def _compile(self):
             if self.tc.full and self.tc.accum > 1:
                 self.g_acc_d.append(cpb(f"{key}-bwd-acc", gm_acc, acc_in))
             if any(self._dev[i] == d for i in self._resident):
-                from .optim_device import ResidentAdamW
+                from .optim_device import ResidentAdamW, resident_shapes
                 ra = ResidentAdamW(b, log=log)
-                for x in w:
-                    if x.dim() >= 2:
-                        ra.compile(tuple(x.shape))
+                for shape in resident_shapes(self.W, c):
+                    ra.compile(shape)
                 self.adamw_d.append(ra)
             else:
                 self.adamw_d.append(None)
@@ -408,6 +408,7 @@ def _resident_update(self, i, dev_grads, k, clip, lr):
         from .optim_device import scalars
         be, ra = self.bes[self._dev[i]], self.adamw_d[self._dev[i]]
         t0 = time.time()
+        self.dev_W[i] = None                        # as in _layer_job: never hold old and new bf16 weights at once
         tc = self.tc
         sc = [be.to_device(x) for x in scalars(lr, tc.betas, 1e-8, tc.weight_decay, self.step_no)]
         gs = be.to_device(torch.tensor([[clip / k]], dtype=torch.float32))
@@ -536,6 +537,7 @@ def _train_step_full(self, batch):
         t_res = time.time()
         for i in sorted(self._resident):            # device AdamW: graphs run in this thread, card by card
             self.dev_W[i], self._last_times[i] = self._resident_update(i, acc[i], k, clip, lr)
+            acc[i] = sq[i] = None                   # free this layer's accumulated gradients now
         t_res = time.time() - t_res
         for i in range(c.n_layers):
             self._ready[i] = jobs[i]                # 다음 step 의 순전파가 레이어별로 기다린다
diff --git a/tests/test_cpu.py b/tests/test_cpu.py
index 11dfb08..8eed526 100644
--- a/tests/test_cpu.py
+++ b/tests/test_cpu.py
@@ -155,6 +155,38 @@ def run(backends, resident):
         assert float(dr.abs().max()) > 0 and _rel(dr, du) < 5e-2, _rel(dr, du)
 
 
+def test_resident_shapes_from_header():
+    """the resident AdamW graph shapes come from the safetensors header alone (compile --adamw-only)"""
+    try:
+        from transformers import LlamaConfig, LlamaForCausalLM
+    except ImportError:
+        return
+    from npu_lora.optim_device import resident_shapes
+    d = tempfile.mkdtemp()
+    cfg = LlamaConfig(hidden_size=64, intermediate_size=128, num_attention_heads=4, num_key_value_heads=2,
+                      num_hidden_layers=1, vocab_size=1000, tie_word_embeddings=False)
+    LlamaForCausalLM(cfg).to(BF).save_pretrained(d, safe_serialization=True)
+    W = L.Weights(d)
+    assert resident_shapes(W, L.LlamaConfig.from_hf(d)) == [(32, 64), (64, 64), (64, 128), (128, 64)]
+    assert W.shape("model.layers.0.mlp.down_proj.weight") == tuple(W.get("model.layers.0.mlp.down_proj.weight").shape)
+
+
+def test_visible_chip_picks_the_nth_assigned_card():
+    """a two-card pod on a node with npu0, npu1, npu3 assigned npu1,npu3 -> ordinals 1 and 2"""
+    from npu_lora import backends
+    real, env = backends.os.listdir, dict(os.environ)
+    backends.os.listdir = lambda d: ["npu0", "npu0pe0", "npu1", "npu1mgmt", "npu3", "npu3pe0-1"]
+    try:
+        os.environ["NPU_LORA_RNGD"] = "npu1,npu3"
+        assert [backends.visible_chip(0), backends.visible_chip(1)] == [1, 2]
+        os.environ["NPU_LORA_RNGD"] = "npu3"
+        assert backends.visible_chip() == 2
+    finally:
+        backends.os.listdir = real
+        os.environ.clear()
+        os.environ.update(env)
+
+
 def test_rope_mm_equals_rope():
     x = torch.randn(4, 16, 32)
     cos, sin = torch.randn(16, 32), torch.randn(16, 32)