Unfinished code

.gitignore

@@ -49,7 +49,7 @@ htmlcov/
 .tox/
 .coverage
 .coverage.*
-.cache
+.cache*
 nosetests.xml
 coverage.xml
 *.cover

cookbook/exp/embedding/ablation_all.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# RAG Ablation Suite — 串行运行全部 5 个消融实验
+# GPUs: 需要 8 卡（兼容所有配置的最大需求）
+#
+# 用法:
+#   COMPRESS_API_KEY=sk-xxx bash cookbook/exp/embedding/ablation_all.sh
+
+set -euo pipefail
+
+export COMPRESS_API_KEY="${COMPRESS_API_KEY:?Set COMPRESS_API_KEY}"
+
+SCRIPT="cookbook/exp/embedding/eval_gpqa_rag.py"
+N=500
+SEED=100
+SIM=0.6
+TOPK=1
+OUTDIR="./output/thinking_rag"
+DB_PATH="./output.oldemb/thinking_rag/lance.db"
+
+# echo "============================================================"
+# echo " Ablation 1/5: Direct (no RAG)"
+# echo "============================================================"
+# GEN_GPUS=8 python $SCRIPT \
+#   --mode direct --n $N --seed $SEED \
+#   --output $OUTDIR/ablation_direct_65k.jsonl
+
+# echo ""
+# echo "============================================================"
+# echo " Ablation 2/5: RAG + raw thinking (drop >24k, no condenser)"
+# echo "============================================================"
+# python $SCRIPT \
+#   --mode rag --n $N --seed $SEED \
+#   --db-path $DB_PATH \
+#   --sim-threshold $SIM --top-k $TOPK \
+#   --max-trace-len 24000 \
+#   --output $OUTDIR/ablation_rag_raw_24k.jsonl
+
+echo ""
+echo "============================================================"
+echo " Ablation 3/5: RAG + API condenser (qwen3.7-max)"
+echo "============================================================"
+python $SCRIPT \
+  --mode rag --n $N --seed $SEED \
+  --db-path $DB_PATH \
+  --sim-threshold $SIM --top-k $TOPK \
+  --condense \
+  --output $OUTDIR/ablation_rag_api_condenser_65k.jsonl
+
+echo ""
+echo "============================================================"
+echo " Ablation 4/5: RAG + local vLLM condenser (4B) + API fallback"
+echo "============================================================"
+EVAL_CONDENSER_GPUS=2 python $SCRIPT \
+  --mode rag --n $N --seed $SEED \
+  --db-path $DB_PATH \
+  --sim-threshold $SIM --top-k $TOPK \
+  --condense \
+  --output $OUTDIR/ablation_rag_local_condenser_65k.jsonl
+
+# echo ""
+# echo "============================================================"
+# echo " Ablation 5/5: RAG + cot_compressed (pre-compressed, no runtime condenser)"
+# echo "============================================================"
+# python $SCRIPT \
+#   --mode rag --n $N --seed $SEED \
+#   --db-path $DB_PATH \
+#   --sim-threshold $SIM --top-k $TOPK \
+#   --use-cot-compressed \
+#   --max-trace-len 4000 \
+#   --output $OUTDIR/ablation_rag_cot_compressed_65k.jsonl
+
+echo ""
+echo "============================================================"
+echo " All 5 ablations complete. Results:"
+echo "============================================================"
+for f in $OUTDIR/ablation_*_65k.jsonl $OUTDIR/ablation_*_24k.jsonl; do
+  n=$(wc -l < "$f")
+  correct=$(python -c "
+import json
+recs=[json.loads(l) for l in open('$f') if l.strip()]
+print(sum(1 for r in recs if r['is_correct']))
+")
+  echo "  $(basename $f): $correct/$n"
+done

cookbook/exp/embedding/ablation_direct.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Ablation 1: Direct (no RAG, no condenser)
+# GPUs: 4 (gen only)
+# Baseline — model solves problems without any retrieved context.
+
+set -euo pipefail
+
+python cookbook/exp/embedding/eval_gpqa_rag.py \
+  --mode direct \
+  --n 200 \
+  --seed 42 \
+  --output ./output/thinking_rag/ablation_direct.jsonl

cookbook/exp/embedding/ablation_rag_api_condenser.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Ablation 3: RAG + API condenser (qwen3.7-max)
+# GPUs: 6 (emb=2 + gen=4), condenser via API (no local vLLM)
+# Compresses thinking_raw with COMPRESS_SYSTEM + CONDENSE_EVAL_QUERY via API.
+
+set -euo pipefail
+
+export COMPRESS_API_KEY="${COMPRESS_API_KEY:?Set COMPRESS_API_KEY}"
+
+python cookbook/exp/embedding/eval_gpqa_rag.py \
+  --mode rag \
+  --n 200 \
+  --seed 42 \
+  --sim-threshold 0.6 \
+  --top-k 1 \
+  --condense \
+  --output ./output/thinking_rag/ablation_rag_api_condenser.jsonl

cookbook/exp/embedding/ablation_rag_local_condenser.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Ablation 4: RAG + local vLLM condenser (Qwen3.5-4B-CM-v2)
+# GPUs: 8 (emb=2 + gen=4 + condenser=2)
+# Local 4B condenser as primary, API as fallback.
+
+set -euo pipefail
+
+export EVAL_CONDENSER_GPUS=2
+export COMPRESS_API_KEY="${COMPRESS_API_KEY:?Set COMPRESS_API_KEY}"
+
+python cookbook/exp/embedding/eval_gpqa_rag.py \
+  --mode rag \
+  --n 200 \
+  --seed 42 \
+  --sim-threshold 0.6 \
+  --top-k 1 \
+  --condense \
+  --output ./output/thinking_rag/ablation_rag_local_condenser.jsonl

cookbook/exp/embedding/ablation_rag_raw.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Ablation 2: RAG + raw thinking (no condenser, truncated to max-trace-len)
+# GPUs: 6 (emb=2 + gen=4)
+# Uses thinking_raw directly, truncated to 4000 chars.
+
+set -euo pipefail
+
+python cookbook/exp/embedding/eval_gpqa_rag.py \
+  --mode rag \
+  --n 200 \
+  --seed 42 \
+  --sim-threshold 0.6 \
+  --top-k 1 \
+  --max-trace-len 4000 \
+  --output ./output/thinking_rag/ablation_rag_raw.jsonl