ARCQuant/evaluate.sh at main · actypedef/ARCQuant · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
#!/bin/bash
# path to your model
MODEL=${1}

dir=$(pwd)
export CUDA_VISIBLE_DEVICE="0"

# wikitext2 ppl
python ${dir}/model/main.py ${MODEL}\
        --act_sort_metric max\
        --dataset wikitext2\
        --lm_eval_limit -1\
        --eval_ppl\
        --quant_type NVFP4\


# zero-shot
python ${dir}/model/main.py ${MODEL} \
        --act_sort_metric max\
        --dataset wikitext2\
        --tasks piqa,arc_challenge,boolq,hellaswag,winogrande,lambada_openai,arc_easy \
        --lm_eval_num_fewshot 0 \
        --lm_eval_limit -1\
        --quant_type NVFP4\


#5-shot mmlu
python ${dir}/model/main.py ${MODEL}\
        --act_sort_metric max\
        --dataset wikitext2\
        --tasks mmlu\
        --lm_eval_num_fewshot 5\
        --lm_eval_limit -1\
        --quant_type NVFP4\