quickstart.py

"""
adaptive-eval quickstart
========================

1. Calibrate item parameters ONCE from a response matrix (models x items, 0/1)
   you already have — e.g. results of a few models you've already run in full.
2. Then evaluate any NEW model adaptively: administer only the most informative
   items until its ability is pinned.
"""

from adaptive_eval import calibrate, adaptive_test

# --- 1. Calibrate from history you already have ------------------------------
# rows = models you already ran on the full bank; columns = items; 1 = correct.
response_matrix = [
    [1, 1, 0, 1, 0, 0],
    [1, 1, 1, 1, 1, 0],
    [0, 0, 0, 1, 0, 0],
    [1, 1, 1, 1, 1, 1],
    [1, 0, 0, 1, 0, 0],
]
items = calibrate(response_matrix)          # -> [(a, b), ...] per item

# --- 2. Evaluate a NEW model adaptively --------------------------------------
# `answer_fn(i)` runs item i on your model and returns 1 (correct) or 0.
# Here we fake a strong model that gets easy items right.
def answer_fn(i: int) -> int:
    _, difficulty = items[i]
    return 1 if difficulty < 0.5 else 0

result = adaptive_test(items, answer_fn, se_threshold=0.4)
print(f"ability estimate: {result.theta:+.2f}  (SE {result.se:.2f})")
print(f"items administered: {result.n_items} of {len(items)}")
print(f"order: {result.administered}")