diff --git a/studies/study_016/anderson_holt_1997_information_cascades/README.md b/studies/study_016/anderson_holt_1997_information_cascades/README.md new file mode 100644 index 0000000..40e30ec --- /dev/null +++ b/studies/study_016/anderson_holt_1997_information_cascades/README.md @@ -0,0 +1,7 @@ +# Information Cascades in the Laboratory + +This HumanStudy-Hub package was reconstructed from a published paper. + +- Review `study.json` for the study overview and readiness status. +- Review `audit/missing_information.json` before running the study. +- Run `python task/adapter.py --smoke-test` to check the package entry point. diff --git a/studies/study_016/anderson_holt_1997_information_cascades/audit/missing_information.json b/studies/study_016/anderson_holt_1997_information_cascades/audit/missing_information.json new file mode 100644 index 0000000..a49fde7 --- /dev/null +++ b/studies/study_016/anderson_holt_1997_information_cascades/audit/missing_information.json @@ -0,0 +1,68 @@ +{ + "_description": "Authoritative researcher checklist of gaps, assumptions, and decisions still required before this package can be treated as a fully verified reconstruction of Anderson & Holt (1997). Nothing in this list has been silently resolved with invented content; each item names the concrete gap, its downstream impact, and a concrete suggested next step.", + "items": [ + { + "study": "study_2_asymmetric", + "field": "materials/materials.json: study_2_asymmetric.instructions_full_text", + "reason": "The paper prints a full oral-instructions script (Appendix) only for the symmetric design. For the asymmetric design it states sessions used 'the same procedure' with the urn contents in Table 4, but no separate instructions script is printed.", + "impact": "The asymmetric instructions_full_text in materials.json is a DERIVED reconstruction (symmetric script with ball counts substituted), not a verbatim transcript. If a researcher needs to present participants/agents with the historically exact wording used in the asymmetric sessions, this substitution should be verified or replaced.", + "suggested_action": "If the original asymmetric-design instructions script becomes available (e.g., from the authors or a data appendix), replace the derived text and update its evidence_label to verbatim." + }, + { + "study": "study_1_symmetric", + "field": "materials/materials.json: study_1_symmetric.public_draws_after_round4_condition", + "reason": "The paper explicitly describes the 2-public-draws-after-round-4 procedure only when analyzing representativeness bias for 'sessions 4 and 5'. It is not fully confirmed whether the third public-draw session used an identical insertion point/count in every period, or whether the insertion happened in only some periods.", + "impact": "task/adapter.py's public_draws_after_round4 condition assumes the same insertion rule (always after round 4, always exactly 2 balls, every period) for all 3 public-draw sessions. If the true procedure varied, cascade/representativeness statistics computed on this condition will not exactly reproduce the paper's Section V.A numbers.", + "suggested_action": "Confirm the exact procedure for the third public-draw session (and whether the 36-case representativeness sample in the paper drew from 2 or 3 sessions) via the authors' data appendix, then adjust task/adapter.py if it differs." + }, + { + "study": "both", + "field": "source/evidence.json: raw per-subject, per-round decision and private-draw dataset", + "reason": "The paper states 'a complete data appendix is available from the authors on request' (footnote 18) but does not print the underlying 540 (symmetric) + 540 (asymmetric) individual decisions/draws anywhere in the PDF. This job was not authorized to seek external sources, so this raw dataset could not be obtained.", + "impact": "evaluation/evaluation.py cannot refit the paper's Table 3 recursive logit error-rate model (Section IV), since that requires the original raw human data (or a comparably large, independent multi-session agent dataset) rather than the aggregated coefficients printed in the paper. evaluation.py reports this check as a structured not_ready result rather than fabricating a number.", + "suggested_action": "If the authors' raw data appendix is obtained (with explicit authorization to fetch an external/OSF/author-supplied URL), add a logit-fitting stage to evaluation.py and update this checklist." + }, + { + "study": "both", + "field": "source/evidence.json / Figure 1 (the physical urn setup diagram)", + "reason": "Figure 1 is referenced in the text but its image content is not present in the machine-readable PDF text extraction (pdftotext).", + "impact": "materials.json's urn_setup fields were reconstructed from two independent verbatim textual descriptions (Section I prose and the Appendix instructions) that happen to agree, rather than from the figure itself. Low impact since both textual sources agree exactly, but the original figure has not been visually verified.", + "suggested_action": "If a higher-resolution PDF or the original figure image becomes available, visually confirm Figure 1 matches the reconstructed urn_setup values." + }, + { + "study": "both", + "field": "source/paper_metadata.json: journal, volume, issue, page range, DOI", + "reason": "No running header/footer, journal name, volume/issue, page range, or DOI string appears anywhere in the 27 extracted PDF pages. The PDF's embedded metadata (checked via pdfinfo) contains no bibliographic fields, only an Acrobat Distiller creation timestamp.", + "impact": "study.json and paper_metadata.json cannot cite a full, verifiable bibliographic reference for the paper beyond title, authors, and a publication year inferred from the contributor-supplied filename.", + "suggested_action": "If the contributor supplies (or authorizes fetching) the journal citation or DOI, add it to source/paper_metadata.json with an appropriate evidence_label." + }, + { + "study": "both", + "field": "materials/materials.json: record_sheet_columns (decision sheet)", + "reason": "The paper describes the paper decision-record sheet's columns (0)-(9) only in prose within the oral instructions; no scanned image or facsimile of the actual sheet is included in the PDF.", + "impact": "A participant-facing digital record sheet built from materials.json is a faithful prose-derived reconstruction of column semantics, not a verified visual facsimile of the original form.", + "suggested_action": "Low priority; the textual description is sufficiently complete to drive the digital task/adapter.py. Revisit only if an exact visual replica of the paper form is required." + }, + { + "study": "study_1_symmetric", + "field": "task/adapter.py & materials.json: whether 'a'/'b' balls were shown to subjects as 'light'/'dark' in the asymmetric design", + "reason": "The paper confirms the light/dark labeling verbatim only for the symmetric-design Appendix instructions (footnote 13). It does not restate this labeling convention for the six asymmetric sessions, though it is highly likely the same convention was reused given the shared experimental apparatus.", + "impact": "materials.json's derived asymmetric instructions assume the same light/dark labeling. If the true asymmetric-session labeling differed, participant-facing text would need correction, though the underlying probabilities/mechanics (Table 4) are unaffected.", + "suggested_action": "Confirm via the authors' data appendix or original session materials if available; otherwise treat the light/dark assumption as a reasonable, clearly-flagged researcher decision." + }, + { + "study": "both", + "field": "evaluation/evaluation.py: Section IV logit econometric error-rate replication", + "reason": "Implementing a faithful recursive per-round logit MLE (as in Table 3) requires many independent repeated observations per round and the original human raw dataset (see the data-appendix item above); it is not a well-posed check to run against a handful of agent-generated sessions.", + "impact": "evaluation.py's logit_error_analysis() intentionally returns a structured not_ready result rather than a fitted beta, so users should not expect a Table-3-style output from this package without further work.", + "suggested_action": "Revisit once either (a) the original raw dataset is obtained with authorization, or (b) the task is run at large scale (many dozens of independent sessions per condition) so an agent-side logit fit would be statistically meaningful in its own right." + }, + { + "study": "both", + "field": "task/task.json & task/adapter.py: practice periods", + "reason": "The paper describes unpaid, non-private practice periods at the start of each session (continued until each urn had been selected at least once) used to familiarize subjects with the mechanics. These carry no decisions or payment and are not needed for the paper's statistical analyses.", + "impact": "task/adapter.py omits practice periods entirely; the agent's first call in a simulated session is period 1 of the paid phase, with the practice-period narration available only as background text in materials.json.", + "suggested_action": "None required for faithful statistical replication; add only if a researcher specifically wants to reproduce the onboarding experience for a human-facing (not agent) run of this task." + } + ] +} diff --git a/studies/study_016/anderson_holt_1997_information_cascades/evaluation/__pycache__/evaluation.cpython-311.pyc b/studies/study_016/anderson_holt_1997_information_cascades/evaluation/__pycache__/evaluation.cpython-311.pyc new file mode 100644 index 0000000..4a990db Binary files /dev/null and b/studies/study_016/anderson_holt_1997_information_cascades/evaluation/__pycache__/evaluation.cpython-311.pyc differ diff --git a/studies/study_016/anderson_holt_1997_information_cascades/evaluation/evaluation.py b/studies/study_016/anderson_holt_1997_information_cascades/evaluation/evaluation.py new file mode 100644 index 0000000..f93184f --- /dev/null +++ b/studies/study_016/anderson_holt_1997_information_cascades/evaluation/evaluation.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +""" +Evaluation for Anderson & Holt (1997), "Information Cascades in the Laboratory". + +Consumes session logs produced by task/adapter.py's run_session() (schema +documented in task/task.json under "outputs_consumed_by_evaluation") and +computes the paper's core behavioral measures: + + 1. Cascade formation / reverse-cascade classification per period (Section + III, Table 7), using the paper's own "Bayes distribution" method: later + rounds' relevant signals are inferred from EARLIER PUBLIC DECISIONS only + (never from another round's hidden private draw), exactly as a real + subsequent decision-maker would have to do. + 2. Actual / private-information / optimal / random expected-payoff + efficiency (Section III, equations for "actual efficiency" and "private + information efficiency"), and counting efficiency for the asymmetric + design (Section V.B). + 3. Status-quo bias check (Section V.A) and representativeness bias check + (Section V.A, public-draw condition only). + 4. Counting-heuristic-vs-Bayes' rule check (Section V.B, asymmetric design + only). + +Each check reports both the agent's own numbers and, for descriptive +comparison only, the corresponding human benchmark transcribed verbatim from +the paper (source/evidence.json). This module does NOT attempt to reproduce +the paper's Table 3 round-by-round logit error-rate estimates (see +`logit_error_analysis()` below), because that requires many repeated +independent replications per round and an MLE fit the paper performed with a +proprietary GAUSS routine on human subjects' raw per-decision data, which is +not published in the paper (see audit/missing_information.json) and would not +be statistically meaningful on a handful of agent-generated sessions. That +check returns a structured "not_ready" result instead of a fabricated number. + +No network access is used or required. +""" +import json +import sys +from collections import defaultdict + +# --------------------------------------------------------------------------- +# Design parameters (must match task/adapter.py's CONDITIONS) +# --------------------------------------------------------------------------- + +DESIGNS = { + "study_1_symmetric": { + "p_a_given_A": 2 / 3, "p_b_given_A": 1 / 3, + "p_a_given_B": 1 / 3, "p_b_given_B": 2 / 3, + }, + "study_2_asymmetric": { + "p_a_given_A": 6 / 7, "p_b_given_A": 1 / 7, + "p_a_given_B": 5 / 7, "p_b_given_B": 2 / 7, + }, +} + +PAYOFF_CORRECT_USD = 2 +PAYOFF_RANDOM_USD = 1.0 # 0.5 * PAYOFF_CORRECT_USD, expected payoff of a coin flip + +_SIGNAL_OF = {"light": "a", "dark": "b"} + + +def posterior_A(design, n_a, n_b): + """Bayesian posterior probability of urn A given n_a relevant a-signals and n_b relevant b-signals.""" + lr_a = design["p_a_given_A"] / design["p_a_given_B"] + lr_b = design["p_b_given_A"] / design["p_b_given_B"] + odds = (lr_a ** n_a) * (lr_b ** n_b) # prior odds are 1:1 (P(A)=P(B)=0.5), so posterior odds = likelihood ratio + return odds / (odds + 1.0) + + +def is_cascade_state(design, n_a, n_b): + """True iff the Bayes-optimal decision for a hypothetical next round is the same + whether that round's own signal turns out to be 'a' or 'b' -- i.e. a cascade is + already in force given tally (n_a, n_b).""" + p_if_a = posterior_A(design, n_a + 1, n_b) + p_if_b = posterior_A(design, n_a, n_b + 1) + dec_if_a = "A" if p_if_a >= 0.5 else "B" + dec_if_b = "A" if p_if_b >= 0.5 else "B" + if dec_if_a == dec_if_b: + return True, dec_if_a + return False, None + + +def _bayes_optimal_decision(design, n_a, n_b, own_signal): + n_a2, n_b2 = (n_a + 1, n_b) if own_signal == "a" else (n_a, n_b + 1) + p = posterior_A(design, n_a2, n_b2) + if p > 0.5: + return "A", p + if p < 0.5: + return "B", p + return ("A" if own_signal == "a" else "B"), p # tie-break: footnote 10, match own signal + + +def analyze_period(study, period): + """Replays one period's rounds, inferring the 'Bayes distribution' tally from + public decisions only (Section III methodology), and returns a per-round and + per-period analysis.""" + design = DESIGNS[study] + n_a, n_b = 0, 0 + cascade_active, cascade_dir = False, None + round_analyses = [] + + for r in period["rounds"]: + own_signal = _SIGNAL_OF[r["private_draw"]] + pre_cascade_active, pre_cascade_dir = is_cascade_state(design, n_a, n_b) + optimal_decision, posterior_for_A = _bayes_optimal_decision(design, n_a, n_b, own_signal) + actual_decision = r["decision"] + actual_posterior = posterior_for_A if actual_decision == "A" else (1 - posterior_for_A) + signal_only_decision = "A" if own_signal == "a" else "B" + + round_analyses.append({ + "round": r["round"], + "pre_round_relevant_tally": {"a": n_a, "b": n_b}, + "cascade_in_force_pre_round": pre_cascade_active, + "own_signal": own_signal, + "optimal_decision": optimal_decision, + "actual_decision": actual_decision, + "matches_optimal": actual_decision == optimal_decision, + "matches_private_signal_only": actual_decision == signal_only_decision, + "bayes_inconsistent_with_private_info": pre_cascade_active and optimal_decision != signal_only_decision, + "expected_payoff_actual_usd": actual_posterior * PAYOFF_CORRECT_USD, + "expected_payoff_optimal_usd": max(posterior_for_A, 1 - posterior_for_A) * PAYOFF_CORRECT_USD, + "expected_payoff_private_info_only_usd": ( + max(posterior_A(design, 1, 0), 1 - posterior_A(design, 1, 0)) * PAYOFF_CORRECT_USD + if own_signal == "a" else + max(posterior_A(design, 0, 1), 1 - posterior_A(design, 0, 1)) * PAYOFF_CORRECT_USD + ), + }) + + # Update the inferred relevant tally exactly as a subsequent rational + # decision-maker would, using ONLY the announced decision. + if cascade_active: + if actual_decision != cascade_dir: + # deviation: reveals a signal favoring the non-cascade urn + if cascade_dir == "A": + n_b += 1 + else: + n_a += 1 + cascade_active, cascade_dir = is_cascade_state(design, n_a, n_b) + # else: decision matches the cascade, adds no new information + else: + inferred_signal = "a" if actual_decision == "A" else "b" + if inferred_signal == "a": + n_a += 1 + else: + n_b += 1 + cascade_active, cascade_dir = is_cascade_state(design, n_a, n_b) + + cascade_possible = any(ra["cascade_in_force_pre_round"] for ra in round_analyses) + cascade_formed = cascade_possible and all( + ra["matches_optimal"] for ra in round_analyses if ra["cascade_in_force_pre_round"] + ) + reverse_cascade = False + if cascade_formed: + cascade_rounds = [ra for ra in round_analyses if ra["cascade_in_force_pre_round"]] + reverse_cascade = cascade_rounds[0]["optimal_decision"] != period["true_urn"] + + return { + "period": period["period"], + "true_urn": period["true_urn"], + "cascade_possible": cascade_possible, + "cascade_formed": cascade_formed, + "reverse_cascade": reverse_cascade, + "rounds": round_analyses, + } + + +# --------------------------------------------------------------------------- +# Aggregate checks +# --------------------------------------------------------------------------- + +def cascade_summary(study, period_analyses): + normal_formed = sum(1 for p in period_analyses if p["cascade_formed"] and not p["reverse_cascade"]) + reverse_formed = sum(1 for p in period_analyses if p["cascade_formed"] and p["reverse_cascade"]) + possible_not_formed = sum(1 for p in period_analyses if p["cascade_possible"] and not p["cascade_formed"]) + n_possible = sum(1 for p in period_analyses if p["cascade_possible"]) + + benchmark = { + "study_1_symmetric": {"cascades_formed": 41, "periods_possible": 56, "text": "41 of 56 possible periods (Section III); Table 7: 28 normal + 13 reverse formed, 10 normal + 5 reverse possible-but-not-formed."}, + "study_2_asymmetric": {"cascades_formed": 46, "periods_possible": 66, "text": "46 of 66 possible periods (Section V.B); Table 7: 28 normal + 18 reverse formed, 12 normal + 8 reverse possible-but-not-formed."}, + }[study] + + return { + "n_periods_analyzed": len(period_analyses), + "n_periods_cascade_possible": n_possible, + "n_cascades_formed_normal": normal_formed, + "n_cascades_formed_reverse": reverse_formed, + "n_cascade_possible_but_not_formed": possible_not_formed, + "cascade_formation_rate": (normal_formed + reverse_formed) / n_possible if n_possible else None, + "human_benchmark": benchmark, + } + + +def efficiency_summary(study, period_analyses): + pi_actual = pi_optimal = pi_random_total = pi_private = 0.0 + n_rounds = 0 + for p in period_analyses: + for ra in p["rounds"]: + pi_actual += ra["expected_payoff_actual_usd"] + pi_optimal += ra["expected_payoff_optimal_usd"] + pi_private += ra["expected_payoff_private_info_only_usd"] + pi_random_total += PAYOFF_RANDOM_USD + n_rounds += 1 + + denom = pi_optimal - pi_random_total + actual_efficiency = 100 * (pi_actual - pi_random_total) / denom if denom else None + private_info_efficiency = 100 * (pi_private - pi_random_total) / denom if denom else None + + benchmark = { + "study_1_symmetric": {"actual_efficiency_pct": 91.4, "private_information_efficiency_pct": 72.1, + "text": "Averaged over all 36 symmetric-design subjects (Section III)."}, + "study_2_asymmetric": {"actual_efficiency_pct": 67.6, "private_information_efficiency_pct": 45.2, + "text": "Averaged over all 36 asymmetric-design subjects (Section V.B)."}, + }[study] + + return { + "n_rounds": n_rounds, + "sum_expected_payoff_actual_usd": round(pi_actual, 4), + "sum_expected_payoff_optimal_usd": round(pi_optimal, 4), + "sum_expected_payoff_private_info_only_usd": round(pi_private, 4), + "sum_expected_payoff_random_usd": round(pi_random_total, 4), + "actual_efficiency_pct": round(actual_efficiency, 2) if actual_efficiency is not None else None, + "private_information_efficiency_pct": round(private_info_efficiency, 2) if private_info_efficiency is not None else None, + "human_benchmark": benchmark, + } + + +def status_quo_bias_check(period_analyses): + """Section V.A: instances where the pre-round Bayes-distribution posterior for A + was exactly 1/2 and the subject's own signal did NOT match the immediately + preceding announced decision. Checks whether the agent followed its own signal + (rational under the paper's null finding) or the preceding decision (status-quo bias).""" + n_instances = 0 + n_followed_own_signal = 0 + for p in period_analyses: + prev_decision = None + for ra in p["rounds"]: + n_a, n_b = ra["pre_round_relevant_tally"]["a"], ra["pre_round_relevant_tally"]["b"] + is_tie = n_a == n_b # posterior 1/2 for A pre-round tally, symmetric design property + own_label = "A" if ra["own_signal"] == "a" else "B" + if prev_decision is not None and is_tie and own_label != prev_decision: + n_instances += 1 + if ra["actual_decision"] == own_label: + n_followed_own_signal += 1 + prev_decision = ra["actual_decision"] + return { + "n_qualifying_instances": n_instances, + "n_followed_own_private_signal_rather_than_status_quo": n_followed_own_signal, + "share_followed_own_signal": (n_followed_own_signal / n_instances) if n_instances else None, + "human_benchmark": { + "n_qualifying_instances": 68, "n_followed_own_signal": 57, + "share": round(57 / 68, 4), + "text": "57 of 68 qualifying instances, symmetric design, all 6 sessions (Section V.A). Paper concludes there is no detectable status-quo bias beyond Bayesian updating.", + }, + } + + +def counting_heuristic_check(study, period_analyses): + """Section V.B (asymmetric design only): among rounds where the plain + signal-count rule and Bayes' rule prescribe different decisions, does the agent + follow Bayes' rule (correct) or the counting heuristic?""" + if study != "study_2_asymmetric": + return {"applicable": False, "note": "Counting and Bayes' rule always coincide in the symmetric design (paper, Section V.B)."} + + n_disagree = 0 + n_bayes_correct_when_disagree = 0 + for p in period_analyses: + for ra in p["rounds"]: + n_a, n_b = ra["pre_round_relevant_tally"]["a"], ra["pre_round_relevant_tally"]["b"] + own = ra["own_signal"] + total_a = n_a + (1 if own == "a" else 0) + total_b = n_b + (1 if own == "b" else 0) + counting_decision = "A" if total_a > total_b else ("B" if total_b > total_a else ("A" if own == "a" else "B")) + if counting_decision != ra["optimal_decision"]: + n_disagree += 1 + if ra["actual_decision"] == ra["optimal_decision"]: + n_bayes_correct_when_disagree += 1 + return { + "applicable": True, + "n_rounds_bayes_and_counting_disagree": n_disagree, + "n_bayes_correct_decisions_when_disagree": n_bayes_correct_when_disagree, + "share_bayes_correct_when_disagree": (n_bayes_correct_when_disagree / n_disagree) if n_disagree else None, + "human_benchmark": { + "n_disagree": 82, "n_bayes_correct": 41, "share": 0.5, + "text": "41 of 82 cases (50%) where Bayes' rule and counting disagree, subjects made the Bayes-correct decision (Section V.B).", + }, + } + + +def logit_error_analysis(): + """Table 3's round-by-round logit beta estimates were fit on the ORIGINAL HUMAN + subjects' raw per-decision data using a recursive Newton-Raphson MLE in GAUSS. + That raw per-subject dataset is not published in the paper (it states a 'complete + data appendix is available from the authors on request', footnote 18) and is not + included in this package (see audit/missing_information.json). Refitting an + equivalent model on a handful of agent-generated sessions would not be a faithful + replication of Table 3 and could produce a misleadingly precise-looking number + from a tiny, non-independent sample. This check is therefore reported as not_ready.""" + return { + "status": "not_ready", + "check": "round_by_round_logit_error_rate_estimation (paper Table 3, Section IV)", + "missing_requirement": ( + "The original per-subject, per-round raw decision/draw dataset for the six " + "symmetric-design sessions (540 decisions), used by the authors to fit a " + "recursive logit model in GAUSS. The paper reports only the fitted " + "coefficients (Table 3), not the underlying per-decision data, and states " + "the full data appendix is available from the authors on request -- which " + "this job was not authorized to seek out (no external URL was supplied)." + ), + "would_require": "A researcher-supplied raw dataset (or a much larger bank of independent agent-run sessions, e.g. dozens per round) plus a maximum-likelihood logit fitting routine (e.g. via statsmodels/scipy), and a decision about whether to replicate the paper's exact recursive round-by-round procedure or a simpler pooled logit.", + } + + +# --------------------------------------------------------------------------- +# Top-level entry point +# --------------------------------------------------------------------------- + +def evaluate(session_logs): + """session_logs: list of dicts matching task/task.json's session_log_schema.""" + by_study = defaultdict(list) + for session in session_logs: + by_study[session["study"]].append(session) + + report = {} + for study, sessions in by_study.items(): + all_period_analyses = [] + for session in sessions: + all_period_analyses.extend(analyze_period(study, p) for p in session["periods"]) + + study_report = { + "n_sessions": len(sessions), + "cascade_summary": cascade_summary(study, all_period_analyses), + "efficiency_summary": efficiency_summary(study, all_period_analyses), + "counting_heuristic_check": counting_heuristic_check(study, all_period_analyses), + } + if study == "study_1_symmetric": + study_report["status_quo_bias_check"] = status_quo_bias_check(all_period_analyses) + report[study] = study_report + + report["logit_error_analysis"] = logit_error_analysis() + return report + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] == "--smoke-test": + # Exercise the evaluator end-to-end on adapter.py's own smoke-test sessions, + # with no network access and no external agent required. + import os + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "task")) + import adapter # noqa: E402 + + sessions = [] + for (study, condition) in adapter.CONDITIONS: + sessions.append(adapter.run_session( + adapter._baseline_follow_private_signal, + study, condition, n_periods=5, seed=42, + )) + print(json.dumps(evaluate(sessions), indent=2)) + return 0 + + data = json.load(sys.stdin) + print(json.dumps(evaluate(data), indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/studies/study_016/anderson_holt_1997_information_cascades/materials/materials.json b/studies/study_016/anderson_holt_1997_information_cascades/materials/materials.json new file mode 100644 index 0000000..3e06fd6 --- /dev/null +++ b/studies/study_016/anderson_holt_1997_information_cascades/materials/materials.json @@ -0,0 +1,121 @@ +{ + "_description": "Participant-visible materials for the two studies extracted from this paper (study_1_symmetric, study_2_asymmetric). Every item carries an evidence_label and a source pointer into source/evidence.json / input/paper.pdf. Studies share the same procedural mechanics (oral instructions read aloud, then a paper record sheet updated by hand each round); they differ only in urn contents and (for study_1's public-draw condition) two extra publicly-revealed draws inserted after round 4.", + + "study_1_symmetric": { + "label": "Symmetric-design cascade experiment (6 sessions: 3 baseline / private-draws-only, 3 with public draws inserted after round 4)", + "conditions": ["baseline_private_draws_only", "public_draws_after_round4"], + + "urn_setup": { + "evidence_label": "verbatim", + "source": "ev-design-1, ev-mat-4 in source/evidence.json", + "urn_A": {"selected_if_die_roll_in": [1, 2, 3], "contents": "2 light (a) balls, 1 dark (b) ball"}, + "urn_B": {"selected_if_die_roll_in": [4, 5, 6], "contents": "1 light (a) ball, 2 dark (b) balls"}, + "note": "'Light'/'Dark' are the labels actually shown to subjects during the experiment; 'a'/'b' are the paper's analytical labels for the same two ball colors." + }, + + "instructions_full_text": { + "evidence_label": "verbatim", + "source": "ev-mat-1 in source/evidence.json; input/paper.pdf pp.23-25 (Appendix)", + "audience": "read aloud by the experimenter to all 6 decision-making subjects plus the monitor before the session begins", + "text": "This is an experiment in the economics of decision making. Various agencies have provided funds for the experiment. Your earnings will depend partly on your decisions and partly on chance. If you are careful and make good decisions, you may earn a considerable amount of money, which will be paid to you, privately, in cash, at the end of the experiment. At this time, we will give you $5. This payment is to compensate you for showing up today.\n\nBefore beginning, we will choose one of you to assist us in the experiment today. This person, who will be called the monitor, will help us by throwing dice and drawing colored balls from a container. The monitor will also observe procedures to insure that the instructions are followed. The monitor will be paid $15 at the end of the experiment in addition to the $5 already paid. We will now assign each of you a number, and we will throw a multi-sided die to select the monitor.\n\nIn this experiment, you will be asked to predict from which randomly chosen urn a ball was drawn. We will begin by rolling a 6 sided die. If the roll of the die yields a 1, 2, or 3, we will draw from urn A which contains 2 light balls and 1 dark ball. If the roll of the die yields a 4, 5, or 6, we will draw from urn B which contains 1 light ball and 2 dark balls. Therefore, it is equally likely that either urn will be selected.\n\nUrn A (used if the die is 1, 2, or 3): 2 Light Balls, 1 Dark Ball.\nUrn B (used if the die is 4, 5, or 6): 1 Light Ball, 2 Dark Balls.\n\nOnce an urn is determined by the roll of the die we will empty the contents of that urn into a container. (The container is always the same, regardless of which urn is being used.) Then we will come around to each of you and draw a ball from the container. The result of this draw will be your private information and should not be shared with other participants. After each draw, we will return the ball to the container before making the next private draw. Each person will have one private draw, with the ball being replaced after each draw.\n\nAfter each person has seen his or her own draw we will ask them to record the letter of the urn (A or B) that they think is more likely to have been used. When the first person approached has indicated a letter, we will announce that letter. After announcing the first person's decision, we will approach the second person and ask this person to record a letter (A or B), which will then be announced. This process will be repeated until all remaining people have made decisions. Finally, the monitor will inform everyone of the urn that was actually used. Everyone who correctly recorded the letter of the urn used earns $2. All others earn nothing.\n\nThe experiment will consist of 15 periods. The results for each period are recorded on a separate row on the decision sheet that follows. The period numbers are listed on the left side of each row. Next to the period number is a blank that should be used to record the draw (Light or Dark) that you see when we come to your desk. Write L (for Light) or D (for Dark) in column (0) at the time the draw is made. The columns numbered (1) through (6) should be used to record the decisions as they are announced. When you are asked to record the letter of an urn, you will be able to see the decisions, if any, that have been made previously by other participants. Write your decision in the column, (1) through (6), that corresponds to the order in which you are approached, and circle your decision to distinguish it from others' decisions. When all six participants have made their choices, the monitor will announce the letter of the urn that was actually used. Record this letter in column (7). If your (circled) decision matches the letter of the urn used, record earnings of $2 in column (8). Otherwise, record earnings of zero for this period. You should keep track of your cumulative earnings in column (9).\n\nAt this time, we will draw a colored marble for each participant; this color will serve as your identification during the experiment. Please write this color in the blank indicated at the top of your decision sheet. In each period, the order in which decisions are made will be determined by drawing these same colored marbles in sequence.\n\nBefore we begin the periods that determine your earnings, we will go through several practice periods. In these practice periods, the monitor will throw the die that determines which urn will be used, and you will each see a draw from that urn. However, unlike in the periods that determine your earnings, you will observe the throw of the die, your draw will not be private, and you will not be asked to make a decision in these practice periods.\n\n[Practice period walkthrough, repeated until each urn has been selected at least once:] At this time the monitor will throw the die that determines which urn is to be used. Remember that urn A is used if the throw is 1, 2, or 3, and urn B is used if the throw is 4, 5, or 6. Now we will draw a colored marble to determine who will see the first draw. The color is _____. We will bring the container to the desk of the person assigned this color and we will draw a ball for this person to see. If this were not a practice period, this person would record the color of this ball (L or D) in column (0), make a decision (A or B), enter it in column (1), and circle it. Then, everyone else would record this decision in column (1), but would not circle this decision since it is not your own. Now we will draw a colored marble to determine who will see the next draw. The color is _____. We will now draw a ball for this person to see. If this were not a practice period, this person would record the color of this ball (L or D) in column (0), make a decision (A or B), enter it in the appropriate column, and circle it. Then, everyone else would record this decision in the appropriate column.\n\nAre there any questions before we begin the periods that determine your earnings? Please do not talk with anyone during the experiment. We will insist that everyone remain silent until the end of the last period. If we observe you communicating with anyone else during the experiment we will pay you your cumulative earnings at that point and ask you to leave without completing the experiment.\n\n[Paid-period walkthrough for round 1 and round 2 of the first paid period, repeated in structure for every subsequent round/period:] At this time the monitor will throw the die that determines which urn is to be used. Remember that urn A is used if the throw is 1, 2, or 3, and urn B is used if the throw is 4, 5, or 6. Now we will draw a colored marble to determine who makes the first decision. The color is _____. We will bring the container to the desk of the person assigned this color and we will draw a ball for this person to see. This person should record the color of this ball (L or D) in column (0), make a decision (A or B), enter it in column (1), and circle it. The first decision is _____. Everyone else should now record this decision in column (1), but do not circle this decision since it is not your own. Now we will draw a colored marble to determine who makes the next decision. The color is _____. We will now draw a ball for this person to see. Record the color of this ball (L or D) in column (0), make a decision (A or B), enter it in the appropriate column, and circle it. This decision is _____. Everyone else should now record this decision in the appropriate column." + }, + + "record_sheet_columns": { + "evidence_label": "reported", + "source": "ev-mat-3 in source/evidence.json (described in prose only; no scanned form exists in the PDF)", + "columns": [ + {"col": "0", "meaning": "own private draw this period: L (light) or D (dark)"}, + {"col": "1-6", "meaning": "the urn decision (A or B) announced by whichever subject was approached in that ordinal position this period; the subject's own decision is circled"}, + {"col": "7", "meaning": "the true urn used this period, revealed by the monitor at period end"}, + {"col": "8", "meaning": "earnings this period: $2 if column (7) matches the subject's circled decision, else $0"}, + {"col": "9", "meaning": "running cumulative earnings"} + ] + }, + + "public_draws_after_round4_condition": { + "evidence_label": "reported", + "source": "ev-design-10 in source/evidence.json", + "description": "In this condition, after the 4th subject in the random sequence has made and announced a decision, two additional balls are drawn from the same container and their colors (light/dark) are announced to all remaining subjects (publicly, not attributed to any individual's private draw). The 5th and 6th subjects in the sequence then decide with a total effective sample of 3 draws (their own private draw plus the 2 newly-revealed public draws) in addition to the public decisions already announced by subjects 1-4.", + "researcher_decision_flag": "The paper only explicitly walks through this for 'sessions 4 and 5' when analyzing representativeness bias; whether all 3 public-draw sessions used this exact insertion point (always after round 4, always exactly 2 balls) in every period is not fully confirmed in the text. See audit/missing_information.json." + }, + + "posterior_lookup_table_1": { + "evidence_label": "verbatim", + "source": "ev-model-3 in source/evidence.json; Table I, input/paper.pdf p.5", + "description": "Posterior probability of event A given the number of relevant a-signals (rows) and b-signals (columns), symmetric design.", + "table": { + "rows_are": "number of a signals (0-3)", + "cols_are": "number of b signals (0-3)", + "values": [ + [0.5, 0.333, 0.2, 0.111], + [0.667, 0.5, 0.333, 0.2], + [0.8, 0.667, 0.5, 0.333], + [0.889, 0.8, 0.667, 0.5] + ] + } + } + }, + + "study_2_asymmetric": { + "label": "Asymmetric-design cascade experiment (6 sessions), testing the counting heuristic against Bayes' rule", + "conditions": ["asymmetric_urns"], + + "urn_setup": { + "evidence_label": "verbatim", + "source": "ev-design-9 in source/evidence.json; Table 4, input/paper.pdf p.18", + "urn_A": {"selected_if_die_roll_in": [1, 2, 3], "contents": "6 a balls, 1 b ball"}, + "urn_B": {"selected_if_die_roll_in": [4, 5, 6], "contents": "5 a balls, 2 b balls"}, + "note": "The paper does not restate whether 'a'/'b' were again shown to subjects as 'light'/'dark' in this design (that labeling is only confirmed verbatim for the symmetric design's Appendix instructions). materials below assume the same light/dark labeling for consistency, and this assumption is flagged in audit/missing_information.json." + }, + + "instructions_full_text": { + "evidence_label": "derived", + "source": "Derived from study_1_symmetric.instructions_full_text (ev-mat-1) by substituting the urn contents from Table 4 (ev-design-9); see ev-mat-2 in source/evidence.json.", + "derivation": "The paper states the asymmetric sessions used 'the same procedure' (input/paper.pdf p.18) but prints no separate appendix script. This text takes the verbatim symmetric-design script and replaces every mention of urn contents ('2 light balls and 1 dark ball' / '1 light ball and 2 dark balls') with the asymmetric counts ('6 light balls and 1 dark ball' / '5 light balls and 2 dark balls'). No other wording is changed. This is a plausible reconstruction, NOT a verbatim transcript, and is marked accordingly.", + "text": "This is an experiment in the economics of decision making. Various agencies have provided funds for the experiment. Your earnings will depend partly on your decisions and partly on chance. If you are careful and make good decisions, you may earn a considerable amount of money, which will be paid to you, privately, in cash, at the end of the experiment. At this time, we will give you $5. This payment is to compensate you for showing up today.\n\nBefore beginning, we will choose one of you to assist us in the experiment today. This person, who will be called the monitor, will help us by throwing dice and drawing colored balls from a container. The monitor will also observe procedures to insure that the instructions are followed. The monitor will be paid $15 at the end of the experiment in addition to the $5 already paid. We will now assign each of you a number, and we will throw a multi-sided die to select the monitor.\n\nIn this experiment, you will be asked to predict from which randomly chosen urn a ball was drawn. We will begin by rolling a 6 sided die. If the roll of the die yields a 1, 2, or 3, we will draw from urn A which contains 6 light balls and 1 dark ball. If the roll of the die yields a 4, 5, or 6, we will draw from urn B which contains 5 light balls and 2 dark balls. Therefore, it is equally likely that either urn will be selected.\n\nUrn A (used if the die is 1, 2, or 3): 6 Light Balls, 1 Dark Ball.\nUrn B (used if the die is 4, 5, or 6): 5 Light Balls, 2 Dark Balls.\n\n[Remaining instructions are identical in structure to study_1_symmetric.instructions_full_text from 'Once an urn is determined by the roll of the die...' onward, including the private-draw procedure, decision/announcement procedure, record sheet columns (0)-(9), colored-marble identification, practice periods, and paid-period walkthroughs. See study_1_symmetric.instructions_full_text for the full wording of those unchanged sections.]" + }, + + "record_sheet_columns": { + "evidence_label": "derived", + "source": "Same structure as study_1_symmetric.record_sheet_columns; no separate asymmetric-design record sheet is described in the paper.", + "columns": "identical to study_1_symmetric.record_sheet_columns" + }, + + "posterior_lookup_table_5": { + "evidence_label": "verbatim", + "source": "ev-model-5b in source/evidence.json; Table 5, input/paper.pdf p.19", + "description": "Bayesian posterior probability of urn A given the number of relevant a-signals (rows, 0-6) and b-signals (columns, 0-6), asymmetric design. Blank cells indicate combinations not tabulated in the source (more signals than the 6-round sequence could produce in the direction shown).", + "table": { + "rows_are": "number of a signals (0-6)", + "cols_are": "number of b signals (0-6)", + "values": [ + [0.50, 0.33, 0.20, 0.11, 0.06, 0.03, 0.02], + [0.55, 0.38, 0.23, 0.13, 0.07, 0.04, null], + [0.59, 0.42, 0.27, 0.15, 0.08, null, null], + [0.63, 0.46, 0.30, 0.18, null, null, null], + [0.68, 0.51, 0.34, null, null, null, null], + [0.71, 0.55, null, null, null, null, null], + [0.75, null, null, null, null, null, null] + ] + }, + "counting_heuristic_definition": { + "evidence_label": "reported", + "source": "input/paper.pdf p.18-19, paper.txt lines 671-694, 728-733", + "text": "The 'counting heuristic' predicts urn A whenever the number of relevant a-signals exceeds the number of relevant b-signals (i.e., it treats every signal as equally informative, ignoring that a b-signal is more diagnostic than an a-signal in this asymmetric design). It disagrees with the Bayes-optimal prediction exactly in the 4 shaded cells of Table 5, where more a-signals than b-signals still yields a Bayesian posterior for A below 0.5." + } + } + }, + + "shared_procedural_notes": { + "payment": { + "evidence_label": "verbatim", + "source": "ev-design-5 in source/evidence.json", + "text": "$5 show-up fee for every subject; $2 for a correct urn prediction in a given period, $0 otherwise; monitor receives an additional $15; average total earnings were about $20 per subject." + }, + "confidentiality_and_conduct_rules": { + "evidence_label": "verbatim", + "source": "ev-design-8 in source/evidence.json", + "text": "Subjects must not share their private draw with anyone. Subjects must remain silent throughout the paid periods; anyone observed communicating is paid their cumulative earnings at that point and asked to leave without finishing the experiment." + } + } +} diff --git a/studies/study_016/anderson_holt_1997_information_cascades/source/evidence.json b/studies/study_016/anderson_holt_1997_information_cascades/source/evidence.json new file mode 100644 index 0000000..a875ee0 --- /dev/null +++ b/studies/study_016/anderson_holt_1997_information_cascades/source/evidence.json @@ -0,0 +1,283 @@ +{ + "_description": "Complete evidence/provenance record. Every substantive claim, number, or material used elsewhere in this package points to a specific location in input/paper.pdf here, with an evidence_label (verbatim | reported | derived | missing). Page numbers refer to the printed page numbers at the bottom of each page of the PDF (the PDF's Introduction begins on printed page 1 / PDF page 1); pdftotext line numbers into input/paper.txt are also given for exact lookup.", + "evidence_labels_defined": { + "verbatim": "directly quoted or transcribed from the source", + "reported": "faithfully paraphrased from the source", + "derived": "transformed from reported information, with the derivation stated", + "missing": "unavailable in the source and requiring researcher input" + }, + "design_overview": [ + { + "id": "ev-design-1", + "claim": "The experiment used a private-information cascade paradigm: two urns (A and B), each containing 3 balls in a 2:1 ratio of 'majority' to 'minority' colored balls (symmetric design), equally likely a priori (chosen by 6-sided die: 1-3 = urn A, 4-6 = urn B).", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.4, para 'I. A SYMMETRIC MODEL'; paper.txt lines 132-148", + "table_reference": "Figure 1 (physical setup, described in text; the actual figure image was not machine-readable, so its content is reconstructed from the surrounding verbatim text and Appendix instructions, which state Urn A = 2 light/1 dark, Urn B = 1 light/2 dark)." + }, + { + "id": "ev-design-2", + "claim": "12 total experimental sessions were run: 3 baseline symmetric sessions (private draws only), 3 symmetric sessions with two public draws inserted after the 4th decision round, and 6 sessions using an asymmetric urn design.", + "evidence_label": "reported", + "source": "input/paper.pdf p.7-8 (Section II, paper.txt lines 250-320) for the 3+3 symmetric sessions; p.18 (Section V.B, paper.txt lines 671-677) for the 6 asymmetric sessions.", + "derivation": "The paper states '72 subjects' overall (p.7) and separately reports 36 decision-making subjects for the symmetric design (p.11, 'Of the 36 subjects') and implies an equal-sized asymmetric-design sample ('Twenty-one out of thirty-six subjects in the asymmetric design', p.21). 36 + 36 = 72, consistent with 6 decision-makers x 6 symmetric sessions and 6 decision-makers x 6 asymmetric sessions = 72 decision-making subjects total (monitors are excluded from this count, one additional monitor per session)." + }, + { + "id": "ev-design-3", + "claim": "Each session had 6 decision-making subjects and 1 subject randomly chosen as 'monitor' (assists with dice/marbles, does not make predictions).", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.7, paper.txt lines 254-258" + }, + { + "id": "ev-design-4", + "claim": "Subjects were undergraduates recruited from economics courses at the University of Virginia with no previous experience with this experiment.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.7, paper.txt lines 251-252" + }, + { + "id": "ev-design-5", + "claim": "Payment: $5 show-up fee; $2 per period for a correct urn prediction, $0 for incorrect; monitor paid an additional $15; average total earnings about $20; paid privately in cash at the end.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.7 and p.23-24 (Appendix), paper.txt lines 252-254, 862-863, 868-869, 899-900" + }, + { + "id": "ev-design-6", + "claim": "A session consisted of 15 (paid) periods and lasted about 1.5 hours, preceded by several unpaid practice periods (continued until each urn had been selected at least once during practice).", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.7-8, paper.txt lines 260-261, 301-306" + }, + { + "id": "ev-design-7", + "claim": "Within a period: a 6-sided die determines the urn (1,2,3 -> urn A; 4,5,6 -> urn B); urn contents are emptied into an unmarked, visually identical container; subjects are approached one at a time in random order to see one private draw WITH replacement; after each private draw the subject records the draw and writes/announces an urn prediction (A or B); the announcement (not the draw) is relayed to other subjects via an experimenter/announcer who does not know the urn or the private draw; at the end of the period the monitor reveals the true urn and subjects record earnings.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.7-8, paper.txt lines 272-286" + }, + { + "id": "ev-design-8", + "claim": "Physical isolation: partitions extending 3 feet beyond each desk, subjects at least 10 feet apart, monitor isolated behind a front partition so no one could see the die throw or others' private draws; subjects instructed to remain silent throughout, with early dismissal (paid only cumulative earnings to that point) as the consequence for communicating.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.7 footnote 12 and p.24-25 (Appendix), paper.txt lines 265-269, 941-949" + }, + { + "id": "ev-design-9", + "claim": "Asymmetric-design urn contents: Urn A (die 1,2,3) = 6 'a' balls + 1 'b' ball; Urn B (die 4,5,6) = 5 'a' balls + 2 'b' balls.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.18, Table 4, paper.txt lines 679-687" + }, + { + "id": "ev-design-10", + "claim": "Public-draw variant: in (at least) two of the three public-draw sessions (referred to in-text as 'sessions 4 and 5'), two additional public draws were inserted after the 4th decision round, so the 5th and 6th decision-makers in the sequence saw a sample of 3 draws (their own private draw plus the 2 public draws) rather than just their own private draw.", + "evidence_label": "reported", + "source": "input/paper.pdf p.17-18, paper.txt lines 634-639", + "derivation_or_gap": "The paper does not fully specify whether the identical 2-public-draws-after-round-4 procedure was used in every period of all three public-draw sessions, or only in the periods/sessions analyzed for the representativeness test. This procedural gap is logged in audit/missing_information.json (item on public-draw procedure)." + } + ], + "theoretical_model": [ + { + "id": "ev-model-1", + "claim": "Signal informativeness (symmetric design): P(signal a | urn A) = 2/3, P(signal b | urn A) = 1/3, and symmetrically for urn B; priors P(A) = P(B) = 1/2.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.4, paper.txt lines 133-141" + }, + { + "id": "ev-model-2", + "claim": "Bayesian posterior formula: Pr(A | n a-signals, m b-signals) = 2^n / (2^n + 2^m), where n and m count only the 'relevant' signals (those before a cascade starts, the two decisions that start a cascade, and any non-Bayesian deviation from a cascade).", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.6, equation (1), paper.txt lines 213-240" + }, + { + "id": "ev-model-3", + "claim": "Table 1: Posterior probability of event A as a function of the number of relevant a-signals (rows 0-3) and b-signals (columns 0-3), symmetric design.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.5, Table I, paper.txt lines 195-206" + }, + { + "id": "ev-model-4", + "claim": "An information cascade forms once an imbalance of 2 in relevant signals in one direction occurs (e.g., the first two decisions agree); all subsequent decision-makers should then rationally ignore their own private signal and follow the established pattern.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.5-6, paper.txt lines 173-184" + }, + { + "id": "ev-model-5", + "claim": "A deviation from an established cascade pattern is interpreted (in equilibrium) as revealing that the deviator's private signal favored the non-cascade urn, because deviating is more costly when the signal agrees with the cascade.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.6, paper.txt lines 210-213" + }, + { + "id": "ev-model-5b", + "claim": "Table 5 / asymmetric posteriors: Bayesian posterior probability of urn A as a function of the number of a-signals (rows 0-6) and b-signals (columns 0-6) for the asymmetric design; 4 cells are highlighted where the Bayesian posterior and a simple counting-of-signals heuristic disagree.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.19, Table 5, paper.txt lines 708-724" + } + ], + "logit_error_model": [ + { + "id": "ev-logit-1", + "claim": "Logit choice rule: Pr(D_i = A) = e^{beta * pi_A} / (e^{beta*pi_A} + e^{beta*pi_B}) = 1 / (1 + e^{-beta(pi_A - pi_B)}), where pi_A, pi_B are expected payoffs of predicting A or B and beta parameterizes sensitivity to payoff differences (beta -> infinity = fully rational, beta -> 0 = random).", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.13, paper.txt lines 471-487" + }, + { + "id": "ev-logit-2", + "claim": "Table 3: round-by-round maximum-likelihood logit beta estimates (recursive across rounds 1-6) fit with a Newton-Raphson routine in GAUSS on the symmetric-design data: round1 beta=2.84 (se 0.36, 4 errors); round2 beta=8.67 (se 2.03, 3 errors); round3 beta=4.95 (se 1.02, 6 errors); round4 beta=2.94 (se 0.64, 14 errors); round5 beta=3.03 (se 0.58, 13 errors); round6 beta=4.29 (se 1.15, 7 errors).", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.14, Table 3, paper.txt lines 516-522" + }, + { + "id": "ev-logit-3", + "claim": "The fitted model correctly predicts 493 of 540 decisions (540 = 36 symmetric-design subjects x 15 periods); allowing for error reclassifies about 5% of decisions relative to the no-error analysis, but does not change the cascade/no-cascade classification of any period. A constrained single-beta-for-all-rounds alternative gives beta=3.78 but fits worse by a likelihood-ratio test than the round-specific model.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.14-15, paper.txt lines 532-536, 546-549" + }, + { + "id": "ev-logit-4", + "claim": "Using the round-1 beta, the model predicts the 2nd-round decision-maker will follow their own private draw (rather than an inconsistent 1st-round announcement) with probability 0.96; empirically, 2nd-round subjects followed their own private draw in 95% of cases where their draw conflicted with the 1st-round announcement.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.14-15, paper.txt lines 541-553" + }, + { + "id": "ev-logit-5", + "claim": "15 of 16 deviations from an established cascade pattern (in the logit/error analysis) were made by a subject who had just seen a private draw favoring the non-cascade urn.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.15, paper.txt lines 565-567" + }, + { + "id": "ev-logit-6", + "claim": "Illustrative posteriors incorporating decision error: after two A decisions and a b draw, Pr(A)=0.5745 (cascade still starts); with two A decisions and no B decisions, Pr(A|a draw)=0.84; with three A decisions, Pr(A|a draw)=0.85; with two A decisions and one (third) B decision before an a draw, Pr(A|a draw) falls to 0.73.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.15, paper.txt lines 556-575" + } + ], + "results_symmetric_design": [ + { + "id": "ev-res-sym-1", + "claim": "Cascade behavior (matching the Bayesian-rational pattern) was observed in 41 of the 56 periods (symmetric design) in which an imbalance made a cascade possible.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.9, paper.txt lines 324-326" + }, + { + "id": "ev-res-sym-2", + "claim": "Table 2 gives round-by-round subject IDs, decisions, private draws, and cascade outcome for periods 5-9 of Session 2, including 3 normal cascades (periods 5,6,7), a cascade delayed by a deviation (period 8, flagged with *), and a reverse cascade (period 9).", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.9, Table 2, paper.txt lines 357-382" + }, + { + "id": "ev-res-sym-3", + "claim": "Deviation from the Bayesian/cascade-consistent decision in favor of one's own private signal occurred in 26% of cases where the optimal Bayesian decision was inconsistent with the private-information-only decision; about 4% of decisions (over all six symmetric-design sessions) were inconsistent with BOTH Bayes' rule and private information.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.9-10, paper.txt lines 339-354" + }, + { + "id": "ev-res-sym-4", + "claim": "Efficiency definitions: actual efficiency = 100*(pi_actual - pi_random)/(pi_optimal - pi_random); private-information efficiency = 100*(pi_privateInfo - pi_random)/(pi_optimal - pi_random), where each pi is summed expected payoff over all 15 periods under that decision rule, using the 'Bayes distribution' (posterior assuming others are Bayesian, with obvious deviations revealing private signals) at the time of each decision.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.11-12, paper.txt lines 385-436" + }, + { + "id": "ev-res-sym-5", + "claim": "Symmetric design: actual efficiency averaged 91.4%, private-information efficiency averaged 72.1%, over all subjects. 22 of 36 subjects achieved 100% actual efficiency (perfect conformity with Bayes' rule); of the other 14, 9 did better than private-information-only decisions would predict. 4 of the 22 perfectly-efficient subjects also had 100% private-information efficiency (i.e., following only private information happened to be optimal for them throughout).", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.11-12, paper.txt lines 441-465" + }, + { + "id": "ev-res-sym-6", + "claim": "Status-quo bias test: across all 6 symmetric-design sessions there were 68 instances where the Bayes-distribution posterior for A was exactly 1/2 and the subject's private signal did not match the label of the immediately preceding decision. In 57 of 68 cases the subject did NOT follow the preceding decision and instead decided consistently with their own private signal -- i.e., no support for a status-quo/conformity bias beyond what Bayesian updating implies.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.16-17, paper.txt lines 611-627" + }, + { + "id": "ev-res-sym-7", + "claim": "Representativeness-bias test: in the two public-draw sessions analyzed ('sessions 4 and 5'), the two public draws plus the subject's own private draw matched the exact contents of one urn (i.e., were a 'representative' sample) in 36 cases; in 10 of those 36 cases the Bayesian posterior for the urn that the sample 'represented' was below 1/2, and the subject decided consistently with Bayes' rule (not representativeness) in all 10 of those cases -- i.e., no support found for a representativeness bias in this setting.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.17-18, paper.txt lines 623-668" + } + ], + "results_asymmetric_design": [ + { + "id": "ev-res-asym-1", + "claim": "Table 6 gives round-by-round subject IDs, decisions, private draws, and Bayesian posteriors for periods 2-5 of Session 10 (asymmetric design), illustrating cases where Bayes' rule and simple signal-counting disagree (marked with + or *) and a reverse cascade in period 4.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.20, Table 6, paper.txt lines 751-786" + }, + { + "id": "ev-res-asym-2", + "claim": "Over all 6 asymmetric-design sessions, cascades formed in 46 of the 66 periods in which they were possible; reverse cascades made up 18 of those 46 cascade periods (vs. 13 of 41 in the symmetric design).", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.20, paper.txt lines 786-790" + }, + { + "id": "ev-res-asym-3", + "claim": "Accounting for the counting heuristic reduces the share of periods with a rational Bayesian cascade from about 73% (symmetric design) to about 70% (asymmetric design). When Bayes' rule and counting prescribe different decisions, subjects made the Bayes-correct decision in 41 of 82 such cases (50%); when counting made no distinct prediction, 66% of decisions were Bayes-correct. Overall, 115 of 540 asymmetric-design decisions were inconsistent with Bayes' rule, and 'over a third' of those inconsistencies are explained by the counting heuristic; 29 of these 41 counting-consistent/Bayes-inconsistent decisions were made by about a third of the subjects.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.20-21, paper.txt lines 796-829" + }, + { + "id": "ev-res-asym-4", + "claim": "Asymmetric design: actual efficiency averaged 67.6%, private-information efficiency averaged 45.2%. Counting efficiency (expected-payoff gain from a pure counting rule over random choice, as a % of the optimal-over-random gain) was, averaged over subjects, approximately equal to actual efficiency; 21 of 36 subjects had actual efficiency exceeding their own counting efficiency.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.21, paper.txt lines 804-819" + }, + { + "id": "ev-res-summary-1", + "claim": "Table 7 (Summary of Cascade Results): Symmetric setup - 28 normal-cascade periods, 13 reverse-cascade periods, 10 possible-but-not-formed normal, 5 possible-but-not-formed reverse (total 56). Asymmetric setup - 28 normal-cascade periods, 18 reverse-cascade periods, 12 possible-but-not-formed normal, 8 possible-but-not-formed reverse (total 66). Overall, cascades formed in 87 of 122 periods across all 12 sessions in which a cascade was possible.", + "evidence_label": "verbatim", + "source": "input/paper.pdf p.22, Table 7, paper.txt lines 832-845" + } + ], + "materials_and_instructions": [ + { + "id": "ev-mat-1", + "claim": "Full oral instructions script for the symmetric design (read aloud to participants), including the cover story, payment terms, urn/die mechanics, private-draw procedure, decision/announcement procedure, description of the paper record sheet columns (0)-(9), assignment of subject identity via colored marbles, and two worked practice-period walkthroughs.", + "evidence_label": "verbatim", + "source": "input/paper.pdf pp.23-25 (Appendix: Instructions for Symmetric Design), paper.txt lines 857-964", + "used_in": "materials/materials.json field 'symmetric_instructions_full_text'" + }, + { + "id": "ev-mat-2", + "claim": "No equivalent full instructions script is printed for the asymmetric design; only the differing urn contents (Table 4) are given. The paper states the asymmetric sessions used the 'same procedure' otherwise.", + "evidence_label": "missing", + "source": "input/paper.pdf p.18, paper.txt lines 671-677 ('we conducted six additional sessions with an asymmetric design'); no appendix text follows for this design.", + "note": "materials.json provides a DERIVED asymmetric instructions script obtained by substituting Table 4's ball counts into the symmetric verbatim script and is explicitly labeled 'derived', not verbatim. Logged in audit/missing_information.json." + }, + { + "id": "ev-mat-3", + "claim": "No decision-sheet image/scan is included in the PDF; the record-sheet layout (columns 0-9) is described only in prose within the instructions.", + "evidence_label": "missing", + "source": "input/paper.pdf p.24, paper.txt lines 901-913" + }, + { + "id": "ev-mat-4", + "claim": "Figure 1 ('The Physical Setup') is referenced but its image content was not machine-readable from the extracted PDF text; its content is reconstructed from the surrounding verbatim prose (2 light/1 dark in urn A, 1 light/2 dark in urn B) and cross-checked against the identical numbers given verbatim in the Appendix instructions.", + "evidence_label": "derived", + "source": "input/paper.pdf p.4-5 and p.23, paper.txt lines 137-148, 872-882", + "derivation": "Combined two verbatim textual descriptions of the same urn contents (Section I prose and Appendix instructions) since the figure image itself could not be extracted as text." + }, + { + "id": "ev-mat-5", + "claim": "No post-experiment exit-questionnaire instrument is reproduced; the paper only reports that at the end of the first session subjects were asked (informally) whether the instructions/procedures were confusing or biased and had no complaints besides wanting less repetition in the oral instructions.", + "evidence_label": "reported", + "source": "input/paper.pdf p.8, footnote 16, paper.txt lines 311-315" + } + ], + "references": { + "evidence_label": "verbatim", + "source": "input/paper.pdf pp.26-27, paper.txt lines 970-1009", + "note": "Transcribed for provenance completeness only. None of these cited works were fetched, browsed, or otherwise consulted; they are not part of the runnable task.", + "list": [ + "Anderson, Lisa R. Information Cascades, unpublished doctoral dissertation, University of Virginia, 1994.", + "Bannerjee, A. V. \"A Simple Model of Herd Behavior,\" Quarterly Journal of Economics, August 1992, 107(3), pp. 797-817.", + "Bikhchandani, Sushil; Hirshleifer, David and Welch, Ivo. \"A Theory of Fads, Fashion, Custom, and Cultural Change as Informational Cascades,\" Journal of Political Economy, October 1992, 100(5), pp. 992-1026.", + "Camerer, Colin F. \"Individual Decision Making,\" in J. Kagel and A. Roth, eds., Handbook of Experimental Economics. Princeton: Princeton University Press, 1995, pp. 587-616.", + "Camerer, Colin F. and Weigelt, Keith. \"Information Mirages in Experimental Asset Markets,\" Journal of Business, October 1991, 64(4), pp. 463-93.", + "Davis, Douglas D. and Holt, Charles A. Experimental Economics. Princeton: Princeton University Press, 1993.", + "Grether, David M. \"Bayes' Rule as a Descriptive Model: The Representativeness Heuristic,\" Quarterly Journal of Economics, March 1980, 95, pp. 537-557.", + "Grether, David M. \"Testing Bayes Rule and the Representativeness Heuristic: Some Experimental Evidence,\" Journal of Economic Behavior and Organization, January 1992, 17, pp. 31-57.", + "Holt, Charles A. and Anderson, Lisa R. \"Classroom Games: Understanding Bayes' Rule,\" Journal of Economic Perspectives, Spring 1996, 10(2), pp. 179-187.", + "Kahneman, Daniel and Tversky, Amos. \"On the Psychology of Prediction,\" Psychological Review, July 1973, 80(4), pp. 237-251.", + "Keynes, John Maynard. The General Theory of Employment, Interest, and Money. New York: Harcourt, Brace & World, 1965.", + "Plott, Charles R. and Sunder, Shyam. \"Efficiency of Experimental Security Markets with Insider Information: An Application of Rational-Expectations Models,\" Journal of Political Economy, August 1982, 90, pp. 663-698.", + "Samuelson, William and Zeckhauser, Richard. \"Status Quo Bias in Decision Making,\" Journal of Risk and Uncertainty, March 1988, 1, pp. 7-59.", + "Stern, Steve. \"The Effects of Firm Optimizing Behavior in Firm Matching Models,\" Review of Economic Studies, October 1990, 57(4), pp. 647-660.", + "Welch, Ivo. \"Sequential Sales, Learning, and Cascades,\" Journal of Finance, June 1992, 47(2), pp. 695-732." + ] + } +} diff --git a/studies/study_016/anderson_holt_1997_information_cascades/source/paper_metadata.json b/studies/study_016/anderson_holt_1997_information_cascades/source/paper_metadata.json new file mode 100644 index 0000000..d9ce3e3 --- /dev/null +++ b/studies/study_016/anderson_holt_1997_information_cascades/source/paper_metadata.json @@ -0,0 +1,54 @@ +{ + "_description": "Bibliographic metadata for the source paper. All fields are drawn only from the uploaded PDF text plus the contributor-supplied filename; no web search, DOI lookup, or OSF/author-page lookup was performed. 'external_sources' is empty because the job did not authorize any external URL.", + "title": { + "value": "Information Cascades in the Laboratory", + "evidence_label": "verbatim", + "source": "input/paper.pdf, p.1 (title line)" + }, + "authors": [ + { + "name": "Lisa R. Anderson", + "affiliation_at_time_of_writing": "Assistant Professor of Economics, American University, Washington, D.C.", + "evidence_label": "verbatim", + "source": "input/paper.pdf, p.1 footnote *" + }, + { + "name": "Charles A. Holt", + "affiliation_at_time_of_writing": "Professor of Economics, University of Virginia, Charlottesville, VA", + "evidence_label": "verbatim", + "source": "input/paper.pdf, p.1 footnote *" + } + ], + "publication_year": { + "value": 1997, + "evidence_label": "reported", + "source": "Derived from the contributor-supplied filename 'anderson_holt_1997_information_cascades.pdf'. The PDF text itself contains no printed journal name, volume, issue, or page range, and the PDF's embedded metadata (checked via pdfinfo) contains no title/author/date fields beyond a 1999 Acrobat Distiller creation timestamp for the file itself, which is not the same as the publication date." + }, + "journal": { + "value": null, + "evidence_label": "missing", + "note": "Not present anywhere in the extracted PDF text (no running header/footer with journal name, volume, issue, or page numbers was found on any of the 27 pages). Not looked up externally per job policy." + }, + "doi": { + "value": null, + "evidence_label": "missing", + "note": "No DOI string appears in the PDF text. DOI resolution/lookup was not performed because no external URL was authorized for this job." + }, + "abstract": { + "value": "When a series of individuals with private information announce public predictions, initial conformity can create an \"information cascade\" in which later predictions match the early announcements. This paper reports an experiment in which private signals are draws from an unobserved urn. Subjects make predictions in sequence and are paid if they correctly guess which of two urns was used for the draws. If initial decisions coincide, then it is rational for subsequent decision makers to follow the established pattern, regardless of their private information. Rational cascades formed in most periods in which such an imbalance occurred.", + "evidence_label": "verbatim", + "source": "input/paper.pdf, p.1 (Abstract)" + }, + "funding": { + "value": "University of Virginia Bankard Fund; U.S. National Science Foundation grants SES90-12694 and SES93-20617.", + "evidence_label": "verbatim", + "source": "input/paper.pdf, p.1 footnote *" + }, + "reference_list_of_cited_works": { + "evidence_label": "verbatim", + "source": "input/paper.pdf, pp.26-27 (References)", + "note": "Full reference list transcribed in source/evidence.json under 'references'. Provided for context only; none of these external works were fetched or consulted." + }, + "external_sources": [], + "external_source_policy_note": "job.json / task instructions supplied no OSF or open-materials URL, so per policy no web search, web fetch, DOI resolution, or author/repository lookup was performed. This metadata file and the entire package were built solely from input/paper.pdf." +} diff --git a/studies/study_016/anderson_holt_1997_information_cascades/study.json b/studies/study_016/anderson_holt_1997_information_cascades/study.json new file mode 100644 index 0000000..2c1a7da --- /dev/null +++ b/studies/study_016/anderson_holt_1997_information_cascades/study.json @@ -0,0 +1,99 @@ +{ + "_description": "Researcher-facing overview of this package. Full provenance for every claim below is in source/evidence.json; participant-facing text is in materials/materials.json; the runnable task is task/task.json + task/adapter.py; scoring is evaluation/evaluation.py; open items are audit/missing_information.json.", + + "paper": { + "title": "Information Cascades in the Laboratory", + "authors": ["Lisa R. Anderson", "Charles A. Holt"], + "publication_year": 1997, + "source_file": "input/paper.pdf", + "note": "Bibliographic details beyond title/authors/year (journal, volume, pages, DOI) are not present anywhere in the extracted PDF text and are marked missing in source/paper_metadata.json." + }, + + "external_sources_consulted": [], + "external_source_policy": "No OSF or open-materials URL was supplied for this job, so no web search, web fetch, DOI resolution, or repository/author-page lookup was performed. This package was built solely from input/paper.pdf.", + + "empirical_studies": [ + { + "id": "study_1_symmetric", + "name": "Symmetric-design information cascade experiment", + "sessions": 6, + "conditions": [ + { + "id": "baseline_private_draws_only", + "n_sessions": 3, + "description": "Standard sequential urn-prediction task with only private draws; no public information beyond announced decisions." + }, + { + "id": "public_draws_after_round4", + "n_sessions": 3, + "description": "Same task, but two additional balls are drawn and their colors announced PUBLICLY after the 4th decision-maker's round (procedure fully confirmed in-text only for 2 of these 3 sessions; see audit/missing_information.json)." + } + ], + "sample": "36 decision-making undergraduate subjects (6 per session, University of Virginia economics courses, no prior experience with this experiment) plus 1 non-deciding 'monitor' per session.", + "task_summary": "Each period, a fair 6-sided die (simulated) selects urn A (2 light/1 dark balls) or urn B (1 light/2 dark balls), equally likely. Six subjects are approached in random order, each sees one private ball draw with replacement, then announces an A/B urn prediction, observable by later subjects. $2 for a correct prediction, $0 otherwise, plus a $5 show-up fee; 15 paid periods per session after unpaid practice periods.", + "key_measures": [ + "Cascade formation rate among periods where a cascade was possible (paper: 41/56)", + "Reverse-cascade share of formed cascades (paper: 13 of 41 formed cascades)", + "Actual efficiency (paper: 91.4%) and private-information-only efficiency (paper: 72.1%)", + "Status-quo bias check: does the subject follow the preceding decision or their own signal when the posterior is exactly 1/2? (paper: 57 of 68 followed own signal -- no bias detected)", + "Representativeness bias check, public-draw condition only (paper: 10 of 10 qualifying cases followed Bayes' rule, not representativeness -- no bias detected)" + ], + "key_findings_reported": "Cascades, including reverse cascades, formed rationally in most possible periods. A round-by-round logit analysis of decision errors (Table 3) shows small but non-negligible error rates that do not change any period's cascade classification. No support was found for status-quo bias or representativeness bias in this setting." + }, + { + "id": "study_2_asymmetric", + "name": "Asymmetric-design information cascade experiment (counting-heuristic test)", + "sessions": 6, + "conditions": [ + { + "id": "asymmetric_urns", + "n_sessions": 6, + "description": "Same sequential task, but urn A contains 6 a-balls/1 b-ball and urn B contains 5 a-balls/2 b-balls, so a b-signal is much more diagnostic than an a-signal and a simple count of signal types can disagree with Bayes' rule." + } + ], + "sample": "36 decision-making undergraduate subjects (6 per session) plus 1 monitor per session; same subject pool/recruitment as study_1.", + "task_summary": "Identical mechanics to study_1's baseline condition, with the asymmetric urn contents above and no public-draw insertion.", + "key_measures": [ + "Cascade formation rate (paper: 46/66) and reverse-cascade share (paper: 18 of 46 formed cascades)", + "Actual efficiency (paper: 67.6%) and private-information-only efficiency (paper: 45.2%)", + "Counting-heuristic check: when Bayes' rule and simple signal-counting disagree, does the subject follow Bayes' rule? (paper: 41 of 82 disagreement cases, i.e. 50%)", + "Counting efficiency (paper: approximately equal to actual efficiency, averaged over subjects; 21 of 36 subjects exceeded their own counting efficiency)" + ], + "key_findings_reported": "Cascades remained prevalent but reverse cascades were relatively more common than in the symmetric design. About a third of all deviations from Bayes' rule can be explained by a simple counting heuristic rather than correct Bayesian updating." + } + ], + + "participant_flow": { + "recruitment": "Undergraduate economics students at the University of Virginia, no previous experience with this experiment.", + "per_session_composition": "6 decision-making subjects + 1 randomly chosen monitor (assists with dice/marbles, does not make predictions).", + "session_structure": "Show-up payment ($5) -> instructions read aloud -> monitor selection -> seating behind partitions -> several unpaid practice periods (until each urn selected at least once) -> 15 paid periods, each consisting of a die roll (urn selection, hidden), 6 sequential private-draw-then-announce rounds, and a public reveal of the true urn -> private cash payment at the end (average ~$20 total, monitor +$15).", + "isolation_and_conduct": "Physical partitions and spacing enforced anonymity; subjects were required to remain silent, with early dismissal (paid only cumulative earnings) as the penalty for communicating." + }, + + "conditions_overview": { + "study_1_symmetric": ["baseline_private_draws_only", "public_draws_after_round4"], + "study_2_asymmetric": ["asymmetric_urns"] + }, + + "outcomes_overview": [ + "Binary per-period cascade classification (none possible / formed normal cascade / formed reverse cascade / possible but not formed)", + "Per-round decision correctness against the true urn", + "Expected-payoff-based efficiency measures: actual, private-information-only, optimal, random-choice, and (asymmetric design only) counting-rule efficiency", + "Status-quo bias and representativeness bias indicator rates (symmetric design)", + "Counting-heuristic-vs-Bayes' rule indicator rate (asymmetric design)", + "Round-by-round logit decision-error rates (Table 3) -- reported in the paper, but marked not_ready in evaluation.py because the raw per-subject dataset needed to refit it is not published in the paper and was not sought (no external source authorized)" + ], + + "package_entry_point": { + "task": "task/task.json (interaction contract) + task/adapter.py (runnable simulation; supports `python task/adapter.py --smoke-test`)", + "evaluation": "evaluation/evaluation.py (consumes task/adapter.py's session logs; supports `python evaluation/evaluation.py --smoke-test`)" + }, + + "readiness_status": { + "status": "needs_researcher_review", + "reason": "The package is fully runnable end-to-end (adapter + evaluation smoke tests pass with no network access) and every required file is present, but two categories of researcher decision remain open before this should be treated as a verified, citation-ready replication: (1) study_2_asymmetric's participant-facing instructions are a derived reconstruction, not a verbatim transcript, and (2) the paper's Table 3 logit error-rate analysis cannot be replicated without the authors' unpublished raw dataset. See audit/missing_information.json for the full list (9 items) and suggested next steps.", + "safe_to_run_task_smoke_test": true, + "safe_to_run_full_replication": true, + "safe_to_claim_exact_replication_of_table_3": false + } +} diff --git a/studies/study_016/anderson_holt_1997_information_cascades/task/__pycache__/adapter.cpython-311.pyc b/studies/study_016/anderson_holt_1997_information_cascades/task/__pycache__/adapter.cpython-311.pyc new file mode 100644 index 0000000..1265084 Binary files /dev/null and b/studies/study_016/anderson_holt_1997_information_cascades/task/__pycache__/adapter.cpython-311.pyc differ diff --git a/studies/study_016/anderson_holt_1997_information_cascades/task/adapter.py b/studies/study_016/anderson_holt_1997_information_cascades/task/adapter.py new file mode 100644 index 0000000..4eb57d2 --- /dev/null +++ b/studies/study_016/anderson_holt_1997_information_cascades/task/adapter.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +""" +Adapter for Anderson & Holt (1997), "Information Cascades in the Laboratory". + +Reconstructs the sequential urn-prediction task described in +materials/materials.json: each period, a die roll (simulated) picks urn A or +urn B; six decision-maker "seats" are approached in a random order; each seat +sees one private ball draw (with replacement) and the public decisions +announced so far this period, then announces its own decision (A or B). + +This module never makes network calls. It exposes: + - run_session(agent_fn, study, condition, n_periods=15, seed=None) + - run_smoke_test() -> dict (uses a built-in baseline policy, no agent_fn needed) + - CLI: `python adapter.py --smoke-test` + +`agent_fn` is supplied by the harness running the model under test. It must be +a callable: agent_fn(round_input: dict) -> {"decision": "A"|"B"}. See +task/task.json ("per_round_interaction") for the exact schema of round_input. + +If a study/condition combination requires a researcher decision that has not +been made (see audit/missing_information.json), run_session raises +AdapterBlocked with a clear message rather than guessing. +""" +import argparse +import json +import os +import random +import sys + +CONDITIONS = { + ("study_1_symmetric", "baseline_private_draws_only"): { + "urn_A": {"a": 2, "b": 1, "die_rolls": (1, 2, 3)}, + "urn_B": {"a": 1, "b": 2, "die_rolls": (4, 5, 6)}, + "public_draws_after_round": None, + }, + ("study_1_symmetric", "public_draws_after_round4"): { + "urn_A": {"a": 2, "b": 1, "die_rolls": (1, 2, 3)}, + "urn_B": {"a": 1, "b": 2, "die_rolls": (4, 5, 6)}, + "public_draws_after_round": 4, + "n_public_draws": 2, + }, + ("study_2_asymmetric", "asymmetric_urns"): { + "urn_A": {"a": 6, "b": 1, "die_rolls": (1, 2, 3)}, + "urn_B": {"a": 5, "b": 2, "die_rolls": (4, 5, 6)}, + "public_draws_after_round": None, + }, +} + +N_SEATS_PER_SESSION = 6 +N_PAID_PERIODS_DEFAULT = 15 +PAYOFF_CORRECT_USD = 2 +PAYOFF_INCORRECT_USD = 0 + + +class AdapterBlocked(Exception): + """Raised when faithful execution requires a missing researcher decision.""" + + +def _draw_ball(urn, rng): + total = urn["a"] + urn["b"] + return "light" if rng.random() < urn["a"] / total else "dark" + + +def _roll_urn(config, rng): + roll = rng.randint(1, 6) + if roll in config["urn_A"]["die_rolls"]: + return "A", config["urn_A"] + return "B", config["urn_B"] + + +def _load_materials(): + materials_path = os.path.join(os.path.dirname(__file__), "..", "materials", "materials.json") + with open(materials_path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _instructions_text(study, materials): + if study == "study_1_symmetric": + return materials["study_1_symmetric"]["instructions_full_text"]["text"] + if study == "study_2_asymmetric": + return materials["study_2_asymmetric"]["instructions_full_text"]["text"] + raise AdapterBlocked(f"Unknown study '{study}'") + + +def run_period(agent_fn, study, condition, config, period_index, rng, instructions_text): + true_urn_label, urn = _roll_urn(config, rng) + seat_order = list(range(1, N_SEATS_PER_SESSION + 1)) + rng.shuffle(seat_order) + + rounds_log = [] + decisions_so_far = [] + public_draws_after = config.get("public_draws_after_round") + n_public_draws = config.get("n_public_draws", 0) + inserted_public_draws = None + + for round_number in range(1, N_SEATS_PER_SESSION + 1): + private_draw = _draw_ball(urn, rng) + + if public_draws_after and round_number == public_draws_after + 1 and inserted_public_draws is None: + inserted_public_draws = [_draw_ball(urn, rng) for _ in range(n_public_draws)] + + extra_public_draws = inserted_public_draws if (public_draws_after and round_number > public_draws_after) else None + + round_input = { + "study": study, + "condition": condition, + "instructions_text": instructions_text, + "period": period_index, + "round": round_number, + "seat": seat_order[round_number - 1], + "private_draw": private_draw, + "decisions_so_far_this_period": list(decisions_so_far), + "extra_public_draws": extra_public_draws, + } + + result = agent_fn(round_input) + decision = result.get("decision") if isinstance(result, dict) else result + if decision not in ("A", "B"): + raise ValueError(f"agent_fn returned invalid decision {decision!r}; expected 'A' or 'B'") + + correct = decision == true_urn_label + decisions_so_far.append({"round": round_number, "decision": decision}) + rounds_log.append({ + "round": round_number, + "private_draw": private_draw, + "decision": decision, + "correct": correct, + "payoff_usd": PAYOFF_CORRECT_USD if correct else PAYOFF_INCORRECT_USD, + }) + + return { + "period": period_index, + "true_urn": true_urn_label, + "public_draws_inserted": inserted_public_draws, + "rounds": rounds_log, + } + + +def run_session(agent_fn, study, condition, n_periods=N_PAID_PERIODS_DEFAULT, seed=None, session_id=None): + key = (study, condition) + if key not in CONDITIONS: + raise AdapterBlocked( + f"No configuration for study={study!r} condition={condition!r}. " + f"Valid combinations: {sorted(CONDITIONS.keys())}" + ) + config = CONDITIONS[key] + materials = _load_materials() + instructions_text = _instructions_text(study, materials) + + rng = random.Random(seed) + periods = [ + run_period(agent_fn, study, condition, config, p, rng, instructions_text) + for p in range(1, n_periods + 1) + ] + return { + "study": study, + "condition": condition, + "session_id": session_id or f"{study}:{condition}:seed={seed}", + "periods": periods, + } + + +# --------------------------------------------------------------------------- +# Built-in baseline policies, used only for --smoke-test (no external agent +# required). These are NOT claims about human behavior; they only exist to +# exercise the simulation mechanics end-to-end. +# --------------------------------------------------------------------------- + +def _baseline_follow_private_signal(round_input): + """Always predicts the urn matching the subject's own private draw, ignoring public history.""" + return {"decision": "A" if round_input["private_draw"] == "light" else "B"} + + +def run_smoke_test(): + results = {} + for (study, condition) in CONDITIONS: + session = run_session( + _baseline_follow_private_signal, + study, + condition, + n_periods=3, + seed=12345, + session_id=f"smoke:{study}:{condition}", + ) + n_rounds = sum(len(p["rounds"]) for p in session["periods"]) + n_correct = sum(r["correct"] for p in session["periods"] for r in p["rounds"]) + results[f"{study}::{condition}"] = { + "n_periods": len(session["periods"]), + "n_rounds": n_rounds, + "n_correct": n_correct, + "sample_period_0": session["periods"][0], + } + return {"status": "ok", "policy": "baseline_follow_private_signal", "results": results} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--smoke-test", action="store_true", help="Run a network-free smoke test of the simulation mechanics and exit.") + parser.add_argument("--study", default=None, help="Study key, e.g. study_1_symmetric") + parser.add_argument("--condition", default=None, help="Condition key, e.g. baseline_private_draws_only") + parser.add_argument("--periods", type=int, default=N_PAID_PERIODS_DEFAULT) + parser.add_argument("--seed", type=int, default=None) + args = parser.parse_args() + + if args.smoke_test: + print(json.dumps(run_smoke_test(), indent=2)) + return 0 + + if args.study and args.condition: + session = run_session(_baseline_follow_private_signal, args.study, args.condition, n_periods=args.periods, seed=args.seed) + print(json.dumps(session, indent=2)) + return 0 + + parser.print_help() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/studies/study_016/anderson_holt_1997_information_cascades/task/task.json b/studies/study_016/anderson_holt_1997_information_cascades/task/task.json new file mode 100644 index 0000000..44b6d1b --- /dev/null +++ b/studies/study_016/anderson_holt_1997_information_cascades/task/task.json @@ -0,0 +1,105 @@ +{ + "_description": "Defines the runnable agent interaction that reconstructs Anderson & Holt (1997)'s sequential urn-prediction cascade task. The agent under test stands in, one decision at a time, for the human 'decision maker' role described in materials/materials.json: on every call it receives one private ball draw plus the public record of decisions already announced earlier in that period, and must announce an urn prediction (A or B). task/adapter.py implements the surrounding simulation (die rolls, ball draws with replacement, bookkeeping, payment) and calls the agent once per decision point.", + "package_entry_point": "task/adapter.py", + "evaluation_entry_point": "evaluation/evaluation.py", + "materials_reference": "materials/materials.json", + + "studies": { + "study_1_symmetric": { + "conditions": { + "baseline_private_draws_only": { + "urn_A": {"a_balls": 2, "b_balls": 1, "die_rolls": [1, 2, 3]}, + "urn_B": {"a_balls": 1, "b_balls": 2, "die_rolls": [4, 5, 6]}, + "public_draws_after_round": null, + "n_sessions_in_paper": 3, + "instructions_key": "study_1_symmetric.instructions_full_text" + }, + "public_draws_after_round4": { + "urn_A": {"a_balls": 2, "b_balls": 1, "die_rolls": [1, 2, 3]}, + "urn_B": {"a_balls": 1, "b_balls": 2, "die_rolls": [4, 5, 6]}, + "public_draws_after_round": 4, + "n_public_draws_inserted": 2, + "n_sessions_in_paper": 3, + "instructions_key": "study_1_symmetric.instructions_full_text", + "researcher_decision_flag": "public-draw insertion procedure confirmed verbatim/reported only for a subset of sessions ('sessions 4 and 5'); see audit/missing_information.json" + } + }, + "n_decision_makers_per_session": 6, + "n_paid_periods_per_session": 15, + "payoff_per_correct_decision_usd": 2, + "payoff_per_incorrect_decision_usd": 0, + "showup_fee_usd": 5 + }, + "study_2_asymmetric": { + "conditions": { + "asymmetric_urns": { + "urn_A": {"a_balls": 6, "b_balls": 1, "die_rolls": [1, 2, 3]}, + "urn_B": {"a_balls": 5, "b_balls": 2, "die_rolls": [4, 5, 6]}, + "public_draws_after_round": null, + "n_sessions_in_paper": 6, + "instructions_key": "study_2_asymmetric.instructions_full_text", + "instructions_evidence_label": "derived" + } + }, + "n_decision_makers_per_session": 6, + "n_paid_periods_per_session": 15, + "payoff_per_correct_decision_usd": 2, + "payoff_per_incorrect_decision_usd": 0, + "showup_fee_usd": 5 + } + }, + + "condition_assignment": { + "how_urn_is_chosen": "For each period, a fair 6-sided die roll (simulated with a seeded PRNG in task/adapter.py) selects urn A (rolls 1-3) or urn B (rolls 4-6), each with probability 1/2, matching the paper's procedure. The realized urn is held fixed for all 6 rounds within that period and is never shown to the agent during the period -- only revealed in the round-result record after round 6, matching the human procedure of the monitor announcing the true urn at period end.", + "how_private_draws_are_generated": "Each round, a ball is drawn with replacement from the period's urn using the urn's a:b ball ratio as the draw probability (e.g. baseline symmetric urn A -> P(a)=2/3, P(b)=1/3). This is the single piece of private information given to the agent for that round.", + "how_round_order_is_chosen": "A random permutation of decision-maker seats (1..6) is drawn once per period (uniformly, matching the paper's colored-marble draw), fixing the order in which rounds occur within the period.", + "agent_role_across_a_session": "The agent under test plays every decision-maker seat, one at a time and one call at a time -- there is no single continuous human 'self' to imitate across rounds, since in the original study each of the 6 seats was a different, mutually anonymous person who only ever saw their own single private draw. Consequently the adapter does NOT give the agent any memory of a prior round's private draw when calling it for a later round; it only ever passes (a) the current round's own private draw and (b) the public decision history announced so far this period, exactly as a newly-approached human subject would see it." + }, + + "per_round_interaction": { + "input_to_agent": { + "study": "study_1_symmetric | study_2_asymmetric", + "condition": "one of the condition keys above", + "instructions_text": "verbatim (study 1) or derived (study 2) oral instructions from materials.json, shown once per session before round 1", + "period": "integer, 1-15", + "round": "integer, 1-6 (this decision-maker's position in this period's random order)", + "private_draw": "'light' | 'dark' (called 'a'/'b' in the paper's notation)", + "decisions_so_far_this_period": "ordered list of {round, decision} for rounds 1..round-1 of the current period, decision in {'A','B'}", + "extra_public_draws": "present only under study_1_symmetric/public_draws_after_round4, populated for rounds 5 and 6 with the 2 draws revealed after round 4" + }, + "output_expected_from_agent": { + "decision": "'A' | 'B'", + "schema_note": "task/adapter.py's agent_fn callable must return exactly one of the two strings; adapter.py records it, has the simulated announcer relay it to subsequent rounds, and scores it against the period's true urn once round 6 completes." + } + }, + + "session_and_run_structure": { + "unit_of_replication": "one 'session' = 1 condition x 15 paid periods x 6 rounds/period = 90 agent calls, matching one human experimental session", + "full_replication_recommendation": "run 3 sessions of baseline_private_draws_only, 3 of public_draws_after_round4, and 6 of asymmetric_urns to mirror the paper's 12-session sample; task/adapter.py exposes n_sessions as a parameter so smaller runs are possible for cheaper/faster smoke-testing", + "smoke_test": "task/adapter.py --smoke-test runs 1 session x 3 periods x 6 rounds per condition using a built-in deterministic baseline policy (no external agent needed) purely to verify the simulation mechanics, JSON I/O, and scoring pipeline execute end-to-end without network access" + }, + + "outputs_consumed_by_evaluation": { + "session_log_schema": { + "study": "string", + "condition": "string", + "session_id": "string", + "periods": [ + { + "period": "integer", + "true_urn": "'A' | 'B'", + "rounds": [ + {"round": "integer", "private_draw": "'light'|'dark'", "decision": "'A'|'B'", "correct": "boolean"} + ] + } + ] + }, + "note": "evaluation/evaluation.py consumes exactly this schema (as emitted by task/adapter.py's run_session()) to compute cascade formation rates, efficiency measures, and bias checks against the benchmarks recorded in source/evidence.json." + }, + + "known_limitations": [ + "The adapter cannot reproduce genuine multi-human anonymity/social dynamics (footnote 14's report of subjects using visual/voice cues to detect agonizing decisions); it only reproduces the informational structure of the task.", + "Practice periods are omitted from the runnable task (they carried no decisions or payment in the original study; see materials.json) -- the agent's very first scored call in a session is period 1 with no prior practice.", + "study_2_asymmetric's instructions_text is a derived reconstruction, not a verbatim transcript (see audit/missing_information.json)." + ] +}