diff --git a/studies/study_016/README.md b/studies/study_016/README.md new file mode 100644 index 0000000..465c62a --- /dev/null +++ b/studies/study_016/README.md @@ -0,0 +1,7 @@ +# Analyzing Frictions in Generalized Second-Price Auction Markets + +This HumanStudy-Hub package was reconstructed from a published paper. + +- Review `study.json` for the study overview and readiness status. +- Review `audit/missing_information.json` before running the study. +- Run `python task/adapter.py --smoke-test` to check the package entry point. diff --git a/studies/study_016/audit/missing_information.json b/studies/study_016/audit/missing_information.json new file mode 100644 index 0000000..799fc38 --- /dev/null +++ b/studies/study_016/audit/missing_information.json @@ -0,0 +1,85 @@ +{ + "_purpose": "Authoritative researcher checklist of missing verbatim content and outstanding researcher decisions. Referenced by id (e.g. 'AUD-01') from study.json, materials/materials.json, task/task.json, and evaluation/evaluation.py. All items arise because this package was built ONLY from the uploaded PDF (input/paper.pdf) -- no OSF/open-materials URL was supplied for this job, so the paper's own online appendix (hosted at the paper's DOI landing page) was not retrieved, per the package builder's external-research policy.", + "checklist": [ + { + "id": "AUD-01", + "study": "human_main", + "field": "Verbatim participant instructions handout (Online Appendix L)", + "reason": "The paper states an experimenter read a printed instructions handout aloud, referenced as 'Online Appendix L', which is not included in the uploaded PDF (main text only).", + "impact": "materials/materials.json ships a 'derived' reconstruction of the instructions built only from the procedural facts reported in Section 4.1-4.5; it is not the wording actual subjects read, so any linguistic/framing effects of the real handout cannot be reproduced.", + "suggested_action": "Retrieve Online Appendix L from the paper's supplemental material (https://doi.org/10.1287/isre.2022.1187) once the researcher explicitly authorizes external retrieval for this job, and replace materials.json#mat.human_main.instructions_derived with the verbatim text (evidence_label -> verbatim)." + }, + { + "id": "AUD-02", + "study": "human_main", + "field": "Exact on-screen interface text/layout (Figure 2 screenshot)", + "reason": "Figure 2 is a screenshot image; pdftotext extraction only preserves the printed figure caption's five labeled callouts, not the pixel-level on-screen text.", + "impact": "The task's decision-point state (task/task.json#io_schema) is a faithful functional description of what information participants saw, but cannot reproduce the literal on-screen wording/formatting.", + "suggested_action": "If pixel-level UI fidelity is required (e.g., for a UI-replication study rather than a decision-task replication), re-extract Figure 2 as an image and have a researcher transcribe on-screen text, or request the original interface code from the authors." + }, + { + "id": "AUD-03", + "study": "human_mechanism", + "field": "Verbatim oTree instructions and decision screens (Online Appendix N.1), and the counterfactual-information-plot screenshots (Online Appendix N.2)", + "reason": "Section 5 references these as being in the online appendix, not included in the uploaded PDF.", + "impact": "materials/materials.json ships a 'derived' reconstruction for human_mechanism instructions; the exact look of the interactive counterfactual-payoff plot and its screen text is unknown.", + "suggested_action": "Same as AUD-01: retrieve from the paper's supplemental material once external retrieval is authorized." + }, + { + "id": "AUD-04", + "study": "human_main, human_mechanism", + "field": "Individual human bid trajectories / raw session data for the other two group members", + "reason": "The paper reports only aggregated statistics (means, bootstrapped SEs, permutation-test p-values) per treatment cell; it does not publish subject-level bid time series, and no OSF/data repository URL was supplied for this job.", + "impact": "task/adapter.py must simulate the two co-players per match using stylized built-in policies ('truthful', 'softmax_explorer') rather than replaying real human behavior. This makes the task runnable and lets it score the agent-under-test's OWN behavior (exploration, bid-to-value ratio, responsiveness to cost), but the resulting environment is NOT a validated behavioral clone of the paper's actual human co-players, so multi-agent outcome-level claims (e.g., 'this LLM produces the same allocative efficiency a group of 3 real subjects would') are out of scope for this package.", + "suggested_action": "If a data-sharing repository for this paper's raw session data becomes available and is explicitly supplied by the researcher, replace the built-in co-player policies with a replay/behavioral-clone module and re-validate." + }, + { + "id": "AUD-05", + "study": "human_main", + "field": "Continuous-time, mouse-driven bid adjustment protocol", + "reason": "human_main used continuous real-time mouse movement across a slider with a silent protocol (Remark 4) and second-by-second random termination (Remark 1); a turn-based agent interaction has no literal analogue to continuous mouse position.", + "impact": "task/adapter.py discretizes the match into a bounded sequence of decision checkpoints (task.json derived_design_decisions D1). Bid-ADJUSTMENT COUNTS from adapter.py runs are therefore not numerically comparable to the paper's Table 2/5 counts; only the qualitative direction of effects (e.g., cost lowers adjustment counts) is intended to be comparable.", + "suggested_action": "If checkpoint-level fidelity to the paper's real-time protocol is required, a researcher should replace `_decision_schedule()` / the per-period loop in adapter.py with a continuous-time discrete-event simulation calibrated to the paper's reported ~1%/second termination hazard, and clearly re-derive what 'one bid adjustment' means in that finer-grained simulation." + }, + { + "id": "AUD-06", + "study": "human_main", + "field": "Matches 5-10 common-random-seed value matching across treatments (Remark 2), and its Online Table M.1 exact termination-second sequences", + "reason": "The paper deliberately matched private values and match-duration realizations across sessions/treatments in matches 5-10 for a clean cross-treatment comparison; the specific seeds/sequences are in Online Table M.1, not included in the uploaded PDF. Also note the paper's own Endnote 19 documents a software bug in this seed-matching scheme (see source/evidence.json#ev.human_main.value_generation.bug).", + "impact": "adapter.py draws private values independently at random every match (task.json derived_design_decisions D2) since a single run has no other session to match against; this is a reasonable simplification for scoring one agent's behavior but means a researcher cannot use two separate adapter.py invocations to recreate the paper's exact matched-value cross-treatment design without supplying their own shared seed logic.", + "suggested_action": "If exact matched-value replication across two adapter.py runs is needed, add a shared value-sequence file/seed argument so two runs (one per cost treatment) consume the same per-match value draws; document whether to replicate or intentionally avoid the paper's reported seed bug." + }, + { + "id": "AUD-07", + "study": "human_mechanism", + "field": "The 4 canonical decision-period-length sequences (Online Table N.1)", + "reason": "Section 5 states 4 fixed duration-sequence realizations were used so results would not depend on one arbitrary sequence; the table itself is in the online appendix, not included in the uploaded PDF.", + "impact": "adapter.py draws a fresh geometric-like period count each match (termination probability 0.1/period, task.json derived_design_decisions D3) instead of replaying the paper's 4 fixed sequences, so run-to-run period-count variance will not exactly match the paper's design.", + "suggested_action": "If the 4 sequences become available, hardcode them into adapter.py's `_decision_schedule()` for study='human_mechanism' and cycle through them exactly as the paper did." + }, + { + "id": "AUD-08", + "study": "human_main, human_mechanism", + "field": "Bootstrapped standard errors and two-tailed permutation-test p-values (Good 2013) across independent human sessions (Tables 5-10)", + "reason": "These statistics require multiple independent SESSIONS of real subjects (the paper ran 8 sessions for human_main, additional sessions for human_mechanism); a single adapter.py run, or even several runs with simulated co-players, is not a valid stand-in for that sampling design.", + "impact": "evaluation/evaluation.py implements only descriptive, single/paired-run directional checks and explicitly returns a `not_ready` result (check_significance_not_ready) for any inferential replication of Tables 5-10, rather than fabricating a p-value.", + "suggested_action": "This gap can only be closed by running the SAME agent-under-test across many independent adapter.py sessions (analogous to the paper's 8+ sessions) and applying the same bootstrap/permutation procedure to the resulting session-level means -- a data-collection protocol decision for the researcher, not something this package can supply on its own." + }, + { + "id": "AUD-09", + "study": "comp_main / comp_additional (context only, not modeled by the runnable task)", + "field": "Ad-quality extension (Online Appendix E) and the exponential-distribution robustness check (Online Appendix C.3)", + "reason": "The main text only summarizes that these robustness analyses exist and hold qualitatively; the appendix details are not included in the uploaded PDF.", + "impact": "study.json and evidence.json note these robustness checks exist but cannot describe their exact specification; they are not implemented in task/adapter.py, which sticks to the bid-only ranking model reported in the main text (Section 3.1, Endnote 5).", + "suggested_action": "Not required for the current runnable task; retrieve only if a researcher specifically wants to extend adapter.py with quality-score-based ranking." + }, + { + "id": "AUD-10", + "study": "human_main, human_mechanism", + "field": "Exchange rate between experimental payoff units and cash payment; total/average subject earnings; IRB approval and informed-consent text; participant demographics", + "reason": "The paper reports that subjects were 'paid in cash' and gives sample sizes/session counts, but does not state the payoff-to-currency conversion rate, average earnings, IRB protocol details, consent-form wording, or participant demographic breakdowns (age, gender, major, etc.) anywhere in the main text.", + "impact": "task/adapter.py reports payoffs in the paper's own abstract payoff units (Equations 1-3), not real currency; a researcher wanting to state real monetary stakes, or to reproduce IRB/consent procedures for a live human-subject re-run, cannot do so from this package alone.", + "suggested_action": "If a live (non-agent) human-subject re-run or a monetarily-calibrated agent incentive is desired, the researcher must supply the conversion rate, consent text, and IRB protocol independently; these are outside what any paper text search can supply." + } + ] +} diff --git a/studies/study_016/evaluation/evaluation.py b/studies/study_016/evaluation/evaluation.py new file mode 100644 index 0000000..9838018 --- /dev/null +++ b/studies/study_016/evaluation/evaluation.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +""" +Evaluation checks for the GSP bid-adjustment-friction task +(task/adapter.py, task/task.json). + +Implements the descriptive checks that a single adapter run (or a small set +of runs across conditions) actually supports, modeled on the paper's own +qualitative Results: + + Result 1/6: lower bid-adjustment cost -> more bid adjustments (exploration) + Result 2/7: bid-to-value ratio is higher for lower-valued participants + Result 3/8: for the lowest-valued participants, bid-to-value ratio + increases as friction (cost) decreases + Result 5/10: allocative efficiency increases as friction cost increases + +Each check needs one or more run logs produced by +`task/adapter.py ... --output run.json` (or the `run_session()` / +`run(...)` return value in-process). Checks that need data this package +does not have (e.g., the paper's bootstrapped permutation-test p-values +computed over real independent human sessions) return a structured +`not_ready` result naming the missing requirement, rather than fabricating +a p-value. See audit/missing_information.json (AUD-08, AUD-09). + +Runs fully offline (stdlib only). +""" + +from __future__ import annotations + +import json +import sys +from typing import Optional + + +RANK_LABELS = ("highest", "medium", "lowest") + + +def _rank_metrics(run_output: dict) -> dict: + """Per-rank (highest/medium/lowest private value) descriptive means, + pooling across ALL THREE participants (agent-under-test + the two + simulated co-players) in every match of one run. This mirrors how the + paper pools across all subjects who happened to hold a given rank + (Tables 2/3/5/6/8/9), except the paper pools across real human subjects + and this pools across one agent-under-test plus simulated co-players. + """ + adj = {r: [] for r in RANK_LABELS} + ratio = {r: [] for r in RANK_LABELS} + efficiency = [] + + for m in run_output["matches"]: + participant_by_rank = m.get("participant_by_rank") + if participant_by_rank is None: + raise ValueError( + "match record missing 'participant_by_rank'; run log was produced " + "by an incompatible/older adapter.py version" + ) + for label, participant in zip(RANK_LABELS, participant_by_rank): + adj[label].append(m["num_bid_adjustments_by_participant"][participant]) + bv = m["bid_to_value_by_participant"][participant] + if bv is not None: + ratio[label].append(bv) + if m["allocative_efficiency"] is not None: + efficiency.append(m["allocative_efficiency"]) + + def mean(xs): + return round(sum(xs) / len(xs), 4) if xs else None + + return { + "mean_bid_adjustments_by_rank": {r: mean(adj[r]) for r in RANK_LABELS}, + "mean_bid_to_value_ratio_by_rank": {r: mean(ratio[r]) for r in RANK_LABELS}, + "mean_allocative_efficiency": mean(efficiency), + "n_matches": len(run_output["matches"]), + } + + +def check_exploration_direction(run_low_cost: dict, run_high_cost: dict) -> dict: + """Result 1/6: lower cost -> more bid adjustments, checked within a + matched pair of runs that differ ONLY in `cost` (same study/alpha/ + info_cost/seed-generating conditions otherwise).""" + c_low = run_low_cost["condition"]["cost"] + c_high = run_high_cost["condition"]["cost"] + if c_low >= c_high: + return { + "check": "exploration_direction", + "status": "not_ready", + "reason": "run_low_cost.condition.cost must be strictly less than run_high_cost.condition.cost", + } + low = _rank_metrics(run_low_cost) + high = _rank_metrics(run_high_cost) + per_rank = {} + all_consistent = True + for r in RANK_LABELS: + lo, hi = low["mean_bid_adjustments_by_rank"][r], high["mean_bid_adjustments_by_rank"][r] + consistent = (lo is not None and hi is not None and lo >= hi) + all_consistent = all_consistent and consistent + per_rank[r] = {"low_cost_mean_adjustments": lo, "high_cost_mean_adjustments": hi, + "direction_matches_paper": consistent} + return { + "check": "exploration_direction", + "paper_result_ids": ["Result 1", "Result 6"], + "evidence_ids": ["ev.comp.result1", "ev.human_main.result6"], + "status": "computed", + "direction_matches_paper_for_all_ranks": all_consistent, + "per_rank": per_rank, + "caveat": "Descriptive comparison of two simulated-co-player runs; not a significance test over independent human sessions (see check_significance_not_ready below).", + } + + +def check_overbidding_by_lowest_valued(run: dict) -> dict: + """Result 2/7: bid-to-value ratio is higher for lower-valued participants + within a single run.""" + metrics = _rank_metrics(run) + r = metrics["mean_bid_to_value_ratio_by_rank"] + if any(r[k] is None for k in RANK_LABELS): + return {"check": "overbidding_by_lowest_valued", "status": "not_ready", + "reason": "one or more ranks had no valid bid-to-value observations in this run"} + ordered_correctly = r["lowest"] >= r["medium"] >= r["highest"] + return { + "check": "overbidding_by_lowest_valued", + "paper_result_ids": ["Result 2", "Result 7"], + "evidence_ids": ["ev.comp.result2to4", "ev.human_main.result7to8"], + "status": "computed", + "mean_bid_to_value_ratio_by_rank": r, + "direction_matches_paper": ordered_correctly, + } + + +def check_friction_reduces_lowest_valued_overbidding(run_low_cost: dict, run_high_cost: dict) -> dict: + """Result 3/8: for the lowest-valued participants, bid-to-value ratio + increases as friction (cost) decreases, i.e. is higher in the low-cost + run than the high-cost run.""" + c_low = run_low_cost["condition"]["cost"] + c_high = run_high_cost["condition"]["cost"] + if c_low >= c_high: + return { + "check": "friction_reduces_lowest_valued_overbidding", + "status": "not_ready", + "reason": "run_low_cost.condition.cost must be strictly less than run_high_cost.condition.cost", + } + low = _rank_metrics(run_low_cost)["mean_bid_to_value_ratio_by_rank"]["lowest"] + high = _rank_metrics(run_high_cost)["mean_bid_to_value_ratio_by_rank"]["lowest"] + if low is None or high is None: + return {"check": "friction_reduces_lowest_valued_overbidding", "status": "not_ready", + "reason": "missing lowest-rank bid-to-value observations in one of the two runs"} + return { + "check": "friction_reduces_lowest_valued_overbidding", + "paper_result_ids": ["Result 3", "Result 8"], + "evidence_ids": ["ev.comp.result2to4", "ev.human_main.result7to8"], + "status": "computed", + "lowest_rank_bid_to_value_ratio": {"low_cost_run": low, "high_cost_run": high}, + "direction_matches_paper": low >= high, + } + + +def check_efficiency_increases_with_friction(run_low_cost: dict, run_high_cost: dict) -> dict: + """Result 5/10: allocative efficiency increases as friction cost + increases.""" + c_low = run_low_cost["condition"]["cost"] + c_high = run_high_cost["condition"]["cost"] + if c_low >= c_high: + return { + "check": "efficiency_increases_with_friction", + "status": "not_ready", + "reason": "run_low_cost.condition.cost must be strictly less than run_high_cost.condition.cost", + } + low = _rank_metrics(run_low_cost)["mean_allocative_efficiency"] + high = _rank_metrics(run_high_cost)["mean_allocative_efficiency"] + if low is None or high is None: + return {"check": "efficiency_increases_with_friction", "status": "not_ready", + "reason": "missing allocative-efficiency observations in one of the two runs"} + return { + "check": "efficiency_increases_with_friction", + "paper_result_ids": ["Result 5", "Result 10"], + "evidence_ids": ["ev.comp.result5", "ev.human_main.result10"], + "status": "computed", + "mean_allocative_efficiency": {"low_cost_run": low, "high_cost_run": high}, + "direction_matches_paper": high >= low, + } + + +def check_significance_not_ready(reason: Optional[str] = None) -> dict: + """The paper's inferential statistics (bootstrapped standard errors and + two-tailed permutation tests across MULTIPLE INDEPENDENT SESSIONS of + real human subjects; Tables 5-10) cannot be honestly reproduced from + adapter.py runs, because: + (a) co-players are simulated policies, not independent human sessions + (audit AUD-04), so there is no valid cross-session sampling + variation to bootstrap or permute; and + (b) the paper's raw session-level data / Online Appendix M robustness + tables are not included in the uploaded PDF (audit AUD-08). + This function documents that gap structurally instead of fabricating a + p-value. + """ + return { + "check": "significance_tests_matching_tables_5_through_10", + "status": "not_ready", + "missing_requirement": ( + reason or + "Independent per-session human (or validated behavioral-clone) data " + "across the C in {0.0,0.1} x alpha in {0.2,0.5,0.8} (human_main) or " + "C x info_cost (human_mechanism) cells, to bootstrap standard errors " + "and run two-tailed permutation tests as in the paper (Good 2013)." + ), + "audit_ref": "AUD-08", + } + + +def evaluate_from_files(paths: list[str]) -> dict: + runs = [] + for path in paths: + with open(path) as f: + payload = json.load(f) + runs.append(payload["run"] if "run" in payload else payload) + return evaluate(runs) + + +def evaluate(runs: list[dict]) -> dict: + """Top-level entry point. `runs` is a list of run-log dicts as produced + by adapter.run_session(...) (or the "run" field of adapter.py's JSON + stdout/--output). Automatically pairs up runs that share study/alpha/ + info_cost but differ in cost, to run the cost-contrast checks; always + runs the single-run checks on every run provided.""" + results = {"single_run_checks": [], "paired_run_checks": [], "not_ready": []} + + for run in runs: + results["single_run_checks"].append({ + "condition": run["condition"], + "result": check_overbidding_by_lowest_valued(run), + }) + + by_key = {} + for run in runs: + c = run["condition"] + key = (c["study"], c.get("alpha"), c.get("info_cost")) + by_key.setdefault(key, {})[c["cost"]] = run + + for key, by_cost in by_key.items(): + if 0.0 in by_cost and (0.1 in by_cost): + low, high = by_cost[0.0], by_cost[0.1] + results["paired_run_checks"].append({ + "matched_on": {"study": key[0], "alpha": key[1], "info_cost": key[2]}, + "exploration_direction": check_exploration_direction(low, high), + "friction_reduces_lowest_valued_overbidding": check_friction_reduces_lowest_valued_overbidding(low, high), + "efficiency_increases_with_friction": check_efficiency_increases_with_friction(low, high), + }) + + if not results["paired_run_checks"]: + results["not_ready"].append({ + "check": "cost_contrast_checks", + "status": "not_ready", + "missing_requirement": ( + "Need at least one matched pair of runs with cost=0.0 and cost=0.1 " + "(same study/alpha/info_cost) to evaluate Results 1/3/5/6/8/10." + ), + }) + + results["not_ready"].append(check_significance_not_ready()) + return results + + +def main(argv=None) -> int: + argv = sys.argv[1:] if argv is None else argv + if not argv: + print(json.dumps({ + "usage": "evaluation.py RUN_LOG.json [RUN_LOG2.json ...]", + "note": "Each RUN_LOG.json is produced by: python3 ../task/adapter.py ... --output RUN_LOG.json", + }, indent=2)) + return 0 + print(json.dumps(evaluate_from_files(argv), indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/studies/study_016/materials/materials.json b/studies/study_016/materials/materials.json new file mode 100644 index 0000000..2e4e513 --- /dev/null +++ b/studies/study_016/materials/materials.json @@ -0,0 +1,117 @@ +{ + "_purpose": "Participant-visible materials, grouped by study and condition. Every item carries an evidence_label and a source pointer into source/evidence.json (by 'evidence_id') or directly into input/paper.pdf. Items labeled 'derived' are researcher-facing reconstructions built ONLY from the reported procedural facts in the paper (never invented facts); the derivation is stated explicitly so a researcher can replace them with the verbatim handout text if/when it becomes available (see audit/missing_information.json).", + + "human_main": { + "study_name": "Main laboratory experiment (Section 4)", + "conditions": { + "between_subjects": { "variable": "C (bid-adjustment cost)", "levels": [0.0, 0.1] }, + "within_subjects": { "variable": "alpha (slot-2/slot-1 value ratio)", "levels": [0.2, 0.5, 0.8] } + }, + "materials": [ + { + "id": "mat.human_main.overview_text", + "title": "Task framing given to participants", + "evidence_label": "reported", + "evidence_id": "ev.human_main.auction_mechanics", + "content": "Participants are told they will bid, as one of three anonymous 'participants' in a group, on two 'goods' across 10 paid 'matches'. Advertisers/ad-slots/auction are relabeled participants/goods/match for the human-subject audience (the paper is explicit that this relabeling was used to make the task intuitive for subjects; the underlying GSP structure is unchanged)." + }, + { + "id": "mat.human_main.instructions_derived", + "title": "Participant instructions (derived reconstruction; verbatim handout is missing)", + "evidence_label": "derived", + "derivation": "Reconstructed strictly from the procedural facts reported in Section 4.1-4.5 and Remarks 1-4 of the paper (see evidence ids ev.human_main.treatments, ev.human_main.matches, ev.human_main.random_termination, ev.human_main.value_generation, ev.human_main.auction_mechanics, ev.human_main.payment_mechanism, ev.human_main.silent_protocol, ev.human_main.training). No wording, phrasing, or additional facts were invented; this is not the verbatim instructions handout (Online Appendix L), which is unavailable to this package (see audit entry AUD-01).", + "content": [ + "You will take part in 10 matches. In each match you are grouped with two other participants (regrouped at random each match).", + "In every match there are two goods for sale: Good 1 and Good 2. Good 2 is worth a fraction (called alpha) of Good 1's value; alpha will be the same for everyone in your group for that match and will be shown on screen.", + "At the start of a match you will be told your own private value for Good 1 (an integer). Only you know your own value.", + "You submit a bid for Good 1 (a number). Your first bid in a match is free to place. If you change your bid later in the same match, you pay a bid-adjustment cost of C for each change (C is announced for the whole session and does not change during the session).", + "Whoever bids highest wins Good 1 and pays an amount equal to the second-highest bid; whoever bids second-highest wins Good 2 and pays alpha times the third-highest bid; whoever bids lowest wins nothing and pays no purchase price (but still pays any bid-adjustment costs already incurred).", + "You may keep revising your bid until the match ends. You will not be told when the match will end in advance; each match lasts at least a minimum amount of time and then may end at any moment. You cannot hear or see other participants change their bids.", + "You will be paid, in cash, based on the outcome of one randomly chosen moment within the match (not necessarily the final moment), summed across all 10 matches." + ] + }, + { + "id": "mat.human_main.practice_structure", + "title": "Practice / comprehension-check structure before paid matches", + "evidence_label": "reported", + "evidence_id": "ev.human_main.training", + "content": "Before real matches: an experimenter reads the instructions handout aloud; participants complete 6 sequential practice tasks on placing/modifying a bid (must answer correctly to advance); participants work through 5 randomly generated example scenarios practicing bid placement/revision against hypothetical opponent bids; calculator, pen, and paper are available throughout." + }, + { + "id": "mat.human_main.interface_description", + "title": "On-screen interface elements (from Figure 2)", + "evidence_label": "reported", + "evidence_id": "ev.human_main.interface", + "content": "Screen shows, per Figure 2's five labeled callouts: (1) that match's information (private value, alpha) shown before the match starts; (2) the participant's current bid/action and the live outcome and payoff associated with it, updated as group members act; (3) a slider/scale on the left used to place and revise the bid; (4) blank placeholders for the other two participants' scales, revealed only once the match ends; (5) an on-screen reminder of the rules (e.g., that alpha is common knowledge).", + "note": "The exact pixel layout and any additional on-screen text beyond these five captioned elements is not machine-readable from the PDF screenshot; see audit entry AUD-02." + }, + { + "id": "mat.human_main.silent_protocol_note", + "title": "Silent bid-adjustment protocol", + "evidence_label": "reported", + "evidence_id": "ev.human_main.silent_protocol", + "content": "To prevent other participants' mouse-click sounds from leaking information about when someone adjusted a bid, participants adjusted bids by moving the mouse back and forth across the scale's border rather than clicking; physical dividers also blocked visual cues between adjacent terminals." + }, + { + "id": "mat.human_main.common_knowledge_reminder", + "title": "Common-knowledge announcement of alpha", + "evidence_label": "reported", + "evidence_id": "ev.human_main.auction_mechanics", + "content": "Experimenters explicitly announced, and kept an on-screen reminder, that alpha (the value of Good 2 relative to Good 1) is the same for every participant in the group for that match, so that this fact is common knowledge among group members." + } + ] + }, + + "human_mechanism": { + "study_name": "Discrete-time / information-cost follow-up experiment (Section 5)", + "conditions": { + "factorial_2x2": { + "factor_1": { "variable": "C (bid-adjustment cost)", "levels": [0.0, 0.1] }, + "factor_2": { "variable": "information cost (access to counterfactual-payoff plot)", "levels": ["free", "costly ($0.5 per period accessed)"] } + }, + "fixed": { "alpha": 0.5 } + }, + "materials": [ + { + "id": "mat.human_mechanism.overview_text", + "title": "Task framing given to participants", + "evidence_label": "reported", + "evidence_id": "ev.human_mechanism.recruitment", + "content": "Same participant-facing GSP-as-match framing as human_main (new, non-overlapping subject pool, 183 participants), but implemented in oTree with a discrete sequence of decision periods instead of continuous time, and a fixed alpha=0.5 in all four treatments." + }, + { + "id": "mat.human_mechanism.instructions_derived", + "title": "Participant instructions (derived reconstruction; verbatim oTree screens are missing)", + "evidence_label": "derived", + "derivation": "Reconstructed strictly from Section 5's procedural description (evidence ids ev.human_mechanism.design, ev.human_mechanism.info_tool, ev.human_mechanism.discrete_time_protocol). The verbatim oTree instructions/decision screens (Online Appendix N.1) are unavailable to this package (see audit entry AUD-03).", + "content": [ + "You will take part in several matches. In each match you are grouped with two other participants and each of you privately knows only your own value for Good 1 (Good 2 is worth alpha=0.5 times Good 1's value for everyone this session).", + "Each match has an uncertain number of decision periods. In each period you have unlimited time to submit a bid or change your previous bid (or leave it unchanged); once everyone in your group has submitted, you all see that period's outcome before the next period begins.", + "Depending on your session's treatment, changing your bid may cost C per change (as in the main experiment), and/or you may be offered a plot showing what your payoff would have been for every possible bid, holding the other two participants' previous bids fixed; viewing this plot in a period may cost $0.5, or may be free, depending on your treatment.", + "Private values are redrawn independently at random for you at the start of every match (they are not matched to other participants' values as in the main experiment's later matches)." + ] + }, + { + "id": "mat.human_mechanism.info_tool_description", + "title": "Counterfactual-information tool", + "evidence_label": "reported", + "evidence_id": "ev.human_mechanism.info_tool", + "content": "An interactive plot showing the participant's possible payoff for any bid they might place, holding the other two group members' bids at the values observed in the previous period. Screenshots of this tool are in Online Appendix N.2, which is unavailable to this package." + } + ] + }, + + "computational_context": { + "study_name": "Q-learning simulations (Section 3) and calibrated follow-up simulations (Section 6)", + "note": "Not human-facing materials; included so the runnable task's payoff/environment logic (task/adapter.py) matches the paper's own model rather than an invented one.", + "materials": [ + { + "id": "mat.comp.payoff_formulas", + "title": "GSP environment and payoff formulas reused by the runnable task", + "evidence_label": "verbatim", + "evidence_id": "ev.comp.payoff.formula", + "content": "pi_j = alpha_k*(v_j - b_(k+1)) - C_j; Psi = sum_k(alpha_k*v_(k)) / sum_j(alpha_j*v^j); Omega_j = b_j / v_j." + } + ] + } +} diff --git a/studies/study_016/source/evidence.json b/studies/study_016/source/evidence.json new file mode 100644 index 0000000..9b28aee --- /dev/null +++ b/studies/study_016/source/evidence.json @@ -0,0 +1,278 @@ +{ + "_purpose": "Complete evidence and provenance record. Every substantive claim or material used elsewhere in this package (study.json, materials/materials.json, task/task.json) points back to an entry here by its 'id'. 'source_location' always refers to input/paper.pdf (page numbers as printed in the running header/footer of the PDF, which pdftotext preserves). evidence_label is one of verbatim | reported | derived | missing, per the package builder's evidence scheme.", + "studies": [ + { + "study_id": "comp_main", + "name": "Agent-based Q-learning simulation of GSP with frictions (Section 3)", + "is_human_subject_study": false, + "role_in_package": "Provides the theoretical/computational auction model that the human-subject experiments (human_main, human_mechanism) instantiate for people. Included for context and because task/adapter.py reuses this paper's own GSP payoff formulas. Not itself a human study and not the target of task/adapter.py's participant-facing procedure." + }, + { + "study_id": "human_main", + "name": "Main laboratory human-subject experiment (Section 4)", + "is_human_subject_study": true, + "role_in_package": "Primary human-subject study modeled by task/task.json and task/adapter.py." + }, + { + "study_id": "human_mechanism", + "name": "Discrete-time / information-cost follow-up human-subject experiment (Section 5)", + "is_human_subject_study": true, + "role_in_package": "Second human-subject study; adapter.py exposes its 2x2 (bid-adjustment cost x information cost) design as an alternate condition set." + }, + { + "study_id": "comp_additional", + "name": "Additional calibrated computational simulations (Section 6)", + "is_human_subject_study": false, + "role_in_package": "Context only (overbidding-restriction and market-size simulations calibrated to human_main data). Not a human study; not implemented as a runnable task." + } + ], + "evidence": [ + { + "id": "ev.comp.env.setup", + "study_id": "comp_main", + "claim": "J=3 advertisers compete for K=2 ad slots; slot-1 click-through rate alpha1=1, slot-2 alpha2=alpha in {0.2,0.5,0.8}, slot-3 alpha3=0; each advertiser has private value v_j ~ U{1,10} redrawn each match/episode; winner of slot k pays the (k+1)-th highest bid; friction cost C is charged whenever an agent changes its bid from t-1 to t.", + "source_location": "p.1440-1442 (Section 3.1-3.2.1); Table 1, p.1441", + "evidence_label": "reported" + }, + { + "id": "ev.comp.payoff.formula", + "study_id": "comp_main", + "claim": "Payoff for advertiser j allotted slot k: pi_j = alpha_k*(v_j - b_(k+1)) - C_j. Allocative efficiency Psi = sum_k(alpha_k * v_(k)) / sum_j(alpha_j * v^j) [realized welfare over max possible welfare]. Bid-to-value ratio Omega_j = b_j / v_j.", + "source_location": "p.1440-1441, Equations (1)-(3)", + "evidence_label": "verbatim", + "notes": "Formulas transcribed as printed; OCR artifacts in the PDF's math typesetting were checked against the surrounding prose description." + }, + { + "id": "ev.comp.qlearning.params", + "study_id": "comp_main", + "claim": "Q-learning parameters: learning rate delta=0.1, discount factor gamma=0.99, Boltzmann exploration parameter lambda=1. M=2000 matches, T=100 periods per match, n=100,000 simulation runs. Treatment grid: C in {0.0, 0.5, 1.0} x alpha in {0.2, 0.5, 0.8}.", + "source_location": "p.1441-1442, Table 1 and Section 3.2.1", + "evidence_label": "verbatim" + }, + { + "id": "ev.comp.result1", + "study_id": "comp_main", + "claim": "Result 1: Lower bid-adjustment costs lead to more exploration (more bid adjustments), for all three value-ranked agent types.", + "source_location": "p.1442, Table 2 and surrounding text", + "evidence_label": "verbatim" + }, + { + "id": "ev.comp.result2to4", + "study_id": "comp_main", + "claim": "Result 2: bid-to-value ratio is higher for lower-valued agents. Result 3: for the lowest-valued agents, bid-to-value ratio increases as friction (C) decreases. Result 4: bid-to-value ratio decreases as alpha increases.", + "source_location": "p.1443, Table 3 and surrounding text", + "evidence_label": "verbatim" + }, + { + "id": "ev.comp.result5", + "study_id": "comp_main", + "claim": "Result 5: allocative efficiency of the market increases as friction costs increase.", + "source_location": "p.1443-1444, Figure 1", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.recruitment", + "study_id": "human_main", + "claim": "138 participants recruited via ORSEE software on the campus of Purdue University; 8 sessions with 15-18 participants each.", + "source_location": "p.1446, Section 4.5", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.treatments", + "study_id": "human_main", + "claim": "Two between-subjects treatments on bid-adjustment cost: costless (C=0.0) and costly (C=0.1). Initial bid in a match is free; every subsequent change to the bid within a match costs C. Three within-subjects levels of alpha (slot-2/slot-1 value similarity): low=0.2, medium=0.5, high=0.8; each group shares the same alpha for the duration of a match.", + "source_location": "p.1444-1445, Section 4.1 and Table 4", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.matches", + "study_id": "human_main", + "claim": "Each session consisted of M=10 matches. At the start of each match, participants were randomly re-grouped into groups of J=3 and stayed in that group until the match ended (regrouping is random each match to avoid systematic learning about specific partners' behavior). Session earnings were the sum of payoffs across all 10 matches.", + "source_location": "p.1445, Section 4.2", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.random_termination", + "study_id": "human_main", + "claim": "Remark 1: random-termination protocol. Each match lasted at least 20 seconds; after that, each second had a 1% chance of ending the match, giving an expected match duration of 2 minutes. The same sequence of random termination seconds was reused across matches within every session (for comparability across sessions), per Online Table M.1 (not available in this PDF).", + "source_location": "p.1445, Section 4.2 and Endnote 16 (p.1451)", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.value_generation", + "study_id": "human_main", + "claim": "For each match, participants were given a randomly drawn private value for good 1 (slot 1); the value of good 2 (slot 2) equaled the value of good 1 multiplied by alpha. Matches 1-4 used unrestricted random draws. Matches 5-10 used common random seeds so that, within a match, all groups across the two cost treatments faced the same three valuations {v1_m, v2_m, v3_m}, and, for each match, at least one group experienced each alpha value. This is Remark 2.", + "source_location": "p.1445, Section 4.2 and Remarks 1-2", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.value_generation.bug", + "study_id": "human_main", + "claim": "A software bug in the common-random-seed generator for matches 5-10 meant the seed incremented by 1 instead of 3, so for each group, two of the three valuations repeated from the previous match and only one was newly drawn (then randomly reassigned within the group), giving ~20% chance of a repeated value across consecutive matches. The bug was identical across all treatments, so the authors judge it does not create a systematic cross-treatment confound, though it may make the reported overbidding/exploration effects conservative (more within-match learning opportunity about the same values).", + "source_location": "p.1451, Endnote 19", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.auction_mechanics", + "study_id": "human_main", + "claim": "At the start of a match each participant submits an initial bid at no cost, then may revise the bid continuously in time during the match (each revision after the first costs C in the costly treatment). If a participant's bid is highest, they win good 1 and pay the second-highest bid, netting (v(1) - b(2)) - C(1). If second-highest, they win good 2 and pay alpha times the third-highest bid, netting alpha*(v(2)-b(3)) - C(2). If lowest, they win no good and pay only any friction cost incurred. Alpha is common knowledge, announced in instructions and reminded on-screen.", + "source_location": "p.1445-1446, Section 4.3", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.payment_mechanism", + "study_id": "human_main", + "claim": "Remark 3: participants were compensated based on the outcome of a single, last, randomly occurring period within the (conceptually infinite-horizon) match, citing Chandrasekhar and Xandri (2022) for incentive-compatibility of this scheme for infinite-horizon dynamic games.", + "source_location": "p.1446, Section 4.3", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.silent_protocol", + "study_id": "human_main", + "claim": "Remark 4: silent protocol. Instead of clicking to submit a new bid (which other participants could hear and use as an information cue), subjects placed/adjusted bids by moving the mouse back and forth across a scale border, keeping the room quiet; physical dividers additionally limited what participants could see of neighbors' screens.", + "source_location": "p.1446-1447, Section 4.4 and Endnote 18", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.interface", + "study_id": "human_main", + "claim": "The experiment ran on a custom author-built interface (screenshot in Figure 2, p.1446) that displayed: (1) match information before the match began; (2) the participant's current action and live outcome, updated as all three group members act; (3) a scale on the left side of the screen used to place/revise the bid; (4) blank scales for the other two participants, revealed only after the match ends; (5) on-screen reminders of the rules (e.g., common knowledge of alpha).", + "source_location": "p.1446, Figure 2 caption", + "evidence_label": "reported", + "notes": "The figure itself is a screenshot image; exact on-screen wording/layout beyond the caption's five labeled elements is not machine-readable from this PDF and is recorded as missing in audit/missing_information.json." + }, + { + "id": "ev.human_main.training", + "study_id": "human_main", + "claim": "Before the paid matches, subjects: (1) received a printed instructions handout (Online Appendix L) read aloud by an experimenter for common knowledge; (2) completed 6 practice tasks on placing/modifying bids, advancing only after each was completed correctly; (3) worked through 5 randomly generated examples practicing bid placement/revision against hypothetical opponent actions; (4) had a calculator, pen and paper available throughout instructions and the experiment. These steps took about 20 minutes; before each match subjects reviewed that match's info and placed the initial bid only when ready; the 10 matches took about 30 minutes; subjects were paid in cash afterward.", + "source_location": "p.1447, Section 4.5", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.instructions_text", + "study_id": "human_main", + "claim": "The verbatim instructions handout read to participants is Online Appendix L, which is not included in the uploaded PDF (main text only).", + "source_location": "p.1447, Section 4.5 ('Online Appendix L')", + "evidence_label": "missing" + }, + { + "id": "ev.human_main.result6", + "study_id": "human_main", + "claim": "Result 6: number of bid adjustments in the C=0.0 treatment is roughly an order of magnitude greater than in C=0.1; lowest-valued participants explore (adjust) more than medium- and highest-valued participants. Table 5 gives means with bootstrapped SEs and two-tailed permutation-test significance stars for matches 5-10.", + "source_location": "p.1447, Table 5 and Section 4.6.1", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.result7to8", + "study_id": "human_main", + "claim": "Result 7: bid-to-value ratio is higher for lower-valued participants (lowest-valued substantially overbid relative to others). Result 8: for the lowest-valued participants, the bid-to-value ratio is significantly higher when C=0.0 than C=0.1 (increases as friction decreases). Table 6 gives means/SEs and significance by alpha x C cell.", + "source_location": "p.1448, Table 6 and Section 4.6.2", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.result9", + "study_id": "human_main", + "claim": "Result 9: bid-to-value ratio decreases as alpha increases, for medium- and highest-valued agents (partial support: significant in 4 of 6 comparisons; the other two are directionally consistent but not significant, both for lower-valued agents).", + "source_location": "p.1448, Section 4.6.2", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_main.result10", + "study_id": "human_main", + "claim": "Result 10: allocative efficiency is significantly higher when C=0.1 than C=0.0 (Table 7: average efficiency 0.94 at C=0.0 vs 0.98 at C=0.1, computed on the last bid subjects placed in each match, unit of observation = group of 3, matches 5-10).", + "source_location": "p.1448-1449, Table 7 and Section 4.6.3", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_mechanism.rationale", + "study_id": "human_mechanism", + "claim": "Motivation: because continuous-time bidding, immediate feedback, and the fact that adjusting one's own bid was the only way to learn about counterfactual outcomes might have encouraged excessive exploration in the main experiment, the authors ran a second experiment making learning information available by design (an interactive counterfactual-payoff plot) and switching to a discrete-time protocol, to test robustness and isolate mechanism.", + "source_location": "p.1449, Section 5 (opening paragraphs)", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_mechanism.recruitment", + "study_id": "human_mechanism", + "claim": "183 new participants recruited on the campus of Purdue University, none of whom had participated in the original (human_main) experiment. Programmed in oTree software (Chen, Schonger, and Wickens 2016).", + "source_location": "p.1449, Section 5", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_mechanism.design", + "study_id": "human_mechanism", + "claim": "2x2 factorial design: bid-adjustment cost C in {0.0, 0.1} (as in human_main) crossed with counterfactual-information cost: free vs. costly ($0.5 charged every period the subject accessed an interactive counterfactual-payoff plot). Alpha fixed at 0.5 for all four treatments (not varied, unlike human_main).", + "source_location": "p.1449, Section 5 (design paragraph) and Table with 'Regarding the design for new experiments...'", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_mechanism.info_tool", + "study_id": "human_mechanism", + "claim": "The purchasable information was an interactive plot displaying possible payoffs for any own bid, conditional on the other two group members holding the bids they used in the previous period (screenshots in Online Appendix N.2, not included in this PDF).", + "source_location": "p.1449, Section 5", + "evidence_label": "reported" + }, + { + "id": "ev.human_mechanism.differences_from_main", + "study_id": "human_mechanism", + "claim": "Differences from human_main: (i) discrete-time setup instead of continuous-time; (ii) different duration/number of decision periods; (iii) multiple duration-sequence realizations (4 sequences, Online Table N.1, not included); (iv) private values drawn independently at random for each subject in each match, rather than fixed/matched across groups as in human_main matches 5-10.", + "source_location": "p.1449, Section 5", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_mechanism.discrete_time_protocol", + "study_id": "human_mechanism", + "claim": "In the discrete-time setting, subjects faced an uncertain number of decision periods; within each period they had unlimited time to make a desired bid adjustment (if any); once all group members submitted a decision for the period, all saw that period's outcome. The expected number of periods was reduced to 10 (termination probability 0.1 per period, vs. the implicit ~1% per second / ~120 one-second-periods-in-expectation of human_main) to keep session length within lab time constraints. Four different period-length sequences were used (Online Table N.1) so that no single sequence realization would bias results.", + "source_location": "p.1449, Section 5", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_mechanism.instructions_text", + "study_id": "human_mechanism", + "claim": "Instructions and decision screens for this experiment are presented in Online Appendix N.1, which is not included in the uploaded PDF (main text only).", + "source_location": "p.1449, Section 5", + "evidence_label": "missing" + }, + { + "id": "ev.human_mechanism.result_adjustments", + "study_id": "human_mechanism", + "claim": "Consistent with human_main and comp_main: number of bid adjustments significantly greater when C=0.0 than C=0.1; also significantly greater when information is costly than when it is free (Table 8, second half of the experiment).", + "source_location": "p.1449-1450, Table 8 and Section 5 discussion", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_mechanism.result_bidtovalue", + "study_id": "human_mechanism", + "claim": "Lowest-valued agents again show the highest bid-to-value ratio; in the costly-information/no-bid-cost cell they significantly overbid. When information costs are present, bid-adjustment costs substantially mitigate overbidding (Table 9).", + "source_location": "p.1450, Table 9 and Section 5.6.2 discussion", + "evidence_label": "verbatim" + }, + { + "id": "ev.human_mechanism.result_efficiency", + "study_id": "human_mechanism", + "claim": "Across both free- and costly-information treatments, bid-adjustment costs improve allocative efficiency (Table 10: e.g., NoInfoCost 0.960->0.982, InfoCost 0.943->0.968 as C goes 0.0->0.1); the free-information effect is directionally consistent but weaker in significance than the costly-information effect.", + "source_location": "p.1450, Table 10 and Section 5 discussion", + "evidence_label": "verbatim" + }, + { + "id": "ev.comp_additional.restriction_sim", + "study_id": "comp_additional", + "claim": "A computational experiment restricting the lowest-valued agent to never overbid (b3 <= v3), holding alpha=0.5, C=0.5, J=3, K=2, raises allocative efficiency from 0.951 to 0.958 (about 5% of the relevant range, since random-bidding efficiency floor is ~0.85).", + "source_location": "p.1450, Section 6 and Endnote 23", + "evidence_label": "verbatim" + }, + { + "id": "ev.comp_additional.market_size_sim", + "study_id": "comp_additional", + "claim": "Simulations calibrated to human_main data (delta=0.05, lambda=0.9 calibrated from experimental data; alpha=0.5, K=2, C=0.5 for the bid-to-value part) varying the number of advertisers J from 3 to 6 show: (i) allocative efficiency decreases as J increases; (ii) lowest-valued agents overbid more; (iii) that overbidding cascades into higher-valued advertisers' bids; the core result (efficiency increases with friction cost) holds across market sizes; efficiency does not change linearly with J (Figure 3).", + "source_location": "p.1451, Section 6 and Figure 3 caption", + "evidence_label": "verbatim" + }, + { + "id": "ev.limitations", + "study_id": "human_main", + "claim": "Author-stated limitations: (1) slot rank determined only by bid, ignoring real-world quality-score factors (partially addressed for ad-quality in Online Appendix E); (2) all advertisers assumed to face the same bid-adjustment cost, ignoring heterogeneity; (3) the paper studies two endpoints only (fully computational vs. fully human decision-making), not mixed human+AI-tool bidding; (4) focus is specifically on bid-adjustment cost as the friction, not participation or model-evaluation/estimation costs, and only in the online-ad-market context (not e-commerce, dating, ride-sharing, etc.).", + "source_location": "p.1451-1452, Section 7 (Conclusion) and Endnote 24", + "evidence_label": "verbatim" + } + ] +} diff --git a/studies/study_016/source/paper_metadata.json b/studies/study_016/source/paper_metadata.json new file mode 100644 index 0000000..edc9e60 --- /dev/null +++ b/studies/study_016/source/paper_metadata.json @@ -0,0 +1,76 @@ +{ + "_notes": "Bibliographic metadata transcribed from the first page and header/footer of the uploaded PDF (input/paper.pdf). Evidence label for each field is given alongside the value.", + "title": { + "value": "Analyzing Frictions in Generalized Second-Price Auction Markets", + "evidence": "verbatim", + "source": "input/paper.pdf p.1437 (title)" + }, + "authors": { + "value": [ + { "name": "Karthik Kannan", "affiliation": "Eller College of Management, University of Arizona" }, + { "name": "Vandith Pamuru", "affiliation": "Indian School of Business, Hyderabad" }, + { "name": "Yaroslav Rosokha", "affiliation": "Krannert School of Management, Purdue University" } + ], + "evidence": "verbatim", + "source": "input/paper.pdf p.1437 (byline)" + }, + "journal": { + "value": "Information Systems Research", + "evidence": "verbatim", + "source": "input/paper.pdf p.1437 (masthead)" + }, + "volume_issue_pages": { + "value": "Vol. 34, No. 4, December 2023, pp. 1437-1454", + "evidence": "verbatim", + "source": "input/paper.pdf p.1437 (masthead)" + }, + "issn": { + "value": "1047-7047 (print), 1526-5536 (online)", + "evidence": "verbatim", + "source": "input/paper.pdf p.1437 (masthead)" + }, + "doi": { + "value": "10.1287/isre.2022.1187", + "evidence": "verbatim", + "source": "input/paper.pdf p.1437 (masthead)" + }, + "publisher": { + "value": "INFORMS (Institute for Operations Research and the Management Sciences)", + "evidence": "verbatim", + "source": "input/paper.pdf p.1437, p.1539 (copyright notice)" + }, + "dates": { + "value": { + "received": "August 9, 2019", + "revised": "December 21, 2020; May 31, 2022; September 26, 2022", + "accepted": "October 12, 2022", + "published_online": "December 14, 2022" + }, + "evidence": "verbatim", + "source": "input/paper.pdf p.1437 (masthead)" + }, + "keywords": { + "value": [ + "auctions", + "generalized second-price auctions", + "human-subject experiments", + "Q-learning", + "machine learning", + "reinforcement learning" + ], + "evidence": "verbatim", + "source": "input/paper.pdf p.1437" + }, + "copyright": { + "value": "Copyright 2023 by INFORMS, all rights reserved. Users may print, download, or email articles for individual use; may not be copied or emailed to multiple sites or posted to a listserv without the copyright holder's express written permission.", + "evidence": "verbatim", + "source": "input/paper.pdf p.1539 (footer)" + }, + "supplemental_material_statement": { + "value": "The masthead states: 'Supplemental Material: The online appendix is available at https://doi.org/10.1287/isre.2022.1187.' This URL was NOT retrieved for this package (see external_sources policy below). The online appendix is referenced throughout the main text (e.g., Online Appendices A-N) as the location of Q-learning implementation/robustness details, ad-quality extensions, calibration details, match-duration tables, and the two experiment instruction handouts (Online Appendix L for the main experiment, Online Appendix N.1 for the discrete-time/oTree experiment).", + "evidence": "verbatim", + "source": "input/paper.pdf p.1437 masthead; multiple in-text citations, e.g. p.1441 ('Online Appendix A'), p.1445 ('Online Appendix C'), p.1447 ('Online Appendix L'), p.1449 ('Online Appendix N.1'), p.1451 ('Online Appendix E')" + }, + "external_sources": [], + "external_sources_policy": "No OSF or open-materials URL was supplied by the contributor in job.json for this job. Per the package-builder's external-research policy, no web search, web fetch, DOI resolution, or repository/author-page discovery was performed. The DOI and supplemental-material URL printed on the paper's masthead were read as text but were never fetched or followed. This package is built entirely from input/paper.pdf." +} diff --git a/studies/study_016/study.json b/studies/study_016/study.json new file mode 100644 index 0000000..9f0722b --- /dev/null +++ b/studies/study_016/study.json @@ -0,0 +1,108 @@ +{ + "_purpose": "Researcher-facing overview of this package. Points into source/evidence.json for provenance, materials/materials.json for participant-visible content, and task/ + evaluation/ for the runnable contract.", + "paper": { + "title": "Analyzing Frictions in Generalized Second-Price Auction Markets", + "authors": [ + "Karthik Kannan", + "Vandith Pamuru", + "Yaroslav Rosokha" + ], + "venue": "Information Systems Research, Vol. 34, No. 4, December 2023, pp. 1437-1454", + "doi": "10.1287/isre.2022.1187", + "source_file": "input/paper.pdf", + "metadata_file": "source/paper_metadata.json" + }, + "external_sources_consulted": [], + "external_sources_policy_note": "No OSF or open-materials URL was supplied by the contributor for this job. Per the package-builder's opt-in policy, no web search, web fetch, DOI resolution, or repository/author-page discovery was performed. This entire package is built from input/paper.pdf only. The paper's own online appendix (referenced repeatedly in-text as Online Appendices A-N) was therefore NOT retrieved; see audit/missing_information.json for every place this leaves a gap.", + "empirical_studies": [ + { + "study_id": "comp_main", + "name": "Agent-based Q-learning simulation of the GSP environment with bid-adjustment frictions", + "section": "Section 3", + "is_human_subject_study": false, + "sample": "100,000 simulated runs of M=2,000 matches x T=100 periods per match, J=3 Q-learning agents per match competing for K=2 ad slots", + "conditions": "Full factorial C in {0.0, 0.5, 1.0} x alpha in {0.2, 0.5, 0.8}", + "key_findings": [ + "Result 1: lower bid-adjustment costs lead to more exploration (more bid adjustments).", + "Result 2: bid-to-value ratio is higher for lower-valued agents.", + "Result 3: for the lowest-valued agents, bid-to-value ratio increases as friction decreases.", + "Result 4: bid-to-value ratio decreases as alpha increases.", + "Result 5: allocative efficiency of the market increases as friction costs increase." + ], + "role_in_package": "Supplies the GSP environment/payoff formulas that task/adapter.py reuses; not itself modeled as a runnable human-study task." + }, + { + "study_id": "human_main", + "name": "Main laboratory human-subject experiment", + "section": "Section 4", + "is_human_subject_study": true, + "sample": "138 participants, 8 lab sessions of 15-18 participants each, recruited via ORSEE at Purdue University", + "conditions": "Between-subjects: bid-adjustment cost C in {0.0 (costless), 0.1 (costly)}. Within-subjects: alpha (slot-2/slot-1 value ratio) in {0.2 (low), 0.5 (medium), 0.8 (high)}. 10 matches per session, groups of J=3 re-randomized each match.", + "participant_flow": "Handout instructions read aloud -> 6 sequential practice tasks (must pass to advance) -> 5 randomized worked examples -> 10 real paid matches (continuous-time bidding within each match, silent mouse-based bid revision, random termination, single randomly-drawn-period payment) -> cash payment.", + "outcomes_measured": [ + "Number of bid adjustments per participant per match (exploration), by value rank", + "Bid-to-value ratio (Omega_j = bid/value) per participant per match, by value rank", + "Allocative efficiency (Psi) of the match (realized welfare / max possible welfare), by treatment" + ], + "key_findings": [ + "Result 6: bid adjustments are an order of magnitude higher at C=0.0 than C=0.1; lowest-valued participants adjust most.", + "Result 7: lowest-valued participants substantially overbid relative to others.", + "Result 8: lowest-valued participants' bid-to-value ratio is significantly higher at C=0.0 than C=0.1.", + "Result 9: bid-to-value ratio decreases as alpha increases, for medium/highest-valued participants (partial support for the analogous computational result).", + "Result 10: allocative efficiency is significantly higher at C=0.1 (0.98) than C=0.0 (0.94)." + ], + "role_in_package": "Primary human-subject study modeled by task/task.json + task/adapter.py (condition set 'human_main')." + }, + { + "study_id": "human_mechanism", + "name": "Discrete-time / information-cost follow-up experiment ('Experiment to Study the Underlying Mechanism')", + "section": "Section 5", + "is_human_subject_study": true, + "sample": "183 new participants (no overlap with human_main), recruited at Purdue University, programmed in oTree", + "conditions": "2x2 factorial: bid-adjustment cost C in {0.0, 0.1} x counterfactual-information cost in {free, $0.5/period}. Alpha fixed at 0.5 for all four cells (not varied). Discrete-time decision periods (expected 10 per match, termination probability 0.1/period) instead of continuous time; private values drawn independently each match (not matched across groups/treatments as in human_main's later matches).", + "participant_flow": "oTree-based instructions and decision screens (Online Appendix N.1, not available in this PDF) -> repeated matches with discrete decision periods, optional purchase of a counterfactual-payoff plot each period depending on treatment -> payment.", + "outcomes_measured": [ + "Number of bid adjustments per participant per match, by value rank and treatment cell", + "Bid-to-value ratio per participant per match, by value rank and treatment cell", + "Allocative efficiency of the match, by treatment cell" + ], + "key_findings": [ + "Bid adjustments are significantly higher when C=0.0 than C=0.1, and higher when information is costly than when it is free.", + "Lowest-valued participants again show the highest bid-to-value ratio, especially with costly information and no bid-adjustment cost; bid-adjustment costs substantially mitigate this overbidding when information is costly.", + "Allocative efficiency improves with bid-adjustment costs in both the free- and costly-information conditions (weaker significance in the free-information condition)." + ], + "role_in_package": "Second human-subject study; modeled by task/task.json + task/adapter.py (condition set 'human_mechanism')." + }, + { + "study_id": "comp_additional", + "name": "Additional calibrated computational simulations", + "section": "Section 6", + "is_human_subject_study": false, + "sample": "Computational simulations, Q-learning parameters (delta, lambda) calibrated to human_main data", + "conditions": "(a) Lowest-valued agent restricted to never overbid (b3<=v3), alpha=0.5, C=0.5, J=3, K=2. (b) Number of advertisers J varied over {3,4,5,6}, alpha=0.5, C=0.5, K=2.", + "key_findings": [ + "(a) Restricting the lowest-valued agent from overbidding raises allocative efficiency from 0.951 to 0.958 (~5% of the relevant range).", + "(b) As the number of advertisers increases, allocative efficiency decreases, lowest-valued agents overbid more, and this cascades into higher-valued advertisers' bids; the core friction-increases-efficiency result holds across market sizes." + ], + "role_in_package": "Context/robustness only; not implemented as a runnable task (no new human-subject data)." + } + ], + "package_entry_point": { + "runnable_task": "task/adapter.py", + "task_definition": "task/task.json", + "evaluation": "evaluation/evaluation.py", + "smoke_test_command": "python3 task/adapter.py --smoke-test", + "example_full_run": "python3 task/adapter.py --study human_main --cost 0.1 --alpha 0.5 --num-matches 10 --agent-policy truthful --seed 1 --output run.json && python3 evaluation/evaluation.py run.json" + }, + "readiness_status": { + "overall": "runnable_with_documented_limitations", + "summary": "task/adapter.py runs fully offline, implements both human-subject studies' condition sets and the paper's own GSP payoff formulas, and passes --smoke-test. It cannot reproduce the paper's exact continuous-time UI, the verbatim instructions handouts, or its cross-session inferential statistics (bootstrapped SEs / permutation tests), because those depend on materials or raw session data not present in the uploaded PDF and not authorized for external retrieval on this job. See audit/missing_information.json for the itemized list (AUD-01 through AUD-10) and task/task.json#derived_design_decisions for the specific simplifications made to keep the task runnable.", + "blocking_for_full_paper_replication": true, + "blocking_for_descriptive_agent_evaluation": false + }, + "contributors": [ + { + "name": "Yuanjun" + } + ] +} diff --git a/studies/study_016/task/adapter.py b/studies/study_016/task/adapter.py new file mode 100644 index 0000000..adb7b84 --- /dev/null +++ b/studies/study_016/task/adapter.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +""" +Runnable adapter for the GSP bid-adjustment-friction task. + +Models the two human-subject studies from Kannan, Pamuru, and Rosokha (2023), +"Analyzing Frictions in Generalized Second-Price Auction Markets", +Information Systems Research 34(4):1437-1454. + +- study "human_main": Section 4 main experiment (between-subjects cost C in + {0.0, 0.1}; within-subjects alpha in {0.2, 0.5, 0.8}). +- study "human_mechanism": Section 5 discrete-time / information-cost + follow-up (2x2: cost C in {0.0, 0.1} x info_cost in {0.0, 0.5}; alpha fixed + at 0.5). + +See task/task.json for the full I/O contract, condition tables, and the +documented derived design decisions (continuous time is discretized into +decision checkpoints; the other two group members each match are simulated +policies, not replayed human data -- see audit/missing_information.json). + +Runs fully offline (stdlib only, no network access) and exposes --smoke-test. +""" + +from __future__ import annotations + +import argparse +import json +import random +import sys +from dataclasses import dataclass, field +from typing import Callable, Optional + + +# --------------------------------------------------------------------------- +# Environment model (payoff formulas verbatim from the paper, Section 3.1, +# Equations 1-3; see source/evidence.json#ev.comp.payoff.formula) +# --------------------------------------------------------------------------- + +ALPHA1 = 1.0 # click-through rate of slot 1, fixed by the paper (Section 3.1) + + +@dataclass +class Condition: + study: str # "human_main" or "human_mechanism" + cost: float # bid-adjustment cost C + alpha: float # value of Good 2 relative to Good 1 + info_cost: Optional[float] = None # human_mechanism only + + def validate(self) -> None: + if self.study not in ("human_main", "human_mechanism"): + raise ValueError(f"unknown study '{self.study}'") + if self.study == "human_main" and self.cost not in (0.0, 0.1): + raise ValueError("human_main defines cost in {0.0, 0.1} (ev.human_main.treatments)") + if self.study == "human_main" and self.alpha not in (0.2, 0.5, 0.8): + raise ValueError("human_main defines alpha in {0.2, 0.5, 0.8} (ev.human_main.treatments)") + if self.study == "human_mechanism": + if self.cost not in (0.0, 0.1): + raise ValueError("human_mechanism defines cost in {0.0, 0.1} (ev.human_mechanism.design)") + if self.info_cost not in (0.0, 0.5): + raise ValueError("human_mechanism defines info_cost in {0.0, 0.5} (ev.human_mechanism.design)") + if self.alpha != 0.5: + raise ValueError("human_mechanism fixes alpha = 0.5 (ev.human_mechanism.design)") + + +PolicyFn = Callable[[dict, random.Random], float] + + +def policy_truthful(state: dict, rng: random.Random) -> float: + """Bid = own value. The weakly-dominant theoretical benchmark the paper + itself invokes for the lowest-valued bidder (ev.comp.env.setup). Used as + the default co-player and as the smoke-test agent-under-test stand-in.""" + return float(state["own_value_good1"]) + + +def policy_softmax_explorer(state: dict, rng: random.Random) -> float: + """Stylized Boltzmann explorer parameterized like the paper's OWN + Q-learning agents (delta/gamma/lambda from Table 1, + ev.comp.qlearning.params). This is a stress-test opponent provided for + runnability -- it is NOT a claim about real human bidding, which the + paper does not release at the individual-trajectory level. + + Candidates near the current bid get a cheap-proxy payoff of + -|candidate - value|; any candidate that differs from the current bid + additionally pays the session's bid_adjustment_cost, so -- consistent + with the paper's own qualitative logic (Result 1/6) -- higher cost makes + this policy adjust its bid less often.""" + value = state["own_value_good1"] + current = state["current_own_bid"] + cost = state["bid_adjustment_cost"] + base = current if current is not None else value + candidates = [max(0.0, min(10.0, base + step)) for step in (-2.0, -1.0, 0.0, 1.0, 2.0)] + lam = 1.0 + payoffs = [] + for c in candidates: + p = -(abs(c - value)) + if current is not None and c != current: + p -= cost + payoffs.append(p) + weights = [pow(2.718281828, lam * p) for p in payoffs] + total = sum(weights) + r = rng.random() * total + acc = 0.0 + for c, w in zip(candidates, weights): + acc += w + if r <= acc: + return round(c, 2) + return candidates[-1] + + +BUILTIN_POLICIES = { + "truthful": policy_truthful, + "softmax_explorer": policy_softmax_explorer, +} + + +@dataclass +class ParticipantLog: + values_seen: list = field(default_factory=list) + bids_history: list = field(default_factory=list) + num_adjustments: int = 0 + num_info_purchases: int = 0 + payoff: float = 0.0 + + +def _decision_schedule(study: str, rng: random.Random) -> int: + """Draw the number of decision checkpoints for one match. + + human_main: derived discretization of the continuous-time / random- + termination protocol (min 20s, 1%/s thereafter, expected 2 minutes) -- + see task.json derived_design_decisions D1 and ev.human_main.random_termination. + Capped at 100 checkpoints (matches T=100 in the paper's own computational + model, ev.comp.qlearning.params) purely to bound runtime. + + human_mechanism: reported discrete-time protocol -- termination + probability 0.1 per period, no stated minimum, expected 10 periods + (ev.human_mechanism.discrete_time_protocol). Exact duration-sequence + realizations (Online Table N.1) are unavailable; periods are drawn fresh + (task.json derived_design_decisions D3). + """ + if study == "human_main": + periods = 20 + while periods < 100 and rng.random() > 0.01: + periods += 1 + return periods + else: + periods = 1 + while periods < 200 and rng.random() > 0.1: + periods += 1 + return periods + + +def run_match(condition: Condition, match_index: int, agent_fn: PolicyFn, + co_player_fn: PolicyFn, rng: random.Random) -> dict: + """Simulate one match: agent-under-test plus two simulated co-players.""" + values = [rng.randint(1, 10) for _ in range(3)] + values.sort(reverse=True) + v_good1 = values # [v_highest, v_medium, v_lowest] for good 1 + alpha = condition.alpha + + participants = ["agent", "co_player_1", "co_player_2"] + fns = {"agent": agent_fn, "co_player_1": co_player_fn, "co_player_2": co_player_fn} + logs = {p: ParticipantLog() for p in participants} + current_bids = {p: None for p in participants} + + n_periods = _decision_schedule(condition.study, rng) + + for period in range(1, n_periods + 1): + for idx, p in enumerate(participants): + state = { + "study": condition.study, + "match_index": match_index, + "period_index": period, + "is_first_decision_in_match": current_bids[p] is None, + "own_value_good1": v_good1[idx], + "alpha": alpha, + "bid_adjustment_cost": condition.cost, + "current_own_bid": current_bids[p], + "info_cost": condition.info_cost if condition.study == "human_mechanism" else None, + "counterfactual_payoff_table": None, + } + new_bid = float(fns[p](state, rng)) + new_bid = max(0.0, min(10.0, new_bid)) + if current_bids[p] is not None and new_bid != current_bids[p]: + logs[p].num_adjustments += 1 + logs[p].payoff -= condition.cost + current_bids[p] = new_bid + logs[p].bids_history.append(new_bid) + logs[p].values_seen.append(v_good1[idx]) + + ranked = sorted(zip(participants, current_bids.values(), v_good1), + key=lambda t: t[1], reverse=True) + (p1, b1, val1), (p2, b2, val2), (p3, b3, val3) = ranked + + payoff1 = ALPHA1 * (val1 - b2) + payoff2 = alpha * (val2 - b3) + payoff3 = 0.0 + logs[p1].payoff += payoff1 + logs[p2].payoff += payoff2 + logs[p3].payoff += payoff3 + + max_welfare = ALPHA1 * v_good1[0] + alpha * v_good1[1] + realized_welfare = ALPHA1 * val1 + alpha * val2 + allocative_efficiency = (realized_welfare / max_welfare) if max_welfare > 0 else None + + bid_to_value = {p: (current_bids[p] / v) if v > 0 else None + for p, v in zip(participants, v_good1)} + + return { + "match_index": match_index, + "n_periods": n_periods, + "values_by_rank": [val1, val2, val3], + "final_bids_by_rank": [b1, b2, b3], + "participant_by_rank": [p1, p2, p3], + "winner_by_rank": [p1, p2, None], + "payoff_by_participant": {p: round(logs[p].payoff, 4) for p in participants}, + "num_bid_adjustments_by_participant": {p: logs[p].num_adjustments for p in participants}, + "bid_to_value_by_participant": bid_to_value, + "allocative_efficiency": allocative_efficiency, + "rank_of_agent": [p1, p2, p3].index("agent") + 1, + } + + +def run_session(condition: Condition, num_matches: int, agent_fn: PolicyFn, + co_player_fn: PolicyFn, seed: int) -> dict: + condition.validate() + rng = random.Random(seed) + match_logs = [run_match(condition, m + 1, agent_fn, co_player_fn, rng) + for m in range(num_matches)] + return { + "condition": { + "study": condition.study, + "cost": condition.cost, + "alpha": condition.alpha, + "info_cost": condition.info_cost, + }, + "num_matches": num_matches, + "seed": seed, + "matches": match_logs, + "fidelity_note": ( + "Co-players are simulated policies, not replayed human data " + "(see task/task.json 'co_players' and audit/missing_information.json AUD-04). " + "Metrics below describe the agent-under-test's behavior in this " + "simulated environment; they are not a claim of statistical " + "equivalence to the paper's human sessions." + ), + } + + +def summarize(run_output: dict) -> dict: + """Descriptive metrics analogous to the paper's Tables 2/5/8 (adjustments), + 3/6/9 (bid-to-value ratio), and Figure 1 / Tables 7/10 (allocative + efficiency), computed for the agent-under-test only.""" + matches = run_output["matches"] + if not matches: + return {"error": "no matches to summarize"} + agent_adjustments = [m["num_bid_adjustments_by_participant"]["agent"] for m in matches] + agent_ratios = [m["bid_to_value_by_participant"]["agent"] for m in matches + if m["bid_to_value_by_participant"]["agent"] is not None] + efficiencies = [m["allocative_efficiency"] for m in matches if m["allocative_efficiency"] is not None] + rank_counts = {1: 0, 2: 0, 3: 0} + for m in matches: + rank_counts[m["rank_of_agent"]] += 1 + + def mean(xs): + return round(sum(xs) / len(xs), 4) if xs else None + + return { + "mean_agent_bid_adjustments": mean(agent_adjustments), + "mean_agent_bid_to_value_ratio": mean(agent_ratios), + "mean_allocative_efficiency": mean(efficiencies), + "agent_rank_distribution": rank_counts, + "num_matches_summarized": len(matches), + } + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def build_arg_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--smoke-test", action="store_true", + help="Run a short offline sanity check across both studies and exit.") + p.add_argument("--study", choices=["human_main", "human_mechanism"], default="human_main") + p.add_argument("--cost", type=float, default=0.0, help="Bid-adjustment cost C.") + p.add_argument("--alpha", type=float, default=0.5, help="alpha (ignored/must be 0.5 for human_mechanism).") + p.add_argument("--info-cost", type=float, default=None, + help="Information cost for human_mechanism (0.0 or 0.5). Required if --study human_mechanism.") + p.add_argument("--num-matches", type=int, default=10) + p.add_argument("--agent-policy", choices=list(BUILTIN_POLICIES.keys()), default="truthful", + help="Built-in stand-in for the agent-under-test (a real harness should inject its own agent_fn).") + p.add_argument("--co-player-policy", choices=list(BUILTIN_POLICIES.keys()), default="truthful") + p.add_argument("--seed", type=int, default=1) + p.add_argument("--output", type=str, default=None, help="Optional path to write full JSON run log.") + return p + + +def run_smoke_test() -> dict: + """Offline, dependency-free sanity check: runs a few matches for each + study's default condition using built-in policies and returns True/False + fitness plus a readiness note. No network access is used or required.""" + results = {} + ok = True + try: + out_main = run_session( + Condition(study="human_main", cost=0.1, alpha=0.5), + num_matches=3, agent_fn=policy_truthful, co_player_fn=policy_softmax_explorer, seed=42, + ) + results["human_main"] = summarize(out_main) + except Exception as exc: # pragma: no cover - smoke test must not crash the CLI + ok = False + results["human_main"] = {"error": str(exc)} + + try: + out_mech = run_session( + Condition(study="human_mechanism", cost=0.0, alpha=0.5, info_cost=0.5), + num_matches=3, agent_fn=policy_truthful, co_player_fn=policy_softmax_explorer, seed=42, + ) + results["human_mechanism"] = summarize(out_mech) + except Exception as exc: # pragma: no cover + ok = False + results["human_mechanism"] = {"error": str(exc)} + + return { + "smoke_test_passed": ok, + "results": results, + "readiness_status": "runnable_with_documented_limitations", + "limitations": ( + "Co-players are simulated (not replayed human data); continuous-time " + "bidding in human_main is discretized into decision checkpoints. " + "See task/task.json#derived_design_decisions and " + "audit/missing_information.json for the full list." + ), + } + + +def main(argv=None) -> int: + args = build_arg_parser().parse_args(argv) + + if args.smoke_test: + print(json.dumps(run_smoke_test(), indent=2)) + return 0 + + if args.study == "human_mechanism" and args.info_cost is None: + print(json.dumps({ + "blocked": True, + "reason": "study=human_mechanism requires --info-cost {0.0 or 0.5} (ev.human_mechanism.design).", + }, indent=2)) + return 1 + + condition = Condition( + study=args.study, + cost=args.cost, + alpha=(0.5 if args.study == "human_mechanism" else args.alpha), + info_cost=args.info_cost, + ) + try: + condition.validate() + except ValueError as exc: + print(json.dumps({"blocked": True, "reason": str(exc)}, indent=2)) + return 1 + + agent_fn = BUILTIN_POLICIES[args.agent_policy] + co_player_fn = BUILTIN_POLICIES[args.co_player_policy] + + run_output = run_session(condition, args.num_matches, agent_fn, co_player_fn, args.seed) + result = {"run": run_output, "summary": summarize(run_output)} + + if args.output: + with open(args.output, "w") as f: + json.dump(result, f, indent=2) + print(json.dumps({"written_to": args.output, "summary": result["summary"]}, indent=2)) + else: + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/studies/study_016/task/task.json b/studies/study_016/task/task.json new file mode 100644 index 0000000..ebb9076 --- /dev/null +++ b/studies/study_016/task/task.json @@ -0,0 +1,108 @@ +{ + "_purpose": "Defines the runnable agent interaction implemented by task/adapter.py: what role the agent-under-test plays, what it is shown, what it must output, how conditions are assigned, and which materials/evidence back each design choice.", + + "task_id": "gsp_bid_adjustment_friction", + "title": "Generalized second-price auction bidding under bid-adjustment frictions", + "studies_modeled": ["human_main", "human_mechanism"], + "entry_point": "task/adapter.py", + "how_to_run": "python3 task/adapter.py --smoke-test # offline sanity check, no network\npython3 task/adapter.py --study human_main --cost 0.1 --alpha 0.5 --num-matches 10 --agent-policy truthful --seed 1\npython3 task/adapter.py --study human_mechanism --cost 0.0 --info-cost 0.5 --num-matches 10 --agent-policy truthful --seed 1", + + "participant_role": { + "description": "The agent-under-test plays the role of ONE of three anonymous group members ('participants' in the paper's own subject-facing language) across a sequence of matches. Each match, the agent is privately shown its own value for Good 1 (Good 2's value is alpha * Good 1's value), and must submit an initial bid and may revise it at successive decision points, subject to the session's bid-adjustment cost C.", + "source": "materials/materials.json#human_main.mat.human_main.instructions_derived and #human_mechanism.mat.human_mechanism.instructions_derived", + "evidence_id": "ev.human_main.auction_mechanics" + }, + + "conditions": { + "human_main": { + "between_subjects": { "cost": [0.0, 0.1] }, + "within_subjects": { "alpha": [0.2, 0.5, 0.8] }, + "group_size_J": 3, + "slots_K": 2, + "matches_per_session": 10, + "evidence_id": "ev.human_main.treatments" + }, + "human_mechanism": { + "factorial_2x2": { "cost": [0.0, 0.1], "info_cost": [0.0, 0.5] }, + "alpha_fixed": 0.5, + "group_size_J": 3, + "slots_K": 2, + "evidence_id": "ev.human_mechanism.design" + } + }, + + "environment_parameters": { + "value_distribution": "U{1,...,10} for Good 1's value, redrawn independently per match per participant (matches 1-4 style in human_main; the seed-matching used by the paper in matches 5-10 is NOT reproduced here -- see derived_design_decisions D2)", + "good2_value": "alpha * value_of_good_1", + "payoff_formula": "winner of slot k pays the (k+1)-th highest bid; payoff = alpha_k*(value - price) - bid_adjustment_costs_incurred_this_match [+/- information_costs_incurred_this_match for human_mechanism]", + "evidence_id": "ev.comp.payoff.formula" + }, + + "io_schema": { + "decision_point_state_shown_to_agent": { + "study": "human_main | human_mechanism", + "match_index": "int, 1-based", + "period_index": "int, 1-based decision checkpoint within the match", + "is_first_decision_in_match": "bool (true => placing the initial bid is free regardless of condition)", + "own_value_good1": "int in [1,10]", + "alpha": "float, value of Good 2 relative to Good 1", + "bid_adjustment_cost": "float, C for this session", + "current_own_bid": "float or null if no bid placed yet", + "info_cost": "float or null; only present for study=human_mechanism", + "counterfactual_payoff_table": "object or null; only present for study=human_mechanism AND the agent chose to purchase it this period (see agent action 'view_info')", + "match_may_end_any_period": true + }, + "agent_action": { + "bid": "number in [0,10], required each period (repeating current_own_bid means no change / no cost)", + "purchase_info": "bool, only meaningful for study=human_mechanism; requests the counterfactual_payoff_table on the NEXT call, at cost info_cost for this period" + }, + "match_end_record": { + "final_bids_by_rank": "[b_highest, b_medium, b_lowest]", + "participant_by_rank": "[participant_id_highest, participant_id_medium, participant_id_lowest]", + "values_by_rank": "[v_highest, v_medium, v_lowest]", + "winner_slot_by_participant": "mapping of participant id -> 1, 2, or null", + "payoff_by_participant": "mapping of participant id -> float", + "num_bid_adjustments_by_participant": "mapping of participant id -> int", + "num_info_purchases_by_participant": "mapping of participant id -> int (human_mechanism only)" + }, + "run_output_metrics": { + "mean_bid_adjustments_by_rank": "as in paper Table 2 / Table 5 / Table 8", + "mean_bid_to_value_ratio_by_rank": "as in paper Table 3 / Table 6 / Table 9", + "mean_allocative_efficiency": "as in paper Figure 1 / Table 7 / Table 10" + } + }, + + "co_players": { + "description": "The other two group members each match are simulated policies, NOT replayed human data (the paper does not release individual human bid trajectories, and this package was built without external/OSF access -- see audit AUD-04). Available built-in policies are documented in task/adapter.py: 'truthful' (bid = value; the theoretical weakly-dominant benchmark the paper itself cites for the lowest-valued bidder, evidence_id ev.comp.env.setup) and 'softmax_explorer' (a stylized Boltzmann explorer parameterized like the paper's OWN Q-learning agents in Section 3.2.1, evidence_id ev.comp.qlearning.params, provided only as a stress-test opponent, not a claim about human behavior).", + "fidelity_caveat": "Because co-players are simulated rather than real humans or a verified behavioral-clone of the paper's subjects, this task can score the agent-under-test's OWN exploration / bid-to-value / responsiveness-to-C behavior, but it cannot be used to reproduce the paper's between-subjects human-vs-human significance tests (Tables 5-10). evaluation.py reflects this: descriptive metrics run; inferential replication returns not_ready." + }, + + "derived_design_decisions": [ + { + "id": "D1", + "decision": "Discretize the continuous-time bidding window (human_main) into a bounded sequence of decision checkpoints instead of true continuous mouse-tracked time, since an LLM-style agent interacts turn-by-turn rather than via continuous mouse position.", + "parameters_used": "min 20 checkpoints, then 1%-per-checkpoint termination probability, capped at 100 checkpoints (mirrors T=100 used in the paper's OWN computational model, ev.comp.qlearning.params), matching the reported EXPECTED duration (2 minutes / ~120 one-second periods) but not the literal continuous silent-mouse protocol (ev.human_main.silent_protocol, ev.human_main.random_termination), which has no turn-based analogue.", + "impact": "Bid-adjustment COUNTS from this task are not directly comparable to Table 2/5 counts; ratios and directional effects of C and alpha are the intended comparison, not absolute adjustment counts.", + "audit_ref": "AUD-05" + }, + { + "id": "D2", + "decision": "Private values are drawn independently at random each match for every simulated run, rather than reproducing the paper's matches-5-10 common-random-seed matching across treatments (ev.human_main.value_generation) -- that scheme exists to let the PAPER compare human sessions apples-to-apples; a single adapter run has no cross-session comparison to protect.", + "impact": "Researchers who want the exact matched-value design for cross-run comparability must supply their own seeding scheme; adapter.py exposes --seed for run-level reproducibility only.", + "audit_ref": "AUD-06" + }, + { + "id": "D3", + "decision": "For study=human_mechanism, the discrete-period termination process (no stated minimum duration, probability 0.1 per period, expected 10 periods) is implemented directly per ev.human_mechanism.discrete_time_protocol, but the paper's own 4 duration-SEQUENCE realizations (Online Table N.1) are not available, so periods are drawn fresh via the RNG rather than replayed from the paper's fixed sequences.", + "impact": "Run-to-run period-count variance will not exactly match the paper's 4 canonical sequences.", + "audit_ref": "AUD-07" + } + ], + + "readiness_status": "runnable_with_documented_limitations", + "materials_used": [ + "../materials/materials.json#human_main.mat.human_main.instructions_derived", + "../materials/materials.json#human_mechanism.mat.human_mechanism.instructions_derived", + "../materials/materials.json#computational_context.mat.comp.payoff_formulas" + ] +}