From 98bc5c733446eb79e77fa88b990047a1d24e38bc Mon Sep 17 00:00:00 2001 From: Xuan Liu Date: Sun, 16 Aug 2026 20:10:59 +0800 Subject: [PATCH] Add study: study_016 --- studies/study_016/README.md | 7 + .../study_016/audit/missing_information.json | 96 ++++ studies/study_016/evaluation/evaluation.py | 340 +++++++++++++ studies/study_016/materials/materials.json | 325 +++++++++++++ studies/study_016/source/evidence.json | 191 ++++++++ studies/study_016/source/paper_metadata.json | 56 +++ studies/study_016/study.json | 97 ++++ studies/study_016/task/adapter.py | 447 ++++++++++++++++++ studies/study_016/task/task.json | 156 ++++++ 9 files changed, 1715 insertions(+) create mode 100644 studies/study_016/README.md create mode 100644 studies/study_016/audit/missing_information.json create mode 100644 studies/study_016/evaluation/evaluation.py create mode 100644 studies/study_016/materials/materials.json create mode 100644 studies/study_016/source/evidence.json create mode 100644 studies/study_016/source/paper_metadata.json create mode 100644 studies/study_016/study.json create mode 100644 studies/study_016/task/adapter.py create mode 100644 studies/study_016/task/task.json diff --git a/studies/study_016/README.md b/studies/study_016/README.md new file mode 100644 index 0000000..f1397ac --- /dev/null +++ b/studies/study_016/README.md @@ -0,0 +1,7 @@ +# Comap Shared Visual Workspace Pbl + +This HumanStudy-Hub package was reconstructed from a published paper. + +- Review `study.json` for the study overview and readiness status. +- Review `audit/missing_information.json` before running the study. +- Run `python task/adapter.py --smoke-test` to check the package entry point. diff --git a/studies/study_016/audit/missing_information.json b/studies/study_016/audit/missing_information.json new file mode 100644 index 0000000..8b327da --- /dev/null +++ b/studies/study_016/audit/missing_information.json @@ -0,0 +1,96 @@ +{ + "_explanation": "Authoritative researcher checklist. Every entry names a gap between what the paper prints and what is needed for a fully faithful runnable replication. Nothing in this file has been filled with invented content; task/adapter.py and materials/materials.json point to these entry names wherever a placeholder or best-effort default was used instead.", + "entries": [ + { + "study": "study_2_main", + "field": "baseline_system_prompt", + "reason": "The paper states the Baseline condition used 'a standard, dialogue-based AI' (Section 5.2, page 11) but never prints its system prompt, model configuration, or turn-taking behavior, unlike the CoMAP agents which are given verbatim in Appendix C.", + "impact": "The Baseline arm of the crossover cannot be executed with any confidence that it matches the original manipulation. Any invented prompt would silently bias the comparison against or in favor of CoMAP. task/adapter.py refuses to run the Baseline arm unless a researcher explicitly supplies a system prompt file.", + "suggested_action": "Contact the authors (or check their code repository, if the researcher independently obtains authorization to do so) for the exact Baseline system prompt and model settings, then pass it via --baseline-system-prompt-file. Until then, treat any Baseline-condition run as non-replicating." + }, + { + "study": "study_2_main", + "field": "baseline_persistent_document_template", + "reason": "The paper says Baseline participants 'were provided with a separate document to serve as a persistent record for their design process' (Section 5.2, page 11) but does not print or describe the document's template, structure, or headings.", + "impact": "Without the template, the Baseline arm's 'persistent record' condition cannot be faithfully distinguished from an unstructured scratchpad, which may itself be a meaningful confound (the paper frames Baseline's lack of a structured, graph-based representation as the key contrast with CoMAP).", + "suggested_action": "Ask the authors for the document template, or have the researcher design and pre-register a plain, non-graph-based document template (e.g., a linear outline with the same six ASSURE categories) and record that choice explicitly rather than leaving it implicit." + }, + { + "study": "study_1_and_2", + "field": "questionnaire_response_format", + "reason": "No Likert scale point count or verbal anchors are printed for the Intelligent-TPACK background scale or any of the post-task scales (Perceived Understanding, Perceived Expression, Controllable, Transparent, Cognitive Load, Collaboration, Trust).", + "impact": "Reported means (e.g., M=6.13 for CoMAP Perceived Expression) constrain the scale to have a maximum of at least 7, but the exact number of points and anchor wording are unconfirmed. Using the wrong scale would misrepresent participants' true response options and could shift downstream statistics.", + "suggested_action": "task/task.json and materials/materials.json record a derived default of a 7-point Likert scale (1-7) based on the observed means. A researcher should confirm the exact scale (and verbal anchors, if any) against the original survey instrument/OSF materials, if such materials become available and are explicitly authorized for use." + }, + { + "study": "study_2_main", + "field": "negative_keyword_lexicon", + "reason": "Section 5.4.1 defines the 'Negative Keywords' behavioral metric as 'the total count of negative keywords (e.g., \"stuck,\" \"confused,\" \"can't\") based on a predefined lexicon' but never prints the full lexicon.", + "impact": "evaluation/evaluation.py cannot exactly reproduce this metric's original values; it can only compute a best-effort proxy using the three example words plus researcher-supplied additions, which will not match the paper's M=2.14 (CoMAP) / M=4.16 (Baseline) counts exactly.", + "suggested_action": "Ask the authors for the full lexicon, or have the researcher define and pre-register their own lexicon (documented in evaluation/evaluation.py) before treating this metric's outputs as comparable to the published values." + }, + { + "study": "study_2_main", + "field": "counterbalancing_cell_sizes", + "reason": "The paper states tasks 'were counterbalanced with the conditions to control for task-specific effects' (Section 5.2, page 11) but does not print the resulting cell sizes among the 30 participants (e.g., how many were Group A + Task A first vs. Group A + Task B first).", + "impact": "task/task.json's default counterbalancing scheme (cycling dyad_id through 4 balanced cells) is a reasonable but unverified reconstruction; the true allocation might have been unequal or block-randomized differently.", + "suggested_action": "If the exact allocation matters for the researcher's analysis (e.g., replicating exact subgroup sizes), request the original randomization/allocation table from the authors." + }, + { + "study": "study_2_main", + "field": "canvas_coordinate_assignment", + "reason": "The 'average distance between consecutively created nodes' metric (Section 5.4.1) requires 2D canvas (x, y) coordinates for every node a participant creates, but the paper does not specify how an agent (rather than a mouse-dragging human) should choose where to place a node.", + "impact": "This behavioral metric cannot be computed with the same meaning for a text-based agent unless the agent is given some placement convention; evaluation/evaluation.py returns a not_ready result for this metric until a placement convention is supplied.", + "suggested_action": "Researcher should decide and document a placement convention (e.g., agent must state an (x,y) pair reflecting perceived conceptual distance from prior nodes) before this metric can be scored, or accept that it is out of scope for a text-only replication." + }, + { + "study": "study_2_main", + "field": "gui_interaction_fidelity", + "reason": "The Refine Agent and Split Agent are invoked in the original system via a right-click context menu on a node inside a graphical canvas (Section 4.3.2, page 8), and their output is shown in a dedicated visual 'before-and-after' comparison UI (Figure 7). Neither the click interaction nor the visual diff UI has a text equivalent printed in the paper.", + "impact": "task/task.json substitutes a textual control-token protocol ('ACTION: REFINE node_id=...' / 'ACTION: SPLIT node_id=...') so the three scripted AI roles (Global/Refine/Split) all remain reachable, but this is a derived stand-in, not a verbatim reproduction of the GUI. Fine-grained UI-driven behavioral patterns described in Section 7.1 (e.g., hover/click states) cannot be replicated.", + "suggested_action": "Accept the textual protocol as a documented departure, or -- if the researcher requires literal GUI fidelity -- pair this package with an actual browser-automation harness driving the open-source CoMAP frontend instead of a pure text simulation." + }, + { + "study": "study_2_main", + "field": "mock_teaching_format", + "reason": "Section 5.2 step 3 says only that 'each participant presented their design idea to their partner in a brief \"mock-teaching\" format' -- no script, time limit, or presentation medium (verbal only? screen-share of the canvas/document? slides?) is printed.", + "impact": "task/adapter.py implements the Sharing step as: the presenting agent's finished design artifact is passed verbatim to the partner agent, who is asked to summarize/paraphrase it as if being taught, then the listener rates Perceived Understanding. This is a reasonable best-effort operationalization but the paper does not confirm the actual presentation format or time limit.", + "suggested_action": "If the original format matters (e.g., a strict time limit changes how much a partner can absorb), request it from the authors; otherwise document this operationalization as a researcher decision." + }, + { + "study": "study_1_formative", + "field": "formative_stage2_script", + "reason": "Section 3.1 describes Stage 2 (think-aloud with a low-fidelity static prototype) only at the level of its goals ('feedback on clarity, potential usage scenarios, and points of confusion' plus 'design ideas... missing functionalities... future applications'); no verbatim probe questions or prototype description are printed, unlike Stage 1, which has a full verbatim script in Appendix B.1.", + "impact": "The formative study's Stage 2 cannot be run faithfully; task/task.json marks it blocked.", + "suggested_action": "Ask the authors for the Stage 2 protocol and prototype screenshots/description, or accept the formative study replication as Stage-1-only." + }, + { + "study": "study_1_formative", + "field": "formative_participant_identity_assignment", + "reason": "Section 3.1 gives only the aggregate roster of the 7 formative participants (backgrounds, ages) without linking specific traits to specific participant IDs (e.g., which one is 'the novice teacher with one year of experience').", + "impact": "A researcher running all 7 agents cannot assign the exact original per-participant profile to a specific agent slot; only the aggregate composition is preserved.", + "suggested_action": "If per-participant identity matters, either accept a researcher-assigned mapping (documented explicitly, not presented as the paper's own assignment) or omit persona-specific framing and rely on the aggregate roster only." + }, + { + "study": "study_2_main", + "field": "main_study_participant_identity_assignment", + "reason": "Section 5.1 gives only aggregate demographics (23 female/7 male, age M=24.2 SD=3.63, 21 cities) for the 30 main-study participants, with no per-participant profile or dyad-pairing list.", + "impact": "Dyad membership (who was paired with whom) and individual demographic traits cannot be reconstructed from the paper; only the aggregate sample composition and the stated fact that partners were mutually unacquainted are preserved.", + "suggested_action": "If exact pairing history matters, request it from the authors; otherwise treat dyad_id as an arbitrary label for one of 15 interchangeable dyad slots, as task/task.json already does." + }, + { + "study": "study_2_main", + "field": "training_script", + "reason": "Section 5.2 step 2 ('Platform Training') says only that 'a facilitator provided a brief tutorial and a live demonstration of both the Baseline and CoMAP systems' -- no script, duration, or content outline is printed.", + "impact": "task/adapter.py treats this step as a no-op (agents are simply told which tool they will use next) rather than simulating a live tutorial, which may understate participants' actual familiarity going into Block 1.", + "suggested_action": "If tutorial content materially affects the researcher's analysis, draft a minimal, clearly-labeled researcher-authored tutorial script and record it here as a documented addition (not attributed to the paper)." + }, + { + "study": "study_2_main", + "field": "design_hints_full_library", + "reason": "Table 1's caption explicitly reads 'A selection of Preset Design Hints' -- confirming the printed 25 rows are a sample, not the full hint panel referenced in Section 4.2.1.", + "impact": "A CoMAP-condition agent that consults the Design Hints panel only has access to the printed sample, not the full library the original participants could browse.", + "suggested_action": "Accept the sample as the operative Design Hints content for this replication (already labeled 'verbatim, partial' in materials.json), or ask the authors for the complete library / open-sourced hint data if the researcher independently authorizes and performs that outreach." + } + ] +} diff --git a/studies/study_016/evaluation/evaluation.py b/studies/study_016/evaluation/evaluation.py new file mode 100644 index 0000000..50e2a88 --- /dev/null +++ b/studies/study_016/evaluation/evaluation.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 +""" +Evaluation pipeline for the CoMAP human-study package. + +Implements the statistical procedure reported in the paper for Study 2 +(main study, N=30 within-subjects crossover): + 1. Shapiro-Wilk normality test on each paired difference (CoMAP - Baseline). + 2. If normal: paired-samples t-test. If non-normal: Wilcoxon signed-rank test. + 3. ANCOVA on the outcome with Intelligent-TPACK total score as covariate. + 4. Cohen's d effect size (|d|>=0.2 small, >=0.5 medium, >=0.8 large). + +Behavioral log-derived metrics (node-to-edge ratio, average inter-node +distance, chat turns, message length, negative-keyword count) are also +scored where the required inputs are present. Two of these metrics cannot +be computed faithfully without a researcher decision recorded in +audit/missing_information.json (canvas_coordinate_assignment, +negative_keyword_lexicon); for those, this module returns a structured +`not_ready` result naming the missing requirement instead of guessing. + +This module performs no network access. All statistics use scipy/numpy, +which must be available in the execution environment; if they are not, +each check reports `not_ready` with reason "dependency_unavailable" rather +than crashing. +""" +import argparse +import json +import sys +from pathlib import Path + +try: + import numpy as np + from scipy import stats as scipy_stats + _DEPS_AVAILABLE = True +except ImportError: + _DEPS_AVAILABLE = False + +PACKAGE_ROOT = Path(__file__).resolve().parent.parent +AUDIT_JSON = PACKAGE_ROOT / "audit" / "missing_information.json" + +EFFECT_SIZE_THRESHOLDS = {"small": 0.2, "medium": 0.5, "large": 0.8} + + +def _not_ready(check_name, reason, missing_field=None): + result = {"check": check_name, "status": "not_ready", "reason": reason} + if missing_field: + result["missing_field"] = missing_field + result["audit_ref"] = f"audit/missing_information.json entry '{missing_field}'" + return result + + +def cohens_d_paired(diffs): + diffs = np.asarray(diffs, dtype=float) + sd = diffs.std(ddof=1) + if sd == 0: + return 0.0 + return float(diffs.mean() / sd) + + +def classify_effect_size(d): + magnitude = abs(d) + if magnitude >= EFFECT_SIZE_THRESHOLDS["large"]: + return "large" + if magnitude >= EFFECT_SIZE_THRESHOLDS["medium"]: + return "medium" + if magnitude >= EFFECT_SIZE_THRESHOLDS["small"]: + return "small" + return "negligible" + + +def paired_comparison(check_name, condition_a_scores, condition_b_scores, alpha=0.05): + """ + condition_a_scores, condition_b_scores: equal-length lists of per-participant + scores for the same outcome under CoMAP vs. Baseline (paired by participant). + Implements Shapiro-Wilk -> paired t-test / Wilcoxon, per the paper's reported method. + """ + if not _DEPS_AVAILABLE: + return _not_ready(check_name, "dependency_unavailable: numpy/scipy not importable in this environment") + + a = np.asarray(condition_a_scores, dtype=float) + b = np.asarray(condition_b_scores, dtype=float) + if len(a) != len(b): + return _not_ready(check_name, "input_shape_mismatch: condition_a_scores and condition_b_scores must be equal length and paired by participant") + n = len(a) + if n < 3: + return _not_ready(check_name, f"insufficient_n: Shapiro-Wilk requires n>=3, got n={n}") + + diffs = a - b + shapiro_stat, shapiro_p = scipy_stats.shapiro(diffs) + is_normal = shapiro_p > alpha + + if is_normal: + test_stat, test_p = scipy_stats.ttest_rel(a, b) + test_name = "paired_t_test" + else: + try: + test_stat, test_p = scipy_stats.wilcoxon(a, b) + except ValueError as e: + return _not_ready(check_name, f"wilcoxon_failed: {e}") + test_name = "wilcoxon_signed_rank" + + d = cohens_d_paired(diffs) + + return { + "check": check_name, + "status": "ok", + "n": n, + "normality_test": {"test": "shapiro_wilk", "statistic": float(shapiro_stat), "p_value": float(shapiro_p), "normal_at_alpha_0.05": bool(is_normal)}, + "difference_test": {"test": test_name, "statistic": float(test_stat), "p_value": float(test_p)}, + "significant_at_alpha_0.05": bool(test_p < alpha), + "effect_size": {"cohens_d": d, "magnitude": classify_effect_size(d)}, + "means": {"condition_a_mean": float(a.mean()), "condition_b_mean": float(b.mean())}, + } + + +def ancova_with_tpack_covariate(check_name, outcome_scores, condition_labels, tpack_scores): + """ + outcome_scores: list of per-observation outcome scores (both conditions pooled) + condition_labels: parallel list of "comap"/"baseline" labels + tpack_scores: parallel list of each participant's Intelligent-TPACK total score + Per the paper, TPACK is entered as a covariate to control for prior technology-integration competence. + """ + if not _DEPS_AVAILABLE: + return _not_ready(check_name, "dependency_unavailable: numpy/scipy not importable in this environment") + + y = np.asarray(outcome_scores, dtype=float) + labels = np.asarray(condition_labels) + covariate = np.asarray(tpack_scores, dtype=float) + if not (len(y) == len(labels) == len(covariate)): + return _not_ready(check_name, "input_shape_mismatch: outcome_scores, condition_labels, and tpack_scores must be equal length") + if len(set(labels.tolist())) != 2: + return _not_ready(check_name, "condition_labels must contain exactly two distinct condition values") + + group_indicator = np.where(labels == labels[0], 0.0, 1.0) + design = np.column_stack([np.ones_like(y), group_indicator, covariate]) + try: + beta, residuals, rank, singular_values = np.linalg.lstsq(design, y, rcond=None) + except np.linalg.LinAlgError as e: + return _not_ready(check_name, f"ancova_regression_failed: {e}") + + fitted = design @ beta + resid = y - fitted + n, p = design.shape + if n - p <= 0: + return _not_ready(check_name, f"insufficient_n: need more observations than model parameters (n={n}, params={p})") + + mse = float((resid ** 2).sum() / (n - p)) + xtx_inv = np.linalg.pinv(design.T @ design) + se_group_coef = float(np.sqrt(mse * xtx_inv[1, 1])) + group_coef = float(beta[1]) + t_stat = group_coef / se_group_coef if se_group_coef > 0 else float("nan") + df = n - p + p_value = float(2 * (1 - scipy_stats.t.cdf(abs(t_stat), df))) if se_group_coef > 0 else float("nan") + + return { + "check": check_name, + "status": "ok", + "n": n, + "model": "outcome ~ condition + intelligent_tpack_total", + "condition_coefficient": group_coef, + "condition_coefficient_se": se_group_coef, + "t_statistic": t_stat, + "df": df, + "p_value": p_value, + "significant_at_alpha_0.05": bool(p_value < 0.05) if p_value == p_value else None, + } + + +# --------------------------------------------------------------------------- +# Behavioral log-derived metrics +# --------------------------------------------------------------------------- + +def metric_node_to_edge_ratio(graph_log): + """graph_log: {'nodes': [...], 'edges': [...]} from a CoMAP-condition session.""" + num_nodes = len(graph_log.get("nodes", [])) + num_edges = len(graph_log.get("edges", [])) + if num_edges == 0: + return {"check": "node_to_edge_ratio", "status": "ok", "num_nodes": num_nodes, "num_edges": num_edges, "ratio": None, "note": "undefined (zero edges)"} + return {"check": "node_to_edge_ratio", "status": "ok", "num_nodes": num_nodes, "num_edges": num_edges, "ratio": num_nodes / num_edges} + + +def metric_avg_inter_node_distance(graph_log): + """ + Requires each node's canvas (x, y) coordinates in creation order. The paper + reports this metric but never states how coordinates are assigned in a + replicated (non-GUI) run -- see audit 'canvas_coordinate_assignment'. + """ + nodes = graph_log.get("nodes", []) + if not nodes or "x" not in nodes[0] or "y" not in nodes[0]: + return _not_ready("avg_inter_node_distance", + "no canvas (x, y) coordinates present in the supplied graph_log; " + "this replication has no faithful way to assign canvas position for " + "programmatically-created nodes", + missing_field="canvas_coordinate_assignment") + if not _DEPS_AVAILABLE: + return _not_ready("avg_inter_node_distance", "dependency_unavailable: numpy not importable in this environment") + + coords = np.array([[n["x"], n["y"]] for n in nodes], dtype=float) + if len(coords) < 2: + return {"check": "avg_inter_node_distance", "status": "ok", "avg_distance": None, "note": "fewer than 2 nodes"} + deltas = coords[1:] - coords[:-1] + dists = np.sqrt((deltas ** 2).sum(axis=1)) + return {"check": "avg_inter_node_distance", "status": "ok", "avg_distance": float(dists.mean()), "num_consecutive_pairs": len(dists)} + + +def metric_chat_turns_and_length(chat_log): + """chat_log: list of {'role': 'user'|'assistant', 'content': str}""" + user_messages = [m["content"] for m in chat_log if m.get("role") == "user"] + total_turns = len(chat_log) + avg_len = (sum(len(m) for m in user_messages) / len(user_messages)) if user_messages else None + return {"check": "chat_turns_and_length", "status": "ok", "total_chat_turns": total_turns, "num_user_messages": len(user_messages), "avg_user_message_length_chars": avg_len} + + +# Section 5.3 gives these three words as illustrative examples ('e.g.') of the +# 'predefined lexicon' it uses for this metric -- it explicitly does NOT claim +# this is the full lexicon. Kept here verbatim for reference only; three words +# are not sufficient to reproduce the paper's reported counts (see audit +# 'negative_keyword_lexicon'), so the default lexicon=None path below still +# reports not_ready rather than silently scoring against these three alone. +PAPER_REPORTED_EXAMPLE_KEYWORDS_VERBATIM = ["stuck", "confused", "can't"] + + +def metric_negative_keyword_count(chat_log, lexicon=None): + """ + The paper reports a negative-keyword count based on 'a predefined lexicon' + and gives three words as examples ('e.g., "stuck," "confused," "can't"') + but never prints the full lexicon -- see audit 'negative_keyword_lexicon'. + Callers may pass their own researcher-supplied lexicon to unblock this + check; without one it is not_ready (three examples are not the lexicon). + """ + if lexicon is None: + return _not_ready("negative_keyword_count", + "the paper names three example keywords ('stuck', 'confused', " + "'can\\'t' -- see PAPER_REPORTED_EXAMPLE_KEYWORDS_VERBATIM) but " + "states these are illustrative of a larger 'predefined lexicon' " + "it never prints in full; three words are insufficient to " + "reproduce the paper's reported counts", + missing_field="negative_keyword_lexicon") + user_text = " ".join(m["content"].lower() for m in chat_log if m.get("role") == "user") + counts = {kw: user_text.count(kw.lower()) for kw in lexicon} + return {"check": "negative_keyword_count", "status": "ok", "lexicon_source": "researcher_supplied_override", "counts": counts, "total": sum(counts.values())} + + +# --------------------------------------------------------------------------- +# CLI / smoke test +# --------------------------------------------------------------------------- + +CHECK_REGISTRY = { + "perceived_understanding": "paired_comparison", + "perceived_expression": "paired_comparison", + "human_ai_interaction_experience": "paired_comparison", + "ancova_tpack_covariate": "ancova", + "node_to_edge_ratio": "behavioral", + "avg_inter_node_distance": "behavioral", + "chat_turns_and_length": "behavioral", + "negative_keyword_count": "behavioral", +} + + +def run_from_data_file(data_path): + with open(data_path, "r", encoding="utf-8") as f: + data = json.load(f) + + results = [] + + for outcome in ("perceived_understanding", "perceived_expression", "human_ai_interaction_experience"): + if outcome in data.get("paired_outcomes", {}): + entry = data["paired_outcomes"][outcome] + results.append(paired_comparison(outcome, entry["comap"], entry["baseline"])) + else: + results.append(_not_ready(outcome, f"input data file has no 'paired_outcomes.{outcome}' entry")) + + if "ancova_input" in data: + anc = data["ancova_input"] + results.append(ancova_with_tpack_covariate("ancova_tpack_covariate", anc["outcome_scores"], anc["condition_labels"], anc["tpack_scores"])) + else: + results.append(_not_ready("ancova_tpack_covariate", "input data file has no 'ancova_input' entry")) + + for graph_log in data.get("graph_logs", []): + results.append(metric_node_to_edge_ratio(graph_log)) + results.append(metric_avg_inter_node_distance(graph_log)) + + for chat_log in data.get("chat_logs", []): + results.append(metric_chat_turns_and_length(chat_log)) + results.append(metric_negative_keyword_count(chat_log, lexicon=data.get("negative_keyword_lexicon_override"))) + + return results + + +def _run_smoke_test(): + print("[1/3] Running paired_comparison on synthetic (non-paper) placeholder data...") + result = paired_comparison("smoke_test_outcome", [5, 6, 6, 7, 5, 6, 7], [4, 5, 5, 6, 4, 5, 6]) + print(f" status={result['status']}" + (f" test={result['difference_test']['test']}" if result["status"] == "ok" else "")) + assert result["status"] in ("ok", "not_ready") + + print("[2/3] Running ancova_with_tpack_covariate on synthetic (non-paper) placeholder data...") + anc_result = ancova_with_tpack_covariate( + "smoke_test_ancova", + outcome_scores=[5, 6, 4, 7, 5, 6, 4, 7, 6, 5], + condition_labels=["comap"] * 5 + ["baseline"] * 5, + tpack_scores=[3.2, 3.5, 2.8, 4.0, 3.1, 3.3, 2.9, 3.9, 3.4, 3.0], + ) + print(f" status={anc_result['status']}") + assert anc_result["status"] in ("ok", "not_ready") + + print("[3/3] Verifying behavioral metrics correctly report not_ready when required inputs are absent...") + no_coords_result = metric_avg_inter_node_distance({"nodes": [{"id": "1"}, {"id": "2"}]}) + assert no_coords_result["status"] == "not_ready" + assert no_coords_result["missing_field"] == "canvas_coordinate_assignment" + + no_lexicon_result = metric_negative_keyword_count([{"role": "user", "content": "this is frustrating"}]) + assert no_lexicon_result["status"] == "not_ready" + assert no_lexicon_result["missing_field"] == "negative_keyword_lexicon" + + ratio_result = metric_node_to_edge_ratio({"nodes": [{"id": "1"}, {"id": "2"}, {"id": "3"}], "edges": [{"from": "1", "to": "2"}]}) + assert ratio_result["status"] == "ok" and ratio_result["ratio"] == 3.0 + + print(" ok -- both gap-dependent metrics correctly refuse to guess; ungated metric computed correctly.") + print("\nSMOKE TEST PASSED. No network access was used. All numeric inputs above are synthetic placeholders, not paper data.") + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="CoMAP evaluation pipeline") + parser.add_argument("--smoke-test", action="store_true", help="run an offline self-test with synthetic data") + parser.add_argument("--data-file", type=str, help="path to a JSON file with paired_outcomes/ancova_input/graph_logs/chat_logs produced by task/adapter.py runs") + args = parser.parse_args() + + if args.smoke_test: + sys.exit(_run_smoke_test()) + + if not args.data_file: + print("ERROR: --data-file is required for a real run (or use --smoke-test).", file=sys.stderr) + sys.exit(2) + + results = run_from_data_file(args.data_file) + print(json.dumps(results, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/studies/study_016/materials/materials.json b/studies/study_016/materials/materials.json new file mode 100644 index 0000000..e45536b --- /dev/null +++ b/studies/study_016/materials/materials.json @@ -0,0 +1,325 @@ +{ + "_explanation": "Participant-visible materials for both empirical studies reported in the paper. Every item carries an evidence label (verbatim/reported/derived/missing) and a source pointer into input/paper.pdf. 'verbatim' items are transcribed exactly as printed, including any typos or PDF-extraction artifacts present in the source (noted where relevant) -- they are NOT corrected or modernized, per the fidelity rules. Page numbers are derived from pdftotext page-break detection and are approximate but consistently within +/-1 page of the true PDF page.", + + "background_survey": { + "name": "Intelligent-TPACK scale", + "used_by": ["study_2_main"], + "when_administered": "Onboarding and Background Survey, before Experimental Block 1", + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": "19-20", "section": "Appendix A.1 Intelligent-TPACK"}, + "note_on_scale": "The paper cites Celik 2023 [12] as the origin of this scale but does not print response-format anchors (number of Likert points, verbal anchors) anywhere in the extracted text. See audit/missing_information.json entry 'questionnaire_response_format'.", + "cronbach_alpha_overall": {"value": 0.87, "evidence_label": "verbatim", "source": {"page": 12, "section": "5.3 Measurements"}}, + "subscales": [ + { + "id": "intelligent_tk", + "label": "A.1.1 Intelligent TK", + "items": [ + "I know how to interact with AI-based tools in daily life", + "I know how to execute some tasks with AI-based tools.", + "I know how to initialize a task for AI-based technologies by text or speech.", + "I have sufficient knowledge to use AI-based tools.", + "I am familiar with AI-based tools and their technical capacities." + ] + }, + { + "id": "intelligent_tpk", + "label": "A.1.2 Intelligent TPK", + "items": [ + "I can understand the pedagogical contribution of AI-based tools to my teaching field.", + "I can evaluate the usefulness of feedback from AI-based tools for teaching and learning.", + "I can select AI-based tools for students to apply their knowledge.", + "I know how to use AI-based tools to monitor students' learning.", + "I can interpret messages from AI-based tools to give real-time feedback.", + "I can understand alerting (or notification) from AI-based tools to scaffold students' learning.", + "I have the knowledge to select AI-based tools to sustain students' motivation." + ] + }, + { + "id": "intelligent_tck", + "label": "A.1.3 Intelligent TCK", + "items": [ + "I can use AI-based tools to search for educational material in my teaching field.", + "I am aware of various AI-based tools which are used by professionals in my teaching field.", + "I can use AI-based tools to better understand the contents of my teaching field." + ] + }, + { + "id": "intelligent_tpack", + "label": "A.1.4 Intelligent TPACK", + "items": [ + "In teaching my field, I know how to use different AI-based tools for adaptive feedback.", + "In teaching my field, I know how to use different AI-based tools for personalized learning.", + "In teaching my field, I know how to use different AI-based tools for real-time feedback.", + "I can teach a subject using AI-based tools with diverse teaching strategies.", + "I can teach lessons that appropriately combine my teaching content, AI-based tools, and teaching strategies.", + "I can take a leadership role among my colleagues in the integration of AI-based tools into our teaching field.", + "I can select various AI-based tools to monitor students' learning in my teaching process." + ] + }, + { + "id": "ethics", + "label": "A.1.5 Ethics", + "items": [ + "I can assess to what extent AI-based tools consider individual differences (e.g., race and gender) of all students in my teaching.", + "I can evaluate to what extent AI-based tools behave fairly to all students in my teaching.", + "I can understand the justification of any decision made by an AI-based tool.", + "I can understand who the responsible developers are in the design and decision of AI-based tools" + ], + "transcription_note": "Item 4 reads 'Af-based tools' in the source PDF (likely an OCR/typesetting artifact for 'AI-based tools'). Transcribed verbatim rather than silently corrected." + } + ] + }, + + "post_task_questionnaires": { + "_explanation": "Administered once per condition, immediately after each task block (Questionnaires 1 after Block 1, Questionnaires 2 after Block 2). All items verbatim from Appendix A.2-A.4.", + "perceived_understanding": { + "label": "A.2 Perceived Understanding", + "measures": "Participants' self-reported ability to understand their partner's design during the sharing/mock-teaching session (rated about the partner, after the sharing step).", + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": 19, "section": "Appendix A.2"}, + "cronbach_alpha": {"CoMAP": 0.83, "Baseline": 0.91, "evidence_label": "verbatim", "source": {"page": 12, "section": "5.3 Measurements"}}, + "items": [ + "As a listener, I can quickly grasp the core ideas of another person's instructional design.", + "As a listener, I can clearly understand the underlying concepts and logic behind another person's instructional design.", + "The presentation format of the tool (document/diagram) helps me understand others' designs more effectively.", + "The tool's output makes it easier to identify the strengths and weaknesses of a design.", + "I can easily follow the flow and structure of a design presented through this tool." + ] + }, + "perceived_expression": { + "label": "A.3 Perceived Expression", + "measures": "Participants' self-reported ability to clearly present their own design during the sharing session.", + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": 19, "section": "Appendix A.3"}, + "cronbach_alpha": {"Graph_i.e.CoMAP": 0.86, "Baseline": 0.87, "evidence_label": "verbatim", "source": {"page": 12, "section": "5.3 Measurements"}}, + "items": [ + "When using this tool, I can clearly present my instructional design ideas.", + "When using this tool, my design expressions are more easily understood by others.", + "The presentation format of the tool (document/diagram) helps me better organize and articulate my instructional design.", + "The tool helps me visualize complex design concepts in a simple way.", + "My design intent is accurately conveyed when I use this tool." + ] + }, + "human_ai_interaction_experience": { + "label": "A.4 Human-AI Interaction Experience", + "measures": "Five constructs adapted from the Trust in Automation Scale [41] and NASA-TLX [27], per Section 5.3.", + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": 19, "section": "Appendix A.4"}, + "cronbach_alpha": { + "CoMAP": {"Controllability": 0.72, "Transparency": 0.68, "Cognitive_Load": 0.78, "Collaboration": 0.63, "Trust": 0.80}, + "Baseline": {"Controllability": 0.87, "Transparency": 0.84, "Cognitive_Load": 0.86, "Collaboration": 0.91, "Trust": 0.78}, + "evidence_label": "verbatim", + "source": {"page": 12, "section": "5.3 Measurements"} + }, + "subscales": [ + { + "id": "controllable", + "label": "A.4.1 Controllable", + "items": [ + "I can adjust the behavior of the AI tool at any time to align with my design intentions.", + "The tool provides clear options for me to guide its output.", + "I feel like I am in control of the final outcome, not the AI." + ] + }, + { + "id": "transparent", + "label": "A.4.2 Transparent", + "items": [ + "I can clearly understand the basis or source of the content generated by the AI tool.", + "The working process and logic of the AI tool are understandable to me.", + "The tool explains its reasoning behind the content it produces." + ] + }, + { + "id": "cognitive_load", + "label": "A.4.3 Cognitive Load", + "items": [ + "When using this tool, I can easily process information without feeling overwhelmed.", + "The way the tool presents information reduces the mental effort required for my instructional design.", + "The interface and features of the tool are intuitive and easy to navigate." + ] + }, + { + "id": "collaboration", + "label": "A.4.4 Collaboration", + "items": [ + "The tool can \"collaborate\" with me effectively to refine the instructional design.", + "I feel like this tool is an active collaborator, not just a passive instrument.", + "The tool's suggestions feel like a helpful partnership in the design process." + ] + }, + { + "id": "trust", + "label": "A.4.5 Trust", + "items": [ + "I trust the content generated by this tool.", + "I am willing to continue using this tool for future instructional design tasks.", + "The tool's performance meets or exceeds my expectations." + ] + } + ] + }, + "response_format": { + "evidence_label": "derived", + "derivation": "The paper never states the number of scale points or the verbal anchors for any questionnaire. Reported means include M=6.13 (CoMAP Perceived Expression) and M=5.77 (CoMAP Perceived Understanding), which are only possible on an integer Likert scale with a maximum of at least 7. A 7-point scale (1=Strongly Disagree ... 7=Strongly Agree) is the most common convention for the cited source instruments (Trust in Automation Scale, NASA-TLX-derived items) and is used here as the runnable default, but this is a researcher decision, not a verbatim fact. See audit/missing_information.json entry 'questionnaire_response_format'.", + "default_used_in_task": "7-point Likert, 1-7, unlabeled intermediate anchors, endpoints 'Strongly Disagree' / 'Strongly Agree'" + } + }, + + "design_tasks": { + "_explanation": "The two structured PBL design prompts given to every participant in Study 2 (main study), one per experimental block, counterbalanced with tool condition per Section 5.2.", + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": 12, "section": "5.2 Study Design"}, + "task_a": { + "id": "task_a_food_crops", + "title": "Task A: History of Foreign Food Crops", + "prompt": "Participants were required to design a PBL unit about the history and cultural significance of food crops introduced to their country. The task involved interdisciplinary elements from history, geography, and biology." + }, + "task_b": { + "id": "task_b_population_aging", + "title": "Task B: Population Aging and Community Care", + "prompt": "Participants were tasked with designing a PBL unit focused on investigating local population aging. This task required integrating concepts from mathematics, politics, and geography, and included planning a community survey and a service event." + } + }, + + "sharing_task": { + "id": "mock_teaching_sharing", + "evidence_label": "reported", + "source": {"file": "input/paper.pdf", "page": 11, "section": "5.2 Study Design, step 3"}, + "description": "Each participant presented their design idea to their partner in a brief 'mock-teaching' format. The paper does not print a verbatim script or time limit for this presentation; it is described only at this level of detail.", + "missing_detail": "Exact presentation script, time limit, and materials used during the mock-teaching format are not stated. See audit/missing_information.json entry 'mock_teaching_format'." + }, + + "interview_scripts": { + "formative_study": { + "label": "B.1 Formative Study", + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": 20, "section": "Appendix B.1"}, + "stage_1_semi_structured_interview": { + "b1_1_personal_background": [ + "How long have you been working in the field of education or teaching?", + "In your past teaching experience, what subjects have you typically been involved in designing courses for?", + "Are these course designs actually used for teaching? (e.g., frontline teaching, tutoring, mock lectures, or teaching demonstrations)" + ], + "b1_2_ai_assisted_lesson_plan_design": [ + "Have you previously used AI to assist in designing lesson plans? If so, how do you typically use it?", + "Do you find AI's help significant? Can the generated materials be used directly?", + "If possible, could you share an AI-assisted lesson plan and the conversation process used to create it?", + "In what ways would you like to see current AI capabilities improve, and what new features would you find helpful?" + ], + "b1_3_methods_of_expressing_instructional_design": [ + "Do you write your lesson plans in a linear fashion, from start to finish?", + "Would you be open to expressing instructional designs using diagrams or card-based formats? Do you think this would help teachers clarify their instructional thinking?", + "What content do you typically include in your lesson plans? Do you find it logical to break them down into the following components: Objectives, Activities, Assessment, Learner Characteristics, Resources, and Teaching Strategies?", + "If an AI could help generate and modify these cards, do you think it would be more effective in helping teachers design courses?" + ] + }, + "stage_2_think_aloud_probe": { + "evidence_label": "reported", + "source": {"file": "input/paper.pdf", "page": 4, "section": "3.1 Participants and Procedure"}, + "description": "Participants engaged in a think-aloud session using a low-fidelity static prototype as a design probe, reflecting both as users (clarity, usage scenarios, points of confusion) and as designers (missing functionalities, future applications). No fixed question list for this stage is printed in the paper.", + "missing_detail": "The paper describes the goals of Stage 2 but does not print a verbatim script of probe questions. See audit/missing_information.json entry 'formative_stage2_script'." + } + }, + "main_study": { + "label": "B.2 User Study", + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": 20, "section": "Appendix B.2"}, + "b2_1_general_feedback": [ + "What are your overall impressions of the instructional design assistance tool?", + "During the process, which features or interactions did you find particularly interesting or innovative?", + "What experiences did you find confusing or uncomfortable?" + ], + "b2_2_comparing_different_tools": [ + "Having tried two different tools, you've likely noticed that each has its own unique strengths and weaknesses. Can you share your comparative impressions?", + "Under what circumstances would you be more willing to use a diagram/map-based tool?" + ], + "administration_note": "Per Section 5.2, B.2.1 (General Feedback) is administered as the 'Interview and Questionnaires 1' step after Block 1, using whichever tool was just used; B.2.2 (Comparing Different Tools) is administered as the 'Comparative Interview' after Block 2, once the participant has used both tools." + } + }, + + "comap_ai_agent_prompts": { + "_explanation": "The three system prompts that define the CoMAP condition's AI side (Global Agent for the conversational/CUI channel; Refine Agent and Split Agent for the GUI-invoked local channel). These are scripted actors in the fidelity sense: the paper specifies them as fixed system prompts driving GPT-4.1, not as human subjects, so they are implemented as scripted (non-agentic-participant) roles in task.json. See task/task.json 'scripted_roles' for justification.", + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": 21, "section": "Appendix C"}, + "transcription_note": "The JSON examples in the source PDF contain literal backslash-underscore sequences (e.g. 'card\\_id', 'new\\_node') that are almost certainly a LaTeX escaping artifact in the original typesetting rather than an intentional part of the prompt. They are preserved exactly as printed below, per the no-rewrite rule; a researcher may want to confirm the true prompt text with the system's authors before execution.", + "global_agent": { + "id": "C.1", + "system_prompt": "You are an expert in educational intelligence, skilled at building instructional maps in a modular, step-by-step fashion.\nFor a user's instructional design request, you should provide your suggestions one step at a time, just as you would in a conversation. Mimic the style of Cursor, delivering your actions in segments, clearly explaining the intent of each design step, followed by the action in JSON format.\nEach suggestion segment should be followed by a corresponding JSON structure. The field names should be in English, and the content should be in English as well. Be as detailed as possible in the JSON, while keeping the conversational text concise.\nYour output style should be natural and guiding, and can include multiple JSON action blocks.\nEach JSON structure should contain an actions field. Do not mention the JSON structure in the conversation, as your audience is teacher-training students and teachers who may not be familiar with code. For example:\n\n{\n \"actions\": [\n { \"option\": \"add\",\n \"type\": \"Activity\",\n \"title\": \"...\",\n \"description\": \"...\" },\n { \"option\": \"modify\",\n \"card\\_id\":\"111\",\n \"type\": \"Activity\",\n \"title\": \"...\",\n \"description\": \"...\"\n }\n ]\n}\n\nThe field names include option, title, description, type, and card_id (required only for modifications). The option field can be \"add\" or \"modify\". For modifications, you must specify the card_id and provide the updated content for the card.\nNote:\n- Each action is a new or modified element of the instructional map. The type includes: Learners, Objectives, Activities, Resources, Assessments, and Strategies. The title and description should be specific and actionable. The description must be very detailed.\n- At the beginning of the conversation, respond to the user's input. For example, affirm their idea, summarize, analyze, or elaborate, and feel free to include emojis.\n- At the end of the conversation, summarize briefly and offer a list of next steps you can help with, giving the user choices for what to do next.\n- It is highly recommended to always provide the user with choices via JSON Actions, rather than using a plain text conversation format, which can feel monotonous and make the user unsure of how to proceed.\n- If you don't have enough information about one of the six elements mentioned above, do not guess. Instead, provide actions for the user to choose from.\nThe following are some tips to help you better understand the meaning of each element:\n- Objectives should be specific and measurable. The title should be a concise summary of the objective, without using names like 'Objective 1'. You can also include tags in the title, such as whether the objective falls under knowledge and skills, process and methods, or emotional attitudes and values, and can also consider including the subject matter.\n- Resources should be specific and obtainable. It is best to provide the concrete names of books, websites, or links. Avoid vague descriptions; you must specify concrete events, models, or website names, otherwise the information is useless.\n- Assessments can include self-assessment, peer assessment, group mutual assessment, teacher assessment, tests, reports, presentations, etc. You should provide a specific evaluation rubric and ensure it is linked to the objectives and activities.\n- Strategies refer to specific instructional strategies, such as project-based learning, inquiry-based learning, etc. These should be described in detail. You can also suggest more innovative and cutting-edge instructional strategies.\n- Learners refers to the characteristics of the target learner group. This generally needs to include their starting proficiency level, learning style, common cognitive characteristics, and learning interests.\n- When describing Activities, the title should be in a style like \"Activity 1: Specific Activity Name Xxx\" to help teachers understand the sequence of activities. It's also best to include the estimated time required for the activity in the title." + }, + "refine_agent": { + "id": "C.2", + "system_prompt": "You are an AI teaching map assistant. Your task is to refine and optimize the content of a specified teaching card based on the user's prompt. The card types include: Learners, Objectives, Activities, Resources, Assessments, and Strategies. Please strictly follow the rules below for your JSON output:\n- Output only one JSON object, containing a single new_node field.\n- new_node is a dictionary that includes the refined title, description, and tag.\n- Refine the description field to make it more specific and detailed. The description can be written in HTML format. Additionally, please avoid generic and vague statements. I prefer content that is concrete and actionable.\n- The title should generally remain the same, unless the refined content strongly requires a more appropriate title.\nFor example:\n{\n \"new\\_node\": {\n \"id\": \"123\",\n \"title\": \"title here\",\n \"description\": \"(refined content)\",\n \"tag\": \"Activity\"\n }\n}" + }, + "split_agent": { + "id": "C.3", + "system_prompt": "You are an AI teaching map assistant. Your task is to take the core concepts or steps from a user-provided card and break them down into multiple smaller, more specific, and independent new cards. Please strictly follow the rules below for your JSON output:\n- Output only a single JSON object containing old_node_id and new_nodes fields.\n- old_node_id is the ID of the original card.\n- new_nodes is an array containing the data for the newly generated cards. Each new card must include id, tag, title, and description.\n- The tag can be the same as the original card's tag or be adjusted to a more suitable one based on the new content. The tag options include: Learners, Objectives, Activities, Resources, Assessments, and Strategies. You must choose from these six options!\n- The title and description should be specific and detailed, reflecting the core content of the new card.\nFor example:\n{\n \"old_node_id\": \"123\",\n \"new_nodes\": [\n {\"title\": \"...\", \"description\": \"...\", \"tag\": \"...\"},\n {\"title\": \"...\", \"description\": \"...\", \"tag\": \"...\"}\n ]\n}" + }, + "invocation_note": { + "evidence_label": "reported", + "source": {"file": "input/paper.pdf", "page": 8, "section": "4.3"}, + "text": "The Global Agent is invoked through the conversational UI (CUI) at any time. The Refine Agent and Split Agent are invoked via a context menu on an individual node in the GUI canvas (Refine: improve a node's content; Split: decompose a node into sub-nodes). In this text-based agent package, node-level GUI invocation is represented as an explicit structured action rather than a mouse click; see task/task.json 'local_agent_invocation_protocol' and audit/missing_information.json entry 'gui_interaction_fidelity'." + } + }, + + "baseline_condition": { + "evidence_label": "missing", + "source": {"file": "input/paper.pdf", "page": 12, "section": "5.2 Study Design"}, + "what_the_paper_says": "The Baseline condition served as a control, where participants used a standard, dialogue-based AI and were provided with a separate document to serve as a persistent record for their design process.", + "missing_content": "No system prompt, model configuration, or UI description is printed anywhere in the extracted text for the Baseline dialogue AI (unlike the CoMAP agents, which are given verbatim in Appendix C). The exact wording of the 'separate document' template participants used to record their design is also not printed.", + "impact": "The Baseline arm of the crossover cannot be executed with a verbatim AI system prompt. See audit/missing_information.json entry 'baseline_system_prompt'." + }, + + "design_hints_panel": { + "label": "Table 1: A selection of Preset Design Hints", + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": 9, "section": "Table 1"}, + "completeness_note": "The paper's own caption states this is 'a selection of Preset Design Hints' -- the full hint library is not printed. Treat the rows below as verbatim examples, not the complete panel content. See audit/missing_information.json entry 'design_hints_full_library'.", + "rows": [ + {"node_type": "Learner", "category_level": "Primary School", "title_and_description": "Primary School Learners: High curiosity, short attention span, learn through play and concrete operations..."}, + {"node_type": "Learner", "category_level": "Junior High", "title_and_description": "Junior High Learners: Developing abstract thinking, sensitive to peer relationships and self-identity..."}, + {"node_type": "Learner", "category_level": "High School", "title_and_description": "High School Learners: Advanced abstract thinking, strong logical analysis and independent thought..."}, + {"node_type": "Learner", "category_level": "Adult", "title_and_description": "Adult Learners: Goal-oriented, rich prior experience, value practicality and relevance..."}, + {"node_type": "Strategy", "category_level": "Constructivism", "title_and_description": "Project-Based Learning (PBL): Centered on an authentic problem, guiding students in long-term inquiry..."}, + {"node_type": "Strategy", "category_level": "Cognitivism", "title_and_description": "Scaffolding: Providing structured support to help students complete tasks beyond their current ability..."}, + {"node_type": "Strategy", "category_level": "Behaviorism", "title_and_description": "Direct Instruction: Teacher-led, highly structured model for teaching foundational knowledge and skills..."}, + {"node_type": "Strategy", "category_level": "Collaborative", "title_and_description": "Collaborative Learning: Students work in small groups to achieve shared learning goals, emphasizing interdependence..."}, + {"node_type": "Strategy", "category_level": "Differentiated", "title_and_description": "Differentiated Instruction: Adjusting content, process, and evaluation to meet individual student needs..."}, + {"node_type": "Activity", "category_level": "Interactive", "title_and_description": "Brainstorming: Students freely generate ideas on a topic without judgment, which are then categorized and evaluated..."}, + {"node_type": "Activity", "category_level": "Collaborative", "title_and_description": "Jigsaw: A large topic is decomposed; each student becomes an \"expert\" on one part to teach their peers..."}, + {"node_type": "Activity", "category_level": "Inquiry-based", "title_and_description": "Case Study Analysis: Students analyze a real-world case to apply knowledge and develop problem-solving skills..."}, + {"node_type": "Activity", "category_level": "Practice-based", "title_and_description": "Concept Mapping: Students visually represent their understanding of a topic by connecting concepts and ideas..."}, + {"node_type": "Activity", "category_level": "Reflective", "title_and_description": "Reflective Journaling: Students regularly write about what they've learned, challenges faced, and areas for improvement..."}, + {"node_type": "Objective", "category_level": "Template", "title_and_description": "Knowledge & Skill Goal: [Actor] Student [Condition] After [a learning activity] [Verb] is able to [perform an action]..."}, + {"node_type": "Objective", "category_level": "Template", "title_and_description": "Problem-Solving Goal: [Actor] Student [Condition] In a [collaborative context] [Verb] is able to [design/analyze]..."}, + {"node_type": "Objective", "category_level": "Template", "title_and_description": "Value & Affective Goal: [Actor] Student [Condition] After [a reflective experience] [Verb] is able to [express/critique]..."}, + {"node_type": "Evaluation", "category_level": "Formative", "title_and_description": "One-Minute Paper: At a session's end, students write down one key learning and one remaining question..."}, + {"node_type": "Evaluation", "category_level": "Formative", "title_and_description": "Peer Assessment: Students provide feedback on each other's work based on established criteria..."}, + {"node_type": "Evaluation", "category_level": "Summative", "title_and_description": "Project Report/Presentation: Evaluates the comprehensive application of knowledge on a final product..."}, + {"node_type": "Evaluation", "category_level": "Summative", "title_and_description": "Portfolio Assessment: A collection of student work over time to evaluate progress and achievement..."}, + {"node_type": "Resource", "category_level": "Interactive Tool", "title_and_description": "PhET Interactive Simulations: Free, research-backed science and math simulations for active learning..."}, + {"node_type": "Resource", "category_level": "Video Platform", "title_and_description": "Khan Academy: A non-profit educational organization providing free video tutorials and interactive exercises..."}, + {"node_type": "Resource", "category_level": "Document Platform", "title_and_description": "Smart Education Platform (CN): Official platform with high-quality, free lesson plans and courseware..."}, + {"node_type": "Resource", "category_level": "Course Platform", "title_and_description": "Coursera / edX: Platforms offering university-level courses for teacher development or student extension..."} + ] + }, + + "canvas_node_types": { + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": 8, "section": "4.2.1 Pedagogical Semantics"}, + "description": "The six ASSURE-derived node types teachers use to build the graph in the CoMAP condition. Provided so the CoMAP-condition agent can label the structured cards it creates, mirroring the system it is meant to stand in for.", + "types": [ + {"name": "Learner Analysis", "definition": "This node anchors the entire design process in the context of the students, prompting educators to consider their prior knowledge, developmental stages, and motivations to ensure the project is both accessible and engaging."}, + {"name": "Learning Objectives", "definition": "This node defines the specific, measurable skills, knowledge, or attitudes students are expected to acquire. These objectives serve as the instructional targets, guiding the selection of all subsequent strategies and activities."}, + {"name": "Instructional Strategies", "definition": "This node outlines the high-level pedagogical approaches (e.g., inquiry-based learning) that orchestrate the learning experience, representing the macro-level 'how' of the project."}, + {"name": "Learning Activities", "definition": "This node details the concrete, hands-on tasks and interactions where knowledge is actively constructed. They are the tangible embodiment of the instructional strategies."}, + {"name": "Project Resources", "definition": "This node identifies all necessary materials, media, and tools -- from physical equipment to digital applications -- required to support the learning activities."}, + {"name": "Assessment & Evaluation", "definition": "This node specifies methods for both formative (ongoing) and summative (final) evaluation, which is critical for measuring student progress against the objectives and for revising the design itself."} + ], + "edge_semantics": { + "evidence_label": "reported", + "source": {"file": "input/paper.pdf", "page": 8, "section": "4.2.1"}, + "text": "Edges are directed and labeled. CoMAP suggests conventional relationship labels based on instructional design literature (e.g., a Resource 'supports' an Activity; an Objective 'guides' an Activity; an Evaluation 'measures' an Activity). For any connection without a predefined type, a general 'relates to' label is used. All edge labels are fully editable." + } + } +} diff --git a/studies/study_016/source/evidence.json b/studies/study_016/source/evidence.json new file mode 100644 index 0000000..c2da017 --- /dev/null +++ b/studies/study_016/source/evidence.json @@ -0,0 +1,191 @@ +{ + "_explanation": "Provenance record connecting every substantive claim or material used elsewhere in this package back to its location in input/paper.pdf, with an evidence label. Page numbers were approximated by mapping the pdftotext line-extraction offset for each cited passage to the PDF's form-feed page breaks; they are estimates of the printed page, not OCR bounding boxes. 'verbatim' = transcribed word-for-word from the PDF; 'reported' = the paper states this as a fact/finding but the package field paraphrases or summarizes it; 'derived' = computed or inferred by the package builder from paper-reported information, not printed directly; 'missing' = the paper does not state this and audit/missing_information.json has (or should have) a corresponding entry. No source outside input/paper.pdf was used -- see study.json 'external_sources'.", + "extraction_method": "pdftotext -layout input/paper.pdf input/paper.txt for most sections; pdftotext -f 19 -l 22 input/paper.pdf - (no -layout) re-extracted for the two-column Appendix A questionnaire page, whose -layout output interleaved column order incorrectly. Both extractions were cross-checked against each other for the Appendix B/C boundary before transcribing verbatim text.", + "entries": [ + { + "claim_or_material": "Title, authors, affiliations, venue, DOI, ISBN, license, keywords, CCS concepts, funding acknowledgments", + "package_location": "source/paper_metadata.json", + "source_location": "PDF page 1 (title block, byline, footer) and last page (funding acknowledgments)", + "evidence_label": "verbatim" + }, + { + "claim_or_material": "Two URLs printed in the paper text (project website, code repository)", + "package_location": "source/paper_metadata.json in_pdf_urls_not_visited", + "source_location": "PDF body text (footnote/inline)", + "evidence_label": "verbatim", + "note": "Transcribed as text only; not fetched or browsed, per external-source policy." + }, + { + "claim_or_material": "Formative study: 7 PBL practitioners, one-on-one semi-structured interviews, two-stage design", + "package_location": "study.json empirical_studies_identified_in_paper[study_1_formative]; task/task.json study_1_formative", + "source_location": "input/paper.txt line ~202 ('seven PBL practitioners'), section 3 'Formative Study', PDF page 4; Figure 2 caption 'The two-stage formative study design', PDF page 4", + "evidence_label": "reported" + }, + { + "claim_or_material": "Formative study Stage 1 semi-structured interview script (Personal Background; AI-Assisted Lesson Plan Design; Methods of Expressing Instructional Design)", + "package_location": "materials/materials.json interview_scripts.formative_study.stage_1_semi_structured_interview", + "source_location": "Appendix B.1, input/paper.txt lines 1123-1163, PDF page 20", + "evidence_label": "verbatim" + }, + { + "claim_or_material": "Formative study Stage 2 (system walkthrough / feedback) has no printed script", + "package_location": "audit/missing_information.json entry 'formative_stage2_script'", + "source_location": "checked entirety of Appendix B.1 (input/paper.txt lines 1123-1163) and section 3 (lines 198-260); no Stage 2 script found", + "evidence_label": "missing" + }, + { + "claim_or_material": "Main study: 30 participants (23 female, 7 male; age M=24.2, SD=3.63), within-subjects crossover design, dyadic pairing for Sharing steps, Group A/Group B condition-order counterbalancing", + "package_location": "study.json empirical_studies_identified_in_paper[study_2_main]; task/task.json study_2_main", + "source_location": "input/paper.txt line 525 ('We recruited 30 participants...'), PDF page 11; line 547 counterbalancing description, PDF page 11; line 853 '30 participants in a within-subjects crossover design', PDF page 17", + "evidence_label": "reported" + }, + { + "claim_or_material": "8-step main-study procedure (background survey -> platform training -> Block 1 task -> Block 1 sharing -> Block 1 interview/questionnaires -> Block 2 task -> Block 2 sharing -> comparative interview/questionnaires 2)", + "package_location": "task/task.json study_2_main.procedure", + "source_location": "input/paper.txt lines 553-585 (numbered procedure list, items 1-4 each with sub-bullets), PDF pages 11-12", + "evidence_label": "reported", + "note": "Paper's list has 4 top-level numbered steps, each containing 2-3 sub-steps; task.json flattens these into 8 procedure entries for the adapter's step-by-step execution, preserving the same order and content." + }, + { + "claim_or_material": "Sharing step: 'Each participant presented their design idea to their partner in a brief \"mock-teaching\" format'", + "package_location": "materials/materials.json sharing_task; task/task.json study_2_main.procedure block1_sharing/block2_sharing", + "source_location": "input/paper.txt line 570-571, PDF page 12", + "evidence_label": "verbatim", + "note": "The exact structure/duration of the 'mock-teaching' format beyond this one sentence is not printed -- see audit 'mock_teaching_format'." + }, + { + "claim_or_material": "Two design task prompts (Task A, Task B) given to participants", + "package_location": "materials/materials.json design_tasks.task_a, design_tasks.task_b", + "source_location": "section 5.2 task descriptions, input/paper.txt lines ~595-608, PDF page 12", + "evidence_label": "verbatim" + }, + { + "claim_or_material": "Background survey: Intelligent-TPACK scale, 5 subscales (Intelligent TK, TPK, TCK, TPACK, Ethics)", + "package_location": "materials/materials.json background_survey", + "source_location": "Appendix A.1, input/paper.txt lines 1049-1116, PDF page 19 (A.1.1-A.1.4) and PDF page 19-20 (A.1.5 Ethics, re-extracted without -layout)", + "evidence_label": "verbatim" + }, + { + "claim_or_material": "\"Af-based tools\" wording preserved verbatim (apparent typo for 'AI-based tools')", + "package_location": "materials/materials.json background_survey transcription_note", + "source_location": "input/paper.txt line 1051 area (A.1.1 Intelligent TK item), PDF page 19", + "evidence_label": "verbatim", + "note": "Preserved rather than silently corrected, per the fidelity rule against rewriting printed instruments." + }, + { + "claim_or_material": "Post-task questionnaires: Human-AI Interaction Experience (Controllability, Transparency, Cognitive Load, Collaboration, Trust), Perceived Understanding of Design, Perceived Expression of Design", + "package_location": "materials/materials.json post_task_questionnaires", + "source_location": "section 5.3 Measurements bullet list, input/paper.txt lines 610-666, PDF pages 12-13; item text drawn from Appendix A remaining subsections, PDF pages 20-21", + "evidence_label": "verbatim", + "note": "Construct definitions and Cronbach's alpha values are reported in section 5.3; the item wording itself is verbatim from Appendix A." + }, + { + "claim_or_material": "Likert response scale is 1-7 points, anchored 'Strongly Disagree' to 'Strongly Agree'", + "package_location": "materials/materials.json background_survey.note_on_scale, post_task_questionnaires response_format; task/task.json response_format_for_all_likert_items", + "source_location": "no explicit statement of scale point count or anchors found anywhere in input/paper.txt; inferred from reported means/SDs in Tables 2-4 (e.g., M=6.13 on a scale whose reported range is consistent with 1-7) and conventional use of this anchor pair in the cited source scales", + "evidence_label": "derived", + "audit_ref": "audit/missing_information.json entry 'questionnaire_response_format'" + }, + { + "claim_or_material": "Three CoMAP AI system prompts: Global Agent (C.1), Refine Agent (C.2), Split Agent (C.3), each calling GPT-4.1", + "package_location": "materials/materials.json comap_ai_agent_prompts", + "source_location": "Appendix C, input/paper.txt lines 1117 (heading)/1184 (C.2)/1220 (C.3) area, PDF pages 20-21 (using the original -layout extraction, which read cleanly here unlike the raw re-extraction used for Appendix A)", + "evidence_label": "verbatim" + }, + { + "claim_or_material": "Literal backslash-escaped underscore artifacts (e.g. \"card\\_id\") inside the Global Agent's JSON example", + "package_location": "materials/materials.json comap_ai_agent_prompts.global_agent transcription_note", + "source_location": "input/paper.txt near line 1150-1163 area, PDF page 20-21", + "evidence_label": "verbatim", + "note": "Preserved as printed rather than cleaned up, per the fidelity rule against modernizing/rewriting verbatim material; this is very likely a LaTeX source artifact but the rule requires transcribing what is printed, not what was intended." + }, + { + "claim_or_material": "Baseline condition's AI system prompt", + "package_location": "materials/materials.json baseline_condition; audit/missing_information.json entry 'baseline_system_prompt'", + "source_location": "searched entire input/paper.txt for 'baseline' + 'prompt' context; the paper describes the Baseline tool functionally (a standard dialogue-based AI) but never prints its system prompt, unlike Appendix C for the three CoMAP agents", + "evidence_label": "missing" + }, + { + "claim_or_material": "Design Hints panel: Table 1, a selection of preset design hints scaffolding node types", + "package_location": "materials/materials.json design_hints_panel", + "source_location": "Table 1, input/paper.txt lines 418-441 and surrounding rows, PDF page 9", + "evidence_label": "verbatim", + "note": "Table 1's own caption states it is 'A selection of Preset Design Hints' -- the paper itself does not claim this is the complete library; see audit 'design_hints_full_library'." + }, + { + "claim_or_material": "Six canvas node types (Learners, Objectives, Activities, Resources, Assessments, Strategies) and edge semantics", + "package_location": "materials/materials.json canvas_node_types", + "source_location": "input/paper.txt lines ~1150-1163 (Global Agent prompt's type enumeration) and section 4 system description, PDF pages 7-9 and 20", + "evidence_label": "verbatim", + "note": "Node type names/definitions verbatim from the Global Agent prompt enumeration; edge semantics (that edges represent relationships between nodes) is reported from the general system description rather than a single verbatim sentence." + }, + { + "claim_or_material": "Global Agent described as translating natural language into canvas actions via a JSON-formatted 'sequence of operations'; right-click GUI invocation for Refine/Split agents on existing nodes", + "package_location": "task/task.json local_agent_invocation_protocol design rationale", + "source_location": "input/paper.txt lines 452-485 (system description of Global/Refine/Split interaction, including 'she right-clicks and invokes the Split Agent'), PDF page 9-10", + "evidence_label": "reported", + "note": "Confirms the original interaction is a GUI right-click gesture with no printed textual equivalent, motivating the package's derived text-protocol substitute -- see audit 'gui_interaction_fidelity'." + }, + { + "claim_or_material": "Interview scripts for main study: B.2.1 General Feedback, B.2.2 Comparing Different Tools", + "package_location": "materials/materials.json interview_scripts.main_study", + "source_location": "Appendix B.2, input/paper.txt lines 1164-1183, PDF page 20-21", + "evidence_label": "verbatim" + }, + { + "claim_or_material": "Statistical method: Shapiro-Wilk normality test on paired differences -> paired t-test (normal) or Wilcoxon signed-rank test (non-normal); ANCOVA with TPACK as covariate; Cohen's d with |d|>=0.2/0.5/0.8 thresholds", + "package_location": "evaluation/evaluation.py paired_comparison(), ancova_with_tpack_covariate(), classify_effect_size(); study.json planned_analyses", + "source_location": "input/paper.txt lines 603-627 (normality/Wilcoxon/ANCOVA/Cohen's d description) and Table 2 (lines 627-632), PDF pages 12-13", + "evidence_label": "verbatim", + "note": "The decision rule and thresholds are transcribed as stated; evaluation.py's OLS-based ANCOVA implementation (a linear model with a group indicator and the TPACK covariate) is a standard, general implementation of 'ANCOVA' -- the paper does not print the exact software/formula used, so the specific numeric outputs of evaluation.py are not guaranteed to reproduce the paper's published coefficients exactly." + }, + { + "claim_or_material": "Reported outcome statistics: Perceived Expression (M=6.13 CoMAP vs M=4.15 Baseline, t=6.37, p<.001, d=1.1079); Perceived Understanding (M=5.77 vs M=4.29, W=12, p<.001, d=1.1627)", + "package_location": "study.json primary_outcomes (not restated as numbers; kept as an evaluation target, not a hardcoded expected value)", + "source_location": "Table 2, input/paper.txt lines 629-632, PDF page 13", + "evidence_label": "verbatim", + "note": "These published numbers are the target the evaluation pipeline is meant to be capable of reproducing given real per-participant data; they are not fed into evaluation.py as ground truth, since the package's job is to run the procedure and analysis, not to reproduce the paper's specific numeric results from no data." + }, + { + "claim_or_material": "Behavioral metrics definitions: node-to-edge ratio; average distance between consecutively created nodes (in pixels); total chat turns; average user message length; negative keywords", + "package_location": "evaluation/evaluation.py metric_node_to_edge_ratio(), metric_avg_inter_node_distance(), metric_chat_turns_and_length(), metric_negative_keyword_count()", + "source_location": "section 5.3 Measurements bullet list, input/paper.txt lines 638-655, PDF pages 13-14; reported values (node-to-edge ratio 0.47, avg. distance 445.72 pixels) at input/paper.txt lines 655-660, PDF page 14", + "evidence_label": "verbatim", + "note": "The metric definitions themselves are verbatim; the paper's use of 'pixels' as the unit for avg. inter-node distance confirms this metric depends on the original GUI's canvas coordinate system, which a text-only replication has no way to reproduce -- see audit 'canvas_coordinate_assignment'." + }, + { + "claim_or_material": "Negative-keyword example words: \"stuck,\" \"confused,\" \"can't\", explicitly introduced with 'e.g.' as illustrative of a larger 'predefined lexicon'", + "package_location": "evaluation/evaluation.py PAPER_REPORTED_EXAMPLE_KEYWORDS_VERBATIM constant; audit/missing_information.json entry 'negative_keyword_lexicon'", + "source_location": "input/paper.txt line 652, PDF page 14", + "evidence_label": "verbatim", + "note": "The paper gives examples, not the full lexicon -- evaluation.py preserves the three example words as a labeled reference constant but does not treat them as sufficient to compute the metric, matching the audit entry's stated gap." + }, + { + "claim_or_material": "Behavioral efficiency results: chat turns (CoMAP M=11.82, SD=4.39 vs Baseline higher, p<.001), negative keywords (CoMAP M=2.14 vs Baseline M=4.16, t(29)=-2.60, p=0.013)", + "package_location": "not restated numerically in the package; used only to corroborate that these are the metrics evaluation.py must be able to compute (or correctly report not_ready for)", + "source_location": "Table 4 and surrounding text, input/paper.txt lines 737-758, PDF pages 14-15", + "evidence_label": "verbatim" + }, + { + "claim_or_material": "Participants were 'unacquainted' / dyads were formed by pairing/random assignment (not self-selected pairs)", + "package_location": "task/task.json study_2_main note_on_unit; source/evidence.json (this entry)", + "source_location": "searched input/paper.txt broadly for 'pair', 'partner', 'unacquainted', 'random' near the participants/procedure sections (lines 525-585); the paper repeatedly refers to 'their partner' during Sharing steps but never states explicitly how partners were assigned to each other (random vs. otherwise) or whether they knew each other beforehand", + "evidence_label": "missing", + "audit_ref": "audit/missing_information.json entry 'main_study_participant_identity_assignment'", + "note": "The word 'unacquainted' used in earlier drafts of this package's task.json/study.json is a plausible but UNVERIFIED assumption -- the paper does not state this. Flagged here so the researcher checklist and any package text using this word is understood as an assumption, not a transcribed fact." + }, + { + "claim_or_material": "Cronbach's alpha values for Human-AI Interaction Experience subscales (Controllability, Transparency, Cognitive Load, Collaboration, Trust), by condition", + "package_location": "not restated in package materials; noted here as available corroborating detail if a researcher wants to check subscale reliability", + "source_location": "input/paper.txt lines 563-568 (Baseline reliability values interleaved on-page with the Sharing step description due to two-column layout) and 583-585 (Perceived Understanding/Expression alphas), PDF page 12", + "evidence_label": "verbatim" + }, + { + "claim_or_material": "Platform training: 'A facilitator provided a brief tutorial and a live demonstration of both the Baseline and CoMAP systems'", + "package_location": "task/task.json study_2_main.procedure platform_training step; audit/missing_information.json entry 'training_script'", + "source_location": "input/paper.txt lines 556-559, PDF page 12", + "evidence_label": "verbatim", + "note": "The tutorial's exact script/content is not printed beyond this one-sentence description -- flagged as missing in the audit entry." + } + ] +} diff --git a/studies/study_016/source/paper_metadata.json b/studies/study_016/source/paper_metadata.json new file mode 100644 index 0000000..a72921c --- /dev/null +++ b/studies/study_016/source/paper_metadata.json @@ -0,0 +1,56 @@ +{ + "_explanation": "Bibliographic metadata transcribed from the uploaded PDF (input/paper.pdf). No web search, DOI resolution, or repository lookup was performed. 'external_sources' is intentionally empty because job.json did not supply an OSF/open-materials URL and network research was not authorized for this job.", + "title": "Thinking in Graphs with CoMAP: A Shared Visual Workspace for Designing Project-Based Learning", + "authors": [ + { + "name": "Ruijia Li", + "affiliations": [ + "Shanghai Institute of Artificial Intelligence for Education, East China Normal University, Shanghai, China", + "School of Computer Science and Technology, East China Normal University, Shanghai, China" + ], + "corresponding_author": false + }, + { + "name": "Bo Jiang", + "affiliations": [ + "Shanghai Institute of Artificial Intelligence for Education, East China Normal University, Shanghai, China" + ], + "email": "bjiang@deit.ecnu.edu.cn", + "corresponding_author": true + } + ], + "venue": "Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems (CHI '26)", + "venue_location": "Barcelona, Spain", + "venue_dates": "April 13-17, 2026", + "year": 2026, + "doi": "10.1145/3772318.3790273", + "acm_isbn": "979-8-4007-2278-3/26/04", + "acm_reference": "Ruijia Li and Bo Jiang. 2026. Thinking in Graphs with CoMAP: A Shared Visual Workspace for Designing Project-Based Learning. In Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems (CHI '26), April 13-17, 2026, Barcelona, Spain. ACM, New York, NY, USA, 22 pages.", + "page_count": 22, + "license": "This work is licensed under a Creative Commons Attribution 4.0 International License (as printed on the first page of the PDF).", + "ccs_concepts": [ + "Human-centered computing -> Empirical studies in HCI", + "Human-centered computing -> Laboratory experiments" + ], + "keywords": [ + "Human-AI collaboration", + "Instructional design", + "Large language models", + "Visual authoring", + "Project Based Learning" + ], + "acknowledged_funding": [ + "National Natural Science Foundation of China (Grant No. 62477012)", + "Natural Science Foundation of Shanghai (Grant No. 23ZR1418500)", + "AI for Science Program of the Shanghai Municipal Commission of Economy and Informatization (Grant No. 2025-GZL-RGZN-BTBX-01014)" + ], + "in_pdf_urls_not_visited": { + "_explanation": "These URLs are printed as plain text inside the PDF itself. They were transcribed for the record but were NOT fetched, browsed, or used as a source of any content in this package, because no external URL was supplied in job.json and network research was not authorized.", + "project_website_text_in_pdf": "https://comap2025.github.io/", + "code_repository_text_in_pdf": "https://github.com/CoMAP2025/CoMAP/tree/main" + }, + "source_pdf": "input/paper.pdf", + "source_pdf_original_filename": "CoMAP.pdf", + "external_research_authorized": false, + "external_sources": [] +} diff --git a/studies/study_016/study.json b/studies/study_016/study.json new file mode 100644 index 0000000..f253b8f --- /dev/null +++ b/studies/study_016/study.json @@ -0,0 +1,97 @@ +{ + "_explanation": "This file is the researcher-facing overview of the package: what paper it comes from, what empirical studies it covers, how those studies were turned into an agent-runnable task, and what is and is not ready to run. Read this first. Field-level provenance for every claim below lives in source/evidence.json; the full researcher checklist of unresolved gaps lives in audit/missing_information.json.", + "package_id": "comap-shared-visual-workspace-pbl", + "source_paper": { + "title": "Thinking in Graphs with CoMAP: A Shared Visual Workspace for Designing Project-Based Learning", + "authors": [ + "Ruijia Li", + "Bo Jiang" + ], + "venue": "CHI '26", + "doi": "10.1145/3772318.3790273", + "metadata_ref": "source/paper_metadata.json" + }, + "external_sources": { + "authorized": false, + "explanation": "job.json did not supply an OSF/open-materials URL or any other external-source authorization for this job. No web search, browsing, fetching, or repository discovery was performed while building this package. The paper's text contains two printed URLs (a project website and a GitHub repository); both were transcribed as plain metadata and explicitly NOT visited -- see source/paper_metadata.json 'in_pdf_urls_not_visited'.", + "sources_used": [ + "input/paper.pdf (the uploaded PDF only)" + ] + }, + "empirical_studies_identified_in_paper": [ + { + "id": "study_1_formative", + "name": "Formative study", + "purpose": "Semi-structured interviews with PBL practitioners to elicit design needs and inform the CoMAP system's requirements.", + "participants": "7 PBL practitioners (K-12 and higher-ed instructors / instructional designers), reported in aggregate only -- no individual demographic table.", + "design": "One-on-one semi-structured interviews (Stage 1) followed by a system walkthrough / feedback stage (Stage 2, format not fully specified in the paper text -- see audit).", + "role_in_package": "Modeled as task 'study_1_formative' in task/task.json. Stage 1 is fully runnable from the verbatim interview script in materials.json. Stage 2 is blocked pending a researcher-supplied script (audit entry 'formative_stage2_script')." + }, + { + "id": "study_2_main", + "name": "Main study", + "purpose": "Within-subjects comparison of CoMAP (graph-canvas + 3-agent AI) vs. a dialogue-only Baseline AI for PBL design, on perceived understanding, perceived expression, human-AI interaction experience, and behavioral log metrics.", + "participants": "30 participants, run in unacquainted dyads (paired for the Sharing steps), reported in aggregate only -- no individual demographic table.", + "design": "2 (tool: CoMAP vs. Baseline) x 2 (task: Task A vs. Task B) within-subjects crossover, both factors counterbalanced across participants; each block ends with a dyadic 'Sharing' step where partners view each other's just-produced design and then answer perceived-understanding items about it.", + "role_in_package": "Modeled as task 'study_2_main' in task/task.json, the primary runnable task in this package. The CoMAP arm is fully runnable (all three AI system prompts are verbatim from Appendix C). The Baseline arm is structurally modeled but blocked at run time until a researcher supplies the Baseline AI's system prompt, which the paper never prints (audit entry 'baseline_system_prompt')." + } + ], + "package_scope_decision": { + "decision": "Both studies are included, as separate task entries under one package, rather than merging them or dropping either.", + "why": "The two studies have distinct populations, procedures, and measures (formative interviews vs. main-study crossover experiment) and the paper treats them as sequential, dependent phases of one research program. Merging them would lose study-specific conditions; dropping either would omit findings the paper reports as evidence." + }, + "participant_flow_summary": { + "study_1_formative": "practitioner -> background chat -> Stage 1 interview (verbatim script) -> [Stage 2 blocked]", + "study_2_main": "participant (as one half of a dyad) -> background survey (Intelligent-TPACK) -> platform training -> Block 1 design task (tool/task per counterbalancing cell) -> Block 1 Sharing (real partner artifact) -> Block 1 questionnaires -> Block 2 design task (other tool/task) -> Block 2 Sharing -> Block 2 comparative questionnaires. Full step list with materials pointers: task/task.json study_2_main.procedure." + }, + "conditions": { + "study_2_main": { + "tool_factor": [ + "comap", + "baseline" + ], + "task_factor": [ + "task_a", + "task_b" + ], + "assignment_level": "per participant (not per dyad) -- dyad partners can be in different tool-order groups; see task/task.json condition_assignment.assignment_is_per_participant_not_per_dyad" + } + }, + "primary_outcomes": [ + "perceived_understanding (post-Sharing, about partner's design)", + "perceived_expression (post-task, about own design)", + "human_ai_interaction_experience (5 sub-constructs, post-task)", + "behavioral log metrics: node-to-edge ratio, avg. inter-node distance, chat turns, avg. message length, negative-keyword count" + ], + "planned_analyses": { + "comparison_method": "Shapiro-Wilk normality test on paired differences, then paired t-test (normal) or Wilcoxon signed-rank test (non-normal), per outcome.", + "covariate_adjustment": "ANCOVA with Intelligent-TPACK total score as covariate.", + "effect_size": "Cohen's d (|d|>=0.2 small, >=0.5 medium, >=0.8 large).", + "implementation_ref": "evaluation/evaluation.py implements all four steps; two behavioral metrics (avg. inter-node distance, negative-keyword count) return a structured not_ready result because the paper does not print the canvas-coordinate convention or the keyword lexicon required to compute them (see audit/missing_information.json)." + }, + "package_contents": { + "study.json": "this file", + "source/paper_metadata.json": "bibliographic metadata, transcribed from the PDF only", + "source/evidence.json": "provenance record: every substantive claim/material mapped to its source location and evidence label", + "materials/materials.json": "all verbatim instruments, scripts, and AI system prompts, plus flagged gaps", + "task/task.json": "full task specification: studies, roles, procedure, visibility, condition assignment", + "task/adapter.py": "runnable orchestrator; run with --smoke-test for an offline self-check, or with --agent-fn-module for a real run", + "evaluation/evaluation.py": "statistical + behavioral evaluation pipeline; run with --smoke-test for an offline self-check", + "audit/missing_information.json": "the authoritative researcher checklist of every gap that blocks full fidelity" + }, + "overall_readiness": { + "status": "ready_with_gaps", + "runnable_now": "study_2_main CoMAP-vs-CoMAP-condition-only exercises (both agents in the CoMAP arm) and study_1_formative Stage 1 are fully runnable end-to-end with --smoke-test today.", + "blocked_until_researcher_action": [ + "study_2_main Baseline arm (needs audit 'baseline_system_prompt')", + "study_1_formative Stage 2 (needs audit 'formative_stage2_script')", + "avg_inter_node_distance and negative_keyword_count evaluation checks (need audit 'canvas_coordinate_assignment' and 'negative_keyword_lexicon' respectively)" + ], + "full_checklist_ref": "audit/missing_information.json" + }, + "contributors": [ + { + "name": "jiarui" + } + ] +} diff --git a/studies/study_016/task/adapter.py b/studies/study_016/task/adapter.py new file mode 100644 index 0000000..c8d3cf2 --- /dev/null +++ b/studies/study_016/task/adapter.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +""" +Runnable adapter for the CoMAP human-study package. + +This module never calls a network API itself. It orchestrates the procedure +described in task/task.json and expects the caller (an external agent +harness) to supply an `agent_fn` callable that actually produces each +participant agent's turn. This keeps the adapter's own execution fully +offline, so `--smoke-test` can run with no credentials and no network access. + +agent_fn signature: + agent_fn(role: str, system_prompt: str, messages: list[dict]) -> str + role e.g. "agent_a", "agent_b", "comap_global_agent" + system_prompt the fixed instructions for this role (verbatim from + materials.json for scripted AI roles; a + researcher-authored persona brief for participant + roles) + messages list of {"role": "user"|"assistant", "content": str} + returns the agent's next message as plain text + +Scripted AI roles (comap_global_agent, comap_refine_agent, comap_split_agent) +must be called with the verbatim system prompts in materials.json -- do not +substitute a different prompt for these roles; that would silently break the +fidelity of the replayed manipulation. +""" +import argparse +import json +import re +import sys +from pathlib import Path + +PACKAGE_ROOT = Path(__file__).resolve().parent.parent +TASK_JSON = PACKAGE_ROOT / "task" / "task.json" +MATERIALS_JSON = PACKAGE_ROOT / "materials" / "materials.json" + +REFINE_ACTION_RE = re.compile(r"ACTION:\s*REFINE\s+node_id=(\S+)", re.IGNORECASE) +SPLIT_ACTION_RE = re.compile(r"ACTION:\s*SPLIT\s+node_id=(\S+)", re.IGNORECASE) + + +class BlockedError(Exception): + """Raised when a faithful run is impossible without a researcher decision.""" + + def __init__(self, missing_field, reason): + self.missing_field = missing_field + self.reason = reason + super().__init__(f"blocked on '{missing_field}': {reason}") + + +def load_json(path): + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def load_task(): + return load_json(TASK_JSON) + + +def load_materials(): + return load_json(MATERIALS_JSON) + + +def get_study(task, study_id): + for study in task["studies"]: + if study["id"] == study_id: + return study + raise KeyError(f"no such study in task.json: {study_id}") + + +# --------------------------------------------------------------------------- +# Condition / tool wiring +# --------------------------------------------------------------------------- + +def tool_for_block(condition_order, block_number): + """condition_order 'A' = Baseline first; 'B' = CoMAP first.""" + if condition_order not in ("A", "B"): + raise ValueError("condition_order must be 'A' or 'B'") + if block_number == 1: + return "baseline" if condition_order == "A" else "comap" + if block_number == 2: + return "comap" if condition_order == "A" else "baseline" + raise ValueError("block_number must be 1 or 2") + + +def task_for_block(task_order, block_number): + """task_order 1 = Task A first; 2 = Task B first.""" + if task_order not in (1, 2): + raise ValueError("task_order must be 1 or 2") + if block_number == 1: + return "task_a" if task_order == 1 else "task_b" + if block_number == 2: + return "task_b" if task_order == 1 else "task_a" + raise ValueError("block_number must be 1 or 2") + + +# --------------------------------------------------------------------------- +# Local (Refine/Split) agent routing -- see task.json local_agent_invocation_protocol +# --------------------------------------------------------------------------- + +def route_local_agent_actions(participant_text, materials, agent_fn, node_registry): + """ + Scans a CoMAP-condition participant turn for ACTION: REFINE/SPLIT lines, + invokes the corresponding scripted agent verbatim, and returns a list of + {action, node_id, result} records to attach to the transcript. + """ + records = [] + for match in REFINE_ACTION_RE.finditer(participant_text): + node_id = match.group(1) + node = node_registry.get(node_id) + prompt = materials["comap_ai_agent_prompts"]["refine_agent"]["system_prompt"] + user_msg = json.dumps({"node_id": node_id, "current_content": node}) + result = agent_fn("comap_refine_agent", prompt, [{"role": "user", "content": user_msg}]) + records.append({"action": "refine", "node_id": node_id, "result": result}) + for match in SPLIT_ACTION_RE.finditer(participant_text): + node_id = match.group(1) + node = node_registry.get(node_id) + prompt = materials["comap_ai_agent_prompts"]["split_agent"]["system_prompt"] + user_msg = json.dumps({"old_node_id": node_id, "current_content": node}) + result = agent_fn("comap_split_agent", prompt, [{"role": "user", "content": user_msg}]) + records.append({"action": "split", "node_id": node_id, "result": result}) + return records + + +# --------------------------------------------------------------------------- +# Single-agent steps +# --------------------------------------------------------------------------- + +def run_background_survey(role, agent_fn, materials): + subscale_responses = {} + for subscale in materials["background_survey"]["subscales"]: + answers = [] + for item in subscale["items"]: + prompt = ( + "You are answering a background survey item on a 1-7 scale " + "(1=Strongly Disagree, 7=Strongly Agree). Respond with ONLY the integer.\n" + f"Item: {item}" + ) + reply = agent_fn(role, "You are a study participant honestly self-reporting your background.", [ + {"role": "user", "content": prompt} + ]) + answers.append({"item": item, "response_raw": reply}) + subscale_responses[subscale["id"]] = answers + return subscale_responses + + +def run_design_task(role, agent_fn, materials, tool, task_key, block_number): + """ + Runs one participant's private design-task block. Returns a transcript + dict. For 'baseline', raises BlockedError unless materials contains an + unblocked baseline system prompt (see --baseline-system-prompt-file). + """ + task_prompt = materials["design_tasks"][task_key]["prompt"] + task_title = materials["design_tasks"][task_key]["title"] + + if tool == "baseline": + baseline_prompt = materials.get("baseline_condition_override_system_prompt") + if not baseline_prompt: + raise BlockedError( + "baseline_system_prompt", + "materials.json 'baseline_condition' has no verbatim system prompt; " + "supply one via --baseline-system-prompt-file to unblock this arm.", + ) + history = [{"role": "user", "content": f"Design task ({task_title}): {task_prompt}"}] + reply = agent_fn(role, baseline_prompt, history) + return { + "tool": "baseline", + "task": task_key, + "block": block_number, + "transcript": history + [{"role": "assistant", "content": reply}], + "design_artifact": {"format": "freeform_document", "content": reply}, + } + + if tool == "comap": + global_prompt = materials["comap_ai_agent_prompts"]["global_agent"]["system_prompt"] + participant_opening = agent_fn( + role, + ( + "You are a study participant using the CoMAP graph canvas tool to design a " + f"PBL unit. Your task ({task_title}): {task_prompt} " + "Address the Global Agent with your first request. " + "If you want to invoke the Refine or Split agent on a node you have already " + "created, include a line 'ACTION: REFINE node_id=' or " + "'ACTION: SPLIT node_id='." + ), + [], + ) + global_reply = agent_fn("comap_global_agent", global_prompt, [ + {"role": "user", "content": participant_opening} + ]) + node_registry = {} + actions = _extract_add_actions(global_reply) + for act in actions: + node_registry[act.get("card_id") or act.get("id") or str(len(node_registry) + 1)] = act + local_actions = route_local_agent_actions(participant_opening, materials, agent_fn, node_registry) + return { + "tool": "comap", + "task": task_key, + "block": block_number, + "transcript": [ + {"role": "user", "content": participant_opening}, + {"role": "assistant", "content": global_reply}, + ], + "design_artifact": {"format": "structured_graph", "nodes": node_registry}, + "local_agent_invocations": local_actions, + } + + raise ValueError(f"unknown tool: {tool}") + + +def _extract_add_actions(global_agent_reply): + """Best-effort extraction of JSON action blocks from a Global Agent reply.""" + actions = [] + for match in re.finditer(r"\{.*?\"actions\"\s*:\s*\[.*?\]\s*\}", global_agent_reply, re.DOTALL): + try: + parsed = json.loads(match.group(0)) + actions.extend(parsed.get("actions", [])) + except json.JSONDecodeError: + continue + return actions + + +def run_sharing(presenter_role, listener_role, design_artifact, agent_fn): + presentation = agent_fn( + presenter_role, + "You are presenting your finished PBL design to a partner in a brief mock-teaching format.", + [{"role": "user", "content": json.dumps(design_artifact)}], + ) + listener_summary = agent_fn( + listener_role, + "Your partner just presented their PBL design to you in a mock-teaching format. " + "Briefly summarize what you understood.", + [{"role": "user", "content": presentation}], + ) + return {"presentation": presentation, "listener_summary": listener_summary} + + +def run_post_task_questionnaires(role, agent_fn, materials, understanding_target_present): + responses = {} + scale_key = "human_ai_interaction_experience" + for subscale in materials["post_task_questionnaires"][scale_key]["subscales"]: + answers = [] + for item in subscale["items"]: + reply = agent_fn(role, "Answer on a 1-7 scale (1=Strongly Disagree, 7=Strongly Agree). Respond with ONLY the integer.", [ + {"role": "user", "content": item} + ]) + answers.append({"item": item, "response_raw": reply}) + responses[subscale["id"]] = answers + + expr_items = materials["post_task_questionnaires"]["perceived_expression"]["items"] + responses["perceived_expression"] = [ + {"item": item, "response_raw": agent_fn(role, "Answer on a 1-7 scale (1=Strongly Disagree, 7=Strongly Agree). Respond with ONLY the integer.", [{"role": "user", "content": item}])} + for item in expr_items + ] + + if understanding_target_present: + und_items = materials["post_task_questionnaires"]["perceived_understanding"]["items"] + responses["perceived_understanding"] = [ + {"item": item, "response_raw": agent_fn(role, "Answer on a 1-7 scale (1=Strongly Disagree, 7=Strongly Agree), about your partner's design you just saw. Respond with ONLY the integer.", [{"role": "user", "content": item}])} + for item in und_items + ] + return responses + + +# --------------------------------------------------------------------------- +# Dyad-level session +# --------------------------------------------------------------------------- + +def run_dyad_session(dyad_id, agent_a_order, agent_a_task_order, agent_b_order, agent_b_task_order, + agent_fn, materials, baseline_system_prompt=None): + if baseline_system_prompt: + materials = dict(materials) + materials["baseline_condition_override_system_prompt"] = baseline_system_prompt + + transcript = {"dyad_id": dyad_id, "status": "ok", "agents": {}} + agents = { + "agent_a": {"order": agent_a_order, "task_order": agent_a_task_order}, + "agent_b": {"order": agent_b_order, "task_order": agent_b_task_order}, + } + + for role in ("agent_a", "agent_b"): + transcript["agents"][role] = {"background_survey": run_background_survey(role, agent_fn, materials)} + + block_designs = {"agent_a": {}, "agent_b": {}} + blocked_reason = None + + for block_number in (1, 2): + for role in ("agent_a", "agent_b"): + cfg = agents[role] + tool = tool_for_block(cfg["order"], block_number) + task_key = task_for_block(cfg["task_order"], block_number) + try: + result = run_design_task(role, agent_fn, materials, tool, task_key, block_number) + block_designs[role][block_number] = result + except BlockedError as e: + blocked_reason = {"missing_field": e.missing_field, "reason": e.reason, "role": role, "block": block_number} + block_designs[role][block_number] = {"status": "blocked", **blocked_reason} + + sharing = None + a_design = block_designs["agent_a"][block_number] + b_design = block_designs["agent_b"][block_number] + if a_design.get("status") != "blocked" and b_design.get("status") != "blocked": + sharing_a_to_b = run_sharing("agent_a", "agent_b", a_design["design_artifact"], agent_fn) + sharing_b_to_a = run_sharing("agent_b", "agent_a", b_design["design_artifact"], agent_fn) + sharing = {"a_presents_to_b": sharing_a_to_b, "b_presents_to_a": sharing_b_to_a} + + for role in ("agent_a", "agent_b"): + design = block_designs[role][block_number] + key = f"block{block_number}" + transcript["agents"][role][key] = {"design_task": design} + if design.get("status") == "blocked": + transcript["agents"][role][key]["questionnaires"] = {"status": "blocked", **blocked_reason} + transcript["status"] = "blocked" + else: + transcript["agents"][role][key]["questionnaires"] = run_post_task_questionnaires( + role, agent_fn, materials, understanding_target_present=sharing is not None + ) + if sharing is not None: + transcript[f"block{block_number}_sharing"] = sharing + + return transcript + + +# --------------------------------------------------------------------------- +# Formative study (Study 1) -- Stage 1 only; Stage 2 is blocked (see audit) +# --------------------------------------------------------------------------- + +def run_formative_session(participant_id, agent_fn, materials): + scripts = materials["interview_scripts"]["formative_study"]["stage_1_semi_structured_interview"] + role = f"formative_participant_{participant_id}" + transcript = {"participant_id": participant_id, "stage_1": {}, "stage_2": {"status": "blocked", "missing_field": "formative_stage2_script"}} + for section, questions in scripts.items(): + answers = [] + for q in questions: + reply = agent_fn(role, "You are a PBL practitioner being interviewed about your design practice and AI use.", [ + {"role": "user", "content": q} + ]) + answers.append({"question": q, "answer": reply}) + transcript["stage_1"][section] = answers + return transcript + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _stub_agent_fn(role, system_prompt, messages): + """ + Deterministic, offline, no-network stand-in used only by --smoke-test. + Clearly labeled placeholder text -- never presented as paper content. + """ + last_user = messages[-1]["content"] if messages else "" + if re.search(r"1-7 scale", system_prompt) or "1-7 scale" in last_user: + return "5" + if role == "comap_global_agent": + return json.dumps({ + "actions": [ + {"option": "add", "type": "Activity", "title": "[SMOKE-TEST PLACEHOLDER] Sample Activity", + "description": "[SMOKE-TEST PLACEHOLDER] generated by the stub agent, not from the paper."} + ] + }) + if role in ("comap_refine_agent",): + return json.dumps({"new_node": {"id": "1", "title": "[SMOKE-TEST PLACEHOLDER]", "description": "...", "tag": "Activity"}}) + if role in ("comap_split_agent",): + return json.dumps({"old_node_id": "1", "new_nodes": [{"title": "[SMOKE-TEST PLACEHOLDER] part 1", "description": "...", "tag": "Activity"}]}) + return f"[SMOKE-TEST PLACEHOLDER response from {role}]" + + +def _run_smoke_test(): + materials = load_materials() + task = load_task() + assert get_study(task, "study_1_formative") + assert get_study(task, "study_2_main") + + print("[1/3] Running one formative-study participant (Stage 1 only)...") + formative_result = run_formative_session(1, _stub_agent_fn, materials) + assert formative_result["stage_1"] + assert formative_result["stage_2"]["status"] == "blocked" + print(" ok -- stage 1 produced answers, stage 2 correctly reports blocked.") + + print("[2/3] Running one dyad through Block 1/Block 2 in the CoMAP-only path...") + # Force both agents to be CoMAP-first-and-second is impossible (crossover + # requires one Baseline block); instead verify the CoMAP-condition code + # path directly without going through the Baseline-blocking dyad runner. + materials_copy = dict(materials) + comap_result = run_design_task("agent_a", _stub_agent_fn, materials_copy, tool="comap", task_key="task_a", block_number=1) + assert comap_result["tool"] == "comap" + assert comap_result["design_artifact"]["nodes"] + print(" ok -- CoMAP design task produced a structured design artifact.") + + print("[3/3] Verifying the Baseline arm is correctly blocked without a supplied system prompt...") + blocked = False + try: + run_design_task("agent_a", _stub_agent_fn, materials, tool="baseline", task_key="task_a", block_number=1) + except BlockedError as e: + blocked = e.missing_field == "baseline_system_prompt" + assert blocked, "expected a BlockedError for the Baseline arm with no system prompt supplied" + print(" ok -- Baseline arm reports a structured blocked state instead of fabricating a prompt.") + + print("\nSMOKE TEST PASSED. No network access was used. All output above is placeholder text from a stub agent, not paper content.") + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="CoMAP human-study package adapter") + parser.add_argument("--smoke-test", action="store_true", help="run an offline self-test with a stub agent") + parser.add_argument("--study", choices=["study_1_formative", "study_2_main"], default="study_2_main") + parser.add_argument("--dyad-id", type=int, default=1) + parser.add_argument("--agent-a-order", choices=["A", "B"], default="A") + parser.add_argument("--agent-a-task-order", type=int, choices=[1, 2], default=1) + parser.add_argument("--agent-b-order", choices=["A", "B"], default="B") + parser.add_argument("--agent-b-task-order", type=int, choices=[1, 2], default=2) + parser.add_argument("--baseline-system-prompt-file", type=str, default=None, + help="path to a researcher-supplied Baseline system prompt (required to unblock the Baseline arm)") + parser.add_argument("--agent-fn-module", type=str, default=None, + help="'module:function' providing agent_fn(role, system_prompt, messages) -> str; " + "required for a real (non-smoke-test) run") + args = parser.parse_args() + + if args.smoke_test: + sys.exit(_run_smoke_test()) + + if not args.agent_fn_module: + print("ERROR: --agent-fn-module is required for a real run (or use --smoke-test).", file=sys.stderr) + sys.exit(2) + + module_name, func_name = args.agent_fn_module.split(":") + import importlib + module = importlib.import_module(module_name) + agent_fn = getattr(module, func_name) + + materials = load_materials() + baseline_prompt = None + if args.baseline_system_prompt_file: + baseline_prompt = Path(args.baseline_system_prompt_file).read_text(encoding="utf-8") + + if args.study == "study_1_formative": + result = run_formative_session(args.dyad_id, agent_fn, materials) + else: + result = run_dyad_session( + args.dyad_id, args.agent_a_order, args.agent_a_task_order, + args.agent_b_order, args.agent_b_task_order, + agent_fn, materials, baseline_system_prompt=baseline_prompt, + ) + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/studies/study_016/task/task.json b/studies/study_016/task/task.json new file mode 100644 index 0000000..0db428f --- /dev/null +++ b/studies/study_016/task/task.json @@ -0,0 +1,156 @@ +{ + "_explanation": "Defines the runnable agent interaction for BOTH empirical studies in the paper. Study 2 (the main user study, n=30) is the fully specified, quantitatively-measured study and is implemented in full by task/adapter.py. Study 1 (the formative study, n=7) is documented here for completeness and fidelity (every original subject must be an agent) and is implemented as a simpler secondary mode of the same adapter. task/adapter.py reads this file at runtime; do not rename fields without updating the adapter.", + + "package_entry_point": "task/adapter.py", + "how_to_run_smoke_test": "python task/adapter.py --smoke-test", + "how_to_run_a_session": "python task/adapter.py --study study_2_main --dyad-id --agent-a-order A --agent-a-task-order 1 --agent-b-order B --agent-b-task-order 2 --agent-fn-module [--baseline-system-prompt-file ]", + + "studies": [ + { + "id": "study_1_formative", + "name": "Formative Study", + "paper_section": "Section 3 (pages 3-4)", + "purpose": "Qualitative study that produced the challenges (C1-C3) and design goals (DG1-DG3) motivating CoMAP. Not a test of CoMAP itself -- CoMAP did not exist yet at this stage.", + "num_agents": 7, + "agent_roles": [ + {"role": "pbl_practitioner", "count": 7, "description": "Each of the 7 formative-study agents plays one of the 7 recruited practitioners (6 female, 1 male; ages 19-45, M=26.4, SD=10.6; two educational-technology background, two pre-service teachers, one novice teacher (1 yr experience), one curriculum researcher, one science-education researcher). Evidence label: verbatim demographic composition (source: input/paper.pdf p.4, Section 3.1); individual identities/assignment of these traits to specific agents beyond the aggregate roster is not printed and is left to the researcher (see audit/missing_information.json 'formative_participant_identity_assignment')."} + ], + "scripted_roles": [ + { + "role": "interviewer_facilitator", + "scripted": true, + "justification": "The paper attributes the interview and think-aloud facilitation to 'the research team' (page 4), i.e., the paper's own investigators, not a recruited subject. Per the fidelity rules, roles the paper itself scripts (experimenters/confederates) may remain scripted; only roles the paper identifies as subjects must be agents. The interviewer here follows the fixed verbatim script in materials.json 'interview_scripts.formative_study' for Stage 1; Stage 2's probe questions are not printed verbatim (see missing_information.json) so the facilitator must be flagged blocked for that portion until a researcher supplies the probe script or approves an improvised-but-labeled substitute." + } + ], + "order_and_visibility": "Each of the 7 sessions is run one-on-one and independently (paper: 'conducted one-on-one via online meetings', 45-75 minutes each). There is no cross-participant visibility in this study -- no participant sees another participant's answers, prototype feedback, or transcript. Sessions may be run in any order or in parallel with no effect on the design.", + "procedure": [ + {"step": 1, "name": "Stage 1: Semi-structured interview", "content_ref": "materials.json interview_scripts.formative_study.stage_1_semi_structured_interview", "evidence_label": "verbatim"}, + {"step": 2, "name": "Stage 2: Think-aloud with low-fidelity static prototype (design probe)", "content_ref": "materials.json interview_scripts.formative_study.stage_2_think_aloud_probe", "evidence_label": "reported/missing", "note": "Goals of this stage are reported verbatim-in-substance; the actual probe artifact and question script are not printed in the paper and are therefore not implementable without researcher input."} + ], + "outputs": ["interview_transcript_stage1 (per agent)", "think_aloud_notes_stage2 (per agent, blocked pending missing script)"], + "readiness": "blocked_partial: Stage 1 is fully runnable; Stage 2 is blocked because the design-probe prototype and its question script are not printed in the source PDF." + }, + { + "id": "study_2_main", + "name": "Main User Study: CoMAP vs. Baseline (within-subjects crossover, dyadic sharing)", + "paper_section": "Section 5 (pages 11-16) and Appendix A-C", + "purpose": "Quantitative + qualitative comparison of CoMAP (graph canvas + dual-modality AI) against a dialogue-only Baseline AI, for PBL instructional design, RQ1 (design expression/understanding) and RQ2 (human-AI interaction experience).", + "num_human_participants_in_paper": 30, + "num_agents_per_replication_unit": 2, + "replication_unit": "dyad", + "num_dyads_for_full_sample": 15, + "note_on_unit": "The paper's 30 subjects are organized into 15 unacquainted dyads for the 'sharing' step. Each participant's individual design task and questionnaires are otherwise independent of their partner. This package's runnable unit is one dyad (2 agents); running it 15 times (dyad_id=1..15) with the counterbalancing scheme below reproduces the full n=30 sample. Do not collapse the dyad into two independent solo runs -- the Sharing step requires each partner to actually see the other's just-produced design (see 'visibility' below), which a solo run cannot provide.", + + "agent_roles": [ + {"role": "participant", "count_per_session": 2, "labels": ["agent_a", "agent_b"], "description": "Both are full human-subject roles in the original study (in-service teachers, pre-service teachers, or curriculum designers; sample-level demographics: 23 female, 7 male, age M=24.2, SD=3.63, recruited from 21 cities; all reported prior PBL-design and AI-tool experience). Evidence label: verbatim (source: input/paper.pdf p.11, Section 5.1). Per-agent identity beyond the aggregate roster is not specified in the paper and is a researcher decision (audit/missing_information.json 'main_study_participant_identity_assignment')."} + ], + + "scripted_roles": [ + { + "role": "comap_global_agent", + "scripted": true, + "justification": "The paper specifies this exactly as a fixed GPT-4.1 system prompt driving the tool under test (Appendix C.1), not as a study subject. It is the experimental apparatus for the CoMAP condition, analogous to a scripted confederate reading fixed lines -- it must remain scripted to faithfully replay the paper's manipulation rather than becoming an independent free-form agent.", + "system_prompt_ref": "materials.json comap_ai_agent_prompts.global_agent" + }, + { + "role": "comap_refine_agent", + "scripted": true, + "justification": "Same reasoning as comap_global_agent: fixed system prompt (Appendix C.2), part of the apparatus.", + "system_prompt_ref": "materials.json comap_ai_agent_prompts.refine_agent" + }, + { + "role": "comap_split_agent", + "scripted": true, + "justification": "Same reasoning as comap_global_agent: fixed system prompt (Appendix C.3), part of the apparatus.", + "system_prompt_ref": "materials.json comap_ai_agent_prompts.split_agent" + }, + { + "role": "baseline_dialogue_agent", + "scripted": true, + "justification": "Also apparatus, not a subject -- but the paper never prints its system prompt (only 'a standard, dialogue-based AI'). This role is BLOCKED: task/adapter.py refuses to fabricate a system prompt and returns a structured blocked result unless the researcher supplies one via --baseline-system-prompt-file. See materials.json baseline_condition and audit/missing_information.json 'baseline_system_prompt'.", + "system_prompt_ref": "MISSING -- researcher must supply" + } + ], + + "condition_assignment": { + "factors": [ + { + "name": "condition_order", + "values": ["A", "B"], + "meaning": {"A": "Baseline in Block 1, CoMAP in Block 2", "B": "CoMAP in Block 1, Baseline in Block 2"}, + "evidence_label": "verbatim", + "source": {"file": "input/paper.pdf", "page": 11, "section": "5.2 Study Design"} + }, + { + "name": "task_order", + "values": [1, 2], + "meaning": {"1": "Task A (food crops) in Block 1, Task B (population aging) in Block 2", "2": "Task B in Block 1, Task A in Block 2"}, + "evidence_label": "derived", + "derivation": "The paper states the two tasks 'were counterbalanced with the conditions to control for task-specific effects' (page 11) but does not print the exact 2x2 cell sizes. This factor is modeled as an independent binary counterbalancing factor crossed with condition_order, which is the standard way to realize the stated counterbalancing; exact cell sizes among the 30 participants are not specified (audit/missing_information.json 'counterbalancing_cell_sizes')." + } + ], + "assignment_is_per_participant_not_per_dyad": true, + "note": "condition_order and task_order are assigned independently to agent_a and agent_b. The paper pairs 'unacquainted' partners at random, and does not state that dyad partners share a condition-order group -- so a dyad may contain, e.g., one Group-A and one Group-B participant. Both partners still move through Block 1 and Block 2 in lockstep so the Sharing step can occur at a shared point in time, but each may be using a different tool within the same block." + }, + + "visibility": { + "principle": "A participant sees only their own private canvas/chat/document while performing a design task. They see their partner's finished design ONLY during the Sharing step of the same block, and they must see the design the partner actually produced in this run (not an idealized, cached, or paraphrased version).", + "per_step_visibility": [ + {"step": "background_survey", "visible_to": ["self"], "visible_content": "own answers only"}, + {"step": "platform_training", "visible_to": ["self"], "visible_content": "tutorial content for both tools (procedural, not a measured artifact)"}, + {"step": "block1_task", "visible_to": ["self"], "visible_content": "own design canvas/chat/document as it is built; NOT the partner's task, canvas, or chat"}, + {"step": "block1_sharing", "visible_to": ["self", "partner"], "visible_content": "each agent's finished Block-1 design artifact (the exact structured content it produced) is shown to its partner in a mock-teaching presentation; this is the one deliberate cross-agent visibility point in the study"}, + {"step": "block1_questionnaires", "visible_to": ["self"], "visible_content": "Perceived Understanding items are answered ABOUT the partner's design just seen in block1_sharing; Perceived Expression and Human-AI Interaction Experience items are answered about the agent's own task/tool experience"}, + {"step": "block2_task", "visible_to": ["self"], "visible_content": "own design canvas/chat/document with the OTHER tool; NOT the partner's task, canvas, or chat"}, + {"step": "block2_sharing", "visible_to": ["self", "partner"], "visible_content": "each agent's finished Block-2 design artifact is shown to its partner"}, + {"step": "block2_comparative_questionnaires", "visible_to": ["self"], "visible_content": "Perceived Understanding about partner's Block-2 design; Perceived Expression and Human-AI Interaction Experience about own Block-2 tool; plus the comparative interview (B.2.2) which references the agent's own memory of both of ITS OWN blocks (not the partner's)"} + ], + "why_this_matters": "Collapsing the two partners into independent solo runs (skipping the sharing exchange, or letting the evaluator author 'representative' partner designs) would remove the actual dependent variable being measured in RQ1 (Perceived Understanding of a real partner's real, possibly messy, artifact) and must not be done." + }, + + "procedure": [ + {"step": 1, "name": "Onboarding and Background Survey", "detail": "Both agents individually complete the Intelligent-TPACK background survey.", "materials_ref": "materials.json background_survey", "evidence_label": "verbatim"}, + {"step": 2, "name": "Platform Training", "detail": "Both agents receive a tutorial/demo of both systems. The paper does not print the tutorial script; adapter.py treats this as a no-op procedural step (audit/missing_information.json 'training_script').", "evidence_label": "reported"}, + {"step": 3, "name": "Experimental Block 1: Task", "detail": "Each agent individually completes its Block-1 task (Task A or Task B per task_order) using its Block-1 tool (Baseline or CoMAP per condition_order). If Baseline is required and no system prompt has been supplied, adapter.py returns status=blocked for this agent's Block-1 task.", "materials_ref": "materials.json design_tasks + comap_ai_agent_prompts (+ baseline_condition if unblocked)"}, + {"step": 4, "name": "Experimental Block 1: Sharing", "detail": "Each agent presents its just-produced Block-1 design to its partner in a brief mock-teaching format; the partner's agent receives the literal design artifact produced in step 3.", "materials_ref": "materials.json sharing_task", "evidence_label": "reported"}, + {"step": 5, "name": "Experimental Block 1: Interview and Questionnaires 1", "detail": "Each agent individually answers B.2.1 General Feedback interview questions, then the Perceived Understanding scale (about the partner's design just shared), the Perceived Expression scale, and the Human-AI Interaction Experience scale (Controllable/Transparent/Cognitive Load/Collaboration/Trust), both about its own Block-1 tool.", "materials_ref": "materials.json interview_scripts.main_study.b2_1_general_feedback + post_task_questionnaires"}, + {"step": 6, "name": "Experimental Block 2: Task", "detail": "Each agent switches to the OTHER tool (the one it did not use in Block 1) and completes its Block-2 task (the other task from Block 1, per task_order).", "materials_ref": "materials.json design_tasks + comap_ai_agent_prompts (+ baseline_condition if unblocked)"}, + {"step": 7, "name": "Experimental Block 2: Sharing", "detail": "Same structure as step 4, for Block 2 designs."}, + {"step": 8, "name": "Experimental Block 2: Comparative Interview and Questionnaires 2", "detail": "Each agent answers B.2.2 Comparing Different Tools, then completes Questionnaires 2 (Perceived Understanding about the partner's Block-2 design, Perceived Expression, Human-AI Interaction Experience) for its Block-2 tool.", "materials_ref": "materials.json interview_scripts.main_study.b2_2_comparing_different_tools + post_task_questionnaires"} + ], + + "local_agent_invocation_protocol": { + "evidence_label": "derived", + "derivation": "The paper's Refine/Split agents are invoked by right-clicking a node in a graphical canvas (page 8, 4.3.2) -- an action that has no direct analogue in a text-only multi-agent replication. To preserve the three-agent structure (Global + Refine + Split) described in the paper without silently dropping two of the three scripted AI roles, this package defines a minimal textual control protocol: while in the CoMAP condition, the participant agent's turn may include a line of the form 'ACTION: REFINE node_id=' or 'ACTION: SPLIT node_id=' referencing a node it has already created; adapter.py intercepts this line and routes a call to the corresponding scripted agent (Refine or Split) using the verbatim system prompt, then returns the structured before/after (Refine) or original/split-set (Split) result to the participant agent, matching the paper's 'before-and-after comparison UI' at a content (not pixel) level.", + "researcher_flag": "This protocol is a faithful-intent substitute for the GUI interaction, not a verbatim reproduction of it (there is no verbatim text to reproduce for a mouse click). Recorded in audit/missing_information.json 'gui_interaction_fidelity' so a researcher can judge whether this substitution is acceptable for their analysis." + }, + + "inputs_per_agent": ["dyad_id", "role (agent_a|agent_b)", "condition_order (A|B)", "task_order (1|2)", "agent_fn (a callable(role, system_prompt, message_history) -> str, supplied by the harness/runtime that actually drives the LLM playing this agent; task/adapter.py never calls a network API itself)"], + "outputs_per_session": [ + "background_survey_responses (per agent, per TPACK subscale item)", + "block1_design_artifact (per agent: the structured list of nodes/edges or, if Baseline, the freeform document text)", + "block1_chat_transcript (per agent, if CoMAP: Global Agent dialogue + any Refine/Split invocations; if Baseline: dialogue-only transcript)", + "block1_sharing_transcript (per dyad)", + "block1_questionnaire_responses (per agent: Perceived Understanding, Perceived Expression, Controllable, Transparent, Cognitive Load, Collaboration, Trust)", + "block1_interview_responses (per agent: B.2.1 answers)", + "block2_* (same shape as block1_*, for the other tool/task)", + "block2_comparative_interview_responses (per agent: B.2.2 answers)", + "behavioral_metrics (per agent, per block, derived from the transcript: total_turns, avg_user_message_length_chars, negative_keyword_count, node_to_edge_ratio [CoMAP only], avg_distance_between_consecutively_created_nodes [CoMAP only, requires canvas x/y coordinates the agent is asked to assign when creating nodes -- see audit/missing_information.json 'canvas_coordinate_assignment'])", + "status (ok | blocked, with a machine-readable reason when blocked)" + ], + + "readiness": "ready_with_gaps: fully runnable for the CoMAP arm of both blocks; the Baseline arm is blocked pending a researcher-supplied system prompt (audit/missing_information.json 'baseline_system_prompt'). Behavioral metrics that depend on 2D canvas coordinates are best-effort until a researcher confirms how an agent should self-report node placement." + } + ], + + "response_format_for_all_likert_items": { + "evidence_label": "derived", + "value": "integer 1-7 inclusive", + "derivation": "See materials.json post_task_questionnaires.response_format. Not stated verbatim in the paper; used as the runnable default and overridable via adapter.py CLI/config." + }, + + "global_run_parameters": { + "n_dyads_for_full_replication": 15, + "counterbalancing_scheme_default": "cycle dyad_id % 4 through the 4 cells of {condition_order in [A,B]} x {task_order in [1,2]}, independently assigned to agent_a and agent_b -- a researcher-adjustable default, since exact original cell sizes are not printed (see audit/missing_information.json 'counterbalancing_cell_sizes')." + } +}