From 63f5928f5c2b426989de2ec89aad67a83f0c06d1 Mon Sep 17 00:00:00 2001 From: LaoKuiZe <3064554432@qq.com> Date: Tue, 7 Jul 2026 23:50:16 +0800 Subject: [PATCH 01/13] feat: migrate generation pipeline into Hub --- .gitignore | 16 + README.md | 60 + config/settings.example.yaml | 100 + generation_pipeline/__init__.py | 1 + generation_pipeline/connectors/__init__.py | 17 + .../connectors/base_connector.py | 415 ++ .../connectors/osf_connector.py | 150 + .../connectors/osf_link_discovery.py | 127 + generation_pipeline/connectors/registry.py | 505 +++ generation_pipeline/effect_selector.py | 211 + generation_pipeline/extractors/__init__.py | 4 + .../extractors/base_extractor.py | 34 + .../extractors/grounded_slot_extractor.py | 907 ++++ .../extractors/study_data_extractor.py | 292 ++ generation_pipeline/fetch_osf.py | 127 + generation_pipeline/fill_osf.py | 51 + generation_pipeline/filters/__init__.py | 4 + generation_pipeline/filters/base_filter.py | 39 + .../filters/replicability_filter.py | 272 ++ generation_pipeline/generators/__init__.py | 3 + .../generators/config_generator.py | 358 ++ generation_pipeline/normalize_schema.py | 41 + generation_pipeline/outputs/.gitkeep | 0 generation_pipeline/parsers/__init__.py | 51 + .../parsers/effect_consolidator.py | 213 + .../parsers/material_assembler.py | 766 ++++ generation_pipeline/parsers/qsf_parser.py | 430 ++ generation_pipeline/parsers/sav_parser.py | 294 ++ generation_pipeline/parsers/source_linker.py | 163 + generation_pipeline/parsers/stage3_adapter.py | 290 ++ generation_pipeline/patchers/__init__.py | 3 + generation_pipeline/patchers/patch_runner.py | 267 ++ generation_pipeline/patchers/slot_filler.py | 684 ++++ generation_pipeline/pipeline.py | 735 ++++ generation_pipeline/pipeline_interface.py | 20 + generation_pipeline/run.py | 625 +++ generation_pipeline/settings.py | 537 +++ generation_pipeline/simulation_targets.py | 342 ++ generation_pipeline/split_replicable.py | 141 + generation_pipeline/stage1_study_contract.py | 127 + generation_pipeline/stage1_verifier.py | 179 + .../stage2_finding_contract.py | 275 ++ generation_pipeline/stage2_findings.py | 181 + generation_pipeline/stage2_verifier.py | 240 ++ generation_pipeline/stage3.py | 1634 ++++++++ .../stage3_material_contract.py | 341 ++ generation_pipeline/stage3_pdf_materials.py | 1592 ++++++++ generation_pipeline/stage4.py | 2127 ++++++++++ generation_pipeline/stage5.py | 289 ++ generation_pipeline/study_selector.py | 245 ++ generation_pipeline/utils/__init__.py | 6 + generation_pipeline/utils/document_loader.py | 42 + generation_pipeline/utils/json_generator.py | 1032 +++++ generation_pipeline/utils/osf_crawler.py | 544 +++ generation_pipeline/utils/osf_fill.py | 218 + generation_pipeline/utils/output_formatter.py | 314 ++ generation_pipeline/utils/pdf_chunker.py | 309 ++ generation_pipeline/utils/pdf_extractor.py | 221 + generation_pipeline/utils/review_parser.py | 134 + generation_pipeline/verification/__init__.py | 9 + .../verification/schema_validator.py | 1083 +++++ .../verification/stage_quality.py | 324 ++ .../verification/verbatim_verifier.py | 1294 ++++++ pyproject.toml | 7 + requirements.txt | 5 + src/agents/__init__.py | 5 + src/agents/background_loader.py | 142 + src/agents/background_template.txt | 33 + src/agents/base_agent.py | 72 + src/agents/custom_methods/__init__.py | 15 + src/agents/custom_methods/example_v4.py | 48 + src/agents/custom_methods/template.py | 51 + src/agents/custom_methods/v1_empty.py | 25 + src/agents/custom_methods/v2_human.py | 24 + .../custom_methods/v3_human_plus_demo.py | 51 + src/agents/custom_methods/v4_background.py | 151 + src/agents/generate_agent_background.py | 313 ++ src/agents/llm_participant_agent.py | 569 +-- src/agents/prompt_builder.py | 227 ++ src/agents/prompt_registry.py | 28 +- src/core/__init__.py | 6 + src/core/benchmark.py | 204 +- src/core/exceptions.py | 5 + src/core/study.py | 77 +- src/core/study_config.py | 547 ++- src/evaluation/__init__.py | 4 + src/evaluation/evaluator_runner.py | 135 +- src/evaluation/finding_explainer.py | 150 + src/evaluation/metrics.py | 84 + src/evaluation/response_validator.py | 250 ++ src/evaluation/sanity_check.py | 773 ++++ src/evaluation/scorer.py | 51 + src/evaluation/standardizers.py | 194 + src/evaluation/stats_lib.py | 3629 +++++++++++++++++ src/llm/__init__.py | 18 + src/llm/anthropic_client.py | 132 + src/llm/base.py | 80 + src/llm/factory.py | 57 + src/llm/helpers.py | 294 ++ src/llm/openai_client.py | 123 + src/llm/xai_client.py | 28 + 101 files changed, 30055 insertions(+), 607 deletions(-) create mode 100644 config/settings.example.yaml create mode 100644 generation_pipeline/__init__.py create mode 100644 generation_pipeline/connectors/__init__.py create mode 100644 generation_pipeline/connectors/base_connector.py create mode 100644 generation_pipeline/connectors/osf_connector.py create mode 100644 generation_pipeline/connectors/osf_link_discovery.py create mode 100644 generation_pipeline/connectors/registry.py create mode 100644 generation_pipeline/effect_selector.py create mode 100644 generation_pipeline/extractors/__init__.py create mode 100644 generation_pipeline/extractors/base_extractor.py create mode 100644 generation_pipeline/extractors/grounded_slot_extractor.py create mode 100644 generation_pipeline/extractors/study_data_extractor.py create mode 100644 generation_pipeline/fetch_osf.py create mode 100644 generation_pipeline/fill_osf.py create mode 100644 generation_pipeline/filters/__init__.py create mode 100644 generation_pipeline/filters/base_filter.py create mode 100644 generation_pipeline/filters/replicability_filter.py create mode 100644 generation_pipeline/generators/__init__.py create mode 100644 generation_pipeline/generators/config_generator.py create mode 100644 generation_pipeline/normalize_schema.py create mode 100644 generation_pipeline/outputs/.gitkeep create mode 100644 generation_pipeline/parsers/__init__.py create mode 100644 generation_pipeline/parsers/effect_consolidator.py create mode 100644 generation_pipeline/parsers/material_assembler.py create mode 100644 generation_pipeline/parsers/qsf_parser.py create mode 100644 generation_pipeline/parsers/sav_parser.py create mode 100644 generation_pipeline/parsers/source_linker.py create mode 100644 generation_pipeline/parsers/stage3_adapter.py create mode 100644 generation_pipeline/patchers/__init__.py create mode 100644 generation_pipeline/patchers/patch_runner.py create mode 100644 generation_pipeline/patchers/slot_filler.py create mode 100644 generation_pipeline/pipeline.py create mode 100644 generation_pipeline/pipeline_interface.py create mode 100644 generation_pipeline/run.py create mode 100644 generation_pipeline/settings.py create mode 100644 generation_pipeline/simulation_targets.py create mode 100644 generation_pipeline/split_replicable.py create mode 100644 generation_pipeline/stage1_study_contract.py create mode 100644 generation_pipeline/stage1_verifier.py create mode 100644 generation_pipeline/stage2_finding_contract.py create mode 100644 generation_pipeline/stage2_findings.py create mode 100644 generation_pipeline/stage2_verifier.py create mode 100644 generation_pipeline/stage3.py create mode 100644 generation_pipeline/stage3_material_contract.py create mode 100644 generation_pipeline/stage3_pdf_materials.py create mode 100644 generation_pipeline/stage4.py create mode 100644 generation_pipeline/stage5.py create mode 100644 generation_pipeline/study_selector.py create mode 100644 generation_pipeline/utils/__init__.py create mode 100644 generation_pipeline/utils/document_loader.py create mode 100644 generation_pipeline/utils/json_generator.py create mode 100644 generation_pipeline/utils/osf_crawler.py create mode 100644 generation_pipeline/utils/osf_fill.py create mode 100644 generation_pipeline/utils/output_formatter.py create mode 100644 generation_pipeline/utils/pdf_chunker.py create mode 100644 generation_pipeline/utils/pdf_extractor.py create mode 100644 generation_pipeline/utils/review_parser.py create mode 100644 generation_pipeline/verification/__init__.py create mode 100644 generation_pipeline/verification/schema_validator.py create mode 100644 generation_pipeline/verification/stage_quality.py create mode 100644 generation_pipeline/verification/verbatim_verifier.py create mode 100644 src/agents/__init__.py create mode 100644 src/agents/background_loader.py create mode 100644 src/agents/background_template.txt create mode 100644 src/agents/base_agent.py create mode 100644 src/agents/custom_methods/__init__.py create mode 100644 src/agents/custom_methods/example_v4.py create mode 100644 src/agents/custom_methods/template.py create mode 100644 src/agents/custom_methods/v1_empty.py create mode 100644 src/agents/custom_methods/v2_human.py create mode 100644 src/agents/custom_methods/v3_human_plus_demo.py create mode 100644 src/agents/custom_methods/v4_background.py create mode 100644 src/agents/generate_agent_background.py create mode 100644 src/agents/prompt_builder.py create mode 100644 src/core/__init__.py create mode 100644 src/evaluation/finding_explainer.py create mode 100644 src/evaluation/metrics.py create mode 100644 src/evaluation/response_validator.py create mode 100644 src/evaluation/sanity_check.py create mode 100644 src/evaluation/scorer.py create mode 100644 src/evaluation/standardizers.py create mode 100644 src/evaluation/stats_lib.py create mode 100644 src/llm/__init__.py create mode 100644 src/llm/anthropic_client.py create mode 100644 src/llm/base.py create mode 100644 src/llm/factory.py create mode 100644 src/llm/helpers.py create mode 100644 src/llm/openai_client.py create mode 100644 src/llm/xai_client.py diff --git a/.gitignore b/.gitignore index 71cb687..1e6147a 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,22 @@ logs/ .env .env.local +# Local pipeline settings may contain API keys +config/settings.yaml +config/settings.yml +config/settings.json + +# Generated pipeline outputs and simulation runs +generation_pipeline/outputs/* +!generation_pipeline/outputs/.gitkeep +generation_pipeline/old_outputs/ +generation_pipeline/**/__pycache__/ +runs/ +runs_*/ +experiments/ +experiments_*/ +.cache/stage5/ + # OS Thumbs.db diff --git a/README.md b/README.md index b261a49..66f4ee4 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,66 @@ HumanStudy-Bench treats participant simulation as an *agent design problem* and provides a **standardized testbed** — combining an **Execution Engine** that reconstructs full experimental protocols from published studies and a **Benchmark** with standardized evaluation metrics — for *replaying human-subject experiments end-to-end* with alignment evaluation at the level of scientific inference. +## Automated Study Generation Pipeline + +This repo also includes a staged generation pipeline for drafting a Hub study +folder from a paper PDF and optional OSF/supplementary sources. + +The pipeline is intentionally human-in-the-loop: + +- Stage 1 identifies paper metadata and study inventory. +- Stage 2 extracts study-level findings, effects, samples, and statistics. +- Stage 3 assembles participant-facing materials from OSF/QSF/SAV/PDF sources. +- Stage 4 writes a Hub study folder under `studies//`. +- Stage 5 can run simulation after `scripts/config.py` exists and the materials + are ready enough to consume. + +Copy the settings template and configure API credentials locally: + +```bash +cp config/settings.example.yaml config/settings.yaml +``` + +Do not commit `config/settings.yaml`. + +Example Stage 4 Hub package generation from an existing `stage3.json`: + +```bash +python generation_pipeline/run.py \ + --settings config/settings.yaml \ + --stage 4 \ + --json generation_pipeline/outputs//stage3.json \ + --pdf /absolute/path/to/paper.pdf \ + --study-id study_my_generated_study \ + --hub-layout \ + --hub-studies-dir studies +``` + +In Hub layout, Stage 4 writes: + +```text +studies// + index.json + README.md + source/ + metadata.json + specification.json + ground_truth.json + source_extraction.json + audit.json + materials/*.json + scripts/ + study_utils.py + config.py # generated by the Stage 4 LLM config generator when configured + evaluator.py # placeholder; fill after inspecting Stage 5 responses +``` + +`scripts/config.py` is Stage 5 input, not Stage 5 output. It is generated by +the Stage 4 config generator when a Stage 4 LLM is configured; otherwise Stage 4 +marks config generation as skipped. `scripts/evaluator.py` is deliberately a +placeholder because reliable evaluator code usually needs response samples from +a Stage 5 smoke run plus human review. + ## How to Contribute a Study ### 1. Fork and clone diff --git a/config/settings.example.yaml b/config/settings.example.yaml new file mode 100644 index 0000000..ad596dc --- /dev/null +++ b/config/settings.example.yaml @@ -0,0 +1,100 @@ +# Copy this file to config/settings.yaml and edit as needed. +# Do not commit config/settings.yaml if it contains secrets. +# The real settings files are ignored by .gitignore: +# config/settings.yaml, config/settings.yml, config/settings.json + +# Global fallback LLM. Any stage-specific section below can override it. +llm: + # provider: openai | google | anthropic | openrouter | vllm + provider: openai + + # The model name sent to the selected provider. + model: gpt-4o + + # Optional. Prefer environment variables for shared machines, but local + # config/settings.yaml may also hold a key because it is gitignored. + # Provider env alternatives: + # OPENAI_API_KEY, GOOGLE_API_KEY, ANTHROPIC_API_KEY, OPENROUTER_API_KEY + api_key: null + + # Optional. If null, the provider default is used: + # openai -> https://api.openai.com/v1 + # google -> https://generativelanguage.googleapis.com/v1beta/openai/ + # anthropic -> https://api.anthropic.com + # openrouter -> https://openrouter.ai/api/v1 + # vllm -> http://localhost:8000/v1 + base_url: null + +# Stage-specific LLM settings. Leave fields null to inherit from llm when the +# provider matches; if provider differs, use that provider's env/defaults. +stage1_llm: + provider: null + model: null + api_key: null + base_url: null + +stage2_llm: + provider: null + model: null + api_key: null + base_url: null + +# Stage 3 patching can use a cheaper model than Stage 1/2. +stage3_llm: + provider: openai + model: gpt-4o-mini + api_key: null + base_url: null + +# Stage 4 HumanStudy-Bench package/config generation. If this is not configured, +# Stage 4 can still emit a deterministic data/studies/ JSON package, +# but it will skip or fallback from LLM-generated study adapters. +stage4_llm: + provider: null + model: null + api_key: null + base_url: null + +# Stage 5 HumanStudy-Bench participant simulation. This can intentionally use a +# different provider/model/key/base_url from the extraction/config LLM above. +stage5_llm: + provider: null + model: null + api_key: null + base_url: null + +# Backward-compatible alias for Stage 5. +agent_llm: + provider: null + model: null + api_key: null + base_url: null + +paths: + output_dir: generation_pipeline/outputs + # HumanStudy-Bench-Hub stores studies directly under ./studies. + data_dir: . + runs_dir: runs + +verification: + threshold: 90 + +stage5: + # If null, Stage 5 uses the study adapter/specification participant count. + n_participants: null + n_agents: null + + # Explicit cost cap for n_participants/n_agents. + max_agents: null + + repeats: 1 + temperature: 1.0 + seed: 42 + num_workers: null + use_cache: false + cache_dir: null + profiles_json: null + system_prompt_file: null + system_prompt_preset: v3_human_plus_demo + reasoning: default + enable_reasoning: false diff --git a/generation_pipeline/__init__.py b/generation_pipeline/__init__.py new file mode 100644 index 0000000..48bb737 --- /dev/null +++ b/generation_pipeline/__init__.py @@ -0,0 +1 @@ +"""Generation pipeline (ai-ethics): Stage 1 filter + Stage 2 ethics-effect extraction.""" diff --git a/generation_pipeline/connectors/__init__.py b/generation_pipeline/connectors/__init__.py new file mode 100644 index 0000000..5b6c586 --- /dev/null +++ b/generation_pipeline/connectors/__init__.py @@ -0,0 +1,17 @@ +"""Source connector backbone for fetching paper supplementary materials.""" + +from generation_pipeline.connectors.base_connector import ( + BaseSourceConnector, + ConnectorFetchResult, + FetchedSource, + SourceFetchPlan, +) +from generation_pipeline.connectors.registry import SourceConnectorRegistry + +__all__ = [ + "BaseSourceConnector", + "ConnectorFetchResult", + "FetchedSource", + "SourceFetchPlan", + "SourceConnectorRegistry", +] diff --git a/generation_pipeline/connectors/base_connector.py b/generation_pipeline/connectors/base_connector.py new file mode 100644 index 0000000..abdadf6 --- /dev/null +++ b/generation_pipeline/connectors/base_connector.py @@ -0,0 +1,415 @@ +"""Base classes and shared utilities for source connectors. + +Connectors are deterministic fetchers for external paper materials. Phase 1 is +OSF-only: stages fetch OSF sources once, store them under +``stage4//sources/``, and then feed that source directory into +verification or patching logic. +""" + +from __future__ import annotations + +import json +import mimetypes +import re +import urllib.parse +import zipfile +import xml.etree.ElementTree as ET +from abc import ABC, abstractmethod +from dataclasses import asdict, dataclass, field +from pathlib import Path, PurePosixPath +from typing import Any, Iterable + +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + + +TEXT_SUFFIXES = { + ".txt", + ".md", + ".json", + ".qsf", + ".csv", + ".tsv", + ".html", + ".htm", + ".xml", + ".do", + ".r", + ".py", + ".js", + ".css", +} +DOCX_SUFFIXES = {".docx"} +PDF_SUFFIXES = {".pdf"} + + +@dataclass(frozen=True) +class SourceFetchPlan: + """Connector-independent plan for fetching a paper's external source.""" + + paper_folder: str + json_path: str | None + paper_title: str + link: str + connector: str | None = None + doi: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + options: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class FetchedSource: + """One fetched file, with provenance.""" + + path: str + connector: str + source_url: str + kind: str = "file" + size: int | None = None + content_type: str | None = None + text_path: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass +class ConnectorFetchResult: + """Fetch result for one connector invocation.""" + + connector: str + plan: SourceFetchPlan + dest: str + fetched: list[FetchedSource] = field(default_factory=list) + skipped: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return not self.errors + + def to_dict(self) -> dict[str, Any]: + return { + "connector": self.connector, + "plan": self.plan.to_dict(), + "dest": self.dest, + "fetched": [item.to_dict() for item in self.fetched], + "skipped": list(self.skipped), + "errors": list(self.errors), + } + + +class BaseSourceConnector(ABC): + """Base class for external source connectors.""" + + name = "base" + + @abstractmethod + def detect(self, link: str) -> bool: + """Return True when this connector can handle ``link``.""" + + @abstractmethod + def fetch(self, plan: SourceFetchPlan, dest: Path) -> list[FetchedSource]: + """Fetch source files for ``plan`` into ``dest`` and return files.""" + + def extract_text(self, file: Path) -> str: + """Extract text from a fetched file. + + Unsupported binary formats return an empty string rather than raising. + Parse failures raise so the registry can record precise provenance. + """ + return extract_text_from_file(file) + + +def extract_text_from_file(path: Path, *, max_chars: int = 200000) -> str: + """Extract text from PDF, DOCX, Qualtrics/text, and simple text-like files.""" + path = Path(path) + suffix = path.suffix.lower() + if suffix in PDF_SUFFIXES: + return extract_pdf_text(path, max_chars=max_chars) + if suffix in DOCX_SUFFIXES: + return _docx_text(path)[:max_chars] + if suffix == ".qsf": + # Qualtrics Survey Format: JSON containing HTML-wrapped question text. + # Raw JSON is 90 %+ metadata noise — parse to human-readable Q&A. + return _qsf_text(path)[:max_chars] + if suffix in TEXT_SUFFIXES: + return path.read_text(encoding="utf-8", errors="ignore")[:max_chars] + + # Try lightweight text detection for unknown files, but avoid binary dumps. + raw = path.read_bytes()[:max_chars] + if not raw: + return "" + if raw.count(b"\x00") > max(1, len(raw) // 20): + return "" + try: + text = raw.decode("utf-8", errors="ignore") + except Exception: + return "" + printable = sum(1 for ch in text if ch.isprintable() or ch.isspace()) + if printable / max(len(text), 1) < 0.8: + return "" + return text + + +def write_json(path: Path, data: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +def safe_filename(name: str, *, fallback: str = "source") -> str: + """Return a filesystem-safe filename while preserving useful suffixes.""" + name = urllib.parse.unquote(name or "").strip() + if not name: + name = fallback + name = re.sub(r"[\\/:*?\"<>|\x00-\x1f]+", "_", name) + name = re.sub(r"\s+", " ", name).strip(" .") + return name or fallback + + +def safe_relative_path(path: str, *, fallback: str = "source") -> Path: + """Sanitize a remote relative path and prevent directory traversal.""" + parts: list[str] = [] + for part in PurePosixPath(path or fallback).parts: + if part in {"", ".", "..", "/"}: + continue + parts.append(safe_filename(part, fallback=fallback)) + return Path(*parts) if parts else Path(safe_filename(fallback)) + + +def filename_from_url(url: str, *, fallback: str = "source") -> str: + parsed = urllib.parse.urlparse(url) + name = Path(urllib.parse.unquote(parsed.path)).name + if not name: + guessed = mimetypes.guess_extension(parsed.path) or "" + name = f"{fallback}{guessed}" + return safe_filename(name, fallback=fallback) + + +def iter_text_files(files: Iterable[Path]) -> Iterable[Path]: + """Yield files that are likely to produce text.""" + for path in files: + suffix = Path(path).suffix.lower() + if suffix in PDF_SUFFIXES | DOCX_SUFFIXES | TEXT_SUFFIXES: + yield Path(path) + + +def _docx_text(path: Path) -> str: + with zipfile.ZipFile(path) as zf: + root = ET.fromstring(zf.read("word/document.xml")) + return "\n".join(node.text for node in root.iter() if node.tag.endswith("}t") and node.text) + + +_HTML_TAG = re.compile(r"<[^>]+>") +_LEFTOP = re.compile(r'class="LeftOpDesc">([^<]+)<') +_OPDESC = re.compile(r'class="OpDesc">([^<]+)<') + + +def _qsf_clean(s: str) -> str: + import html as _html + + return re.sub(r"\s+", " ", _html.unescape(_HTML_TAG.sub(" ", s or ""))).strip() + + +def _qsf_readable_logic(logic: Any) -> str: + """Render Qualtrics BranchLogic / DisplayLogic as a short human-readable condition.""" + descriptions: list[str] = [] + + def collect(node: Any) -> None: + if isinstance(node, dict): + for key, val in node.items(): + if key == "Description" and isinstance(val, str): + descriptions.append(val) + else: + collect(val) + elif isinstance(node, list): + for item in node: + collect(item) + + collect(logic) + + parts: list[str] = [] + for desc in descriptions: + left_m = _LEFTOP.search(desc) + if not left_m: + continue + left = _qsf_clean(left_m.group(1)) + op_m = _OPDESC.search(desc) + op = _qsf_clean(op_m.group(1)) if op_m else "" + if left: + parts.append(f"{left} {op}".strip()) + # Dedupe while preserving order + seen: set[str] = set() + uniq = [p for p in parts if not (p in seen or seen.add(p))] + return "; ".join(uniq) + + +def _render_sq(payload: dict[str, Any]) -> str: + """Render one Survey Question (SQ) payload to clean text with choices + display logic.""" + qtext = _qsf_clean(payload.get("QuestionText", "")) + if not qtext: + return "" + sub = _qsf_clean(payload.get("QuestionDescription", "")) + header = f"Q: {qtext}" + if sub and sub != qtext and len(sub) < 120: + header += f" ({sub})" + parts = [header] + + cond = _qsf_readable_logic(payload.get("DisplayLogic")) + if cond: + parts.append(f" [shown if: {cond}]") + + choices = payload.get("Choices") or payload.get("Answers") or {} + if isinstance(choices, dict): + for key in sorted(choices, key=lambda k: (int(k) if str(k).isdigit() else 0, str(k))): + val = choices[key] + lbl = _qsf_clean(val.get("Display") or val.get("Text") or "") if isinstance(val, dict) else "" + if lbl: + parts.append(f" {key}. {lbl}") + labels = payload.get("Labels") or {} + if isinstance(labels, dict): + for key, val in labels.items(): + lbl = _qsf_clean(val.get("Display") or "") if isinstance(val, dict) else "" + if lbl: + parts.append(f" [{key}] {lbl}") + return "\n".join(parts) + + +def _qsf_text(path: Path) -> str: + try: + raw = path.read_text(encoding="utf-8", errors="ignore") + data = json.loads(raw) + except Exception: + return path.read_text(encoding="utf-8", errors="ignore") + + elements = data.get("SurveyElements") if isinstance(data, dict) else [] + if not isinstance(elements, list): + return "" + + # --- index questions by QID and parse block definitions ------------------ + qid_map: dict[str, dict[str, Any]] = {} + block_map: dict[str, dict[str, Any]] = {} + flow: list[Any] = [] + survey_name = "" + for el in elements: + if not isinstance(el, dict): + continue + kind = el.get("Element") + if kind == "SQ": + payload = el.get("Payload") + qid = el.get("PrimaryAttribute") or (payload.get("QuestionID") if isinstance(payload, dict) else None) + if qid and isinstance(payload, dict): + qid_map[qid] = payload + elif kind == "BL": + payload = el.get("Payload") + iterable = payload.values() if isinstance(payload, dict) else (payload if isinstance(payload, list) else []) + for block in iterable: + if not isinstance(block, dict): + continue + bid = block.get("ID") + if not bid: + continue + qids = [ + item.get("QuestionID") + for item in (block.get("BlockElements") or []) + if isinstance(item, dict) and item.get("Type") == "Question" and item.get("QuestionID") + ] + block_map[bid] = { + "description": _qsf_clean(block.get("Description") or ""), + "type": block.get("Type") or "Standard", + "qids": qids, + } + elif kind == "FL": + payload = el.get("Payload") + if isinstance(payload, dict) and isinstance(payload.get("Flow"), list): + flow = payload["Flow"] + elif kind == "SO": + payload = el.get("Payload") + if isinstance(payload, dict): + survey_name = _qsf_clean(payload.get("SurveyName") or "") + + def render_block(bid: str, context: list[str]) -> str: + block = block_map.get(bid) + if not block or block.get("type") == "Trash": + return "" + rendered = [_render_sq(qid_map[q]) for q in block["qids"] if q in qid_map] + rendered = [r for r in rendered if r] + if not rendered: + return "" + desc = block["description"] or bid + ctx = f" | condition: {', '.join(context)}" if context else "" + return f"\n--- BLOCK: {desc}{ctx} ---\n" + "\n".join(rendered) + + # --- walk the flow, accumulating branch/embedded-data context ------------ + body_parts: list[str] = [] + condition_vars: dict[str, set] = {} + branch_labels: list[str] = [] + + def walk(items: list[Any], context: list[str]) -> None: + for it in items: + if not isinstance(it, dict): + continue + t = it.get("Type") + if t == "Block": + bid = it.get("ID") + if bid: + rendered = render_block(bid, context) + if rendered: + body_parts.append(rendered) + elif t == "EmbeddedData": + local = list(context) + for ed in it.get("EmbeddedData", []) or []: + if not isinstance(ed, dict): + continue + field_name, value = ed.get("Field"), ed.get("Value") + if field_name and value not in (None, ""): + condition_vars.setdefault(field_name, set()).add(str(value)) + local.append(f"{field_name}={value}") + # ED assignments apply to siblings that follow; mutate context in place + context[:] = local + elif t in ("Branch",): + label = _qsf_readable_logic(it.get("BranchLogic")) + if label: + branch_labels.append(label) + child_ctx = context + ([label] if label else []) + walk(it.get("Flow", []) or [], child_ctx) + elif t in ("BlockRandomizer", "Randomizer", "Group", "Root"): + walk(it.get("Flow", []) or [], list(context)) + else: + # Unknown node may still carry a nested Flow + if isinstance(it.get("Flow"), list): + walk(it["Flow"], list(context)) + + if flow: + walk(flow, []) + + # --- assemble final text ------------------------------------------------- + out: list[str] = [] + if survey_name: + out.append(f"=== QSF SURVEY: {survey_name} ===") + if condition_vars or branch_labels: + lines = ["CONDITIONS defined in the survey flow (manipulation variables):"] + for field_name in sorted(condition_vars): + values = sorted(condition_vars[field_name]) + if len(values) == 1: + lines.append(f" - {field_name} = {values[0]}") + else: + lines.append(f" - {field_name} ∈ {{{', '.join(values)}}} (randomized between conditions)") + seen_b: set[str] = set() + for label in branch_labels: + if label not in seen_b: + seen_b.add(label) + lines.append(f" - branch: {label}") + out.append("\n".join(lines)) + + if body_parts: + out.extend(body_parts) + else: + # Fallback: flat SQ dump in element order (old behavior) when flow parsing + # produced nothing (e.g. malformed flow or single implicit block). + flat = [_render_sq(qid_map[q]) for q in qid_map] + out.extend(r for r in flat if r) + + return "\n\n".join(out) diff --git a/generation_pipeline/connectors/osf_connector.py b/generation_pipeline/connectors/osf_connector.py new file mode 100644 index 0000000..efe5953 --- /dev/null +++ b/generation_pipeline/connectors/osf_connector.py @@ -0,0 +1,150 @@ +"""OSF source connector. + +This wraps the existing OSF crawler primitives while changing the storage +layout to the Phase-1 source backbone convention: + + stage4//sources/osf/files/... + stage4//sources/osf/osf_manifest.json +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +from generation_pipeline.connectors.base_connector import ( + BaseSourceConnector, + FetchedSource, + SourceFetchPlan, + safe_relative_path, + write_json, +) +from generation_pipeline.utils.osf_crawler import ( + OsfClient, + PaperOsfPlan, + build_paper_plan, + parse_osf_url, + select_files, +) + + +class OsfConnector(BaseSourceConnector): + name = "osf" + + def __init__( + self, + *, + token: str | None = None, + download_all: bool = True, + dry_run: bool = False, + ): + self.token = token if token is not None else os.environ.get("OSF_TOKEN") + self.download_all = download_all + self.dry_run = dry_run + + def detect(self, link: str) -> bool: + return "osf.io" in (link or "").lower() + + def fetch(self, plan: SourceFetchPlan, dest: Path) -> list[FetchedSource]: + osf_plan = self._to_osf_plan(plan) + if not osf_plan.can_download: + return [] + + dest = Path(dest) + files_dir = dest / "files" + client = OsfClient(token=self.token, view_only=osf_plan.view_only) + + entity_type, title, files_index = client.resolve_entity(osf_plan.osf_id) + osf_plan.entity_type = entity_type + osf_plan.entity_title = title + + providers = client._request(files_index).get("data", []) + all_files = [] + for provider in providers: + provider_files_url = ( + provider.get("relationships", {}) + .get("files", {}) + .get("links", {}) + .get("related", {}) + .get("href") + ) + if provider_files_url: + all_files.extend(client.list_files_recursive(provider_files_url)) + + matched = select_files(all_files, osf_plan.effects_needing_osf, download_all=self.download_all) + osf_plan.matched_files = matched + if not matched: + reason = ( + "OSF project has no files in storage" + if not all_files + else "No OSF files matched selection criteria" + ) + osf_plan.errors.append(reason) + write_json( + dest / "osf_manifest.json", + { + "plan": osf_plan.to_dict(), + "all_osf_files": [item.to_dict() for item in all_files], + "source_layout": "sources/osf/files", + "dry_run": self.dry_run, + }, + ) + raise RuntimeError(reason) + + fetched: list[FetchedSource] = [] + skipped: list[str] = [] + for item in matched: + if not item.download_url: + continue + rel = safe_relative_path(item.path) + out_path = files_dir / rel + if self.dry_run: + # Dry-run: record intent but do NOT add to fetched (file not on disk). + osf_plan.skipped_files.append(str(out_path)) + skipped.append(str(out_path)) + continue + if out_path.exists(): + # Already downloaded on a previous run — skip network round-trip. + osf_plan.skipped_files.append(str(out_path)) + else: + try: + client.download(item.download_url, out_path) + osf_plan.downloaded_files.append(str(out_path)) + except Exception as exc: + osf_plan.errors.append(f"Failed to download {item.path}: {exc}") + continue + fetched.append( + FetchedSource( + path=str(out_path), + connector=self.name, + source_url=item.download_url, + size=item.size, + ) + ) + + manifest = { + "plan": osf_plan.to_dict(), + "all_osf_files": [item.to_dict() for item in all_files], + "source_layout": "sources/osf/files", + "dry_run": self.dry_run, + "skipped_existing": [s for s in osf_plan.skipped_files if not self.dry_run], + } + write_json(dest / "osf_manifest.json", manifest) + return fetched + + def _to_osf_plan(self, plan: SourceFetchPlan) -> PaperOsfPlan: + if plan.json_path: + osf_plan = build_paper_plan(Path(plan.json_path)) + if osf_plan.osf_id: + return osf_plan + + osf_id, view_only = parse_osf_url(plan.link) + return PaperOsfPlan( + paper_folder=plan.paper_folder, + json_path=plan.json_path or "", + paper_title=plan.paper_title, + osf_url=plan.link, + osf_id=osf_id, + view_only=view_only, + ) diff --git a/generation_pipeline/connectors/osf_link_discovery.py b/generation_pipeline/connectors/osf_link_discovery.py new file mode 100644 index 0000000..bfcc6f6 --- /dev/null +++ b/generation_pipeline/connectors/osf_link_discovery.py @@ -0,0 +1,127 @@ +"""OSF link discovery from paper JSON, PDF text, and optional LLM assistance.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any, Iterable + +from generation_pipeline.utils.osf_crawler import OSF_URL_RE, parse_osf_url +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + +PDF_TEXT_MAX_CHARS = 400000 +DISCOVERY_PROMPT_MAX_CHARS = 80000 + + +def discover_osf_links_from_text(text: str) -> list[str]: + """Return unique normalized OSF links found in free text.""" + links: list[str] = [] + seen: set[str] = set() + for match in OSF_URL_RE.finditer(text or ""): + # The shared OSF regex intentionally accepts 5-char node ids, but in + # free text it can otherwise match the prefix of longer OSF paths such + # as /preprints/. Treat an immediately-following alphanumeric + # character as a non-node path and skip it. + if match.end() < len(text or "") and (text or "")[match.end()].isalnum(): + continue + raw = match.group(0).rstrip(").,;]") + try: + osf_id, view_only = parse_osf_url(raw) + except ValueError: + continue + normalized = f"https://osf.io/{osf_id}/" + if view_only: + normalized = f"{normalized}?view_only={view_only}" + if normalized not in seen: + seen.add(normalized) + links.append(normalized) + return links + + +def discover_osf_links_from_json(data: dict[str, Any]) -> list[str]: + """Scan paper metadata and string values for OSF links.""" + return discover_osf_links_from_text(json.dumps(data, ensure_ascii=False)) + + +def discover_osf_links_from_pdf(pdf_path: Path) -> list[str]: + """Extract PDF text and scan it for OSF links.""" + text = extract_pdf_text(Path(pdf_path), max_chars=PDF_TEXT_MAX_CHARS) + return discover_osf_links_from_text(text) + + +class OsfLinkDiscovery: + """Rule-first OSF link discovery with optional LLM fallback.""" + + def __init__(self, llm_client: Any | None = None): + self.llm_client = llm_client + + def discover( + self, + *, + paper_json: dict[str, Any] | None = None, + json_path: Path | None = None, + pdf_path: Path | None = None, + extra_texts: Iterable[str] = (), + ) -> list[str]: + data = paper_json + if data is None and json_path is not None and Path(json_path).exists(): + data = json.loads(Path(json_path).read_text(encoding="utf-8")) + + texts: list[str] = [] + if data is not None: + texts.append(json.dumps(data, ensure_ascii=False)) + texts.extend(text for text in extra_texts if text) + + links = self._unique_links(*(discover_osf_links_from_text(text) for text in texts)) + if links: + return links + + pdf_text = "" + if pdf_path is not None and Path(pdf_path).exists(): + try: + pdf_text = extract_pdf_text(Path(pdf_path), max_chars=PDF_TEXT_MAX_CHARS) + except Exception: + pdf_text = "" + links = discover_osf_links_from_text(pdf_text) + if links: + return links + + if self.llm_client is None or not pdf_text: + return [] + return self._discover_with_llm(pdf_text) + + def _discover_with_llm(self, text: str) -> list[str]: + try: + from src.llm.helpers import generate_json + except Exception: + return [] + + system = ( + "You locate Open Science Framework (OSF) data/material links in paper text. " + "Only return explicit osf.io URLs that appear in the provided text." + ) + prompt = ( + "Find every explicit OSF URL in this paper text. Do not infer links. " + "Respond as JSON: {\"osf_links\": [\"https://osf.io/...\"]}.\n\n" + f"TEXT:\n{text[:DISCOVERY_PROMPT_MAX_CHARS]}" + ) + try: + data = generate_json(self.llm_client, [{"role": "user", "content": prompt}], system=system) + except Exception: + return [] + links = data.get("osf_links") if isinstance(data, dict) else None + if not isinstance(links, list): + return [] + return self._unique_links(discover_osf_links_from_text("\n".join(str(item) for item in links))) + + @staticmethod + def _unique_links(*groups: Iterable[str]) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for group in groups: + for link in group: + if link not in seen: + seen.add(link) + out.append(link) + return out diff --git a/generation_pipeline/connectors/registry.py b/generation_pipeline/connectors/registry.py new file mode 100644 index 0000000..2111cc4 --- /dev/null +++ b/generation_pipeline/connectors/registry.py @@ -0,0 +1,505 @@ +"""OSF-only source connector registry""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +from pathlib import Path +from typing import Any, Iterable + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from generation_pipeline.connectors.base_connector import ( + BaseSourceConnector, + ConnectorFetchResult, + FetchedSource, + SourceFetchPlan, + iter_text_files, + safe_relative_path, + write_json, +) +from generation_pipeline.connectors.osf_connector import OsfConnector +from generation_pipeline.connectors.osf_link_discovery import OsfLinkDiscovery +from generation_pipeline.utils.osf_crawler import find_paper_jsons + + +class SourceConnectorRegistry: + """Registry that currently dispatches only OSF source connectors.""" + + def __init__( + self, + connectors: Iterable[BaseSourceConnector] | None = None, + *, + link_discovery: OsfLinkDiscovery | None = None, + extract_text: bool = True, + llm_client: Any | None = None, + ): + self.connectors = list(connectors) if connectors is not None else [OsfConnector()] + self.link_discovery = link_discovery or OsfLinkDiscovery(llm_client=llm_client) + self.extract_text = extract_text + + def connector_for(self, link: str) -> BaseSourceConnector | None: + for connector in self.connectors: + if connector.detect(link): + return connector + return None + + def plans_for_json( + self, + json_path: Path, + *, + pdf_path: Path | None = None, + ) -> list[SourceFetchPlan]: + data = json.loads(Path(json_path).read_text(encoding="utf-8")) + links = self.link_discovery.discover(paper_json=data, json_path=json_path, pdf_path=pdf_path) + folder = Path(json_path).parent.name + metadata = data.get("paper_metadata") if isinstance(data.get("paper_metadata"), dict) else {} + plans: list[SourceFetchPlan] = [] + seen: set[str] = set() + for link in links: + connector = self.connector_for(link) + if connector is None or link in seen: + continue + seen.add(link) + plans.append( + SourceFetchPlan( + paper_folder=folder, + json_path=str(json_path), + paper_title=str(data.get("paper_title") or ""), + link=link, + connector=connector.name, + doi=metadata.get("doi"), + metadata=metadata, + ) + ) + return plans + + def fetch_for_json( + self, + json_path: Path, + *, + pdf_path: Path | None = None, + dest_dir: Path | None = None, + ) -> dict[str, Any]: + json_path = Path(json_path) + dest_dir = Path(dest_dir) if dest_dir is not None else json_path.parent / "sources" + + reused = self._reuse_paired_pdf_sources(json_path, pdf_path, dest_dir, plans=[]) + if reused is not None: + return reused + + plans = self.plans_for_json(json_path, pdf_path=pdf_path) + + dest_dir.mkdir(parents=True, exist_ok=True) + results: list[ConnectorFetchResult] = [] + for plan in plans: + connector = self.connector_for(plan.link) + if connector is None: + results.append( + ConnectorFetchResult( + connector=plan.connector or "unknown", + plan=plan, + dest=str(dest_dir), + errors=[f"No connector registered for link: {plan.link}"], + ) + ) + continue + connector_dest = dest_dir / connector.name + try: + fetched = connector.fetch(plan, connector_dest) + result = ConnectorFetchResult( + connector=connector.name, + plan=plan, + dest=str(connector_dest), + fetched=fetched, + ) + except Exception as exc: + result = ConnectorFetchResult( + connector=connector.name, + plan=plan, + dest=str(connector_dest), + errors=[str(exc)], + ) + if self.extract_text: + self._extract_text_bundle(connector, result, dest_dir) + results.append(result) + + manifest = { + "json_path": str(json_path), + "pdf_path": str(pdf_path) if pdf_path else None, + "sources_dir": str(dest_dir), + "plans": [plan.to_dict() for plan in plans], + "results": [result.to_dict() for result in results], + "summary": { + "plans": len(plans), + "connectors": sorted({result.connector for result in results}), + "fetched_files": sum(len(result.fetched) for result in results), + "skipped_files": sum(len(result.skipped) for result in results), + "errors": sum(len(result.errors) for result in results), + }, + } + write_json(dest_dir / "source_manifest.json", manifest) + return manifest + + def _reuse_paired_pdf_sources( + self, + json_path: Path, + pdf_path: Path | None, + dest_dir: Path, + plans: list[SourceFetchPlan], + ) -> dict[str, Any] | None: + """Reuse local sources next to the paired PDF before downloading.""" + pdf_dir = Path(pdf_path).parent if pdf_path else None + + complete_candidates: list[Path] = [] + if pdf_dir is not None: + complete_candidates.append(pdf_dir / "sources") + complete_candidates.append(dest_dir) + for source_dir in complete_candidates: + if (source_dir / "combined_sources.txt").exists(): + files = _source_payload_files(source_dir) + return self._local_reuse_manifest( + json_path=json_path, + pdf_path=pdf_path, + sources_dir=source_dir, + plans=plans, + files=files, + reuse_mode="existing_source_bundle", + reused_from=source_dir, + write_manifest=False, + ) + + osf_candidates: list[Path] = [dest_dir / "osf"] + if pdf_dir is not None: + osf_candidates.extend([pdf_dir / "sources" / "osf", pdf_dir / "osf"]) + for osf_dir in osf_candidates: + files_dir = osf_dir / "files" + source_files = _files_under(files_dir) + if not source_files: + continue + + target_osf_dir = dest_dir / "osf" + target_files_dir = target_osf_dir / "files" + if not _same_path(osf_dir, target_osf_dir): + shutil.copytree(osf_dir, target_osf_dir, dirs_exist_ok=True) + source_files = _files_under(target_files_dir) + + connector = self._connector_by_name("osf") + result = ConnectorFetchResult( + connector="osf", + plan=_reuse_plan(json_path, plans), + dest=str(target_osf_dir), + fetched=[ + FetchedSource( + path=str(path), + connector="osf", + source_url="local", + size=path.stat().st_size if path.exists() else None, + ) + for path in source_files + ], + ) + if self.extract_text: + self._extract_text_bundle(connector, result, dest_dir) + manifest = self._local_reuse_manifest( + json_path=json_path, + pdf_path=pdf_path, + sources_dir=dest_dir, + plans=plans, + files=[Path(item.path) for item in result.fetched], + reuse_mode="legacy_osf_files", + reused_from=osf_dir, + result=result, + write_manifest=True, + ) + return manifest + + return None + + def _connector_by_name(self, name: str) -> BaseSourceConnector: + for connector in self.connectors: + if connector.name == name: + return connector + return OsfConnector() + + def _local_reuse_manifest( + self, + *, + json_path: Path, + pdf_path: Path | None, + sources_dir: Path, + plans: list[SourceFetchPlan], + files: list[Path], + reuse_mode: str, + reused_from: Path, + result: ConnectorFetchResult | None = None, + write_manifest: bool, + ) -> dict[str, Any]: + results = [result.to_dict()] if result is not None else [] + manifest = { + "json_path": str(json_path), + "pdf_path": str(pdf_path) if pdf_path else None, + "sources_dir": str(sources_dir), + "plans": [plan.to_dict() for plan in plans], + "results": results, + "reused_from": str(reused_from), + "reuse_mode": reuse_mode, + "summary": { + "plans": len(plans), + "connectors": ["osf"] if files else [], + "fetched_files": len(files), + "skipped_files": 0, + "reused_files": len(files), + "errors": 0, + }, + } + if write_manifest: + write_json(sources_dir / "source_manifest.json", manifest) + return manifest + + def _extract_text_bundle( + self, + connector: BaseSourceConnector, + result: ConnectorFetchResult, + sources_dir: Path, + ) -> None: + text_records: list[dict[str, Any]] = [] + combined_blocks: list[str] = [] + + # Build the candidate file list from BOTH freshly-fetched files and files + # that were skipped because they already existed on disk. Re-running + # ``--fetch-sources`` on an already-downloaded paper must still regenerate + # combined_sources.txt (e.g. after the QSF parser is upgraded) without + # forcing a re-download. + candidate_paths: list[str] = [item.path for item in result.fetched] + seen_paths = set(candidate_paths) + for skipped in result.skipped: + if skipped not in seen_paths: + candidate_paths.append(skipped) + seen_paths.add(skipped) + + text_capable = set(iter_text_files(Path(p) for p in candidate_paths)) + + # Sort files so the most informative come first in combined_sources.txt. + # Priority (highest first): DOCX/QSF/pre-reg PDFs → other PDFs → text/code → data. + def _file_priority(path_str: str) -> int: + ext = Path(path_str).suffix.lower() + if ext in {".docx"}: return 0 + if ext in {".qsf"}: return 1 + if "pre-regist" in path_str.lower(): return 2 + if ext == ".pdf": return 3 + if ext in {".txt", ".md", ".html", ".htm"}: return 4 + if ext in {".do", ".r", ".py", ".js"}: return 5 + return 6 # .xlsx, .sav, .csv, .tsv — data files, least informative + + sorted_paths = sorted(candidate_paths, key=_file_priority) + path_to_orig_index = {item.path: i for i, item in enumerate(result.fetched)} + for path_str in sorted_paths: + file_path = Path(path_str) + if not file_path.exists() or file_path not in text_capable: + continue + try: + text = connector.extract_text(file_path) + except Exception as exc: + result.errors.append(f"Text extraction failed for {file_path}: {exc}") + continue + if not text.strip(): + continue + rel = safe_relative_path(str(file_path.relative_to(sources_dir))) + text_path = sources_dir / "text" / rel.with_suffix(rel.suffix + ".txt") + text_path.parent.mkdir(parents=True, exist_ok=True) + text_path.write_text(text, encoding="utf-8") + orig_idx = path_to_orig_index.get(path_str) + if orig_idx is not None: + item = result.fetched[orig_idx] + result.fetched[orig_idx] = FetchedSource( + path=item.path, + connector=item.connector, + source_url=item.source_url, + kind=item.kind, + size=item.size, + content_type=item.content_type, + text_path=str(text_path), + ) + text_records.append( + { + "source_path": path_str, + "text_path": str(text_path), + "connector": connector.name, + "chars": len(text), + } + ) + combined_blocks.append(f"\n--- Source: {path_str} ---\n{text}") + + if text_records: + write_json(sources_dir / "source_text_index.json", text_records) + (sources_dir / "combined_sources.txt").write_text("\n".join(combined_blocks), encoding="utf-8") + + +def _same_path(left: Path, right: Path) -> bool: + try: + return left.resolve() == right.resolve() + except Exception: + return Path(left) == Path(right) + + +def _files_under(path: Path) -> list[Path]: + path = Path(path) + if not path.is_dir(): + return [] + return sorted(item for item in path.rglob("*") if item.is_file()) + + +def _source_payload_files(sources_dir: Path) -> list[Path]: + osf_files = _files_under(Path(sources_dir) / "osf" / "files") + if osf_files: + return osf_files + combined = Path(sources_dir) / "combined_sources.txt" + return [combined] if combined.exists() else [] + + +def _reuse_plan(json_path: Path, plans: list[SourceFetchPlan]) -> SourceFetchPlan: + if plans: + return plans[0] + return SourceFetchPlan( + paper_folder=Path(json_path).parent.name, + json_path=str(json_path), + paper_title="", + link="local", + connector="osf", + ) + + +def infer_pdf_for_json(json_path: Path) -> Path | None: + """Find a sibling PDF for a per-paper JSON.""" + pdfs = sorted(Path(json_path).parent.glob("*.pdf")) + return pdfs[0] if pdfs else None + + +def default_sources_dir_for_json(json_path: Path) -> Path: + """Return the default source directory for a JSON file. + + Stage4 and current pipeline outputs store each paper in its own folder, so + ``/sources`` is the intended layout. Legacy flat ``outputs`` files use + a stem-specific directory to avoid collisions. + """ + json_path = Path(json_path) + stage_paper_id = json_path.stem.split("_stage", 1)[0] if "_stage" in json_path.stem else None + if json_path.parent.name == "outputs": + return json_path.parent / f"{json_path.stem}_sources" + if stage_paper_id and json_path.parent.name == stage_paper_id: + return json_path.parent / "sources" + if "_stage1_" in json_path.name or "_stage2_" in json_path.name: + return json_path.parent / f"{json_path.stem}_sources" + return json_path.parent / "sources" + + +def paired_pdf_for_json(json_path: Path, pdf_paths: list[Path], index: int) -> Path | None: + """Pair explicit PDFs to JSONs, falling back to a sibling PDF.""" + if len(pdf_paths) > 1: + return pdf_paths[index] if index < len(pdf_paths) else infer_pdf_for_json(json_path) + if len(pdf_paths) == 1: + return pdf_paths[0] + return infer_pdf_for_json(json_path) + + +def fetch_json_sources( + json_paths: list[Path], + *, + pdf_paths: list[Path] | None = None, + token: str | None = None, + dry_run: bool = False, + download_all: bool = True, + extract_text: bool = True, + output_dir: Path | None = None, + llm_client: Any | None = None, +) -> list[dict[str, Any]]: + """Fetch OSF sources for explicit JSON/PDF pairs. + + This is the path used by Stage 1 source discovery and by standalone checks + of existing JSON files where the JSON may not live under ``stage4//`` + yet. If the JSON lacks an OSF link, the registry still scans the paired PDF + for explicit ``osf.io`` URLs. + """ + connector = OsfConnector(token=token, dry_run=dry_run, download_all=download_all) + registry = SourceConnectorRegistry( + [connector], extract_text=extract_text and not dry_run, llm_client=llm_client + ) + pdf_paths = [Path(path) for path in (pdf_paths or [])] + results: list[dict[str, Any]] = [] + for index, json_path in enumerate(Path(path) for path in json_paths): + pdf_path = paired_pdf_for_json(json_path, pdf_paths, index) + if output_dir is None: + dest_dir = default_sources_dir_for_json(json_path) + else: + out_root = Path(output_dir) + dest_dir = out_root if len(json_paths) == 1 else out_root / json_path.stem + results.append(registry.fetch_for_json(json_path, pdf_path=pdf_path, dest_dir=dest_dir)) + return results + + +def fetch_stage4_sources( + stage4_dir: Path, + *, + only: list[str] | None = None, + token: str | None = None, + dry_run: bool = False, + download_all: bool = True, + extract_text: bool = True, + llm_client: Any | None = None, +) -> list[dict[str, Any]]: + """Fetch OSF sources for every matching paper JSON under stage4.""" + connector = OsfConnector(token=token, dry_run=dry_run, download_all=download_all) + registry = SourceConnectorRegistry( + [connector], extract_text=extract_text and not dry_run, llm_client=llm_client + ) + results: list[dict[str, Any]] = [] + for json_path in find_paper_jsons(Path(stage4_dir)): + if only and not any(part in json_path.as_posix() for part in only): + continue + results.append( + registry.fetch_for_json( + json_path, + pdf_path=infer_pdf_for_json(json_path), + dest_dir=default_sources_dir_for_json(json_path), + ) + ) + return results + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fetch OSF sources for stage4 paper JSONs") + parser.add_argument("--stage4-dir", type=Path, default=Path("stage4")) + parser.add_argument("--only", action="append", default=None) + parser.add_argument("--token", default=os.environ.get("OSF_TOKEN")) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--no-extract-text", action="store_true") + args = parser.parse_args() + + results = fetch_stage4_sources( + args.stage4_dir, + only=args.only, + token=args.token, + dry_run=args.dry_run, + extract_text=not args.no_extract_text, + ) + ok = sum(1 for result in results if result["summary"]["errors"] == 0) + for result in results: + summary = result["summary"] + label = Path(result["json_path"]).parent.name + reused = summary.get("reused_files", 0) + reused_part = f" reused={reused}" if reused else "" + print( + f"{label}: plans={summary['plans']} fetched={summary['fetched_files']} " + f"errors={summary['errors']}{reused_part} sources={result['sources_dir']}" + ) + print(f"Done: {ok}/{len(results)} paper(s) without connector errors.") + if any(result["summary"]["errors"] for result in results): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/generation_pipeline/effect_selector.py b/generation_pipeline/effect_selector.py new file mode 100644 index 0000000..e75f6bd --- /dev/null +++ b/generation_pipeline/effect_selector.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import json +import re +from typing import Any, Dict, List, Optional + +from src.llm.helpers import call_with_timeout + +EFFECT_SELECTION_RUBRIC = """Select the PRIMARY findings of ONE study for a simulation benchmark. The +"effecttype" label MAY BE WRONG; judge from IV/DV semantics. + +STEP 1: From the paper title + study, name the study's FOCAL OUTCOME construct(s) - the +dependent measure(s) the study is fundamentally about (often named in the title). + +STEP 2: KEEP an effect iff ALL hold: + - its IV is the study's experimental MANIPULATION / assigned condition (a manipulated + between- or within-subjects factor), not a measured individual-difference scale; + - its DV is a FOCAL outcome (or a distinct focal outcome), NOT a manipulation check + (a DV that merely re-measures the manipulated construct itself), NOT a secondary / + auxiliary measure outside the focal outcome(s); + - keep main effects AND interactions of the manipulation; a condition->focal-outcome + effect is primary even if mislabelled 'correlation' / 'mediation'. +DROP everything else (manipulation checks, non-focal secondary DVs, pure correlations of +measured scales, mediation / process tests). + +Return ONLY JSON: +{"focal_outcomes":["..."],"effects":[{"effect_index":int,"keep":true,"role":"primary|manip_check|secondary_dv|correlation|mediation","reason":"<=12 words"}]}""" + + +def build_prompt(title: str, study_name: str, effects: List[Dict[str, Any]]) -> str: + lines = [ + f" [{i}] effecttype={e.get('effecttype')} | IV={e.get('IV')} | DV={e.get('DV')}" + for i, e in enumerate(effects) + ] + return ( + f"{EFFECT_SELECTION_RUBRIC}\n\nPaper: {title}\nStudy: {study_name}\n" + f"Effects:\n" + "\n".join(lines) + ) + + +def _parse(response: str) -> Optional[List[Dict[str, Any]]]: + text = response if isinstance(response, str) else str(response) + m = re.search(r"\{.*\}", text, re.DOTALL) + if not m: + return None + try: + obj = json.loads(m.group()) + except json.JSONDecodeError: + return None + eff = obj.get("effects") if isinstance(obj, dict) else None + return eff if isinstance(eff, list) else None + + +_NON_PRIMARY_EFFECT_TYPES = { + "simple", + "simple_effect", + "mediation", + "correlation", + "manipulation_check", + "check", +} + + +def _stats_p_value(stats: Dict[str, Any]) -> Optional[float]: + raw = stats.get("p_value") if isinstance(stats, dict) else None + if raw in (None, "", [None, None]): + return None + text = str(raw).strip().lower() + if text in {"ns", "n.s.", "not significant", "non-significant"}: + return 1.0 + match = re.search(r"([<>]=?)?\s*\.?(\d+(?:\.\d+)?)", text) + if not match: + return None + number = match.group(2) + if "." in match.group(0) and "." not in number: + return float("0." + number) + return float(number) + + +def _deterministic_primary_veto(effect: Dict[str, Any]) -> Optional[str]: + etype = str(effect.get("effecttype") or "").strip().lower() + if etype in _NON_PRIMARY_EFFECT_TYPES: + return f"drop deterministic {etype}" + stats = effect.get("stats") if isinstance(effect.get("stats"), dict) else {} + sig = str(stats.get("sig") or effect.get("sig") or "").strip().lower() + if any(token in sig for token in ("marginal", "ns", "n.s", "non", "not sig")): + return "drop deterministic non-significant" + p_value = _stats_p_value(stats) + if p_value is not None and p_value > 0.05: + return "drop deterministic p>.05" + dv = str(effect.get("DV") or "").lower() + iv = str(effect.get("IV") or "").lower() + if "manipulation check" in dv or "manipulation check" in iv: + return "drop deterministic manipulation check" + return None + + +def select_effects( + title: str, + study_name: str, + effects: List[Dict[str, Any]], + client: Any, + *, + votes: int = 1, + timeout: Optional[float] = 60.0, +) -> Dict[int, Dict[str, Any]]: + """Return {effect_index: {"keep": bool, "role": str, "reason": str}}. + + Fail-open: indices the model could not score keep their default (keep=True). + With votes>1 each effect's keep decision is a majority vote. + """ + n = len(effects) + keep_votes = [0] * n + drop_votes = [0] * n + roles: List[str] = ["primary"] * n + reasons: List[str] = [""] * n + rounds = 0 + + for _ in range(max(1, votes)): + try: + response = _generate_selection_response( + client, + prompt=build_prompt(title, study_name, effects), + timeout=timeout, + ) + parsed = _parse(response) + except Exception: # pragma: no cover - network/LLM dependent + parsed = None + if parsed is None: + continue + rounds += 1 + for d in parsed: + idx = d.get("effect_index") + if not isinstance(idx, int) or not (0 <= idx < n): + continue + if bool(d.get("keep", True)): + keep_votes[idx] += 1 + else: + drop_votes[idx] += 1 + if d.get("role"): + roles[idx] = str(d.get("role")) + if d.get("reason"): + reasons[idx] = str(d.get("reason"))[:200] + + out: Dict[int, Dict[str, Any]] = {} + for i in range(n): + if rounds == 0: + out[i] = {"keep": True, "role": "primary", "reason": "selection skipped: LLM unavailable"} + continue + keep = keep_votes[i] >= drop_votes[i] # ties -> keep (fail-open) + veto = _deterministic_primary_veto(effects[i]) + if veto: + out[i] = {"keep": False, "role": roles[i], "reason": veto} + else: + out[i] = {"keep": keep, "role": roles[i], "reason": reasons[i]} + return out + + +def _generate_selection_response(client: Any, *, prompt: str, timeout: Optional[float]) -> str: + def _call() -> str: + try: + return client.generate_content(prompt=prompt, timeout=timeout, max_tokens=1200) + except TypeError: + try: + return client.generate_content(prompt=prompt, timeout=timeout) + except TypeError: + return client.generate_content(prompt=prompt) + + return call_with_timeout(_call, timeout=timeout) + + +def apply_to_study( + study: Dict[str, Any], + title: str, + client: Any, + *, + votes: int = 1, + timeout: Optional[float] = 60.0, +) -> Dict[int, Dict[str, Any]]: + """Re-judge a consolidated study's effects and override the primary flags.""" + effects = [e for e in study.get("effects", []) if isinstance(e, dict)] + study_name = str(study.get("study") or study.get("study_id") or "") + decisions = select_effects(title, study_name, effects, client, votes=votes, timeout=timeout) + + for i, e in enumerate(effects): + dec = decisions.get(i) + cons = e.get("consolidation") + if dec and isinstance(cons, dict): + cons["llm_role"] = dec["role"] + cons["llm_keep"] = dec["keep"] + cons["llm_reason"] = dec["reason"] + + summary = study.get("consolidation_summary") + if isinstance(summary, dict): + primary = 0 + for g in summary.get("groups", []): + rep = g.get("representative_effect_index") + dec = decisions.get(rep) if isinstance(rep, int) else None + if dec is not None: + g["is_primary_simulation_target"] = dec["keep"] + g["analysis_role"] = dec["role"] + # mirror onto the effect-level consolidation blocks + for idx in g.get("effect_indices", []): + if 0 <= idx < len(effects) and isinstance(effects[idx].get("consolidation"), dict): + effects[idx]["consolidation"]["is_primary_simulation_target"] = dec["keep"] + effects[idx]["consolidation"]["analysis_role"] = dec["role"] + if g.get("is_primary_simulation_target"): + primary += 1 + summary["primary_simulation_targets"] = primary + summary["llm_reselected"] = True + return decisions diff --git a/generation_pipeline/extractors/__init__.py b/generation_pipeline/extractors/__init__.py new file mode 100644 index 0000000..6e16ccb --- /dev/null +++ b/generation_pipeline/extractors/__init__.py @@ -0,0 +1,4 @@ +from generation_pipeline.extractors.base_extractor import BaseExtractor +from generation_pipeline.extractors.study_data_extractor import StudyDataExtractor + +__all__ = ["BaseExtractor", "StudyDataExtractor"] diff --git a/generation_pipeline/extractors/base_extractor.py b/generation_pipeline/extractors/base_extractor.py new file mode 100644 index 0000000..3b03359 --- /dev/null +++ b/generation_pipeline/extractors/base_extractor.py @@ -0,0 +1,34 @@ +""" +Base Extractor Class +""" + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Dict, Any + + +class BaseExtractor(ABC): + """Base class for all extractors""" + + def __init__(self, llm_client): + """ + Initialize extractor. + + Args: + llm_client: LLM client instance (GeminiClient) + """ + self.client = llm_client + + @abstractmethod + def process(self, stage1_json: Dict[str, Any], pdf_path: Path) -> Dict[str, Any]: + """ + Process stage1 results and extract study data. + + Args: + stage1_json: Results from stage1 filter + pdf_path: Path to PDF file + + Returns: + Dictionary with extracted data + """ + raise NotImplementedError diff --git a/generation_pipeline/extractors/grounded_slot_extractor.py b/generation_pipeline/extractors/grounded_slot_extractor.py new file mode 100644 index 0000000..aa6481b --- /dev/null +++ b/generation_pipeline/extractors/grounded_slot_extractor.py @@ -0,0 +1,907 @@ +from __future__ import annotations + +import argparse +import json +import re +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from copy import deepcopy +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from generation_pipeline.utils.pdf_chunker import ( + Chunk, + build_chunks, + chunk_pdf, + format_evidence, + retrieve, +) +from generation_pipeline.utils.pdf_extractor import extract_pdf_text +from generation_pipeline.verification.verbatim_verifier import ( + best_partial_ratio, + normalize_for_match, +) +from generation_pipeline.verification.schema_validator import SLOT_NAMES +from src.llm.helpers import generate_json + +DEFAULT_THRESHOLD = 90.0 +PDF_TEXT_MAX_CHARS = 400000 + +# Slot-specific retrieval keywords + a short description of what the span is. +SLOT_PROFILE: dict[str, dict[str, Any]] = { + "materials": { + "desc": "the exact stimulus text shown to participants (e.g. the vignette, scenario, " + "passage, or prompt they read)", + "keywords": ["vignette", "scenario", "stimuli", "stimulus", "material", "passage", + "presented", "read", "described", "condition", "instructions"], + }, + "manipulation": { + "desc": "the exact description of what was manipulated across conditions " + "(the independent-variable operationalization)", + "keywords": ["manipulat", "condition", "randomly assigned", "random assignment", + "varied", "between-subjects", "within-subjects", "we manipulated", "we varied"], + }, + "items": { + "desc": "the exact dependent-variable measure: the question wording AND the response " + "scale / anchors / options", + "keywords": ["scale", "item", "items", "measured", "rated", "response", "responses", + "slider", "sliding", "likert", "anchor", "questionnaire", "question", + "0 =", "1 =", "7 =", "point", "percentage"], + }, +} + + +@dataclass +class SlotResult: + status: str + content: Optional[str] + score: Optional[float] + action: str # "grounded_verbatim" | "kept_paraphrased" | "downgraded" | "not_present" | "unchanged" + evidence_pages: list[int] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "status": self.status, + "score": round(self.score, 2) if isinstance(self.score, float) else self.score, + "action": self.action, + "evidence_pages": self.evidence_pages, + } + + +@dataclass(frozen=True) +class GroundingTask: + order: int + study_index: int + effect_index: int + slot: str + path: str + study_label: str + iv: str + dv: str + location_hint: str + materials_notes: str + prev_status: Optional[str] + prev_content: Any + had_content: bool + + +class GroundedSlotExtractor: + """Evidence-grounded extractor/repairer for materials/manipulation/items slots.""" + + def __init__( + self, + client, + *, + threshold: float = DEFAULT_THRESHOLD, + k: int = 8, + chunk_size: int = 2500, + overlap: int = 600, + retries: int = 1, + temperature: float = 0.0, + slot_timeout: float | None = 60.0, + max_workers: int = 1, + ): + self.client = client + self.threshold = threshold + self.k = k + self.chunk_size = chunk_size + self.overlap = overlap + self.retries = retries + self.temperature = temperature + self.slot_timeout = slot_timeout + self.max_workers = max(int(max_workers or 1), 1) + self._chunks: list[Chunk] = [] + self._source_norm: str = "" + self._source_paths_by_study: dict[str, set[str]] = {} + self._source_paths_by_study_non_prereg: dict[str, set[str]] = {} + self._shared_source_paths: set[str] = set() + + # -- preparation ------------------------------------------------------- + def prepare(self, pdf_path: Path) -> None: + """Parse + chunk the PDF once; cache normalized raw source for verification.""" + pdf_path = Path(pdf_path) + self._chunks = chunk_pdf(pdf_path, chunk_size=self.chunk_size, overlap=self.overlap) + raw = extract_pdf_text(pdf_path, max_chars=PDF_TEXT_MAX_CHARS) + self._source_norm = normalize_for_match(raw) + self._source_paths_by_study = {} + self._source_paths_by_study_non_prereg = {} + self._shared_source_paths = set() + + def prepare_from_text(self, text: str) -> None: + """Chunk pre-extracted text (e.g. combined_sources.txt from OSF files). + + Use this when the evidence is supplementary source material rather than + a PDF — the same chunking + lexical retrieval logic applies. + """ + # Treat the whole block as a single "page" for provenance purposes. + pages = [(1, text)] + self._chunks = build_chunks(pages, chunk_size=self.chunk_size, overlap=self.overlap) + self._source_norm = normalize_for_match(text) + self._source_paths_by_study = {} + self._source_paths_by_study_non_prereg = {} + self._shared_source_paths = set() + + def prepare_from_sources(self, records: list[dict[str, str]]) -> None: + """Chunk source files separately, preserving source/study provenance. + + Each record must contain ``source_path`` and ``text``. Optional + ``text_path`` is ignored here but can be useful in reports. + """ + chunks: list[Chunk] = [] + source_texts: list[str] = [] + by_study: dict[str, set[str]] = {} + by_study_non_prereg: dict[str, set[str]] = {} + shared: set[str] = set() + + for record in records: + source_path = str(record.get("source_path") or record.get("text_path") or "").strip() + text = record.get("text") or "" + if not source_path or not text.strip(): + continue + study_keys = _study_keys_from_source_path(source_path) + source_kind = _source_kind_from_path(source_path, study_keys) + built = build_chunks( + [(1, text)], + chunk_size=self.chunk_size, + overlap=self.overlap, + source_path=source_path, + source_kind=source_kind, + study_keys=study_keys, + ) + for chunk in built: + chunk.id = len(chunks) + chunks.append(chunk) + source_texts.append(text) + if study_keys: + for key in study_keys: + by_study.setdefault(key, set()).add(source_path) + if source_kind != "prereg": + by_study_non_prereg.setdefault(key, set()).add(source_path) + elif source_kind == "shared": + shared.add(source_path) + + self._chunks = chunks + self._source_norm = normalize_for_match("\n\n".join(source_texts)) + self._source_paths_by_study = by_study + self._source_paths_by_study_non_prereg = by_study_non_prereg + self._shared_source_paths = shared + + # -- core -------------------------------------------------------------- + def extract_slot( + self, + *, + study: str, + iv: str, + dv: str, + slot: str, + location_hint: str = "", + materials_notes: str = "", + prev_status: Optional[str] = None, + prev_content: Optional[str] = None, + paraphrase_threshold: Optional[float] = None, + assemble: bool = False, + ) -> SlotResult: + """Extract a slot from prepared source text. + + Args: + paraphrase_threshold: If provided, a second tier below ``self.threshold`` + where extracted content is accepted as ``paraphrased`` rather than + discarded. Useful for supplementary OSF files (DOCX/QSF) where the + text format differs from the paper (e.g. Qualtrics piped-text + variables, pre-registration prose). Typical value: 75. + assemble: When True (and ``paraphrase_threshold`` is set), first try a + MULTI-SPAN assembly: ask the model to gather every verbatim span + that together constitutes the material (e.g. donation intro + game + rules + per-condition manipulation + DV items, which in a Qualtrics + survey live in different blocks/conditions), then verify EACH span + individually against the source. This is the PaperQA-style + "gather evidence then compose" + LangExtract-style per-span + grounding path. Falls back to single-span extraction. + """ + if slot not in SLOT_PROFILE: + raise ValueError(f"Unknown slot: {slot}") + if not self._chunks: + raise RuntimeError("Call prepare(pdf_path) before extract_slot().") + + profile = SLOT_PROFILE[slot] + + if isinstance(prev_content, str) and prev_content.strip() and prev_status == "verbatim": + pre = best_partial_ratio(normalize_for_match(prev_content), self._source_norm) + if pre >= self.threshold: + return SlotResult( + status="verbatim", content=prev_content, score=pre, action="kept_verbatim_prechecked" + ) + + draft = prev_content if isinstance(prev_content, str) else "" + query_parts = (iv, dv, slot, profile["desc"], location_hint, materials_notes, draft[:600]) + if not self._has_source_metadata(): + query_parts = (study, *query_parts) + query = " ".join(str(x) for x in query_parts if x) + # Multi-span assembly retrieves a WIDER candidate set so pieces scattered + # across blocks/conditions all enter the evidence window. + eff_k = min(self.k * 3, len(self._chunks)) if assemble else self.k + scored = self._retrieve_scoped(study, query, k=eff_k, keywords=profile["keywords"]) + if not scored: + if self._has_source_metadata(): + return SlotResult( + status="source_missing", + content=None, + score=None, + action="source_missing_no_scoped_evidence", + ) + return self._fallback(prev_status, prev_content, score=None, action="not_present") + + evidence = format_evidence(scored) + evidence_pages = sorted({c.page_start for c, _ in scored} | {c.page_end for c, _ in scored}) + + # ---- Multi-span assembly (primary path for composite materials) ------ + if assemble and paraphrase_threshold is not None: + assembled = self._assemble_spans( + slot, study, iv, dv, profile["desc"], evidence, + paraphrase_threshold=paraphrase_threshold, + evidence_pages=evidence_pages, + ) + if assembled is not None: + return assembled + if slot == "materials" and self._only_prereg_sources_for_study(study): + return SlotResult( + status="source_missing", + content=None, + score=0.0, + action="source_missing_only_prereg_no_material_span", + evidence_pages=evidence_pages, + ) + feedback = "" + best_score = 0.0 + best_quote: str = "" + for attempt in range(self.retries + 1): + parsed = self._ask(slot, study, dv, profile["desc"], evidence, feedback, draft=draft) + if parsed.get("error"): + return SlotResult( + status=prev_status or "not_in_paper", + content=prev_content, + score=None, + action="skipped_api_error", + ) + if not parsed.get("found"): + break + quote = (parsed.get("quote") or "").strip() + if not quote: + break + score = best_partial_ratio(normalize_for_match(quote), self._source_norm) + if score > best_score: + best_score = score + best_quote = quote + if score >= self.threshold: + return SlotResult( + status="verbatim", + content=quote, + score=score, + action="grounded_verbatim", + evidence_pages=evidence_pages, + ) + feedback = ( + "Your previous answer was NOT found verbatim in the evidence " + f"(match score {score:.0f} < {self.threshold:.0f}). You MUST copy characters " + "exactly as they appear in the evidence passages, with no edits. If the text is " + "not present in the evidence, set found=false." + ) + + # Paraphrase tier A: verbatim extraction scored below strict threshold + # but above the paraphrase threshold — the text is clearly from the source. + if ( + paraphrase_threshold is not None + and best_quote + and best_score >= paraphrase_threshold + ): + return SlotResult( + status="paraphrased", + content=best_quote, + score=best_score, + action="grounded_paraphrased", + evidence_pages=evidence_pages, + ) + + # Paraphrase tier B: verbatim extraction returned NOT_PRESENT, but the + # caller allows paraphrasing (e.g. OSF/QSF sources with piped text). + # Fall back to a "describe from evidence" prompt that doesn't require + # a clean verbatim span — suitable for QSF survey format and + # pre-registration prose descriptions. + if paraphrase_threshold is not None: + described = self._ask_describe(slot, study, dv, profile["desc"], evidence) + if described.get("error"): + return SlotResult( + status=prev_status or "not_in_paper", + content=prev_content, + score=None, + action="skipped_api_error", + ) + desc_text = (described.get("description") or "").strip() + if desc_text: + desc_score = best_partial_ratio(normalize_for_match(desc_text), self._source_norm) + if desc_score >= paraphrase_threshold: + return SlotResult( + status="paraphrased", + content=desc_text, + score=desc_score, + action="grounded_paraphrased_assembled", + evidence_pages=evidence_pages, + ) + + return self._fallback(prev_status, prev_content, score=best_score, action="downgraded") + + def _has_source_metadata(self) -> bool: + return any(chunk.source_path for chunk in self._chunks) + + def _retrieve_scoped( + self, + study: str, + query: str, + *, + k: int, + keywords: list[str], + ) -> list[tuple[Chunk, float]]: + """Metadata-filtered retrieval: exact study sources, then shared, then global.""" + study_key = _study_key_from_label(study) + if not self._has_source_metadata() or not study_key: + return retrieve(self._chunks, query, k=k, keywords=keywords) + + tiers: list[set[str] | None] = [] + exact_non_prereg = self._source_paths_by_study_non_prereg.get(study_key, set()) + exact_all = self._source_paths_by_study.get(study_key, set()) + exact_prereg = exact_all - exact_non_prereg + if exact_non_prereg: + tiers.append(exact_non_prereg) + if exact_prereg: + tiers.append(exact_prereg) + if self._shared_source_paths: + tiers.append(self._shared_source_paths) + tiers.append(None) + + seen: set[str] = set() + for allowed in tiers: + key = "__global__" if allowed is None else "|".join(sorted(allowed)) + if key in seen: + continue + seen.add(key) + scored = retrieve( + self._chunks, + query, + k=k, + keywords=keywords, + allowed_sources=allowed, + use_bm25=True, + ) + if scored: + return scored + return [] + + def _only_prereg_sources_for_study(self, study: str) -> bool: + study_key = _study_key_from_label(study) + if not study_key: + return False + exact_all = self._source_paths_by_study.get(study_key, set()) + if not exact_all: + return False + exact_non_prereg = self._source_paths_by_study_non_prereg.get(study_key, set()) + return not exact_non_prereg + + def _fallback( + self, prev_status: Optional[str], prev_content: Optional[str], *, score, action: str + ) -> SlotResult: + """Honest downgrade: never present unverifiable text as verbatim.""" + if isinstance(prev_content, str) and prev_content.strip(): + keep_status = prev_status if prev_status in {"cited_scale", "osf_only"} else "paraphrased" + return SlotResult( + status=keep_status, + content=prev_content, + score=score, + action="kept_paraphrased" if keep_status == "paraphrased" else "unchanged", + ) + return SlotResult(status="not_in_paper", content=None, score=score, action=action) + + def _ask( + self, slot: str, study: str, dv: str, desc: str, evidence: str, feedback: str, draft: str = "" + ) -> dict[str, Any]: + system = ( + "You are a meticulous research assistant extracting VERBATIM text from a scientific " + "paper. You never paraphrase, summarize, normalize, or invent text. You only copy " + "characters exactly as they appear in the provided evidence." + ) + fb = f"\n\nIMPORTANT FEEDBACK:\n{feedback}\n" if feedback else "" + draft_block = "" + if draft and draft.strip(): + draft_block = ( + "\n\nAPPROXIMATE DRAFT (may be paraphrased or contain errors) — use it ONLY to " + "LOCATE the right passage; do NOT copy from it, copy from the EVIDENCE:\n" + f'"""{draft.strip()[:1500]}"""\n' + ) + prompt = f"""TASK: From the evidence passages below (extracted from the paper), copy {desc} +for "{study}" (dependent variable: {dv}). + +Rules: +- Copy the EXACT contiguous span of text from the evidence. Do not paraphrase, shorten, + reorder, fix typos, or merge separate passages. +- Prefer the single most complete contiguous passage that fully states the {slot}. +- The same materials/measure may be shared across studies; a passage describing it counts + even if it names a different study, as long as it states the {slot} for this measure. +- If the {slot} text is NOT contained in the evidence, set "found": false. +- Do NOT use any outside knowledge. Only the evidence below may be quoted.{fb}{draft_block} + +EVIDENCE: +{evidence} + +Respond with JSON exactly: +{{"found": true|false, "quote": "", "reason": ""}}""" + try: + data = generate_json( + self.client, + [{"role": "user", "content": prompt}], + system=system, + temperature=self.temperature, + retries=0, + timeout=self.slot_timeout, + ) + except Exception as exc: + return {"error": f"{type(exc).__name__}: {exc}"} + if not isinstance(data, dict): + return {"found": False, "quote": "", "reason": "bad_response"} + return data + + def _ask_describe( + self, slot: str, study: str, dv: str, desc: str, evidence: str + ) -> dict[str, Any]: + """Fallback for sources (e.g. QSF) where verbatim extraction fails. + + Asks the LLM to DESCRIBE the slot content based on the evidence rather + than copy a verbatim span. This handles QSF piped-text and pre-registration + prose. The result is tagged ``paraphrased``. + """ + system = ( + "You extract experimental measurement information from research materials. " + "You summarize what you find in the provided evidence. " + "Do not invent information not present in the evidence." + ) + prompt = f"""TASK: Based ONLY on the evidence below (from a study's supplementary materials), +describe {desc} for "{study}" (dependent variable: {dv}). + +Rules: +- Summarize what you can find in the evidence about this {slot}. Include question wording, + scale anchors, and response options if present. +- Do NOT use outside knowledge. Base your answer solely on the evidence. +- If the {slot} is NOT described in the evidence at all, set "found": false. +- Keep the description concise but complete (max 400 words). + +EVIDENCE: +{evidence} + +Respond with JSON exactly: +{{"found": true|false, "description": "", "reason": ""}}""" + try: + data = generate_json( + self.client, + [{"role": "user", "content": prompt}], + system=system, + temperature=self.temperature, + retries=0, + timeout=self.slot_timeout, + ) + except Exception as exc: + return {"error": f"{type(exc).__name__}: {exc}"} + if not isinstance(data, dict): + return {"found": False, "description": "", "reason": "bad_response"} + return data + + # -- multi-span assembly ----------------------------------------------- + def _assemble_spans( + self, + slot: str, + study: str, + iv: str, + dv: str, + desc: str, + evidence: str, + *, + paraphrase_threshold: float, + evidence_pages: list[int], + ) -> Optional[SlotResult]: + """Gather multiple labeled verbatim spans and verify each individually. + + Returns a SlotResult when at least one span grounds in the source, or + ``None`` to signal the caller should fall back to single-span extraction. + + Honesty guarantee: every span kept in the assembled content is + individually present in the source at >= paraphrase_threshold. Spans the + model returns that do NOT ground (likely edited/paraphrased/invented) are + dropped, never shown as verbatim. + """ + parsed = self._ask_assemble(slot, study, iv, dv, desc, evidence) + if parsed.get("error"): + return SlotResult(status="not_in_paper", content=None, score=None, action="skipped_api_error") + spans = parsed.get("spans") + if not isinstance(spans, list) or not spans: + return None + + grounded: list[tuple[str, str, float]] = [] + for span in spans: + if not isinstance(span, dict): + continue + quote = (span.get("quote") or "").strip() + label = (span.get("label") or "").strip() + if not quote: + continue + score = best_partial_ratio(normalize_for_match(quote), self._source_norm) + if score >= paraphrase_threshold: + grounded.append((label, quote, score)) + + if not grounded: + return None # nothing grounded — let single-span path try + + all_verbatim = all(score >= self.threshold for _, _, score in grounded) + content = "\n\n".join( + (f"[{label}]\n{quote}" if label else quote) for label, quote, _ in grounded + ) + weakest = min(score for _, _, score in grounded) + return SlotResult( + status="verbatim" if all_verbatim else "paraphrased", + content=content, + score=weakest, + action="grounded_assembled" if all_verbatim else "grounded_assembled_paraphrased", + evidence_pages=evidence_pages, + ) + + def _ask_assemble( + self, slot: str, study: str, iv: str, dv: str, desc: str, evidence: str + ) -> dict[str, Any]: + """Ask the model to gather ALL verbatim spans composing the material. + + Designed for composite stimuli (e.g. a behavioral paradigm whose intro, + rules, per-condition manipulation, and DV items live in separate Qualtrics + blocks/conditions). The model returns a list of {label, quote}; the caller + verifies each quote against the source. + """ + system = ( + "You are a meticulous research assistant assembling the COMPLETE set of stimulus " + "materials for one experimental measure from supplementary materials (which may be a " + "Qualtrics survey export with multiple blocks and conditions). You copy text VERBATIM " + "and never paraphrase, summarize, or invent. You gather EVERY relevant span, including " + "one span per experimental condition when the manipulation differs across conditions." + ) + prompt = f"""TASK: From the evidence passages below (extracted from a study's supplementary +materials), gather the COMPLETE set of verbatim spans that together constitute {desc} +for "{study}" (independent variable: {iv}; dependent variable: {dv}). + +These materials are often SPREAD ACROSS MULTIPLE blocks/conditions. For example a behavioral +paradigm may have: an introduction, the rules/instructions, a per-condition manipulation +(e.g. one version shown when condition=public and another when condition=private, or when a +variable like ambiguous=1 vs ambiguous=0), and the response items. The condition labels may +appear in a "CONDITIONS" outline or in "[shown if: ...]" tags rather than in the stimulus text. + +Rules: +- Return a LIST of spans. Each span = {{"label": "", "quote": ""}}. +- Copy each quote EXACTLY as it appears in the evidence — do not paraphrase, shorten, reorder, fix typos, or merge separate passages into one quote. One contiguous passage per span. +- Include a SEPARATE span for each condition's version of the manipulation when they differ. +- Do NOT use outside knowledge. Only quote text present in the evidence below. +- If none of the evidence contains this material, return an empty list. + +EVIDENCE: +{evidence} + +Respond with JSON exactly: +{{"spans": [{{"label": "...", "quote": "..."}}], "reason": ""}}""" + try: + data = generate_json( + self.client, + [{"role": "user", "content": prompt}], + system=system, + temperature=self.temperature, + retries=0, + timeout=self.slot_timeout, + ) + except Exception as exc: + return {"error": f"{type(exc).__name__}: {exc}"} + if not isinstance(data, dict): + return {"spans": [], "reason": "bad_response"} + return data + + def reground_paper( + self, + paper: dict[str, Any], + pdf_path: Path, + *, + slots: Optional[list[str]] = None, + only_verbatim_and_empty: bool = True, + ) -> tuple[dict[str, Any], dict[str, Any]]: + """Re-ground slots of an existing paper dict. Returns (new_paper, report).""" + self.prepare(pdf_path) + data = deepcopy(paper) + slots = slots or list(SLOT_NAMES) + records: list[dict[str, Any]] = [] + counts = {"grounded_verbatim": 0, "kept_paraphrased": 0, "not_in_paper": 0, "skipped": 0} + tasks, skipped = _collect_grounding_tasks( + data, + slots, + only_verbatim_and_empty=only_verbatim_and_empty, + ) + counts["skipped"] = skipped + total_slots = len(tasks) + worker_count = min(self.max_workers, total_slots) if total_slots else 0 + if total_slots: + print( + f" Stage 2 grounding pass: checking {total_slots} material slots " + f"with {worker_count} worker(s)", + flush=True, + ) + + task_results = self._run_grounding_tasks(tasks, worker_count) + for task, result, record in sorted(task_results, key=lambda item: item[0].order): + obj = data["eligible_studies"][task.study_index]["effects"][task.effect_index][task.slot] + obj["status"] = result.status + obj["content"] = result.content + counts[result.status] = counts.get(result.status, 0) + 1 + records.append(record) + + report = { + "threshold": self.threshold, + "summary": { + "slots_processed": len(records), + "slots_skipped": counts.get("skipped", 0), + "workers": worker_count, + "grounded_verbatim": counts.get("verbatim", counts.get("grounded_verbatim", 0)), + "downgraded_or_kept": sum( + 1 for r in records if r["action"] in {"kept_paraphrased", "downgraded", "not_present"} + ), + }, + "records": records, + } + return data, report + + def _run_grounding_tasks( + self, + tasks: list[GroundingTask], + worker_count: int, + ) -> list[tuple[GroundingTask, SlotResult, dict[str, Any]]]: + if not tasks: + return [] + if worker_count <= 1: + results = [] + total = len(tasks) + for idx, task in enumerate(tasks, start=1): + print( + f" grounding {idx}/{total}: " + f"{task.study_label} effect {task.effect_index + 1} {task.slot}", + flush=True, + ) + results.append(self._run_grounding_task(task)) + return results + + results: list[tuple[GroundingTask, SlotResult, dict[str, Any]]] = [] + with ThreadPoolExecutor(max_workers=worker_count) as executor: + future_to_task = {executor.submit(self._run_grounding_task, task): task for task in tasks} + for idx, future in enumerate(as_completed(future_to_task), start=1): + task, result, record = future.result() + print( + f" grounding completed {idx}/{len(tasks)}: " + f"{task.study_label} effect {task.effect_index + 1} {task.slot}", + flush=True, + ) + results.append((task, result, record)) + return results + + def _run_grounding_task(self, task: GroundingTask) -> tuple[GroundingTask, SlotResult, dict[str, Any]]: + result = self.extract_slot( + study=task.study_label, + iv=task.iv, + dv=task.dv, + slot=task.slot, + location_hint=task.location_hint, + materials_notes=task.materials_notes, + prev_status=task.prev_status, + prev_content=task.prev_content, + ) + record = { + "path": task.path, + "study": task.study_label, + "effect_index": task.effect_index, + "slot": task.slot, + "before": {"status": task.prev_status, "had_content": task.had_content}, + **result.to_dict(), + } + return task, result, record + + +def _collect_grounding_tasks( + paper: dict[str, Any], + slots: list[str], + *, + only_verbatim_and_empty: bool, +) -> tuple[list[GroundingTask], int]: + tasks: list[GroundingTask] = [] + skipped = 0 + order = 0 + for si, study in enumerate(paper.get("eligible_studies", [])): + if not isinstance(study, dict): + continue + study_label = study.get("study", f"study[{si}]") + for ei, effect in enumerate(study.get("effects", [])): + if not isinstance(effect, dict): + continue + iv, dv = effect.get("IV", ""), effect.get("DV", "") + loc = effect.get("table_or_page_location", "") + notes = effect.get("materials_notes", "") + for slot in slots: + obj = effect.get(slot) + if not isinstance(obj, dict): + continue + status = obj.get("status") + content = obj.get("content") + has_content = isinstance(content, str) and content.strip() != "" + if only_verbatim_and_empty and status not in {"verbatim", None} and not ( + status in {"paraphrased"} and has_content + ): + skipped += 1 + continue + tasks.append( + GroundingTask( + order=order, + study_index=si, + effect_index=ei, + slot=slot, + path=f"$.eligible_studies[{si}].effects[{ei}].{slot}", + study_label=study_label, + iv=iv, + dv=dv, + location_hint=loc, + materials_notes=notes, + prev_status=status, + prev_content=content, + had_content=has_content, + ) + ) + order += 1 + return tasks, skipped + + +def _write_json_atomic(path: Path, data: dict[str, Any], *, backup: bool = True) -> None: + import shutil + + if backup and path.exists(): + shutil.copy2(path, path.with_suffix(path.suffix + ".bak")) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + tmp.replace(path) + + +def _study_key_from_label(study_label: Any) -> str | None: + text = normalize_for_match(str(study_label or "")) + match = re.search(r"\b(?:study|experiment)\s*(\d+[a-z]?)\b", text) + if match: + return match.group(1) + match = re.search(r"\b(\d+[a-z]?)\b", text) + return match.group(1) if match else None + + +def _study_keys_from_source_path(source_path: str) -> tuple[str, ...]: + """Infer study identifiers from a source filename/path. + + Examples: + Study 3.qsf -> ("3",) + Studies 1a-b.qsf -> ("1a", "1b") + Pre-registrations/Study 5.pdf -> ("5",) + """ + text = normalize_for_match(Path(source_path).as_posix()) + keys: list[str] = [] + + # Ranges like "Studies 1a-b" or "Studies 1-3". + for match in re.finditer( + r"\bstudies?\s*(?P\d+[a-z]?)(?:\s*[-–]\s*(?P\d+[a-z]?|[a-z]))?", + text, + ): + start = match.group("start") + end = match.group("end") + if end: + keys.extend(_expand_study_range(start, end)) + else: + keys.append(start) + + # Directory-style paths such as ".../Study 2a/Code.txt". + for match in re.finditer(r"\bstudy\s*(?P\d+[a-z]?)\b", text): + keys.append(match.group("key")) + + seen: set[str] = set() + return tuple(key for key in keys if not (key in seen or seen.add(key))) + + +def _expand_study_range(start: str, end: str) -> list[str]: + start_match = re.fullmatch(r"(\d+)([a-z]?)", start) + end_match = re.fullmatch(r"(\d+)?([a-z]?)", end) + if not start_match or not end_match: + return [start] + start_num = int(start_match.group(1)) + start_suffix = start_match.group(2) + end_num = int(end_match.group(1) or start_num) + end_suffix = end_match.group(2) + + if start_suffix and end_suffix and start_num == end_num: + return [f"{start_num}{chr(code)}" for code in range(ord(start_suffix), ord(end_suffix) + 1)] + if not start_suffix and not end_suffix and start_num <= end_num and end_num - start_num <= 20: + return [str(num) for num in range(start_num, end_num + 1)] + return [start, f"{end_num}{end_suffix}"] + + +def _source_kind_from_path(source_path: str, study_keys: tuple[str, ...]) -> str: + text = normalize_for_match(Path(source_path).as_posix()) + if "pre-registration" in text or "pre registrations" in text or "prereg" in text: + return "prereg" + if "experimental materials" in text or Path(source_path).suffix.lower() in {".qsf", ".docx"}: + return "experimental" + if not study_keys: + return "shared" + return "study" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Re-ground verbatim slots of a paper JSON against its PDF") + parser.add_argument("--json", required=True, type=Path) + parser.add_argument("--pdf", required=True, type=Path) + parser.add_argument("--provider", default=None, help="Override; else resolved from settings/env") + parser.add_argument("--model", default=None, help="Override; else resolved from settings/env") + parser.add_argument("--base-url", default=None, help="Override OpenAI-compatible base URL") + parser.add_argument("--api-key", default=None, help="Override; prefer the provider env var") + parser.add_argument("--settings", type=Path, default=None, help="Path to settings.yaml") + parser.add_argument("--k", type=int, default=8) + parser.add_argument("--dry-run", action="store_true", help="Do not write JSON back") + parser.add_argument("--report", type=Path, help="Where to write the reground report") + args = parser.parse_args() + + from src.llm.factory import get_client + from generation_pipeline.settings import load_settings, resolve_llm_config + + settings = load_settings(args.settings) + cfg = resolve_llm_config( + settings, + provider=args.provider, + model=args.model, + api_key=args.api_key, + api_base=args.base_url, + ) + print(f"LLM: provider={cfg.provider} model={cfg.model} base_url={cfg.api_base}") + client = get_client(provider=cfg.provider, model=cfg.model, api_key=cfg.api_key, api_base=cfg.api_base) + extractor = GroundedSlotExtractor(client, threshold=DEFAULT_THRESHOLD, k=args.k) + + paper = json.loads(args.json.read_text(encoding="utf-8")) + new_paper, report = extractor.reground_paper(paper, args.pdf) + + if not args.dry_run: + _write_json_atomic(args.json, new_paper, backup=True) + report_path = args.report or args.json.parent / "reground_report.json" + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + print(json.dumps(report["summary"], indent=2, ensure_ascii=False)) + print(f"report -> {report_path}") + + +if __name__ == "__main__": + main() diff --git a/generation_pipeline/extractors/study_data_extractor.py b/generation_pipeline/extractors/study_data_extractor.py new file mode 100644 index 0000000..dc60f91 --- /dev/null +++ b/generation_pipeline/extractors/study_data_extractor.py @@ -0,0 +1,292 @@ +""" +Study & Data Extractor - Stage 2 (ai-ethics) + +Extracts study-level samples plus effect/finding/statistics records. The legacy +`materials / manipulation / items` effect slots remain in the schema for +backward compatibility, but participant-facing material recovery is a Stage 3 +study-level responsibility. +""" + +import json +import re +import sys +from pathlib import Path +from typing import Any, Dict, Optional + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from generation_pipeline.extractors.base_extractor import BaseExtractor +from generation_pipeline.utils.document_loader import DocumentLoader +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + + +PDF_TEXT_MAX_CHARS = 400000 + + +class StudyDataExtractor(BaseExtractor): + """Extract ethics findings/effect records for downstream material search.""" + + def process( + self, + stage1_json: Dict[str, Any], + pdf_path: Path, + regeneration_instructions: Optional[Dict[str, Any]] = None, + *, + grounded: bool = False, + ground_threshold: float = 90.0, + ground_k: int = 8, + ground_timeout: float | None = 60.0, + ground_workers: int = 4, + ) -> Dict[str, Any]: + loader = DocumentLoader() + pdf_info = loader.get_pdf_pages(pdf_path) + pdf_text = extract_pdf_text(pdf_path, max_chars=PDF_TEXT_MAX_CHARS) + prompt = self._build_prompt(stage1_json, pdf_path.name, len(pdf_info), regeneration_instructions) + full_prompt = f"PDF content:\n\n{pdf_text}\n\n{prompt}" + + try: + response = self.client.generate_content(prompt=full_prompt) + except Exception as e: + raise RuntimeError(f"Error calling LLM API: {e}.") + if response is None: + raise ValueError("LLM returned None response.") + + result = self._parse_response(response, stage1_json) + if _effect_count(result) == 0: + raise ValueError( + "Stage 2 extraction returned zero effects for eligible Stage 1 studies. " + "Re-run with regeneration feedback or inspect the Stage 1 experiment anchors." + ) + if grounded: + result = self._ground_slots( + result, + pdf_path, + threshold=ground_threshold, + k=ground_k, + timeout=ground_timeout, + workers=ground_workers, + ) + return result + + def _ground_slots( + self, + result: Dict[str, Any], + pdf_path: Path, + *, + threshold: float, + k: int, + timeout: float | None, + workers: int, + ) -> Dict[str, Any]: + try: + from generation_pipeline.extractors.grounded_slot_extractor import GroundedSlotExtractor + except Exception as exc: # pragma: no cover - import guard + import warnings + + warnings.warn(f"Grounding pass unavailable ({exc}); returning ungrounded slots.", RuntimeWarning) + return result + if not result.get("eligible_studies"): + return result + extractor = GroundedSlotExtractor( + self.client, + threshold=threshold, + k=k, + slot_timeout=timeout, + max_workers=workers, + ) + grounded, report = extractor.reground_paper( + result, pdf_path, only_verbatim_and_empty=True + ) + grounded["_grounding_report"] = report + return grounded + + def _build_prompt( + self, + stage1_json: Dict[str, Any], + pdf_name: str, + num_pages: int, + regeneration_instructions: Optional[Dict[str, Any]] = None, + ) -> str: + experiments_info = json.dumps(stage1_json.get("experiments", []), indent=2, ensure_ascii=False) + + feedback_section = "" + if regeneration_instructions: + feedback_section = "\n\n" + "=" * 80 + "\nVALIDATION FEEDBACK FROM PREVIOUS EXTRACTION:\n" + "=" * 80 + "\n" + if regeneration_instructions.get("missing_effects"): + feedback_section += "MISSING EFFECTS:\n" + for x in regeneration_instructions["missing_effects"]: + feedback_section += f" - {x}\n" + if regeneration_instructions.get("exact_stats_needed"): + feedback_section += "EXACT STATISTICS REQUIRED:\n" + for item in regeneration_instructions["exact_stats_needed"]: + feedback_section += f" - {item.get('reason', '')}\n" + if regeneration_instructions.get("data_corrections"): + feedback_section += "DATA CORRECTIONS NEEDED:\n" + for item in regeneration_instructions["data_corrections"]: + feedback_section += f" - {item.get('reason', '')}\n" + feedback_section += "=" * 80 + "\n\n" + + return f"""Extract per-effect records from the paper: {pdf_name} ({num_pages} pages). + +STAGE 1 FILTER RESULTS (only extract studies that were marked replicable / eligible): +{experiments_info} +{feedback_section} +This corpus tracks moral / ethical / prosocial behavior effects. Stage 2 is +responsible for study/effect/finding/statistics extraction, not final +participant-facing materials. The output JSON must match the project schema +exactly. For EACH eligible study, list every reported statistical effect +separately under `effects[]`; downstream code will consolidate those rows into +study-level findings and simulation targets. + +CRITICAL RULES: +- Numeric fields that are not reported MUST be `null` (not omitted, not empty string). +- `stats.ci` is ALWAYS a two-element array `[low, high]` — use `[null, null]` if not reported. +- `p_value` is stored as a STRING (e.g. ".03", "<.001"). +- `sig` is one of: "sig", "ns", "marginal". +- `direction` is one of: "pos", "neg", "null". +- `effecttype` codes: "main" | "int" (interaction) | "simple" | "mediation" | "correlation". +- `size` is not effect size. It is only an integer participant/sample N for this + effect analysis, or null. Put Cohen's d in `stats.D`, standardized beta/r in + `stats.b`, eta-square in `stats.eta_square`, and exact result text in + `reported_statistics_text`. +- `materials_notes` is only a one-line paper/source hint for Stage 3 material search. +- `materials`, `manipulation`, `items` are legacy compatibility slots. Do NOT + reconstruct participant-facing materials here. Use + `{{"status": null, "content": null}}` unless the paper prints a short exact + quote that is directly useful as a source locator. Full instructions, + stimuli, response options, anchors, and condition levels are recovered at + study/sub-study level in Stage 3. +- `table_or_page_location`: e.g. "Table 1, p. 489" or "Study 2 Results, Openness". + +SAMPLE FIELD — CRITICAL RULES: +- `sample` lives at the STUDY level (not inside each effect). +- It describes the TOTAL participant pool for the whole study — the number of + people recruited/consented, not any subgroup or analysis cell. +- Most fields are nullable — only fill what the paper actually reports. +- `total_n`: The FULL study recruited / final-sample N (e.g., "We recruited + 4,001 participants via Prolific" → total_n=4001). Do NOT use a subgroup n + or cell n that appears only in a statistics table or condition row. +- `analyzed_n`: Only fill when the paper reports a DIFFERENT n after exclusions. + If not stated separately, leave null. +- `female_percent` and `male_percent`: ALWAYS 0-100 scale (e.g. 55.4, not 0.554). +- `platform` (STRICTLY controlled vocab — output EXACTLY one token, no sentences): + MTurk | Prolific | CloudResearch | Undergraduate | Graduate | + Lab | Organizational | Online | Field | Archival | Mixed | Other + WRONG: "we recruited subjects via the platform CloudResearch." + RIGHT: "CloudResearch" + Guide: political/moral psychology online → MTurk or Prolific; + campus samples → Undergraduate; workplace studies → Organizational; + real-world interventions → Field; secondary records → Archival. +- `inclusion_criteria`: VERBATIM quote of who was eligible (e.g. "self-identified + Democrats", "U.S. adults who had been fired from a job"). null if not stated. +- `exclusion_criteria`: VERBATIM quote about who was excluded (attention checks, + manipulation failure, outliers). null if not stated. +- `notes`: VERBATIM quote for any other unusual sampling detail. null otherwise. +- NEVER write LLM-composed summaries. Every string field is a direct quote or null. + +ANALYSIS SAMPLE FIELDS (effect level, both optional): +- `analysis_n` (integer or null): fill ONLY when this specific analysis used a + SUBSET of the study sample (e.g., Democrats-only, a condition arm, a cell). + If the full study sample was used, leave null. +- `analysis_scope` (string or null, controlled vocab): + full_sample | subgroup | condition | cell | simple_effect | other + Fill only when analysis_n is non-null. Examples: + Democrats-only subgroup → analysis_n=958, analysis_scope="subgroup" + One arm of between-subjects → analysis_n=200, analysis_scope="condition" + 2×2 cell → analysis_n=101, analysis_scope="cell" + +OUTPUT FORMAT — respond with ONLY this JSON (no markdown fences): + +{{ + "paper_title": "...", + "paper_metadata": {{ + "authors": ["..."], + "year": 2024, + "journal": "...", + "doi": "...", + "link": "..." + }}, + "eligible_studies": [ + {{ + "study": "Study 1", + "eligibility_rationale": "...", + "sample": {{ + "total_n": 183, + "analyzed_n": null, + "mean_age": 36.5, + "female_percent": 55.2, + "male_percent": 44.8, + "platform": "MTurk", + "country": "United States", + "inclusion_criteria": "", + "exclusion_criteria": "", + "notes": null + }}, + "effects": [ + {{ + "platform": "MTurk", + "effecttype": "int", + "IV": "...", + "DV": "...", + "size": null, + "direction": "pos", + "mean_group1": null, + "sd_group1": null, + "mean_group2": null, + "sd_group2": null, + "stats": {{ + "B": 0.35, "b": null, "chi_square": null, "D": null, + "eta_square": 0.027, "f": null, "t": 2.19, "z": null, + "ci": [null, null], + "p_value": ".03", + "sig": "sig" + }}, + "materials_notes": "One-line source/search hint for Stage 3 material recovery.", + "table_or_page_location": "Study 1 Results", + "analysis_n": null, + "analysis_scope": null, + "materials": {{ "status": null, "content": null }}, + "manipulation": {{ "status": null, "content": null }}, + "items": {{ "status": null, "content": null }} + }} + ] + }} + ] +}}""" + + def _parse_response(self, response: str, stage1_json: Dict[str, Any]) -> Dict[str, Any]: + response_text = response.strip() if isinstance(response, str) else str(response).strip() + if "```json" in response_text: + response_text = response_text.split("```json")[1].split("```")[0].strip() + elif "```" in response_text: + response_text = response_text.split("```")[1].split("```")[0].strip() + try: + result = _loads_llm_json(response_text) + except json.JSONDecodeError: + m = re.search(r"\{.*\}", response_text, re.DOTALL) + if m: + result = _loads_llm_json(m.group()) + else: + raise ValueError(f"Could not parse JSON: {response_text[:200]}") + + if not isinstance(result, dict): + result = {} + result.setdefault("paper_id", stage1_json.get("paper_id", "unknown")) + return result + + +def _effect_count(result: Dict[str, Any]) -> int: + return sum( + len(study.get("effects", []) or []) + for study in result.get("eligible_studies", []) or [] + if isinstance(study, dict) + ) + + +def _loads_llm_json(text: str) -> Any: + """Parse JSON with a small repair pass for common LLM formatting mistakes.""" + try: + return json.loads(text) + except json.JSONDecodeError: + repaired = re.sub(r",(\s*[}\]])", r"\1", text) + return json.loads(repaired) diff --git a/generation_pipeline/fetch_osf.py b/generation_pipeline/fetch_osf.py new file mode 100644 index 0000000..9533675 --- /dev/null +++ b/generation_pipeline/fetch_osf.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +""" +Download OSF supplementary materials for stage4 papers whose experiments +reference materials not present in the paper (osf_only / not_in_paper / cited_scale). + +Usage: + # Preview which papers need OSF materials + python generation_pipeline/fetch_osf.py --list + + # Dry-run download plan for one paper + python generation_pipeline/fetch_osf.py --only TheBenefit --dry-run + + # Download all OSF files for papers with OSF links (includes all-linked by default) + python generation_pipeline/fetch_osf.py + + # Only papers still missing materials in JSON + python generation_pipeline/fetch_osf.py --missing-only + +Private / view-only projects: + export OSF_TOKEN=... # optional bearer token for authenticated projects +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from generation_pipeline.utils.osf_crawler import run_batch + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fetch OSF materials for stage4 corpus") + parser.add_argument( + "--stage4-dir", + type=Path, + default=Path("stage4"), + help="Root directory containing per-paper JSON folders (default: stage4)", + ) + parser.add_argument( + "--only", + action="append", + default=None, + help="Only process papers whose path/folder matches substring (repeatable)", + ) + parser.add_argument( + "--list", + action="store_true", + help="List papers with missing OSF-referenced materials; do not download", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Resolve OSF and show matched files without writing to disk", + ) + parser.add_argument( + "--missing-only", + action="store_true", + help="Only fetch papers with missing materials in JSON (legacy behavior)", + ) + parser.add_argument( + "--token", + default=os.environ.get("OSF_TOKEN"), + help="OSF API bearer token (or set OSF_TOKEN env var)", + ) + args = parser.parse_args() + + if not args.stage4_dir.exists(): + raise SystemExit(f"stage4 dir not found: {args.stage4_dir}") + + results = run_batch( + args.stage4_dir, + only=args.only, + token=args.token, + download_all=True, + dry_run=args.dry_run, + list_only=args.list, + all_linked=not args.missing_only, + missing_only=args.missing_only, + ) + + if args.list: + if not results: + print("No papers with OSF link found.") + return + label = "missing materials" if args.missing_only else "OSF link" + print(f"Found {len(results)} paper(s) with {label}:\n") + for plan in results: + print(f"- {plan.paper_folder}") + print(f" title: {plan.paper_title[:80]}") + print(f" osf: {plan.osf_url or '(no link in JSON)'}") + files_dir = args.stage4_dir / plan.paper_folder / "osf" / "files" + n_dl = sum(1 for _ in files_dir.rglob("*") if _.is_file()) if files_dir.exists() else 0 + print(f" missing slots: {len(plan.effects_needing_osf)} | osf/files on disk: {n_dl}") + for need in plan.effects_needing_osf[:3]: + print(f" • {need.study} [{need.slot}={need.status}] IV={need.iv[:50]}") + if len(plan.effects_needing_osf) > 3: + print(f" • ... +{len(plan.effects_needing_osf) - 3} more") + print() + return + + ok, err = 0, 0 + for plan in results: + if plan.errors and not plan.downloaded_files and not plan.skipped_files: + err += 1 + print(f"✗ {plan.paper_folder}: {'; '.join(plan.errors)}") + continue + ok += 1 + action = "would download" if args.dry_run else "downloaded" + n = len(plan.skipped_files) if args.dry_run else len(plan.downloaded_files) + print(f"✓ {plan.paper_folder}: {action} {n} file(s)") + files = plan.skipped_files if args.dry_run else plan.downloaded_files + for path in files[:8]: + print(f" {path}") + if len(files) > 8: + print(f" ... +{len(files) - 8} more") + for msg in plan.errors: + print(f" ⚠ {msg}") + + print(f"\nDone: {ok} paper(s) processed, {err} error(s).") + + +if __name__ == "__main__": + main() diff --git a/generation_pipeline/fill_osf.py b/generation_pipeline/fill_osf.py new file mode 100644 index 0000000..4f7ffe1 --- /dev/null +++ b/generation_pipeline/fill_osf.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Fill JSON slots from downloaded OSF materials.""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from generation_pipeline.utils.osf_crawler import run_batch as fetch_osf +from generation_pipeline.utils.osf_fill import run_fill + + +def main(): + parser = argparse.ArgumentParser(description="Fill OSF materials into stage4 JSON") + parser.add_argument("--stage4-dir", type=Path, default=Path("stage4")) + parser.add_argument("--only", action="append", default=None) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--fetch", action="store_true", help="Download OSF files first") + parser.add_argument("--token", default=None, help="OSF_TOKEN for private projects") + args = parser.parse_args() + + if args.fetch: + print("Downloading OSF files...") + plans = fetch_osf( + args.stage4_dir, + only=args.only, + token=args.token, + dry_run=False, + ) + for p in plans: + if p.errors: + print(f" ⚠ {p.paper_folder}: {p.errors[0]}") + + print("Filling JSON from OSF files...") + results = run_fill(args.stage4_dir, only=args.only, dry_run=args.dry_run) + for r in results: + if r.get("skipped"): + print(f"- {r['paper']}: skipped ({r['skipped']})") + continue + ch = r.get("changes", []) + if ch: + print(f"✓ {r['paper']}: {len(ch)} update(s)" + (" (dry-run)" if r.get("dry_run") else "")) + for c in ch: + print(f" • {c}") + else: + print(f"- {r['paper']}: no slots filled") + + +if __name__ == "__main__": + main() diff --git a/generation_pipeline/filters/__init__.py b/generation_pipeline/filters/__init__.py new file mode 100644 index 0000000..fbd522f --- /dev/null +++ b/generation_pipeline/filters/__init__.py @@ -0,0 +1,4 @@ +from generation_pipeline.filters.base_filter import BaseFilter +from generation_pipeline.filters.replicability_filter import ReplicabilityFilter + +__all__ = ["BaseFilter", "ReplicabilityFilter"] diff --git a/generation_pipeline/filters/base_filter.py b/generation_pipeline/filters/base_filter.py new file mode 100644 index 0000000..19a2ea5 --- /dev/null +++ b/generation_pipeline/filters/base_filter.py @@ -0,0 +1,39 @@ +""" +Base Filter Class +""" + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Dict, Any, Optional + + +class BaseFilter(ABC): + """Base class for all filters""" + + def __init__(self, llm_client): + """ + Initialize filter. + + Args: + llm_client: LLM client instance (GeminiClient) + """ + self.client = llm_client + + @abstractmethod + def process( + self, + pdf_path: Path, + regeneration_instructions: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """ + Process PDF and return filter results. + + Args: + pdf_path: Path to PDF file + regeneration_instructions: Optional verifier feedback from a prior + extraction attempt. + + Returns: + Dictionary with filter results + """ + raise NotImplementedError diff --git a/generation_pipeline/filters/replicability_filter.py b/generation_pipeline/filters/replicability_filter.py new file mode 100644 index 0000000..82545a4 --- /dev/null +++ b/generation_pipeline/filters/replicability_filter.py @@ -0,0 +1,272 @@ +""" +Replicability Filter - Stage 1 (ai-ethics) + +Identifies which experiments in a moral / ethical / prosocial-behavior paper +are eligible for our extraction corpus, and pulls paper metadata. +""" + +import json +import re +import sys +from pathlib import Path +from typing import Dict, Any, List, Optional + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from generation_pipeline.filters.base_filter import BaseFilter +from generation_pipeline.stage1_study_contract import slugify +from generation_pipeline.utils.document_loader import DocumentLoader +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + + +PDF_TEXT_MAX_CHARS = 400000 + + +class ReplicabilityFilter(BaseFilter): + """Stage 1 filter for ethics papers.""" + + def process( + self, + pdf_path: Path, + regeneration_instructions: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + loader = DocumentLoader() + pdf_info = loader.get_pdf_pages(pdf_path) + pdf_text = extract_pdf_text(pdf_path, max_chars=PDF_TEXT_MAX_CHARS) + prompt = self._build_prompt(pdf_path.name, len(pdf_info), regeneration_instructions) + full_prompt = f"PDF content:\n\n{pdf_text}\n\n{prompt}" + + try: + response = self.client.generate_content(prompt=full_prompt) + except Exception as e: + raise RuntimeError( + f"Error calling LLM API: {e}. Check config/settings.yaml, provider API key env, " + "and optional BASE_URL." + ) + if response is None: + raise ValueError("LLM returned None response.") + + return self._parse_response(response, pdf_path) + + def _build_prompt( + self, + pdf_name: str, + num_pages: int, + regeneration_instructions: Optional[Dict[str, Any]] = None, + ) -> str: + feedback_section = "" + if regeneration_instructions: + feedback_section = "\n\n" + "=" * 80 + "\nVALIDATION FEEDBACK FROM PREVIOUS STAGE 1 INVENTORY:\n" + "=" * 80 + "\n" + if regeneration_instructions.get("missing_studies"): + feedback_section += "MISSING STUDIES TO ADD OR RECHECK:\n" + for item in regeneration_instructions["missing_studies"]: + feedback_section += f" - {item}\n" + if regeneration_instructions.get("split_merge_corrections"): + feedback_section += "SPLIT/MERGE CORRECTIONS:\n" + for item in regeneration_instructions["split_merge_corrections"]: + if isinstance(item, dict): + feedback_section += f" - {item.get('study', '')}: {item.get('reason', '')}\n" + else: + feedback_section += f" - {item}\n" + if regeneration_instructions.get("eligibility_corrections"): + feedback_section += "ELIGIBILITY CORRECTIONS:\n" + for item in regeneration_instructions["eligibility_corrections"]: + if isinstance(item, dict): + feedback_section += ( + f" - {item.get('study', '')}: expected={item.get('expected_label', '')}; " + f"{item.get('reason', '')}\n" + ) + else: + feedback_section += f" - {item}\n" + feedback_section += "=" * 80 + "\n\n" + + return f"""Analyze the research paper in the attached PDF: {pdf_name} ({num_pages} pages). + +This corpus studies MORAL, ETHICAL, and PROSOCIAL BEHAVIOR in psychology / management. +Identify each experiment/study and decide whether it is ELIGIBLE for our extraction. +{feedback_section} + +ELIGIBILITY CRITERIA (a study is eligible only if ALL hold): +- Empirical / experimental / quasi-experimental — not purely theoretical or qualitative. +- Reports at least one quantitative effect with a usable statistic (t, F, B/b, chi-square, z, d, eta_square, etc.). +- Outcome variable concerns moral / ethical / prosocial / norm-related behavior, judgment, intention, or perception + (e.g. cheating, lying, helping, fairness, punishment, moral judgment, ethical leadership, CSR perception, etc.). +- Sample size and design are recoverable from the paper. + +For EACH study/experiment, report: +- experiment_id (e.g. "Study 1", "Experiment 2a") +- study_id: stable machine id, snake_case if possible (e.g. "study_1", + "study_2a"); keep unique within the paper +- experiment_name: short description +- study_name: short human-readable study name; usually same as experiment_name +- design_type: concise study design label (e.g. "between-subjects", + "within-subjects", "mixed", "correlational", "field", "archival"), or null + if the design cannot be recovered +- conditions_or_factors: array of concise strings naming the manipulated, + measured, or repeated factors and their levels (e.g. + ["message sidedness: one-sided vs two-sided", "attitude strength: continuous"]), + or [] if no clear factors are recoverable +- input: what participants saw/read/did +- participant_task: one sentence describing the participant-facing task, + including the stimulus category and response action when recoverable +- participants: brief description (N, source) +- output: what was measured +- candidate_source_hints: array of objects describing where Stage 3 should look + for participant-facing materials, each with: + {{"kind": "paper|appendix|supplement|osf|cited_scale|unknown", + "description": "short source hint", + "expected_fields": ["instructions", "stimulus", "items", "options", "conditions"]}} +- replicable: YES / NO / UNCERTAIN +- has_self_contained_materials: true/false (full stimulus / scale items present in paper or supplement) +- exclusion_reasons: [] if eligible, else list of reasons (e.g. "no quantitative outcome", "outcome not moral/ethical") +- missing_materials: empty string or description of what is missing (e.g. "scale items referenced from Reynolds 2008") + +Also extract: paper_title, paper_authors, paper_abstract. + +Respond ONLY with valid JSON (no markdown fences): +{{ + "paper_title": "...", + "paper_authors": ["..."], + "paper_abstract": "...", + "experiments": [ + {{ + "experiment_id": "Study 1", + "study_id": "study_1", + "experiment_name": "...", + "study_name": "...", + "design_type": "between-subjects", + "conditions_or_factors": ["condition: control vs treatment"], + "input": "...", + "participant_task": "...", + "participants": "...", + "output": "...", + "candidate_source_hints": [ + {{ + "kind": "paper", + "description": "stimulus and outcome items appear in Study 1 method section", + "expected_fields": ["instructions", "stimulus", "items", "options", "conditions"] + }} + ], + "replicable": "YES", + "has_self_contained_materials": true, + "exclusion_reasons": [], + "missing_materials": "" + }} + ], + "overall_replicable": true, + "confidence": 0.85, + "notes": "..." +}}""" + + def _parse_response(self, response: str, pdf_path: Path) -> Dict[str, Any]: + response_text = response.strip() if isinstance(response, str) else str(response).strip() + if "```json" in response_text: + response_text = response_text.split("```json")[1].split("```")[0].strip() + elif "```" in response_text: + response_text = response_text.split("```")[1].split("```")[0].strip() + try: + result = json.loads(response_text) + except json.JSONDecodeError: + m = re.search(r"\{.*\}", response_text, re.DOTALL) + if m: + result = json.loads(m.group()) + else: + raise ValueError(f"Could not parse JSON: {response_text[:200]}") + + if not isinstance(result, dict): + result = {} + _normalize_experiments(result) + paper_id = pdf_path.stem.replace(" ", "_").replace("-", "_").lower() + result["paper_id"] = paper_id + return result + + +def _normalize_experiments(result: Dict[str, Any]) -> None: + experiments = result.get("experiments") + if not isinstance(experiments, list): + return + seen_ids: set[str] = set() + for index, exp in enumerate(experiments, start=1): + if not isinstance(exp, dict): + continue + label = str(exp.get("experiment_id") or exp.get("study_name") or exp.get("experiment_name") or f"Study {index}").strip() + exp["experiment_id"] = label + study_id = slugify(exp.get("study_id") or label, fallback=f"study_{index}") + if study_id in seen_ids: + study_id = f"{study_id}_{index}" + seen_ids.add(study_id) + exp["study_id"] = study_id + name = str(exp.get("study_name") or exp.get("experiment_name") or label).strip() + exp["study_name"] = name + exp.setdefault("experiment_name", name) + exp.setdefault("design_type", None) + conditions = exp.get("conditions_or_factors", []) + if conditions is None: + exp["conditions_or_factors"] = [] + elif isinstance(conditions, str): + text = conditions.strip() + exp["conditions_or_factors"] = [text] if text else [] + elif isinstance(conditions, list): + exp["conditions_or_factors"] = [ + str(item).strip() for item in conditions if str(item).strip() + ] + else: + text = str(conditions).strip() + exp["conditions_or_factors"] = [text] if text else [] + exp["participant_task"] = _normalize_participant_task(exp) + exp["candidate_source_hints"] = _normalize_source_hints(exp) + + +def _normalize_participant_task(exp: Dict[str, Any]) -> str: + task = str(exp.get("participant_task") or "").strip() + if task: + return task + parts: List[str] = [] + input_text = str(exp.get("input") or "").strip() + output_text = str(exp.get("output") or "").strip() + if input_text: + parts.append(f"participants saw/did: {input_text}") + if output_text: + parts.append(f"measured: {output_text}") + return "; ".join(parts) + + +def _hint_dict(kind: str, description: str, expected_fields: List[str]) -> Dict[str, Any]: + return { + "kind": kind, + "description": description, + "expected_fields": expected_fields, + } + + +def _normalize_source_hints(exp: Dict[str, Any]) -> List[Dict[str, Any]]: + raw = exp.get("candidate_source_hints") + hints: List[Dict[str, Any]] = [] + if isinstance(raw, str) and raw.strip(): + hints.append(_hint_dict("unknown", raw.strip(), ["instructions", "stimulus", "items", "options", "conditions"])) + elif isinstance(raw, list): + for item in raw: + if isinstance(item, dict): + kind = str(item.get("kind") or "unknown").strip().lower() or "unknown" + description = str(item.get("description") or item.get("source") or item.get("hint") or "").strip() + expected = item.get("expected_fields") + if isinstance(expected, str): + expected_fields = [expected.strip()] if expected.strip() else [] + elif isinstance(expected, list): + expected_fields = [str(field).strip() for field in expected if str(field).strip()] + else: + expected_fields = [] + hints.append(_hint_dict(kind, description, expected_fields)) + elif str(item).strip(): + hints.append(_hint_dict("unknown", str(item).strip(), ["instructions", "stimulus", "items", "options", "conditions"])) + if hints: + return hints + + expected_fields = ["instructions", "stimulus", "items", "options", "conditions"] + missing = str(exp.get("missing_materials") or "").strip() + if missing: + return [_hint_dict("unknown", missing, expected_fields)] + if exp.get("has_self_contained_materials") is True: + desc = str(exp.get("input") or exp.get("participant_task") or "participant-facing materials should be in the paper").strip() + return [_hint_dict("paper", desc, expected_fields)] + return [_hint_dict("unknown", "participant-facing material source not specified by Stage 1", expected_fields)] diff --git a/generation_pipeline/generators/__init__.py b/generation_pipeline/generators/__init__.py new file mode 100644 index 0000000..7fa3860 --- /dev/null +++ b/generation_pipeline/generators/__init__.py @@ -0,0 +1,3 @@ +""" +Generator modules for generation pipeline. +""" diff --git a/generation_pipeline/generators/config_generator.py b/generation_pipeline/generators/config_generator.py new file mode 100644 index 0000000..aa34200 --- /dev/null +++ b/generation_pipeline/generators/config_generator.py @@ -0,0 +1,358 @@ +""" +Config Generator - Generates StudyConfig classes from extraction results using LLM +""" + +import json +import re +import sys +from pathlib import Path +from typing import Dict, Any, Optional + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) +from src.llm.factory import get_client + + +class ConfigGenerator: + """Generate StudyConfig classes from extraction results using LLM""" + + def __init__( + self, + provider: str = "gemini", + model: str = "models/gemini-3-flash-preview", + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ): + """ + Initialize config generator. + + Args: + provider: One of gemini, openai, anthropic, xai, openrouter + model: Model name + api_key: Optional API key + api_base: Optional API base URL + """ + self.client = get_client(provider=provider, model=model, api_key=api_key, api_base=api_base) + + def generate( + self, + extraction_result: Dict[str, Any], + study_id: str, + output_path: Path, + pdf_path: Optional[Path] = None, + study_dir: Optional[Path] = None, + standalone: bool = False, + ) -> Path: + """ + Generate StudyConfig class file using a combination of Template and LLM. + """ + studies = extraction_result.get('studies', []) + if not studies: + raise ValueError("No studies found in extraction result") + + if pdf_path is None and study_dir: + pdf_files = list(study_dir.glob("*.pdf")) + if pdf_files: + pdf_path = pdf_files[0] + + study_context = {} + material_previews = [] + if study_dir: + for json_file in ["metadata.json", "specification.json", "ground_truth.json"]: + p = study_dir / json_file + if p.exists(): + try: + study_context[json_file] = json.loads(p.read_text(encoding='utf-8')) + except: pass + + materials_dir = study_dir / "materials" + if materials_dir.exists(): + for json_file in materials_dir.glob("*.json"): + try: + content = json_file.read_text(encoding='utf-8') + preview = "\n".join(content.splitlines()[:20]) + material_previews.append(f"FILE: {json_file.name}\n{preview}\n...") + except: pass + + material_context = "\n\n".join(material_previews) + + prompt = self._build_logic_only_prompt( + json.dumps(extraction_result, indent=2, ensure_ascii=False), + study_id, + json.dumps(study_context, indent=2, ensure_ascii=False), + material_context + ) + + try: + response = self.client.generate_content(prompt=prompt) + except Exception as e: + raise RuntimeError(f"LLM Error: {e}") + + logic_code = self._extract_code_from_response(response) + + class_name = f"Study{study_id.replace('_', '').capitalize()}Config" + if standalone: + imports = """import json +import random +import re +import numpy as np +from scipy import stats +from pathlib import Path +from typing import Dict, Any, List, Optional + +from study_utils import BaseStudyConfig, PromptBuilder +""" + else: + imports = """import json +import random +import re +import numpy as np +from scipy import stats +from pathlib import Path +from typing import Dict, Any, List, Optional + +from src.core.study_config import BaseStudyConfig, StudyConfigRegistry +from src.agents.prompt_builder import PromptBuilder +""" + + final_code = f"""{imports} +{logic_code} +""" + if f"class {class_name}" not in final_code: + final_code = re.sub(r"class StudyConfig\s*:", f"class {class_name}(BaseStudyConfig):", final_code) + final_code = re.sub(r"class \w+Config\s*:", f"class {class_name}(BaseStudyConfig):", final_code) + final_code = re.sub(r"class \w+Config\(BaseStudyConfig\)\s*:", f"class {class_name}(BaseStudyConfig):", final_code) + + if standalone: + final_code = re.sub(r"^@StudyConfigRegistry\.register\([^\n]+\)\n", "", final_code, flags=re.MULTILINE) + final_code = final_code.replace( + "from src.core.study_config import BaseStudyConfig, StudyConfigRegistry", + "from study_utils import BaseStudyConfig, PromptBuilder", + ) + final_code = final_code.replace( + "from src.agents.prompt_builder import PromptBuilder\n", + "", + ) + final_code = final_code.replace("StudyConfigRegistry.register", "# StudyConfigRegistry.register") + elif f'@StudyConfigRegistry.register("{study_id}")' not in final_code: + final_code = final_code.replace(f"class {class_name}", f'@StudyConfigRegistry.register("{study_id}")\nclass {class_name}') + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(final_code, encoding='utf-8') + return output_path + + def _build_logic_only_prompt(self, extraction_summary, study_id, context_summary, material_context): + # Use a template string and manual replacement to avoid f-string curly brace errors with JSON/Code + template = """You are a Python expert for HumanStudyBench. Your task is to write the CORE LOGIC for `[[STUDY_ID]]_config.py`. + +STUDY ID: [[STUDY_ID]] + +### Core Principles +1. **Match the human experimental design exactly** - One trial per participant with all items (unless within-subjects explicitly requires multiple trials) +2. **Use class attributes** - `prompt_builder_class` and `PROMPT_VARIANT` must be class attributes, not instance attributes +3. **Never skip sub-studies** - If `n=0` in specification, use a default (e.g., `n=50`) to ensure all experiments run + +### Available Methods (from BaseStudyConfig) +- `self.load_material(sub_id)` - Load material JSON (sub_id is filename without .json extension) +- `self.load_specification()` - Returns `{"participants": {"n": ..., "by_sub_study": {...}}, ...}` +- `self.load_ground_truth()` - Returns `{"studies": [{"findings": [...]}], ...}` +- `self.extract_numeric(text)`, `self.extract_choice(text, options)` - Parse responses + +### Note on Findings +- The study's `metadata.json` contains a `findings` array with finding-level weights (used for evaluation aggregation) +- Each finding has a `finding_id` that matches the `finding_id` in `ground_truth.json` +- This information is primarily used by evaluators, not config generation + +### EXTRACTION SUMMARY (Goal) +[[EXTRACTION_SUMMARY]] + +### MATERIALS (Context) +[[MATERIAL_CONTEXT]] + +### Working Examples + +**Example 1: Simple Study (study_001)** +```python +class CustomPromptBuilder(PromptBuilder): + def __init__(self, study_path: Path): + super().__init__(study_path) + + def build_trial_prompt(self, trial_metadata): + # Note: System prompt is now handled separately by SystemPromptRegistry + # This method only builds the task/trial content + items = trial_metadata.get("items", []) + + prompt = "" + + # Add task context introduction (similar to study_003/004) + prompt += "You are participating in a psychology study on decision-making. Please read the following instructions and provide your responses.\n\n" + + # Build questions with Q indices + q_counter = 1 + for item in items: + prompt += f"Q{q_counter} (answer with letter: A or B): {item['question']}\n" + item["q_idx"] = q_counter + q_counter += 1 + + # RESPONSE_SPEC + prompt += "\nRESPONSE_SPEC: Output Q1=, Q2=, etc.\n" + return prompt + +@StudyConfigRegistry.register("study_001") +class StudyStudy001Config(BaseStudyConfig): + prompt_builder_class = CustomPromptBuilder # Class attribute + PROMPT_VARIANT = "v3" # Class attribute + + def __init__(self, study_path: Path, specification: Dict[str, Any]): + super().__init__(study_path, specification) + + def create_trials(self, n_trials=None): + trials = [] + material = self.load_material("study_1_hypothetical_stories") + n = 80 if n_trials is None else n_trials + + for item in material["items"]: + for _ in range(n): + trials.append({ + "sub_study_id": "study_1_hypothetical_stories", + "scenario_id": item["id"], + "scenario": item["id"], + "items": [item], + "profile": {"age": random.randint(18, 22), "gender": random.choice(["Male", "Female"])}, + "variant": self.PROMPT_VARIANT + }) + return trials + + def dump_prompts(self, output_dir): + trials = self.create_trials(n_trials=1) + for idx, trial in enumerate(trials): + prompt = self.prompt_builder.build_trial_prompt(trial) # Use self.prompt_builder + with open(f"{output_dir}/study_001_trial_{idx}.txt", "w") as f: + f.write(prompt) +``` + +**Example 2: Between-Subjects Design (study_002)** +```python +class CustomPromptBuilder(PromptBuilder): + def __init__(self, study_path: Path): + super().__init__(study_path) + + def build_trial_prompt(self, trial_metadata): + # Note: System prompt is now handled separately by SystemPromptRegistry + # This method only builds the task/trial content + items = trial_metadata.get("items", []) + + prompt = "" + + # Add task context introduction + prompt += "You are participating in a psychology study on judgment and decision-making. Please read the following instructions and provide your responses.\n\n" + + q_counter = 1 + + for item in items: + # For anchored studies, assign anchor type at participant level + anchor_type = item.get("assigned_anchor_type") + anchor_val = item.get("metadata", {}).get(f"{anchor_type}_anchor") + + prompt += f"Q{q_counter}.1 (A/B): Is value higher/lower than {anchor_val}?\n" + prompt += f"Q{q_counter}.2 (number): Your estimate?\n" + item["q_idx_choice"] = f"Q{q_counter}.1" + item["q_idx_estimate"] = f"Q{q_counter}.2" + q_counter += 1 + + prompt += f"\nRESPONSE_SPEC: Q1.1=, Q1.2=, Q2.1=, Q2.2=\n" + return prompt + +@StudyConfigRegistry.register("study_002") +class StudyStudy002Config(BaseStudyConfig): + prompt_builder_class = CustomPromptBuilder + PROMPT_VARIANT = "v3" + + def __init__(self, study_path: Path, specification: Dict[str, Any]): + super().__init__(study_path, specification) + + def create_trials(self, n_trials=None): + trials = [] + material = self.load_material("exp_1_anchored_estimation") + spec = self.specification + n = spec["participants"]["by_sub_study"]["exp_1_anchored_estimation"]["n"] + if n == 0: + n = 50 # Default to ensure experiment runs + + for i in range(n): + # Assign anchor type at PARTICIPANT level (all items get same anchor) + assigned_anchor_type = random.choice(["low", "high"]) + assigned_items = [] + for item in material["items"]: + item_copy = item.copy() + item_copy["assigned_anchor_type"] = assigned_anchor_type + assigned_items.append(item_copy) + + # ONE trial per participant with ALL items + trials.append({ + "sub_study_id": "exp_1_anchored_estimation", + "scenario_id": "exp_1_anchored_estimation", + "scenario": "exp_1_anchored_estimation", + "items": assigned_items, + "profile": {"age": random.randint(18, 25), "gender": random.choice(["male", "female"])}, + "variant": self.PROMPT_VARIANT + }) + return trials +``` + +### Your Task +Generate the complete `CustomPromptBuilder` and `StudyConfig` classes following these patterns: +- Use class attributes for `prompt_builder_class` and `PROMPT_VARIANT` +- Call `super().__init__()` in both `__init__` methods +- Create ONE trial per participant with ALL items (unless within-subjects design) +- Use `Qk=` or `Qk.n=` format for responses. Please specify the exact format expected in the RESPONSE_SPEC. +- Include RESPONSE_SPEC with exact format expected +- Use `self.prompt_builder` (not `self.prompt_builder_class()`) in `dump_prompts` +- If `n=0`, use default `n=50` to ensure experiments run + +DO NOT write import statements or the registration decorator - these will be added automatically. + +Generate the code now: +""" + return template.replace("[[STUDY_ID]]", study_id)\ + .replace("[[EXTRACTION_SUMMARY]]", extraction_summary)\ + .replace("[[MATERIAL_CONTEXT]]", material_context) + + def _extract_code_from_response(self, response: str) -> str: + """Extract Python code from LLM response""" + response_text = response.strip() + + if '```python' in response_text: + response_text = response_text.split('```python')[1].split('```')[0].strip() + elif '```' in response_text: + response_text = response_text.split('```')[1].split('```')[0].strip() + + lines = response_text.split('\n') + + start_idx = 0 + for i, line in enumerate(lines): + stripped = line.strip() + if (stripped.startswith('\"\"\"') or + stripped.startswith("'''") or + stripped.startswith('import ') or + stripped.startswith('from ') or + stripped.startswith('class ') or + stripped.startswith('def ') or + stripped.startswith('@')): + start_idx = i + break + + end_idx = len(lines) + for i in range(len(lines) - 1, -1, -1): + stripped = lines[i].strip() + if stripped and not stripped.startswith('#'): + end_idx = i + 1 + break + + code = '\n'.join(lines[start_idx:end_idx]) + + if not code: + print(f"Warning: Extracted code is empty! Raw response length: {len(response)}") + if response.strip(): + code = response.strip() + + return code diff --git a/generation_pipeline/normalize_schema.py b/generation_pipeline/normalize_schema.py new file mode 100644 index 0000000..cc5b1d3 --- /dev/null +++ b/generation_pipeline/normalize_schema.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Normalize historical stage4 JSON schema and write a repair report.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from generation_pipeline.verification.schema_validator import normalize_stage4_tree + + +def main() -> None: + parser = argparse.ArgumentParser(description="Scan stage4/**/*.json with schema_validator") + parser.add_argument("--stage4-dir", type=Path, default=Path("stage4")) + parser.add_argument( + "--report", + type=Path, + default=Path("stage4/schema_normalization_report.json"), + help="Where to write the normalization report", + ) + parser.add_argument("--write", action="store_true", help="Write deterministic repairs back to paper JSONs") + parser.add_argument("--no-backup", action="store_true", help="Skip .bak backups when --write is used") + args = parser.parse_args() + + report = normalize_stage4_tree( + args.stage4_dir, + write=args.write, + backup=not args.no_backup, + report_path=args.report, + ) + print(json.dumps(report["summary"], indent=2, ensure_ascii=False)) + print(f"Report: {args.report}") + raise SystemExit(1 if report["summary"]["invalid"] else 0) + + +if __name__ == "__main__": + main() diff --git a/generation_pipeline/outputs/.gitkeep b/generation_pipeline/outputs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/generation_pipeline/parsers/__init__.py b/generation_pipeline/parsers/__init__.py new file mode 100644 index 0000000..cfd94fc --- /dev/null +++ b/generation_pipeline/parsers/__init__.py @@ -0,0 +1,51 @@ +from generation_pipeline.parsers.qsf_parser import ( + ParsedSurvey, + QsfBlock, + QsfItem, + parse_qsf, + parse_qsf_file, +) +from generation_pipeline.parsers.sav_parser import ( + ParsedDataset, + SavVariable, + parse_sav_file, + parse_sav_meta, +) +from generation_pipeline.parsers.source_linker import ( + LinkedFile, + LinkResult, + link_sources, + route_ext, +) +from generation_pipeline.parsers.material_assembler import assemble_study_materials +from generation_pipeline.parsers.effect_consolidator import ( + annotate_study, + consolidate_effects, +) +from generation_pipeline.parsers.stage3_adapter import ( + load_stage3_studies, + match_stage3_study, + materials_from_stage3, +) + +__all__ = [ + "ParsedSurvey", + "QsfBlock", + "QsfItem", + "parse_qsf", + "parse_qsf_file", + "ParsedDataset", + "SavVariable", + "parse_sav_file", + "parse_sav_meta", + "LinkedFile", + "LinkResult", + "link_sources", + "route_ext", + "assemble_study_materials", + "load_stage3_studies", + "match_stage3_study", + "materials_from_stage3", + "annotate_study", + "consolidate_effects", +] diff --git a/generation_pipeline/parsers/effect_consolidator.py b/generation_pipeline/parsers/effect_consolidator.py new file mode 100644 index 0000000..7b7e9ab --- /dev/null +++ b/generation_pipeline/parsers/effect_consolidator.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Set + +# effecttype -> (analysis_role, is_primary_simulation_target) +_ROLE_MAP: Dict[str, tuple[str, bool]] = { + "main": ("primary_finding", True), + "int": ("interaction", True), + "interaction": ("interaction", True), + "simple": ("simple_effect", False), + "simple_effect": ("simple_effect", False), + "correlation": ("correlation", False), + "mediation": ("mediation", False), + "moderation": ("moderation", True), + "manipulation_check": ("manipulation_check", False), + "check": ("manipulation_check", False), +} + +# tokens that mark a factor LEVEL, not the construct itself -> stripped before +_LEVEL_NOISE = re.compile( + r"\b(\d[\d,\.]*\s*(years?|yrs?|%|percent|days?|months?)?|" + r"standardi[sz]ed|raw|within[- ]subjects?|between[- ]subjects?|stated|" + r"per[- ]trial|omnibus|overall|baseline|post|pre)\b", + re.IGNORECASE, +) +_PARENS = re.compile(r"\([^)]*\)") +_NONWORD = re.compile(r"[^a-z0-9]+") +_AS_ABOVE = re.compile(r"\b(as above|as previous|same as (above|previous)|ditto)\b", re.IGNORECASE) + +_STOP = { + "the", "a", "an", "of", "to", "in", "on", "for", "and", "or", "by", "with", + "that", "this", "is", "are", "be", "vs", "versus", "choose", "choice", + "rating", "scale", "measure", "score", "level", "average", "mean", +} + + +def _norm(text: Any) -> str: + return str(text or "").strip() + + +def _construct_tokens(text: Any, *, drop_parens: bool = False) -> Set[str]: + s = _norm(text).lower() + if drop_parens: + s = _PARENS.sub(" ", s) + s = _LEVEL_NOISE.sub(" ", s) + s = _NONWORD.sub(" ", s) + toks = {t for t in s.split() if t and t not in _STOP and len(t) > 1} + return toks + + +def _table_base(loc: Any) -> str: + s = _norm(loc).lower() + s = _PARENS.sub("", s) + s = re.sub(r"\bp\.?\s*\d+\b", "", s) # drop page refs + s = re.sub(r"[,;].*$", "", s) # drop trailing clauses + return _NONWORD.sub(" ", s).strip() + + +def _jaccard(a: Set[str], b: Set[str]) -> float: + if not a and not b: + return 1.0 + if not a or not b: + return 0.0 + return len(a & b) / len(a | b) + + +def _role_for(effecttype: Any) -> tuple[str, bool]: + return _ROLE_MAP.get(str(effecttype or "").strip().lower(), ("other", False)) + + +def _level_label(effect: Dict[str, Any]) -> str: + """A short label distinguishing this row from its siblings (the moderator).""" + for key in ("IV", "DV", "table_or_page_location"): + m = _LEVEL_NOISE.search(_norm(effect.get(key))) + if m: + return m.group(0).strip() + return _norm(effect.get("table_or_page_location"))[:40] + + +@dataclass +class EffectGroup: + merge_group_id: str + representative_index: int + effect_indices: List[int] = field(default_factory=list) + effecttype: str = "" + analysis_role: str = "other" + is_primary_simulation_target: bool = False + iv: str = "" + dv: str = "" + table: str = "" + moderator_levels: List[str] = field(default_factory=list) + + +def consolidate_effects( + effects: List[Dict[str, Any]], + *, + iv_threshold: float = 0.6, + dv_threshold: float = 0.6, + study_id: str = "study", +) -> List[EffectGroup]: + """Group a study's effect rows into consolidated effects.""" + groups: List[EffectGroup] = [] + prev_idx: Optional[int] = None + + for idx, eff in enumerate(effects): + if not isinstance(eff, dict): + continue + etype = str(eff.get("effecttype") or "").strip().lower() + role, primary = _role_for(etype) + iv = _norm(eff.get("IV")) + dv = _norm(eff.get("DV")) + table = _table_base(eff.get("table_or_page_location")) + + # explicit "as above" duplicate -> attach to the previous group + merged = False + if (_AS_ABOVE.search(dv) or _AS_ABOVE.search(iv)) and prev_idx is not None and groups: + g = next((g for g in groups if prev_idx in g.effect_indices), None) + if g is not None: + g.effect_indices.append(idx) + g.moderator_levels.append(_level_label(eff)) + merged = True + + if not merged: + # Keep parentheticals + iv_tok = _construct_tokens(iv) + dv_tok = _construct_tokens(dv) + for g in groups: + if g.effecttype != etype or g.table != table or not table: + continue + if (_jaccard(_construct_tokens(g.iv), iv_tok) >= iv_threshold + and _jaccard(_construct_tokens(g.dv), dv_tok) >= dv_threshold): + g.effect_indices.append(idx) + g.moderator_levels.append(_level_label(eff)) + merged = True + break + + if not merged: + groups.append(EffectGroup( + merge_group_id=f"{study_id}::g{len(groups) + 1}", + representative_index=idx, + effect_indices=[idx], + effecttype=etype, + analysis_role=role, + is_primary_simulation_target=primary, + iv=iv, dv=dv, table=table, + )) + prev_idx = idx + + # choose representative = member with the most complete reported statistics + for g in groups: + g.representative_index = _pick_representative(effects, g.effect_indices) + g.moderator_levels = [m for m in dict.fromkeys(g.moderator_levels) if m] + return groups + + +def _completeness(effect: Dict[str, Any]) -> int: + stats = effect.get("stats") if isinstance(effect.get("stats"), dict) else {} + return sum(1 for v in stats.values() if v not in (None, "", [None, None])) + + +def _pick_representative(effects: List[Dict[str, Any]], indices: List[int]) -> int: + return max(indices, key=lambda i: _completeness(effects[i])) if indices else (indices or [0])[0] + + +def annotate_study(study: Dict[str, Any], *, study_id: Optional[str] = None) -> Dict[str, Any]: + """Attach `consolidation` to each effect and a `consolidation_summary`. + + Returns the same study dict (mutated in place) for convenience. + """ + effects = [e for e in study.get("effects", []) if isinstance(e, dict)] + sid = study_id or str(study.get("study") or study.get("study_id") or "study") + groups = consolidate_effects(effects, study_id=sid) + + for g in groups: + for i in g.effect_indices: + effects[i]["consolidation"] = { + "merge_group_id": g.merge_group_id, + "representative_effect_index": g.representative_index, + "effect_indices": g.effect_indices, + "is_representative": i == g.representative_index, + "analysis_role": g.analysis_role, + "is_primary_simulation_target": g.is_primary_simulation_target, + "moderator_levels": g.moderator_levels, + } + + primary = [g for g in groups if g.is_primary_simulation_target] + study["consolidation_summary"] = { + "raw_effects": len(effects), + "consolidated_effects": len(groups), + "primary_simulation_targets": len(primary), + "by_role": _role_histogram(groups), + "groups": [ + { + "merge_group_id": g.merge_group_id, + "analysis_role": g.analysis_role, + "is_primary_simulation_target": g.is_primary_simulation_target, + "representative_effect_index": g.representative_index, + "effect_indices": g.effect_indices, + "n_merged": len(g.effect_indices), + } + for g in groups + ], + } + return study + + +def _role_histogram(groups: List[EffectGroup]) -> Dict[str, int]: + out: Dict[str, int] = {} + for g in groups: + out[g.analysis_role] = out.get(g.analysis_role, 0) + 1 + return out diff --git a/generation_pipeline/parsers/material_assembler.py b/generation_pipeline/parsers/material_assembler.py new file mode 100644 index 0000000..d32580b --- /dev/null +++ b/generation_pipeline/parsers/material_assembler.py @@ -0,0 +1,766 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Dict, List, Optional + +from generation_pipeline.parsers.qsf_parser import ParsedSurvey, parse_qsf_file +from generation_pipeline.parsers.sav_parser import ParsedDataset, parse_sav_file +from generation_pipeline.parsers.source_linker import LinkedFile, LinkResult +from generation_pipeline.parsers.stage3_adapter import materials_from_stage3 + +# Blocks / items that are survey plumbing, not the experimental task. +_ADMIN_BLOCK_RE = re.compile( + r"consent|debrief|demograph|prolific|mturk|\battention\b|attn|comprehension|" + r"manipulation check|^instructions?$|practice|captcha|short demos?|timing", + re.IGNORECASE, +) +_ADMIN_ITEM_RE = re.compile( + r"^(?:english|gender|sex|age|race|ethnic|education|income|prolific|mturk|" + r"attention|attn|duration|finished|progress|obj_?side)$", + re.IGNORECASE, +) +_TOKEN_RE = re.compile(r"[a-z][a-z0-9_'-]{2,}", re.IGNORECASE) +_TARGET_STOP_TERMS = { + "the", "and", "for", "with", "that", "this", "from", "into", "about", + "study", "studies", "effect", "effects", "condition", "conditions", + "measure", "measures", "item", "items", "index", "scale", "article", + "message", "messages", "counter", "attitudinal", "dependent", "variable", + "variables", "participant", "participants", "toward", "after", "before", + "interaction", "direct", "indirect", +} + + +def _slug(text: str, fallback: str) -> str: + s = re.sub(r"[^a-z0-9]+", "_", str(text or "").lower()).strip("_") + return s or fallback + + +def _is_admin_block(name: Optional[str]) -> bool: + return bool(name and _ADMIN_BLOCK_RE.search(name)) + + +def _has_truncation_marker(text: str) -> bool: + return bool(text and ("..." in text or "…" in text)) + + +def _scale_key_text(value: Any) -> str: + try: + n = float(value) + except (TypeError, ValueError): + return str(value).strip() + return str(int(n)) if n == int(n) else str(n) + + +def _clean_scale_label(key: Any, label: Any) -> str: + key_text = _scale_key_text(key) + text = re.sub(r"\s+", " ", str(label or "")).strip() + if not text: + return key_text + + leading = re.match(r"^([+-]?\d+(?:\.\d+)?)\s*(.*)$", text) + if leading: + leading_key = _scale_key_text(leading.group(1)) + rest = leading.group(2).strip() + if leading_key == key_text: + rest = re.sub( + r"^(?:[?=:.\-]|\u2010|\u2011|\u2012|\u2013|\u2014|\u2015|\u2212)+\s*", + "", + rest, + ).strip() + return f"{key_text} - {rest}" if rest else key_text + + return f"{key_text} - {text}" if text != key_text else key_text + + +def _normalized_scale_anchors(scale: Optional[Dict[str, Any]]) -> Dict[str, str]: + if not isinstance(scale, dict) or not isinstance(scale.get("anchors"), dict): + return {} + return { + _scale_key_text(key): _clean_scale_label(key, value) + for key, value in scale.get("anchors", {}).items() + } + + +def _scale_sort_key(value: Any) -> tuple[int, float | str]: + try: + return (0, float(value)) + except (TypeError, ValueError): + return (1, str(value)) + + +def _scale_options(scale: Optional[Dict[str, Any]], *, enumerate_discrete: bool) -> List[str]: + if not isinstance(scale, dict): + return [] + anchors = _normalized_scale_anchors(scale) + try: + lo = int(float(scale["min"])) + hi = int(float(scale["max"])) + except (KeyError, TypeError, ValueError): + return [str(v) for _, v in sorted(anchors.items(), key=lambda item: _scale_sort_key(item[0]))] + + if enumerate_discrete and 0 <= hi - lo <= 20: + return [_clean_scale_label(n, anchors.get(str(n), str(n))) for n in range(lo, hi + 1)] + + return [str(v) for _, v in sorted(anchors.items(), key=lambda item: _scale_sort_key(item[0]))] + + +def _response_format(ntype: str, scale: Optional[Dict[str, Any]], options: List[str]) -> Dict[str, Any]: + fmt: Dict[str, Any] = {"answer_type": ntype} + if scale: + fmt.update({ + "scale_min": scale.get("min"), + "scale_max": scale.get("max"), + "anchors": _normalized_scale_anchors(scale), + }) + if ntype == "slider": + fmt["response_mode"] = "numeric_slider" + elif ntype == "scale": + fmt["response_mode"] = "discrete_scale" + if options: + fmt["options"] = options + return fmt + + +def _clean_matrix_text(text: str) -> str: + text = re.sub(r"\s*(?:\.\.\.|…)\s*", " ", text or "") + return re.sub(r"\s+", " ", text).strip() + + +def _token_stem(token: str) -> str: + token = token.lower().strip("'_-") + for suffix in ("iveness", "ation", "ness", "ment", "ity", "ing", "ed", "es", "s"): + if len(token) > len(suffix) + 4 and token.endswith(suffix): + token = token[:-len(suffix)] + break + return token[:6] if len(token) > 6 else token + + +def _target_stems(*values: Any) -> set[str]: + stems: set[str] = set() + for value in values: + for token in _TOKEN_RE.findall(str(value or "").lower()): + token = token.strip("'_-") + if len(token) < 4 or token in _TARGET_STOP_TERMS or token.isdigit(): + continue + stems.add(_token_stem(token)) + return stems + + +def _looks_instruction_like(name: str, label: str) -> bool: + if re.search(r"(?:^|_)(instruction|intro)(?:_|$)", name or "", flags=re.IGNORECASE): + return True + return bool(re.match(r"^\s*(now|thank you|to minimize)\b", label or "", flags=re.IGNORECASE)) + +# QSF -> materials + +def _expand_question(it: Any) -> str: + stem = it.text + if it.rows: + clean_stem = _clean_matrix_text(stem) + trials = " ".join( + f"{i}. {_clean_matrix_text(row)}" + for i, row in enumerate(it.rows, start=1) + ) + return f"{clean_stem} {trials}".strip() if clean_stem else trials + return stem + + +def _qsf_condition_candidates(survey: ParsedSurvey) -> List[Dict[str, Any]]: + conditions: List[Dict[str, Any]] = [dict(c) for c in getattr(survey, "flow_conditions", [])] + names = {c.get("name") for c in conditions} + + vignette_groups: set[str] = set() + vignette_numbers: set[str] = set() + for block in survey.blocks: + if block.is_trash or _is_admin_block(block.description): + continue + match = re.match(r"\s*([A-Za-z][\w-]*)\s+Vignette\s+(\d+)", block.description or "") + if match: + vignette_groups.add(match.group(1)) + vignette_numbers.add(match.group(2)) + + if len(vignette_groups) > 1 and "qsf_vignette_group" not in names: + conditions.append({ + "name": "qsf_vignette_group", + "levels": sorted(vignette_groups), + "source": "qsf_block_description", + "review_required": True, + }) + if len(vignette_numbers) > 1 and "qsf_vignette" not in names: + conditions.append({ + "name": "qsf_vignette", + "levels": sorted(vignette_numbers, key=lambda item: int(item)), + "source": "qsf_block_description", + "review_required": True, + }) + return conditions + + +def _from_qsf(survey: ParsedSurvey, source_file: str) -> Dict[str, Any]: + items: List[Dict[str, Any]] = [] + instructions: List[str] = [] + instruction_blocks: List[Dict[str, Any]] = [] + + for block in survey.blocks: + if block.is_trash or _is_admin_block(block.description): + continue + for it in block.items: + # stimulus / framing text -> instructions + if not it.is_response_item(): + if len(it.text) >= 60: + instructions.append(it.text) + instruction_blocks.append( + { + "block": it.block or block.description, + "qid": it.qid, + "data_export_tag": it.data_export_tag, + "text": it.text, + "source": "osf_qsf", + "source_file": source_file, + } + ) + continue + tag = it.data_export_tag or it.qid + question = _expand_question(it) + options = list(it.choices) + if it.scale and it.type in ("scale", "matrix"): + options = options or _scale_options(it.scale, enumerate_discrete=True) + elif it.scale and it.type == "slider": + options = options or _scale_options(it.scale, enumerate_discrete=False) + item: Dict[str, Any] = { + "id": _slug(tag, it.qid.lower()), + "question": question, + "options": options, + "type": it.type, + "source": "osf_qsf", + "source_file": source_file, + "data_export_tag": it.data_export_tag, + "block": it.block, + } + if it.scale: + item["scale"] = it.scale + item["response_format"] = _response_format(it.type, it.scale, options) + items.append(item) + + return { + "instructions": "\n\n".join(instructions[:8]), + "instruction_blocks": instruction_blocks, + "items": items, + "conditions": _qsf_condition_candidates(survey), + "response_schema": _response_schema(items), + } + + +# SAV -> materials + +def _from_sav(dataset: ParsedDataset, source_file: str) -> Dict[str, Any]: + items: List[Dict[str, Any]] = [] + skipped: List[str] = [] + for v in dataset.response_items(): + if _looks_instruction_like(v.name, v.label): + skipped.append(v.name) + continue + opts = v.choices + if not opts and v.scale: + opts = _scale_options(v.scale, enumerate_discrete=True) + item = { + "id": _slug(v.name, v.name), + "question": v.label, + "options": opts, + "type": v.type, + "source": "osf_sav", + "source_file": source_file, + "data_export_tag": v.name, + **({"scale": v.scale} if v.scale else {}), + } + item["response_format"] = _response_format(v.type, v.scale, opts) + if _has_truncation_marker(v.label): + item.setdefault("quality_flags", []).append("label_contains_ellipsis") + items.append(item) + + conditions = [ + {"name": v.name, "label": v.label or v.name, "levels": v.choices} + for v in dataset.condition_variables() + ] + return { + "instructions": "", # .sav carries no stimulus text (provides_stimulus=False) + "instruction_blocks": [], + "items": items, + "conditions": conditions, + "response_schema": _response_schema(items), + "skipped_items": skipped, + } + + +# Shared + +def _response_schema(items: List[Dict[str, Any]]) -> Dict[str, Any]: + """Infer a dominant response schema from the assembled items.""" + if not items: + return {} + from collections import Counter + dominant = Counter(it["type"] for it in items).most_common(1)[0][0] + rep = next((it for it in items if it["type"] == dominant), items[0]) + schema: Dict[str, Any] = {"answer_type": dominant} + if rep.get("scale"): + schema["scale_min"] = rep["scale"]["min"] + schema["scale_max"] = rep["scale"]["max"] + schema["anchors"] = _normalized_scale_anchors(rep["scale"]) + if rep.get("options"): + schema["options"] = rep["options"] + return schema + + +def _primary_effects(stage3_study: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]: + if not isinstance(stage3_study, dict): + return [] + effects = [effect for effect in stage3_study.get("effects", []) if isinstance(effect, dict)] + primary = [] + for effect in effects: + cons = effect.get("consolidation") + if isinstance(cons, dict): + if cons.get("is_primary_simulation_target", True) and cons.get("is_representative", True) is not False: + primary.append(effect) + return primary or effects + + +def _focal_item_stems(stage3_study: Optional[Dict[str, Any]]) -> set[str]: + effects = _primary_effects(stage3_study) + stems: set[str] = set() + for effect in effects: + stems |= _target_stems(effect.get("DV")) + return stems + + +def _item_match_score(item: Dict[str, Any], stems: set[str]) -> int: + if not stems: + return 0 + haystack = " ".join( + str(item.get(key) or "") + for key in ("id", "data_export_tag", "question") + ) + item_stems = _target_stems(haystack) + score = len(item_stems & stems) + # Short analysis tags often use an abbreviated construct prefix + # (e.g. open_1 for openness, recep1 for receptiveness). + tag = str(item.get("data_export_tag") or item.get("id") or "").lower() + tag_parts = [_token_stem(part) for part in re.split(r"[^a-z0-9]+", tag) if part] + for stem in stems: + if any(part and (stem.startswith(part) or part.startswith(stem[:4])) for part in tag_parts): + score += 2 + return score + + +def _annotate_focal_item_candidates( + items: List[Dict[str, Any]], + stage3_study: Optional[Dict[str, Any]], + warnings: List[str], + source_trace: Dict[str, Any], +) -> List[Dict[str, Any]]: + if len(items) < 8: + return items + stems = _focal_item_stems(stage3_study) + if not stems: + return items + scored = [(item, _item_match_score(item, stems)) for item in items] + selected = [ + item for item, score in scored + if score > 0 and not _ADMIN_ITEM_RE.match(str(item.get("id") or item.get("data_export_tag") or "")) + ] + if len(selected) < 2 or len(selected) == len(items): + return items + warnings.append( + f"Identified {len(selected)}/{len(items)} likely focal response items; full source instrument retained." + ) + source_trace["focal_item_candidates"] = { + "mode": "focal_outcome_terms_non_destructive", + "target_stems": sorted(stems), + "candidate_items": [item.get("data_export_tag") or item.get("id") for item in selected], + "instrument_items_retained": len(items), + } + return items + + +def _stage3_item_texts(stage3_study: Optional[Dict[str, Any]]) -> List[str]: + texts: List[str] = [] + for effect in _primary_effects(stage3_study): + slot = effect.get("items") + content = slot.get("content") if isinstance(slot, dict) else slot + if content: + texts.append(str(content)) + return texts + + +def _extract_question_phrases(text: str) -> List[str]: + """Pull participant-facing item phrases out of prose item summaries.""" + text = _clean_matrix_text(text) + if not text: + return [] + candidates: List[str] = [] + for sentence in re.split(r"(?<=[.!?])\s+", text): + if "used to measure" in sentence.lower() and "all participants rated" not in sentence.lower(): + continue + if not re.search(r"\b(how|what extent|rate|rated|likely|appreciat|recogniz|share|want)\b", sentence, re.I): + continue + if re.search(r"\ball participants rated\b", sentence, flags=re.I): + sentence = re.sub(r"^.*?\ball participants rated\b\s+", "", sentence, flags=re.I) + elif re.search(r"\b(?:please\s+)?rate\b", sentence, flags=re.I): + sentence = re.sub(r"^.*?\b(?:please\s+)?rate\b\s+", "", sentence, flags=re.I) + elif not re.match(r"\s*(?:how|to what extent|what)\b", sentence, flags=re.I): + continue + parts = re.split(r",\s+|\s+and\s+", sentence) + for part in parts: + part = part.strip(" ;.") + part = re.sub(r"^(?:and|or)\s+", "", part, flags=re.IGNORECASE) + if not re.search(r"\b(how|what extent|likely|appreciat|recogniz|share|want)\b", part, re.I): + continue + if len(part) < 18: + continue + part = _participantize_question(part) + if not part.endswith("?"): + part += "?" + candidates.append(part[:1].upper() + part[1:]) + deduped: List[str] = [] + seen = set() + for item in candidates: + key = re.sub(r"\s+", " ", item.lower()) + if key not in seen: + seen.add(key) + deduped.append(item) + return deduped + + +def _participantize_question(text: str) -> str: + """Convert prose summaries of participant items back into direct questions.""" + text = re.sub(r"\b[hH]ow likely they were to\b", "how likely are you to", text) + text = re.sub(r"\b[hH]ow much they appreciated\b", "how much do you appreciate", text) + text = re.sub(r"\b[hH]ow much they recognized\b", "how much do you recognize", text) + text = re.sub(r"\bthey just read\b", "you just read", text, flags=re.IGNORECASE) + text = re.sub(r"\bthey read\b", "you read", text, flags=re.IGNORECASE) + text = re.sub(r"\btheir\b", "your", text, flags=re.IGNORECASE) + return text + + +def _item_numeric_group_key(item: Dict[str, Any]) -> tuple[str, Optional[int]]: + tag = str(item.get("data_export_tag") or item.get("id") or "") + match = re.match(r"^([A-Za-z_]+?)[_-]?(\d+)$", tag) + if not match: + return tag.lower(), None + return match.group(1).lower().rstrip("_"), int(match.group(2)) + + +def _record_truncated_item_patch_hints( + items: List[Dict[str, Any]], + stage3_study: Optional[Dict[str, Any]], + warnings: List[str], + source_trace: Dict[str, Any], +) -> List[Dict[str, Any]]: + if not any(_has_truncation_marker(item.get("question", "")) for item in items): + return items + phrases: List[str] = [] + for text in _stage3_item_texts(stage3_study): + phrases.extend(_extract_question_phrases(text)) + if not phrases: + return items + + groups: Dict[str, List[tuple[int, Optional[int], Dict[str, Any]]]] = {} + for idx, item in enumerate(items): + key, number = _item_numeric_group_key(item) + groups.setdefault(key, []).append((idx, number, item)) + + hints: List[Dict[str, str]] = [] + for _, group in groups.items(): + numbered = [(idx, number, item) for idx, number, item in group if number is not None] + if len(numbered) < 2 or len(phrases) < len(numbered): + continue + numbered.sort(key=lambda entry: entry[1] or 0) + group_questions = [entry[2].get("question", "") for entry in numbered] + if not any(_has_truncation_marker(question) for question in group_questions): + continue + for idx, number, item in numbered: + old = str(item.get("question") or "") + if not _has_truncation_marker(old): + continue + phrase_index = (number or 1) - 1 + if phrase_index < 0 or phrase_index >= len(phrases): + continue + new = phrases[phrase_index] + hints.append({ + "item": str(item.get("data_export_tag") or item.get("id")), + "old": old.replace("...", "[truncated]").replace("…", "[truncated]"), + "suggested_question": new, + "suggested_source": "stage3_effect_item_slot", + }) + break + + if hints: + warnings.append( + f"Recorded {len(hints)} Stage 3 item patch hint(s); SAV codebook labels were left unchanged." + ) + source_trace["item_patch_hints"] = hints + return items + + +def _norm_condition_label(value: str) -> str: + value = value.lower() + value = value.replace("two-sided", "twosided").replace("one-sided", "onesided") + value = value.replace("two-side", "twosided").replace("one-side", "onesided") + value = re.sub(r"\b(condition|conditions|strong|weak|high|low)\b", " ", value) + return re.sub(r"[^a-z0-9]+", "", value) + + +def _record_stage3_condition_patch_hints( + conditions: List[Dict[str, Any]], + s3: Optional[Dict[str, Any]], + warnings: List[str], + source_trace: Dict[str, Any], +) -> List[Dict[str, Any]]: + if not s3 or not s3.get("conditions"): + return conditions + stage3_conditions = s3.get("conditions") or [] + stage3_levels: Dict[str, str] = {} + for cond in stage3_conditions: + descriptions = cond.get("level_descriptions") if isinstance(cond, dict) else None + if isinstance(descriptions, dict): + for level, desc in descriptions.items(): + stage3_levels[_norm_condition_label(str(level))] = str(desc) + if not stage3_levels: + return conditions + hints: List[Dict[str, Any]] = [] + for cond in conditions: + level_desc: Dict[str, str] = {} + for level in cond.get("levels", []): + norm = _norm_condition_label(str(level)) + match = next( + ( + desc for key, desc in stage3_levels.items() + if norm in key or key in norm or bool(set(re.findall(r"[a-z]+", norm)) & set(re.findall(r"[a-z]+", key))) + ), + None, + ) + if match: + level_desc[str(level)] = match + if level_desc: + hints.append({ + "condition": cond.get("name") or cond.get("label"), + "suggested_level_descriptions": level_desc, + "suggested_source": "stage3_condition_slot", + }) + if hints: + warnings.append( + f"Recorded {len(hints)} Stage 3 condition patch hint(s); source conditions were left unchanged." + ) + source_trace["condition_patch_hints"] = hints + return conditions + + +def assemble_study_materials( + study_id: str, + link_result: LinkResult, + *, + repo_root: Optional[Path] = None, + stage3_study: Optional[Dict[str, Any]] = None, + allow_pdf_slot_fallback: bool = False, +) -> Dict[str, Any]: + """Assemble one materials dict for `study_id` from its linked sources. + + Source priority: + 1. OSF .qsf instrument - verbatim participant-facing survey. + 2. OSF .sav codebook - items + conditions; stimulus can be + filled from legacy Stage 3 slots only + when `allow_pdf_slot_fallback=True`. + 3. Stage 3 PDF-extracted slots - optional legacy fallback. The new + pipeline keeps incomplete packages as + `ready=false` instead of treating + per-effect slots as final materials. + """ + sub_id = _slug(study_id, "study") + if not sub_id.startswith("study") and not sub_id.startswith("pilot"): + sub_id = f"study_{sub_id}" + + mats = link_result.material_files(study_id) + base = { + "sub_study_id": sub_id, + "instructions": "", + "instruction_blocks": [], + "items": [], + "conditions": [], + "response_schema": {}, + "readiness": {"ready": False, "blocking_issues": [], "warnings": []}, + "source_trace": {"primary_source": None, "source_file": None, "candidates": [ + {"kind": f.kind, "file": _relpath(f.path, repo_root)} for f in mats + ]}, + } + s3 = materials_from_stage3(stage3_study) if stage3_study else None + + primary = _pick_primary(mats) + + # --- 3. no OSF instrument: keep a draft unless legacy fallback is explicit + if primary is None: + if allow_pdf_slot_fallback and s3 and s3["items"]: + base.update({k: s3[k] for k in ("instructions", "items", "conditions")}) + base["response_schema"] = _response_schema(s3["items"]) + base["source_trace"]["primary_source"] = "pdf_stage3" + base["source_trace"]["source_file"] = "stage3.json" + base["source_trace"]["slot_summary"] = s3["slot_summary"] + base["readiness"]["ready"] = True + base["readiness"]["warnings"].append( + "No OSF instrument; materials taken from Stage 3 PDF slots " + f"(status: {s3['slot_summary']}). Review verbatim-ness." + ) + _apply_quality_gate(base, primary_kind="pdf_stage3") + base["readiness"]["ready"] = bool(base["items"]) and not base["readiness"]["blocking_issues"] + else: + base["readiness"]["blocking_issues"].append("no_structured_material_source") + base["readiness"]["warnings"].append( + "No structured OSF/supplement instrument was linked. Legacy " + "per-effect slots were not used as final materials; recover the " + "participant-facing stimulus and response items from the PDF or " + "supplement in the Stage 4 draft." + ) + if s3: + base["source_trace"]["legacy_slot_summary"] = s3["slot_summary"] + return base + + # --- 1/2. OSF instrument present ----------------------------------------- + src_rel = _relpath(primary.path, repo_root) + try: + if primary.kind == "qsf": + built = _from_qsf(parse_qsf_file(primary.path), src_rel) + else: # sav + built = _from_sav(parse_sav_file(primary.path), src_rel) + except Exception as exc: # pragma: no cover - parser robustness guard + base["readiness"]["blocking_issues"].append(f"parse_error:{type(exc).__name__}") + return base + + base.update({k: built[k] for k in ("instructions", "items", "conditions", "response_schema")}) + if "instruction_blocks" in built: + base["instruction_blocks"] = built["instruction_blocks"] + base["source_trace"]["primary_source"] = f"osf_{primary.kind}" + base["source_trace"]["source_file"] = src_rel + if built.get("skipped_items"): + base["readiness"]["warnings"].append( + "Skipped non-response SAV fields: " + ", ".join(built["skipped_items"]) + ) + + # --- 2. enrich .sav (no stimulus) with Stage 3 PDF material -------------- + if primary.kind == "sav" and not base["instructions"]: + if allow_pdf_slot_fallback and s3 and s3["instructions"]: + base["instructions"] = s3["instructions"] + base["source_trace"]["stimulus_source"] = "pdf_stage3" + base["source_trace"]["slot_summary"] = s3["slot_summary"] + base["readiness"]["warnings"].append( + "Items from .sav codebook; stimulus filled from Stage 3 PDF " + f"slots (status: {s3['slot_summary'].get('materials_status')})." + ) + else: + if s3: + base["source_trace"]["legacy_slot_summary"] = s3["slot_summary"] + base["readiness"]["warnings"].append( + "Items recovered from .sav codebook; stimulus text not available " + "from data file. Legacy per-effect slots were not used as final " + "stimulus; complete from PDF/supplement." + ) + + base["items"] = _annotate_focal_item_candidates( + base["items"], + stage3_study, + base["readiness"]["warnings"], + base["source_trace"], + ) + if primary.kind == "sav": + base["items"] = _record_truncated_item_patch_hints( + base["items"], + stage3_study, + base["readiness"]["warnings"], + base["source_trace"], + ) + base["conditions"] = _record_stage3_condition_patch_hints( + base["conditions"], + s3, + base["readiness"]["warnings"], + base["source_trace"], + ) + base["response_schema"] = _response_schema(base["items"]) + + n = len(base["items"]) + if n == 0: + base["readiness"]["blocking_issues"].append("no_participant_items") + _apply_quality_gate(base, primary_kind=primary.kind) + base["readiness"]["ready"] = n > 0 and not base["readiness"]["blocking_issues"] + return base + + +def _apply_quality_gate(material: Dict[str, Any], *, primary_kind: str) -> None: + issues = material["readiness"]["blocking_issues"] + warnings = material["readiness"]["warnings"] + + if primary_kind == "sav": + truncated = [ + item.get("data_export_tag") or item.get("id") + for item in material.get("items", []) + if _has_truncation_marker(item.get("question", "")) + ] + if truncated: + issues.append("truncated_sav_labels") + warnings.append( + "SAV variable labels contain ellipses/truncation markers: " + + ", ".join(str(tag) for tag in truncated[:12]) + + (" ..." if len(truncated) > 12 else "") + ) + + trace = material.get("source_trace", {}) if isinstance(material.get("source_trace"), dict) else {} + stimulus_source = trace.get("stimulus_source") + has_instructions = bool(str(material.get("instructions") or "").strip()) + if not has_instructions or not stimulus_source: + issues.append("stimulus_not_verbatim") + warnings.append( + "SAV files do not contain participant-facing stimulus text; " + "complete stimulus from a participant-facing PDF/supplement source." + ) + + if primary_kind == "qsf": + ellipsis = [ + item.get("data_export_tag") or item.get("id") + for item in material.get("items", []) + if _has_truncation_marker(item.get("question", "")) + ] + if ellipsis: + warnings.append( + "QSF item text still contains ellipsis markers; verify matrix expansion: " + + ", ".join(str(tag) for tag in ellipsis[:12]) + + (" ..." if len(ellipsis) > 12 else "") + ) + + if primary_kind == "pdf_stage3": + cited = [ + item.get("data_export_tag") or item.get("id") + for item in material.get("items", []) + if str(item.get("slot_status") or "").lower() == "cited_scale" + ] + if cited: + warnings.append( + "PDF Stage 3 items include cited/non-verbatim scales: " + + ", ".join(str(tag) for tag in cited[:12]) + + (" ..." if len(cited) > 12 else "") + ) + if cited and len(cited) == len(material.get("items", [])): + issues.append("pdf_items_not_verbatim") + + +def _pick_primary(mats: List[LinkedFile]) -> Optional[LinkedFile]: + for kind in ("qsf", "sav"): + for f in mats: + if f.kind == kind: + return f + return None + + +def _relpath(path: Path, repo_root: Optional[Path]) -> str: + if repo_root: + try: + return str(Path(path).resolve().relative_to(Path(repo_root).resolve())) + except ValueError: + pass + return str(path) diff --git a/generation_pipeline/parsers/qsf_parser.py b/generation_pipeline/parsers/qsf_parser.py new file mode 100644 index 0000000..9ebef84 --- /dev/null +++ b/generation_pipeline/parsers/qsf_parser.py @@ -0,0 +1,430 @@ +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field, asdict +from html import unescape +from pathlib import Path +from typing import Any, Dict, List, Optional + +_TYPE_MAP: Dict[str, str] = { + "MC": "multiple_choice", # multiple choice (single/multi answer) + "Matrix": "matrix", # grid of rows x scale + "Slider": "slider", + "TE": "text_entry", + "DB": "descriptive_text", # display-only block of text (no response) + "Timing": "meta", # page timers + "Captcha": "meta", + "Meta": "meta", + "RO": "rank_order", + "CS": "constant_sum", + "DD": "drill_down", + "PGR": "pick_group_rank", +} + +NON_RESPONSE_TYPES = {"descriptive_text", "meta"} + + +@dataclass +class QsfItem: + qid: str + data_export_tag: Optional[str] + block: Optional[str] + qualtrics_type: str + type: str + text: str + choices: List[str] = field(default_factory=list) + rows: List[str] = field(default_factory=list) + scale: Optional[Dict[str, Any]] = None + + def is_response_item(self) -> bool: + return self.type not in NON_RESPONSE_TYPES + + +@dataclass +class QsfBlock: + id: str + description: str + is_trash: bool + items: List[QsfItem] = field(default_factory=list) + + +@dataclass +class ParsedSurvey: + survey_name: str + blocks: List[QsfBlock] = field(default_factory=list) + flow_conditions: List[Dict[str, Any]] = field(default_factory=list) + + @property + def items(self) -> List[QsfItem]: + out: List[QsfItem] = [] + for block in self.blocks: + out.extend(block.items) + return out + + def response_items(self, *, include_trash: bool = False) -> List[QsfItem]: + """Participant-answerable items, trash blocks excluded by default.""" + out: List[QsfItem] = [] + for block in self.blocks: + if block.is_trash and not include_trash: + continue + out.extend(it for it in block.items if it.is_response_item()) + return out + + def to_dict(self) -> Dict[str, Any]: + return { + "survey_name": self.survey_name, + "flow_conditions": self.flow_conditions, + "blocks": [ + { + "id": b.id, + "description": b.description, + "is_trash": b.is_trash, + "items": [asdict(it) for it in b.items], + } + for b in self.blocks + ], + "items": [asdict(it) for it in self.items], + } + +# Text normalization + +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"[ \t ]+") + + +def clean_html(raw: Any) -> str: + """Strip Qualtrics rich-text HTML to readable plain text. + + Block tags become line breaks so multi-paragraph prompts stay legible; + inline tags are removed; entities are unescaped; whitespace collapsed. + """ + if raw is None: + return "" + text = str(raw) + # turn block-level boundaries into newlines before stripping tags + text = re.sub(r"(?i)<\s*br\s*/?\s*>", "\n", text) + text = re.sub(r"(?i)", "\n", text) + text = _TAG_RE.sub("", text) + text = unescape(text) + # collapse intra-line whitespace, trim each line, drop empties + lines = [_WS_RE.sub(" ", ln).strip() for ln in text.split("\n")] + lines = [ln for ln in lines if ln] + return "\n".join(lines) + + +def _choice_displays(choices: Any) -> List[str]: + """Qualtrics Choices/Answers dicts -> ordered list of display strings. + + Keys are numeric strings ("1","2","10"); preserve numeric order, not the + dict's insertion/lexical order. + """ + if not isinstance(choices, dict): + return [] + def _key(k: str) -> Any: + try: + return (0, int(k)) + except (TypeError, ValueError): + return (1, str(k)) + out: List[str] = [] + for k in sorted(choices.keys(), key=_key): + v = choices[k] + disp = v.get("Display") if isinstance(v, dict) else v + out.append(clean_html(disp)) + return out + + +def _scale_from_answers(answers: Any) -> Optional[Dict[str, Any]]: + """Likert/matrix scale points -> {min, max, anchors}. + + Qualtrics blanks intermediate anchors with " "; keep only the points + that carry a real label as the anchors map. + """ + if not isinstance(answers, dict): + return None + numeric = [] + anchors: Dict[str, str] = {} + for k, v in answers.items(): + disp = clean_html(v.get("Display") if isinstance(v, dict) else v) + try: + n = int(k) + except (TypeError, ValueError): + continue + numeric.append(n) + if disp: + anchors[str(n)] = disp + if not numeric: + return None + return {"min": min(numeric), "max": max(numeric), "anchors": anchors} + + +def _slider_scale(payload: Dict[str, Any]) -> Optional[Dict[str, Any]]: + cfg = payload.get("Configuration") if isinstance(payload.get("Configuration"), dict) else {} + lo, hi = cfg.get("CSSliderMin"), cfg.get("CSSliderMax") + try: + lo = int(lo) + hi = int(hi) + except (TypeError, ValueError): + # fall back to label-based scale if the config range is absent + return _scale_from_answers(payload.get("Labels")) or _scale_from_answers( + payload.get("Answers") + ) + anchors: Dict[str, str] = {} + labels = payload.get("Labels") + if isinstance(labels, dict): + displays = [] + def label_key(item: Any) -> tuple[int, Any]: + return (0, int(item)) if str(item).isdigit() else (1, str(item)) + + for k in sorted(labels.keys(), key=label_key): + v = labels[k] + disp = clean_html(v.get("Display") if isinstance(v, dict) else v) + if disp: + displays.append(disp) + anchors = _slider_anchors(displays, lo, hi) + return {"min": lo, "max": hi, "anchors": anchors} + + +def _slider_anchors(labels: List[str], lo: int, hi: int) -> Dict[str, str]: + if not labels: + return {} + explicit: Dict[str, str] = {} + for label in labels: + match = re.match(r"^\s*(-?\d+(?:\.\d+)?)\s*(?:[-:–—]\s*)?(.*)$", label, flags=re.S) + if not match: + explicit = {} + break + value = float(match.group(1)) + if value < lo or value > hi: + explicit = {} + break + key = str(int(value)) if value.is_integer() else str(value) + explicit[key] = label + if explicit and len(explicit) == len(labels): + return explicit + + if len(labels) == 1: + return {str(lo): labels[0]} + span = hi - lo + anchors: Dict[str, str] = {} + for idx, label in enumerate(labels): + value = lo + (span * idx / (len(labels) - 1)) + key = str(int(round(value))) if abs(value - round(value)) < 1e-9 else str(value) + anchors[key] = label + return anchors + + +def _extract_flow_conditions(flow_payload: Any) -> List[Dict[str, Any]]: + """Recover simple embedded-data condition factors from Qualtrics Survey Flow.""" + values: Dict[str, set[str]] = {} + + def visit(node: Any) -> None: + if isinstance(node, list): + for item in node: + visit(item) + return + if not isinstance(node, dict): + return + if node.get("Type") == "EmbeddedData": + for entry in node.get("EmbeddedData") or []: + if not isinstance(entry, dict): + continue + field = entry.get("Field") or entry.get("Description") + if not field: + continue + values.setdefault(str(field), set()).add(str(entry.get("Value", ""))) + for key in ("Flow", "Payload"): + if key in node: + visit(node[key]) + + visit(flow_payload) + out = [] + for field, levels in sorted(values.items()): + clean_levels = sorted(level for level in levels if level != "") + if len(clean_levels) > 1: + out.append({ + "name": field, + "levels": clean_levels, + "source": "qsf_flow_embedded_data", + }) + return out + + +# Core parse + +def _normalized_type(qtype: str, selector: str) -> str: + base = _TYPE_MAP.get(qtype, "other") + # Bipolar matrices are really a forced A/B choice, not a rating grid. + if qtype == "Matrix" and selector == "Bipolar": + return "multiple_choice" + return base + + +def _bipolar_choices(payload: Dict[str, Any]) -> List[str]: + """Bipolar matrix choices encode 'A...:B...' poles; reduce to ['A','B'].""" + rows = _choice_displays(payload.get("Choices")) + for row in rows: + if ":" in row: + left, right = row.split(":", 1) + la = left.strip()[:1].upper() + rb = right.strip()[:1].upper() + if la and rb: + return [la, rb] + return ["A", "B"] if rows else [] + + +def _build_item(payload: Dict[str, Any], qid: str, block_name: Optional[str]) -> QsfItem: + qtype = str(payload.get("QuestionType") or "") + selector = str(payload.get("Selector") or "") + norm = _normalized_type(qtype, selector) + qualtrics_type = f"{qtype}/{selector}" if selector else qtype + + text = clean_html(payload.get("QuestionText")) + choices: List[str] = [] + rows: List[str] = [] + scale: Optional[Dict[str, Any]] = None + + if qtype == "Matrix" and selector == "Bipolar": + choices = _bipolar_choices(payload) + rows = _choice_displays(payload.get("Choices")) # the per-trial rows + elif qtype == "Matrix": + rows = _choice_displays(payload.get("Choices")) # statement rows + scale = _scale_from_answers(payload.get("Answers")) + choices = [a for a in (scale or {}).get("anchors", {}).values()] if scale else [] + elif qtype == "MC": + choices = _choice_displays(payload.get("Choices")) + elif qtype == "Slider": + scale = _slider_scale(payload) + rows = _choice_displays(payload.get("Choices")) # statement(s) being rated + + return QsfItem( + qid=qid, + data_export_tag=payload.get("DataExportTag") or None, + block=block_name, + qualtrics_type=qualtrics_type, + type=norm, + text=text, + choices=[c for c in choices if c], + rows=[r for r in rows if r], + scale=scale, + ) + + +def _iter_blocks(payload: Any) -> List[Dict[str, Any]]: + if isinstance(payload, list): + return [b for b in payload if isinstance(b, dict)] + if isinstance(payload, dict): + return [b for b in payload.values() if isinstance(b, dict)] + return [] + + +def parse_qsf(data: Dict[str, Any]) -> ParsedSurvey: + """Parse a loaded .qsf dict into a `ParsedSurvey`.""" + elements = data.get("SurveyElements", []) + survey_name = "" + entry = data.get("SurveyEntry") + if isinstance(entry, dict): + survey_name = clean_html(entry.get("SurveyName") or "") + + # 1) index every question payload by QID + questions: Dict[str, Dict[str, Any]] = {} + for el in elements: + if el.get("Element") == "SQ": + qid = el.get("PrimaryAttribute") or (el.get("Payload") or {}).get("QuestionID") + payload = el.get("Payload") + if qid and isinstance(payload, dict): + questions[qid] = payload + + # 2) find the block element (ordering + grouping) + block_defs: List[Dict[str, Any]] = [] + for el in elements: + if el.get("Element") == "BL": + block_defs = _iter_blocks(el.get("Payload")) + break + + blocks: List[QsfBlock] = [] + seen: set[str] = set() + + for bdef in block_defs: + desc = clean_html(bdef.get("Description") or "") or "Block" + is_trash = str(bdef.get("Type") or "").lower() == "trash" or "trash" in desc.lower() + block_id = str(bdef.get("ID") or desc) + block = QsfBlock(id=block_id, description=desc, is_trash=is_trash) + for be in bdef.get("BlockElements", []) or []: + if be.get("Type") != "Question": + continue + qid = be.get("QuestionID") + payload = questions.get(qid) + if not payload: + continue + block.items.append(_build_item(payload, qid, desc)) + seen.add(qid) + blocks.append(block) + + # 3) any question not referenced by a block (rare) -> "Unblocked" + orphans = [qid for qid in questions if qid not in seen] + if orphans: + block = QsfBlock(id="__unblocked__", description="Unblocked", is_trash=False) + for qid in orphans: + block.items.append(_build_item(questions[qid], qid, None)) + blocks.append(block) + + flow_conditions: List[Dict[str, Any]] = [] + for el in elements: + if el.get("Element") == "FL": + flow_conditions = _extract_flow_conditions(el.get("Payload")) + break + + return ParsedSurvey(survey_name=survey_name, blocks=blocks, flow_conditions=flow_conditions) + + +def parse_qsf_file(path: str | Path) -> ParsedSurvey: + """Load and parse a .qsf file from disk.""" + p = Path(path) + data = json.loads(p.read_text(encoding="utf-8")) + return parse_qsf(data) + +# CLI + +def _main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser(description="Parse a Qualtrics .qsf survey file.") + ap.add_argument("qsf", help="Path to the .qsf file") + ap.add_argument("--json", action="store_true", help="Emit full parsed JSON") + ap.add_argument("--all", action="store_true", help="Include trash/meta items") + args = ap.parse_args(argv) + + survey = parse_qsf_file(args.qsf) + + if args.json: + out = survey.to_dict() + if not args.all: + out["blocks"] = [b for b in out["blocks"] if not b["is_trash"]] + print(json.dumps(out, ensure_ascii=False, indent=2)) + return 0 + + print(f"Survey: {survey.survey_name or '(unnamed)'}") + for block in survey.blocks: + if block.is_trash and not args.all: + continue + marker = " [trash]" if block.is_trash else "" + print(f"\n## Block: {block.description}{marker} ({len(block.items)} items)") + for it in block.items: + if not args.all and not it.is_response_item(): + continue + tag = it.data_export_tag or it.qid + preview = it.text.replace("\n", " ") + if len(preview) > 90: + preview = preview[:90] + "…" + extra = "" + if it.choices: + extra = f" choices={it.choices[:4]}" + elif it.scale: + extra = f" scale={it.scale['min']}-{it.scale['max']}" + print(f" - [{tag}] ({it.type}) {preview}{extra}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/generation_pipeline/parsers/sav_parser.py b/generation_pipeline/parsers/sav_parser.py new file mode 100644 index 0000000..0f9385e --- /dev/null +++ b/generation_pipeline/parsers/sav_parser.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +import argparse +import json +import re +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + + +# Variables that are survey plumbing / identifiers / quality flags, not measures. +_META_NAME_RE = re.compile( + r"^(id|.*_id|.*id\d*|prolific.*|mturk.*|worker.*|ip(addr)?|" + r"duration.*|finished|progress|status|recorded.*|response.*|" + r"start.*|end.*|date|consent|exclude.*|attn.*|attention.*|" + r"timer.*|.*_timer|q_.*|distribution.*|channel|latitude|longitude|" + r"unnamed.*|index|row)$", + re.IGNORECASE, +) + +# Demographic variables: categorical but not experimental conditions. +_DEMO_NAME_RE = re.compile( + r"^(gender|sex|male|female|age|race.*|ethnic.*|income|education|edu|" + r"degree|party|politic.*|conserv.*|liberal|religio.*|ses|" + r"hispanic|nationality|country|state|zip|marital|employ.*)$", + re.IGNORECASE, +) + + +@dataclass +class SavVariable: + name: str + label: str + measure: str + role: str + type: str + choices: List[str] = field(default_factory=list) + scale: Optional[Dict[str, Any]] = None + + def is_response_item(self) -> bool: + return self.role == "item" + + +@dataclass +class ParsedDataset: + dataset_name: str + n_rows: int + variables: List[SavVariable] = field(default_factory=list) + source_kind: str = "sav_data" + provides_stimulus: bool = False + + def response_items(self) -> List[SavVariable]: + return [v for v in self.variables if v.role == "item"] + + def condition_variables(self) -> List[SavVariable]: + return [v for v in self.variables if v.role == "condition"] + + def demographics(self) -> List[SavVariable]: + return [v for v in self.variables if v.role == "demographic"] + + def to_dict(self) -> Dict[str, Any]: + return { + "dataset_name": self.dataset_name, + "source_kind": self.source_kind, + "provides_stimulus": self.provides_stimulus, + "n_rows": self.n_rows, + "variables": [asdict(v) for v in self.variables], + } + +# Value-label interpretation + +def _ordered_value_labels(value_labels: Optional[Dict[Any, str]]) -> List[tuple]: + """[(numeric_key, label), ...] sorted by numeric key when possible.""" + if not value_labels: + return [] + pairs = [] + for k, v in value_labels.items(): + try: + key = (0, float(k)) + except (TypeError, ValueError): + key = (1, str(k)) + pairs.append((key, k, str(v).strip())) + pairs.sort(key=lambda t: t[0]) + return [(k, lab) for _, k, lab in pairs] + + +def _scale_key_text(value: Any) -> str: + try: + n = float(value) + except (TypeError, ValueError): + return str(value).strip() + return str(int(n)) if n == int(n) else str(n) + + +def _clean_scale_anchor(key: Any, label: Any) -> str: + key_text = _scale_key_text(key) + text = re.sub(r"\s+", " ", str(label or "")).strip() + if not text: + return key_text + + leading = re.match(r"^([+-]?\d+(?:\.\d+)?)\s*(.*)$", text) + if leading: + leading_key = _scale_key_text(leading.group(1)) + rest = leading.group(2).strip() + if leading_key == key_text: + rest = re.sub( + r"^(?:[?=:.\-]|\u2010|\u2011|\u2012|\u2013|\u2014|\u2015|\u2212)+\s*", + "", + rest, + ).strip() + return rest or key_text + + return text + + +def _looks_like_rating_scale(value_labels: Optional[Dict[Any, str]], measure: str) -> bool: + """A rating scale has >=3 numeric, monotonically keyed anchors.""" + ordered = _ordered_value_labels(value_labels) + if len(ordered) < 3: + return False + nums = [] + for k, _ in ordered: + try: + nums.append(float(k)) + except (TypeError, ValueError): + return False + return measure in ("scale", "ordinal") or nums == sorted(nums) + + +def _scale_from_value_labels(value_labels: Dict[Any, str]) -> Optional[Dict[str, Any]]: + nums: List[float] = [] + anchors: Dict[str, str] = {} + for k, lab in value_labels.items(): + try: + n = float(k) + except (TypeError, ValueError): + continue + nums.append(n) + if lab and str(lab).strip(): + ik = int(n) if n == int(n) else n + anchors[str(ik)] = _clean_scale_anchor(ik, lab) + if not nums: + return None + lo, hi = min(nums), max(nums) + return { + "min": int(lo) if lo == int(lo) else lo, + "max": int(hi) if hi == int(hi) else hi, + "anchors": anchors, + } + + +# Role + type classification + +def _classify(name: str, label: str, measure: str, + value_labels: Optional[Dict[Any, str]]) -> tuple[str, str]: + """Return (role, normalized_type). + + role: id_meta | demographic | condition | item | derived + """ + has_labels = bool(value_labels) + is_scale = _looks_like_rating_scale(value_labels, measure) + + if _META_NAME_RE.match(name): + return "id_meta", "meta" + + if re.search(r"_x_|_X_|interaction", name) or (not label and not has_labels): + return "derived", "numeric" + + if _DEMO_NAME_RE.match(name): + return "demographic", "multiple_choice" if has_labels else "numeric" + + if has_labels and not is_scale and measure == "nominal": + return "condition", "multiple_choice" + + if is_scale: + return "item", "scale" + + if has_labels: + return "item", "multiple_choice" + + # No value labels: a measured numeric or free response. + if measure == "scale": + return "item", "numeric" + return "item", "text" + + +def _build_variable(name: str, label: Optional[str], measure: str, + value_labels: Optional[Dict[Any, str]]) -> SavVariable: + label = (label or "").strip() + role, ntype = _classify(name, label, measure, value_labels) + + choices: List[str] = [] + scale: Optional[Dict[str, Any]] = None + if value_labels: + if ntype == "scale": + scale = _scale_from_value_labels(value_labels) + else: + choices = [lab for _, lab in _ordered_value_labels(value_labels) if lab] + + return SavVariable( + name=name, + label=label, + measure=measure or "unknown", + role=role, + type=ntype, + choices=choices, + scale=scale, + ) + +# Core parse + +def parse_sav_meta(meta: Any, dataset_name: str = "") -> ParsedDataset: + """Build a ParsedDataset from a pyreadstat metadata object.""" + names = list(meta.column_names) + labels = dict(getattr(meta, "column_names_to_labels", {}) or {}) + measures = dict(getattr(meta, "variable_measure", {}) or {}) + value_labels = dict(getattr(meta, "variable_value_labels", {}) or {}) + + variables = [ + _build_variable(n, labels.get(n), measures.get(n, "unknown"), value_labels.get(n)) + for n in names + ] + return ParsedDataset( + dataset_name=dataset_name or getattr(meta, "file_label", "") or "", + n_rows=int(getattr(meta, "number_rows", 0) or 0), + variables=variables, + ) + + +def parse_sav_file(path: str | Path) -> ParsedDataset: + """Load a .sav file's metadata (no data rows) and parse it. + + Requires `pyreadstat`. Raises a clear error if it is not installed. + """ + try: + import pyreadstat # noqa: WPS433 (lazy import; optional dependency) + except ImportError as exc: # pragma: no cover - environment dependent + raise ImportError( + "Reading .sav files requires pyreadstat. Install it with " + "`pip install pyreadstat`." + ) from exc + + p = Path(path) + _, meta = pyreadstat.read_sav(str(p), metadataonly=True, output_format="dict") + return parse_sav_meta(meta, dataset_name=p.stem) + +# CLI + +def _main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser(description="Parse an SPSS .sav data file's codebook.") + ap.add_argument("sav", help="Path to the .sav file") + ap.add_argument("--json", action="store_true", help="Emit full parsed JSON") + ap.add_argument("--all", action="store_true", help="Include id/meta variables") + args = ap.parse_args(argv) + + ds = parse_sav_file(args.sav) + + if args.json: + out = ds.to_dict() + if not args.all: + out["variables"] = [v for v in out["variables"] if v["role"] != "id_meta"] + print(json.dumps(out, ensure_ascii=False, indent=2)) + return 0 + + print(f"Dataset: {ds.dataset_name} ({len(ds.variables)} vars, {ds.n_rows} rows)") + print(f"provides_stimulus={ds.provides_stimulus} (codebook only; stimulus must come from PDF)") + + groups = [ + ("Conditions", ds.condition_variables()), + ("Items (DVs / measures)", ds.response_items()), + ("Demographics", ds.demographics()), + ] + if args.all: + groups.append(("ID / meta", [v for v in ds.variables if v.role == "id_meta"])) + + for title, vs in groups: + if not vs: + continue + print(f"\n## {title} ({len(vs)})") + for v in vs: + preview = v.label.replace("\n", " ") + if len(preview) > 78: + preview = preview[:78] + "…" + extra = "" + if v.scale: + extra = f" scale={v.scale['min']}-{v.scale['max']}" + elif v.choices: + extra = f" options={v.choices[:4]}" + print(f" - {v.name:18s} ({v.type}) {preview}{extra}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/generation_pipeline/parsers/source_linker.py b/generation_pipeline/parsers/source_linker.py new file mode 100644 index 0000000..7804e42 --- /dev/null +++ b/generation_pipeline/parsers/source_linker.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional, Sequence, Set + +# Extensions we can parse structurally vs. treat as free text. +STRUCTURED_EXTS = {".qsf": "qsf", ".sav": "sav"} +TEXT_EXTS = {".pdf": "pdf", ".docx": "docx", ".txt": "text", ".md": "text"} + +DATA_EXTS = {".csv", ".xlsx", ".xls", ".do", ".r", ".sps", ".dta", ".json"} + +# Named (non-numeric) study tokens that recur across papers. +_NAMED_TOKENS = ("pilot", "validation") + +_TOKEN_RE = re.compile(r"\b(?:study|studies|exp(?:eriment)?s?)[\s_-]*" + r"(\d+[a-z]?(?:-[a-z0-9]+)?)(?![a-z0-9])", re.IGNORECASE) + +_SHORT_RE = re.compile(r"(? str: + return re.sub(r"\s+", "", tok).lower() + + +def _expand_range(raw: str) -> Set[str]: + """Expand compact study refs to a token set. + + "1a-b" -> {1a, 1b} "2-3" -> {2, 3} "1a/1b" -> {1a, 1b} + "2a" -> {2a} "1" -> {1} + """ + raw = _norm_token(raw) + m = re.match(r"^(\d+)([a-z]?)[-–](\d*)([a-z]?)$", raw) + if not m: + return {raw} + n1, l1, n2, l2 = m.groups() + base = n1 + out: Set[str] = set() + # letter range on same base number: 1a-b -> 1a, 1b + if l1 and l2 and not n2: + for code in range(ord(l1), ord(l2) + 1): + out.add(f"{base}{chr(code)}") + return out + # numeric range: 2-3 -> 2, 3 + if n2 and not l1 and not l2: + for num in range(int(n1), int(n2) + 1): + out.add(str(num)) + return out + # explicit two endpoints: 1a/1b style already split by separator + out.add(f"{n1}{l1}") + if n2: + out.add(f"{n2}{l2}") + elif l2: + out.add(f"{n1}{l2}") + return out + + +def study_tokens(study_id: str) -> Set[str]: + """Token set identifying a sub-study, e.g. 'Study 2a' -> {'2a'}.""" + text = (study_id or "").lower() + tokens: Set[str] = set() + for m in _TOKEN_RE.finditer(text): + tokens |= _expand_range(m.group(1)) + for m in _SHORT_RE.finditer(study_id or ""): + tokens.add(_norm_token(m.group(1))) + for named in _NAMED_TOKENS: + if named in text: + tokens.add(named) + return tokens + + +def file_tokens(path: str) -> Set[str]: + """Token set a file/folder name refers to, ranges expanded.""" + name = str(path) + tokens: Set[str] = set() + for m in _TOKEN_RE.finditer(name): + tokens |= _expand_range(m.group(1)) + for m in _SHORT_RE.finditer(name): + tokens.add(_norm_token(m.group(1))) + low = name.lower() + for named in _NAMED_TOKENS: + if named in low: + tokens.add(named) + return tokens + + +def route_ext(path: str | Path) -> Optional[str]: + """Return parser kind for a path: 'qsf'|'sav'|'pdf'|'docx'|'text'|'data'|None.""" + ext = Path(path).suffix.lower() + if ext in STRUCTURED_EXTS: + return STRUCTURED_EXTS[ext] + if ext in TEXT_EXTS: + return TEXT_EXTS[ext] + if ext in DATA_EXTS: + return "data" + return None + + +@dataclass +class LinkedFile: + path: Path + kind: str # qsf | sav | pdf | docx | text | data + tokens: Set[str] = field(default_factory=set) + + @property + def is_material(self) -> bool: + """Whether this file can yield participant-facing materials.""" + return self.kind in ("qsf", "sav", "pdf", "docx", "text") + + +@dataclass +class LinkResult: + by_study: Dict[str, List[LinkedFile]] + unlinked: List[LinkedFile] + + def material_files(self, study_id: str) -> List[LinkedFile]: + files = self.by_study.get(study_id, []) + # Prefer structured instruments first, then text fallback. + order = {"qsf": 0, "sav": 1, "docx": 2, "pdf": 3, "text": 4, "data": 9} + return sorted([f for f in files if f.is_material], + key=lambda f: order.get(f.kind, 8)) + + +def _iter_files(root: str | Path) -> List[Path]: + base = Path(root) + if not base.exists(): + return [] + return sorted(p for p in base.rglob("*") if p.is_file()) + + +def link_sources( + study_ids: Sequence[str], + files_root: str | Path, + *, + include_data: bool = False, +) -> LinkResult: + studies = {sid: study_tokens(sid) for sid in study_ids} + by_study: Dict[str, List[LinkedFile]] = {sid: [] for sid in study_ids} + unlinked: List[LinkedFile] = [] + + for path in _iter_files(files_root): + kind = route_ext(path) + if kind is None: + continue + if kind == "data" and not include_data: + continue + # match on the path relative to root so folder names count + try: + rel = path.relative_to(files_root) + except ValueError: + rel = path + ftoks = file_tokens(str(rel)) + lf = LinkedFile(path=path, kind=kind, tokens=ftoks) + + matched = [sid for sid, stoks in studies.items() if stoks & ftoks] + if matched: + for sid in matched: + by_study[sid].append(lf) + else: + unlinked.append(lf) + + return LinkResult(by_study=by_study, unlinked=unlinked) diff --git a/generation_pipeline/parsers/stage3_adapter.py b/generation_pipeline/parsers/stage3_adapter.py new file mode 100644 index 0000000..bf82e75 --- /dev/null +++ b/generation_pipeline/parsers/stage3_adapter.py @@ -0,0 +1,290 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any, Dict, List, Optional + +from generation_pipeline.parsers.source_linker import study_tokens + +# Status ranked by trust; used to pick the best when several effects disagree. +_STATUS_RANK = { + "verbatim": 0, + "osf_only": 1, + "paraphrased": 2, + "not_in_paper": 8, + "source_missing": 8, + None: 9, +} +_TOKEN_RE = re.compile(r"[a-z][a-z0-9_'-]{2,}", re.IGNORECASE) +_STOP_TERMS = { + "the", "and", "for", "with", "that", "this", "from", "into", "about", + "study", "studies", "participants", "participant", "condition", + "conditions", "message", "messages", "article", "articles", "measure", + "measures", "item", "items", "scale", "were", "was", "are", "they", + "their", "then", "after", "before", "effect", "effects", + "one", "two", "one-sided", "two-sided", "sidedness", "side", "sides", + "author", "authors", "argued", "arguments", "acknowledged", "briefly", + "reasonable", "points", "same", "first", "fact", "few", "some", "such", + "version", "versions", "written", "read", "presented", +} +_PARTICIPANT_TEXT_RE = re.compile( + r"\b(participants?|you|your|please|read|rate|respond|answer|choose|" + r"write|presented|assigned|told|condition|scenario|message|article|task)\b", + re.IGNORECASE, +) +_ANALYSIS_TEXT_RE = re.compile( + r"\b(regression|model|anova|coefficient|hypothes(?:is|ized|es)|" + r"independent variables?|dependent variable|p\s*[<=>]|B\s*=|t\(|" + r"partial\s+η|figure\s+\d+|results?)\b", + re.IGNORECASE, +) + + +def load_stage3_studies(stage3_path: str | Path) -> List[Dict[str, Any]]: + """Return the list of study dicts from a stage3.json file.""" + data = json.loads(Path(stage3_path).read_text(encoding="utf-8")) + return data.get("eligible_studies") or data.get("studies") or [] + + +def match_stage3_study(study_id: str, studies: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Find the Stage 3 study whose name shares a study-token with `study_id`.""" + want = study_tokens(study_id) + if not want: + return None + for s in studies: + name = s.get("study") or s.get("study_id") or s.get("experiment_id") or "" + if study_tokens(str(name)) & want: + return s + return None + + +def _slot(effect: Dict[str, Any], name: str) -> tuple[Optional[str], str]: + """Return (status, content) for a slot, tolerating string or dict shape.""" + v = effect.get(name) + if isinstance(v, dict): + return v.get("status"), _clean_slot_text(v.get("content")) + if isinstance(v, str): + return ("verbatim" if v.strip() else None), _clean_slot_text(v) + return None, "" + + +def _clean_slot_text(value: Any) -> str: + text = str(value or "").replace("…", " ").replace("...", " ") + text = re.sub(r"\s+", " ", text).strip() + if not text: + return "" + if re.match(r"^[a-z]{2,}\b", text): + colon = text.find(":") + if 0 < colon < 240: + after = text[colon + 1:].strip() + if re.match(r'^[“"A-Z]|\b(please|imagine|read|rate|you|your)\b', after, flags=re.IGNORECASE): + text = after + return text + + +def _status_rank(status: Optional[str]) -> int: + return _STATUS_RANK.get(status, 5) + + +def _text_tokens(text: str) -> set[str]: + return { + tok.strip("'_-") + for tok in _TOKEN_RE.findall(str(text or "").lower()) + if tok.strip("'_-") not in _STOP_TERMS and not tok.isdigit() + } + + +def _similarity(a: set[str], b: set[str]) -> float: + if not a or not b: + return 0.0 + return len(a & b) / max(1, len(a | b)) + + +def _is_analysis_text(text: str) -> bool: + return bool(_ANALYSIS_TEXT_RE.search(text)) + + +def _is_participant_text(text: str) -> bool: + return bool(_PARTICIPANT_TEXT_RE.search(text)) + + +def _effect_is_primary(effect: Dict[str, Any]) -> bool: + cons = effect.get("consolidation") + if isinstance(cons, dict): + return bool(cons.get("is_primary_simulation_target", True)) and cons.get("is_representative", True) is not False + return True + + +def _slot_records(effects: List[Dict[str, Any]], name: str) -> List[Dict[str, Any]]: + records: List[Dict[str, Any]] = [] + source_effects = [effect for effect in effects if _effect_is_primary(effect)] or effects + for idx, effect in enumerate(source_effects): + status, content = _slot(effect, name) + if not content: + continue + if name in {"materials", "manipulation"} and (_is_analysis_text(content) or not _is_participant_text(content)): + continue + records.append({ + "status": status or "unknown", + "content": content, + "tokens": _text_tokens(content), + "index": idx, + }) + return records + + +def _dedup_records(records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + best: Dict[str, Dict[str, Any]] = {} + for rec in records: + key = re.sub(r"\s+", " ", rec["content"].lower()).strip() + if key not in best or _status_rank(rec["status"]) < _status_rank(best[key]["status"]): + best[key] = rec + return list(best.values()) + + +def _coherent_records(records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + records = _dedup_records(records) + if len(records) <= 1: + return records + + clusters: List[List[Dict[str, Any]]] = [] + for rec in records: + placed = False + for cluster in clusters: + if any(_similarity(rec["tokens"], other["tokens"]) >= 0.16 for other in cluster): + cluster.append(rec) + placed = True + break + if not placed: + clusters.append([rec]) + + def score(cluster: List[Dict[str, Any]]) -> tuple[int, int, int]: + status_bonus = sum(max(0, 8 - _status_rank(rec["status"])) for rec in cluster) + length_bonus = min(3000, sum(len(rec["content"]) for rec in cluster)) // 250 + return (len(cluster), status_bonus, length_bonus) + + chosen = max(clusters, key=score) + return sorted(chosen, key=lambda rec: (_status_rank(rec["status"]), rec["index"])) + + +def _best_unique(slot_values: List[tuple[Optional[str], str]]) -> List[tuple[str, str]]: + """Dedup (status, content) by normalized content, keeping best status.""" + best: Dict[str, tuple[str, str]] = {} + for status, content in slot_values: + if not content: + continue + key = re.sub(r"\s+", " ", content.lower()).strip() + rank = _status_rank(status) + if key not in best or rank < _status_rank(best[key][0]): + best[key] = (status or "unknown", content) + return list(best.values()) + + +def _slug(text: str, fallback: str) -> str: + s = re.sub(r"[^a-z0-9]+", "_", str(text or "").lower()).strip("_") + return s or fallback + + +def _condition_levels_from_texts(texts: List[str]) -> List[Dict[str, Any]]: + """Extract generic condition-level descriptions from participant text.""" + level_map: Dict[str, List[str]] = {} + sentences = re.split(r"(?<=[.!?])\s+", " ".join(texts)) + current_level: Optional[str] = None + for sentence in sentences: + sent = sentence.strip() + if not sent: + continue + match = re.search( + r"\b(?:in|under|for)\s+the\s+([A-Za-z0-9][A-Za-z0-9 /_-]{1,70}?\s+condition)\b", + sent, + flags=re.IGNORECASE, + ) + if match: + current_level = re.sub(r"\s+", " ", match.group(1)).strip() + level_map.setdefault(current_level, []).append(sent) + continue + if current_level and re.match(r"^(?:In addition|Then|Next|The author|Participants|They)\b", sent): + level_map[current_level].append(sent) + if len(level_map) < 2: + return [] + return [ + { + "level": level, + "description": " ".join(parts), + "source": "pdf_stage3_slot", + } + for level, parts in level_map.items() + ] + + +def materials_from_stage3(stage3_study: Dict[str, Any]) -> Dict[str, Any]: + """Consolidate a Stage 3 study's effect slots into a materials block. + + Returns {instructions, items, conditions, slot_summary} where each item and + the instructions carry their Stage 3 `status` so trust is visible downstream. + """ + effects = [e for e in stage3_study.get("effects", []) if isinstance(e, dict)] + + materials = [(rec["status"], rec["content"]) for rec in _coherent_records(_slot_records(effects, "materials"))] + manipulations = [(rec["status"], rec["content"]) for rec in _coherent_records(_slot_records(effects, "manipulation"))] + + # instructions = stimulus (materials) followed by manipulation framing + instr_parts: List[str] = [c for _, c in materials] + [c for _, c in manipulations] + instructions = "\n\n".join(instr_parts) + + # items: one per distinct DV measure wording, labelled by the effect's DV + items: List[Dict[str, Any]] = [] + seen: set[str] = set() + for i, e in enumerate(effects, start=1): + status, content = _slot(e, "items") + if not content: + continue + key = re.sub(r"\s+", " ", content.lower()).strip() + if key in seen: + continue + seen.add(key) + dv = e.get("DV") or f"measure {i}" + items.append({ + "id": _slug(dv, f"item_{i}"), + "question": content, + "options": [], + "type": "open_ended", + "source": "pdf_stage3", + "source_file": "stage3.json", + "data_export_tag": None, + "slot_status": status or "unknown", + "dv": dv, + }) + + # conditions from manipulation slots (named, when distinguishable) + condition_levels = _condition_levels_from_texts([c for _, c in manipulations]) + conditions: List[Dict[str, Any]] = [] + if condition_levels: + conditions.append({ + "name": "pdf_stage3_conditions", + "levels": [level["level"] for level in condition_levels], + "level_descriptions": { + level["level"]: level["description"] for level in condition_levels + }, + "source": "pdf_stage3_slot", + "review_required": True, + }) + + instruction_statuses = [status for status, _ in materials + manipulations] + best_instruction_status = ( + min(instruction_statuses, key=_status_rank) if instruction_statuses else "source_missing" + ) + + return { + "instructions": instructions, + "items": items, + "conditions": conditions, + "slot_summary": { + "instructions_status": best_instruction_status, + "materials_status": materials[0][0] if materials else "source_missing", + "manipulation_status": manipulations[0][0] if manipulations else "source_missing", + "items_status": items[0]["slot_status"] if items else "source_missing", + "n_effects": len(effects), + }, + } diff --git a/generation_pipeline/patchers/__init__.py b/generation_pipeline/patchers/__init__.py new file mode 100644 index 0000000..9c3e5fc --- /dev/null +++ b/generation_pipeline/patchers/__init__.py @@ -0,0 +1,3 @@ +from generation_pipeline.patchers.slot_filler import SlotFiller, find_study_section + +__all__ = ["SlotFiller", "find_study_section"] diff --git a/generation_pipeline/patchers/patch_runner.py b/generation_pipeline/patchers/patch_runner.py new file mode 100644 index 0000000..0940c7d --- /dev/null +++ b/generation_pipeline/patchers/patch_runner.py @@ -0,0 +1,267 @@ +""" +Patch Runner — orchestrates Stage 3 across the existing 30-paper corpus. + +Pairs each JSON in with a PDF in by filename-stem prefix, +runs SlotFiller, and writes back atomically (.tmp → rename), with optional +.bak backup of the original. +""" + +import json +import shutil +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from generation_pipeline.patchers.slot_filler import SlotFiller +from generation_pipeline.verification.schema_validator import summarize_report, validate_paper + + +# --------------------------------------------------------------------------- +# Pairing +# --------------------------------------------------------------------------- + +def _stem_key(name: str, n_chars: int = 60) -> str: + """Normalize a filename for matching (matches extractor.html's keyOf).""" + s = Path(name).stem + s = s.replace(",", "").replace(" ", "_").lower() + return s[:n_chars] + + +def pair_json_pdf(json_dir: Path, pdf_dir: Path, *, only: Optional[List[str]] = None) -> List[Tuple[Path, Path]]: + """Return [(json_path, pdf_path), ...] paired by filename-stem prefix.""" + json_dir = Path(json_dir) + jsons = list(json_dir.glob("*.json")) + jsons.extend(path for path in json_dir.glob("*/*.json") if path not in jsons) + if only: + jsons = [j for j in jsons if any(o in j.name for o in only)] + pdfs = list(Path(pdf_dir).glob("*.pdf")) + pdf_index = {_stem_key(p.name): p for p in pdfs} + + out: List[Tuple[Path, Path]] = [] + for j in jsons: + jk = _stem_key(j.name) + # Exact match first, then prefix match + match = pdf_index.get(jk) + if match is None: + for pk, p in pdf_index.items(): + if pk.startswith(jk[:40]) or jk.startswith(pk[:40]): + match = p + break + if match is not None: + out.append((j, match)) + else: + print(f" ⚠ no PDF match for {j.name}") + return out + + +def _source_dir_from_candidate(path: Path) -> Path | None: + """Resolve a source path to a directory containing combined_sources.txt. + + Accept both the actual source directory (``.../sources``) and a paper + directory containing ``sources/combined_sources.txt``. + """ + path = Path(path) + if path.is_dir() and (path / "combined_sources.txt").exists(): + return path + nested = path / "sources" + if nested.is_dir() and (nested / "combined_sources.txt").exists(): + return nested + return None + + +def discover_source_dirs( + json_path: Path, + pdf_path: Path, + *, + explicit_source_dirs: Optional[List[Path]] = None, + use_fetched_sources: bool = True, +) -> List[Path]: + """Find OSF/source directories for a JSON/PDF pair.""" + out: List[Path] = [] + seen: set[str] = set() + candidates: list[Path] = [] + if explicit_source_dirs: + candidates.extend(Path(item) for item in explicit_source_dirs) + if use_fetched_sources: + candidates.extend( + [ + Path(json_path).parent / "sources", + Path(pdf_path).parent / "sources", + ] + ) + + for candidate in candidates: + resolved = _source_dir_from_candidate(candidate) + if resolved is None: + continue + key = str(resolved.resolve()) + if key in seen: + continue + seen.add(key) + out.append(resolved) + return out + + +# --------------------------------------------------------------------------- +# Atomic write +# --------------------------------------------------------------------------- + +def write_json_atomic(path: Path, data: dict, *, backup: bool) -> None: + backup_path = path.with_suffix(path.suffix + ".bak") + if backup and path.exists() and not backup_path.exists(): + shutil.copy2(path, backup_path) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + tmp.replace(path) + + +def format_patch_update(update: Any) -> str: + """Render one incremental Stage 3 update for CLI logs.""" + if not isinstance(update, dict): + return str(update) + + path = update.get("path") or ".".join( + str(part) for part in (update.get("study"), update.get("slot")) if part + ) + parts = [path or "unknown"] + status = update.get("status") + score = update.get("score") + source = update.get("source") + iv = update.get("iv") + dv = update.get("dv") + filled_nulls = update.get("filled_nulls") + updated_fields = update.get("updated_fields") + content_chars = update.get("content_chars") + + if status is not None: + parts.append(f"status={status}") + if score is not None: + try: + parts.append(f"score={float(score):.0f}") + except (TypeError, ValueError): + parts.append(f"score={score}") + if iv: + parts.append(f"iv={_short_log_value(iv)}") + if dv: + parts.append(f"dv={_short_log_value(dv)}") + if source: + parts.append(f"source={source}") + if filled_nulls is not None: + parts.append(f"filled_nulls={filled_nulls}") + if updated_fields is not None: + parts.append(f"updated_fields={updated_fields}") + if content_chars is not None: + parts.append(f"content_chars={content_chars}") + return " ".join(parts) + + +def _short_log_value(value: Any, limit: int = 48) -> str: + text = " ".join(str(value).split()) + if len(text) > limit: + text = text[: limit - 3] + "..." + return json.dumps(text, ensure_ascii=False) + + +# --------------------------------------------------------------------------- +# Runner +# --------------------------------------------------------------------------- + +def run_patch( + json_dir: Path, + pdf_dir: Path, + *, + filler: SlotFiller, + only: Optional[List[str]] = None, + overwrite_filled: bool = False, + backup: bool = True, + dry_run: bool = False, + use_fetched_sources: bool = True, + source_dirs: Optional[List[Path]] = None, +) -> Dict[str, str]: + """ + Patch every JSON in `json_dir` whose stem matches a PDF in `pdf_dir`. + + Args: + only: optional filename substrings — only patch matching JSONs + overwrite_filled: re-run already-filled slots + backup: write .bak before overwriting + dry_run: pair + run LLM but don't write back + use_fetched_sources: if True (default), automatically look for a + ``sources/`` directory adjacent to each JSON and paired PDF, then + pass any ``combined_sources.txt`` found there to the slot filler. + This lets Stage 3 use OSF/supplementary materials fetched during + Stage 1 source discovery or stored under ``stage4//sources``. + source_dirs: optional explicit OSF/source directories. Each item may be + either a directory containing ``combined_sources.txt`` or a paper + directory containing ``sources/combined_sources.txt``. + + Returns map of {json_filename: status_string}. + """ + pairs = pair_json_pdf(json_dir, pdf_dir, only=only) + + print(f"Pairing: {len(pairs)} JSON↔PDF pairs found.") + if not pairs: + return {} + + results: Dict[str, str] = {} + for i, (jpath, ppath) in enumerate(pairs, 1): + print(f"\n[{i}/{len(pairs)}] {jpath.name}") + try: + paper = json.loads(jpath.read_text(encoding="utf-8")) + except Exception as e: + print(f" ✗ could not load JSON: {e}") + results[jpath.name] = f"load-error: {e}" + continue + + pair_source_dirs = discover_source_dirs( + jpath, + ppath, + explicit_source_dirs=source_dirs, + use_fetched_sources=use_fetched_sources, + ) + for source_dir in pair_source_dirs: + print(f" ↳ using fetched sources: {source_dir / 'combined_sources.txt'}") + + def write_incremental(current: dict, update: Any) -> None: + if dry_run: + return + detail = format_patch_update(update) + repaired, report = validate_paper(current, repair=True, path=jpath) + if not report.valid: + raise ValueError(f"Incremental schema validation failed after {detail}: {summarize_report(report)}") + write_json_atomic(jpath, repaired, backup=backup) + print(f" ↳ wrote incremental patch: {detail}") + + try: + patched = filler.patch_paper( + paper, + ppath, + overwrite_filled=overwrite_filled, + source_dirs=pair_source_dirs or None, + on_update=write_incremental, + ) + except Exception as e: + print(f" ✗ patch failed: {e}") + results[jpath.name] = f"patch-error: {e}" + continue + + patched, schema_report = validate_paper(patched, repair=True, path=jpath) + if not schema_report.valid: + print(f" ✗ schema validation failed: {summarize_report(schema_report)}") + results[jpath.name] = f"schema-error: {summarize_report(schema_report)}" + continue + if schema_report.changed or schema_report.issues: + print(f" schema validation: {summarize_report(schema_report)}") + + if dry_run: + print(f" ✓ (dry-run) would write {jpath.name}") + results[jpath.name] = "dry-run-ok" + continue + + write_json_atomic(jpath, patched, backup=backup) + print(f" ✓ wrote {jpath.name}") + results[jpath.name] = "ok" + + return results diff --git a/generation_pipeline/patchers/slot_filler.py b/generation_pipeline/patchers/slot_filler.py new file mode 100644 index 0000000..240bdc2 --- /dev/null +++ b/generation_pipeline/patchers/slot_filler.py @@ -0,0 +1,684 @@ +""" +Slot Filler - Stage 3 (ai-ethics) + +Patches existing per-paper JSON files (the `输出/*.json` corpus) by: + 1. Filling empty `materials / manipulation / items` slots (status == null) + with verbatim/paraphrased content from the PDF. + 2. Upgrading the `sample` field from a string to a structured participants + object. Current canonical schema stores this object at study level; this + class still accepts old effect-level `sample` strings for migration. + +The patcher only writes slots that are currently empty (status == null); +human-reviewed slots are preserved. +""" + +import json +import re +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from copy import deepcopy +from pathlib import Path +from typing import Any, Callable, Dict, Optional + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + + +STUDY_SECTION_CHARS = 24000 # window per study fed to LLM +PDF_TEXT_MAX_CHARS = 400000 +PatchUpdate = Dict[str, Any] + + +def _count_non_null_leaves(value: Any) -> int: + if isinstance(value, dict): + return sum(_count_non_null_leaves(item) for item in value.values()) + if isinstance(value, list): + return sum(_count_non_null_leaves(item) for item in value) + return 0 if value is None else 1 + + +def _count_newly_filled_nulls(previous: Any, current: Any) -> int: + if previous is None: + return _count_non_null_leaves(current) + if isinstance(previous, dict) and isinstance(current, dict): + keys = set(previous) | set(current) + return sum(_count_newly_filled_nulls(previous.get(key), current.get(key)) for key in keys) + if isinstance(previous, list) and isinstance(current, list): + total = 0 + max_len = max(len(previous), len(current)) + for index in range(max_len): + before = previous[index] if index < len(previous) else None + after = current[index] if index < len(current) else None + total += _count_newly_filled_nulls(before, after) + return total + return 0 + + +def _content_chars(value: Any) -> int: + if isinstance(value, str): + return len(value) + if value is None: + return 0 + return len(json.dumps(value, ensure_ascii=False)) + + +def _make_patch_update( + study_label: str, + slot_name: str, + *, + source: str, + previous: Any, + current: Any, + effect_index: int | None = None, + iv: str | None = None, + dv: str | None = None, + score: float | None = None, +) -> PatchUpdate: + content = current.get("content") if isinstance(current, dict) else current + path = f"{study_label}.{slot_name}" + if effect_index is not None: + path = f"{study_label}.effect[{effect_index}].{slot_name}" + return { + "path": path, + "study": study_label, + "effect_index": effect_index, + "slot": slot_name, + "iv": iv, + "dv": dv, + "source": source, + "status": current.get("status") if isinstance(current, dict) else None, + "score": score, + "filled_nulls": _count_newly_filled_nulls(previous, current), + "updated_fields": _count_non_null_leaves(current), + "content_chars": _content_chars(content), + } + + +# --------------------------------------------------------------------------- +# Section locator: pull a window around "Study N" / first Method section +# --------------------------------------------------------------------------- + +def find_study_section(pdf_text: str, study_label: str, window: int = STUDY_SECTION_CHARS) -> str: + """Return a substring of PDF text centered on the study's Method section. + + Tries (in order): "Study N", "Experiment N", "Method"; falls back to the + first `window` chars of the document. + """ + # Normalize: "Study 1" → patterns ["Study 1", "Experiment 1", "Study1"] + candidates = [study_label] + m = re.search(r"(study|experiment)\s*(\d+[a-z]?)", study_label, re.I) + if m: + n = m.group(2) + candidates.extend([f"Study {n}", f"Experiment {n}", f"Study{n}", f"Experiment{n}"]) + candidates.append("Method") + + for cand in candidates: + match = re.search(rf"\b{re.escape(cand)}\b", pdf_text, re.I) + if match: + start = max(0, match.start() - 200) + return pdf_text[start : start + window] + + return pdf_text[:window] + + +class SlotFiller: + """Patch one paper JSON in place (returns new dict, does not mutate input).""" + + VALID_STATUSES = { + "verbatim", + "paraphrased", + "cited_scale", + "osf_only", + "not_in_paper", + "source_missing", + } + + def __init__( + self, + llm_client, + *, + source_workers: int = 4, + source_slot_timeout: float | None = 60.0, + ): + self.client = llm_client + self.source_workers = max(int(source_workers or 1), 1) + self.source_slot_timeout = source_slot_timeout + + # Statuses that mean "slot was assessed but content isn't in the PDF" — + # these become candidates for re-patching when OSF source files are provided. + OSF_RETRYABLE_STATUSES = {"osf_only", "not_in_paper", "cited_scale"} + + def patch_paper( + self, + paper_json: Dict[str, Any], + pdf_path: Path, + *, + overwrite_filled: bool = False, + source_dirs: Optional[list] = None, + on_update: Optional[Callable[[Dict[str, Any], Any], None]] = None, + ) -> Dict[str, Any]: + """ + Args: + paper_json: parsed JSON from `输出/*.json` + pdf_path: matched PDF (same filename stem) + overwrite_filled: if True, also re-run already-filled slots + (default: False — preserve human-reviewed work) + source_dirs: optional list of Path-like source directories (e.g. + ``stage4//sources``). When provided: + 1. Pass 1 (PDF): fill status=None slots from the PDF. + 2. Pass 2 (OSF grounded): for slots still + ``osf_only``/``not_in_paper``/``cited_scale`` with + no content, run the paper-qa-style GroundedSlotExtractor + against the combined source text. Only verified-verbatim + quotes are accepted; unverifiable content is honestly kept + as the original status with content=null. + """ + result = deepcopy(paper_json) + pdf_text = extract_pdf_text(pdf_path, max_chars=PDF_TEXT_MAX_CHARS) + + for study in result.get("eligible_studies", []): + study_label = study.get("study", "") + section = find_study_section(pdf_text, study_label) + for effect_index, effect in enumerate(study.get("effects", []), 1): + updates = self._patch_effect( + effect, + section, + study_label, + effect_index=effect_index, + overwrite=overwrite_filled, + ) + if on_update: + for update in updates: + on_update(result, update) + + source_records = self._load_supplementary_sources(source_dirs) + if source_records: + self._patch_from_sources(result, source_records, on_update=on_update) + + return result + + @staticmethod + def _load_supplementary(source_dirs: Optional[list]) -> str: + """Load text from combined_sources.txt in each source directory. + + No hard character cap is applied here: the text is chunked and only + the top-k retrieved chunks (typically ~20 KB) are ever sent to the + LLM, so a large corpus does not inflate prompt costs. + """ + if not source_dirs: + return "" + parts: list[str] = [] + for src_dir in source_dirs: + combined = Path(src_dir) / "combined_sources.txt" + if combined.exists(): + try: + parts.append(combined.read_text(encoding="utf-8", errors="ignore")) + except Exception: + pass + return "\n\n".join(parts) + + @staticmethod + def _load_supplementary_sources(source_dirs: Optional[list]) -> list[dict[str, str]]: + """Load source text records while preserving source-file provenance. + + Preferred input is ``source_text_index.json`` written by the connector + registry; it maps each original source file to its extracted text file. + Fall back to the legacy anonymous ``combined_sources.txt`` only when the + index is unavailable. + """ + if not source_dirs: + return [] + records: list[dict[str, str]] = [] + for src_dir in source_dirs: + src_dir = Path(src_dir) + added = False + index_path = src_dir / "source_text_index.json" + if index_path.exists(): + try: + index = json.loads(index_path.read_text(encoding="utf-8")) + except Exception: + index = [] + if isinstance(index, list): + for item in index: + if not isinstance(item, dict): + continue + text_path = Path(str(item.get("text_path") or "")) + if not text_path.exists(): + continue + try: + text = text_path.read_text(encoding="utf-8", errors="ignore") + except Exception: + continue + if text.strip(): + records.append( + { + "source_path": str(item.get("source_path") or text_path), + "text_path": str(text_path), + "text": text, + } + ) + added = True + if added: + continue + combined = src_dir / "combined_sources.txt" + if combined.exists(): + try: + text = combined.read_text(encoding="utf-8", errors="ignore") + except Exception: + text = "" + if text.strip(): + records.append({"source_path": str(combined), "text_path": str(combined), "text": text}) + return records + + # Lower fuzzy-match threshold for OSF supplementary content. + # QSF piped text (${e://Field/text}) and pre-registration prose won't + # score ≥90 against the PDF, but ≥75 indicates the content is present. + OSF_PARAPHRASE_THRESHOLD = 75.0 + + def _patch_from_sources( + self, + result: Dict[str, Any], + source_text: str | list[dict[str, str]], + *, + on_update: Optional[Callable[[Dict[str, Any], Any], None]] = None, + ) -> None: + """Paper-qa-style grounded fill from OSF/supplementary source text. + + Chunks the source text and uses GroundedSlotExtractor (with a two-tier + threshold: ≥90 → verbatim, ≥75 → paraphrased) to fill slots that + were previously unfillable from the PDF alone. + + OSF supplementary files come in several formats: + - DOCX (e.g. manipulation scripts) → clean text, often verbatim-matchable + - Pre-registration PDFs → abstract prose, often paraphrase-tier + - QSF Qualtrics surveys → piped variables, needs assembly → paraphrase-tier + """ + try: + from generation_pipeline.extractors.grounded_slot_extractor import GroundedSlotExtractor + except ImportError: + return # grounded extractor unavailable — degrade gracefully + + extractor = GroundedSlotExtractor( + self.client, + threshold=90.0, + k=8, + retries=1, + slot_timeout=self.source_slot_timeout, + max_workers=self.source_workers, + ) + if isinstance(source_text, list): + extractor.prepare_from_sources(source_text) + else: + extractor.prepare_from_text(source_text) + + studies = [s for s in result.get("eligible_studies", []) if isinstance(s, dict)] + + per_study_filled: dict[int, int] = {} + per_study_empty_osf_only: dict[int, list[tuple[dict, int, str]]] = {} + source_tasks: list[dict[str, Any]] = [] + + for study_index, study in enumerate(studies): + study_label = study.get("study", "") + per_study_filled.setdefault(study_index, 0) + per_study_empty_osf_only.setdefault(study_index, []) + for effect_index, effect in enumerate(study.get("effects", []), 1): + if not isinstance(effect, dict): + continue + iv = effect.get("IV", "") + dv = effect.get("DV", "") + loc = effect.get("table_or_page_location", "") + notes = effect.get("materials_notes", "") + for slot_name in ("materials", "manipulation", "items"): + obj = effect.get(slot_name) or {} + status = obj.get("status") + content = obj.get("content") + if status not in self.OSF_RETRYABLE_STATUSES: + continue + if content: # already has content (e.g. human-filled) + continue + source_tasks.append( + { + "order": len(source_tasks), + "study_index": study_index, + "study_label": study_label, + "effect": effect, + "effect_index": effect_index, + "slot_name": slot_name, + "status": status, + "iv": iv, + "dv": dv, + "location_hint": loc, + "materials_notes": notes, + } + ) + + if source_tasks: + worker_count = min(self.source_workers, len(source_tasks)) + print( + f" source patch: checking {len(source_tasks)} OSF/source slots " + f"with {worker_count} worker(s)", + flush=True, + ) + + source_results = self._run_source_tasks(extractor, source_tasks) + for task, res in sorted(source_results, key=lambda item: item[0]["order"]): + effect = task["effect"] + slot_name = task["slot_name"] + obj = effect.get(slot_name) or {} + if res.content and res.status in {"verbatim", "paraphrased"}: + previous = deepcopy(obj) + current = {"status": res.status, "content": res.content} + effect[slot_name] = current + per_study_filled[task["study_index"]] += 1 + update = _make_patch_update( + task["study_label"], + slot_name, + source="osf", + previous=previous, + current=current, + effect_index=task["effect_index"], + iv=task["iv"], + dv=task["dv"], + score=res.score, + ) + if on_update: + on_update(result, update) + else: + print(f" ✓ filled from sources: {task['study_label']}.{slot_name} " + f"status={res.status} score={res.score:.0f}") + elif res.status == "source_missing": + previous = deepcopy(obj) + current = {"status": "source_missing", "content": None} + effect[slot_name] = current + update = _make_patch_update( + task["study_label"], + slot_name, + source="osf", + previous=previous, + current=current, + effect_index=task["effect_index"], + iv=task["iv"], + dv=task["dv"], + score=res.score, + ) + if on_update: + on_update(result, update) + else: + print(f" ⚠ source_missing: {task['study_label']}.{slot_name} " + f"(no study-scoped source evidence)") + elif task["status"] == "osf_only": + per_study_empty_osf_only[task["study_index"]].append( + (effect, task["effect_index"], slot_name) + ) + + # Mark study-local OSF gaps only after at least one sibling study was filled. + any_study_filled = any(count > 0 for count in per_study_filled.values()) + for study_index, study in enumerate(studies): + if per_study_filled.get(study_index, 0) > 0: + continue + if not any_study_filled: + continue + study_label = study.get("study", "") + for effect, effect_index, slot_name in per_study_empty_osf_only.get(study_index, []): + obj = effect.get(slot_name) or {} + previous = deepcopy(obj) + current = {"status": "source_missing", "content": None} + effect[slot_name] = current + update = _make_patch_update( + study_label, + slot_name, + source="osf", + previous=previous, + current=current, + effect_index=effect_index, + iv=effect.get("IV"), + dv=effect.get("DV"), + ) + if on_update: + on_update(result, update) + else: + print(f" ⚠ source_missing: {study_label}.{slot_name} " + f"(no material in fetched sources; sibling studies filled)") + + def _run_source_tasks(self, extractor: Any, tasks: list[dict[str, Any]]) -> list[tuple[dict[str, Any], Any]]: + if not tasks: + return [] + + def run_one(task: dict[str, Any]) -> tuple[dict[str, Any], Any]: + res = extractor.extract_slot( + study=task["study_label"], + iv=task["iv"], + dv=task["dv"], + slot=task["slot_name"], + location_hint=task["location_hint"], + materials_notes=task["materials_notes"], + prev_status=task["status"], + prev_content=None, + paraphrase_threshold=self.OSF_PARAPHRASE_THRESHOLD, + assemble=True, + ) + return task, res + + worker_count = min(self.source_workers, len(tasks)) + if worker_count <= 1: + results = [] + for index, task in enumerate(tasks, start=1): + print( + f" source patch {index}/{len(tasks)}: " + f"{task['study_label']} effect {task['effect_index']} {task['slot_name']}", + flush=True, + ) + results.append(run_one(task)) + return results + + results = [] + with ThreadPoolExecutor(max_workers=worker_count) as executor: + future_to_task = {executor.submit(run_one, task): task for task in tasks} + for index, future in enumerate(as_completed(future_to_task), start=1): + task, res = future.result() + print( + f" source patch completed {index}/{len(tasks)}: " + f"{task['study_label']} effect {task['effect_index']} {task['slot_name']}", + flush=True, + ) + results.append((task, res)) + return results + + def _patch_effect( + self, + effect: Dict[str, Any], + section_text: str, + study_label: str, + *, + effect_index: int | None = None, + overwrite: bool, + ) -> list[PatchUpdate]: + """Mutate `effect` in place: fill empty slots + upgrade sample.""" + updates: list[PatchUpdate] = [] + needs_slots = self._empty_slots(effect, overwrite=overwrite) + needs_sample_upgrade = isinstance(effect.get("sample"), str) + if not needs_slots and not needs_sample_upgrade: + return updates + + filled = self._call_llm( + section_text=section_text, + study_label=study_label, + effect=effect, + need_slots=needs_slots, + need_sample=needs_sample_upgrade, + ) + if filled is None: + return updates + + for slot_name in ("materials", "manipulation", "items"): + if slot_name in needs_slots and slot_name in filled: + new = filled[slot_name] + if isinstance(new, dict) and self._is_valid_slot(new): + previous = deepcopy(effect.get(slot_name) or {}) + effect[slot_name] = new + updates.append( + _make_patch_update( + study_label, + slot_name, + source="pdf", + previous=previous, + current=new, + effect_index=effect_index, + iv=effect.get("IV"), + dv=effect.get("DV"), + ) + ) + + # Legacy upgrade: old effect.sample string → flat sample object. + # The schema validator later lifts this to study.sample. + if needs_sample_upgrade and "sample" in filled and isinstance(filled["sample"], dict): + original = effect["sample"] + new_sample = filled["sample"] + description = new_sample.pop("description", None) + if not new_sample.get("notes"): + new_sample["notes"] = description or original + effect["sample"] = new_sample + updates.append( + _make_patch_update( + study_label, + "sample", + source="pdf", + previous=original, + current=new_sample, + effect_index=effect_index, + iv=effect.get("IV"), + dv=effect.get("DV"), + ) + ) + + return updates + + def _empty_slots(self, effect: Dict[str, Any], *, overwrite: bool) -> set[str]: + """Slots to fill from the PDF (status=None, or all if overwrite=True).""" + out = set() + for slot_name in ("materials", "manipulation", "items"): + slot = effect.get(slot_name) or {} + if overwrite or slot.get("status") is None: + out.add(slot_name) + return out + + def _is_valid_slot(self, slot: Dict[str, Any]) -> bool: + return ( + "status" in slot + and (slot["status"] is None or slot["status"] in self.VALID_STATUSES) + and "content" in slot + ) + + def _call_llm( + self, + *, + section_text: str, + study_label: str, + effect: Dict[str, Any], + need_slots: set[str], + need_sample: bool, + ) -> Optional[Dict[str, Any]]: + prompt = self._build_prompt(section_text, study_label, effect, need_slots, need_sample) + try: + response = self.client.generate_content(prompt=prompt) + except Exception as e: + print(f" LLM error on {study_label}: {e}") + return None + if response is None: + return None + return self._parse_response(response) + + def _build_prompt( + self, + section_text: str, + study_label: str, + effect: Dict[str, Any], + need_slots: set[str], + need_sample: bool, + ) -> str: + wanted = [] + if "materials" in need_slots: + wanted.append('"materials" (the stimuli / scenarios / texts participants saw)') + if "manipulation" in need_slots: + wanted.append('"manipulation" (exactly what differed between conditions for the IV)') + if "items" in need_slots: + wanted.append('"items" (the DV scale items / response options)') + if need_sample: + wanted.append('"sample" (structured participants profile, see schema below)') + + schema_slot = """{ + "status": "verbatim | paraphrased | cited_scale | osf_only | not_in_paper", + "content": "" +}""" + + sample_schema = """{ + "total_n": , + "analyzed_n": , + "mean_age": , + "female_percent": , + "male_percent": , + "platform": "", + "country": "", + "inclusion_criteria": "", + "exclusion_criteria": "", + "notes": "" +}""" + + return f"""You are extracting research-study details from a PDF text excerpt for the ai-ethics corpus. + +STUDY: {study_label} +KNOWN EFFECT (for context — do not re-extract): +- IV: {effect.get('IV')} +- DV: {effect.get('DV')} +- N: {effect.get('size')} +- materials_notes: {effect.get('materials_notes')} +- table_or_page_location: {effect.get('table_or_page_location')} + +PDF EXCERPT FOR THIS STUDY: +\"\"\" +{section_text} +\"\"\" + +YOUR TASK — extract the following fields: +{chr(10).join('- ' + w for w in wanted)} + +RULES (CRITICAL): +- Each of (materials, manipulation, items) is an object with this schema: +{schema_slot} +- For `content`, use VERBATIM TEXT from the excerpt when status="verbatim". +- If the paper only describes the material in summary form, use status="paraphrased". +- If the paper cites a published scale without reproducing it (e.g. "Reynolds 2008"), + status="cited_scale" and content=. +- If the paper says materials are on OSF / supplementary only, status="osf_only", content=null. +- If the paper does not include this material at all, status="not_in_paper", content=null. +- DO NOT FABRICATE: never invent stimulus text not present in the excerpt. + +- `sample` schema (structured participants profile; canonical location is study.sample): +{sample_schema} + Fill only fields the paper reports. All others = null. `total_n` must be the + full final/recruited study N. Do not put subgroup / condition / simple-effect + table Ns into sample.total_n. + +OUTPUT — respond with ONLY this JSON (no markdown fences). Include ONLY the +keys requested above (omit keys you were not asked to fill): +{{ + {", ".join(f'"{k}": ...' for k in (['materials','manipulation','items','sample'] if False else list(need_slots) + (['sample'] if need_sample else [])))} +}}""" + + def _parse_response(self, response: str) -> Optional[Dict[str, Any]]: + text = response.strip() if isinstance(response, str) else str(response).strip() + if "```json" in text: + text = text.split("```json", 1)[1].split("```", 1)[0].strip() + elif "```" in text: + text = text.split("```", 1)[1].split("```", 1)[0].strip() + try: + return json.loads(text) + except json.JSONDecodeError: + m = re.search(r"\{.*\}", text, re.DOTALL) + if m: + try: + return json.loads(m.group()) + except json.JSONDecodeError: + return None + return None diff --git a/generation_pipeline/pipeline.py b/generation_pipeline/pipeline.py new file mode 100644 index 0000000..e16b2dd --- /dev/null +++ b/generation_pipeline/pipeline.py @@ -0,0 +1,735 @@ +""" +Generation Pipeline Orchestrator for ai-ethics. +""" + +import json +import sys +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from generation_pipeline.filters.replicability_filter import ReplicabilityFilter +from generation_pipeline.extractors.study_data_extractor import StudyDataExtractor +from generation_pipeline.patchers.slot_filler import SlotFiller +from generation_pipeline.patchers.patch_runner import run_patch +from generation_pipeline.stage1_study_contract import apply_stage1_study_contract +from generation_pipeline.stage2_findings import annotate_stage2_findings +from generation_pipeline.stage1_verifier import verify_stage1_inventory +from generation_pipeline.stage2_verifier import verify_stage2_findings +from generation_pipeline.stage4 import build_human_study_package +from generation_pipeline.stage5 import Stage5Options, run_stage5 +from generation_pipeline.utils.output_formatter import OutputFormatter +from generation_pipeline.utils.review_parser import ReviewParser +from generation_pipeline.verification.schema_validator import summarize_report, validate_paper +from generation_pipeline.verification.stage_quality import build_stage1_quality, build_stage2_quality +from generation_pipeline.settings import AppSettings, load_settings, resolve_data_dir, resolve_llm_config, resolve_output_dir, resolve_runs_dir +from src.llm.factory import get_client + + +def paper_id_from_pdf(pdf_path: Path) -> str: + return Path(pdf_path).stem.replace(" ", "_").replace("-", "_").lower() + + +def _count_regeneration_items(regeneration: Dict[str, Any]) -> int: + count = 0 + for value in regeneration.values(): + if isinstance(value, list): + count += len(value) + elif isinstance(value, dict): + count += len(value) + elif value not in (None, "", False): + count += 1 + return count + + +def _stage1_regeneration_feedback(verification: Dict[str, Any]) -> Dict[str, Any]: + """Convert Stage 1 verifier output into filter regeneration feedback.""" + raw = verification.get("regeneration_instructions") + raw = raw if isinstance(raw, dict) else {} + feedback: Dict[str, Any] = { + "missing_studies": list(raw.get("missing_studies") or []), + "split_merge_corrections": list(raw.get("split_merge_corrections") or []), + "eligibility_corrections": list(raw.get("eligibility_corrections") or []), + } + + for item in verification.get("inventory_checks", []) or []: + if not isinstance(item, dict): + continue + verdict = str(item.get("verdict") or "").lower() + if verdict in {"", "ok"}: + continue + study = item.get("study") or "unknown study" + issue = item.get("issue") or item.get("evidence") or verdict + if verdict == "missing": + feedback["missing_studies"].append(f"{study}: {issue}") + elif verdict in {"split_issue", "merge_issue", "extra"}: + feedback["split_merge_corrections"].append({"study": study, "reason": issue}) + else: + feedback["split_merge_corrections"].append({"study": study, "reason": f"{verdict}: {issue}"}) + + for item in verification.get("eligibility_checks", []) or []: + if not isinstance(item, dict): + continue + verdict = str(item.get("verdict") or "").lower() + if verdict in {"", "ok"}: + continue + study = item.get("study") or "unknown study" + issue = item.get("issue") or item.get("evidence") or verdict + feedback["eligibility_corrections"].append( + { + "study": study, + "expected_label": item.get("expected_label") or "UNCERTAIN", + "reason": f"{verdict}: {issue}", + } + ) + + return { + "missing_studies": [item for item in feedback["missing_studies"] if item], + "split_merge_corrections": [item for item in feedback["split_merge_corrections"] if item], + "eligibility_corrections": [item for item in feedback["eligibility_corrections"] if item], + } + + +def _stage1_should_auto_refine(verification: Dict[str, Any]) -> bool: + if verification.get("status") != "ok": + return False + if verification.get("overall") not in {"pass", "needs_review", "fail"}: + return False + return _count_regeneration_items(_stage1_regeneration_feedback(verification)) > 0 + + +def _stage1_refinement_snapshot( + *, + attempt: int, + trigger: str, + verification: Dict[str, Any], + quality: Dict[str, Any], + regeneration_feedback: Dict[str, Any], +) -> Dict[str, Any]: + return { + "attempt": attempt, + "trigger": trigger, + "verifier_status": verification.get("status"), + "verifier_overall": verification.get("overall"), + "verifier_confidence": verification.get("confidence"), + "verifier_notes": verification.get("notes", ""), + "quality_needs_human_review": quality.get("needs_human_review"), + "quality_issues": quality.get("issues", [])[:20], + "regeneration_instructions": regeneration_feedback, + } + + +def _stage2_regeneration_feedback(verification: Dict[str, Any]) -> Dict[str, Any]: + """Convert verifier output into the extractor feedback shape.""" + raw = verification.get("regeneration_instructions") + raw = raw if isinstance(raw, dict) else {} + feedback: Dict[str, Any] = { + "missing_effects": list(raw.get("missing_effects") or []), + "exact_stats_needed": list(raw.get("exact_stats_needed") or []), + "data_corrections": list(raw.get("data_corrections") or []), + } + + for item in verification.get("study_coverage", []) or []: + if not isinstance(item, dict): + continue + verdict = str(item.get("verdict") or "").lower() + if verdict in {"", "ok"}: + continue + study = item.get("study") or "unknown study" + issue = item.get("issue") or item.get("evidence") or verdict + reason = f"{study}: verifier marked study coverage as {verdict}; {issue}" + if verdict == "missing": + feedback["missing_effects"].append(reason) + else: + feedback["data_corrections"].append({"path": "$.eligible_studies", "reason": reason}) + + for item in verification.get("finding_checks", []) or []: + if not isinstance(item, dict): + continue + verdict = str(item.get("verdict") or "").lower() + if verdict in {"", "ok"}: + continue + finding_id = item.get("finding_id") or "unknown finding" + issue = item.get("issue") or item.get("evidence") or verdict + reason = f"{finding_id}: verifier marked finding as {verdict}; {issue}" + if verdict == "wrong_stats": + feedback["exact_stats_needed"].append({"finding_id": finding_id, "reason": reason}) + elif verdict == "unsupported": + feedback["data_corrections"].append({"path": "$.eligible_studies[*].findings", "reason": reason}) + elif verdict == "missing": + feedback["missing_effects"].append(reason) + else: + feedback["data_corrections"].append({"path": "$.eligible_studies[*].findings", "reason": reason}) + + return { + "missing_effects": [item for item in feedback["missing_effects"] if item], + "exact_stats_needed": [item for item in feedback["exact_stats_needed"] if item], + "data_corrections": [item for item in feedback["data_corrections"] if item], + } + + +def _stage2_should_auto_refine(verification: Dict[str, Any]) -> bool: + if verification.get("status") != "ok": + return False + if verification.get("overall") not in {"pass", "needs_review", "fail"}: + return False + return _count_regeneration_items(_stage2_regeneration_feedback(verification)) > 0 + + +def _stage2_refinement_snapshot( + *, + attempt: int, + trigger: str, + verification: Dict[str, Any], + quality: Dict[str, Any], + regeneration_feedback: Dict[str, Any], +) -> Dict[str, Any]: + return { + "attempt": attempt, + "trigger": trigger, + "verifier_status": verification.get("status"), + "verifier_overall": verification.get("overall"), + "verifier_confidence": verification.get("confidence"), + "verifier_notes": verification.get("notes", ""), + "quality_needs_human_review": quality.get("needs_human_review"), + "quality_issues": quality.get("issues", [])[:20], + "regeneration_instructions": regeneration_feedback, + } + + +def _write_stage2_schema_failure( + *, + paper_dir: Path, + json_path: Path, + raw_result: Dict[str, Any], + schema_report: Any, + label: str, +) -> None: + """Persist invalid Stage 2 output so real-API failures are inspectable.""" + invalid_path = paper_dir / "stage2.invalid.json" + report_path = paper_dir / "stage2.schema_report.json" + invalid_path.write_text(json.dumps(raw_result, indent=2, ensure_ascii=False), encoding="utf-8") + report_payload = schema_report.to_dict() + report_payload["stage"] = 2 + report_payload["attempt_label"] = label + report_payload["target_json"] = str(json_path) + report_path.write_text(json.dumps(report_payload, indent=2, ensure_ascii=False), encoding="utf-8") + + +def _schema_error_preview(schema_report: Any, *, limit: int = 8) -> str: + errors = [ + issue + for issue in schema_report.to_dict().get("issues", []) + if issue.get("severity") == "error" and not issue.get("fixed") + ] + if not errors: + return "" + lines = [] + for issue in errors[:limit]: + lines.append(f" - {issue.get('path')}: {issue.get('message')}") + if len(errors) > limit: + lines.append(f" - ... {len(errors) - limit} more error(s)") + return "\n".join(lines) + + +class GenerationPipeline: + """Two-stage ethics-extraction pipeline.""" + + def __init__( + self, + provider: Optional[str] = None, + model: Optional[str] = None, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + output_dir: Optional[Path] = None, + settings: Optional[AppSettings] = None, + ): + settings = settings or load_settings() + llm_config = resolve_llm_config( + settings, + provider=provider, + model=model, + api_key=api_key, + api_base=api_base, + ) + self.provider = llm_config.provider + self.model = llm_config.model + self.client = get_client( + provider=self.provider, + model=self.model, + api_key=llm_config.api_key, + api_base=llm_config.api_base, + ) + self.output_dir = Path(output_dir) if output_dir else resolve_output_dir(settings) + self.output_dir.mkdir(parents=True, exist_ok=True) + + self.filter = ReplicabilityFilter(self.client) + self.extractor = StudyDataExtractor(self.client) + self.slot_filler = SlotFiller(self.client) + + def paper_output_dir(self, paper_id: str) -> Path: + out = self.output_dir / paper_id + out.mkdir(parents=True, exist_ok=True) + return out + + def run_stage1( + self, + pdf_path: Path, + *, + verify_inventory: bool = True, + verifier_timeout: float | None = 60.0, + auto_refine_attempts: int = 1, + ) -> Tuple[Path, Path, Dict[str, Any]]: + print(f"Running Stage 1: Replicability Filter for {pdf_path.name}") + paper_id = paper_id_from_pdf(pdf_path) + paper_dir = self.paper_output_dir(paper_id) + + md_path = paper_dir / "stage1.md" + json_path = paper_dir / "stage1.json" + + def run_attempt(regeneration_feedback: Optional[Dict[str, Any]], label: str) -> Dict[str, Any]: + result = self.filter.process(pdf_path, regeneration_instructions=regeneration_feedback) + result["paper_id"] = paper_id + apply_stage1_study_contract(result) + if verify_inventory: + print(f" Stage 1 verifier ({label}): checking study inventory coverage", flush=True) + try: + result["stage1_verification"] = verify_stage1_inventory( + result, + pdf_path, + self.client, + timeout=verifier_timeout, + ) + except Exception as exc: + result["stage1_verification"] = { + "version": "stage1_verifier_v1", + "status": "error", + "overall": "needs_review", + "error": f"{type(exc).__name__}: {exc}", + "inventory_checks": [], + "eligibility_checks": [], + "regeneration_instructions": { + "missing_studies": [], + "split_merge_corrections": [], + "eligibility_corrections": [], + }, + } + else: + result["stage1_verification"] = { + "version": "stage1_verifier_v1", + "status": "skipped", + "overall": "needs_review", + "notes": "Disabled by --no-stage1-verifier.", + "inventory_checks": [], + "eligibility_checks": [], + "regeneration_instructions": { + "missing_studies": [], + "split_merge_corrections": [], + "eligibility_corrections": [], + }, + } + result["stage1_quality"] = build_stage1_quality(result) + return result + + result = run_attempt(None, "initial") + refinement_history = [] + max_refines = max(0, int(auto_refine_attempts or 0)) + for attempt in range(1, max_refines + 1): + verification = result.get("stage1_verification") if isinstance(result.get("stage1_verification"), dict) else {} + if not verify_inventory or not _stage1_should_auto_refine(verification): + break + feedback = _stage1_regeneration_feedback(verification) + refinement_history.append( + _stage1_refinement_snapshot( + attempt=attempt, + trigger="stage1_verifier", + verification=verification, + quality=result.get("stage1_quality") if isinstance(result.get("stage1_quality"), dict) else {}, + regeneration_feedback=feedback, + ) + ) + print( + f" Stage 1 auto-refine {attempt}/{max_refines}: rerunning with " + f"{_count_regeneration_items(feedback)} verifier feedback item(s)", + flush=True, + ) + result = run_attempt(feedback, f"auto_refine_{attempt}") + + if refinement_history: + result["stage1_refinement"] = { + "version": "stage1_auto_refine_v1", + "attempts": len(refinement_history), + "max_attempts": max_refines, + "final_verifier_overall": result.get("stage1_verification", {}).get("overall") + if isinstance(result.get("stage1_verification"), dict) + else None, + } + result["stage1_refinement_history"] = refinement_history + + md_content = OutputFormatter.format_stage1_review(result) + + md_path.write_text(md_content, encoding="utf-8") + json_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") + print(f"✓ Stage 1 complete.\n MD : {md_path}\n JSON: {json_path}") + return md_path, json_path, result + + def check_stage1_review(self, review_file: Path) -> Dict[str, Any]: + parsed = ReviewParser.parse(review_file) + return { + "action": ReviewParser.get_action(parsed["review_status"]), + "review_status": parsed["review_status"], + "comments": parsed["comments"], + "checklists": parsed["checklists"], + } + + def run_stage2( + self, + stage1_json_path: Path, + pdf_path: Path, + regeneration_instructions_path: Optional[Path] = None, + *, + grounded: bool = False, + ground_threshold: float = 90.0, + ground_k: int = 8, + ground_timeout: float | None = 60.0, + ground_workers: int = 4, + verify_findings: bool = True, + verifier_timeout: float | None = 60.0, + auto_refine_attempts: int = 1, + ) -> Tuple[Path, Path, Dict[str, Any]]: + print(f"Running Stage 2: Ethics Effect Extraction for {pdf_path.name}") + if not stage1_json_path.exists(): + raise FileNotFoundError(f"Stage1 JSON not found: {stage1_json_path}") + stage1_result = json.loads(stage1_json_path.read_text(encoding="utf-8")) + if not isinstance(stage1_result, dict): + raise ValueError(f"Stage1 result not a dict: {type(stage1_result)}") + + regen = None + if regeneration_instructions_path: + if regeneration_instructions_path.exists(): + regen = json.loads(regeneration_instructions_path.read_text(encoding="utf-8")) + print(f"Using regeneration feedback: {regeneration_instructions_path.name}") + else: + print(f"Warning: regeneration file not found: {regeneration_instructions_path}") + + if grounded: + print( + " Stage 2 legacy grounding: checking per-effect material slots " + "(disabled by default; Stage 3 owns study-level materials)", + flush=True, + ) + + paper_id = stage1_result.get("paper_id", "unknown") + paper_dir = self.paper_output_dir(str(paper_id)) + md_path = paper_dir / "stage2.md" + json_path = paper_dir / "stage2.json" + + def run_attempt(regeneration_feedback: Optional[Dict[str, Any]], label: str) -> Dict[str, Any]: + raw_result = self.extractor.process( + stage1_result, + pdf_path, + regeneration_instructions=regeneration_feedback, + grounded=grounded, + ground_threshold=ground_threshold, + ground_k=ground_k, + ground_timeout=ground_timeout, + ground_workers=ground_workers, + ) + result, schema_report = validate_paper(raw_result, repair=True, path=json_path) + if not schema_report.valid: + _write_stage2_schema_failure( + paper_dir=paper_dir, + json_path=json_path, + raw_result=raw_result, + schema_report=schema_report, + label=label, + ) + preview = _schema_error_preview(schema_report) + if preview: + print(f"Stage 2 schema validation errors ({label}):\n{preview}", flush=True) + print( + "Stage 2 invalid artifacts written:\n" + f" JSON : {paper_dir / 'stage2.invalid.json'}\n" + f" Report: {paper_dir / 'stage2.schema_report.json'}", + flush=True, + ) + raise ValueError(f"Stage 2 schema validation failed: {summarize_report(schema_report)}") + if schema_report.changed or schema_report.issues: + print(f"Stage 2 schema validation ({label}): {summarize_report(schema_report)}") + + result = annotate_stage2_findings(result, stage1_json=stage1_result) + if verify_findings: + print(f" Stage 2 verifier ({label}): checking study/finding coverage", flush=True) + try: + result["stage2_verification"] = verify_stage2_findings( + result, + stage1_result, + pdf_path, + self.client, + timeout=verifier_timeout, + ) + except Exception as exc: + result["stage2_verification"] = { + "version": "stage2_verifier_v1", + "status": "error", + "overall": "needs_review", + "error": f"{type(exc).__name__}: {exc}", + "study_coverage": [], + "finding_checks": [], + "regeneration_instructions": { + "missing_effects": [], + "exact_stats_needed": [], + "data_corrections": [], + }, + } + else: + result["stage2_verification"] = { + "version": "stage2_verifier_v1", + "status": "skipped", + "overall": "needs_review", + "notes": "Disabled by --no-stage2-verifier.", + "study_coverage": [], + "finding_checks": [], + "regeneration_instructions": { + "missing_effects": [], + "exact_stats_needed": [], + "data_corrections": [], + }, + } + result["stage2_quality"] = build_stage2_quality(result, stage1_result) + return result + + result = run_attempt(regen, "initial") + refinement_history = [] + max_refines = max(0, int(auto_refine_attempts or 0)) + for attempt in range(1, max_refines + 1): + verification = result.get("stage2_verification") if isinstance(result.get("stage2_verification"), dict) else {} + if not verify_findings or not _stage2_should_auto_refine(verification): + break + feedback = _stage2_regeneration_feedback(verification) + refinement_history.append( + _stage2_refinement_snapshot( + attempt=attempt, + trigger="stage2_verifier", + verification=verification, + quality=result.get("stage2_quality") if isinstance(result.get("stage2_quality"), dict) else {}, + regeneration_feedback=feedback, + ) + ) + print( + f" Stage 2 auto-refine {attempt}/{max_refines}: rerunning with " + f"{_count_regeneration_items(feedback)} verifier feedback item(s)", + flush=True, + ) + result = run_attempt(feedback, f"auto_refine_{attempt}") + + if refinement_history: + result["stage2_refinement"] = { + "version": "stage2_auto_refine_v1", + "attempts": len(refinement_history), + "max_attempts": max_refines, + "final_verifier_overall": result.get("stage2_verification", {}).get("overall") + if isinstance(result.get("stage2_verification"), dict) + else None, + } + result["stage2_refinement_history"] = refinement_history + + md_content = OutputFormatter.format_stage2_review(result) + + md_path.write_text(md_content, encoding="utf-8") + json_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") + print(f"✓ Stage 2 complete.\n MD : {md_path}\n JSON: {json_path}") + return md_path, json_path, result + + def check_stage2_review(self, review_file: Path) -> Dict[str, Any]: + parsed = ReviewParser.parse(review_file) + return { + "action": ReviewParser.get_action(parsed["review_status"]), + "review_status": parsed["review_status"], + "comments": parsed["comments"], + "checklists": parsed["checklists"], + } + + # ------------------------------------------------------------------ + # Stage 3 — patch existing JSON corpus (fill slots + upgrade sample) + # ------------------------------------------------------------------ + def run_stage3( + self, + json_dir: Path, + pdf_dir: Path, + *, + only: Optional[list] = None, + overwrite_filled: bool = False, + backup: bool = True, + dry_run: bool = False, + use_fetched_sources: bool = True, + source_dirs: Optional[list] = None, + ) -> Dict[str, str]: + print(f"Running Stage 3: Patch slots in {json_dir} using PDFs from {pdf_dir}") + results = run_patch( + Path(json_dir), + Path(pdf_dir), + filler=self.slot_filler, + only=only, + overwrite_filled=overwrite_filled, + backup=backup, + dry_run=dry_run, + use_fetched_sources=use_fetched_sources, + source_dirs=[Path(item) for item in source_dirs] if source_dirs else None, + ) + ok = sum(1 for v in results.values() if v == "ok" or v == "dry-run-ok") + print(f"✓ Stage 3 complete: {ok}/{len(results)} papers patched.") + return results + + # ------------------------------------------------------------------ + # Stage 3 (OSF) - assemble per-study materials from OSF instruments + # ------------------------------------------------------------------ + def run_stage3_paper( + self, + paper_dir: Path, + *, + stage2_path: Optional[Path] = None, + pdf_path: Optional[Path] = None, + osf_files_dir: Optional[Path] = None, + slot_fill: bool = False, + select_studies: bool = True, + backup: bool = True, + write: bool = True, + allow_effect_slot_fallback: bool = False, + selection_votes: int = 3, + selection_timeout: Optional[float] = 60.0, + pdf_material_timeout: Optional[float] = 120.0, + ) -> Dict[str, Any]: + """Build canonical Stage 3 artifacts for one paper directory. + + This is the OSF-linker path: it writes `stage3.json` and `stage3.md` + without rewriting `stage2.json`, then adds `study_materials`, + consolidation annotations, and simulation-target metadata. + """ + from generation_pipeline.stage3 import run_stage3 as _run_stage3 + + print(f"Running Stage 3 (OSF material assembly): {paper_dir}") + result = _run_stage3( + Path(paper_dir), + stage2_path=stage2_path, + pdf_path=pdf_path, + osf_files_dir=osf_files_dir, + filler=self.slot_filler if slot_fill else None, + llm_client=self.client if select_studies else None, + backup=backup, + write=write, + allow_effect_slot_fallback=allow_effect_slot_fallback, + selection_votes=selection_votes, + selection_timeout=selection_timeout, + pdf_material_timeout=pdf_material_timeout, + ) + materials = result["materials"] + ready = sum(1 for m in materials.values() if m["readiness"]["ready"]) + kept = sum(1 for m in materials.values() if m.get("selection", {}).get("keep", True)) + print( + f"✓ Stage 3 complete: wrote {Path(result['stage3_json']).name} + " + f"{Path(result['stage3_md']).name}; {ready}/{len(materials)} ready, " + f"{kept}/{len(materials)} selected." + ) + return result + + def build_osf_study_materials( + self, + paper_dir: Path, + *, + select_studies: bool = True, + write: bool = True, + allow_effect_slot_fallback: bool = False, + osf_files_dir: Optional[Path] = None, + ) -> Dict[str, Any]: + """Rebuild study-level materials in an existing `stage3.json`.""" + from generation_pipeline.stage3 import build_study_materials + + print(f"Running Stage 3 (OSF): assembling materials for {paper_dir}") + materials = build_study_materials( + Path(paper_dir), + osf_files_dir=osf_files_dir, + llm_client=self.client if select_studies else None, + write=write, + allow_effect_slot_fallback=allow_effect_slot_fallback, + ) + ready = sum(1 for m in materials.values() if m["readiness"]["ready"]) + kept = sum(1 for m in materials.values() if m.get("selection", {}).get("keep", True)) + print( + f"✓ Stage 3 (OSF) complete: {ready}/{len(materials)} ready; " + f"{kept}/{len(materials)} selected for simulation." + ) + return materials + + # ------------------------------------------------------------------ + # Stage 4 - build HumanStudy-Bench study package + # ------------------------------------------------------------------ + def run_stage4( + self, + json_path: Path, + *, + data_dir: Path, + study_dir: Optional[Path] = None, + study_id: Optional[str] = None, + pdf_path: Optional[Path] = None, + max_repair_iters: int = 3, + use_llm: bool = True, + generate_config: bool = True, + update_registry: bool = True, + ) -> Dict[str, Any]: + del max_repair_iters + print(f"Running Stage 4: Build HumanStudy-Bench study package for {json_path}") + summary = build_human_study_package( + Path(json_path), + data_dir=Path(data_dir), + study_dir=Path(study_dir) if study_dir else None, + study_id=study_id, + pdf_path=pdf_path, + provider=self.provider, + model=self.model, + api_key=self.client.api_key, + api_base=self.client.api_base, + use_llm=use_llm, + selection_llm_client=self.client if use_llm else None, + generate_config=generate_config, + update_registry=update_registry, + ) + print( + "✓ Stage 4 complete: " + f"study_id={summary['study_id']} json_generation={summary['json_generation']} " + f"config={summary['config_status']}" + ) + print(f" Study package: {summary['study_dir']}") + return summary + + # Stage 5 - run HumanStudy-Bench participant simulation + def run_stage5( + self, + study: str | Path, + *, + data_dir: Path | None = None, + runs_dir: Path | None = None, + models: list[str] | None = None, + options: Stage5Options | None = None, + ) -> Dict[str, Any]: + model_names = models or [self.model] + + stage5_options = options or Stage5Options() + stage5_options.api_key = stage5_options.api_key or self.client.api_key + stage5_options.api_base = stage5_options.api_base or self.client.api_base + + print(f"Running Stage 5: HumanStudy-Bench simulation for {study}") + summary = run_stage5( + study, + runs_dir=Path(runs_dir) if runs_dir else resolve_runs_dir(), + models=model_names, + options=stage5_options, + data_dir=Path(data_dir) if data_dir else resolve_data_dir(), + ) + print( + "✓ Stage 5 complete: " + f"study_id={summary['study_id']} models={summary['completed']} " + f"runs={summary['run_count']} use_real_llm={summary['use_real_llm']}" + ) + return summary diff --git a/generation_pipeline/pipeline_interface.py b/generation_pipeline/pipeline_interface.py new file mode 100644 index 0000000..9fbeb69 --- /dev/null +++ b/generation_pipeline/pipeline_interface.py @@ -0,0 +1,20 @@ +""" +Pipeline interface for ai-ethics generation stages. + +Stages: + 1 - Replicability filter (PDF → stage1 review JSON+MD) + 2 - Study/finding extraction (stage1 + PDF → stage2 review JSON+MD) + 3 - Source/material evidence assembly (stage2 + PDF/OSF → stage3 JSON+MD) + 4 - HumanStudy-Bench package draft (stage3 → metadata/spec/ground_truth/materials) + 5 - HumanStudy-Bench simulation run +""" + +__all__ = ["STAGES"] + +STAGES = { + 1: "Filter (replicability + ethics relevance)", + 2: "Extraction (study-level findings/effects/statistics)", + 3: "Source/material evidence assembly (study-level fields)", + 4: "HumanStudy-Bench study package draft", + 5: "HumanStudy-Bench simulation", +} diff --git a/generation_pipeline/run.py b/generation_pipeline/run.py new file mode 100644 index 0000000..5ef1bc1 --- /dev/null +++ b/generation_pipeline/run.py @@ -0,0 +1,625 @@ +""" +CLI for ai-ethics Generation Pipeline (Stages 1 & 2). + +Usage: + # Stage 1 — filter a single PDF in current dir or data/papers/ + python generation_pipeline/run.py --stage 1 --pdf "data/papers/2024_Xu_Petty.pdf" + + # Stage 1 source discovery/fetch — find OSF links from Stage 1 JSON/PDF and fetch files + python generation_pipeline/run.py --stage 1 --pdf "data/papers/2024_Xu_Petty.pdf" --fetch-sources + + # Stage 2 — extract effects (uses latest stage1 JSON for the paper) + python generation_pipeline/run.py --stage 2 --pdf "data/papers/2024_Xu_Petty.pdf" + + # Refine: re-run stage based on a manually-edited review .md + python generation_pipeline/run.py --stage 2 --pdf --refine + + # Stage 4 - build HumanStudy-Bench study package + python generation_pipeline/run.py --stage 4 --json outputs//stage3.json --study-id study_my_paper + + # Stage 5 - simulate participants with the generated HumanStudy-Bench adapter + python generation_pipeline/run.py --stage 5 --study-id study_my_paper --mock-agent + + # Verification gate for already extracted JSON + python generation_pipeline/run.py --verify --json stage4/runnable//*.json --pdf stage4/runnable//*.pdf +""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + + +def find_pdf(pdf_arg: str | list[str] | None) -> Path: + if isinstance(pdf_arg, list): + if len(pdf_arg) == 0: + pdf_arg = None + elif len(pdf_arg) == 1: + pdf_arg = pdf_arg[0] + else: + pdf_arg = " ".join(pdf_arg) + if pdf_arg: + p = Path(pdf_arg) + if p.exists(): + return p + raise FileNotFoundError(f"PDF not found: {p}") + # Fallback: look in data/papers/ then cwd + for d in (Path("data/papers"), Path.cwd()): + if d.exists(): + pdfs = list(d.glob("*.pdf")) + if len(pdfs) == 1: + return pdfs[0] + if len(pdfs) > 1: + raise ValueError(f"Multiple PDFs in {d}, use --pdf to pick one.") + raise FileNotFoundError("No PDF found. Pass --pdf .") + + +def find_latest_stage_file(stage: int, output_dir: Path, paper_id: str | None = None) -> Path: + filename = f"stage{stage}.json" + candidates: list[Path] = [output_dir / filename] + if paper_id: + candidates.insert(0, output_dir / paper_id / filename) + else: + candidates.extend(output_dir.glob(f"*/{filename}")) + files = [path for path in candidates if path.is_file()] + if not files: + raise FileNotFoundError( + f"No canonical {filename} in {output_dir} (paper_id={paper_id})" + ) + return max(files, key=lambda p: p.stat().st_mtime) + + +def print_source_fetch_results(results: list[dict]) -> None: + ok = sum(1 for item in results if item["summary"]["errors"] == 0) + for item in results: + summary = item["summary"] + reused = summary.get("reused_files", 0) + reused_part = f" reused={reused}" if reused else "" + print( + f"{Path(item['json_path']).parent.name}: " + f"plans={summary['plans']} fetched={summary['fetched_files']} " + f"errors={summary['errors']}{reused_part} sources={item['sources_dir']}" + ) + print(f"Source fetch complete: {ok}/{len(results)} paper(s) without connector errors.") + + +def _stage4_llm_requested(settings) -> bool: + for section_name in ("stage4_llm", "config_llm", "gym_llm"): + section = settings.section(section_name) + for key in ("provider", "model", "api_key", "base_url", "api_base"): + value = section.get(key) + if value not in (None, ""): + return True + return False + + +def _first_cli_or_setting(*values): + for value in values: + if value is None: + continue + if isinstance(value, str) and value.strip() == "": + continue + return value + return None + + +def _optional_float(value): + if value is None: + return None + if isinstance(value, str) and value.strip().lower() in {"", "null", "none", "default", "default(unset)"}: + return None + return float(value) + + +def main(): + parser = argparse.ArgumentParser(description="ai-ethics extraction pipeline (stages 1, 2, 3, 4, 5)") + parser.add_argument("--settings", type=Path, help="Path to settings YAML/JSON (default: config/settings.yaml if present)") + parser.add_argument("--stage", choices=["1", "2", "3", "4", "5"], + help="1=Filter, 2=Extraction, 3=Patch, 4=Human study package, 5=Human simulation") + parser.add_argument("--pdf", nargs="*", help="Path to PDF file(s)") + parser.add_argument("--json", nargs="+", help="Per-paper JSON file(s), used by --verify") + parser.add_argument( + "--fetch-sources", + action="store_true", + help=( + "Fetch OSF sources. With --stage, this is Stage 1 only; without --stage, " + "fetch explicit --json/--pdf pairs or stage4 corpus files." + ), + ) + parser.add_argument("--stage4-dir", type=Path, default=Path("stage4"), help="Root stage4 corpus directory") + parser.add_argument("--osf-token", default=None, help="OSF API bearer token; defaults to OSF_TOKEN env var") + parser.add_argument("--verify", action="store_true", help="Run schema + hallucination verification gate") + parser.add_argument("--verification-report", type=Path, help="Optional aggregate verification report path") + parser.add_argument("--source", nargs="*", default=[], help="Additional source text files for --verify") + parser.add_argument("--source-dir", nargs="*", default=[], help="Additional source directories for --verify") + parser.add_argument("--refine", action="store_true", help="Re-run stage after manual review") + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--experiments-dir", type=Path, help="Deprecated alias for Stage 4 data directory") + parser.add_argument("--data-dir", type=Path, help="HumanStudy-Bench data directory") + parser.add_argument( + "--hub-layout", + action="store_true", + help="Stage 4: write a HumanStudy-Bench-Hub study folder with source/ and scripts/", + ) + parser.add_argument( + "--hub-studies-dir", + type=Path, + default=Path("studies"), + help="Stage 4 --hub-layout output directory for studies/", + ) + parser.add_argument("--runs-dir", type=Path, help="Stage 5 run output directory") + parser.add_argument("--study-id", help="HumanStudy-Bench study id for Stage 4/5") + parser.add_argument("--provider", choices=["openai", "google", "anthropic", "openrouter", "vllm"]) + parser.add_argument("--model") + parser.add_argument("--api-key", default=None) + parser.add_argument("--api-base", "--base-url", dest="api_base", default=None) + parser.add_argument("--regeneration-instructions", type=str, help="Path to regeneration feedback JSON (stage 2)") + parser.add_argument("--no-stage1-verifier", action="store_true", help="Stage 1: skip LLM study-inventory verifier pass") + parser.add_argument( + "--stage1-verifier-timeout", + type=float, + default=60.0, + help="Stage 1 verifier request timeout in seconds; <=0 disables timeout", + ) + parser.add_argument( + "--stage1-auto-refine-attempts", + type=int, + default=1, + help="Stage 1: automatically rerun inventory extraction with verifier feedback this many times", + ) + parser.add_argument( + "--no-stage1-auto-refine", + action="store_true", + help="Stage 1: disable automatic verifier-feedback reruns", + ) + parser.add_argument( + "--stage2-grounding", + action="store_true", + help="Legacy Stage 2: run slow per-effect material-slot grounding. Disabled by default; Stage 3 owns material evidence.", + ) + parser.add_argument( + "--no-grounding", + action="store_true", + help="Deprecated compatibility flag; Stage 2 grounding is already disabled unless --stage2-grounding is set.", + ) + parser.add_argument("--no-stage2-verifier", action="store_true", help="Stage 2: skip LLM study/finding verifier pass") + parser.add_argument( + "--stage2-verifier-timeout", + type=float, + default=60.0, + help="Stage 2 verifier request timeout in seconds; <=0 disables timeout", + ) + parser.add_argument( + "--stage2-auto-refine-attempts", + type=int, + default=1, + help="Stage 2: automatically rerun extraction with verifier feedback this many times", + ) + parser.add_argument( + "--no-stage2-auto-refine", + action="store_true", + help="Stage 2: disable automatic verifier-feedback reruns", + ) + parser.add_argument("--ground-threshold", type=float, default=90.0, help="Legacy Stage 2 grounding verbatim threshold") + parser.add_argument("--ground-k", type=int, default=8, help="Legacy Stage 2 grounding retrieval chunks per slot") + parser.add_argument( + "--ground-timeout", + type=float, + default=60.0, + help="Legacy Stage 2 helper timeout in seconds for each grounding LLM call; <=0 disables timeout", + ) + parser.add_argument( + "--ground-workers", + type=int, + default=4, + help="Legacy Stage 2 parallel grounding workers; use 1 for serial execution", + ) + # Stage 3 options + parser.add_argument("--json-dir", type=str, help="Directory of per-paper JSONs to patch (stage 3)") + parser.add_argument("--pdf-dir", type=str, help="Directory of PDFs to pair with JSONs (stage 3)") + parser.add_argument( + "--osf-dir", + action="append", + type=Path, + default=None, + help=( + "OSF/source directory for Stage 3. May point to a sources/ directory " + "containing combined_sources.txt or a paper directory containing sources/. " + "Repeatable. If omitted, Stage 3 auto-checks JSON and PDF sibling sources/." + ), + ) + parser.add_argument("--only", action="append", default=None, + help="Filename substring filter — only patch matching JSONs (repeatable, stage 3)") + parser.add_argument("--overwrite-filled", action="store_true", + help="Re-run already-filled slots (default: preserve existing fills)") + parser.add_argument("--no-backup", action="store_true", help="Skip .bak backups when overwriting") + parser.add_argument("--no-extract-text", action="store_true", help="Skip source text extraction after fetching") + parser.add_argument("--dry-run", action="store_true", help="Pair + call LLM but don't write back") + parser.add_argument( + "--stage3-slot-fill", + action="store_true", + help="Legacy Stage 3: run PDF slot filling before source/material assembly", + ) + parser.add_argument( + "--allow-effect-slot-fallback", + action="store_true", + help="Legacy Stage 3: allow per-effect materials/manipulation/items slots as material fallback", + ) + parser.add_argument( + "--stage3-select-votes", + type=int, + default=3, + help="Stage 3 LLM study-selection votes; lower to 1 for small smoke tests", + ) + parser.add_argument( + "--stage3-select-timeout", + type=float, + default=60.0, + help="Stage 3 per-call LLM study-selection timeout in seconds; <=0 disables timeout", + ) + parser.add_argument( + "--stage3-pdf-timeout", + type=float, + default=120.0, + help="Stage 3 PDF material extraction timeout in seconds; <=0 disables timeout", + ) + parser.add_argument("--max-repair-iters", type=int, default=3, help="Deprecated old gym Stage 4 option; ignored") + parser.add_argument("--no-generate-config", action="store_true", help="Stage 4: skip LLM-generated StudyConfig adapter") + # Stage 5 options + parser.add_argument("--experiment", type=Path, help="Deprecated Stage 5 alias for --study-id or study directory") + parser.add_argument("--sim-models", nargs="+", help="Stage 5 agent model(s); defaults to stage5_llm.model") + parser.add_argument("--n-agents", type=int, help="Override Stage 5 agent count") + parser.add_argument("--max-agents", type=int, help="Explicit Stage 5 cost cap; omitted means no cap") + parser.add_argument("--repeats", type=int, help="Stage 5 repeats per persona") + parser.add_argument("--temperature", type=float, help="Stage 5 agent LLM temperature") + parser.add_argument("--seed", type=int, help="Stage 5 deterministic seed") + parser.add_argument("--careless-rate", type=float, help="Deprecated old gym Stage 5 option; ignored") + parser.add_argument("--parse-failure-threshold", type=float, help="Deprecated old gym Stage 5 option; ignored") + parser.add_argument("--min-condition-n", type=int, help="Deprecated old gym Stage 5 option; ignored") + parser.add_argument("--include-partial", action="store_true", help="Deprecated old gym Stage 5 option; ignored") + parser.add_argument("--mock-agent", action="store_true", help="Stage 5 use deterministic mock policy instead of LLM") + parser.add_argument("--num-workers", type=int, help="Stage 5 parallel participant workers") + parser.add_argument("--use-cache", action="store_true", help="Stage 5 cache raw repeat results") + parser.add_argument("--cache-dir", type=Path, help="Stage 5 cache directory") + parser.add_argument("--profiles-json", type=Path, help="Stage 5 participant profiles JSON") + parser.add_argument("--system-prompt-file", type=Path, help="Stage 5 custom system prompt file") + parser.add_argument("--system-prompt-preset", help="Stage 5 system prompt preset") + parser.add_argument("--reasoning", help="Stage 5 reasoning setting for compatible models") + parser.add_argument("--enable-reasoning", action="store_true", help="Stage 5 force reasoning for compatible models") + args = parser.parse_args() + + from generation_pipeline.settings import ( + load_settings, + resolve_data_dir, + resolve_output_dir, + resolve_runs_dir, + resolve_stage_llm_config, + resolve_verification_threshold, + ) + + settings = load_settings(args.settings) + output_dir = resolve_output_dir(settings, args.output_dir) + verification_threshold = resolve_verification_threshold(settings) + + if args.fetch_sources and not args.stage: + from generation_pipeline.connectors.registry import fetch_json_sources, fetch_stage4_sources + from src.llm.factory import get_client as _get_client + + # Build an LLM client for link-discovery fallback (papers with no explicit OSF URL). + try: + _llm_cfg = resolve_stage_llm_config( + settings, + stage=1, + provider=args.provider, + model=args.model, + api_key=args.api_key, + api_base=args.api_base, + ) + _llm_client = _get_client(provider=_llm_cfg.provider, model=_llm_cfg.model, + api_key=_llm_cfg.api_key, api_base=_llm_cfg.api_base) + except Exception: + _llm_client = None # no key configured — skip LLM fallback silently + + if args.json: + results = fetch_json_sources( + [Path(item) for item in args.json], + pdf_paths=[Path(item) for item in (args.pdf or [])], + token=args.osf_token, + dry_run=args.dry_run, + extract_text=not args.no_extract_text, + output_dir=args.output_dir, + llm_client=_llm_client, + ) + else: + results = fetch_stage4_sources( + args.stage4_dir, + only=args.only, + token=args.osf_token, + dry_run=args.dry_run, + extract_text=not args.no_extract_text, + llm_client=_llm_client, + ) + print_source_fetch_results(results) + if any(item["summary"]["errors"] for item in results): + sys.exit(1) + return + + if args.verify: + if not args.json: + raise SystemExit("--verify requires --json [more.json]") + from generation_pipeline.verification.verbatim_verifier import verify_files + + aggregate = verify_files( + [Path(item) for item in args.json], + pdf_paths=[Path(item) for item in (args.pdf or [])], + source_paths=[Path(item) for item in args.source], + source_dirs=[Path(item) for item in args.source_dir], + threshold=verification_threshold, + write=not args.dry_run, + backup=not args.no_backup, + aggregate_report_path=args.verification_report, + repair=not args.dry_run, + ) + print(f"Verification complete: {aggregate['summary']}") + if aggregate["summary"]["failed"]: + sys.exit(1) + return + + if not args.stage: + parser.error("--stage is required unless --verify is used") + + stage = int(args.stage) + if args.fetch_sources and stage != 1: + raise SystemExit( + "--fetch-sources belongs to Stage 1 in the staged pipeline. " + "Run `--stage 1 --fetch-sources` before Stage 2, or use standalone " + "`--fetch-sources --json --pdf ` for old JSONs." + ) + + if stage == 5: + study_ref = args.study_id or args.experiment + if not study_ref: + raise SystemExit("Stage 5 requires --study-id or --experiment ") + from generation_pipeline.stage5 import Stage5Options, run_stage5 + + runs_dir = resolve_runs_dir(settings, args.runs_dir) + stage5_cfg = resolve_stage_llm_config( + settings, + stage=5, + provider=args.provider, + model=args.model, + api_key=args.api_key, + api_base=args.api_base, + ) + models = args.sim_models or [stage5_cfg.model] + + stage5_settings = settings.section("stage5") + options = Stage5Options( + n_participants=_first_cli_or_setting(args.n_agents, stage5_settings.get("n_participants")), + n_agents=_first_cli_or_setting(args.n_agents, stage5_settings.get("n_agents")), + max_agents=_first_cli_or_setting(args.max_agents, stage5_settings.get("max_agents")), + repeats=int(_first_cli_or_setting(args.repeats, stage5_settings.get("repeats"), 1)), + temperature=_optional_float(_first_cli_or_setting(args.temperature, stage5_settings.get("temperature"))) or 1.0, + seed=int(_first_cli_or_setting(args.seed, stage5_settings.get("seed"), 42)), + dry_run=args.dry_run, + mock=args.mock_agent, + num_workers=_first_cli_or_setting(args.num_workers, stage5_settings.get("num_workers")), + use_cache=bool(args.use_cache or stage5_settings.get("use_cache", False)), + cache_dir=_first_cli_or_setting(args.cache_dir, stage5_settings.get("cache_dir")), + profiles_json=_first_cli_or_setting(args.profiles_json, stage5_settings.get("profiles_json")), + system_prompt_file=_first_cli_or_setting(args.system_prompt_file, stage5_settings.get("system_prompt_file")), + system_prompt_preset=_first_cli_or_setting( + args.system_prompt_preset, + stage5_settings.get("system_prompt_preset"), + "v3_human_plus_demo", + ), + reasoning=_first_cli_or_setting(args.reasoning, stage5_settings.get("reasoning"), "default"), + enable_reasoning=bool(args.enable_reasoning or stage5_settings.get("enable_reasoning", False)), + api_key=stage5_cfg.api_key, + api_base=stage5_cfg.api_base, + ) + print(f"Running Stage 5: HumanStudy-Bench simulation for {study_ref}") + summary = run_stage5( + study_ref, + runs_dir=runs_dir, + models=models, + options=options, + data_dir=resolve_data_dir(settings, args.data_dir), + ) + print( + "✓ Stage 5 complete: " + f"study_id={summary['study_id']} models={summary['completed']} " + f"runs={summary['run_count']} use_real_llm={summary['use_real_llm']}" + ) + print(f" Runs: {runs_dir / summary['study_id']}") + return + + if stage == 4: + if not args.json: + raise SystemExit("Stage 4 requires --json [more.json]") + if args.study_id and len(args.json) > 1: + raise SystemExit("--study-id can only be used with one Stage 4 JSON input") + from generation_pipeline.stage4 import build_human_study_package, paper_local_stage4_dir + from src.llm.factory import get_client as _get_client + + explicit_data_dir = args.experiments_dir or args.data_dir + if args.hub_layout: + data_dir = args.hub_studies_dir + else: + data_dir = resolve_data_dir(settings, explicit_data_dir) if explicit_data_dir else None + stage4_generation_llm_requested = _stage4_llm_requested(settings) or any( + value is not None for value in (args.provider, args.model, args.api_key, args.api_base) + ) + _llm_cfg = resolve_stage_llm_config( + settings, + stage=4, + provider=args.provider, + model=args.model, + api_key=args.api_key, + api_base=args.api_base, + ) + _selection_llm_client = None + if _llm_cfg.api_key: + _selection_llm_client = _get_client( + provider=_llm_cfg.provider, + model=_llm_cfg.model, + api_key=_llm_cfg.api_key, + api_base=_llm_cfg.api_base, + ) + pdf_path = None + if args.pdf and len(args.pdf) == 1: + pdf_path = Path(args.pdf[0]) + for json_item in args.json: + json_path = Path(json_item) + if args.hub_layout or data_dir: + local_study_dir = None + else: + local_study_dir = paper_local_stage4_dir(json_path, args.study_id) + print(f"Running Stage 4: Build HumanStudy-Bench study package for {json_item}") + summary = build_human_study_package( + json_path, + data_dir=data_dir or json_path.parent, + study_dir=local_study_dir, + study_id=args.study_id, + pdf_path=pdf_path if pdf_path and pdf_path.exists() else None, + provider=_llm_cfg.provider if _llm_cfg else (args.provider or "openai"), + model=_llm_cfg.model if _llm_cfg else (args.model or "gpt-4o-mini"), + api_key=_llm_cfg.api_key if _llm_cfg else args.api_key, + api_base=_llm_cfg.api_base if _llm_cfg else args.api_base, + use_llm=stage4_generation_llm_requested and _selection_llm_client is not None, + selection_llm_client=_selection_llm_client, + generate_config=not args.no_generate_config, + update_registry=bool(data_dir) and not args.hub_layout, + hub_layout=args.hub_layout, + ) + print( + "✓ Stage 4 complete: " + f"study_id={summary['study_id']} json_generation={summary['json_generation']} " + f"config={summary['config_status']}" + ) + print(f" Study package: {summary['study_dir']}") + return + + from generation_pipeline.pipeline import GenerationPipeline, paper_id_from_pdf + llm_config = resolve_stage_llm_config( + settings, + stage=stage, + provider=args.provider, + model=args.model, + api_key=args.api_key, + api_base=args.api_base, + ) + pipeline = GenerationPipeline( + provider=llm_config.provider, + model=llm_config.model, + api_key=llm_config.api_key, + api_base=llm_config.api_base, + output_dir=output_dir, + settings=settings, + ) + + if stage == 3: + stage2_path = Path(args.json[0]) if args.json else None + pdf_path = find_pdf(args.pdf) if args.pdf else None + if stage2_path: + paper_dir = stage2_path.parent + elif pdf_path: + paper_dir = output_dir / paper_id_from_pdf(pdf_path) + elif args.json_dir or args.pdf_dir: + raise SystemExit( + "Legacy Stage 3 corpus patching is not the default OSF-linker path. " + "Run Stage 3 with --pdf or --json /stage2.json." + ) + else: + raise SystemExit("Stage 3 requires --pdf or --json /stage2.json") + try: + pipeline.run_stage3_paper( + paper_dir=paper_dir, + stage2_path=stage2_path, + pdf_path=pdf_path, + osf_files_dir=args.osf_dir[0] if args.osf_dir else None, + slot_fill=args.stage3_slot_fill, + select_studies=True, + backup=not args.no_backup, + write=not args.dry_run, + allow_effect_slot_fallback=args.allow_effect_slot_fallback, + selection_votes=args.stage3_select_votes, + selection_timeout=args.stage3_select_timeout if args.stage3_select_timeout > 0 else None, + pdf_material_timeout=args.stage3_pdf_timeout if args.stage3_pdf_timeout > 0 else None, + ) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + import traceback; traceback.print_exc() + sys.exit(1) + return + + pdf_path = find_pdf(args.pdf) + paper_id = paper_id_from_pdf(pdf_path) + + try: + if stage == 1: + if args.refine: + review = find_latest_stage_file(1, output_dir, paper_id=paper_id).with_suffix(".md") + if not review.exists(): + raise FileNotFoundError(f"Review file not found: {review}") + print(f"Refining Stage 1 from {review.name}") + print(f"Review status: {pipeline.check_stage1_review(review)['action']}") + pipeline.run_stage1( + pdf_path, + verify_inventory=not args.no_stage1_verifier, + verifier_timeout=None if args.stage1_verifier_timeout <= 0 else args.stage1_verifier_timeout, + auto_refine_attempts=0 + if args.no_stage1_auto_refine + else args.stage1_auto_refine_attempts, + ) + if args.fetch_sources: + from generation_pipeline.connectors.registry import fetch_json_sources + + stage1_json = find_latest_stage_file(1, output_dir, paper_id=paper_id) + results = fetch_json_sources( + [stage1_json], + pdf_paths=[pdf_path], + token=args.osf_token, + dry_run=args.dry_run, + extract_text=not args.no_extract_text, + llm_client=pipeline.client, # use same client for link-discovery LLM fallback + ) + print_source_fetch_results(results) + if any(item["summary"]["errors"] for item in results): + sys.exit(1) + + elif stage == 2: + regen = Path(args.regeneration_instructions) if args.regeneration_instructions else None + if args.refine: + review = find_latest_stage_file(2, output_dir, paper_id=paper_id).with_suffix(".md") + if not review.exists(): + raise FileNotFoundError(f"Review file not found: {review}") + print(f"Refining Stage 2 from {review.name}") + print(f"Review status: {pipeline.check_stage2_review(review)['action']}") + stage1_json = find_latest_stage_file(1, output_dir, paper_id=paper_id) + pipeline.run_stage2( + stage1_json, + pdf_path, + regeneration_instructions_path=regen, + grounded=bool(args.stage2_grounding and not args.no_grounding), + ground_threshold=args.ground_threshold, + ground_k=args.ground_k, + ground_timeout=args.ground_timeout, + ground_workers=args.ground_workers, + verify_findings=not args.no_stage2_verifier, + verifier_timeout=None if args.stage2_verifier_timeout <= 0 else args.stage2_verifier_timeout, + auto_refine_attempts=0 + if args.no_stage2_auto_refine + else args.stage2_auto_refine_attempts, + ) + + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + import traceback + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/generation_pipeline/settings.py b/generation_pipeline/settings.py new file mode 100644 index 0000000..fffe44e --- /dev/null +++ b/generation_pipeline/settings.py @@ -0,0 +1,537 @@ +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +DEFAULT_SETTINGS_PATHS = ( + Path("config/settings.yaml"), + Path("config/settings.yml"), + Path("config/settings.json"), +) +SUPPORTED_PROVIDERS = ("openai", "google", "anthropic", "openrouter", "vllm") +DEFAULT_PROVIDER = "openai" +DEFAULT_MODEL_BY_PROVIDER = { + "openai": "gpt-5-mini", + "google": "gemini-3-flash", + "anthropic": "claude-sonnet-4-6", + "openrouter": "anthropic/claude-sonnet-4-6", + "vllm": "local-model", +} +PROVIDER_API_KEY_ENV = { + "openai": "OPENAI_API_KEY", + "google": "GOOGLE_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "openrouter": "OPENROUTER_API_KEY", + "vllm": "OPENAI_API_KEY", +} +PROVIDER_DEFAULT_API_BASE = { + "openai": "https://api.openai.com/v1", + "google": "https://generativelanguage.googleapis.com/v1beta/openai/", + "anthropic": "https://api.anthropic.com", + "openrouter": "https://openrouter.ai/api/v1", + "vllm": "http://localhost:8000/v1", +} +DEFAULT_OUTPUT_DIR = Path("generation_pipeline/outputs") +DEFAULT_DATA_DIR = Path("data") +DEFAULT_EXPERIMENTS_DIR = Path("experiments") +DEFAULT_RUNS_DIR = Path("runs") +DEFAULT_VERIFICATION_THRESHOLD = 90.0 +AGENT_LLM_SECTION_NAMES = ("agent_llm", "simulation_llm") +STAGE_LLM_SECTION_NAMES = { + 1: ("stage1_llm", "filter_llm"), + 2: ("stage2_llm", "extraction_llm"), + 3: ("stage3_llm", "patch_llm"), + 4: ("stage4_llm", "config_llm", "gym_llm"), + 5: ("stage5_llm", "agent_llm", "simulation_llm"), +} +STAGE_ENV_PREFIXES = { + 1: ("STAGE1", "FILTER"), + 2: ("STAGE2", "EXTRACTION"), + 3: ("STAGE3", "PATCH"), + 4: ("STAGE4", "CONFIG", "GYM"), + 5: ("STAGE5", "AGENT", "SIM"), +} + + +@dataclass(frozen=True) +class AppSettings: + data: dict[str, Any] + path: Path | None = None + + def section(self, name: str) -> dict[str, Any]: + value = self.data.get(name, {}) + return value if isinstance(value, dict) else {} + + +@dataclass(frozen=True) +class ResolvedLLMConfig: + provider: str + model: str + api_key: str | None + api_base: str | None + + +def load_settings(path: str | Path | None = None) -> AppSettings: + """Load optional settings file and `.env` variables.""" + _load_dotenv() + settings_path = _resolve_settings_path(path) + if settings_path is None: + return AppSettings(data={}, path=None) + return AppSettings(data=_read_settings_file(settings_path), path=settings_path) + + +def resolve_llm_config( + settings: AppSettings | None = None, + *, + provider: str | None = None, + model: str | None = None, + api_key: str | None = None, + api_base: str | None = None, +) -> ResolvedLLMConfig: + """Resolve LLM provider/model/API settings from CLI/env/config/defaults.""" + settings = settings or load_settings() + llm = settings.section("llm") + + resolved_provider = str( + _first_non_empty( + provider, + os.getenv("PROVIDER"), + llm.get("provider"), + settings.data.get("provider"), + DEFAULT_PROVIDER, + ) + ).lower() + if resolved_provider not in PROVIDER_API_KEY_ENV: + raise ValueError( + f"Unknown provider: {resolved_provider}. " + f"Use one of: {', '.join(SUPPORTED_PROVIDERS)}" + ) + + resolved_model = str( + _first_non_empty( + model, + os.getenv("MODEL"), + llm.get("model"), + settings.data.get("model"), + DEFAULT_MODEL_BY_PROVIDER.get(resolved_provider), + ) + ) + + resolved_api_key = _first_non_empty( + api_key, + os.getenv(PROVIDER_API_KEY_ENV[resolved_provider]), + llm.get("api_key"), + settings.data.get("api_key"), + "EMPTY" if resolved_provider == "vllm" else None, + ) + + resolved_api_base = _first_non_empty( + api_base, + os.getenv("BASE_URL"), + llm.get("base_url"), + llm.get("api_base"), + settings.data.get("base_url"), + settings.data.get("api_base"), + PROVIDER_DEFAULT_API_BASE.get(resolved_provider), + ) + + return ResolvedLLMConfig( + provider=resolved_provider, + model=resolved_model, + api_key=str(resolved_api_key) if resolved_api_key is not None else None, + api_base=str(resolved_api_base) if resolved_api_base is not None else None, + ) + + +def resolve_stage_llm_config( + settings: AppSettings | None = None, + *, + stage: int, + provider: str | None = None, + model: str | None = None, + api_key: str | None = None, + api_base: str | None = None, +) -> ResolvedLLMConfig: + """Resolve the LLM config for one pipeline stage. + + Resolution order: + CLI override → stage-specific env → stage-specific settings section → + provider env / global llm fallback → provider defaults. + """ + settings = settings or load_settings() + base = resolve_llm_config(settings) + sections = [settings.section(name) for name in STAGE_LLM_SECTION_NAMES.get(stage, ())] + if stage == 5: + sections = _stage5_active_sections(sections, base) + prefixes = STAGE_ENV_PREFIXES.get(stage, (f"STAGE{stage}",)) + + def from_sections(key: str) -> Any: + return _first_non_empty(*(section.get(key) for section in sections)) + + provider_override = _first_non_empty( + provider, + *(_env(prefix, "PROVIDER") for prefix in prefixes), + from_sections("provider"), + ) + provider_was_overridden = provider_override is not None + resolved_provider = str(provider_override or base.provider).lower() + if resolved_provider not in PROVIDER_API_KEY_ENV: + raise ValueError( + f"Unknown provider for stage {stage}: {resolved_provider}. " + f"Use one of: {', '.join(SUPPORTED_PROVIDERS)}" + ) + same_provider_as_base = resolved_provider == base.provider + + resolved_model = _first_non_empty( + model, + *(_env(prefix, "MODEL") for prefix in prefixes), + from_sections("model"), + DEFAULT_MODEL_BY_PROVIDER.get(resolved_provider) if provider_was_overridden else base.model, + DEFAULT_MODEL_BY_PROVIDER.get(resolved_provider), + ) + + resolved_api_key = _first_non_empty( + api_key, + *(_env(prefix, "API_KEY") for prefix in prefixes), + from_sections("api_key"), + os.getenv(PROVIDER_API_KEY_ENV[resolved_provider]), + base.api_key if same_provider_as_base else None, + "EMPTY" if resolved_provider == "vllm" else None, + ) + + resolved_api_base = _first_non_empty( + api_base, + *(_env(prefix, "BASE_URL") for prefix in prefixes), + *(_env(prefix, "API_BASE") for prefix in prefixes), + from_sections("base_url"), + from_sections("api_base"), + base.api_base if same_provider_as_base else None, + PROVIDER_DEFAULT_API_BASE.get(resolved_provider), + ) + + return ResolvedLLMConfig( + provider=resolved_provider, + model=str(resolved_model), + api_key=str(resolved_api_key) if resolved_api_key is not None else None, + api_base=str(resolved_api_base) if resolved_api_base is not None else None, + ) + + +def _stage5_active_sections(sections: list[dict[str, Any]], base: ResolvedLLMConfig) -> list[dict[str, Any]]: + """Ignore copied Stage-5 cross-provider defaults that have no usable key. + + Stage 5 should inherit the global `llm` config unless the user explicitly + makes a stage-specific provider runnable by setting a matching key in the + section or environment. Same-provider model overrides are still active and + inherit the global key/base URL. + """ + active: list[dict[str, Any]] = [] + for section in sections: + provider = _first_non_empty(section.get("provider")) + if provider is None: + active.append(section) + continue + provider_name = str(provider).lower() + if provider_name not in PROVIDER_API_KEY_ENV: + active.append(section) + continue + if provider_name == base.provider: + active.append(section) + continue + if provider_name == "vllm": + active.append(section) + continue + if _first_non_empty(section.get("api_key"), os.getenv(PROVIDER_API_KEY_ENV[provider_name])): + active.append(section) + return active + + +def resolve_agent_llm_config( + settings: AppSettings | None = None, + *, + provider: str | None = None, + model: str | None = None, + api_key: str | None = None, + api_base: str | None = None, +) -> ResolvedLLMConfig: + """ Resolve the LLM config reserved for Stage 5 agent simulation.""" + settings = settings or load_settings() + extraction_config = resolve_llm_config(settings) + agent_llm = settings.section(AGENT_LLM_SECTION_NAMES[0]) + simulation_llm = settings.section(AGENT_LLM_SECTION_NAMES[1]) + + provider_override = _first_non_empty( + provider, + os.getenv("AGENT_PROVIDER"), + os.getenv("SIM_PROVIDER"), + agent_llm.get("provider"), + simulation_llm.get("provider"), + ) + provider_was_overridden = provider_override is not None + resolved_provider = str(provider_override or extraction_config.provider).lower() + if resolved_provider not in PROVIDER_API_KEY_ENV: + raise ValueError( + f"Unknown provider: {resolved_provider}. " + f"Use one of: {', '.join(SUPPORTED_PROVIDERS)}" + ) + same_provider_as_extraction = resolved_provider == extraction_config.provider + + resolved_model = _first_non_empty( + model, + os.getenv("AGENT_MODEL"), + os.getenv("SIM_MODEL"), + agent_llm.get("model"), + simulation_llm.get("model"), + DEFAULT_MODEL_BY_PROVIDER.get(resolved_provider) if provider_was_overridden else extraction_config.model, + DEFAULT_MODEL_BY_PROVIDER.get(resolved_provider), + ) + + resolved_api_key = _first_non_empty( + api_key, + os.getenv("AGENT_API_KEY"), + os.getenv("SIM_API_KEY"), + os.getenv(PROVIDER_API_KEY_ENV[resolved_provider]), + agent_llm.get("api_key"), + simulation_llm.get("api_key"), + "EMPTY" if resolved_provider == "vllm" and provider_was_overridden else None, + extraction_config.api_key if same_provider_as_extraction else None, + "EMPTY" if resolved_provider == "vllm" else None, + ) + + resolved_api_base = _first_non_empty( + api_base, + os.getenv("AGENT_BASE_URL"), + os.getenv("AGENT_API_BASE"), + os.getenv("SIM_BASE_URL"), + os.getenv("SIM_API_BASE"), + agent_llm.get("base_url"), + agent_llm.get("api_base"), + simulation_llm.get("base_url"), + simulation_llm.get("api_base"), + extraction_config.api_base if same_provider_as_extraction else None, + PROVIDER_DEFAULT_API_BASE.get(resolved_provider), + ) + + return ResolvedLLMConfig( + provider=resolved_provider, + model=str(resolved_model), + api_key=str(resolved_api_key) if resolved_api_key is not None else None, + api_base=str(resolved_api_base) if resolved_api_base is not None else None, + ) + + +def resolve_output_dir(settings: AppSettings | None = None, output_dir: str | Path | None = None) -> Path: + settings = settings or load_settings() + paths = settings.section("paths") + value = _first_non_empty(output_dir, os.getenv("OUTPUT_DIR"), settings.data.get("output_dir"), paths.get("output_dir"), DEFAULT_OUTPUT_DIR) + return Path(value) + + +def resolve_experiments_dir(settings: AppSettings | None = None, experiments_dir: str | Path | None = None) -> Path: + settings = settings or load_settings() + paths = settings.section("paths") + value = _first_non_empty( + experiments_dir, + os.getenv("EXPERIMENTS_DIR"), + settings.data.get("experiments_dir"), + paths.get("experiments_dir"), + DEFAULT_EXPERIMENTS_DIR, + ) + return Path(value) + + +def resolve_data_dir(settings: AppSettings | None = None, data_dir: str | Path | None = None) -> Path: + settings = settings or load_settings() + paths = settings.section("paths") + value = _first_non_empty( + data_dir, + os.getenv("DATA_DIR"), + settings.data.get("data_dir"), + paths.get("data_dir"), + DEFAULT_DATA_DIR, + ) + return Path(value) + + +def resolve_runs_dir(settings: AppSettings | None = None, runs_dir: str | Path | None = None) -> Path: + settings = settings or load_settings() + paths = settings.section("paths") + value = _first_non_empty( + runs_dir, + os.getenv("RUNS_DIR"), + settings.data.get("runs_dir"), + paths.get("runs_dir"), + DEFAULT_RUNS_DIR, + ) + return Path(value) + + +def resolve_verification_threshold( + settings: AppSettings | None = None, + threshold: float | None = None, +) -> float: + settings = settings or load_settings() + verification = settings.section("verification") + value = _first_non_empty( + threshold, + os.getenv("VERIFICATION_THRESHOLD"), + settings.data.get("verification_threshold"), + verification.get("threshold"), + DEFAULT_VERIFICATION_THRESHOLD, + ) + return float(value) + + +def _resolve_settings_path(path: str | Path | None) -> Path | None: + explicit = path or os.getenv("SETTINGS_PATH") + if explicit: + settings_path = Path(explicit) + if not settings_path.exists(): + raise FileNotFoundError(f"Settings file not found: {settings_path}") + return settings_path + + for candidate in DEFAULT_SETTINGS_PATHS: + if candidate.exists(): + return candidate + return None + + +def _read_settings_file(path: Path) -> dict[str, Any]: + suffix = path.suffix.lower() + text = path.read_text(encoding="utf-8") + if suffix == ".json": + data = json.loads(text) + elif suffix in {".yaml", ".yml"}: + try: + import yaml + except ImportError: + data = _read_simple_yaml(text) + else: + data = yaml.safe_load(text) or {} + else: + raise ValueError(f"Unsupported settings file type: {path.suffix}") + if not isinstance(data, dict): + raise ValueError(f"Settings file must contain an object at top level: {path}") + return data + + +def _read_simple_yaml(text: str) -> dict[str, Any]: + """Parse the small settings YAML subset used by config/settings.yaml.""" + data: dict[str, Any] = {} + current_section: dict[str, Any] | None = None + current_indent = 0 + + for raw_line in text.splitlines(): + if not raw_line.strip() or raw_line.lstrip().startswith("#"): + continue + indent = len(raw_line) - len(raw_line.lstrip(" ")) + line = _strip_inline_comment(raw_line).strip() + if not line: + continue + if ":" not in line: + raise ValueError(f"Unsupported settings YAML line: {raw_line}") + + key, value = line.split(":", 1) + key = key.strip() + value = value.strip() + if not key: + raise ValueError(f"Unsupported settings YAML line: {raw_line}") + + if indent == 0: + if value == "": + current_section = {} + current_indent = indent + data[key] = current_section + else: + data[key] = _parse_scalar(value) + current_section = None + elif current_section is not None and indent > current_indent: + current_section[key] = _parse_scalar(value) + else: + raise ValueError(f"Unsupported nested settings YAML line: {raw_line}") + + return data + + +def _strip_inline_comment(line: str) -> str: + in_single = False + in_double = False + for index, char in enumerate(line): + if char == "'" and not in_double: + in_single = not in_single + elif char == '"' and not in_single: + in_double = not in_double + elif char == "#" and not in_single and not in_double: + return line[:index] + return line + + +def _parse_scalar(value: str) -> Any: + lowered = value.lower() + if lowered in {"null", "none", "~", ""}: + return None + if lowered == "true": + return True + if lowered == "false": + return False + if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")): + return value[1:-1] + try: + return int(value) + except ValueError: + pass + try: + return float(value) + except ValueError: + return value + + +def _load_dotenv() -> None: + try: + from dotenv import load_dotenv + except ImportError: + return + load_dotenv(override=False) + + +def _first_non_empty(*values: Any) -> Any: + for value in values: + if value is None: + continue + if isinstance(value, str) and value.strip() == "": + continue + return value + return None + + +def _env(prefix: str, suffix: str) -> str | None: + return os.getenv(f"{prefix}_{suffix}") + + +def settings_to_public_dict( + settings: AppSettings, + llm_config: ResolvedLLMConfig, + agent_llm_config: ResolvedLLMConfig | None = None, +) -> dict[str, Any]: + """Return a secret-safe view useful for logging/debugging.""" + public = { + "settings_path": str(settings.path) if settings.path else None, + "llm": { + "provider": llm_config.provider, + "model": llm_config.model, + "api_base": llm_config.api_base, + "api_key_set": bool(llm_config.api_key), + }, + } + if agent_llm_config is not None: + public["agent_llm"] = { + "provider": agent_llm_config.provider, + "model": agent_llm_config.model, + "api_base": agent_llm_config.api_base, + "api_key_set": bool(agent_llm_config.api_key), + } + return public diff --git a/generation_pipeline/simulation_targets.py b/generation_pipeline/simulation_targets.py new file mode 100644 index 0000000..ac4d3f1 --- /dev/null +++ b/generation_pipeline/simulation_targets.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional + + +_TOKEN_RE = re.compile(r"[a-z][a-z0-9]+", re.IGNORECASE) +_STOP = { + "the", "and", "for", "with", "that", "this", "from", "into", "using", + "used", "study", "studies", "effect", "effects", "condition", + "conditions", "measure", "measures", "item", "items", "scale", + "participant", "participants", "response", "responses", "reported", + "results", "result", "table", "index", "direct", "focal", "primary", + "continuous", "between", "within", "point", "rating", + "more", "less", "same", "than", "today", "across", + "future", "generat", "people", "person", "persons", "human", "humans", +} +_DROP_ROLES = { + "simple", "simple_effect", "mediation", "correlation", + "manipulation_check", "check", "secondary_dv", +} +_ALLOW_ROLES = {"primary", "primary_finding", "interaction", "moderation", "main"} + + +def _slug(value: Any, fallback: str = "study") -> str: + text = re.sub(r"[^a-z0-9]+", "_", str(value or "").lower()).strip("_") + return text or fallback + + +def _stem(token: str) -> str: + token = token.lower().strip("'_-") + if token in {"sided", "sidedness"}: + return "side" + for suffix in ("iveness", "ation", "ness", "ment", "ility", "ity", "ing", "ed", "es", "s"): + if len(token) > len(suffix) + 4 and token.endswith(suffix): + token = token[:-len(suffix)] + break + return token[:7] if len(token) > 7 else token + + +def _tokens(*values: Any) -> set[str]: + out: set[str] = set() + for value in values: + if isinstance(value, dict): + value = value.get("content") + text = str(value or "") + text = re.sub(r"[-/]+", " ", text) + for token in _TOKEN_RE.findall(text): + stem = _stem(token) + if len(stem) >= 4 and stem not in _STOP and not stem.isdigit(): + out.add(stem) + return out + + +def _soft_overlap(a: set[str], b: set[str]) -> int: + hits = 0 + used: set[str] = set() + for x in a: + for y in b: + if y in used: + continue + if x == y or (min(len(x), len(y)) >= 4 and (x.startswith(y) or y.startswith(x))): + hits += 1 + used.add(y) + break + return hits + + +def _stats_p_value(stats: Dict[str, Any]) -> Optional[float]: + raw = stats.get("p_value") if isinstance(stats, dict) else None + if raw in (None, "", [None, None]): + return None + text = str(raw).strip().lower() + if text in {"ns", "n.s.", "not significant", "non-significant"}: + return 1.0 + match = re.search(r"([<>]=?)?\s*\.?(\d+(?:\.\d+)?)", text) + if not match: + return None + number = match.group(2) + if "." in match.group(0) and "." not in number: + return float("0." + number) + return float(number) + + +def _effect_is_significant(effect: Dict[str, Any]) -> bool: + stats = effect.get("stats") if isinstance(effect.get("stats"), dict) else {} + sig = str(stats.get("sig") or effect.get("sig") or "").strip().lower() + if any(token in sig for token in ("marginal", "ns", "n.s", "non", "not sig")): + return False + p_value = _stats_p_value(stats) + return not (p_value is not None and p_value > 0.05) + + +def _material_selected(material: Any) -> bool: + if not isinstance(material, dict): + return True + selection = material.get("selection") + if not isinstance(selection, dict): + return True + return bool(selection.get("keep", True)) + + +def _study_selected(study: Dict[str, Any]) -> bool: + selection = study.get("selection") + if not isinstance(selection, dict): + return True + return bool(selection.get("keep", True)) + + +def _condition_terms(material: Dict[str, Any]) -> set[str]: + terms: set[str] = set() + for cond in material.get("conditions", []) or []: + if not isinstance(cond, dict): + continue + terms |= _tokens(cond.get("name"), cond.get("label")) + for level in cond.get("levels", []) or []: + terms |= _tokens(level) + descriptions = cond.get("level_descriptions") + if isinstance(descriptions, dict): + for value in descriptions.values(): + terms |= _tokens(value) + return terms + + +def _item_terms(material: Dict[str, Any]) -> set[str]: + terms: set[str] = set() + for item in material.get("items", []) or []: + if not isinstance(item, dict): + continue + terms |= _tokens(item.get("id"), item.get("data_export_tag"), item.get("question")) + return terms + + +def _item_match_count(material: Dict[str, Any], dv_terms: set[str]) -> int: + count = 0 + for item in material.get("items", []) or []: + if not isinstance(item, dict): + continue + item_terms = _tokens(item.get("id"), item.get("data_export_tag"), item.get("question")) + if _soft_overlap(dv_terms, item_terms) > 0: + count += 1 + return count + + +def _role(effect: Dict[str, Any]) -> str: + cons = effect.get("consolidation") if isinstance(effect.get("consolidation"), dict) else {} + return str(cons.get("analysis_role") or effect.get("effecttype") or "").strip().lower() + + +def _is_representative(effect: Dict[str, Any], index: int) -> bool: + cons = effect.get("consolidation") if isinstance(effect.get("consolidation"), dict) else {} + if not cons: + return True + rep = cons.get("representative_effect_index", index) + return bool(cons.get("is_representative", index == rep)) + + +def _candidate( + *, + effect: Dict[str, Any], + effect_index: int, + material: Dict[str, Any], + title_terms: set[str], +) -> Optional[Dict[str, Any]]: + role = _role(effect) + etype = str(effect.get("effecttype") or "").strip().lower() + if role in _DROP_ROLES or etype in _DROP_ROLES: + return None + if role and role not in _ALLOW_ROLES and etype not in {"main", "int", "interaction", "moderation"}: + return None + if not _effect_is_significant(effect): + return None + iv = str(effect.get("IV") or "") + dv = str(effect.get("DV") or "") + materials_notes = str(effect.get("materials_notes") or "") + if "manipulation check" in f"{iv} {dv} {materials_notes}".lower(): + return None + + iv_terms = _tokens(iv) + dv_terms = _tokens(dv) + condition_terms = _condition_terms(material) + item_terms = _item_terms(material) + condition_hits = _soft_overlap(iv_terms, condition_terms) + item_hits = _soft_overlap(dv_terms, item_terms) + item_match_count = _item_match_count(material, dv_terms) + title_hits = _soft_overlap(dv_terms, title_terms) + cons = effect.get("consolidation") if isinstance(effect.get("consolidation"), dict) else {} + llm_keep = cons.get("llm_keep") + primary_flag = bool(cons.get("is_primary_simulation_target", True)) + + score = 0 + score += 35 if role in {"interaction", "moderation"} or etype in {"int", "interaction"} else 20 + score += 20 if primary_flag else 0 + score += 12 if llm_keep is True else 0 + score += 12 * min(condition_hits, 2) + score += 8 * min(item_hits, 2) + score += 5 * min(item_match_count, 5) + score += 14 * min(title_hits, 2) + score += 6 if _effect_is_significant(effect) else 0 + + return { + "effect_index": effect_index, + "effect": effect, + "score": score, + "role": role or etype or "primary", + "condition_hits": condition_hits, + "item_hits": item_hits, + "item_match_count": item_match_count, + "title_hits": title_hits, + "has_conditions": bool(condition_terms), + "dv_key": " ".join(sorted(dv_terms)) or _slug(dv, "dv"), + "merge_group_id": cons.get("merge_group_id"), + } + + +def _pick_study_targets(candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + if not candidates: + return [] + + # If the material exposes condition variables, prefer effects whose IV maps + # to those participant-facing conditions. This removes measured-only direct + # effects while preserving manipulation x moderator interactions. + if any(c["condition_hits"] > 0 for c in candidates): + candidates = [c for c in candidates if c["condition_hits"] > 0] + + if any(c["item_hits"] > 0 for c in candidates): + candidates = [c for c in candidates if c["item_hits"] > 0] + + # When the paper title clearly names an item-backed focal outcome, use it to + # drop secondary DVs such as mediators or appreciation checks. If the only + # title hits are derived metrics with no direct item match, keep them as + # possible additions below instead of letting them suppress item-backed DVs. + title_backed = [c for c in candidates if c["title_hits"] > 0 and c["item_match_count"] > 0] + if title_backed: + candidates = title_backed + elif any(c["has_conditions"] and c["condition_hits"] > 0 for c in candidates): + # Explicit condition structure supports multiple co-primary outcomes in + # the same study (for example manipulation -> beliefs and support). + return sorted( + [c for c in candidates if c["condition_hits"] > 0], + key=lambda item: item["effect_index"], + ) + + by_dv: Dict[str, List[Dict[str, Any]]] = {} + for candidate in candidates: + by_dv.setdefault(candidate["dv_key"], []).append(candidate) + + def group_score(items: List[Dict[str, Any]]) -> tuple[int, int, int]: + return ( + max(int(item["score"]) for item in items), + len(items), + sum(int(item["condition_hits"]) for item in items), + ) + + ranked_groups = sorted(by_dv.values(), key=group_score, reverse=True) + selected = list(ranked_groups[0]) + selected_ids = {id(item) for item in selected} + for candidate in candidates: + if id(candidate) in selected_ids: + continue + if candidate["title_hits"] > 0 and candidate["item_match_count"] == 0 and candidate["score"] >= 55: + selected.append(candidate) + return sorted(selected, key=lambda item: item["effect_index"]) + + +def build_simulation_targets( + eligible_studies: List[Dict[str, Any]], + materials: Dict[str, Dict[str, Any]], + *, + paper_title: str = "", +) -> List[Dict[str, Any]]: + """Build and stamp target records for Stage 4. + + Returns one or more target findings per selected sub-study. Each target is + bound to a material id and to a representative effect index, so Stage 4 no + longer has to infer which paper analysis the prompt should represent. + """ + title_terms = _tokens(paper_title) + targets: List[Dict[str, Any]] = [] + + for study in eligible_studies: + if not isinstance(study, dict) or not _study_selected(study): + continue + study_name = str(study.get("study") or study.get("study_id") or study.get("experiment_id") or "") + sub_id = _slug(study_name, "study") + material = materials.get(sub_id) + if not isinstance(material, dict) or not _material_selected(material): + continue + + candidates: List[Dict[str, Any]] = [] + effects = [effect for effect in study.get("effects", []) or [] if isinstance(effect, dict)] + for index, effect in enumerate(effects): + if not _is_representative(effect, index): + continue + candidate = _candidate( + effect=effect, + effect_index=index, + material=material, + title_terms=title_terms, + ) + if candidate is not None: + candidates.append(candidate) + + picked = _pick_study_targets(candidates) + for rank, candidate in enumerate(picked, start=1): + effect = candidate["effect"] + target_id = f"{sub_id}__effect_{rank:02d}" + target = { + "target_id": target_id, + "sub_study_id": sub_id, + "study_name": study_name or sub_id, + "effect_index": candidate["effect_index"], + "merge_group_id": candidate.get("merge_group_id"), + "analysis_role": candidate["role"], + "selection_basis": { + "score": candidate["score"], + "condition_hits": candidate["condition_hits"], + "item_hits": candidate["item_hits"], + "item_match_count": candidate["item_match_count"], + "title_hits": candidate["title_hits"], + "dv_key": candidate["dv_key"], + }, + } + effect["simulation_target"] = { + "selected": True, + "target_id": target_id, + "rank": rank, + **target["selection_basis"], + } + targets.append(target) + return targets + + +def summarize_simulation_targets(targets: List[Dict[str, Any]]) -> Dict[str, Any]: + by_study: Dict[str, int] = {} + for target in targets: + sid = str(target.get("sub_study_id") or "study") + by_study[sid] = by_study.get(sid, 0) + 1 + return { + "total_targets": len(targets), + "by_sub_study": by_study, + } diff --git a/generation_pipeline/split_replicable.py b/generation_pipeline/split_replicable.py new file mode 100644 index 0000000..a1cbbe2 --- /dev/null +++ b/generation_pipeline/split_replicable.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +""" +Split stage4 corpus into runnable / non_runnable based on material completeness for agent replication. + +Criterion (per effect, all three slots must have non-empty content): + - materials.content + - manipulation.content + - items.content (cited_scale with <80 chars counts as incomplete — scale items missing) +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +RUNNABLE = "runnable" +NOT_RUNNABLE = "non_runnable" +MIN_ITEMS_CHARS = 80 + + +def analyze(json_path: Path) -> tuple[bool, list[dict], int]: + data = json.loads(json_path.read_text(encoding="utf-8")) + issues: list[dict] = [] + n_effects = 0 + for study in data.get("eligible_studies", []): + sname = study.get("study", "") + for ei, eff in enumerate(study.get("effects", [])): + n_effects += 1 + for slot in ("materials", "manipulation", "items"): + obj = eff.get(slot) or {} + st = obj.get("status") + c = obj.get("content") + has = c is not None and str(c).strip() != "" + problem = None + if not has: + problem = st or "empty" + elif slot == "items" and st == "cited_scale" and len(str(c).strip()) < MIN_ITEMS_CHARS: + problem = "cited_scale_partial" + if problem: + issues.append( + { + "study": sname, + "effect_index": ei, + "slot": slot, + "status": problem, + "IV": eff.get("IV", ""), + "DV": eff.get("DV", ""), + } + ) + return (len(issues) == 0 and n_effects > 0, issues, n_effects) + + +def split(stage4_dir: Path, dry_run: bool = False) -> dict: + runnable_dir = stage4_dir / RUNNABLE + not_dir = stage4_dir / NOT_RUNNABLE + skip_names = {RUNNABLE, NOT_RUNNABLE, "replicable", "non_replicable", "能跑", "不能跑"} + + report = {"runnable": [], "not_runnable": [], "moved": []} + + paper_dirs: list[Path] = [] + split_dirs = {RUNNABLE, NOT_RUNNABLE, "能跑", "不能跑"} + for folder in sorted(stage4_dir.iterdir()): + if not folder.is_dir(): + continue + if folder.name in split_dirs: + paper_dirs.extend(sorted(p for p in folder.iterdir() if p.is_dir())) + elif folder.name not in skip_names: + paper_dirs.append(folder) + + for folder in paper_dirs: + jsons = [f for f in folder.glob("*.json") if f.name not in ("sum.json",)] + jsons = [f for f in jsons if "manifest" not in f.name.lower()] + if not jsons: + continue + + can_run, issues, n_eff = analyze(jsons[0]) + entry = { + "folder": folder.name, + "effects": n_eff, + "issues": issues, + } + dest_parent = runnable_dir if can_run else not_dir + if can_run: + report["runnable"].append(entry) + else: + report["not_runnable"].append(entry) + + if not dry_run: + dest_parent.mkdir(parents=True, exist_ok=True) + dest = dest_parent / folder.name + if dest.exists(): + shutil.rmtree(dest) + shutil.move(str(folder), str(dest)) + report["moved"].append({"from": folder.name, "to": str(dest.relative_to(stage4_dir))}) + + if not dry_run: + index_path = stage4_dir / "replication_split.json" + index_path.write_text( + json.dumps( + { + "criterion": ( + "Each effect: materials, manipulation, items all have content; " + f"items cited_scale must be >= {MIN_ITEMS_CHARS} chars" + ), + "runnable_count": len(report["runnable"]), + "not_runnable_count": len(report["not_runnable"]), + "runnable": [x["folder"] for x in report["runnable"]], + "not_runnable": [ + {"folder": x["folder"], "issues": x["issues"]} for x in report["not_runnable"] + ], + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + return report + + +def main(): + import argparse + + p = argparse.ArgumentParser() + p.add_argument("--stage4-dir", type=Path, default=Path("stage4")) + p.add_argument("--dry-run", action="store_true") + args = p.parse_args() + r = split(args.stage4_dir, dry_run=args.dry_run) + print(f"runnable: {len(r['runnable'])}") + for x in r["runnable"]: + print(f" ✓ {x['folder']} ({x['effects']} effects)") + print(f"\nnon_runnable: {len(r['not_runnable'])}") + for x in r["not_runnable"]: + print(f" ✗ {x['folder']} — {len(x['issues'])} gap(s)") + if not args.dry_run: + print(f"\nMoved to stage4/{RUNNABLE}/ and stage4/{NOT_RUNNABLE}/") + print("Index: stage4/replication_split.json") + + +if __name__ == "__main__": + main() diff --git a/generation_pipeline/stage1_study_contract.py b/generation_pipeline/stage1_study_contract.py new file mode 100644 index 0000000..3e9a1f4 --- /dev/null +++ b/generation_pipeline/stage1_study_contract.py @@ -0,0 +1,127 @@ +"""Deterministic contract audit for Stage 1 study inventories.""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List + + +_VALID_REPLICABLE = {"YES", "NO", "UNCERTAIN"} + + +def slugify(value: Any, fallback: str = "study") -> str: + text = re.sub(r"[^a-z0-9]+", "_", str(value or "").lower()).strip("_") + return text or fallback + + +def _has_text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _source_hints(value: Any) -> List[Dict[str, Any]]: + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, dict)] + + +def _add_missing(missing_by_field: Dict[str, List[str]], field: str, ref: str) -> None: + missing_by_field.setdefault(field, []).append(ref) + + +def audit_stage1_study_contract(stage1_json: Dict[str, Any]) -> Dict[str, Any]: + """Return a field-level audit for a Stage 1 inventory.""" + experiments = [exp for exp in stage1_json.get("experiments", []) or [] if isinstance(exp, dict)] + studies: Dict[str, Dict[str, Any]] = {} + missing_by_field: Dict[str, List[str]] = {} + seen_study_ids: set[str] = set() + ready = 0 + eligible_or_uncertain = 0 + + for idx, exp in enumerate(experiments): + path = f"$.experiments[{idx}]" + study_id = str(exp.get("study_id") or exp.get("experiment_id") or f"study_{idx + 1}") + study_ref = study_id or path + replicable = str(exp.get("replicable") or "").strip().upper() + is_downstream_candidate = replicable in {"YES", "UNCERTAIN"} + if is_downstream_candidate: + eligible_or_uncertain += 1 + + blocking: List[str] = [] + warnings: List[str] = [] + + if not _has_text(exp.get("study_id")): + blocking.append("missing_study_id") + _add_missing(missing_by_field, "study_id", study_ref) + if study_id in seen_study_ids: + blocking.append("duplicate_study_id") + _add_missing(missing_by_field, "study_id", study_ref) + seen_study_ids.add(study_id) + + if not _has_text(exp.get("experiment_id")): + blocking.append("missing_experiment_id") + _add_missing(missing_by_field, "experiment_id", study_ref) + if not _has_text(exp.get("study_name")): + blocking.append("missing_study_name") + _add_missing(missing_by_field, "study_name", study_ref) + if replicable not in _VALID_REPLICABLE: + blocking.append("invalid_replicable_label") + _add_missing(missing_by_field, "replicable", study_ref) + + if is_downstream_candidate: + for field in ("design_type", "participant_task", "input", "participants", "output"): + if not _has_text(exp.get(field)): + blocking.append(f"missing_{field}") + _add_missing(missing_by_field, field, study_ref) + if not exp.get("conditions_or_factors"): + blocking.append("missing_conditions_or_factors") + _add_missing(missing_by_field, "conditions_or_factors", study_ref) + hints = _source_hints(exp.get("candidate_source_hints")) + if not hints: + blocking.append("missing_candidate_source_hints") + _add_missing(missing_by_field, "candidate_source_hints", study_ref) + elif not any(_has_text(hint.get("description")) for hint in hints): + warnings.append("source_hints_without_description") + + if "has_self_contained_materials" not in exp or not isinstance(exp.get("has_self_contained_materials"), bool): + warnings.append("self_contained_materials_not_boolean") + _add_missing(missing_by_field, "has_self_contained_materials", study_ref) + + if not blocking: + ready += 1 + studies[study_id] = { + "study_id": exp.get("study_id"), + "experiment_id": exp.get("experiment_id"), + "study_name": exp.get("study_name"), + "replicable": replicable, + "ready": not blocking, + "downstream_candidate": is_downstream_candidate, + "blocking_issues": sorted(set(blocking)), + "warnings": sorted(set(warnings)), + "fields": { + "design_type": exp.get("design_type"), + "conditions_or_factors": len(exp.get("conditions_or_factors") or []), + "participant_task": exp.get("participant_task"), + "input": exp.get("input"), + "output": exp.get("output"), + "candidate_source_hints": len(_source_hints(exp.get("candidate_source_hints"))), + }, + } + + blocking_issue_count = sum(len(study.get("blocking_issues") or []) for study in studies.values()) + return { + "version": "stage1_study_contract_v1", + "total_studies": len(experiments), + "eligible_or_uncertain": eligible_or_uncertain, + "ready": ready, + "needs_review": len(experiments) - ready, + "blocking_issue_count": blocking_issue_count, + "missing_by_field": {field: sorted(set(refs)) for field, refs in sorted(missing_by_field.items())}, + "studies": studies, + } + + +def apply_stage1_study_contract(stage1_json: Dict[str, Any]) -> Dict[str, Any]: + """Attach the Stage 1 study contract in-place and return it.""" + contract = audit_stage1_study_contract(stage1_json) + stage1_json["stage1_study_contract"] = contract + return contract diff --git a/generation_pipeline/stage1_verifier.py b/generation_pipeline/stage1_verifier.py new file mode 100644 index 0000000..b3a1d31 --- /dev/null +++ b/generation_pipeline/stage1_verifier.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import json +import re +import time +from pathlib import Path +from typing import Any, Dict, List, Optional + +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + + +PDF_TEXT_MAX_CHARS = 180000 + + +def _compact_stage1(stage1_json: Dict[str, Any]) -> List[Dict[str, Any]]: + out: List[Dict[str, Any]] = [] + for exp in stage1_json.get("experiments", []) or []: + if not isinstance(exp, dict): + continue + out.append( + { + "experiment_id": exp.get("experiment_id"), + "study_id": exp.get("study_id"), + "experiment_name": exp.get("experiment_name"), + "study_name": exp.get("study_name"), + "design_type": exp.get("design_type"), + "conditions_or_factors": exp.get("conditions_or_factors"), + "input": exp.get("input"), + "participant_task": exp.get("participant_task"), + "participants": exp.get("participants"), + "output": exp.get("output"), + "candidate_source_hints": exp.get("candidate_source_hints"), + "replicable": exp.get("replicable"), + "has_self_contained_materials": exp.get("has_self_contained_materials"), + "exclusion_reasons": exp.get("exclusion_reasons"), + "missing_materials": exp.get("missing_materials"), + } + ) + return out + + +def _loads_json(text: str) -> Dict[str, Any]: + response_text = text.strip() + if "```json" in response_text: + response_text = response_text.split("```json", 1)[1].split("```", 1)[0].strip() + elif "```" in response_text: + response_text = response_text.split("```", 1)[1].split("```", 1)[0].strip() + try: + parsed = json.loads(response_text) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", response_text, re.DOTALL) + if not match: + raise + parsed = json.loads(match.group()) + if not isinstance(parsed, dict): + raise ValueError(f"Verifier response is not a JSON object: {type(parsed)}") + return parsed + + +def _normalize_report(report: Dict[str, Any]) -> Dict[str, Any]: + overall = str(report.get("overall") or "needs_review").strip().lower() + if overall not in {"pass", "needs_review", "fail"}: + overall = "needs_review" + normalized = { + "version": "stage1_verifier_v1", + "status": "ok", + "overall": overall, + "confidence": report.get("confidence"), + "inventory_checks": report.get("inventory_checks") if isinstance(report.get("inventory_checks"), list) else [], + "eligibility_checks": report.get("eligibility_checks") if isinstance(report.get("eligibility_checks"), list) else [], + "regeneration_instructions": report.get("regeneration_instructions") + if isinstance(report.get("regeneration_instructions"), dict) + else {}, + "notes": report.get("notes", ""), + } + regen = normalized["regeneration_instructions"] + regen.setdefault("missing_studies", []) + regen.setdefault("split_merge_corrections", []) + regen.setdefault("eligibility_corrections", []) + return normalized + + +def build_verifier_prompt(stage1_json: Dict[str, Any], pdf_text: str) -> str: + """Build the Stage 1 verifier prompt. Exposed for unit tests.""" + return f"""You are verifying a Stage 1 study inventory from a psychology/management paper. + +Check whether the extracted inventory covers the paper's empirical studies and +whether each eligibility label is defensible for an automated HumanStudy-Bench +pipeline. Focus on study coverage, split/merge errors, design/sample anchors, +and eligibility labels. Do NOT require complete participant-facing materials; +Stage 3 handles material source recovery. + +PAPER TEXT: +{pdf_text} + +STAGE 1 STUDY INVENTORY: +{json.dumps(_compact_stage1(stage1_json), ensure_ascii=False, indent=2)} + +Return ONLY a compact JSON object with this schema. Do not include markdown, +analysis, commentary, or chain-of-thought. Keep evidence strings short. +{{ + "overall": "pass|needs_review|fail", + "confidence": 0.0, + "inventory_checks": [ + {{ + "study": "Study 1", + "verdict": "ok|missing|extra|split_issue|merge_issue|needs_review", + "issue": "short issue or empty", + "evidence": "short paper evidence" + }} + ], + "eligibility_checks": [ + {{ + "study": "Study 1", + "verdict": "ok|wrong_label|unsupported|needs_review", + "expected_label": "YES|NO|UNCERTAIN", + "issue": "short issue or empty", + "evidence": "short paper evidence" + }} + ], + "regeneration_instructions": {{ + "missing_studies": ["..."], + "split_merge_corrections": [{{"study": "...", "reason": "..."}}], + "eligibility_corrections": [{{"study": "...", "expected_label": "...", "reason": "..."}}] + }}, + "notes": "one short summary" +}}""" + + +def verify_stage1_inventory( + stage1_json: Dict[str, Any], + pdf_path: Path, + llm_client: Any, + *, + pdf_text: Optional[str] = None, + timeout: Optional[float] = 60.0, + max_tokens: Optional[int] = 3000, + max_attempts: int = 2, + retry_delay: float = 1.0, +) -> Dict[str, Any]: + if llm_client is None: + return { + "version": "stage1_verifier_v1", + "status": "skipped", + "overall": "needs_review", + "notes": "No LLM client provided.", + "inventory_checks": [], + "eligibility_checks": [], + "regeneration_instructions": { + "missing_studies": [], + "split_merge_corrections": [], + "eligibility_corrections": [], + }, + } + text = pdf_text if pdf_text is not None else extract_pdf_text(Path(pdf_path), max_chars=PDF_TEXT_MAX_CHARS) + prompt = build_verifier_prompt(stage1_json, text) + attempts = max(1, int(max_attempts or 1)) + last_error: Optional[BaseException] = None + for attempt in range(1, attempts + 1): + try: + try: + response = llm_client.generate_content(prompt=prompt, timeout=timeout, max_tokens=max_tokens) + except TypeError: + try: + response = llm_client.generate_content(prompt=prompt, timeout=timeout) + except TypeError: + response = llm_client.generate_content(prompt=prompt) + if response is None: + raise ValueError("Stage 1 verifier LLM returned None.") + return _normalize_report(_loads_json(str(response))) + except Exception as exc: + last_error = exc + if attempt >= attempts: + break + if retry_delay > 0: + time.sleep(retry_delay) + if last_error is not None: + raise last_error + raise ValueError("Stage 1 verifier failed without an exception.") diff --git a/generation_pipeline/stage2_finding_contract.py b/generation_pipeline/stage2_finding_contract.py new file mode 100644 index 0000000..77096a3 --- /dev/null +++ b/generation_pipeline/stage2_finding_contract.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional, Set + + +def _norm(value: Any) -> str: + text = str(value or "").strip().lower() + text = re.sub(r"[^a-z0-9]+", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def _study_key(value: Any) -> str: + text = _norm(value) + match = re.search(r"\b(?:study|experiment|exp)\s+(\d+[a-z]?)\b", text) + if match: + return match.group(1) + match = re.search(r"\b(pilot|validation)\b", text) + if match: + return match.group(1) + text = re.sub(r"\b(study|experiment|exp)\b", "", text) + return re.sub(r"\s+", " ", text).strip() + + +def _has_text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _has_stat_value(stats: Dict[str, Any]) -> bool: + for key in ("B", "b", "t", "f", "z", "chi_square", "D", "eta_square", "p_value", "sig"): + value = stats.get(key) + if value not in (None, "", [], [None, None]): + return True + ci = stats.get("ci") + return isinstance(ci, list) and any(item is not None for item in ci) + + +def _has_reported_statistics(finding: Dict[str, Any]) -> bool: + if _has_text(finding.get("reported_statistics")): + return True + stats = finding.get("stats") if isinstance(finding.get("stats"), dict) else {} + return _has_stat_value(stats) + + +def _valid_index(value: Any, effects: List[Dict[str, Any]]) -> Optional[int]: + try: + idx = int(value) + except (TypeError, ValueError): + return None + if 0 <= idx < len(effects): + return idx + return None + + +def _stage1_eligible_keys(stage1_json: Optional[Dict[str, Any]]) -> Set[str]: + if not isinstance(stage1_json, dict): + return set() + keys: Set[str] = set() + for exp in stage1_json.get("experiments", []) or []: + if not isinstance(exp, dict): + continue + replicable = str(exp.get("replicable") or "").strip().upper() + if replicable not in {"YES", "UNCERTAIN"}: + continue + key = _study_key(exp.get("experiment_id") or exp.get("experiment_name")) + if key: + keys.add(key) + return keys + + +def _add_missing(missing_by_field: Dict[str, List[str]], field: str, ref: str) -> None: + missing_by_field.setdefault(field, []).append(ref) + + +def _finding_audit( + finding: Dict[str, Any], + *, + effects: List[Dict[str, Any]], + path: str, + missing_by_field: Dict[str, List[str]], +) -> Dict[str, Any]: + finding_id = str(finding.get("finding_id") or path) + blocking: List[str] = [] + warnings: List[str] = [] + + for field in ("finding_id", "role", "IV", "DV"): + if finding.get(field) in (None, "", []): + issue = f"missing_{field}" + blocking.append(issue) + _add_missing(missing_by_field, field, finding_id) + + if not _has_reported_statistics(finding): + blocking.append("missing_reported_statistics") + _add_missing(missing_by_field, "reported_statistics", finding_id) + + rep_idx = _valid_index(finding.get("representative_effect_index"), effects) + if rep_idx is None: + blocking.append("invalid_representative_effect_index") + _add_missing(missing_by_field, "representative_effect_index", finding_id) + + raw_indices = finding.get("effect_indices") + if not isinstance(raw_indices, list) or not raw_indices: + blocking.append("missing_effect_indices") + _add_missing(missing_by_field, "effect_indices", finding_id) + valid_indices: List[int] = [] + else: + valid_indices = [] + invalid_indices: List[Any] = [] + for value in raw_indices: + idx = _valid_index(value, effects) + if idx is None: + invalid_indices.append(value) + else: + valid_indices.append(idx) + if invalid_indices: + blocking.append("invalid_effect_indices") + _add_missing(missing_by_field, "effect_indices", finding_id) + + if rep_idx is not None and valid_indices and rep_idx not in valid_indices: + warnings.append("representative_not_in_effect_indices") + + target = finding.get("simulation_target") if isinstance(finding.get("simulation_target"), dict) else {} + if "candidate" not in target: + blocking.append("missing_simulation_target_candidate") + _add_missing(missing_by_field, "simulation_target", finding_id) + + if not _has_text(finding.get("table_or_page_location")): + warnings.append("missing_table_or_page_location") + _add_missing(missing_by_field, "table_or_page_location", finding_id) + + return { + "finding_id": finding.get("finding_id"), + "ready": not blocking, + "blocking_issues": sorted(set(blocking)), + "warnings": sorted(set(warnings)), + "fields": { + "representative_effect_index": rep_idx, + "effect_indices": valid_indices, + "role": finding.get("role"), + "simulation_target_candidate": target.get("candidate"), + "IV": finding.get("IV"), + "DV": finding.get("DV"), + "reported_statistics_present": _has_reported_statistics(finding), + "table_or_page_location": finding.get("table_or_page_location"), + }, + } + + +def audit_stage2_finding_contract( + stage2_json: Dict[str, Any], + stage1_json: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Return a field-level contract audit for a Stage 2 result.""" + studies = [study for study in stage2_json.get("eligible_studies", []) or [] if isinstance(study, dict)] + missing_by_field: Dict[str, List[str]] = {} + audits: Dict[str, Dict[str, Any]] = {} + ready_studies = 0 + total_findings = 0 + primary_targets = 0 + stage2_keys: Set[str] = set() + + for sidx, study in enumerate(studies): + path = f"$.eligible_studies[{sidx}]" + study_id = str(study.get("study_id") or study.get("study") or f"study_{sidx + 1}") + study_name = str(study.get("study_name") or study.get("study") or study_id) + key = _study_key(study.get("study") or study.get("study_name") or study_id) + if key: + stage2_keys.add(key) + + effects = [effect for effect in study.get("effects", []) or [] if isinstance(effect, dict)] + findings = [finding for finding in study.get("findings", []) or [] if isinstance(finding, dict)] + sample = study.get("sample") if isinstance(study.get("sample"), dict) else {} + blocking: List[str] = [] + warnings: List[str] = [] + + if not _has_text(study_id) or not _has_text(study_name): + blocking.append("missing_study_identity") + _add_missing(missing_by_field, "study_identity", study_id or path) + if not effects: + blocking.append("missing_raw_effects") + _add_missing(missing_by_field, "effects", study_id) + if not findings: + blocking.append("missing_consolidated_findings") + _add_missing(missing_by_field, "findings", study_id) + if not sample.get("total_n") and not sample.get("analyzed_n"): + warnings.append("missing_sample_n") + _add_missing(missing_by_field, "sample_n", study_id) + + finding_audits: Dict[str, Dict[str, Any]] = {} + study_primary_targets = 0 + seen_reps: Set[int] = set() + for fidx, finding in enumerate(findings): + total_findings += 1 + target = finding.get("simulation_target") if isinstance(finding.get("simulation_target"), dict) else {} + if target.get("candidate") is True: + study_primary_targets += 1 + primary_targets += 1 + audit = _finding_audit( + finding, + effects=effects, + path=f"{path}.findings[{fidx}]", + missing_by_field=missing_by_field, + ) + rep_idx = audit["fields"]["representative_effect_index"] + if rep_idx is not None: + if rep_idx in seen_reps: + audit["warnings"].append("duplicate_representative_effect_index") + seen_reps.add(rep_idx) + if not audit["ready"]: + blocking.append(f"finding_not_ready:{finding.get('finding_id') or fidx}") + finding_audits[str(finding.get("finding_id") or f"finding_{fidx + 1}")] = audit + + if findings and study_primary_targets == 0: + blocking.append("no_primary_simulation_target") + _add_missing(missing_by_field, "primary_simulation_target", study_id) + + ready = not blocking + if ready: + ready_studies += 1 + audits[study_id] = { + "study_id": study_id, + "study_name": study_name, + "ready": ready, + "blocking_issues": sorted(set(blocking)), + "warnings": sorted(set(warnings)), + "raw_effects": len(effects), + "findings_total": len(findings), + "primary_simulation_targets": study_primary_targets, + "fields": { + "study_identity": "present" if _has_text(study_id) and _has_text(study_name) else "missing", + "sample_n": "present" if sample.get("total_n") or sample.get("analyzed_n") else "missing", + "raw_effects": len(effects), + "consolidated_findings": len(findings), + }, + "findings": finding_audits, + } + + stage1_keys = _stage1_eligible_keys(stage1_json) + coverage = { + "checked": bool(stage1_keys), + "stage1_eligible_or_uncertain": len(stage1_keys), + "stage2_studies": len(stage2_keys), + "missing_from_stage2": sorted(stage1_keys - stage2_keys), + "extra_in_stage2": sorted(stage2_keys - stage1_keys) if stage1_keys else [], + } + for key in coverage["missing_from_stage2"]: + _add_missing(missing_by_field, "stage1_coverage", key) + + contract_blockers = sum( + len(study_audit.get("blocking_issues") or []) + for study_audit in audits.values() + ) + len(coverage["missing_from_stage2"]) + + return { + "version": "stage2_finding_contract_v1", + "total_studies": len(studies), + "ready": ready_studies, + "needs_review": len(studies) - ready_studies, + "total_findings": total_findings, + "primary_simulation_targets": primary_targets, + "blocking_issue_count": contract_blockers, + "missing_by_field": {field: sorted(set(refs)) for field, refs in sorted(missing_by_field.items())}, + "stage1_stage2_coverage": coverage, + "studies": audits, + } + + +def apply_stage2_finding_contract( + stage2_json: Dict[str, Any], + stage1_json: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Attach the Stage 2 finding contract audit in-place and return it.""" + contract = audit_stage2_finding_contract(stage2_json, stage1_json) + stage2_json["stage2_finding_contract"] = contract + return contract diff --git a/generation_pipeline/stage2_findings.py b/generation_pipeline/stage2_findings.py new file mode 100644 index 0000000..fe02985 --- /dev/null +++ b/generation_pipeline/stage2_findings.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import re +from typing import Any, Dict, List + +from generation_pipeline.parsers.effect_consolidator import annotate_study +from generation_pipeline.stage2_finding_contract import apply_stage2_finding_contract + + +def _slug(value: Any, fallback: str = "study") -> str: + text = re.sub(r"[^a-z0-9]+", "_", str(value or "").lower()).strip("_") + return text or fallback + + +def _reported_statistics(effect: Dict[str, Any]) -> str: + stats = effect.get("stats") if isinstance(effect.get("stats"), dict) else {} + parts: List[str] = [] + for key in ("B", "b", "t", "f", "z", "chi_square", "D", "eta_square", "p_value", "sig"): + value = stats.get(key) + if value not in (None, "", [], [None, None]): + parts.append(f"{key}={value}") + ci = stats.get("ci") + if isinstance(ci, list) and any(item is not None for item in ci): + parts.append(f"ci={ci}") + text = effect.get("reported_statistics_text") + if isinstance(text, str) and text.strip(): + parts.append(text.strip()) + return ", ".join(parts) + + +def _effect_at(effects: List[Dict[str, Any]], index: Any) -> Dict[str, Any]: + try: + idx = int(index) + except (TypeError, ValueError): + return {} + if idx < 0 or idx >= len(effects): + return {} + effect = effects[idx] + return effect if isinstance(effect, dict) else {} + + +def _valid_effect_index(effects: List[Dict[str, Any]], index: Any) -> int | None: + try: + idx = int(index) + except (TypeError, ValueError): + return None + if 0 <= idx < len(effects): + return idx + return None + + +def _finding_from_group( + *, + study_id: str, + study_name: str, + effects: List[Dict[str, Any]], + group: Dict[str, Any], + ordinal: int, +) -> Dict[str, Any]: + rep_index = _valid_effect_index(effects, group.get("representative_effect_index")) + representative = _effect_at(effects, rep_index) + effect_indices = [ + idx + for idx in (_valid_effect_index(effects, value) for value in group.get("effect_indices", []) or []) + if idx is not None + ] + if rep_index is not None and rep_index not in effect_indices: + effect_indices.insert(0, rep_index) + role = str(group.get("analysis_role") or "other") + is_target = bool(group.get("is_primary_simulation_target")) + finding_id = f"{_slug(study_id)}__finding_{ordinal:02d}" + return { + "finding_id": finding_id, + "study_id": study_id, + "study_name": study_name, + "merge_group_id": group.get("merge_group_id"), + "representative_effect_index": rep_index, + "effect_indices": effect_indices, + "role": role, + "simulation_target": { + "candidate": is_target, + "reason": "primary consolidated effect" if is_target else f"demoted role: {role}", + }, + "IV": representative.get("IV"), + "DV": representative.get("DV"), + "effecttype": representative.get("effecttype"), + "direction": representative.get("direction"), + "table_or_page_location": representative.get("table_or_page_location"), + "reported_statistics": _reported_statistics(representative), + "stats": representative.get("stats") if isinstance(representative.get("stats"), dict) else {}, + "materials_link_hints": { + "materials_status": (representative.get("materials") or {}).get("status") if isinstance(representative.get("materials"), dict) else None, + "manipulation_status": (representative.get("manipulation") or {}).get("status") if isinstance(representative.get("manipulation"), dict) else None, + "items_status": (representative.get("items") or {}).get("status") if isinstance(representative.get("items"), dict) else None, + }, + } + + +def _has_reported_statistics(finding: Dict[str, Any]) -> bool: + if str(finding.get("reported_statistics") or "").strip(): + return True + stats = finding.get("stats") if isinstance(finding.get("stats"), dict) else {} + for key in ("B", "b", "t", "f", "z", "chi_square", "D", "eta_square", "p_value", "sig"): + if stats.get(key) not in (None, "", [], [None, None]): + return True + ci = stats.get("ci") + return isinstance(ci, list) and any(item is not None for item in ci) + + +def _ensure_fallback_simulation_target(findings: List[Dict[str, Any]]) -> None: + """Promote one valid finding when a study only has demoted simple effects.""" + if any(finding.get("simulation_target", {}).get("candidate") is True for finding in findings): + return + demoted_only_roles = {"secondary_dv", "manipulation_check", "check", "mediation", "correlation"} + for finding in findings: + role = str(finding.get("role") or "").strip().lower() + if role in demoted_only_roles: + continue + if not _has_reported_statistics(finding): + continue + target = finding.setdefault("simulation_target", {}) + target["candidate"] = True + target["reason"] = ( + "fallback primary target: no main/primary target was available; " + f"promoted reported {role or 'finding'}" + ) + target["fallback"] = True + return + + +def annotate_stage2_findings( + paper: Dict[str, Any], + *, + recompute_consolidation: bool = True, + stage1_json: Dict[str, Any] | None = None, +) -> Dict[str, Any]: + """Attach `findings[]` and `stage2_finding_summary` to a Stage 2 paper dict.""" + studies = [study for study in paper.get("eligible_studies", []) or [] if isinstance(study, dict)] + total_findings = 0 + primary_targets = 0 + by_role: Dict[str, int] = {} + + for study_index, study in enumerate(studies, start=1): + study_name = str(study.get("study") or study.get("name") or f"Study {study_index}") + study_id = _slug(study_name, f"study_{study_index}") + study["study_id"] = study_id + study["study_name"] = study_name + if recompute_consolidation or not isinstance(study.get("consolidation_summary"), dict): + annotate_study(study, study_id=study_id) + effects = [effect for effect in study.get("effects", []) or [] if isinstance(effect, dict)] + groups = study.get("consolidation_summary", {}).get("groups", []) + findings = [ + _finding_from_group( + study_id=study_id, + study_name=study_name, + effects=effects, + group=group, + ordinal=ordinal, + ) + for ordinal, group in enumerate(groups, start=1) + if isinstance(group, dict) + ] + _ensure_fallback_simulation_target(findings) + study["findings"] = findings + total_findings += len(findings) + for finding in findings: + role = str(finding.get("role") or "other") + by_role[role] = by_role.get(role, 0) + 1 + if finding.get("simulation_target", {}).get("candidate"): + primary_targets += 1 + + paper["stage2_finding_summary"] = { + "version": "stage2_findings_v1", + "studies_total": len(studies), + "raw_effects_total": sum(len(study.get("effects", []) or []) for study in studies), + "findings_total": total_findings, + "primary_simulation_target_candidates": primary_targets, + "by_role": by_role, + } + apply_stage2_finding_contract(paper, stage1_json) + return paper diff --git a/generation_pipeline/stage2_verifier.py b/generation_pipeline/stage2_verifier.py new file mode 100644 index 0000000..0e1f2d4 --- /dev/null +++ b/generation_pipeline/stage2_verifier.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import json +import re +import time +from pathlib import Path +from typing import Any, Dict, List, Optional + +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + + +PDF_TEXT_MAX_CHARS = 180000 +DEFAULT_MAX_TOKENS = 8000 +RAW_PREVIEW_CHARS = 700 + + +def _compact_stage1(stage1_json: Dict[str, Any]) -> List[Dict[str, Any]]: + out: List[Dict[str, Any]] = [] + for exp in stage1_json.get("experiments", []) or []: + if not isinstance(exp, dict): + continue + out.append( + { + "experiment_id": exp.get("experiment_id"), + "study_id": exp.get("study_id"), + "experiment_name": exp.get("experiment_name"), + "study_name": exp.get("study_name"), + "replicable": exp.get("replicable"), + "design_type": exp.get("design_type"), + "conditions_or_factors": exp.get("conditions_or_factors"), + "input": exp.get("input"), + "participant_task": exp.get("participant_task"), + "output": exp.get("output"), + "candidate_source_hints": exp.get("candidate_source_hints"), + "missing_materials": exp.get("missing_materials"), + } + ) + return out + + +def _compact_stage2(stage2_json: Dict[str, Any]) -> List[Dict[str, Any]]: + studies: List[Dict[str, Any]] = [] + for study in stage2_json.get("eligible_studies", []) or []: + if not isinstance(study, dict): + continue + studies.append( + { + "study": study.get("study"), + "study_id": study.get("study_id"), + "sample": study.get("sample"), + "raw_effects": len(study.get("effects", []) or []), + "findings": [ + { + "finding_id": finding.get("finding_id"), + "role": finding.get("role"), + "simulation_target": finding.get("simulation_target"), + "representative_effect_index": finding.get("representative_effect_index"), + "effect_indices": finding.get("effect_indices"), + "IV": finding.get("IV"), + "DV": finding.get("DV"), + "reported_statistics": finding.get("reported_statistics"), + "location": finding.get("table_or_page_location"), + } + for finding in study.get("findings", []) or [] + if isinstance(finding, dict) + ], + } + ) + return studies + + +def _loads_json(text: str) -> Dict[str, Any]: + response_text = text.strip() + if "```json" in response_text: + response_text = response_text.split("```json", 1)[1].split("```", 1)[0].strip() + elif "```" in response_text: + response_text = response_text.split("```", 1)[1].split("```", 1)[0].strip() + try: + parsed = json.loads(response_text) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", response_text, re.DOTALL) + if not match: + raise + parsed = json.loads(match.group()) + if not isinstance(parsed, dict): + raise ValueError(f"Verifier response is not a JSON object: {type(parsed)}") + return parsed + + +def _raw_preview(value: Any) -> str: + text = str(value or "").replace("\n", "\\n") + return text[:RAW_PREVIEW_CHARS] + + +def _retry_prompt(original_prompt: str, bad_response: str) -> str: + return ( + original_prompt + + "\n\nYour previous response could not be parsed as JSON. " + "Return ONLY the compact JSON object requested above. Do not include " + "markdown, prose, headings, or analysis. Previous response preview:\n" + + _raw_preview(bad_response) + ) + + +def _normalize_report(report: Dict[str, Any]) -> Dict[str, Any]: + overall = str(report.get("overall") or "needs_review").strip().lower() + if overall not in {"pass", "needs_review", "fail"}: + overall = "needs_review" + normalized = { + "version": "stage2_verifier_v1", + "status": "ok", + "overall": overall, + "confidence": report.get("confidence"), + "study_coverage": report.get("study_coverage") if isinstance(report.get("study_coverage"), list) else [], + "finding_checks": report.get("finding_checks") if isinstance(report.get("finding_checks"), list) else [], + "regeneration_instructions": report.get("regeneration_instructions") + if isinstance(report.get("regeneration_instructions"), dict) + else {}, + "notes": report.get("notes", ""), + } + regen = normalized["regeneration_instructions"] + regen.setdefault("missing_effects", []) + regen.setdefault("exact_stats_needed", []) + regen.setdefault("data_corrections", []) + return normalized + + +def build_verifier_prompt(stage1_json: Dict[str, Any], stage2_json: Dict[str, Any], pdf_text: str) -> str: + """Build the Stage 2 verifier prompt. Exposed for unit tests.""" + return f"""You are verifying an extraction from a psychology/management paper. + +Check whether Stage 2 correctly covers the eligible studies and whether each +consolidated finding is supported by the paper text. Focus on study/effect/ +finding/statistics correctness. Do NOT evaluate whether participant materials +are complete; Stage 3 handles materials. + +Keep the verifier output compact: include every study in study_coverage, but in +finding_checks include only findings with problems or the most important +borderline cases. Do not list every ok finding. + +PAPER TEXT: +{pdf_text} + +STAGE 1 STUDY INVENTORY: +{json.dumps(_compact_stage1(stage1_json), ensure_ascii=False, indent=2)} + +STAGE 2 CONSOLIDATED FINDINGS: +{json.dumps(_compact_stage2(stage2_json), ensure_ascii=False, indent=2)} + +Return ONLY a compact JSON object with this schema. Do not include markdown, +analysis, commentary, or chain-of-thought. Keep evidence strings short. +{{ + "overall": "pass|needs_review|fail", + "confidence": 0.0, + "study_coverage": [ + {{ + "study": "Study 1", + "verdict": "ok|missing|extra|split_issue|needs_review", + "issue": "short issue or empty", + "evidence": "short paper evidence" + }} + ], + "finding_checks": [ + {{ + "finding_id": "study_1__finding_01", + "verdict": "ok|unsupported|wrong_stats|wrong_role|duplicate|needs_review", + "issue": "short issue or empty", + "evidence": "short paper evidence" + }} + ], + "regeneration_instructions": {{ + "missing_effects": ["..."], + "exact_stats_needed": [{{"finding_id": "...", "reason": "..."}}], + "data_corrections": [{{"path": "$.eligible_studies[0]...", "reason": "..."}}] + }}, + "notes": "one short summary" +}}""" + + +def verify_stage2_findings( + stage2_json: Dict[str, Any], + stage1_json: Dict[str, Any], + pdf_path: Path, + llm_client: Any, + *, + pdf_text: Optional[str] = None, + timeout: Optional[float] = 60.0, + max_tokens: Optional[int] = DEFAULT_MAX_TOKENS, + max_attempts: int = 2, + retry_delay: float = 1.0, +) -> Dict[str, Any]: + if llm_client is None: + return { + "version": "stage2_verifier_v1", + "status": "skipped", + "overall": "needs_review", + "notes": "No LLM client provided.", + "study_coverage": [], + "finding_checks": [], + "regeneration_instructions": { + "missing_effects": [], + "exact_stats_needed": [], + "data_corrections": [], + }, + } + text = pdf_text if pdf_text is not None else extract_pdf_text(Path(pdf_path), max_chars=PDF_TEXT_MAX_CHARS) + prompt = build_verifier_prompt(stage1_json, stage2_json, text) + attempts = max(1, int(max_attempts or 1)) + last_error: Optional[BaseException] = None + last_response: Optional[str] = None + current_prompt = prompt + for attempt in range(1, attempts + 1): + try: + try: + response = llm_client.generate_content(prompt=current_prompt, timeout=timeout, max_tokens=max_tokens) + except TypeError: + try: + response = llm_client.generate_content(prompt=current_prompt, timeout=timeout) + except TypeError: + response = llm_client.generate_content(prompt=current_prompt) + if response is None: + raise ValueError("Stage 2 verifier LLM returned None.") + last_response = str(response) + return _normalize_report(_loads_json(last_response)) + except Exception as exc: + last_error = exc + if attempt >= attempts: + break + if last_response is not None: + current_prompt = _retry_prompt(prompt, last_response) + if retry_delay > 0: + time.sleep(retry_delay) + if last_error is not None: + if last_response is not None: + raise ValueError( + f"{type(last_error).__name__}: {last_error}; " + f"raw_response_preview={_raw_preview(last_response)}" + ) from last_error + raise last_error + raise ValueError("Stage 2 verifier failed without an exception.") diff --git a/generation_pipeline/stage3.py b/generation_pipeline/stage3.py new file mode 100644 index 0000000..d332e61 --- /dev/null +++ b/generation_pipeline/stage3.py @@ -0,0 +1,1634 @@ +from __future__ import annotations + +import json +import re +import zipfile +from pathlib import Path +from typing import Any, Dict, List, Optional +from xml.etree import ElementTree as ET + +from generation_pipeline.parsers.effect_consolidator import annotate_study +from generation_pipeline.parsers.material_assembler import assemble_study_materials +from generation_pipeline.parsers.source_linker import LinkResult, link_sources +from generation_pipeline.parsers.stage3_adapter import match_stage3_study +from generation_pipeline.stage3_material_contract import apply_material_contracts +from generation_pipeline.stage3_pdf_materials import PDF_TEXT_MAX_CHARS, extract_pdf_study_materials + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _eligible_of(data: Dict[str, Any]) -> List[Dict[str, Any]]: + return data.get("eligible_studies") or data.get("studies") or [] + + +# --------------------------------------------------------------------------- # +# Unified Stage 3 entry: stage2.json -> stage3.json (+ stage3.md) +# --------------------------------------------------------------------------- # + +def _find_stage_file(paper_dir: Path, stem: str) -> Optional[Path]: + """Locate a stage file by canonical name only.""" + canonical = paper_dir / f"{stem}.json" + if canonical.exists(): + return canonical + return None + + +def _find_pdf(paper_dir: Path) -> Optional[Path]: + pdfs = sorted(p for p in paper_dir.glob("*.pdf")) + return pdfs[0] if pdfs else None + + +def _resolve_osf_files_dir(paper_dir: Path, osf_files_dir: Optional[Path]) -> Path: + """Resolve a Stage 3 OSF/source directory to the actual files directory.""" + candidates: List[Path] = [] + if osf_files_dir is not None: + src = Path(osf_files_dir) + candidates.extend( + [ + src / "osf" / "files", + src / "sources" / "osf" / "files", + src / "files", + src, + ] + ) + candidates.extend( + [ + paper_dir / "sources" / "osf" / "files", + paper_dir / "osf" / "files", + ] + ) + for candidate in candidates: + if candidate.exists() and candidate.is_dir(): + return candidate + return candidates[0] if candidates else paper_dir / "sources" / "osf" / "files" + + +def run_stage3( + paper_dir: str | Path, + *, + stage2_path: Optional[Path] = None, + pdf_path: Optional[Path] = None, + osf_files_dir: Optional[Path] = None, + filler: Any = None, + llm_client: Any = None, + backup: bool = True, + write: bool = True, + allow_effect_slot_fallback: bool = False, + selection_votes: int = 3, + selection_timeout: Optional[float] = 60.0, + pdf_material_timeout: Optional[float] = 120.0, +) -> Dict[str, Any]: + """Unified Stage 3 for one paper: produce a canonical `stage3.json` + `stage3.md`. + + Unlike the legacy corpus patch runner (which rewrites the Stage 2 file in + place), this writes Stage 3 as its OWN artifact, so the pipeline contract is + `stage2.json -> stage3.json (+ .md)`: + + 1. Load stage2.json. + 2. Optionally slot-fill from the PDF/sources when a `filler` is explicitly + given. Stage 2 is still left untouched. + 3. Write stage3.json. + 4. Run OSF material assembly + effect consolidation + (optional) study + selection, enriching stage3.json (`study_materials`, consolidation, …). + 5. Write a human-readable stage3.md summary. + + Returns {stage3_json, stage3_md, materials}. + """ + paper_dir = Path(paper_dir) + stage2_path = stage2_path or _find_stage_file(paper_dir, "stage2") + if stage2_path is None or not Path(stage2_path).exists(): + raise FileNotFoundError(f"stage2.json not found under {paper_dir}") + stage3_path = paper_dir / "stage3.json" + + paper = json.loads(Path(stage2_path).read_text(encoding="utf-8")) + + # --- slot filling (stage2 -> stage3 content), never in place on stage2 ---- + if filler is not None: + pdf_path = pdf_path or _find_pdf(paper_dir) + if pdf_path is None: + raise FileNotFoundError(f"No PDF found under {paper_dir} for slot filling") + from generation_pipeline.patchers.patch_runner import discover_source_dirs + from generation_pipeline.verification.schema_validator import validate_paper + + source_dirs = discover_source_dirs(Path(stage2_path), Path(pdf_path)) + patched = filler.patch_paper(paper, Path(pdf_path), source_dirs=source_dirs or None) + patched, _ = validate_paper(patched, repair=True, path=stage3_path) + data = patched + else: + data = paper + + if write: + _atomic_write(stage3_path, data, backup=backup) + + # --- OSF assembly + consolidation + selection (enriches stage3.json) ------ + materials = build_study_materials( + paper_dir, + stage3_path=stage3_path, + osf_files_dir=osf_files_dir, + pdf_path=pdf_path, + llm_client=llm_client, + write=write, + allow_effect_slot_fallback=allow_effect_slot_fallback, + selection_votes=selection_votes, + selection_timeout=selection_timeout, + pdf_material_timeout=pdf_material_timeout, + ) + + stage3_md = paper_dir / "stage3.md" + if write: + # reload to pick up the enrichment build_study_materials wrote back + enriched = json.loads(stage3_path.read_text(encoding="utf-8")) + stage3_md.write_text(_render_stage3_md(enriched, materials), encoding="utf-8") + + return {"stage3_json": stage3_path, "stage3_md": stage3_md, "materials": materials} + + +def _atomic_write(path: Path, data: Dict[str, Any], *, backup: bool) -> None: + if backup and path.exists(): + bak = path.with_suffix(path.suffix + ".bak") + if not bak.exists(): + bak.write_text(path.read_text(encoding="utf-8"), encoding="utf-8") + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + tmp.replace(path) + + +def _render_stage3_md(data: Dict[str, Any], materials: Dict[str, Dict[str, Any]]) -> str: + """Human-readable Stage 3 summary (the canonical stage3.md).""" + title = data.get("paper_title") or data.get("paper_id") or "(untitled)" + lines = [f"# Stage 3 — {title}", ""] + + cons = data.get("effect_consolidation_summary") + if cons: + lines += [ + "## Effects", + f"- raw effects: {cons.get('raw_effects')}", + f"- consolidated: {cons.get('consolidated_effects')}", + f"- primary simulation targets: {cons.get('primary_simulation_targets')}", + f"- by role: {cons.get('by_role')}", + "", + ] + sel = data.get("study_selection_summary") + if sel: + lines += [ + "## Study selection", + f"- kept: {sel.get('kept')}", + f"- dropped: {sel.get('dropped')}", + "", + ] + osf = data.get("osf_materials_summary") + if osf: + lines += [ + "## Materials", + f"- ready: {osf.get('ready')}/{osf.get('total_sub_studies')}", + f"- by source: {osf.get('by_primary_source')}", + f"- needs review: {osf.get('unready')}", + "", + ] + contract = data.get("human_material_contract") + if contract: + missing = contract.get("missing_by_field") or {} + missing_source = contract.get("missing_source_by_field") or {} + selected_missing = contract.get("selected_missing_by_field") or {} + selected_missing_source = contract.get("selected_missing_source_by_field") or {} + lines += [ + "## HumanStudy material contract", + f"- ready: {contract.get('ready')}/{contract.get('total_sub_studies')}", + f"- needs patch: {contract.get('needs_patch')}", + f"- selected ready: {contract.get('selected_ready', 'N/A')}/{contract.get('selected_total', 'N/A')}", + f"- selected needs patch: {contract.get('selected_needs_patch', 'N/A')}", + f"- missing by field: {missing}", + f"- selected missing by field: {selected_missing}", + f"- missing source by field: {missing_source}", + f"- selected missing source by field: {selected_missing_source}", + "", + ] + target_summary = data.get("simulation_target_summary") + if target_summary: + lines += [ + "## Simulation targets", + f"- total targets: {target_summary.get('total_targets')}", + f"- by sub-study: {target_summary.get('by_sub_study')}", + "", + ] + requirements = data.get("material_requirements_summary") + if isinstance(requirements, dict): + lines += [ + "## Material requirements", + f"- selected ready: {requirements.get('selected_ready')}/{requirements.get('selected_total')}", + f"- selected needs patch: {requirements.get('selected_needs_patch')}", + f"- selected missing by field: {requirements.get('selected_missing_by_field')}", + f"- selected blocking by issue: {requirements.get('selected_blocking_by_issue')}", + f"- target-bound sub-studies: {requirements.get('target_bound_sub_studies')}", + "", + ] + + lines += ["## Per sub-study", "", + "| sub_study | selected | source | items | ready |", + "|---|:--:|---|:--:|:--:|"] + for sid, m in materials.items(): + sel_keep = m.get("selection", {}).get("keep", True) + src = m.get("source_trace", {}).get("primary_source") + ready = m.get("readiness", {}).get("ready") + lines.append(f"| {sid} | {'✅' if sel_keep else '—'} | {src} | {len(m.get('items', []))} | {'✅' if ready else '⚠'} |") + lines.append("") + return "\n".join(lines) + + +def _study_ids(eligible: List[Dict[str, Any]], paper_dir: Path) -> List[str]: + ids = [ + str(s.get("study") or s.get("study_id") or s.get("experiment_id") or "").strip() + for s in eligible + ] + ids = [i for i in ids if i] + if ids: + return ids + # fall back to Stage 1 experiment ids + stage1 = paper_dir / "stage1.json" + if stage1.exists(): + data = json.loads(stage1.read_text(encoding="utf-8")) + return [e.get("experiment_id") for e in data.get("experiments", []) if e.get("experiment_id")] + return [] + + +def _load_stage1_json(paper_dir: Path) -> Dict[str, Any]: + stage1 = paper_dir / "stage1.json" + if not stage1.exists(): + return {} + try: + data = json.loads(stage1.read_text(encoding="utf-8")) + except Exception: + return {} + return data if isinstance(data, dict) else {} + + +def _stage1_experiment_map(stage1_json: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + for exp in stage1_json.get("experiments", []) or []: + if not isinstance(exp, dict): + continue + for key in ( + exp.get("study_id"), + exp.get("experiment_id"), + exp.get("study_name"), + exp.get("experiment_name"), + ): + sid = _material_slug(key) + if sid and sid not in out: + out[sid] = exp + return out + + +def _targets_by_sub_id(targets: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]: + out: Dict[str, List[Dict[str, Any]]] = {} + for target in targets: + if not isinstance(target, dict): + continue + sid = _material_slug(target.get("sub_study_id") or target.get("study_name")) + if sid: + out.setdefault(sid, []).append(target) + return out + + +def _stage_study_map(eligible: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + for study in eligible: + if not isinstance(study, dict): + continue + for key in (study.get("study_id"), study.get("study"), study.get("experiment_id")): + sid = _material_slug(key) + if sid and sid not in out: + out[sid] = study + return out + + +def _effect_for_target(study: Optional[Dict[str, Any]], target: Dict[str, Any]) -> Dict[str, Any]: + if not isinstance(study, dict): + return {} + try: + index = int(target.get("effect_index")) + except (TypeError, ValueError): + return {} + effects = study.get("effects") if isinstance(study.get("effects"), list) else [] + if 0 <= index < len(effects) and isinstance(effects[index], dict): + return effects[index] + return {} + + +def _contract_status(material: Dict[str, Any], field: str) -> tuple[str, str]: + contract = material.get("human_material_contract") if isinstance(material.get("human_material_contract"), dict) else {} + fields = contract.get("fields") if isinstance(contract.get("fields"), dict) else {} + evidence = contract.get("field_evidence") if isinstance(contract.get("field_evidence"), dict) else {} + field_payload = fields.get(field) if isinstance(fields.get(field), dict) else {} + evidence_payload = evidence.get(field) if isinstance(evidence.get(field), dict) else {} + return str(field_payload.get("status") or "missing"), str(evidence_payload.get("status") or "missing") + + +def _required_field_record(material: Dict[str, Any], field: str, *, required: bool, purpose: str) -> Dict[str, Any]: + status, source_status = _contract_status(material, field) + requires_source = field != "response_schema" + if not requires_source and source_status == "missing": + source_status = "not_applicable" + missing = status == "missing" or (required and requires_source and source_status == "missing") + return { + "field": field, + "required": required, + "purpose": purpose, + "status": status, + "source_status": source_status, + "missing": missing, + } + + +def apply_material_requirements( + data: Dict[str, Any], + materials: Dict[str, Dict[str, Any]], + *, + eligible: List[Dict[str, Any]], + targets: List[Dict[str, Any]], + stage1_json: Dict[str, Any], +) -> Dict[str, Any]: + """Attach final HumanStudy-Bench field requirements to Stage 3 materials.""" + stage1_map = _stage1_experiment_map(stage1_json) + study_map = _stage_study_map(eligible) + target_map = _targets_by_sub_id(targets) + summary_missing: Dict[str, List[str]] = {} + selected_missing: Dict[str, List[str]] = {} + blocking_by_issue: Dict[str, List[str]] = {} + selected_blocking_by_issue: Dict[str, List[str]] = {} + selected_total = 0 + selected_ready = 0 + + for sid, material in materials.items(): + if not isinstance(material, dict): + continue + study = study_map.get(_material_slug(sid)) + stage1_exp = stage1_map.get(_material_slug(sid)) + study_targets = target_map.get(_material_slug(sid), []) + selected = material.get("selection", {}).get("keep", True) is not False + if selected: + selected_total += 1 + + target_effects = [_effect_for_target(study, target) for target in study_targets] + target_ivs = sorted({str(effect.get("IV")) for effect in target_effects if effect.get("IV")}) + target_dvs = sorted({str(effect.get("DV")) for effect in target_effects if effect.get("DV")}) + conditions_expected = bool((stage1_exp or {}).get("conditions_or_factors") or target_ivs) + source_hints = (stage1_exp or {}).get("candidate_source_hints") + source_hints = source_hints if isinstance(source_hints, list) else [] + + fields = [ + _required_field_record( + material, + "instructions", + required=True, + purpose="participant-facing task framing, stimulus, vignette, or manipulation text", + ), + _required_field_record( + material, + "response_items", + required=True, + purpose="participant response questions/items consumed by the simulator", + ), + _required_field_record( + material, + "response_options", + required=True, + purpose="answer options, anchors, or scale bounds for every response item", + ), + _required_field_record( + material, + "response_schema", + required=True, + purpose="dominant simulator answer type plus options/anchors/scale metadata", + ), + _required_field_record( + material, + "conditions", + required=conditions_expected, + purpose="manipulated/measured factors and levels needed to instantiate trials", + ), + ] + source_record = { + "field": "source_evidence", + "required": True, + "purpose": "field-level source provenance for human review and patching", + "status": "present" if material.get("source_trace") else "missing", + "source_status": "present" if material.get("source_trace") else "missing", + "missing": not bool(material.get("source_trace")), + } + fields.append(source_record) + + missing_required = [field["field"] for field in fields if field["required"] and field["missing"]] + expected_missing = [field["field"] for field in fields if field["missing"]] + readiness = material.get("readiness") if isinstance(material.get("readiness"), dict) else {} + readiness_blocking = [ + str(issue) + for issue in readiness.get("blocking_issues", []) or [] + if str(issue).strip() + ] + ready = not missing_required and not readiness_blocking and readiness.get("ready") is not False + if selected and ready: + selected_ready += 1 + for field in expected_missing: + summary_missing.setdefault(field, []).append(sid) + if selected: + selected_missing.setdefault(field, []).append(sid) + for issue in readiness_blocking: + blocking_by_issue.setdefault(issue, []).append(sid) + if selected: + selected_blocking_by_issue.setdefault(issue, []).append(sid) + + search_terms = sorted( + _tokens( + *(target_ivs + target_dvs), + (stage1_exp or {}).get("participant_task"), + (stage1_exp or {}).get("input"), + *((hint.get("description") if isinstance(hint, dict) else hint) for hint in source_hints), + ) + ) + material["material_requirements"] = { + "version": "human-study-bench-material-requirements-v1", + "sub_study_id": sid, + "selected": selected, + "ready": ready, + "target_ids": [target.get("target_id") for target in study_targets if target.get("target_id")], + "target_effect_indices": [target.get("effect_index") for target in study_targets], + "target_ivs": target_ivs, + "target_dvs": target_dvs, + "required_fields": fields, + "missing_required_fields": missing_required, + "missing_expected_fields": expected_missing, + "readiness_blocking_issues": sorted(set(readiness_blocking)), + "stage1_design": { + "experiment_id": (stage1_exp or {}).get("experiment_id"), + "study_id": (stage1_exp or {}).get("study_id"), + "design_type": (stage1_exp or {}).get("design_type"), + "conditions_or_factors": (stage1_exp or {}).get("conditions_or_factors", []), + "participant_task": (stage1_exp or {}).get("participant_task"), + }, + "source_hints": source_hints, + "search_terms": search_terms[:40], + } + + summary = { + "version": "human-study-bench-material-requirements-v1", + "total_sub_studies": len(materials), + "selected_total": selected_total, + "selected_ready": selected_ready, + "selected_needs_patch": selected_total - selected_ready, + "missing_by_field": {field: sorted(set(sids)) for field, sids in sorted(summary_missing.items())}, + "selected_missing_by_field": {field: sorted(set(sids)) for field, sids in sorted(selected_missing.items())}, + "blocking_by_issue": {issue: sorted(set(sids)) for issue, sids in sorted(blocking_by_issue.items())}, + "selected_blocking_by_issue": { + issue: sorted(set(sids)) + for issue, sids in sorted(selected_blocking_by_issue.items()) + }, + "target_bound_sub_studies": sorted(target_map), + } + data["material_requirements_summary"] = summary + return summary + + +def build_study_materials( + paper_dir: str | Path, + *, + stage3_path: Optional[Path] = None, + osf_files_dir: Optional[Path] = None, + pdf_path: Optional[Path] = None, + llm_client: Any = None, + write: bool = True, + allow_effect_slot_fallback: bool = False, + selection_votes: int = 3, + selection_timeout: Optional[float] = 60.0, + pdf_material_timeout: Optional[float] = 120.0, +) -> Dict[str, Dict[str, Any]]: + """Assemble per-sub-study materials for a paper and (optionally) persist them. + + Returns a mapping {sub_study_id: materials_dict}. When `write=True`, the + mapping plus a short summary is written back into `stage3.json` under + `study_materials` / `osf_materials_summary`. + + By default, per-effect `materials/manipulation/items` slots are not treated + as final materials. They may be recorded in `source_trace.legacy_slot_summary` + for human review, but `ready` remains false until a structured source or + later Stage 4 patch provides participant-facing instructions and items. + + When `llm_client` is provided, also runs the study selector (which sub-studies + are simulation-worthy) and annotates each study with a `selection` block. + """ + paper_dir = Path(paper_dir) + stage3_path = stage3_path or (paper_dir / "stage3.json") + if not stage3_path.exists(): + raise FileNotFoundError(f"stage3.json not found: {stage3_path}") + + osf_files_dir = _resolve_osf_files_dir(paper_dir, osf_files_dir) + data = json.loads(stage3_path.read_text(encoding="utf-8")) + stage1_json = _load_stage1_json(paper_dir) + eligible = _eligible_of(data) + study_ids = _study_ids(eligible, paper_dir) + + # 1. consolidate + select effects (dedup over-segmentation, role tagging) + for study in eligible: + annotate_study(study) + + # 1a. LLM re-judgment of effect roles (manip-checks, mislabelled causal + # effects, secondary DVs) — overrides the deterministic effecttype roles. + if llm_client is not None: + from generation_pipeline.effect_selector import apply_to_study + title = data.get("paper_title") or "" + for study in eligible: + try: + apply_to_study(study, title, llm_client, votes=selection_votes, timeout=selection_timeout) + except Exception: # fail-open: keep deterministic roles + pass + + # 1b. study-level selection (which sub-studies are simulation-worthy) + if llm_client is not None: + _annotate_selection( + paper_dir, + eligible, + study_ids, + data, + llm_client, + votes=selection_votes, + timeout=selection_timeout, + ) + + # Keep study-level findings aligned with any LLM role overrides above. + from generation_pipeline.stage2_findings import annotate_stage2_findings + annotate_stage2_findings(data, recompute_consolidation=False) + + # 2. link OSF sources + assemble per-study materials + if osf_files_dir.exists(): + link: LinkResult = link_sources(study_ids, osf_files_dir) + else: + # no OSF files at all → empty link result; assembler falls back to Stage 3 + link = LinkResult(by_study={sid: [] for sid in study_ids}, unlinked=[]) + + materials: Dict[str, Dict[str, Any]] = {} + material_study_ids: Dict[str, str] = {} + for sid in study_ids: + s3_study = match_stage3_study(sid, eligible) + built = assemble_study_materials( + sid, + link, + repo_root=REPO_ROOT, + stage3_study=s3_study, + allow_pdf_slot_fallback=allow_effect_slot_fallback, + ) + if isinstance(s3_study, dict) and isinstance(s3_study.get("selection"), dict): + built["selection"] = s3_study["selection"] + materials[built["sub_study_id"]] = built + material_study_ids[built["sub_study_id"]] = sid + + if llm_client is not None: + _merge_linked_source_materials( + materials, + data, + link, + material_study_ids, + llm_client, + stage1_json=stage1_json, + timeout=pdf_material_timeout, + ) + + if pdf_path is not None and _selected_material_needs_source_completion(materials): + try: + _merge_pdf_materials( + materials, + extract_pdf_study_materials( + data, + Path(pdf_path), + llm_client, + timeout=pdf_material_timeout, + stage1_json=stage1_json, + ), + ) + except Exception as exc: + data["pdf_material_extraction_error"] = f"{type(exc).__name__}: {exc}" + for material in materials.values(): + readiness = material.setdefault("readiness", {"ready": False, "blocking_issues": [], "warnings": []}) + warnings = readiness.setdefault("warnings", []) + warnings.append(f"PDF material extraction failed; material left as source-only draft: {type(exc).__name__}: {exc}") + + _apply_stage2_material_scaffolds(materials, eligible) + _complete_sparse_materials_from_siblings(materials, eligible) + data["human_material_contract"] = apply_material_contracts(materials) + + from generation_pipeline.simulation_targets import ( + build_simulation_targets, + summarize_simulation_targets, + ) + targets = build_simulation_targets( + eligible, + materials, + paper_title=str(data.get("paper_title") or ""), + ) + data["simulation_targets"] = targets + data["simulation_target_summary"] = summarize_simulation_targets(targets) + apply_material_requirements( + data, + materials, + eligible=eligible, + targets=targets, + stage1_json=stage1_json, + ) + + if write: + _write_back(stage3_path, data, materials) + + return materials + + +def _merge_linked_source_materials( + materials: Dict[str, Dict[str, Any]], + data: Dict[str, Any], + link: LinkResult, + material_study_ids: Dict[str, str], + llm_client: Any, + *, + stage1_json: Dict[str, Any], + timeout: Optional[float], +) -> None: + """Extract missing study materials from linked OSF/source PDF/DOCX/TXT files. + + Structured QSF/SAV remains the first-class source. This pass only fills + study drafts that are missing a participant-facing source or have codebook + items without a stimulus. It uses the same HumanStudy-Bench field prompt as + the paper-PDF fallback, but grounds extraction in linked supplementary + materials before consulting the main article PDF. + """ + attempts: List[Dict[str, Any]] = [] + errors: List[Dict[str, str]] = [] + + for sub_id, material in materials.items(): + selection = material.get("selection") if isinstance(material.get("selection"), dict) else {} + if selection.get("keep") is False: + continue + if not _material_needs_linked_source(material): + continue + study_id = material_study_ids.get(sub_id) or sub_id + candidates = _rank_linked_source_candidates(link.material_files(study_id)) + if not candidates: + continue + for source in candidates[:2]: + attempt = { + "sub_study_id": sub_id, + "study_id": study_id, + "file": _rel_source_file(source.path), + "kind": source.kind, + } + attempts.append(attempt) + try: + extracted = extract_pdf_study_materials( + data, + source.path, + llm_client, + stage1_json=stage1_json, + pdf_text=_source_text_override(source.path, source.kind), + source_label="osf_source_llm", + timeout=timeout, + only_sub_study_id=sub_id, + ) + except Exception as exc: + errors.append({ + **{key: str(value) for key, value in attempt.items()}, + "error": f"{type(exc).__name__}: {exc}", + }) + continue + if not extracted: + continue + extracted_ids = sorted(extracted) + extracted = _filter_pdf_materials_for_target( + extracted, + target_sid=sub_id, + materials=materials, + ) + attempt["extracted_sub_study_ids"] = extracted_ids + attempt["merged_sub_study_ids"] = sorted(extracted) + dropped_ids = sorted(set(extracted_ids) - set(extracted)) + if dropped_ids: + attempt["dropped_cross_study_ids"] = dropped_ids + if not extracted: + attempt["merged"] = False + continue + before = materials.get(sub_id, {}) + before_items = len(before.get("items") or []) if isinstance(before, dict) else 0 + before_instructions = str(before.get("instructions") or "") if isinstance(before, dict) else "" + _merge_pdf_materials(materials, extracted) + merged = materials.get(sub_id, {}) + if isinstance(merged.get("source_trace"), dict): + merged["source_trace"]["source_extraction"] = { + "mode": "linked_osf_source", + "source_file": _rel_source_file(source.path), + "source_kind": source.kind, + } + after_items = len(merged.get("items") or []) if isinstance(merged, dict) else 0 + after_instructions = str(merged.get("instructions") or "") if isinstance(merged, dict) else "" + attempt["merged"] = after_items != before_items or after_instructions != before_instructions + attempt["post_ready"] = merged.get("readiness", {}).get("ready") if isinstance(merged, dict) else None + if not _material_needs_linked_source(materials[sub_id]): + break + + if attempts or errors: + data["linked_source_material_extraction"] = { + "version": "linked-source-material-extraction-v1", + "attempts": attempts, + "errors": errors, + } + + +def _rank_linked_source_candidates(files: List[Any]) -> List[Any]: + source_files = [item for item in files if getattr(item, "kind", None) in {"pdf", "docx", "text"}] + + def score(item: Any) -> tuple[int, str]: + path_text = str(getattr(item, "path", "")).lower() + value = 0 + if any(term in path_text for term in ("material", "materials", "survey", "questionnaire", "instrument", "stimuli", "stimulus")): + value += 6 + if any(term in path_text for term in ("experimental", "experiment")): + value += 3 + if any(term in path_text for term in ("pre-registration", "prereg", "pre_reg", "analysis", "script", "code")): + value -= 6 + if getattr(item, "kind", None) == "text": + value += 2 + elif getattr(item, "kind", None) == "docx": + value += 1 + return (-value, str(getattr(item, "path", ""))) + + return sorted(source_files, key=score) + + +def _filter_pdf_materials_for_target( + extracted: Dict[str, Dict[str, Any]], + *, + target_sid: str, + materials: Dict[str, Dict[str, Any]], +) -> Dict[str, Dict[str, Any]]: + """Keep only extracted materials that map back to the current source target.""" + out: Dict[str, Dict[str, Any]] = {} + target_only = {target_sid: materials[target_sid]} if target_sid in materials else {} + for pdf_sid, pdf_material in extracted.items(): + if _match_pdf_material_id(pdf_sid, target_only) == target_sid: + out[pdf_sid] = pdf_material + continue + if _match_pdf_material_id(pdf_sid, materials) == target_sid: + out[pdf_sid] = pdf_material + return out + + +def _source_text_override(path: Path, kind: str) -> Optional[str]: + if kind == "text": + return Path(path).read_text(encoding="utf-8", errors="ignore")[:PDF_TEXT_MAX_CHARS] + if kind == "docx": + return _extract_docx_text(Path(path))[:PDF_TEXT_MAX_CHARS] + return None + + +def _extract_docx_text(path: Path) -> str: + try: + with zipfile.ZipFile(path) as zf: + raw = zf.read("word/document.xml") + except Exception: + return "" + try: + root = ET.fromstring(raw) + except ET.ParseError: + return "" + chunks = [] + for node in root.iter(): + if node.tag.endswith("}t") and node.text: + chunks.append(node.text) + return "\n".join(chunks) + + +def _rel_source_file(path: Path) -> str: + try: + return str(Path(path).resolve().relative_to(REPO_ROOT.resolve())) + except ValueError: + return str(path) + + +def _material_needs_linked_source(material: Dict[str, Any]) -> bool: + return _material_needs_pdf_fallback(material) or _material_needs_stimulus_completion(material) + + +def _selected_material_needs_source_completion(materials: Dict[str, Dict[str, Any]]) -> bool: + for material in materials.values(): + if not isinstance(material, dict): + continue + selection = material.get("selection") if isinstance(material.get("selection"), dict) else {} + if selection.get("keep") is False: + continue + if _material_needs_linked_source(material): + return True + return False + + +def _material_needs_stimulus_completion(material: Dict[str, Any]) -> bool: + if not isinstance(material, dict): + return False + if str(material.get("instructions") or "").strip(): + return False + return bool(material.get("items")) + + +def _material_needs_pdf_fallback(material: Dict[str, Any]) -> bool: + source = material.get("source_trace", {}).get("primary_source") + if source: + return False + readiness = material.get("readiness") if isinstance(material.get("readiness"), dict) else {} + blocking = set(readiness.get("blocking_issues") or []) + if "no_structured_material_source" in blocking: + return True + if not material.get("instructions") or not material.get("items"): + return True + return readiness.get("ready") is False + + +def _merge_pdf_materials( + materials: Dict[str, Dict[str, Any]], + pdf_materials: Dict[str, Dict[str, Any]], +) -> None: + """Fill empty/no-source study material drafts from PDF-extracted packages.""" + for sid, pdf_material in pdf_materials.items(): + if not isinstance(pdf_material, dict): + continue + target_sid = _match_pdf_material_id(sid, materials) + if target_sid is None: + continue + current = materials.get(target_sid) + if current is not None and _material_needs_pdf_stimulus(current, pdf_material): + _merge_pdf_stimulus(current, pdf_material) + continue + if current is not None and not _material_needs_pdf_fallback(current): + continue + merged = dict(pdf_material) + if isinstance(current, dict): + if isinstance(current.get("selection"), dict): + merged["selection"] = current["selection"] + trace = merged.setdefault("source_trace", {}) + trace["replaced_empty_source_trace"] = current.get("source_trace", {}) + merged["sub_study_id"] = target_sid + materials[target_sid] = merged + + +def _match_pdf_material_id( + pdf_sid: str, + materials: Dict[str, Dict[str, Any]], +) -> Optional[str]: + """Map PDF-extractor ids back to known Stage 3 study ids. + + The PDF extractor may produce descriptive ids such as + `study_1a_vignette_paradigm_democrat_sample`; Stage 3 must still keep the + material unit fixed to the Stage 1/2 study id (`study_1a`). + """ + if pdf_sid in materials: + return pdf_sid + pdf_slug = _material_slug(pdf_sid) + candidates = [] + for sid in materials: + sid_slug = _material_slug(sid) + if pdf_slug == sid_slug: + return sid + if pdf_slug.startswith(f"{sid_slug}_"): + candidates.append((len(sid_slug), sid)) + if candidates: + return max(candidates)[1] + + pdf_tokens = _tokens(pdf_sid) + scored: List[tuple[float, str]] = [] + for sid, material in materials.items(): + score = _jaccard( + pdf_tokens, + _tokens( + sid, + material.get("sub_study_id") if isinstance(material, dict) else "", + ), + ) + if score > 0: + scored.append((score, sid)) + if scored: + score, sid = max(scored) + if score >= 0.5: + return sid + return None + + +def _material_needs_pdf_stimulus( + material: Dict[str, Any], + pdf_material: Dict[str, Any], +) -> bool: + """Whether PDF material can complete a structured-source item package.""" + if not isinstance(material, dict) or not isinstance(pdf_material, dict): + return False + if str(material.get("instructions") or "").strip(): + return False + if not material.get("items"): + return False + pdf_instructions = str(pdf_material.get("instructions") or "").strip() + if not pdf_instructions: + return False + pdf_readiness = pdf_material.get("readiness") if isinstance(pdf_material.get("readiness"), dict) else {} + pdf_blocking = set(pdf_readiness.get("blocking_issues") or []) + return not ({"missing_instructions", "pdf_material_not_found"} & pdf_blocking) + + +def _merge_pdf_stimulus( + material: Dict[str, Any], + pdf_material: Dict[str, Any], +) -> None: + material["instructions"] = str(pdf_material.get("instructions") or "").strip() + if not material.get("conditions") and pdf_material.get("conditions"): + material["conditions"] = pdf_material["conditions"] + elif pdf_material.get("conditions"): + _merge_condition_descriptions(material, pdf_material) + _repair_item_questions_from_pdf_material(material, pdf_material) + + trace = material.setdefault("source_trace", {}) + pdf_trace = pdf_material.get("source_trace") if isinstance(pdf_material.get("source_trace"), dict) else {} + trace["stimulus_source"] = pdf_trace.get("primary_source") or "pdf_llm" + trace["stimulus_source_file"] = pdf_trace.get("source_file") + trace["stimulus_source_trace"] = pdf_trace + + readiness = material.setdefault("readiness", {"ready": False, "blocking_issues": [], "warnings": []}) + issues = readiness.setdefault("blocking_issues", []) + issues[:] = [ + issue for issue in issues + if issue not in {"stimulus_not_verbatim", "missing_instructions", "no_structured_material_source"} + ] + warnings = readiness.setdefault("warnings", []) + warnings.append("Stimulus/instructions completed from PDF material extraction; response items retained from structured source.") + if material.get("items"): + if not _material_has_truncated_questions(material): + issues[:] = [ + issue for issue in issues + if issue not in {"truncated_sav_labels", "truncated_response_question"} + ] + readiness["ready"] = not issues + + +_TOKEN_RE = re.compile(r"[a-z][a-z0-9_'-]{2,}", re.IGNORECASE) +_MATERIAL_STOP_TERMS = { + "the", "and", "for", "with", "that", "this", "from", "into", "about", + "study", "studies", "effect", "effects", "condition", "conditions", + "participant", "participants", "message", "messages", "article", "articles", + "measure", "measures", "item", "items", "scale", "index", "response", + "responses", "dependent", "variable", "independent", "interaction", +} +_GENERIC_ASSIGNMENT_RE = re.compile( + r"participants?\s+were\s+randomly\s+presented\s+with\s+either\s+the\s+[^.]+", + re.IGNORECASE, +) +_CONDITION_DETAIL_RE = re.compile( + r"\b(?:in|under|for)\s+the\s+[A-Za-z0-9][A-Za-z0-9 /_-]{1,80}?\s+condition\b", + re.IGNORECASE, +) + + +def _material_slug(text: Any) -> str: + value = re.sub(r"[^a-z0-9]+", "_", str(text or "").lower()).strip("_") + if value and not value.startswith(("study", "pilot")): + value = f"study_{value}" + return value + + +def _stem(token: str) -> str: + token = token.lower().strip("'_-") + for suffix in ("iveness", "ation", "ness", "ment", "ity", "ing", "ed", "es", "s"): + if len(token) > len(suffix) + 4 and token.endswith(suffix): + token = token[:-len(suffix)] + break + return token[:7] if len(token) > 7 else token + + +def _tokens(*values: Any) -> set[str]: + out: set[str] = set() + for value in values: + for tok in _TOKEN_RE.findall(str(value or "")): + stemmed = _stem(tok) + if len(stemmed) >= 4 and stemmed not in _MATERIAL_STOP_TERMS and not stemmed.isdigit(): + out.add(stemmed) + return out + + +def _jaccard(a: set[str], b: set[str]) -> float: + if not a or not b: + return 0.0 + return len(a & b) / len(a | b) + + +def _material_has_truncated_questions(material: Dict[str, Any]) -> bool: + for item in material.get("items", []) or []: + if not isinstance(item, dict): + continue + question = str(item.get("question") or "") + if "..." in question or "\u2026" in question: + return True + return False + + +def _repair_item_questions_from_pdf_material( + material: Dict[str, Any], + pdf_material: Dict[str, Any], +) -> None: + current_items = [item for item in material.get("items", []) or [] if isinstance(item, dict)] + pdf_items = [item for item in pdf_material.get("items", []) or [] if isinstance(item, dict)] + if not current_items or not pdf_items: + return + + repairs: List[Dict[str, str]] = [] + pdf_trace = pdf_material.get("source_trace") if isinstance(pdf_material.get("source_trace"), dict) else {} + pdf_source = str(pdf_trace.get("primary_source") or "pdf_llm") + pdf_source_file = str(pdf_trace.get("source_file") or "") + for idx, item in enumerate(current_items): + old_question = str(item.get("question") or "") + if "..." not in old_question and "\u2026" not in old_question: + continue + replacement = _best_pdf_item_question(item, pdf_items, fallback_index=idx) + if not replacement: + continue + item["question"] = replacement + flags = [flag for flag in item.get("quality_flags", []) if flag != "label_contains_ellipsis"] + if flags: + item["quality_flags"] = sorted(set(flags)) + else: + item.pop("quality_flags", None) + old_source = str(item.get("source") or "").strip() + if old_source and pdf_source not in old_source: + item["source"] = f"{old_source}+{pdf_source}" + else: + item["source"] = old_source or pdf_source + item["question_source"] = pdf_source + if pdf_source_file: + item["question_source_file"] = pdf_source_file + repairs.append( + { + "item": str(item.get("data_export_tag") or item.get("id") or idx), + "old": old_question.replace("\u2026", "..."), + "new": replacement, + "source": pdf_source, + } + ) + + if repairs: + trace = material.setdefault("source_trace", {}) + trace["item_question_repairs"] = repairs + warnings = material.setdefault("readiness", {}).setdefault("warnings", []) + warnings.append(f"Repaired {len(repairs)} truncated item question(s) from PDF/source material extraction.") + + +def _best_pdf_item_question( + item: Dict[str, Any], + pdf_items: List[Dict[str, Any]], + *, + fallback_index: int, +) -> str: + current_terms = _tokens(item.get("id"), item.get("data_export_tag"), item.get("question")) + scored: List[tuple[float, Dict[str, Any]]] = [] + for pdf_item in pdf_items: + question = str(pdf_item.get("question") or "").strip() + if not question or "..." in question or "\u2026" in question: + continue + pdf_terms = _tokens(pdf_item.get("id"), pdf_item.get("data_export_tag"), question) + scored.append((_jaccard(current_terms, pdf_terms), pdf_item)) + if scored: + score, match = max(scored, key=lambda row: row[0]) + if score >= 0.12: + return str(match.get("question") or "").strip() + if 0 <= fallback_index < len(pdf_items): + question = str(pdf_items[fallback_index].get("question") or "").strip() + if question and "..." not in question and "\u2026" not in question: + return question + return "" + + +def _effect_terms(study: Optional[Dict[str, Any]]) -> set[str]: + if not isinstance(study, dict): + return set() + terms: set[str] = set() + effects = [effect for effect in study.get("effects", []) if isinstance(effect, dict)] + primary = [] + for effect in effects: + cons = effect.get("consolidation") if isinstance(effect.get("consolidation"), dict) else {} + if cons.get("is_primary_simulation_target", True) and cons.get("is_representative", True) is not False: + primary.append(effect) + for effect in primary or effects: + terms |= _tokens(effect.get("IV"), effect.get("DV")) + return terms + + +def _condition_terms(material: Dict[str, Any]) -> set[str]: + terms: set[str] = set() + for cond in material.get("conditions", []) or []: + if not isinstance(cond, dict): + continue + terms |= _tokens(cond.get("name"), cond.get("label")) + for level in cond.get("levels", []) or []: + terms |= _tokens(level) + return terms + + +def _condition_levels(material: Dict[str, Any]) -> set[str]: + levels: set[str] = set() + for cond in material.get("conditions", []) or []: + if not isinstance(cond, dict): + continue + for level in cond.get("levels", []) or []: + norm = re.sub(r"[^a-z0-9]+", "", str(level).lower()) + if norm: + levels.add(norm) + return levels + + +def _item_terms(material: Dict[str, Any]) -> set[str]: + terms: set[str] = set() + for item in material.get("items", []) or []: + if not isinstance(item, dict): + continue + terms |= _tokens(item.get("id"), item.get("data_export_tag"), item.get("question")) + return terms + + +def _instruction_detail_count(text: str) -> int: + return len(_CONDITION_DETAIL_RE.findall(text or "")) + + +def _has_condition_descriptions(material: Dict[str, Any]) -> bool: + for cond in material.get("conditions", []) or []: + if isinstance(cond, dict) and isinstance(cond.get("level_descriptions"), dict): + if any(str(v).strip() for v in cond["level_descriptions"].values()): + return True + return False + + +def _instructions_sparse(material: Dict[str, Any]) -> bool: + text = str(material.get("instructions") or "").strip() + if len(text) < 180: + return True + if _GENERIC_ASSIGNMENT_RE.fullmatch(text.rstrip(".")): + return True + return _instruction_detail_count(text) < 2 and not _has_condition_descriptions(material) + + +def _instructions_rich(material: Dict[str, Any]) -> bool: + text = str(material.get("instructions") or "").strip() + return len(text) >= 160 and (_instruction_detail_count(text) >= 2 or _has_condition_descriptions(material)) + + +def _study_by_sub_id(eligible: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + for study in eligible: + sid = _material_slug(study.get("study") or study.get("study_id") or study.get("experiment_id")) + if sid: + out[sid] = study + return out + + +def _sibling_material_score( + target: Dict[str, Any], + candidate: Dict[str, Any], + target_study: Optional[Dict[str, Any]], + candidate_study: Optional[Dict[str, Any]], +) -> tuple[float, Dict[str, float]]: + effect = _jaccard(_effect_terms(target_study), _effect_terms(candidate_study)) + items = _jaccard(_item_terms(target), _item_terms(candidate)) + cond_terms = _jaccard(_condition_terms(target), _condition_terms(candidate)) + target_levels = _condition_levels(target) + candidate_levels = _condition_levels(candidate) + level_match = 1.0 if target_levels and target_levels == candidate_levels else 0.0 + score = (0.50 * effect) + (0.30 * items) + (0.10 * cond_terms) + (0.10 * level_match) + return score, { + "effect_similarity": round(effect, 3), + "item_similarity": round(items, 3), + "condition_similarity": round(cond_terms, 3), + "level_match": round(level_match, 3), + "score": round(score, 3), + } + + +def _merge_condition_descriptions(target: Dict[str, Any], source: Dict[str, Any]) -> None: + source_by_name = { + str(cond.get("name") or cond.get("label") or "").lower(): cond + for cond in source.get("conditions", []) or [] + if isinstance(cond, dict) + } + merged = [] + for cond in target.get("conditions", []) or []: + if not isinstance(cond, dict): + merged.append(cond) + continue + key = str(cond.get("name") or cond.get("label") or "").lower() + src = source_by_name.get(key) + new_cond = dict(cond) + if isinstance(src, dict) and isinstance(src.get("level_descriptions"), dict): + existing = new_cond.get("level_descriptions") if isinstance(new_cond.get("level_descriptions"), dict) else {} + new_cond["level_descriptions"] = {**src["level_descriptions"], **existing} + new_cond.setdefault("description_source", "sibling_study_material") + merged.append(new_cond) + target["conditions"] = merged + + +def _apply_stage2_material_scaffolds( + materials: Dict[str, Dict[str, Any]], + eligible: List[Dict[str, Any]], +) -> None: + """Preserve Stage 2 study/effect hints when no participant material exists. + + This is deliberately a patch scaffold, not a runnable material source. It + prevents Stage 3/4 from emitting empty files when Stage 2 already identified + the study design, target DVs, source locations, and material notes. + """ + study_map = _study_by_sub_id(eligible) + for sid, material in materials.items(): + if not isinstance(material, dict): + continue + study = study_map.get(sid) + if not isinstance(study, dict): + continue + selection = material.get("selection") if isinstance(material.get("selection"), dict) else {} + if selection.get("keep") is False: + continue + + effects = _stage2_scaffold_effects(study) + if not effects: + continue + + trace = material.setdefault("source_trace", {}) + had_primary_source = bool(trace.get("primary_source")) + scaffold = _build_stage2_scaffold(study, effects) + trace["stage2_scaffold"] = scaffold["source_trace"]["stage2_scaffold"] + + readiness = material.setdefault("readiness", {"ready": False, "blocking_issues": [], "warnings": []}) + blocking = readiness.setdefault("blocking_issues", []) + warnings = readiness.setdefault("warnings", []) + added = False + + if not str(material.get("instructions") or "").strip() and not had_primary_source: + material["instructions"] = scaffold["instructions"] + trace["primary_source"] = "stage2_scaffold" + trace["source_file"] = "stage2.json" + blocking.append("stage2_scaffold_not_participant_facing") + added = True + + if not material.get("items"): + material["items"] = scaffold["items"] + material["response_schema"] = scaffold["response_schema"] + blocking.append("stage2_scaffold_placeholder_items") + added = True + + if not material.get("conditions") and scaffold["conditions"] and not had_primary_source: + material["conditions"] = scaffold["conditions"] + added = True + + if added: + blocking.append("stage2_scaffold_requires_source_patch") + warnings.append( + "Stage 2 scaffold preserved known IV/DV/source-location hints, " + "but verbatim participant-facing wording must still be recovered from PDF/OSF." + ) + readiness["blocking_issues"] = sorted(set(blocking)) + readiness["warnings"] = sorted(set(warnings)) + readiness["ready"] = False + + +def _stage2_scaffold_effects(study: Dict[str, Any]) -> List[Dict[str, Any]]: + effects = [effect for effect in study.get("effects", []) or [] if isinstance(effect, dict)] + if not effects: + return [] + primary: List[Dict[str, Any]] = [] + for effect in effects: + cons = effect.get("consolidation") if isinstance(effect.get("consolidation"), dict) else {} + role = str(cons.get("analysis_role") or effect.get("effecttype") or "").lower() + if cons and cons.get("is_representative", True) is False: + continue + if cons and cons.get("is_primary_simulation_target") is False: + continue + if role in {"manipulation_check", "check", "secondary_dv"}: + continue + if role in {"primary_finding", "primary", "main", "main_effect", "interaction", "moderation", "int"}: + primary.append(effect) + if primary: + return primary[:8] + fallback = [ + effect + for effect in effects + if str(effect.get("effecttype") or "").lower() in {"main", "int", "interaction", "moderation"} + ] + return (fallback or effects)[:8] + + +def _build_stage2_scaffold(study: Dict[str, Any], effects: List[Dict[str, Any]]) -> Dict[str, Any]: + study_name = str(study.get("study") or study.get("study_id") or study.get("experiment_id") or "Study") + source_notes = _stage2_source_notes(effects) + effect_lines = [] + for idx, effect in enumerate(effects, start=1): + line = f"{idx}. IV: {effect.get('IV') or 'unknown'} | DV: {effect.get('DV') or 'unknown'}" + if effect.get("materials_notes"): + line += f" | materials_notes: {effect.get('materials_notes')}" + if effect.get("table_or_page_location"): + line += f" | source_location: {effect.get('table_or_page_location')}" + effect_lines.append(line) + instructions = "\n".join( + [ + "STAGE2 SCAFFOLD ONLY - not participant-facing material.", + "Recover verbatim stimulus, task instructions, response items, options, and anchors from the PDF appendix or supplementary instrument before simulation.", + f"Study: {study_name}", + "Known Stage 2 targets:", + *effect_lines, + ] + ) + items = _stage2_placeholder_items(effects) + conditions = _stage2_conditions_from_effects(effects) + return { + "instructions": instructions, + "items": items, + "conditions": conditions, + "response_schema": {"answer_type": "text", "placeholder": True} if items else {}, + "source_trace": { + "stage2_scaffold": { + "mode": "stage2_design_and_source_hints", + "study": study_name, + "source_file": "stage2.json", + "effects": source_notes, + "limitations": [ + "Not participant-facing wording.", + "Response items are placeholders derived from Stage 2 DVs.", + "Conditions are inferred from Stage 2 IV strings and require PDF/OSF confirmation.", + ], + } + }, + } + + +def _stage2_source_notes(effects: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + notes: List[Dict[str, Any]] = [] + for idx, effect in enumerate(effects): + notes.append( + { + "effect_index": idx, + "effecttype": effect.get("effecttype"), + "IV": effect.get("IV"), + "DV": effect.get("DV"), + "direction": effect.get("direction"), + "materials_notes": effect.get("materials_notes"), + "table_or_page_location": effect.get("table_or_page_location"), + "size": effect.get("size") or effect.get("analysis_n"), + "stats": effect.get("stats"), + } + ) + return notes + + +def _stage2_placeholder_items(effects: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + items: List[Dict[str, Any]] = [] + seen: set[str] = set() + for effect in effects: + dv = str(effect.get("DV") or "").strip() + if not dv: + continue + key = re.sub(r"\s+", " ", dv.lower()) + if key in seen: + continue + seen.add(key) + item_id = re.sub(r"[^a-z0-9]+", "_", dv.lower()).strip("_") or "dv" + items.append( + { + "id": f"stage2_placeholder_{item_id}", + "question": f"PATCH REQUIRED: recover verbatim participant-facing item(s) for '{dv}'.", + "type": "text", + "options": [], + "source": "stage2_scaffold", + "source_file": "stage2.json", + "metadata": { + "placeholder": True, + "dv": dv, + "related_ivs": sorted({str(e.get("IV")) for e in effects if e.get("DV") == dv and e.get("IV")}), + "source_locations": sorted({ + str(e.get("table_or_page_location")) + for e in effects + if e.get("DV") == dv and e.get("table_or_page_location") + }), + }, + } + ) + return items[:12] + + +def _stage2_conditions_from_effects(effects: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + conditions: List[Dict[str, Any]] = [] + seen: set[str] = set() + for effect in effects: + iv = str(effect.get("IV") or "").strip() + if not iv: + continue + for part in re.split(r"\s+(?:×|x|X)\s+", iv): + condition = _condition_from_stage2_iv(part, source_effect=effect) + if not condition: + continue + key = (condition["name"].lower(), tuple(level.lower() for level in condition.get("levels", []))) + if str(key) in seen: + continue + seen.add(str(key)) + conditions.append(condition) + return conditions[:8] + + +def _condition_from_stage2_iv(text: str, *, source_effect: Dict[str, Any]) -> Optional[Dict[str, Any]]: + raw = re.sub(r"\s+", " ", str(text or "")).strip(" ;.") + if not raw: + return None + name = raw + level_text = "" + paren = re.search(r"^(?P.*?)\((?P[^)]*\bvs\.?\b[^)]*)\)", raw, flags=re.IGNORECASE) + colon = re.search(r"^(?P.*?):\s*(?P.*\bvs\.?\b.*)$", raw, flags=re.IGNORECASE) + if paren: + name = paren.group("name").strip() + level_text = paren.group("levels") + elif colon: + name = colon.group("name").strip() + level_text = colon.group("levels") + elif re.search(r"\bvs\.?\b", raw, flags=re.IGNORECASE): + level_text = raw + if not level_text: + return None + level_text = re.split(r",\s*(?:manipulated|measured|reported|randomized)", level_text, flags=re.IGNORECASE)[0] + pieces = [ + re.sub(r"\s+", " ", piece).strip(" .;") + for piece in re.split(r"\bvs\.?\b| versus ", level_text, flags=re.IGNORECASE) + ] + levels = [piece for piece in pieces if piece] + if len(levels) < 2: + return None + clean_name = re.sub(r"\s+", " ", name).strip(" :") or "stage2_condition" + return { + "name": clean_name, + "levels": levels[:6], + "source": "stage2_scaffold", + "source_file": "stage2.json", + "review_required": True, + "metadata": { + "source_iv": source_effect.get("IV"), + "source_location": source_effect.get("table_or_page_location"), + "placeholder": True, + }, + } + + +def _complete_sparse_materials_from_siblings( + materials: Dict[str, Dict[str, Any]], + eligible: List[Dict[str, Any]], +) -> None: + """Record sibling stimulus candidates for sparse studies without patching. + + This handles preregistered replications and repeated paradigms where one + study's PDF slots contain the full manipulation and a sibling only says + "participants were randomly assigned/presented..." even though the OSF + codebook shows the same condition and outcome structure. + + Earlier versions copied the sibling's instructions into the sparse material + and sometimes flipped readiness to true. That is too aggressive for the new + Stage 3 contract: source-backed fields should remain unchanged, and + inferred cross-study reuse should be a patch candidate for human review. + """ + study_map = _study_by_sub_id(eligible) + for sid, material in materials.items(): + source_trace = material.get("source_trace") if isinstance(material.get("source_trace"), dict) else {} + if source_trace.get("stimulus_source") == "pdf_llm" or source_trace.get("primary_source") == "pdf_llm": + continue + if not _instructions_sparse(material): + continue + target_study = study_map.get(sid) + ranked: List[tuple[float, Dict[str, float], str, Dict[str, Any]]] = [] + for candidate_id, candidate in materials.items(): + if candidate_id == sid or not _instructions_rich(candidate): + continue + score, parts = _sibling_material_score( + material, + candidate, + target_study, + study_map.get(candidate_id), + ) + if parts["effect_similarity"] >= 0.25 and parts["item_similarity"] >= 0.20 and score >= 0.42: + ranked.append((score, parts, candidate_id, candidate)) + if not ranked: + continue + _, parts, source_id, source = max(ranked, key=lambda row: row[0]) + trace = material.setdefault("source_trace", {}) + trace["material_completion_candidate"] = { + "mode": "sibling_study_material", + "source_sub_study_id": source_id, + "current_instruction_preview": str(material.get("instructions") or "").strip()[:240], + "suggested_instruction_preview": str(source.get("instructions") or "").strip()[:600], + "suggested_conditions": source.get("conditions") if isinstance(source.get("conditions"), list) else [], + **parts, + } + warnings = material.setdefault("readiness", {}).setdefault("warnings", []) + warnings.append( + f"Candidate sibling stimulus found in {source_id} " + f"(score={parts['score']}, effect_similarity={parts['effect_similarity']}); material left unchanged." + ) + + +def _annotate_selection( + paper_dir: Path, + eligible: List[Dict[str, Any]], + study_ids: List[str], + data: Dict[str, Any], + client: Any, + *, + votes: int = 3, + timeout: Optional[float] = 60.0, +) -> None: + """Run the study selector and attach `selection` to each eligible study.""" + from generation_pipeline.study_selector import select_studies, selection_summary + + stage1 = paper_dir / "stage1.json" + experiments: List[Dict[str, Any]] = [] + if stage1.exists(): + experiments = json.loads(stage1.read_text(encoding="utf-8")).get("experiments", []) + if not experiments: + # fall back to building briefs from the eligible studies themselves + experiments = [{"experiment_id": sid} for sid in study_ids] + + decisions = select_studies(experiments, client, votes=votes, timeout=timeout) + for study in eligible: + name = str(study.get("study") or study.get("study_id") or "") + dec = decisions.get(name) + if dec is not None: + study["selection"] = dec + data["study_selection_summary"] = selection_summary(decisions) + + +def _summarize(materials: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + ready = sum(1 for m in materials.values() if m["readiness"]["ready"]) + by_source: Dict[str, int] = {} + for m in materials.values(): + src = m["source_trace"].get("primary_source") or "none" + by_source[src] = by_source.get(src, 0) + 1 + return { + "total_sub_studies": len(materials), + "ready": ready, + "needs_review": len(materials) - ready, + "by_primary_source": by_source, + "unready": [ + sid for sid, m in materials.items() if not m["readiness"]["ready"] + ], + } + + +def _consolidation_summary(eligible: List[Dict[str, Any]]) -> Dict[str, Any]: + raw = cons = prim = 0 + by_role: Dict[str, int] = {} + for s in eligible: + cs = s.get("consolidation_summary") + if not cs: + continue + raw += cs["raw_effects"] + cons += cs["consolidated_effects"] + prim += cs["primary_simulation_targets"] + for role, n in cs.get("by_role", {}).items(): + by_role[role] = by_role.get(role, 0) + n + return { + "raw_effects": raw, + "consolidated_effects": cons, + "primary_simulation_targets": prim, + "by_role": by_role, + } + + +def _write_back( + stage3_path: Path, + data: Dict[str, Any], + materials: Dict[str, Dict[str, Any]], +) -> None: + # `data` already carries the in-place effect consolidation annotations. + data["study_materials"] = materials + data["osf_materials_summary"] = _summarize(materials) + data["effect_consolidation_summary"] = _consolidation_summary(_eligible_of(data)) + tmp = stage3_path.with_suffix(stage3_path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + tmp.replace(stage3_path) + + +def _main(argv: Optional[List[str]] = None) -> int: + import argparse + + ap = argparse.ArgumentParser(description="Stage 3 OSF material extraction.") + ap.add_argument("paper_dir", help="Output paper dir containing stage3.json") + ap.add_argument("--no-write", action="store_true", help="Do not modify stage3.json") + args = ap.parse_args(argv) + + mats = build_study_materials(args.paper_dir, write=not args.no_write) + summary = _summarize(mats) + print(json.dumps(summary, ensure_ascii=False, indent=2)) + for sid, m in mats.items(): + rd = m["readiness"] + src = m["source_trace"].get("primary_source") + flag = "" if rd["ready"] else f" !! {rd['blocking_issues']}" + print(f" {sid:14s} src={src} items={len(m['items'])} ready={rd['ready']}{flag}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/generation_pipeline/stage3_material_contract.py b/generation_pipeline/stage3_material_contract.py new file mode 100644 index 0000000..9d65932 --- /dev/null +++ b/generation_pipeline/stage3_material_contract.py @@ -0,0 +1,341 @@ +"""HumanStudy-Bench material contract checks for Stage 3 outputs.""" + +from __future__ import annotations + +from typing import Any, Dict, List + + +_RESPONSE_OPTION_TYPES = {"multiple_choice", "likert", "scale", "matrix", "slider"} + + +def _text(value: Any) -> str: + return str(value or "").strip() + + +def _has_ellipsis(value: Any) -> bool: + return "..." in _text(value) or "…" in _text(value) + + +def _item_options(item: Dict[str, Any]) -> List[Any]: + options = item.get("options") + if isinstance(options, list) and options: + return options + choices = item.get("choices") + if isinstance(choices, list) and choices: + return choices + response_format = item.get("response_format") + if isinstance(response_format, dict) and isinstance(response_format.get("options"), list): + return response_format["options"] + return [] + + +def _item_has_scale(item: Dict[str, Any]) -> bool: + scale = item.get("scale") + if isinstance(scale, dict) and (scale.get("min") is not None or scale.get("max") is not None): + return True + response_format = item.get("response_format") + if isinstance(response_format, dict): + return response_format.get("scale_min") is not None or response_format.get("scale_max") is not None + return False + + +def _unique(values: List[Any]) -> List[str]: + return sorted({text for value in values if (text := _text(value))}) + + +def _source_trace(material: Dict[str, Any]) -> Dict[str, Any]: + return material.get("source_trace") if isinstance(material.get("source_trace"), dict) else {} + + +def _base_sources(material: Dict[str, Any]) -> Dict[str, List[str]]: + trace = _source_trace(material) + stimulus_trace = trace.get("stimulus_source_trace") if isinstance(trace.get("stimulus_source_trace"), dict) else {} + if _text(trace.get("stimulus_source")): + return { + "sources": _unique([trace.get("stimulus_source")]), + "files": _unique([trace.get("stimulus_source_file"), stimulus_trace.get("source_file")]), + } + return { + "sources": _unique([trace.get("primary_source")]), + "files": _unique([trace.get("source_file")]), + } + + +def _item_source_summary(items: List[Dict[str, Any]]) -> Dict[str, Any]: + missing: List[str] = [] + sources: List[Any] = [] + files: List[Any] = [] + for index, item in enumerate(items, start=1): + item_id = _text(item.get("id") or item.get("data_export_tag") or f"item_{index}") + source = item.get("source") + source_file = item.get("source_file") + if not _text(source) and not _text(source_file): + missing.append(item_id) + sources.append(source) + files.append(source_file) + return { + "sources": _unique(sources), + "files": _unique(files), + "missing_item_sources": missing, + } + + +def _condition_source_summary(conditions: Any, material: Dict[str, Any]) -> Dict[str, Any]: + normalized = [condition for condition in conditions if isinstance(condition, dict)] if isinstance(conditions, list) else [] + trace = _source_trace(material) + sources: List[Any] = [trace.get("conditions_source"), trace.get("primary_source")] + files: List[Any] = [trace.get("conditions_source_file"), trace.get("source_file")] + missing: List[str] = [] + for index, condition in enumerate(normalized, start=1): + condition_id = _text(condition.get("name") or condition.get("label") or f"condition_{index}") + sources.extend([condition.get("source"), condition.get("description_source")]) + files.extend([condition.get("source_file")]) + if not any(_text(condition.get(key)) for key in ("source", "source_file", "description_source")): + # A source_trace-level primary_source can still provenance extracted + # condition levels from the same instrument. + if not _text(trace.get("primary_source")) and not _text(trace.get("source_file")): + missing.append(condition_id) + return { + "sources": _unique(sources), + "files": _unique(files), + "missing_condition_sources": missing, + } + + +def _field_evidence( + material: Dict[str, Any], + *, + items: List[Dict[str, Any]], + conditions: Any, +) -> Dict[str, Dict[str, Any]]: + base = _base_sources(material) + item_sources = _item_source_summary(items) + condition_sources = _condition_source_summary(conditions, material) + has_instructions = bool(_text(material.get("instructions") or material.get("stimulus"))) + has_conditions = bool(conditions) + return { + "instructions": { + "status": ( + "present" if has_instructions and (base["sources"] or base["files"]) + else "missing" if has_instructions + else "not_applicable" + ), + **base, + }, + "stimulus": { + "status": ( + "present" if has_instructions and (base["sources"] or base["files"]) + else "missing" if has_instructions + else "not_applicable" + ), + **base, + }, + "response_items": { + "status": "present" if item_sources["sources"] or item_sources["files"] else ("missing" if items else "not_applicable"), + **item_sources, + }, + "response_options": { + "status": "present" if item_sources["sources"] or item_sources["files"] else ("missing" if items else "not_applicable"), + **item_sources, + }, + "conditions": { + "status": ( + "present" if has_conditions and (condition_sources["sources"] or condition_sources["files"]) + else "missing" if has_conditions + else "not_applicable" + ), + **condition_sources, + }, + } + + +def _condition_summary(conditions: Any) -> Dict[str, Any]: + if not isinstance(conditions, list): + conditions = [] + normalized = [condition for condition in conditions if isinstance(condition, dict)] + levels = 0 + described = 0 + for condition in normalized: + condition_levels = condition.get("levels") + if isinstance(condition_levels, list): + levels += len(condition_levels) + descriptions = condition.get("level_descriptions") + if isinstance(descriptions, dict): + described += sum(1 for value in descriptions.values() if _text(value)) + return { + "status": "present" if normalized else "missing", + "count": len(normalized), + "levels": levels, + "described_levels": described, + } + + +def audit_material_contract(material: Dict[str, Any]) -> Dict[str, Any]: + """Return a field-level audit for one Stage 3 study material.""" + issues: List[str] = [] + warnings: List[str] = [] + + instructions = _text(material.get("instructions") or material.get("stimulus")) + instruction_field = { + "status": "present" if instructions else "missing", + "chars": len(instructions), + "source": ( + material.get("source_trace", {}).get("stimulus_source") + or material.get("source_trace", {}).get("primary_source") + ) if isinstance(material.get("source_trace"), dict) else None, + } + if not instructions: + issues.append("missing_instructions") + + raw_items = material.get("items") + items = [item for item in raw_items if isinstance(item, dict)] if isinstance(raw_items, list) else [] + item_field = { + "status": "present" if items else "missing", + "count": len(items), + "sources": sorted( + { + _text(item.get("source")) + for item in items + if _text(item.get("source")) + } + ), + "missing_questions": [], + "truncated_questions": [], + } + if not items: + issues.append("missing_response_items") + evidence = _field_evidence(material, items=items, conditions=material.get("conditions")) + + options_present = 0 + options_missing: List[str] = [] + for index, item in enumerate(items, start=1): + item_id = _text(item.get("id") or item.get("data_export_tag") or f"item_{index}") + question = _text(item.get("question")) + if not question: + item_field["missing_questions"].append(item_id) + issues.append("missing_response_question") + if _has_ellipsis(question): + item_field["truncated_questions"].append(item_id) + issues.append("truncated_response_question") + + item_type = _text(item.get("type")).lower() + requires_options = item_type in _RESPONSE_OPTION_TYPES + has_contract = bool(_item_options(item) or _item_has_scale(item)) + if has_contract: + options_present += 1 + elif requires_options: + options_missing.append(item_id) + + option_field = { + "status": "present" if items and not options_missing else ("missing" if options_missing else "not_required"), + "items_with_options_or_scale": options_present, + "items_missing_options_or_scale": options_missing, + } + if options_missing: + issues.append("missing_response_options") + + response_schema = material.get("response_schema") or material.get("response_format") + response_field = { + "status": "present" if isinstance(response_schema, dict) and response_schema else "missing", + "schema_keys": sorted(response_schema.keys()) if isinstance(response_schema, dict) else [], + } + if items and response_field["status"] == "missing": + warnings.append("missing_response_schema") + + conditions = _condition_summary(material.get("conditions")) + if conditions["status"] == "present" and conditions["levels"] == 0: + warnings.append("condition_without_levels") + + source_trace = material.get("source_trace") if isinstance(material.get("source_trace"), dict) else {} + source_field = { + "primary_source": source_trace.get("primary_source"), + "source_file": source_trace.get("source_file"), + "stimulus_source": source_trace.get("stimulus_source"), + "has_candidates": bool(source_trace.get("candidates")), + } + if not source_field["primary_source"] and not source_field["stimulus_source"]: + warnings.append("missing_source_trace") + for field, payload in evidence.items(): + status = payload.get("status") + if status == "missing": + warnings.append(f"missing_source_evidence:{field}") + + blocking = sorted(set(issues)) + return { + "version": "human-study-bench-material-v1", + "sub_study_id": material.get("sub_study_id"), + "ready": not blocking, + "blocking_issues": blocking, + "warnings": sorted(set(warnings)), + "fields": { + "instructions": instruction_field, + "stimulus": instruction_field, + "response_items": item_field, + "response_options": option_field, + "response_schema": response_field, + "conditions": conditions, + "source_evidence": source_field, + }, + "field_evidence": evidence, + } + + +def apply_material_contracts(materials: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + """Annotate materials in-place and return a Stage 3 contract summary.""" + audits: Dict[str, Dict[str, Any]] = {} + ready_count = 0 + selected_ready_count = 0 + field_missing: Dict[str, List[str]] = {} + selected_field_missing: Dict[str, List[str]] = {} + source_missing: Dict[str, List[str]] = {} + selected_source_missing: Dict[str, List[str]] = {} + selected_sub_studies: List[str] = [] + + for sid, material in materials.items(): + audit = audit_material_contract(material) + audits[sid] = audit + material["human_material_contract"] = audit + selection = material.get("selection") if isinstance(material.get("selection"), dict) else {} + selected = selection.get("keep") is not False + if selected: + selected_sub_studies.append(sid) + + readiness = material.setdefault("readiness", {"ready": False, "blocking_issues": [], "warnings": []}) + blocking = list(readiness.get("blocking_issues") or []) + blocking.extend(audit["blocking_issues"]) + readiness["blocking_issues"] = sorted(set(blocking)) + warnings = list(readiness.get("warnings") or []) + warnings.extend(f"contract:{warning}" for warning in audit["warnings"]) + readiness["warnings"] = sorted(set(warnings)) + readiness["ready"] = readiness.get("ready") is not False and not readiness["blocking_issues"] + + if audit["ready"] and readiness["ready"]: + ready_count += 1 + if selected: + selected_ready_count += 1 + for field, payload in audit["fields"].items(): + if payload.get("status") == "missing": + field_missing.setdefault(field, []).append(sid) + if selected: + selected_field_missing.setdefault(field, []).append(sid) + for field, payload in audit.get("field_evidence", {}).items(): + if payload.get("status") == "missing": + source_missing.setdefault(field, []).append(sid) + if selected: + selected_source_missing.setdefault(field, []).append(sid) + + return { + "version": "human-study-bench-material-v1", + "total_sub_studies": len(materials), + "ready": ready_count, + "needs_patch": len(materials) - ready_count, + "selected_sub_studies": selected_sub_studies, + "selected_total": len(selected_sub_studies), + "selected_ready": selected_ready_count, + "selected_needs_patch": len(selected_sub_studies) - selected_ready_count, + "missing_by_field": field_missing, + "selected_missing_by_field": selected_field_missing, + "missing_source_by_field": source_missing, + "selected_missing_source_by_field": selected_source_missing, + "materials": audits, + } diff --git a/generation_pipeline/stage3_pdf_materials.py b/generation_pipeline/stage3_pdf_materials.py new file mode 100644 index 0000000..8be4ebf --- /dev/null +++ b/generation_pipeline/stage3_pdf_materials.py @@ -0,0 +1,1592 @@ +from __future__ import annotations + +import json +import re +import time +from pathlib import Path +from typing import Any, Dict, List, Optional + +from src.llm.helpers import call_with_timeout + +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + + +PDF_TEXT_MAX_CHARS = 500000 +PDF_CONTEXT_MAX_CHARS = 220000 +PDF_STUDY_CONTEXT_MAX_CHARS = 24000 +PDF_APPENDIX_CONTEXT_MAX_CHARS = 16000 +PDF_MATERIAL_MAX_TOKENS = 5000 +_VALID_TYPES = {"multiple_choice", "likert", "scale", "slider", "open_ended", "ranking", "matrix", "text"} +_TOKEN_RE = re.compile(r"[a-z][a-z0-9_'-]{2,}", re.IGNORECASE) +_TARGET_STOP_TERMS = { + "the", "and", "for", "with", "that", "this", "from", "into", "about", + "study", "studies", "effect", "effects", "condition", "conditions", + "measure", "measures", "item", "items", "index", "scale", "article", + "message", "messages", "dependent", "variable", "variables", "participant", + "participants", "toward", "after", "before", "interaction", "direct", + "indirect", "main", "simple", "primary", "reported", "report", "anova", + "correlation", "regression", "decomposition", "analysis", "finding", + "one-way", "two-way", "three-way", +} +_CHECK_ITEM_RE = re.compile( + r"\b(attention|attn|careless|comprehension|manipulation|bot|captcha|" + r"screen(?:er|ing)|instructional|quality)\s*[_ -]?\s*check\b|" + r"(?:^|[_ -])(attention|attn|careless|comprehension|manipulation|captcha)(?:[_ -]|$)|" + r"\bplease\s+(?:select|choose|pick)\b", + re.IGNORECASE, +) +_ADMIN_ITEM_RE = re.compile( + r"(?:^|[_ -])(consent|debrief|demograph|age|gender|sex|race|ethnic|" + r"education|income|prolific|mturk|duration|finished|progress|timer|timing)(?:[_ -]|$)", + re.IGNORECASE, +) +_META_INSTRUCTION_RE = re.compile( + r"\b(" + r"from\s+(?:the\s+)?(?:abstract|overview|paper|method|methods)|" + r"general\s+description|available\s+in\s+the\s+paper|paper\s+text|" + r"conceptual\s+framing|method\s+description|pages?\s+\d+" + r")\b", + re.IGNORECASE, +) + + +def _slug(text: Any, fallback: str = "study") -> str: + value = re.sub(r"[^a-z0-9]+", "_", str(text or "").lower()).strip("_") + return value or fallback + + +def _sub_study_id(value: Any) -> str: + slug = _slug(value) + if not slug.startswith(("study", "pilot")): + slug = f"study_{slug}" + return slug + + +def _loads_json(text: str) -> Dict[str, Any]: + response_text = str(text or "").strip() + if "```json" in response_text: + response_text = response_text.split("```json", 1)[1].split("```", 1)[0].strip() + elif "```" in response_text: + response_text = response_text.split("```", 1)[1].split("```", 1)[0].strip() + try: + parsed = json.loads(response_text) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", response_text, re.DOTALL) + if not match: + raise + parsed = json.loads(match.group()) + if not isinstance(parsed, dict): + raise ValueError(f"PDF material response is not a JSON object: {type(parsed)}") + return parsed + + +def _text(value: Any) -> str: + text = str(value or "").replace("\u2026", "...").strip() + text = re.sub(r"\s+", " ", text) + return text.strip() + + +def _has_ellipsis(value: Any) -> bool: + text = str(value or "") + return "..." in text or "\u2026" in text + + +def _stage1_hint_map(stage1_json: Optional[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + if not isinstance(stage1_json, dict): + return out + for exp in stage1_json.get("experiments", []) or []: + if not isinstance(exp, dict): + continue + compact = { + "experiment_id": exp.get("experiment_id"), + "study_id": exp.get("study_id"), + "design_type": exp.get("design_type"), + "conditions_or_factors": exp.get("conditions_or_factors"), + "participant_task": exp.get("participant_task"), + "input": exp.get("input"), + "output": exp.get("output"), + "candidate_source_hints": exp.get("candidate_source_hints"), + } + for key in (exp.get("study_id"), exp.get("experiment_id"), exp.get("study_name"), exp.get("experiment_name")): + sid = _sub_study_id(key) + if sid and sid not in out: + out[sid] = compact + return out + + +def _selected_or_all_studies(stage_json: Dict[str, Any]) -> List[Dict[str, Any]]: + studies = [ + study + for study in stage_json.get("eligible_studies", []) or stage_json.get("studies", []) or [] + if isinstance(study, dict) + ] + selected = [ + study + for study in studies + if isinstance(study.get("selection"), dict) and study["selection"].get("keep") is True + ] + if selected: + studies = selected + return studies + + +def _study_matches_sub_id(study: Dict[str, Any], sub_id: str) -> bool: + wanted = _sub_study_id(sub_id) + keys = { + _sub_study_id(study.get("study")), + _sub_study_id(study.get("study_id")), + _sub_study_id(study.get("experiment_id")), + _sub_study_id(study.get("study_name")), + _sub_study_id(study.get("experiment_name")), + } + return wanted in keys + + +def _stage_json_with_studies(stage_json: Dict[str, Any], studies: List[Dict[str, Any]]) -> Dict[str, Any]: + subset = dict(stage_json) + subset["eligible_studies"] = studies + subset.pop("studies", None) + return subset + + +def _compact_studies(stage_json: Dict[str, Any], stage1_json: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]: + out: List[Dict[str, Any]] = [] + stage1_hints = _stage1_hint_map(stage1_json) + studies = _selected_or_all_studies(stage_json) + for study in studies: + if not isinstance(study, dict): + continue + findings = [] + for finding in study.get("findings", []) or []: + if isinstance(finding, dict): + findings.append( + { + "finding_id": finding.get("finding_id"), + "role": finding.get("role"), + "IV": finding.get("IV"), + "DV": finding.get("DV"), + "simulation_target": finding.get("simulation_target"), + } + ) + effects = [] + for effect in study.get("effects", []) or []: + if isinstance(effect, dict): + effects.append( + { + "IV": effect.get("IV"), + "DV": effect.get("DV"), + "effecttype": effect.get("effecttype"), + "direction": effect.get("direction"), + } + ) + study_name = study.get("study") or study.get("study_id") or study.get("experiment_id") + item = { + "study": study_name, + "study_id": study.get("study_id"), + "design": study.get("design") or study.get("design_type"), + "sample": study.get("sample"), + "effects": effects[:12], + "findings": findings[:12], + "selection": study.get("selection"), + "required_hsb_fields": [ + "instructions/stimulus/vignette/manipulation", + "participant response items", + "options/anchors/scales", + "conditions/factors/levels", + ], + } + hint = stage1_hints.get(_sub_study_id(study_name)) or stage1_hints.get(_sub_study_id(study.get("study_id"))) + if hint: + item["stage1_source_hints"] = hint + out.append(item) + return out + + +def _material_context(pdf_text: str) -> str: + """Keep material-heavy parts of a long PDF instead of only the front.""" + if len(pdf_text) <= PDF_CONTEXT_MAX_CHARS: + return pdf_text + + markers = [ + "appendix", + "supplement", + "stimuli", + "stimulus", + "materials", + "measures", + "manipulation", + "question stem", + "items were", + "participants were asked", + ] + chunks: List[str] = [pdf_text[:50000]] + appendix = _appendix_context(pdf_text, max_chars=90000) + if appendix: + chunks.append(appendix) + chunks.append(pdf_text[-90000:]) + lowered = pdf_text.lower() + for marker in markers: + start = 0 + while True: + idx = lowered.find(marker, start) + if idx < 0: + break + chunks.append(pdf_text[max(0, idx - 12000): idx + 28000]) + start = idx + len(marker) + if sum(len(c) for c in chunks) >= PDF_CONTEXT_MAX_CHARS * 2: + break + + seen = set() + deduped: List[str] = [] + for chunk in chunks: + key = chunk[:500] + if key in seen: + continue + seen.add(key) + deduped.append(chunk) + text = "\n\n--- MATERIAL CONTEXT BREAK ---\n\n".join(deduped) + if len(text) > PDF_CONTEXT_MAX_CHARS: + text = text[:PDF_CONTEXT_MAX_CHARS] + return text + + +def _appendix_blocks(pdf_text: str) -> List[Dict[str, str]]: + """Return appendix sections as first-class material contexts. + + Many papers place exact stimuli and scale items only in appendices. Treating + appendix text as ordinary tail text makes it easy to truncate away the + participant-facing details after methods/results chunks are added. + """ + text = str(pdf_text or "") + if not text: + return [] + matches = list( + re.finditer( + r"(?im)(?:^|\n)\s*(Appendix\s+[A-Z][^\n]{0,140}|Supplement(?:al|ary)?\s+(?:Materials?|Appendix)[^\n]{0,120})\s*(?=\n)", + text, + ) + ) + blocks: List[Dict[str, str]] = [] + if not matches: + idx = text.lower().find("appendix") + if idx >= 0: + return [{"heading": "Appendix", "text": text[idx:]}] + return [] + for index, match in enumerate(matches): + start = match.start() + end = matches[index + 1].start() if index + 1 < len(matches) else len(text) + heading = re.sub(r"\s+", " ", match.group(1)).strip() + body = text[start:end].strip() + if body: + blocks.append({"heading": heading, "text": body}) + return blocks + + +def _appendix_context( + pdf_text: str, + *, + study: Optional[Dict[str, Any]] = None, + stage1_json: Optional[Dict[str, Any]] = None, + max_chars: int = PDF_APPENDIX_CONTEXT_MAX_CHARS, +) -> str: + blocks = _appendix_blocks(pdf_text) + if not blocks: + return "" + terms = set(_study_context_terms(study, stage1_json)) if isinstance(study, dict) else set() + priority_terms = { + "measure", + "measures", + "item", + "items", + "stimuli", + "stimulus", + "manipulation", + "question stem", + "appendix", + } + + def score(block: Dict[str, str]) -> tuple[int, int]: + haystack = f"{block.get('heading', '')}\n{block.get('text', '')}".lower() + value = 0 + for term in terms: + if len(term) >= 4 and term in haystack: + value += 3 + for term in priority_terms: + if term in haystack: + value += 2 + # Preserve original appendix order as a secondary key. + return value, -len(haystack) + + ranked = sorted(enumerate(blocks), key=lambda item: (score(item[1]), -item[0]), reverse=True) + chunks: List[str] = [] + total = 0 + per_block_limit = max(3000, min(9000, max_chars // max(1, min(len(ranked), 4)))) + for _, block in ranked: + chunk = ( + f"--- PRIORITY APPENDIX MATERIAL BLOCK: {block.get('heading', 'Appendix')} ---\n" + f"{block.get('text', '').strip()}" + ) + if not chunk.strip(): + continue + remaining = max_chars - total + if remaining <= 0: + break + chunks.append(chunk[:min(remaining, per_block_limit)]) + total += len(chunks[-1]) + return "\n\n".join(chunks) + + +def _appendix_retry_context( + pdf_text: str, + *, + study: Optional[Dict[str, Any]], + stage1_json: Optional[Dict[str, Any]], +) -> str: + appendix = _appendix_context( + pdf_text, + study=study, + stage1_json=stage1_json, + max_chars=PDF_STUDY_CONTEXT_MAX_CHARS, + ) + if not appendix: + return "" + return ( + "--- APPENDIX-ONLY MATERIAL RETRY CONTEXT ---\n" + "The prior full-PDF material extraction failed. Use this appendix-only " + "context to recover exact participant-facing stimuli, measures, items, " + "conditions, options, and anchors.\n\n" + f"{appendix}" + ) + + +def _should_appendix_retry(exc: BaseException) -> bool: + if not isinstance(exc, ValueError): + return False + text = str(exc).lower() + return "valid json" in text or "no response" in text or "response_preview='empty'" in text + + +def _clean_pdf_phrase(value: Any) -> str: + text = str(value or "") + text = ( + text.replace("\ufb01", "fi") + .replace("\ufb02", "fl") + .replace("\u0081", "-") + .replace("", "-") + ) + text = re.sub(r"\s+", " ", text).strip() + text = _fix_pdf_ligature_splits(text) + text = re.sub(r"\bCOVID-\s*19\b", "COVID-19", text, flags=re.IGNORECASE) + text = re.sub(r"\s+([,.;:!?])", r"\1", text) + text = re.sub(r"([(\[{])\s+", r"\1", text) + text = re.sub(r"\s+([)\]}])", r"\1", text) + text = re.sub(r"\s+([’'])", r"\1", text) + text = re.sub(r"([’'])\s+", r"\1", text) + text = re.sub(r"\s*/\s*", "/", text) + return text.strip() + + +def _fix_pdf_ligature_splits(text: str) -> str: + """Repair common PDF words split around fi/fl ligatures.""" + replacements = { + "of fice": "office", + "of fices": "offices", + "of ficial": "official", + "of ficials": "officials", + "con firm": "confirm", + "con firmed": "confirmed", + "dif ficult": "difficult", + "ef fect": "effect", + "ef fects": "effects", + "af fect": "affect", + "af fects": "affects", + "speci fic": "specific", + "scienti fic": "scientific", + "signi ficant": "significant", + "coef ficient": "coefficient", + } + out = str(text or "") + for broken, fixed in replacements.items(): + out = re.sub(rf"\b{re.escape(broken)}\b", fixed, out, flags=re.IGNORECASE) + return out + + +def _clean_pdf_multiline(value: Any) -> str: + text = str(value or "") + text = ( + text.replace("\ufb01", "fi") + .replace("\ufb02", "fl") + .replace("\u0081", "-") + .replace("", "-") + ) + lines = [_clean_pdf_phrase(line) for line in text.splitlines()] + lines = [line for line in lines if line] + out = "\n".join(lines) + out = re.sub(r"\n{3,}", "\n\n", out) + return out.strip() + + +def _strip_pdf_artifacts(value: str) -> str: + text = str(value or "") + markers = [ + "This document is copyrighted", + "All rights, including", + "This article is intended solely", + "--- Page", + "LEADERS' ANTI-ASIAN COMMUNICATION", + "LEADERS’ ANTI-ASIAN COMMUNICATION", + "JUN AND WU", + ] + cut = len(text) + lowered = text.lower() + for marker in markers: + idx = lowered.find(marker.lower()) + if idx >= 0: + cut = min(cut, idx) + return text[:cut].strip() + + +def _study_number(value: Any) -> Optional[str]: + match = re.search(r"\bstudy\s+([0-9]+[a-z]?)\b", str(value or ""), flags=re.IGNORECASE) + return match.group(1).lower() if match else None + + +def _stage_study_number(study: Dict[str, Any]) -> Optional[str]: + for key in ("study", "study_id", "experiment_id", "study_name", "experiment_name"): + number = _study_number(study.get(key)) + if number: + return number + return None + + +def _appendix_text_by_heading(pdf_text: str, heading_prefix: str) -> str: + chunks = [ + block.get("text", "") + for block in _appendix_blocks(pdf_text) + if str(block.get("heading") or "").lower().startswith(heading_prefix.lower()) + ] + return "\n\n".join(chunk for chunk in chunks if chunk).strip() + + +def _study_section_from_appendix(appendix_text: str, study_number: Optional[str]) -> str: + text = str(appendix_text or "") + if not text: + return "" + if not study_number: + return text + pattern = re.compile(rf"(?im)^\s*Study\s+{re.escape(study_number)}\b[^\n]*") + match = pattern.search(text) + if not match: + return "" + next_match = re.search(r"(?im)^\s*Study\s+[0-9]+[a-z]?\b[^\n]*", text[match.end():]) + end = match.end() + next_match.start() if next_match else len(text) + return text[match.start():end].strip() + + +def _is_measure_heading(line: str) -> bool: + clean = _clean_pdf_phrase(line) + if not clean: + return False + lowered = clean.lower() + if lowered in {"measures", "control variables"} or lowered.startswith(("study ", "appendix ")): + return False + if lowered.startswith( + ( + "all measures", + "we used", + "we adapted", + "to measure", + "before measuring", + "participants", + "question items", + "items were", + "the items", + "the question", + "carelessness", + ) + ): + return False + if re.search(r"[.;:?]\s*$", clean): + return False + words = clean.split() + if not 1 <= len(words) <= 8: + return False + alpha_words = [word for word in words if re.search(r"[A-Za-z]", word)] + titleish = sum(1 for word in alpha_words if word[:1].isupper()) + return bool(alpha_words and titleish >= max(1, len(alpha_words) // 2)) + + +def _measure_sections(study_section: str) -> List[tuple[str, str]]: + lines = _clean_pdf_multiline(_strip_pdf_artifacts(study_section)).splitlines() + heading_indices = [ + (idx, line) + for idx, line in enumerate(lines) + if _is_measure_heading(line) + ] + sections: List[tuple[str, str]] = [] + for pos, (start, heading) in enumerate(heading_indices): + end = heading_indices[pos + 1][0] if pos + 1 < len(heading_indices) else len(lines) + body = "\n".join(lines[start + 1:end]).strip() + if re.search(r"\b(?:question\s+)?items?\s+were\b", body, flags=re.IGNORECASE): + sections.append((heading, body)) + return sections + + +def _scale_options(scale: Dict[str, Any]) -> List[str]: + try: + lo = int(float(scale["min"])) + hi = int(float(scale["max"])) + except (KeyError, TypeError, ValueError): + return [] + anchors = scale.get("anchors") if isinstance(scale.get("anchors"), dict) else {} + out: List[str] = [] + for value in range(lo, hi + 1): + label = _clean_pdf_phrase(anchors.get(str(value), "")) + out.append(f"{value} - {label}" if label and label != str(value) else str(value)) + return out + + +def _scale_from_text(text: str, fallback: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + base = dict(fallback or {}) + clean = _clean_pdf_phrase(text) + match = re.search( + r"\b([0-9]+)\s*=\s*(.+?)\s+to\s+([0-9]+)\s*=\s*(.+?)(?:[).,;]|$)", + clean, + flags=re.IGNORECASE, + ) + if not match: + return base + lo, left, hi, right = match.groups() + left = _clean_pdf_phrase(left).strip(" .") + right = _clean_pdf_phrase(right).strip(" .") + return { + "min": int(lo), + "max": int(hi), + "anchors": {str(int(lo)): left, str(int(hi)): right}, + } + + +def _extract_stem(section_text: str) -> str: + clean = _clean_pdf_phrase(section_text) + patterns = [ + r"question stem read,?\s*[\"“](.+?)[\"”]", + r"with the following question stem\s*[\"“](.+?)[\"”]", + r"with the question stem,?\s*[\"“](.+?)[\"”]", + ] + for pattern in patterns: + match = re.search(pattern, clean, flags=re.IGNORECASE) + if match: + stem = _clean_pdf_phrase(match.group(1)) + stem = re.sub(r"\?\.$", "?", stem) + return stem + return "" + + +def _split_lettered_items(items_text: str) -> List[str]: + clean = _clean_pdf_phrase(items_text) + matches = list(re.finditer(r"(?:^|[\s;,])(?:and\s+)?\(([a-z])\)\s*", clean, flags=re.IGNORECASE)) + items: List[str] = [] + for idx, match in enumerate(matches): + start = match.end() + end = matches[idx + 1].start() if idx + 1 < len(matches) else len(clean) + item = _clean_pdf_phrase(clean[start:end]) + item = re.sub(r"^(?:and|or)\s+", "", item, flags=re.IGNORECASE) + item = item.strip(" ;,") + item = re.sub(r"\.\s*[\"”]?$", "", item).strip() + if item: + items.append(item) + return items + + +def _extract_items_text(section_text: str) -> str: + clean = _clean_pdf_phrase(section_text) + match = re.search( + r"\b(?:Question\s+items|The\s+(?:three\s+)?items|Items)\s+were,?\s*(.+)$", + clean, + flags=re.IGNORECASE, + ) + if not match: + return "" + tail = match.group(1) + tail = re.split( + r"\b(?:To note|Carelessness Checks|Control Variables)\b|This document is copyrighted", + tail, + maxsplit=1, + flags=re.IGNORECASE, + )[0] + return tail.strip() + + +def _extract_measure_items( + study_section: str, + *, + material_id: str, + pdf_path: Path, + source_label: str, +) -> List[Dict[str, Any]]: + default_scale = _scale_from_text(study_section, {"min": 1, "max": 7, "anchors": {}}) + items: List[Dict[str, Any]] = [] + for heading, body in _measure_sections(study_section): + stem = _extract_stem(body) + scale = _scale_from_text(body, default_scale) + options = _scale_options(scale) + raw_items = _split_lettered_items(_extract_items_text(body)) + for item_index, item_text in enumerate(raw_items, start=1): + reverse_coded = bool(re.search(r"reverse[- ]coded", item_text, flags=re.IGNORECASE)) + item_text = re.sub(r"\s*\(reverse[- ]coded\)\s*", "", item_text, flags=re.IGNORECASE).strip(" ;,.") + question = _clean_pdf_phrase(f"{stem} {item_text}".strip()) if stem else _clean_pdf_phrase(item_text) + item = { + "id": f"{_slug(heading, material_id)}_{item_index}", + "question": question, + "options": options, + "type": "likert", + "scale": scale, + "response_format": { + "answer_type": "likert", + "scale_min": scale.get("min"), + "scale_max": scale.get("max"), + "anchors": scale.get("anchors", {}), + "options": options, + }, + "source": source_label, + "source_file": str(pdf_path), + "block": heading, + "metadata": { + "construct": heading, + "reverse_coded": reverse_coded, + "evidence": {"section": "Appendix A", "heading": heading}, + }, + } + items.append(item) + return items + + +def _condition_label(value: str) -> str: + text = _clean_pdf_phrase(value).replace("-", " ") + text = re.sub(r"\s+", " ", text).strip() + return text[:1].upper() + text[1:] if text else text + + +def _condition_block_study_number(block_text: str) -> Optional[str]: + match = re.search(r"\bStudy\s+([0-9]+[a-z]?)\s+Manipulation\b", block_text, flags=re.IGNORECASE) + return match.group(1).lower() if match else _study_number(block_text) + + +def _extract_bracketed_condition(block_text: str) -> tuple[List[Dict[str, Any]], str]: + text = _clean_pdf_multiline(_strip_pdf_artifacts(block_text)) + if not text or "[" not in text or "/" not in text: + return [], "" + label_match = re.search(r"\[([^/\[\]]+)/([^\[\]]+)\]\s+condition", text, flags=re.IGNORECASE) + if not label_match: + return [], "" + levels = [_condition_label(label_match.group(1)), _condition_label(label_match.group(2))] + body_match = re.search(r"\bTo:\s*.+$", text, flags=re.IGNORECASE | re.DOTALL) + if not body_match: + return [], "" + template = body_match.group(0).strip() + alt_re = re.compile(r"\[([^\[\]/]+)/([^\[\]]+)\]") + descriptions: Dict[str, str] = {} + for idx, level in enumerate(levels): + def repl(match: re.Match[str]) -> str: + return _clean_pdf_phrase(match.group(idx + 1)) + + descriptions[level] = _clean_pdf_phrase(alt_re.sub(repl, template)) + condition = { + "name": "label condition", + "levels": levels, + "level_descriptions": descriptions, + "source": "pdf_appendix_parser", + "source_file": "", + "metadata": { + "evidence": {"section": "Appendix B"}, + "stimulus_template": _clean_pdf_phrase(template), + }, + } + return [condition], _clean_pdf_phrase(template) + + +def _appendix_material_instructions(conditions: List[Dict[str, Any]], study_number: Optional[str]) -> str: + if conditions: + level_text = ", ".join( + str(level) + for condition in conditions + for level in condition.get("levels", []) or [] + ) + return ( + "Read the team manager email assigned by the study condition " + f"({level_text}). Then answer the following questions about your reactions " + "and perceptions after reading the email." + ) + label = f"Study {study_number}" if study_number else "the study" + return f"Complete the participant-facing survey questions for {label}." + + +def _deterministic_appendix_materials( + stage_json: Dict[str, Any], + pdf_path: Path, + pdf_text: str, + *, + source_label: str = "pdf_appendix_parser", +) -> Dict[str, Dict[str, Any]]: + appendix_a = _appendix_text_by_heading(pdf_text, "Appendix A") + if not appendix_a: + return {} + appendix_b = _appendix_text_by_heading(pdf_text, "Appendix B") + condition_study = _condition_block_study_number(appendix_b) if appendix_b else None + parsed_conditions, stimulus_template = _extract_bracketed_condition(appendix_b) + for condition in parsed_conditions: + condition["source_file"] = str(pdf_path) + + materials: Dict[str, Dict[str, Any]] = {} + for study in _selected_or_all_studies(stage_json): + if not isinstance(study, dict): + continue + study_name = study.get("study") or study.get("study_id") or study.get("experiment_id") + study_number = _stage_study_number(study) + study_section = _study_section_from_appendix(appendix_a, study_number) + if not study_section: + continue + sub_id = _sub_study_id(study_name) + items = _extract_measure_items( + study_section, + material_id=sub_id, + pdf_path=pdf_path, + source_label=source_label, + ) + if not items: + continue + conditions = parsed_conditions if parsed_conditions and study_number == condition_study else [] + instructions = _appendix_material_instructions(conditions, study_number) + warnings: List[str] = [] + blocking: List[str] = [] + if not conditions and any("condition" in str(effect.get("IV") or "").lower() for effect in study.get("effects", []) or [] if isinstance(effect, dict)): + warnings.append("No appendix condition block matched this study; verify manipulated conditions manually.") + if any(_has_ellipsis(item.get("question")) for item in items): + blocking.append("truncated_pdf_item_text") + ready = bool(instructions and items) and not blocking + material = { + "sub_study_id": sub_id, + "instructions": instructions, + "items": items, + "conditions": conditions, + "response_schema": _infer_response_schema(items), + "readiness": { + "ready": ready, + "blocking_issues": blocking, + "warnings": warnings, + }, + "source_trace": { + "primary_source": source_label, + "source_file": str(pdf_path), + "extractor": "stage3_pdf_appendix_parser_v1", + "preserve_full_instrument_for_runtime": True, + "conditions_source": "Appendix B" if conditions else None, + "items_source": "Appendix A", + "stimulus_template": stimulus_template if conditions else "", + "evidence": { + "sections": ["Appendix A", *(["Appendix B"] if conditions else [])], + "study_section": f"Study {study_number}" if study_number else "", + "item_constructs": sorted({item.get("block") for item in items if item.get("block")}), + }, + }, + } + materials[sub_id] = material + return materials + + +def _study_context_terms(study: Dict[str, Any], stage1_json: Optional[Dict[str, Any]]) -> List[str]: + values: List[Any] = [ + study.get("study"), + study.get("study_id"), + study.get("experiment_id"), + study.get("study_name"), + study.get("experiment_name"), + study.get("design") or study.get("design_type"), + ] + for key in ("conditions_or_factors", "participant_task", "input", "output"): + values.append(study.get(key)) + for effect in study.get("effects", []) or []: + if isinstance(effect, dict): + values.extend([effect.get("IV"), effect.get("DV"), effect.get("claim")]) + for finding in study.get("findings", []) or []: + if isinstance(finding, dict): + values.extend([finding.get("IV"), finding.get("DV"), finding.get("claim")]) + + stage1_hints = _stage1_hint_map(stage1_json) + for key in ( + _sub_study_id(study.get("study")), + _sub_study_id(study.get("study_id")), + _sub_study_id(study.get("experiment_id")), + ): + hint = stage1_hints.get(key) + if isinstance(hint, dict): + values.extend(hint.values()) + + terms: List[str] = [] + seen = set() + for value in values[:5]: + phrase = str(value or "").lower().strip() + if len(phrase) >= 4 and phrase not in seen: + seen.add(phrase) + terms.append(phrase) + for value in values: + for token in _TOKEN_RE.findall(str(value or "")): + token = token.lower().strip("'_-") + if len(token) < 4 or token in _TARGET_STOP_TERMS or token.isdigit(): + continue + if token not in seen: + seen.add(token) + terms.append(token) + return terms[:40] + + +def _study_material_context( + pdf_text: str, + study: Dict[str, Any], + stage1_json: Optional[Dict[str, Any]], +) -> str: + """Keep a smaller, study-specific source context for one material call.""" + if len(pdf_text) <= PDF_STUDY_CONTEXT_MAX_CHARS: + return pdf_text + + lowered = pdf_text.lower() + chunks: List[str] = [pdf_text[:6000]] + appendix = _appendix_context( + pdf_text, + study=study, + stage1_json=stage1_json, + max_chars=PDF_APPENDIX_CONTEXT_MAX_CHARS, + ) + if appendix: + chunks.append(appendix) + study_terms = _study_context_terms(study, stage1_json) + terms = [ + *study_terms[:12], + "procedure", + "method", + "appendix", + "supplement", + "supplemental", + "stimuli", + "stimulus", + "measures", + "manipulation", + "participants were asked", + *study_terms[12:], + ] + total_chars = sum(len(chunk) for chunk in chunks) + for term in terms: + start = 0 + hits = 0 + needle = str(term or "").lower().strip() + if len(needle) < 4: + continue + while hits < 3: + idx = lowered.find(needle, start) + if idx < 0: + break + chunks.append(pdf_text[max(0, idx - 3500): idx + 9000]) + total_chars += len(chunks[-1]) + hits += 1 + start = idx + len(needle) + if total_chars >= PDF_STUDY_CONTEXT_MAX_CHARS * 2: + break + if total_chars >= PDF_STUDY_CONTEXT_MAX_CHARS * 2: + break + + seen = set() + deduped: List[str] = [] + for chunk in chunks: + key = re.sub(r"\s+", " ", chunk[:600]).strip() + if key in seen: + continue + seen.add(key) + deduped.append(chunk) + context = "\n\n--- STUDY MATERIAL CONTEXT BREAK ---\n\n".join(deduped) + if len(context) > PDF_STUDY_CONTEXT_MAX_CHARS: + context = context[:PDF_STUDY_CONTEXT_MAX_CHARS] + return context + + +def _infer_response_schema(items: List[Dict[str, Any]]) -> Dict[str, Any]: + if not items: + return {} + dominant = str(items[0].get("type") or "open_ended") + schema: Dict[str, Any] = {"answer_type": dominant} + options = items[0].get("options") + if isinstance(options, list) and options: + schema["options"] = options + scale = items[0].get("scale") if isinstance(items[0].get("scale"), dict) else {} + if scale: + if scale.get("min") is not None: + schema["scale_min"] = scale.get("min") + if scale.get("max") is not None: + schema["scale_max"] = scale.get("max") + anchors = scale.get("anchors") + if isinstance(anchors, dict) and anchors: + schema["anchors"] = anchors + return schema + + +def _normalize_item(raw: Dict[str, Any], material_id: str, index: int) -> Dict[str, Any]: + item_type = str(raw.get("type") or raw.get("answer_type") or "open_ended").strip().lower() + if item_type not in _VALID_TYPES: + item_type = "open_ended" + options = raw.get("options") + if not isinstance(options, list): + options = [] + options = [_text(option) for option in options if _text(option)] + scale = raw.get("scale") if isinstance(raw.get("scale"), dict) else {} + anchors = raw.get("anchors") if isinstance(raw.get("anchors"), dict) else {} + if anchors and not scale: + scale = {"anchors": anchors} + item = { + "id": _slug(raw.get("id") or raw.get("data_export_tag") or f"{material_id}_item_{index}", f"{material_id}_item_{index}"), + "question": _text(raw.get("question") or raw.get("text") or raw.get("label")), + "type": item_type, + "options": options, + } + if scale: + item["scale"] = scale + response_format = raw.get("response_format") if isinstance(raw.get("response_format"), dict) else {} + if response_format: + item["response_format"] = response_format + metadata = raw.get("metadata") if isinstance(raw.get("metadata"), dict) else {} + evidence = raw.get("evidence") if isinstance(raw.get("evidence"), dict) else {} + if evidence: + metadata = {**metadata, "evidence": evidence} + if metadata: + item["metadata"] = metadata + return item + + +def _token_stem(token: str) -> str: + token = token.lower().strip("'_-") + for suffix in ("iveness", "ation", "ness", "ment", "ity", "ing", "ed", "es", "s"): + if len(token) > len(suffix) + 4 and token.endswith(suffix): + token = token[:-len(suffix)] + break + return token[:6] if len(token) > 6 else token + + +def _target_stems(*values: Any) -> set[str]: + stems: set[str] = set() + for value in values: + for token in _TOKEN_RE.findall(str(value or "").lower()): + token = token.strip("'_-") + if len(token) < 4 or token in _TARGET_STOP_TERMS or token.isdigit(): + continue + stems.add(_token_stem(token)) + return stems + + +def _study_lookup_key(value: Any) -> str: + return _sub_study_id(value) + + +def _matching_stage_study(stage_json: Dict[str, Any], sub_id: str, study_name: Any) -> Optional[Dict[str, Any]]: + studies = [ + study + for study in stage_json.get("eligible_studies", []) or stage_json.get("studies", []) or [] + if isinstance(study, dict) + ] + wanted = {_study_lookup_key(sub_id), _study_lookup_key(study_name)} + for study in studies: + keys = { + _study_lookup_key(study.get("study")), + _study_lookup_key(study.get("study_id")), + _study_lookup_key(study.get("experiment_id")), + } + if wanted & keys: + return study + return studies[0] if len(studies) == 1 else None + + +def _target_dv_values(study: Optional[Dict[str, Any]]) -> List[Any]: + if not isinstance(study, dict): + return [] + + findings = [finding for finding in study.get("findings", []) or [] if isinstance(finding, dict)] + candidate_findings = [ + finding + for finding in findings + if isinstance(finding.get("simulation_target"), dict) + and finding["simulation_target"].get("candidate") is True + ] + role_findings = [ + finding + for finding in findings + if str(finding.get("role") or "").lower() in {"primary_finding", "interaction", "main_effect"} + ] + selected_findings = candidate_findings or role_findings or findings + values = [finding.get("DV") for finding in selected_findings if finding.get("DV")] + + effects = [effect for effect in study.get("effects", []) or [] if isinstance(effect, dict)] + for effect in effects: + consolidation = effect.get("consolidation") if isinstance(effect.get("consolidation"), dict) else {} + if consolidation.get("is_primary_simulation_target") is True and consolidation.get("is_representative", True) is not False: + if effect.get("DV"): + values.append(effect.get("DV")) + + return values + + +def _item_match_score(item: Dict[str, Any], stems: set[str]) -> int: + if not stems: + return 0 + haystack = " ".join( + str(item.get(key) or "") + for key in ("id", "data_export_tag", "question") + ) + item_stems = _target_stems(haystack) + score = len(item_stems & stems) + tag = str(item.get("data_export_tag") or item.get("id") or "").lower() + tag_parts = [_token_stem(part) for part in re.split(r"[^a-z0-9]+", tag) if part] + for stem in stems: + if any(part and (stem.startswith(part) or part.startswith(stem[:4])) for part in tag_parts): + score += 2 + return score + + +def _non_response_item_reason(item: Dict[str, Any]) -> Optional[str]: + item_id = str(item.get("id") or item.get("data_export_tag") or "") + question = _text(item.get("question")) + haystack = f"{item_id} {question}" + if _ADMIN_ITEM_RE.search(haystack): + return "admin_or_demographic_item" + if _CHECK_ITEM_RE.search(haystack): + return "attention_or_manipulation_check" + item_type = str(item.get("type") or "").lower() + if item_type == "text" and not item.get("options") and not item.get("scale"): + return "display_text_not_response_item" + return None + + +def _filter_pdf_response_items( + items: List[Dict[str, Any]], + *, + stage_json: Dict[str, Any], + sub_id: str, + study_name: Any, +) -> tuple[List[Dict[str, Any]], Dict[str, Any]]: + report: Dict[str, Any] = { + "mode": "none", + "dropped_non_response": [], + "dropped_non_target": [], + "target_stems": [], + "kept": [], + } + + substantive: List[Dict[str, Any]] = [] + for item in items: + reason = _non_response_item_reason(item) + item_id = str(item.get("data_export_tag") or item.get("id") or item.get("question") or "") + if reason: + report["dropped_non_response"].append({"id": item_id, "reason": reason}) + continue + substantive.append(item) + + study = _matching_stage_study(stage_json, sub_id, study_name) + stems = _target_stems(*_target_dv_values(study)) + report["target_stems"] = sorted(stems) + scored = [(item, _item_match_score(item, stems)) for item in substantive] + target_items = [item for item, score in scored if score > 0] + + if target_items and len(target_items) < len(substantive): + target_ids = {id(item) for item in target_items} + report["mode"] = "target_dv_terms" + report["dropped_non_target"] = [ + str(item.get("data_export_tag") or item.get("id") or item.get("question") or "") + for item in substantive + if id(item) not in target_ids + ] + report["kept"] = [ + str(item.get("data_export_tag") or item.get("id") or item.get("question") or "") + for item in target_items + ] + return target_items, report + + report["mode"] = "non_response_removed" if report["dropped_non_response"] else "all_items" + report["kept"] = [ + str(item.get("data_export_tag") or item.get("id") or item.get("question") or "") + for item in substantive + ] + return substantive, report + + +def _tag_pdf_item_sources(items: List[Dict[str, Any]], pdf_path: Path, source_label: str) -> None: + source_file = str(pdf_path) + for item in items: + item.setdefault("source", source_label) + item.setdefault("source_file", source_file) + + +def _normalize_conditions(raw_conditions: Any) -> List[Dict[str, Any]]: + out: List[Dict[str, Any]] = [] + if not isinstance(raw_conditions, list): + return out + for idx, cond in enumerate(raw_conditions, start=1): + if not isinstance(cond, dict): + continue + levels = cond.get("levels") + if not isinstance(levels, list): + levels = [] + item = { + "name": _text(cond.get("name") or cond.get("label") or f"condition_{idx}"), + "levels": [_text(level) for level in levels if _text(level)], + } + descriptions = cond.get("level_descriptions") + if isinstance(descriptions, dict): + item["level_descriptions"] = { + _text(k): _text(v) for k, v in descriptions.items() if _text(k) and _text(v) + } + out.append(item) + return out + + +def _study_mentions(value: Any) -> set[str]: + text = str(value or "") + return { + match.group(1).lower() + for match in re.finditer(r"\bstudy\s+([0-9]+[a-z]?)\b", text, flags=re.IGNORECASE) + } + + +def _expected_study_mentions(stage_json: Dict[str, Any], sub_id: str, study_name: Any) -> set[str]: + study = _matching_stage_study(stage_json, sub_id, study_name) + if not isinstance(study, dict): + return _study_mentions(study_name) + mentions: set[str] = set() + for key in ("study", "study_id", "experiment_id", "study_name", "experiment_name"): + mentions |= _study_mentions(study.get(key)) + return mentions or _study_mentions(study_name) + + +def _has_cross_study_evidence( + *, + stage_json: Dict[str, Any], + sub_id: str, + study_name: Any, + instructions: str, + evidence: Dict[str, Any], +) -> bool: + expected = _expected_study_mentions(stage_json, sub_id, study_name) + if not expected: + return False + evidence_text = instructions + "\n" + json.dumps(evidence, ensure_ascii=False) + mentioned = _study_mentions(evidence_text) + return bool(mentioned and not (mentioned & expected)) + + +def _normalize_material( + raw: Dict[str, Any], + pdf_path: Path, + stage_json: Dict[str, Any], + *, + source_label: str = "pdf_llm", +) -> Optional[Dict[str, Any]]: + study_name = raw.get("study") or raw.get("study_id") or raw.get("sub_study_id") + sub_id = _sub_study_id(study_name) + instructions = _text(raw.get("instructions") or raw.get("stimulus") or raw.get("vignette")) + material_id = sub_id + items = [ + _normalize_item(item, material_id, idx) + for idx, item in enumerate(raw.get("items") or [], start=1) + if isinstance(item, dict) + ] + items = [item for item in items if item.get("question")] + items, item_filter = _filter_pdf_response_items( + items, + stage_json=stage_json, + sub_id=sub_id, + study_name=study_name, + ) + _tag_pdf_item_sources(items, pdf_path, source_label) + conditions = _normalize_conditions(raw.get("conditions")) + + completeness = str(raw.get("completeness") or "partial").strip().lower() + evidence = raw.get("evidence") if isinstance(raw.get("evidence"), dict) else {} + blocking: List[str] = [] + warnings: List[str] = [] + if not instructions: + blocking.append("missing_instructions") + if not items: + blocking.append("no_participant_items") + if completeness in {"none", "missing"}: + blocking.append("pdf_material_not_found") + elif completeness not in {"complete", "ready"}: + blocking.append("pdf_material_incomplete") + if _has_cross_study_evidence( + stage_json=stage_json, + sub_id=sub_id, + study_name=study_name, + instructions=instructions, + evidence=evidence, + ): + blocking.append("pdf_material_cross_study_evidence") + warnings.append("PDF material evidence mentions a different study than the target study.") + if _META_INSTRUCTION_RE.search(instructions): + blocking.append("pdf_material_not_participant_facing") + warnings.append("PDF material instructions contain source commentary rather than participant-facing wording.") + if any(_has_ellipsis(item.get("question")) or any(_has_ellipsis(opt) for opt in item.get("options", [])) for item in items): + blocking.append("truncated_pdf_item_text") + warnings.append("PDF material extraction contains ellipsis markers; recover full wording before simulation.") + if raw.get("verbatim") is False: + warnings.append("PDF material extractor marked the material as non-verbatim or reconstructed.") + if raw.get("missing"): + warnings.append("Missing PDF material details: " + _text(raw.get("missing"))) + + ready = bool(instructions and items) and not blocking + return { + "sub_study_id": sub_id, + "instructions": instructions, + "items": items, + "conditions": conditions, + "response_schema": _infer_response_schema(items), + "readiness": { + "ready": ready, + "blocking_issues": sorted(dict.fromkeys(blocking)), + "warnings": warnings, + }, + "source_trace": { + "primary_source": source_label, + "source_file": str(pdf_path), + "extractor": "stage3_pdf_materials_v1", + "evidence": evidence, + "item_filter": item_filter, + }, + } + + +def build_pdf_material_prompt( + stage_json: Dict[str, Any], + pdf_text: str, + stage1_json: Optional[Dict[str, Any]] = None, +) -> str: + """Build the PDF material extractor prompt. Exposed for unit tests.""" + return f"""Extract participant-facing study materials from this psychology/management paper PDF or supplementary source document. + +Use the Stage 2 study/finding inventory only to decide which studies and +outcomes matter. Extract materials from the source text itself. Do not invent +question wording, options, anchors, condition labels, or vignettes. If the PDF +only names a cited scale without item wording, mark that study as partial or +missing. Do not use ellipses; if wording is truncated, mark it incomplete. +Put display text, vignettes, and stimuli in instructions/conditions, not in +items. Exclude attention checks, careless-response checks, comprehension +checks, manipulation checks, consent, demographics, timers, and debriefing from +items unless Stage 2 explicitly makes that construct the simulation target. + +For each eligible study, return a study-level material package matching the +HumanStudy-Bench material shape: +- instructions: participant-facing task framing, scenario, vignette, or stimulus. +- items: participant response questions/items with type, options or scale/anchors. +- conditions: manipulated/measured factors and condition levels. +- completeness: complete, partial, or none. +- evidence: short source sections/pages/quotes proving the extraction. + +SOURCE TEXT: +{pdf_text} + +STAGE 1/2 STUDY, TARGET, AND SOURCE-HINT INVENTORY: +{json.dumps(_compact_studies(stage_json, stage1_json), ensure_ascii=False, indent=2)} + +Return ONLY JSON with this schema: +{{ + "materials": [ + {{ + "study": "Study 1", + "instructions": "", + "items": [ + {{ + "id": "interpersonal_justice_1", + "question": "", + "type": "likert|scale|slider|multiple_choice|open_ended|ranking|matrix|text", + "options": [], + "scale": {{"min": 1, "max": 7, "anchors": {{"1": "Never", "7": "Always"}}}}, + "evidence": {{"section": "Appendix A"}} + }} + ], + "conditions": [ + {{ + "name": "label condition", + "levels": ["neutral labels", "stigmatizing labels"], + "level_descriptions": {{"neutral labels": "", "stigmatizing labels": ""}} + }} + ], + "completeness": "complete|partial|none", + "verbatim": true, + "missing": "", + "evidence": {{"sections": ["Appendix A"], "quotes": ["short quote"]}} + }} + ], + "notes": "short extraction note" +}}""" + + +def extract_pdf_study_materials( + stage_json: Dict[str, Any], + pdf_path: Path, + llm_client: Any, + *, + stage1_json: Optional[Dict[str, Any]] = None, + pdf_text: Optional[str] = None, + source_label: str = "pdf_llm", + timeout: Optional[float] = 60.0, + max_attempts: int = 2, + retry_delay: float = 1.0, + only_sub_study_id: Optional[str] = None, +) -> Dict[str, Dict[str, Any]]: + text = pdf_text if pdf_text is not None else extract_pdf_text(Path(pdf_path), max_chars=PDF_TEXT_MAX_CHARS) + studies = _selected_or_all_studies(stage_json) + if only_sub_study_id: + studies = [study for study in studies if _study_matches_sub_id(study, only_sub_study_id)] + stage_json = _stage_json_with_studies(stage_json, studies) + if not studies: + return {} + + deterministic = _deterministic_appendix_materials( + stage_json, + Path(pdf_path), + text, + source_label="pdf_appendix_parser", + ) + if deterministic: + target_ids = { + _sub_study_id(study.get("study") or study.get("study_id") or study.get("experiment_id")) + for study in studies + if isinstance(study, dict) + } + if llm_client is None or target_ids <= set(deterministic): + return deterministic + if llm_client is None: + return deterministic or {} + if len(studies) > 1: + try: + materials = _extract_pdf_study_materials_by_study( + stage_json, + Path(pdf_path), + llm_client, + text, + studies=studies, + stage1_json=stage1_json, + source_label=source_label, + timeout=timeout, + max_attempts=max_attempts, + retry_delay=retry_delay, + ) + except Exception: + if deterministic: + return deterministic + raise + materials.update(deterministic) + return materials + + context = _material_context(text) + prompt = build_pdf_material_prompt(stage_json, context, stage1_json=stage1_json) + try: + parsed = _generate_json_with_retries( + llm_client, + prompt=prompt, + timeout=timeout, + max_attempts=max_attempts, + retry_delay=retry_delay, + ) + except Exception as exc: + if not _should_appendix_retry(exc): + raise + retry_context = _appendix_retry_context( + text, + study=studies[0] if studies else None, + stage1_json=stage1_json, + ) + if not retry_context: + raise + parsed = _generate_json_with_retries( + llm_client, + prompt=build_pdf_material_prompt(stage_json, retry_context, stage1_json=stage1_json), + timeout=timeout, + max_attempts=1, + retry_delay=retry_delay, + ) + return _normalize_pdf_material_response(parsed, Path(pdf_path), stage_json, source_label=source_label) + + +def _normalize_pdf_material_response( + parsed: Dict[str, Any], + pdf_path: Path, + stage_json: Dict[str, Any], + *, + source_label: str, +) -> Dict[str, Dict[str, Any]]: + materials: Dict[str, Dict[str, Any]] = {} + for raw in parsed.get("materials") or []: + if not isinstance(raw, dict): + continue + material = _normalize_material(raw, Path(pdf_path), stage_json, source_label=source_label) + if material is None: + continue + materials[material["sub_study_id"]] = material + return materials + + +def _extract_pdf_study_materials_by_study( + stage_json: Dict[str, Any], + pdf_path: Path, + llm_client: Any, + pdf_text: str, + *, + studies: List[Dict[str, Any]], + stage1_json: Optional[Dict[str, Any]], + source_label: str, + timeout: Optional[float], + max_attempts: int, + retry_delay: float, +) -> Dict[str, Dict[str, Any]]: + materials: Dict[str, Dict[str, Any]] = {} + errors: List[str] = [] + total = len(studies) + for index, study in enumerate(studies, start=1): + single_stage = _stage_json_with_studies(stage_json, [study]) + context = _study_material_context(pdf_text, study, stage1_json) + study_name = study.get("study") or study.get("study_id") or study.get("experiment_id") or "unknown" + print( + f" Stage 3 PDF material extraction {index}/{total}: {study_name} " + f"({len(context)} chars)", + flush=True, + ) + prompt = build_pdf_material_prompt( + single_stage, + context, + stage1_json=stage1_json, + ) + try: + parsed = _generate_json_with_retries( + llm_client, + prompt=prompt, + timeout=timeout, + max_attempts=max_attempts, + retry_delay=retry_delay, + ) + materials.update( + _normalize_pdf_material_response( + parsed, + pdf_path, + single_stage, + source_label=source_label, + ) + ) + except Exception as exc: + if not _should_appendix_retry(exc): + errors.append(f"{study_name}: {type(exc).__name__}: {exc}") + continue + retry_context = _appendix_retry_context( + pdf_text, + study=study, + stage1_json=stage1_json, + ) + if retry_context: + print( + f" Stage 3 PDF appendix-only retry {index}/{total}: {study_name} " + f"({len(retry_context)} chars)", + flush=True, + ) + try: + parsed = _generate_json_with_retries( + llm_client, + prompt=build_pdf_material_prompt( + single_stage, + retry_context, + stage1_json=stage1_json, + ), + timeout=timeout, + max_attempts=1, + retry_delay=retry_delay, + ) + materials.update( + _normalize_pdf_material_response( + parsed, + pdf_path, + single_stage, + source_label=source_label, + ) + ) + continue + except Exception as retry_exc: + errors.append( + f"{study_name}: {type(exc).__name__}: {exc}; " + f"appendix_retry={type(retry_exc).__name__}: {retry_exc}" + ) + continue + errors.append(f"{study_name}: {type(exc).__name__}: {exc}") + continue + if materials: + return materials + if errors: + raise RuntimeError("Stage 3 PDF material extraction failed for all study-level calls: " + " | ".join(errors)) + return {} + + +def _generate_json_with_retries( + llm_client: Any, + *, + prompt: str, + timeout: Optional[float], + max_attempts: int, + retry_delay: float, +) -> Dict[str, Any]: + attempts = max(1, int(max_attempts or 1)) + current_prompt = prompt + last_error: Optional[BaseException] = None + last_parse_error: Optional[BaseException] = None + last_preview = "empty" + for attempt in range(1, attempts + 1): + try: + response = call_with_timeout( + lambda: _generate_once(llm_client, prompt=current_prompt, timeout=timeout), + timeout=timeout, + ) + except Exception as exc: + last_error = exc + if type(exc).__name__ == "LLMTimeoutError": + break + if attempt >= attempts: + break + if retry_delay > 0: + time.sleep(retry_delay) + continue + + text = str(response or "").strip() + last_preview = text[:700] if text else "empty" + try: + return _loads_json(text) + except Exception as exc: + last_parse_error = exc + if attempt >= attempts: + break + current_prompt = ( + prompt + + "\n\nYour previous answer was not parseable JSON. " + + "Return ONLY the JSON object matching the requested schema; no prose, no markdown, no commentary." + ) + if retry_delay > 0: + time.sleep(retry_delay) + if last_parse_error is not None: + raise ValueError( + "Stage 3 PDF material extractor did not return valid JSON: " + f"{type(last_parse_error).__name__}: {last_parse_error}; response_preview={last_preview!r}" + ) from last_parse_error + if last_error is not None: + raise last_error + raise ValueError("Stage 3 PDF material extractor returned no response.") + + +def _generate_once(llm_client: Any, *, prompt: str, timeout: Optional[float]) -> Any: + try: + return llm_client.generate_content(prompt=prompt, timeout=timeout, max_tokens=PDF_MATERIAL_MAX_TOKENS) + except TypeError: + try: + return llm_client.generate_content(prompt=prompt, timeout=timeout) + except TypeError: + # Some local test/fallback clients do not accept a timeout kwarg. + return llm_client.generate_content(prompt=prompt) diff --git a/generation_pipeline/stage4.py b/generation_pipeline/stage4.py new file mode 100644 index 0000000..38fe2af --- /dev/null +++ b/generation_pipeline/stage4.py @@ -0,0 +1,2127 @@ +from __future__ import annotations + +import json +import re +import shutil +from copy import deepcopy +from datetime import date +from pathlib import Path +from typing import Any, Dict, List, Optional + +from generation_pipeline.generators.config_generator import ConfigGenerator +from generation_pipeline.utils.json_generator import JSONGenerator +from src.llm.helpers import generate_json + + +STUDY_ID_MAX_LEN = 80 + + +def _slugify(value: Any, *, fallback: str = "study") -> str: + text = str(value or "").strip().lower() + text = re.sub(r"[^a-z0-9]+", "_", text) + text = re.sub(r"_+", "_", text).strip("_") + if not text: + text = fallback + if text[0].isdigit(): + text = f"study_{text}" + return text[:STUDY_ID_MAX_LEN].rstrip("_") or fallback + + +def _default_study_id(payload: Dict[str, Any], json_path: Path) -> str: + candidate = payload.get("study_id") or payload.get("paper_id") + if not candidate: + stem = json_path.stem + stem = re.sub(r"_stage[0-9]+_(?:extraction|filter|patched)$", "", stem) + candidate = stem + slug = _slugify(candidate) + return slug if slug.startswith("study_") else f"study_{slug}" + + +def paper_local_stage4_dir(json_path: Path, study_id: Optional[str] = None) -> Path: + """Default local Stage 4 package directory for a paper output folder.""" + json_path = Path(json_path) + if study_id: + resolved_study_id = study_id + else: + payload = json.loads(json_path.read_text(encoding="utf-8")) + resolved_study_id = _default_study_id(payload, json_path) + return json_path.parent / "stage4" / resolved_study_id + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[1] + + +def _hub_index(metadata: Dict[str, Any], extraction: Dict[str, Any]) -> Dict[str, Any]: + description = metadata.get("description") or extraction.get("paper_abstract") or metadata.get("title") or "" + return { + "title": metadata.get("title") or extraction.get("paper_title") or "", + "authors": metadata.get("authors") or extraction.get("paper_authors") or [], + "year": metadata.get("year") or extraction.get("paper_year"), + "description": str(description).strip(), + } + + +def _hub_readme(metadata: Dict[str, Any], audit: Dict[str, Any]) -> str: + title = metadata.get("title") or metadata.get("id") or "Generated Study" + authors = metadata.get("authors") or [] + author_text = ", ".join(str(item) for item in authors) if authors else "Unknown" + year = metadata.get("year") or "Unknown year" + ready = audit.get("package_audit", {}).get("ready_for_simulation") + patch_required = audit.get("package_audit", {}).get("human_patch_required") + findings = metadata.get("findings") or [] + lines = [ + f"# {title}", + "", + f"**Authors:** {author_text}", + f"**Year:** {year}", + "", + str(metadata.get("description") or "").strip(), + "", + "## Generated Package", + "", + "This study package was generated by the Stage 4 HumanStudy-Bench-Hub compiler.", + f"- Ready for simulation: `{ready}`", + f"- Human patch required: `{patch_required}`", + f"- Findings: `{len(findings)}`", + "", + "Review `source/audit.json` and `source/materials/*.json` before trusting simulation results.", + "", + "## Files", + "", + "- `source/metadata.json`", + "- `source/specification.json`", + "- `source/ground_truth.json`", + "- `source/materials/*.json`", + "- `scripts/config.py`", + "- `scripts/evaluator.py`", + "- `scripts/study_utils.py`", + "", + ] + return "\n".join(lines) + + +def _standard_study_utils_text() -> str: + template_path = _repo_root() / "studies" / "study_001" / "scripts" / "study_utils.py" + if template_path.exists(): + return template_path.read_text(encoding="utf-8") + return '''""" +Standalone study utilities for generated HumanStudy-Bench-Hub studies. +""" +import json +import re +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any, Dict, List, Optional + + +class PromptBuilder: + def __init__(self, study_path: Path): + self.study_path = Path(study_path) + self.materials_path = self.study_path / "materials" + + def build_system_prompt(self, participant_profile: Dict[str, Any] = None) -> Optional[str]: + return None + + def build_trial_prompt(self, trial_data: Dict[str, Any]) -> str: + return f"Trial {trial_data.get('trial_number', '?')}: Please respond to the study material." + + def get_instructions(self) -> str: + return "Read the study material carefully and answer in the requested format." + + +class BaseStudyConfig(ABC): + prompt_builder_class = PromptBuilder + + def __init__(self, study_path: Path, specification: Dict[str, Any]): + self.study_path = Path(study_path) + self.source_path = self.study_path / "source" + self.specification = specification + self.study_id = specification["study_id"] + self.prompt_builder = self.prompt_builder_class(self.source_path) + + def load_material(self, sub_study_id: str) -> Dict[str, Any]: + return json.loads((self.source_path / "materials" / f"{sub_study_id}.json").read_text(encoding="utf-8")) + + def load_specification(self) -> Dict[str, Any]: + return json.loads((self.source_path / "specification.json").read_text(encoding="utf-8")) + + def load_ground_truth(self) -> Dict[str, Any]: + return json.loads((self.source_path / "ground_truth.json").read_text(encoding="utf-8")) + + def extract_numeric(self, text: str, default: float = 0.0) -> float: + match = re.search(r"(-?\\d+\\.?\\d*)", str(text or "")) + return float(match.group(1)) if match else default + + def extract_choice(self, text: str, options: List[str] = None) -> Optional[int]: + text_s = str(text or "").strip() + if options: + for i, opt in enumerate(options): + if opt.lower() in text_s.lower(): + return i + match = re.search(r"\\b([A-Z])\\b", text_s.upper()) + return ord(match.group(1)) - ord("A") if match else None + + @abstractmethod + def create_trials(self, n_trials: Optional[int] = None) -> List[Dict[str, Any]]: + raise NotImplementedError + + def get_prompt_builder(self) -> PromptBuilder: + return self.prompt_builder + + def get_instructions(self) -> str: + return self.prompt_builder.get_instructions() + + def aggregate_results(self, raw_results: Dict[str, Any]) -> Dict[str, Any]: + return raw_results +''' + + +def _write_hub_scripts(study_root: Path, study_id: str) -> List[Path]: + scripts_dir = study_root / "scripts" + scripts_dir.mkdir(parents=True, exist_ok=True) + study_utils_path = scripts_dir / "study_utils.py" + evaluator_path = scripts_dir / "evaluator.py" + study_utils_path.write_text(_standard_study_utils_text(), encoding="utf-8") + evaluator_path.write_text( + f'''""" +Placeholder evaluator for {study_id}. + +Fill this after running a Stage 5 smoke test and inspecting response samples. +""" +from typing import Any, Dict + + +def evaluate_study(results: Dict[str, Any]) -> Dict[str, Any]: + return {{ + "test_results": [], + "note": "Evaluator not implemented yet. Add study-specific statistical checks after Stage 5 response inspection.", + }} +''', + encoding="utf-8", + ) + return [study_utils_path, evaluator_path] + + +def _write_hub_root_files(study_root: Path, metadata: Dict[str, Any], extraction: Dict[str, Any], audit: Dict[str, Any]) -> List[Path]: + index_path = study_root / "index.json" + readme_path = study_root / "README.md" + _write_json(index_path, _hub_index(metadata, extraction)) + readme_path.write_text(_hub_readme(metadata, audit), encoding="utf-8") + return [index_path, readme_path] + + +def _first_text(*values: Any) -> str: + for value in values: + if isinstance(value, str) and value.strip(): + return value.strip() + if isinstance(value, dict): + content = value.get("content") + if isinstance(content, str) and content.strip(): + return content.strip() + return "" + + +def _metadata_value(payload: Dict[str, Any], key: str, default: Any = None) -> Any: + metadata = payload.get("paper_metadata") + if isinstance(metadata, dict) and key in metadata: + return metadata.get(key) + return payload.get(key, default) + + +def _effect_reported_statistics(effect: Dict[str, Any]) -> str: + stats = effect.get("stats") if isinstance(effect.get("stats"), dict) else {} + parts: List[str] = [] + for key in ("B", "b", "t", "f", "z", "chi_square", "D", "eta_square", "p_value", "sig"): + value = stats.get(key) + if value not in (None, "", [None, None]): + parts.append(f"{key}={value}") + ci = stats.get("ci") + if isinstance(ci, list) and any(item is not None for item in ci): + parts.append(f"ci={ci}") + return ", ".join(parts) + + +def _effect_to_statistical_result(effect: Dict[str, Any], index: int) -> Dict[str, Any]: + claim = effect.get("claim") or effect.get("IV") or effect.get("DV") or f"Effect {index}" + hypothesis = effect.get("statistical_hypothesis") or f"{effect.get('IV')} -> {effect.get('DV')}" + test_name = effect.get("effecttype") or "reported_effect" + return { + "claim": claim, + "hypothesis": hypothesis, + "test_name": test_name, + "statistical_hypothesis": hypothesis, + "reported_statistics": _effect_reported_statistics(effect), + "significance_level": 0.05, + "expected_direction": effect.get("direction") or "none", + "original_effect": effect, + } + + +def _effect_to_item(effect: Dict[str, Any], index: int) -> Dict[str, Any]: + fallback = f"Respond to outcome measure {index}." + question = _first_text( + effect.get("items"), + effect.get("materials"), + effect.get("materials_notes"), + effect.get("DV"), + fallback, + ) + return { + "id": f"item_{index}", + "question": question or fallback, + "options": [], + "type": "open_ended", + "metadata": { + "label": _slugify(effect.get("DV") or effect.get("IV") or f"item_{index}", fallback=f"item_{index}"), + "iv": effect.get("IV"), + "dv": effect.get("DV"), + "direction": effect.get("direction"), + }, + } + + +def _slot_status(effect: Dict[str, Any], slot: str) -> Optional[str]: + value = effect.get(slot) + if isinstance(value, dict): + status = value.get("status") + return str(status).strip() if status is not None else None + return None + + +_TOKEN_RE = re.compile(r"[a-z][a-z0-9_'-]{2,}", re.IGNORECASE) +_TOKEN_STOP = { + "the", "and", "for", "with", "that", "this", "from", "into", "about", + "study", "effect", "effects", "condition", "conditions", "participant", + "participants", "measure", "measures", "scale", "item", "items", "index", + "rating", "score", "perception", "perceptions", "variable", +} + + +def _tokens(*values: Any) -> set[str]: + out: set[str] = set() + for value in values: + for token in _TOKEN_RE.findall(str(value or "").lower()): + token = token.strip("'_-") + if len(token) >= 4 and token not in _TOKEN_STOP and not token.isdigit(): + out.add(token[:8]) + return out + + +def _score_item_for_effect(item: Dict[str, Any], effect: Dict[str, Any]) -> float: + dv_terms = _tokens(effect.get("DV")) + iv_terms = _tokens(effect.get("IV")) + item_terms = _tokens( + item.get("id"), + item.get("data_export_tag"), + item.get("question"), + item.get("label"), + item.get("metadata", {}).get("label") if isinstance(item.get("metadata"), dict) else None, + ) + if not item_terms: + return 0.0 + dv_score = len(dv_terms & item_terms) / len(dv_terms) if dv_terms else 0.0 + iv_score = len(iv_terms & item_terms) / len(iv_terms) if iv_terms else 0.0 + # Response items measure the outcome, so DV recall should dominate. IV + # overlap is only a weak tie-breaker for generic scales. + return (0.85 * dv_score) + (0.15 * iv_score) + + +def _select_response_item( + base_material: Optional[Dict[str, Any]], + effect: Dict[str, Any], + *, + candidate_items: Optional[List[Dict[str, Any]]] = None, +) -> Optional[Dict[str, Any]]: + if not isinstance(base_material, dict): + return None + if candidate_items is not None: + items = [item for item in candidate_items if isinstance(item, dict)] + else: + items = [item for item in base_material.get("items", []) or [] if isinstance(item, dict)] + if not items: + return None + scored = sorted( + ((item, _score_item_for_effect(item, effect)) for item in items), + key=lambda pair: pair[1], + reverse=True, + ) + return dict(scored[0][0]) + + +def _item_identifier(item: Dict[str, Any]) -> str: + return str(item.get("id") or item.get("data_export_tag") or "").strip() + + +def _item_lookup_keys(item: Dict[str, Any]) -> List[str]: + keys: List[str] = [] + for value in (item.get("id"), item.get("data_export_tag")): + text = str(value or "").strip() + if text: + keys.extend([text, text.lower()]) + return list(dict.fromkeys(keys)) + + +def _item_condition_suffix(item: Dict[str, Any]) -> str: + """Return compact condition/trial suffixes such as d1/r4 when available.""" + ident = _item_identifier(item).lower() + match = re.search(r"([a-z]\d+)$", ident) + return match.group(1) if match else "" + + +def _items_by_identifier(items: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + for item in items: + for key in _item_lookup_keys(item): + if key not in out: + out[key] = item + return out + + +def _candidate_item_ids(base_material: Dict[str, Any]) -> List[str]: + trace = base_material.get("source_trace") if isinstance(base_material.get("source_trace"), dict) else {} + focal = trace.get("focal_item_candidates") if isinstance(trace.get("focal_item_candidates"), dict) else {} + candidates = focal.get("candidate_items") if isinstance(focal.get("candidate_items"), list) else [] + return [str(item).strip() for item in candidates if str(item).strip()] + + +def _preserve_full_runtime_instrument(base_material: Dict[str, Any]) -> bool: + trace = base_material.get("source_trace") if isinstance(base_material.get("source_trace"), dict) else {} + return bool( + base_material.get("preserve_full_instrument_for_runtime") is True + or trace.get("preserve_full_instrument_for_runtime") is True + ) + + +def _expand_runtime_item_slice( + selected: Dict[str, Any], + all_items: List[Dict[str, Any]], + *, + max_group_items: int = 16, +) -> List[str]: + """Expand one selected item to a small same-trial/same-block item slice.""" + ident = _item_identifier(selected) + suffix = _item_condition_suffix(selected) + if suffix: + suffix_group = [ + _item_identifier(item) + for item in all_items + if _item_condition_suffix(item) == suffix and _item_identifier(item) + ] + if 2 <= len(suffix_group) <= max_group_items: + return suffix_group + + block = str(selected.get("block") or "").strip() + if block: + block_group = [ + _item_identifier(item) + for item in all_items + if str(item.get("block") or "").strip() == block and _item_identifier(item) + ] + if 2 <= len(block_group) <= max_group_items: + return block_group + return [ident] if ident else [] + + +def _canonical_item_id(item: Dict[str, Any]) -> str: + return _item_identifier(item) + + +def _resolve_item_ids(candidate_ids: List[str], by_id: Dict[str, Dict[str, Any]]) -> List[str]: + selected: List[str] = [] + for candidate in candidate_ids: + key = str(candidate or "").strip() + item = by_id.get(key) or by_id.get(key.lower()) + if not isinstance(item, dict): + continue + ident = _canonical_item_id(item) + if ident and ident not in selected: + selected.append(ident) + return selected + + +def _candidate_selection_too_broad( + selected_ids: List[str], + source_items: List[Dict[str, Any]], + *, + max_full_items: int, +) -> bool: + if not selected_ids: + return False + if len(selected_ids) <= max_full_items: + return False + if len(selected_ids) >= max(1, int(len(source_items) * 0.75)): + return True + return len(selected_ids) > max(max_full_items, 20) + + +def _item_preview(item: Dict[str, Any], *, max_question_chars: int = 700) -> Dict[str, Any]: + question = re.sub(r"\s+", " ", str(item.get("question") or "")).strip() + if len(question) > max_question_chars: + question = question[: max_question_chars - 3].rstrip() + "..." + options = item.get("options") if isinstance(item.get("options"), list) else [] + scale = item.get("scale") if isinstance(item.get("scale"), dict) else {} + response_format = item.get("response_format") if isinstance(item.get("response_format"), dict) else {} + return { + "id": _item_identifier(item), + "data_export_tag": str(item.get("data_export_tag") or "").strip() or None, + "block": str(item.get("block") or "").strip() or None, + "type": str(item.get("type") or "").strip() or None, + "question": question, + "options": options[:8], + "scale": scale or None, + "answer_type": response_format.get("answer_type") if response_format else None, + } + + +def _effect_preview(effect: Dict[str, Any], target: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + target = target if isinstance(target, dict) else {} + return { + "target_id": target.get("target_id"), + "effect_index": target.get("effect_index"), + "IV": effect.get("IV"), + "DV": effect.get("DV"), + "effecttype": effect.get("effecttype"), + "direction": effect.get("direction"), + "materials_notes": effect.get("materials_notes"), + "statistical_hypothesis": effect.get("statistical_hypothesis"), + } + + +def _selection_prompt_payload( + *, + base_material: Dict[str, Any], + effects: List[Dict[str, Any]], + targets: List[Dict[str, Any]], + source_items: List[Dict[str, Any]], + candidate_items: List[Dict[str, Any]], +) -> Dict[str, Any]: + aligned_effects: List[Dict[str, Any]] = [] + for index, effect in enumerate(effects): + target = targets[index] if index < len(targets) else None + aligned_effects.append(_effect_preview(effect, target)) + if not aligned_effects: + aligned_effects = [_effect_preview({}, target) for target in targets] + + trace = base_material.get("source_trace") if isinstance(base_material.get("source_trace"), dict) else {} + return { + "study_name": base_material.get("study_name") or base_material.get("sub_study_id"), + "source": trace.get("primary_source"), + "target_effects": aligned_effects, + "source_item_count": len(source_items), + "candidate_item_count": len(candidate_items), + "candidate_items": [_item_preview(item) for item in candidate_items], + } + + +def _select_item_ids_with_llm( + *, + llm_client: Any, + base_material: Dict[str, Any], + effects: List[Dict[str, Any]], + targets: List[Dict[str, Any]], + source_items: List[Dict[str, Any]], + candidate_items: List[Dict[str, Any]], + by_id: Dict[str, Dict[str, Any]], + timeout: Optional[float] = 45, +) -> tuple[List[str], Dict[str, Any]]: + payload = _selection_prompt_payload( + base_material=base_material, + effects=effects, + targets=targets, + source_items=source_items, + candidate_items=candidate_items, + ) + system = ( + "You select existing participant-facing survey items for reproducing " + "reported psychology experiment effects. You never invent new item IDs " + "or rewrite materials." + ) + prompt = f"""Choose the response items that must be shown to a simulated participant. + +Rules: +- Select ONLY item ids that appear in candidate_items. +- Cover every target effect. If the effects require different constructs, keep all required constructs. +- Prefer original experimental task items over attention checks, manipulation checks, demographics, or unrelated attitude scales. +- For matrix, bipolar, discounting, conjoint, or multi-trial tasks, select the whole task/block needed to produce analyzable responses, not a single row. +- If uncertain and the candidate list is reasonably small, keep more items rather than dropping needed task material. + +Input JSON: +{json.dumps(payload, indent=2, ensure_ascii=False)} + +Return JSON with exactly: +{{ + "selected_item_ids": ["existing_id"], + "confidence": "high|medium|low", + "reason": "short explanation" +}}""" + data = generate_json( + llm_client, + [{"role": "user", "content": prompt}], + system=system, + temperature=0, + max_tokens=1200, + retries=1, + timeout=timeout, + ) + selected_raw = data.get("selected_item_ids") if isinstance(data, dict) else [] + selected_ids = _resolve_item_ids([str(item) for item in selected_raw or []], by_id) + expanded: List[str] = [] + for item_id in selected_ids: + item = by_id.get(item_id) or by_id.get(item_id.lower()) + if not isinstance(item, dict): + continue + for expanded_id in _expand_runtime_item_slice(item, source_items): + if expanded_id and expanded_id not in expanded: + expanded.append(expanded_id) + report = { + "used": True, + "confidence": data.get("confidence") if isinstance(data, dict) else None, + "reason": data.get("reason") if isinstance(data, dict) else None, + "raw_selected_item_ids": [str(item) for item in selected_raw or []], + "selected_item_ids": expanded, + } + return expanded, report + + +def _select_runtime_items( + base_material: Optional[Dict[str, Any]], + effects: List[Dict[str, Any]], + targets: List[Dict[str, Any]], + *, + max_full_items: int = 12, + llm_client: Any = None, + llm_timeout: Optional[float] = 45, +) -> tuple[List[Dict[str, Any]], Dict[str, Any]]: + """Select a runtime-sized response item subset from full Stage 3 evidence. + + Stage 3 intentionally preserves complete source instruments. Runtime + simulation prompts should be smaller and target-bound, so Stage 4 keeps the + full material in provenance while emitting only the likely response slice. + """ + if not isinstance(base_material, dict): + return [], {} + raw_items = base_material.get("items") if isinstance(base_material.get("items"), list) else [] + source_items = [item for item in raw_items if isinstance(item, dict)] + by_id = _items_by_identifier(source_items) + selected_ids: List[str] = [] + selection_source = "target_effect_match" + llm_selection: Dict[str, Any] = {"used": False} + + if _preserve_full_runtime_instrument(base_material): + return [deepcopy(item) for item in source_items], { + "mode": "full_source_instrument_preserved", + "source_item_count": len(source_items), + "runtime_item_count": len(source_items), + } + + candidate_ids = _candidate_item_ids(base_material) + resolved_candidate_ids = _resolve_item_ids(candidate_ids, by_id) + broad_candidates = _candidate_selection_too_broad( + resolved_candidate_ids, + source_items, + max_full_items=max_full_items, + ) + if resolved_candidate_ids and not broad_candidates: + selected_ids = resolved_candidate_ids + selection_source = "stage3_focal_item_candidates" + + if not selected_ids and len(source_items) <= max_full_items: + return [deepcopy(item) for item in source_items], { + "mode": "full_source_instrument", + "source_item_count": len(source_items), + "runtime_item_count": len(source_items), + } + + candidate_pool_ids = resolved_candidate_ids if resolved_candidate_ids else [_canonical_item_id(item) for item in source_items] + candidate_pool_items = [ + by_id[item_id] + for item_id in candidate_pool_ids + if item_id in by_id and isinstance(by_id[item_id], dict) + ] + if not candidate_pool_items: + candidate_pool_items = source_items + + if not selected_ids and llm_client is not None and candidate_pool_items: + try: + llm_selected_ids, llm_selection = _select_item_ids_with_llm( + llm_client=llm_client, + base_material=base_material, + effects=effects, + targets=targets, + source_items=source_items, + candidate_items=candidate_pool_items, + by_id=by_id, + timeout=llm_timeout, + ) + if str(llm_selection.get("confidence") or "").lower() == "low": + llm_selection["ignored_due_to_low_confidence"] = True + elif llm_selected_ids: + selected_ids = llm_selected_ids + selection_source = "llm_item_selection" + except Exception as exc: # pragma: no cover - network/LLM dependent + llm_selection = { + "used": True, + "error": f"{type(exc).__name__}: {exc}", + } + + if not selected_ids and resolved_candidate_ids: + # Stage 3 focal candidates are advisory and may be broad, but a broad + # source-grounded slice is safer than collapsing a multi-task instrument + # to a single lexically similar item. + selected_ids = resolved_candidate_ids + selection_source = "stage3_focal_item_candidates_broad" + + if not selected_ids: + for effect in effects: + item = _select_response_item(base_material, effect, candidate_items=candidate_pool_items) + if not isinstance(item, dict): + continue + for item_id in _expand_runtime_item_slice(item, source_items): + if item_id and item_id in by_id and item_id not in selected_ids: + selected_ids.append(item_id) + + if not selected_ids: + return [deepcopy(item) for item in source_items], { + "mode": "full_source_instrument_no_target_match", + "source_item_count": len(source_items), + "runtime_item_count": len(source_items), + } + + selected = [deepcopy(by_id[item_id]) for item_id in selected_ids if item_id in by_id] + target_ids = [str(target.get("target_id")) for target in targets if target.get("target_id")] + return selected, { + "mode": "target_bound_subset", + "selection_source": selection_source, + "source_item_count": len(source_items), + "runtime_item_count": len(selected), + "selected_item_ids": [_item_identifier(item) for item in selected], + "target_ids": target_ids, + "llm_selection": llm_selection, + } + + +def _normalize_block_name(value: Any) -> str: + text = re.sub(r"[^a-z0-9]+", " ", str(value or "").lower()) + return re.sub(r"\s+", " ", text).strip() + + +def _block_family(value: Any) -> str: + text = _normalize_block_name(value) + text = re.sub(r"\bratings?\b", " ", text) + text = re.sub(r"\bpage\s+\d+\b", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def _select_runtime_instructions( + base_material: Optional[Dict[str, Any]], + runtime_items: List[Dict[str, Any]], + fallback_instructions: str, +) -> tuple[str, Dict[str, Any]]: + if not isinstance(base_material, dict): + return fallback_instructions, {} + raw_blocks = base_material.get("instruction_blocks") + instruction_blocks = [block for block in raw_blocks if isinstance(block, dict)] if isinstance(raw_blocks, list) else [] + if not instruction_blocks: + return fallback_instructions, { + "mode": "full_instructions_no_block_evidence", + "runtime_instruction_chars": len(fallback_instructions), + } + + selected_blocks = [ + str(item.get("block") or "").strip() + for item in runtime_items + if isinstance(item, dict) and str(item.get("block") or "").strip() + ] + selected_norms = {_normalize_block_name(block) for block in selected_blocks if _normalize_block_name(block)} + selected_families = {_block_family(block) for block in selected_blocks if _block_family(block)} + + def block_text(block: Dict[str, Any]) -> str: + return str(block.get("text") or "").strip() + + exact = [ + block for block in instruction_blocks + if _normalize_block_name(block.get("block")) in selected_norms and block_text(block) + ] + mode = "exact_item_block_match" + matched = exact + if not matched: + matched = [ + block for block in instruction_blocks + if _block_family(block.get("block")) in selected_families and block_text(block) + ] + mode = "block_family_match" + + if not matched: + return fallback_instructions, { + "mode": "full_instructions_no_matching_blocks", + "source_instruction_block_count": len(instruction_blocks), + "selected_item_blocks": sorted(set(selected_blocks)), + "runtime_instruction_chars": len(fallback_instructions), + } + + texts: List[str] = [] + seen_texts: set[str] = set() + seen_blocks: set[str] = set() + selected_instruction_blocks: List[str] = [] + collapsed_alternatives: Dict[str, int] = {} + for block in matched: + text = block_text(block) + block_name = str(block.get("block") or "") + block_key = _normalize_block_name(block_name) + if block_key in seen_blocks: + collapsed_alternatives[block_name] = collapsed_alternatives.get(block_name, 0) + 1 + continue + seen_blocks.add(block_key) + key = re.sub(r"\s+", " ", text.lower()) + if not text or key in seen_texts: + continue + seen_texts.add(key) + texts.append(text) + selected_instruction_blocks.append(block_name) + + instructions = "\n\n".join(texts).strip() or fallback_instructions + return instructions, { + "mode": mode, + "source_instruction_block_count": len(instruction_blocks), + "runtime_instruction_block_count": len(texts), + "selected_item_blocks": sorted(set(selected_blocks)), + "selected_instruction_blocks": sorted(set(selected_instruction_blocks)), + "collapsed_instruction_alternatives": collapsed_alternatives, + "source_instruction_chars": len(fallback_instructions), + "runtime_instruction_chars": len(instructions), + } + + +def _response_format_from_scale(scale: Dict[str, Any]) -> Dict[str, Any]: + if not isinstance(scale, dict): + return {} + lo = scale.get("min") + hi = scale.get("max") + out: Dict[str, Any] = {} + if lo is not None and hi is not None: + out["scale"] = f"{lo}-{hi} Likert" + anchors = scale.get("anchors") if isinstance(scale.get("anchors"), dict) else {} + labelled = { + str(key): str(value) + for key, value in anchors.items() + if str(value).strip() and str(value).strip() != str(key).strip() + } + if labelled: + out["anchors"] = labelled + return out + + +def _runtime_warnings(base_readiness: Dict[str, Any], item_selection: Dict[str, Any]) -> List[str]: + warnings = list(base_readiness.get("warnings", []) if isinstance(base_readiness.get("warnings"), list) else []) + if item_selection.get("selection_source") != "llm_item_selection": + return warnings + filtered = [ + warning + for warning in warnings + if not ( + isinstance(warning, str) + and "focal response item" in warning.lower() + and "retained" in warning.lower() + ) + ] + filtered.append("Stage 4 LLM selected the runtime item subset; inspect source_trace.runtime_item_selection for coverage.") + return list(dict.fromkeys(filtered)) + + +def _effect_response_contract( + base_material: Optional[Dict[str, Any]], + effect: Dict[str, Any], + *, + candidate_items: Optional[List[Dict[str, Any]]] = None, +) -> tuple[Dict[str, Any], List[str], str, Dict[str, Any]]: + item = _select_response_item(base_material, effect, candidate_items=candidate_items) + if isinstance(item, dict): + response_format = item.get("response_format") if isinstance(item.get("response_format"), dict) else {} + options = item.get("options") if isinstance(item.get("options"), list) else [] + scale = item.get("scale") if isinstance(item.get("scale"), dict) else {} + item_type = str(item.get("type") or "").strip() or "open_ended" + if not response_format and scale: + response_format = _response_format_from_scale(scale) + if not response_format and options: + response_format = {"options": options} + return response_format, options, item_type, scale + + schema = {} + if isinstance(base_material, dict) and isinstance(base_material.get("response_schema"), dict): + schema = dict(base_material["response_schema"]) + if schema.get("scale_min") is not None and schema.get("scale_max") is not None: + scale = { + "min": schema.get("scale_min"), + "max": schema.get("scale_max"), + "anchors": schema.get("anchors") if isinstance(schema.get("anchors"), dict) else {}, + } + return _response_format_from_scale(scale), [], str(schema.get("answer_type") or "scale"), scale + if isinstance(schema.get("options"), list): + return {"options": schema["options"]}, schema["options"], str(schema.get("answer_type") or "multiple_choice"), {} + return {}, [], "open_ended", {} + + +def _source_item_block(effect: Dict[str, Any], source_text: Optional[str]) -> Optional[List[str]]: + if not isinstance(source_text, str) or not source_text.strip(): + return None + target_terms = _tokens(effect.get("DV"), _first_text(effect.get("items"))) + if not target_terms: + return None + lines = [line.strip() for line in source_text.splitlines()] + best_index = None + best_score = 0.0 + for index, line in enumerate(lines): + line_terms = _tokens(line) + if not line_terms: + continue + score = len(target_terms & line_terms) / len(target_terms | line_terms) + if score > best_score: + best_score = score + best_index = index + if best_index is None or best_score <= 0: + return None + + block: List[str] = [lines[best_index]] + saw_item = False + for line in lines[best_index + 1:]: + if not line: + if saw_item: + break + continue + if block and not line.startswith(("•", "-", "*")) and saw_item: + break + block.append(line) + if line.startswith(("•", "-", "*")) or len(block) > 2: + saw_item = True + item_lines = [line for line in block if line.startswith(("•", "-", "*"))] + if len(item_lines) < 2: + return None + return block + + +def _effect_prompt_material( + *, + sub_id: str, + study_name: str, + effect: Dict[str, Any], + effect_number: int, + base_material: Optional[Dict[str, Any]], + target_id: Optional[str] = None, + source_text: Optional[str] = None, +) -> Dict[str, Any]: + material_id = target_id or f"{sub_id}__effect_{effect_number:02d}" + base_material = base_material if isinstance(base_material, dict) else None + base_readiness = base_material.get("readiness", {}) if isinstance(base_material, dict) else {} + source_block = _source_item_block(effect, source_text) if base_material is None else None + stimulus = str((base_material or {}).get("instructions") or "").strip() + if source_block: + stimulus = "\n".join(source_block) + item = _select_response_item(base_material, effect) + response_format, options, item_type, scale = _effect_response_contract(base_material, effect) + question = "" + items: List[Dict[str, Any]] = [] + if source_block: + question = str(effect.get("DV") or f"Outcome measure {effect_number}").strip() + bullet_lines = [line.lstrip("•-* ").strip() for line in source_block if line.startswith(("•", "-", "*"))] + items = [ + { + "id": f"{material_id}_item_{idx}", + "question": line, + "type": "open_ended", + "options": [], + } + for idx, line in enumerate(bullet_lines, start=1) + ] + elif isinstance(item, dict): + question = str(item.get("question") or "").strip() + item = dict(item) + item.setdefault("id", f"{material_id}_item_1") + item.setdefault("question", question) + item.setdefault("type", item_type) + if options and not item.get("options"): + item["options"] = options + items = [item] + elif response_format: + question = str(effect.get("DV") or f"Outcome measure {effect_number}").strip() + items = [{ + "id": f"{material_id}_item_1", + "question": question, + "type": item_type, + "options": options, + }] + + blocking = list(base_readiness.get("blocking_issues", []) if isinstance(base_readiness, dict) else []) + warnings = list(base_readiness.get("warnings", []) if isinstance(base_readiness, dict) else []) + if base_material is None and source_block is None: + blocking.append("missing_experiment_materials") + if not stimulus: + blocking.append("missing_instructions") + if not question: + blocking.append("missing_response_question") + if not items: + blocking.append("missing_response_items") + if item_type in {"multiple_choice", "likert", "scale", "matrix"} and not (options or response_format.get("scale")): + blocking.append("missing_response_options") + if base_material is None and source_block is None and _slot_status(effect, "items") == "cited_scale": + blocking = ["cited_scale_without_verbatim_items"] + blocking = sorted(dict.fromkeys(blocking)) + ready = bool(stimulus and question and items) and not blocking and base_readiness.get("ready", True) is not False + + return { + "sub_study_id": sub_id, + "material_id": material_id, + "target_id": material_id, + "study_name": study_name, + "stimulus": stimulus, + "instructions": stimulus, + "question": question, + "response_format": response_format, + "items": items, + "readiness": { + "ready": ready, + "blocking_issues": blocking, + "warnings": warnings, + }, + "source_trace": { + "primary_source": "stage3_simulation_target", + "base_source_trace": (base_material or {}).get("source_trace", {}), + }, + "metadata": { + "effect_index": effect_number - 1, + "iv": effect.get("IV"), + "dv": effect.get("DV"), + "effecttype": effect.get("effecttype"), + "direction": effect.get("direction"), + "scale": scale, + }, + } + + +def _study_prompt_material( + *, + sub_id: str, + study_name: str, + effects: List[Dict[str, Any]], + targets: List[Dict[str, Any]], + base_material: Optional[Dict[str, Any]], + selection_llm_client: Any = None, +) -> Dict[str, Any]: + """Build one participant-facing material package for a sub-study.""" + base_material = base_material if isinstance(base_material, dict) else None + representative_effect = effects[0] if effects else {} + target_ids = [str(target.get("target_id")) for target in targets if target.get("target_id")] + + if base_material is None: + material = _effect_prompt_material( + sub_id=sub_id, + study_name=study_name, + effect=representative_effect, + effect_number=1, + base_material=None, + target_id=sub_id, + ) + material["target_ids"] = target_ids + material["source_trace"]["targets"] = targets + return material + + base_readiness = base_material.get("readiness", {}) if isinstance(base_material.get("readiness"), dict) else {} + instructions = _first_text( + base_material.get("instructions"), + base_material.get("stimulus"), + base_material.get("scenario"), + base_material.get("vignette"), + ) + items, item_selection = _select_runtime_items( + base_material, + effects, + targets, + llm_client=selection_llm_client, + ) + instructions, instruction_selection = _select_runtime_instructions(base_material, items, instructions) + response_format, _, _, scale = _effect_response_contract( + base_material, + representative_effect, + candidate_items=items, + ) + if not response_format: + if isinstance(base_material.get("response_format"), dict): + response_format = deepcopy(base_material["response_format"]) + elif isinstance(base_material.get("response_schema"), dict): + response_format = deepcopy(base_material["response_schema"]) + + question = _first_text( + base_material.get("question"), + *(item.get("question") for item in items if isinstance(item, dict)), + "Complete the following study questions.", + ) + + source_trace = deepcopy(base_material.get("source_trace", {})) if isinstance(base_material.get("source_trace"), dict) else {} + source_trace.setdefault("primary_source", "stage3_study_material") + source_trace["targets"] = targets + if item_selection: + source_trace["runtime_item_selection"] = item_selection + if instruction_selection: + source_trace["runtime_instruction_selection"] = instruction_selection + + material = { + "sub_study_id": sub_id, + "material_id": sub_id, + "study_name": study_name, + "target_ids": target_ids, + "stimulus": instructions, + "instructions": instructions, + "question": question, + "response_format": response_format, + "items": items, + "conditions": deepcopy(base_material.get("conditions", [])) if isinstance(base_material.get("conditions"), list) else [], + "readiness": { + "ready": False, + "blocking_issues": [], + "warnings": _runtime_warnings(base_readiness, item_selection), + }, + "source_trace": source_trace, + "metadata": { + "target_ids": target_ids, + "primary_target_id": target_ids[0] if target_ids else None, + "scale": scale, + "source_item_count": item_selection.get("source_item_count") if item_selection else len(items), + "runtime_item_count": len(items), + "source_instruction_chars": instruction_selection.get("source_instruction_chars") if instruction_selection else len(instructions), + "runtime_instruction_chars": len(instructions), + }, + } + if isinstance(base_material.get("selection"), dict): + material["selection"] = deepcopy(base_material["selection"]) + + blocking = list(base_readiness.get("blocking_issues", []) if isinstance(base_readiness.get("blocking_issues"), list) else []) + blocking.extend(_runtime_material_issues(material)) + blocking = sorted(dict.fromkeys(blocking)) + material["readiness"]["blocking_issues"] = blocking + material["readiness"]["ready"] = base_readiness.get("ready", True) is not False and not blocking + return material + + +def _target_material_id(target: Dict[str, Any], fallback: str) -> str: + return str(target.get("sub_study_id") or target.get("material_id") or target.get("target_id") or fallback) + + +def normalize_to_human_extraction(payload: Dict[str, Any]) -> Dict[str, Any]: + """ + Convert either HumanStudy-Bench extraction JSON or ai-ethics Stage 2 JSON into + the HumanStudy-Bench extraction shape consumed by the study generators. + """ + if isinstance(payload.get("studies"), list): + normalized = dict(payload) + normalized.setdefault("paper_authors", _metadata_value(payload, "authors", [])) + normalized.setdefault("paper_year", _metadata_value(payload, "year")) + normalized.setdefault("paper_abstract", _metadata_value(payload, "abstract", "")) + normalized.setdefault("study_materials", payload.get("study_materials", {})) + normalized.setdefault("simulation_targets", payload.get("simulation_targets", [])) + return normalized + + eligible = payload.get("eligible_studies") + if not isinstance(eligible, list): + raise ValueError( + "Stage 4 input must contain HumanStudy-Bench 'studies' or ai-ethics 'eligible_studies'." + ) + + stage3_materials = payload.get("study_materials") + stage3_materials = stage3_materials if isinstance(stage3_materials, dict) else {} + + studies: List[Dict[str, Any]] = [] + filtered_stage3_materials: Dict[str, Any] = {} + for study_index, study in enumerate(eligible, start=1): + if not isinstance(study, dict): + continue + study_name = study.get("study") or study.get("name") or f"Study {study_index}" + sub_id = _slugify(study_name, fallback=f"study_{study_index}") + effects = [effect for effect in study.get("effects", []) if isinstance(effect, dict)] + sample = study.get("sample") if isinstance(study.get("sample"), dict) else {} + + has_stage3_material = sub_id in stage3_materials and isinstance(stage3_materials.get(sub_id), dict) + material = stage3_materials.get(sub_id) + material = material if isinstance(material, dict) else {} + material_items = material.get("items") if isinstance(material.get("items"), list) else None + if has_stage3_material: + items = material_items if material_items is not None else [] + else: + items = [_effect_to_item(effect, idx) for idx, effect in enumerate(effects, start=1)] + statistics = [ + _effect_to_statistical_result(effect, idx) + for idx, effect in enumerate(effects, start=1) + ] + instructions = material.get("instructions") if isinstance(material.get("instructions"), str) else "" + if not has_stage3_material and not instructions.strip(): + instructions_parts = [ + study.get("eligibility_rationale"), + *(_first_text(effect.get("manipulation"), effect.get("materials"), effect.get("materials_notes")) for effect in effects), + ] + instructions = "\n\n".join(part for part in instructions_parts if isinstance(part, str) and part.strip()) + + selection = study.get("selection") if isinstance(study.get("selection"), dict) else {} + if selection.get("keep") is False: + continue + material_selection = material.get("selection") if isinstance(material.get("selection"), dict) else {} + if material_selection.get("keep") is False: + continue + if has_stage3_material: + filtered_stage3_materials[sub_id] = material + + studies.append( + { + "study_id": study_name, + "study_name": study_name, + "phenomenon": payload.get("paper_title") or study_name, + "sample": sample, + "effects": effects, + "sub_studies": [ + { + "sub_study_id": sub_id, + "name": study_name, + "content": instructions, + "instructions": instructions, + "items": items, + "human_data": { + "sample": sample, + "statistical_results": statistics, + }, + } + ], + } + ) + + return { + "paper_id": payload.get("paper_id"), + "paper_title": payload.get("paper_title") or _metadata_value(payload, "title", ""), + "paper_authors": _metadata_value(payload, "authors", []), + "paper_year": _metadata_value(payload, "year"), + "paper_abstract": _metadata_value(payload, "abstract", ""), + "source_schema": "ai_ethics_stage2", + "studies": studies, + "study_materials": filtered_stage3_materials or stage3_materials, + "simulation_targets": payload.get("simulation_targets", []), + } + + +def _deterministic_metadata(extraction: Dict[str, Any], study_id: str) -> Dict[str, Any]: + targets = [target for target in extraction.get("simulation_targets", []) or [] if isinstance(target, dict)] + scenarios = [] + seen_scenarios = set() + findings = [] + if targets: + for index, target in enumerate(targets, start=1): + effect, stat, _, _ = _target_effect(extraction, target) + target_id = str(target.get("target_id") or f"target_{index}") + material_id = _target_material_id(target, f"target_{index}") + if material_id not in seen_scenarios: + scenarios.append(material_id) + seen_scenarios.add(material_id) + findings.append( + { + "finding_id": f"F{index}", + "target_id": target_id, + "material_id": material_id, + "sub_study_id": target.get("sub_study_id"), + "main_hypothesis": ( + stat.get("claim") + or stat.get("hypothesis") + or effect.get("claim") + or f"{effect.get('IV')} -> {effect.get('DV')}" + or f"Finding {index}" + ), + "weight": 1.0, + "tests": [{"test_name": stat.get("test_name") or effect.get("effecttype") or "reported_effect", "weight": 1.0}], + } + ) + else: + for study in extraction.get("studies", []): + for sub_study in study.get("sub_studies", []): + sub_id = sub_study.get("sub_study_id") + if sub_id: + scenarios.append(sub_id) + for idx, stat in enumerate(sub_study.get("human_data", {}).get("statistical_results", []), start=1): + findings.append( + { + "finding_id": f"F{len(findings) + 1}", + "main_hypothesis": stat.get("claim") or stat.get("hypothesis") or f"Finding {idx}", + "weight": 1.0, + "tests": [{"test_name": stat.get("test_name") or "reported_effect", "weight": 1.0}], + } + ) + return { + "id": study_id, + "title": extraction.get("paper_title", ""), + "authors": extraction.get("paper_authors", []), + "year": extraction.get("paper_year"), + "domain": None, + "subdomain": None, + "keywords": [], + "difficulty": "medium", + "description": extraction.get("paper_abstract", ""), + "scenarios": scenarios, + "findings": findings, + } + + +def _deterministic_specification(extraction: Dict[str, Any], study_id: str) -> Dict[str, Any]: + by_sub_study: Dict[str, Dict[str, Any]] = {} + total_n = 0 + population = None + recruitment_source = None + for study in extraction.get("studies", []): + sample = study.get("sample") if isinstance(study.get("sample"), dict) else {} + for sub_study in study.get("sub_studies", []): + sub_id = sub_study.get("sub_study_id") + if not sub_id: + continue + sub_sample = sub_study.get("human_data", {}).get("sample") or sample + n = sub_sample.get("analyzed_n") or sub_sample.get("total_n") or sub_sample.get("n") or 0 + try: + n_int = int(n or 0) + except (TypeError, ValueError): + n_int = 0 + total_n += n_int + population = population or sub_sample.get("population") or sub_sample.get("platform") + recruitment_source = recruitment_source or sub_sample.get("recruitment_source") or sub_sample.get("platform") + by_sub_study[sub_id] = { + "n": n_int, + "population": sub_sample.get("population") or sub_sample.get("platform"), + "recruitment_source": sub_sample.get("recruitment_source") or sub_sample.get("platform"), + } + + return { + "study_id": study_id, + "title": extraction.get("paper_title", ""), + "participants": { + "n": total_n, + "population": population, + "recruitment_source": recruitment_source, + "demographics": {}, + "by_sub_study": by_sub_study, + }, + "design": {"type": None, "factors": []}, + "procedure": {"steps": []}, + } + + +def _target_effect( + extraction: Dict[str, Any], + target: Dict[str, Any], +) -> tuple[Dict[str, Any], Dict[str, Any], Dict[str, Any], Dict[str, Any]]: + sub_id = str(target.get("sub_study_id") or "") + try: + effect_index = int(target.get("effect_index") or 0) + except (TypeError, ValueError): + effect_index = 0 + for study in extraction.get("studies", []) or []: + if not isinstance(study, dict): + continue + for sub_study in study.get("sub_studies", []) or []: + if not isinstance(sub_study, dict) or sub_study.get("sub_study_id") != sub_id: + continue + stats = sub_study.get("human_data", {}).get("statistical_results", []) + stat = stats[effect_index] if isinstance(stats, list) and effect_index < len(stats) and isinstance(stats[effect_index], dict) else {} + effect = stat.get("original_effect") if isinstance(stat.get("original_effect"), dict) else {} + if not effect: + effects = study.get("effects", []) if isinstance(study.get("effects"), list) else [] + effect = effects[effect_index] if effect_index < len(effects) and isinstance(effects[effect_index], dict) else {} + return effect, stat, study, sub_study + return {}, {}, {}, {} + + +def _deterministic_ground_truth(extraction: Dict[str, Any], study_id: str) -> Dict[str, Any]: + targets = [target for target in extraction.get("simulation_targets", []) or [] if isinstance(target, dict)] + studies = [] + finding_counter = 1 + if targets: + by_study: Dict[str, Dict[str, Any]] = {} + for target in targets: + effect, stat, study, _ = _target_effect(extraction, target) + study_name = str(target.get("study_name") or study.get("study_name") or study.get("study_id") or "Study") + bucket = by_study.setdefault( + study_name, + {"study_id": study.get("study_id") or study_name, "study_name": study_name, "findings": []}, + ) + data_key = _slugify(effect.get("DV") or stat.get("claim") or f"finding_{finding_counter}") + target_id = str(target.get("target_id") or f"target_{finding_counter}") + material_id = _target_material_id(target, f"target_{finding_counter}") + bucket["findings"].append( + { + "finding_id": f"F{finding_counter}", + "target_id": target_id, + "sub_study_id": target.get("sub_study_id"), + "material_id": material_id, + "main_hypothesis": ( + stat.get("claim") + or stat.get("hypothesis") + or effect.get("claim") + or f"{effect.get('IV')} -> {effect.get('DV')}" + or f"Finding {finding_counter}" + ), + "statistical_tests": [ + { + "test_name": stat.get("test_name") or effect.get("effecttype") or "reported_effect", + "statistical_hypothesis": stat.get("statistical_hypothesis") or stat.get("hypothesis"), + "reported_statistics": stat.get("reported_statistics") or _effect_reported_statistics(effect), + "significance_level": stat.get("significance_level", 0.05), + "expected_direction": stat.get("expected_direction") or effect.get("direction") or "none", + } + ], + "original_data_points": { + "description": effect.get("materials_notes") or "", + "data": { + data_key: { + "mean_group1": effect.get("mean_group1"), + "sd_group1": effect.get("sd_group1"), + "mean_group2": effect.get("mean_group2"), + "sd_group2": effect.get("sd_group2"), + "n": effect.get("size") or effect.get("analysis_n"), + } + }, + }, + } + ) + finding_counter += 1 + studies = list(by_study.values()) + else: + for study in extraction.get("studies", []): + findings = [] + for sub_study in study.get("sub_studies", []): + for stat in sub_study.get("human_data", {}).get("statistical_results", []): + original = stat.get("original_effect") if isinstance(stat.get("original_effect"), dict) else {} + data_key = _slugify(original.get("DV") or stat.get("claim") or f"finding_{finding_counter}") + findings.append( + { + "finding_id": f"F{finding_counter}", + "main_hypothesis": stat.get("claim") or stat.get("hypothesis") or f"Finding {finding_counter}", + "statistical_tests": [ + { + "test_name": stat.get("test_name") or "reported_effect", + "statistical_hypothesis": stat.get("statistical_hypothesis") or stat.get("hypothesis"), + "reported_statistics": stat.get("reported_statistics", ""), + "significance_level": stat.get("significance_level", 0.05), + "expected_direction": stat.get("expected_direction", "none"), + } + ], + "original_data_points": { + "description": original.get("materials_notes") or "", + "data": { + data_key: { + "mean_group1": original.get("mean_group1"), + "sd_group1": original.get("sd_group1"), + "mean_group2": original.get("mean_group2"), + "sd_group2": original.get("sd_group2"), + "n": original.get("size") or original.get("analysis_n"), + } + }, + }, + } + ) + finding_counter += 1 + studies.append( + { + "study_id": study.get("study_id") or study.get("study_name"), + "study_name": study.get("study_name") or study.get("study_id"), + "findings": findings, + } + ) + return { + "study_id": study_id, + "title": extraction.get("paper_title", ""), + "authors": extraction.get("paper_authors", []), + "year": extraction.get("paper_year"), + "studies": studies, + "overall_original_results": {"description": "", "data_tables": [], "all_raw_data": {}}, + } + + +def _deterministic_materials( + extraction: Dict[str, Any], + study_dir: Path, + *, + selection_llm_client: Any = None, +) -> List[Path]: + materials_dir = study_dir / "materials" + materials_dir.mkdir(parents=True, exist_ok=True) + for old_json in materials_dir.glob("*.json"): + old_json.unlink() + paths: List[Path] = [] + targets = [target for target in extraction.get("simulation_targets", []) or [] if isinstance(target, dict)] + source_materials = extraction.get("study_materials") if isinstance(extraction.get("study_materials"), dict) else {} + if targets: + by_sub_study: Dict[str, List[Dict[str, Any]]] = {} + for rank, target in enumerate(targets, start=1): + sub_id = str(target.get("sub_study_id") or "") + if not sub_id: + continue + target.setdefault("target_id", f"{sub_id}__effect_{rank:02d}") + by_sub_study.setdefault(sub_id, []).append(target) + + for sub_id, sub_targets in by_sub_study.items(): + effects: List[Dict[str, Any]] = [] + study_name = sub_id + for target in sub_targets: + effect, _, study, _ = _target_effect(extraction, target) + if effect: + effects.append(effect) + study_name = str(target.get("study_name") or study.get("study_name") or study.get("study_id") or study_name) + base_material = source_materials.get(sub_id) if isinstance(source_materials.get(sub_id), dict) else None + material = _study_prompt_material( + sub_id=sub_id, + study_name=study_name, + effects=effects, + targets=sub_targets, + base_material=base_material, + selection_llm_client=selection_llm_client, + ) + path = materials_dir / f"{sub_id}.json" + path.write_text(json.dumps(material, indent=2, ensure_ascii=False), encoding="utf-8") + paths.append(path) + return paths + + for study in extraction.get("studies", []): + for sub_study in study.get("sub_studies", []): + sub_id = sub_study.get("sub_study_id") + if not sub_id: + continue + source_material = source_materials.get(sub_id) if isinstance(source_materials.get(sub_id), dict) else None + items = sub_study.get("items") if isinstance(sub_study.get("items"), list) else [] + instructions = sub_study.get("instructions") or sub_study.get("content") or "" + material = { + "sub_study_id": sub_id, + "instructions": instructions, + "items": items, + "readiness": { + "ready": bool(str(instructions).strip() and items), + "blocking_issues": [], + "warnings": [], + }, + "source_trace": {"primary_source": "stage4_legacy_fallback"}, + } + if source_material is not None: + for key in ("instructions", "items", "readiness", "source_trace", "response_schema", "conditions"): + if key in source_material: + material[key] = source_material[key] + path = materials_dir / f"{sub_id}.json" + path.write_text(json.dumps(material, indent=2, ensure_ascii=False), encoding="utf-8") + paths.append(path) + return paths + + +def _write_json(path: Path, payload: Dict[str, Any]) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") + return path + + +def _runtime_material_issues(material: Dict[str, Any]) -> List[str]: + issues: List[str] = [] + instructions = material.get("instructions") or material.get("stimulus") + items = material.get("items") + if not isinstance(instructions, str) or not instructions.strip(): + issues.append("missing_instructions") + if material.get("material_id") and not str(material.get("question") or "").strip(): + issues.append("missing_response_question") + if not isinstance(items, list): + issues.append("items_not_list") + return issues + if not items: + issues.append("missing_response_items") + return issues + for idx, item in enumerate(items, start=1): + if not isinstance(item, dict): + issues.append(f"item_{idx}_not_object") + continue + question = item.get("question") + if not isinstance(question, str) or not question.strip(): + issues.append(f"item_{idx}_missing_question") + item_type = str(item.get("type") or "").strip().lower() + options = item.get("options") + response_format = material.get("response_format") if isinstance(material.get("response_format"), dict) else {} + if item_type in {"multiple_choice", "likert", "scale", "matrix"} and not (options or response_format.get("scale")): + issues.append(f"item_{idx}_missing_options") + return issues + + +def _read_json_for_audit(path: Path) -> tuple[Optional[Dict[str, Any]], Optional[str]]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return None, "missing" + except json.JSONDecodeError as exc: + return None, f"invalid_json:{exc.msg}" + except Exception as exc: # pragma: no cover - defensive filesystem guard + return None, f"{type(exc).__name__}:{exc}" + if not isinstance(data, dict): + return None, "json_not_object" + return data, None + + +def _iter_ground_truth_findings(ground_truth: Dict[str, Any]) -> List[Dict[str, Any]]: + findings: List[Dict[str, Any]] = [] + for study in ground_truth.get("studies", []) or []: + if not isinstance(study, dict): + continue + for finding in study.get("findings", []) or []: + if isinstance(finding, dict): + findings.append(finding) + return findings + + +def _package_file_audit(study_dir: Path) -> tuple[Dict[str, Any], Dict[str, Dict[str, Any]]]: + required = {} + loaded: Dict[str, Dict[str, Any]] = {} + for name in ("metadata.json", "specification.json", "ground_truth.json"): + path = study_dir / name + data, error = _read_json_for_audit(path) + required[name] = { + "exists": path.exists(), + "valid_json_object": error is None, + "error": error, + } + if data is not None: + loaded[name] = data + + materials_dir = study_dir / "materials" + material_files = sorted(materials_dir.glob("*.json")) if materials_dir.exists() else [] + required["materials/"] = { + "exists": materials_dir.exists(), + "json_files": [path.name for path in material_files], + "count": len(material_files), + } + return required, loaded + + +def _package_reference_audit( + *, + metadata: Dict[str, Any], + specification: Dict[str, Any], + ground_truth: Dict[str, Any], + material_ids: set[str], + material_by_id: Dict[str, Dict[str, Any]], +) -> Dict[str, Any]: + findings = _iter_ground_truth_findings(ground_truth) + material_refs = sorted( + { + str(finding.get("material_id") or finding.get("sub_study_id")) + for finding in findings + if finding.get("material_id") or finding.get("sub_study_id") + } + ) + spec_sub_studies = set() + participants = specification.get("participants") if isinstance(specification.get("participants"), dict) else {} + by_sub = participants.get("by_sub_study") if isinstance(participants.get("by_sub_study"), dict) else {} + spec_sub_studies.update(str(key) for key in by_sub.keys()) + scenarios = { + str(item) + for item in metadata.get("scenarios", []) or [] + if str(item).strip() + } + missing_material_refs = sorted(ref for ref in material_refs if ref not in material_ids) + unreferenced_materials = sorted(material_ids - set(material_refs)) if material_refs else sorted(material_ids - scenarios) + missing_spec_refs = sorted(ref for ref in material_refs if spec_sub_studies and ref not in spec_sub_studies) + + target_refs_by_material: Dict[str, List[str]] = {} + for finding in findings: + material_ref_raw = finding.get("material_id") or finding.get("sub_study_id") + target_id_raw = finding.get("target_id") + if material_ref_raw in (None, "") or target_id_raw in (None, ""): + continue + material_ref = str(material_ref_raw).strip() + target_id = str(target_id_raw).strip() + if not material_ref or not target_id: + continue + target_refs_by_material.setdefault(material_ref, []).append(target_id) + target_refs_by_material = { + material_ref: sorted(dict.fromkeys(target_ids)) + for material_ref, target_ids in sorted(target_refs_by_material.items()) + } + + material_target_links: Dict[str, List[str]] = {} + missing_material_target_links: Dict[str, List[str]] = {} + for material_ref, target_ids in target_refs_by_material.items(): + material = material_by_id.get(material_ref) + if not isinstance(material, dict): + continue + raw_material_target_ids = material.get("target_ids") + if isinstance(raw_material_target_ids, list): + material_target_ids = { + str(target_id).strip() + for target_id in raw_material_target_ids + if target_id not in (None, "") and str(target_id).strip() + } + else: + material_target_ids = set() + material_target_links[material_ref] = sorted(material_target_ids) + missing_targets = sorted(set(target_ids) - material_target_ids) + if missing_targets: + missing_material_target_links[material_ref] = missing_targets + + return { + "ground_truth_findings": len(findings), + "ground_truth_material_refs": material_refs, + "ground_truth_target_refs_by_material": target_refs_by_material, + "metadata_scenarios": sorted(scenarios), + "specification_sub_studies": sorted(spec_sub_studies), + "missing_material_refs": missing_material_refs, + "unreferenced_materials": unreferenced_materials, + "missing_specification_refs": missing_spec_refs, + "material_target_links": material_target_links, + "missing_material_target_links": missing_material_target_links, + } + + +def _material_file_audit(material_by_id: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + by_material: Dict[str, Dict[str, Any]] = {} + issue_counts: Dict[str, int] = {} + for material_id, material in material_by_id.items(): + issues = _runtime_material_issues(material) + for issue in issues: + issue_counts[issue] = issue_counts.get(issue, 0) + 1 + readiness = material.get("readiness") if isinstance(material.get("readiness"), dict) else {} + by_material[material_id] = { + "runtime_issues": issues, + "readiness": readiness, + "instruction_chars": len(str(material.get("instructions") or material.get("stimulus") or "")), + "items": len(material.get("items") or []) if isinstance(material.get("items"), list) else 0, + } + return { + "materials": by_material, + "issue_counts": issue_counts, + } + + +def _build_package_audit( + *, + study_id: str, + study_dir: Path, + materials_audit: Dict[str, Any], + material_by_id: Dict[str, Dict[str, Any]], +) -> Dict[str, Any]: + required_files, loaded = _package_file_audit(study_dir) + issues: List[Dict[str, Any]] = [] + + for name, info in required_files.items(): + if name == "materials/": + if not info.get("exists"): + issues.append({"severity": "error", "path": "materials/", "message": "materials directory missing"}) + elif not info.get("count"): + issues.append({"severity": "error", "path": "materials/", "message": "no material JSON files"}) + continue + if not info.get("exists"): + issues.append({"severity": "error", "path": name, "message": "required package file missing"}) + elif not info.get("valid_json_object"): + issues.append({"severity": "error", "path": name, "message": f"invalid JSON package file: {info.get('error')}"}) + + metadata = loaded.get("metadata.json", {}) + specification = loaded.get("specification.json", {}) + ground_truth = loaded.get("ground_truth.json", {}) + for name, payload, key in ( + ("metadata.json", metadata, "id"), + ("specification.json", specification, "study_id"), + ("ground_truth.json", ground_truth, "study_id"), + ): + if payload and payload.get(key) != study_id: + issues.append( + { + "severity": "error", + "path": f"{name}.{key}", + "message": f"package id mismatch: {payload.get(key)!r} != {study_id!r}", + } + ) + + material_ids = set(material_by_id.keys()) + refs = _package_reference_audit( + metadata=metadata, + specification=specification, + ground_truth=ground_truth, + material_ids=material_ids, + material_by_id=material_by_id, + ) + for ref in refs["missing_material_refs"]: + issues.append({"severity": "error", "path": "ground_truth.json", "message": f"material reference has no file: {ref}"}) + for ref in refs["missing_specification_refs"]: + issues.append({"severity": "warning", "path": "specification.json", "message": f"material ref missing from participants.by_sub_study: {ref}"}) + for ref, target_ids in refs["missing_material_target_links"].items(): + issues.append( + { + "severity": "error", + "path": f"materials/{ref}.json.target_ids", + "message": "ground_truth target_id not covered by material: " + ", ".join(target_ids), + } + ) + + material_file_audit = _material_file_audit(material_by_id) + for material_id, item in material_file_audit["materials"].items(): + if item["runtime_issues"]: + issues.append( + { + "severity": "error", + "path": f"materials/{material_id}.json", + "message": "material runtime issues: " + ", ".join(item["runtime_issues"]), + } + ) + + material_patch_required = bool(materials_audit.get("materials_needing_patch")) + loadable = not any( + issue["severity"] == "error" + for issue in issues + if not (str(issue["path"]).startswith("materials/") and str(issue["path"]) != "materials/") + ) + ready_for_simulation = loadable and not material_patch_required and not any(issue["severity"] == "error" for issue in issues) + return { + "version": "stage4_package_contract_v1", + "loadable": loadable, + "ready_for_simulation": ready_for_simulation, + "human_patch_required": not ready_for_simulation, + "required_files": required_files, + "references": refs, + "material_files": material_file_audit, + "issues": issues, + } + + +def _build_audit( + payload: Dict[str, Any], + extraction: Dict[str, Any], + study_id: str, + study_dir: Path, + material_paths: List[Path], +) -> Dict[str, Any]: + source_materials = payload.get("study_materials") + source_materials = source_materials if isinstance(source_materials, dict) else {} + material_by_id: Dict[str, Dict[str, Any]] = {} + for path in material_paths: + try: + material = json.loads(Path(path).read_text(encoding="utf-8")) + except Exception as exc: # pragma: no cover - defensive audit guard + material_by_id[Path(path).stem] = { + "sub_study_id": Path(path).stem, + "_load_error": f"{type(exc).__name__}: {exc}", + } + continue + if isinstance(material, dict): + key = material.get("material_id") or material.get("target_id") or material.get("sub_study_id") or Path(path).stem + material_by_id[str(key)] = material + + materials_audit: Dict[str, Any] = {} + targets = [target for target in extraction.get("simulation_targets", []) or [] if isinstance(target, dict)] + audit_units: List[Dict[str, Any]] = [] + if targets: + grouped_targets: Dict[str, List[Dict[str, Any]]] = {} + for target in targets: + sub_id = str(target.get("sub_study_id") or "") + if sub_id: + grouped_targets.setdefault(sub_id, []).append(target) + for sub_id, grouped in grouped_targets.items(): + audit_units.append({"audit_id": sub_id, "sub_study_id": sub_id, "targets": grouped}) + else: + for study in extraction.get("studies", []): + for sub_study in study.get("sub_studies", []): + sub_id = str(sub_study.get("sub_study_id") or "") + if sub_id: + audit_units.append({"audit_id": sub_id, "sub_study_id": sub_id, "targets": []}) + + for unit in audit_units: + audit_id = unit["audit_id"] + sub_id = unit["sub_study_id"] + runtime = material_by_id.get(audit_id, {}) + source = source_materials.get(sub_id) + source = source if isinstance(source, dict) else {} + source_readiness = source.get("readiness") if isinstance(source.get("readiness"), dict) else {} + runtime_readiness = runtime.get("readiness") if isinstance(runtime.get("readiness"), dict) else {} + runtime_issues = _runtime_material_issues(runtime) + source_ready = source_readiness.get("ready") + runtime_ready = runtime_readiness.get("ready") + ready = (source_ready is not False) and (runtime_ready is not False) and not runtime_issues + material_path = study_dir / "materials" / f"{audit_id}.json" + materials_audit[audit_id] = { + "ready": ready, + "human_patch_required": not ready, + "runtime_material": str(material_path), + "runtime_issues": runtime_issues, + "runtime_readiness": runtime_readiness, + "source_readiness": source_readiness, + "selection": source.get("selection", {}), + "targets": unit.get("targets", []), + "target_ids": [ + str(target.get("target_id")) + for target in unit.get("targets", []) + if isinstance(target, dict) and target.get("target_id") + ], + "sub_study_id": sub_id, + "source_trace": source.get("source_trace", {}), + "human_material_contract": source.get("human_material_contract", {}), + "field_evidence": ( + source.get("human_material_contract", {}).get("field_evidence", {}) + if isinstance(source.get("human_material_contract"), dict) + else {} + ), + "conditions": source.get("conditions", []), + "response_schema": source.get("response_schema", {}), + "counts": { + "instruction_chars": len(str(runtime.get("instructions") or runtime.get("stimulus") or "")), + "items": len(runtime.get("items") or []) if isinstance(runtime.get("items"), list) else 0, + }, + } + + ready_count = sum(1 for item in materials_audit.values() if item["ready"]) + material_package_audit = { + "materials_total": len(materials_audit), + "materials_ready": ready_count, + "materials_needing_patch": len(materials_audit) - ready_count, + "materials": materials_audit, + } + package_audit = _build_package_audit( + study_id=study_id, + study_dir=study_dir, + materials_audit=material_package_audit, + material_by_id=material_by_id, + ) + return { + "stage4_version": "human-study-bench-draft", + "study_id": study_id, + "source_schema": extraction.get("source_schema"), + "materials_total": len(materials_audit), + "materials_ready": ready_count, + "materials_needing_patch": len(materials_audit) - ready_count, + "materials": materials_audit, + "package_audit": package_audit, + } + + +def _update_registry(data_dir: Path, metadata: Dict[str, Any]) -> Path: + registry_path = data_dir / "registry.json" + if registry_path.exists(): + registry = json.loads(registry_path.read_text(encoding="utf-8")) + else: + registry = {"version": "0.1.0", "last_updated": None, "total_studies": 0, "studies": []} + + studies = [item for item in registry.get("studies", []) if item.get("id") != metadata["id"]] + studies.append( + { + "id": metadata["id"], + "title": metadata.get("title", ""), + "domain": metadata.get("domain"), + "subdomain": metadata.get("subdomain"), + "difficulty": metadata.get("difficulty", "medium"), + "tags": metadata.get("keywords", []), + "status": "active", + "added_date": date.today().isoformat(), + "version": "1.0", + "estimated_time_minutes": 10, + } + ) + registry["studies"] = sorted(studies, key=lambda item: item.get("id", "")) + registry["total_studies"] = len(registry["studies"]) + registry["last_updated"] = date.today().isoformat() + _write_json(registry_path, registry) + return registry_path + + +def _build_json_package( + extraction: Dict[str, Any], + study_id: str, + study_dir: Path, + *, + use_llm: bool, + provider: str, + model: str, + api_key: Optional[str], + api_base: Optional[str], + pdf_path: Optional[Path], + selection_llm_client: Any = None, +) -> Dict[str, Any]: + if use_llm: + try: + generator = JSONGenerator(provider=provider, model=model, api_key=api_key, api_base=api_base) + print(" Stage 4 generating metadata.json", flush=True) + metadata = generator.generate_metadata(extraction, study_id, pdf_path=pdf_path) + _write_json(study_dir / "metadata.json", metadata) + print(" Stage 4 generating specification.json", flush=True) + specification = generator.generate_specification(extraction, study_id) + _write_json(study_dir / "specification.json", specification) + print(" Stage 4 generating ground_truth.json", flush=True) + ground_truth = generator.generate_ground_truth(extraction, study_id, debug_dir=study_dir) + _write_json(study_dir / "ground_truth.json", ground_truth) + print(" Stage 4 generating materials/*.json", flush=True) + material_paths = generator.generate_materials(extraction, study_dir, pdf_path=pdf_path) + return { + "metadata": metadata, + "specification": specification, + "ground_truth": ground_truth, + "material_paths": material_paths, + "json_generation": "llm", + } + except Exception as exc: + print(f"Warning: HumanStudy-Bench JSON generator failed, using deterministic fallback: {exc}") + + metadata = _deterministic_metadata(extraction, study_id) + specification = _deterministic_specification(extraction, study_id) + ground_truth = _deterministic_ground_truth(extraction, study_id) + material_paths = _deterministic_materials(extraction, study_dir, selection_llm_client=selection_llm_client) + _write_json(study_dir / "metadata.json", metadata) + _write_json(study_dir / "specification.json", specification) + _write_json(study_dir / "ground_truth.json", ground_truth) + return { + "metadata": metadata, + "specification": specification, + "ground_truth": ground_truth, + "material_paths": material_paths, + "json_generation": "deterministic", + } + + +def build_human_study_package( + json_path: Path, + *, + data_dir: Path = Path("data"), + study_dir: Optional[Path] = None, + study_id: Optional[str] = None, + pdf_path: Optional[Path] = None, + provider: str = "openai", + model: str = "gpt-4o-mini", + api_key: Optional[str] = None, + api_base: Optional[str] = None, + use_llm: bool = False, + selection_llm_client: Any = None, + generate_config: bool = False, + config_dir: Path = Path("src/studies"), + update_registry: bool = True, + hub_layout: bool = False, +) -> Dict[str, Any]: + json_path = Path(json_path) + data_dir = Path(data_dir) + payload = json.loads(json_path.read_text(encoding="utf-8")) + resolved_study_id = study_id or _default_study_id(payload, json_path) + extraction = normalize_to_human_extraction(payload) + extraction["study_id"] = resolved_study_id + + if hub_layout: + study_root = Path(study_dir) if study_dir else data_dir / resolved_study_id + package_dir = study_root / "source" + else: + study_root = Path(study_dir) if study_dir else data_dir / "studies" / resolved_study_id + package_dir = study_root + + study_root.mkdir(parents=True, exist_ok=True) + package_dir.mkdir(parents=True, exist_ok=True) + source_path = _write_json(package_dir / "source_extraction.json", extraction) + + package = _build_json_package( + extraction, + resolved_study_id, + package_dir, + use_llm=use_llm, + provider=provider, + model=model, + api_key=api_key, + api_base=api_base, + pdf_path=pdf_path, + selection_llm_client=selection_llm_client, + ) + audit = _build_audit(payload, extraction, resolved_study_id, package_dir, package["material_paths"]) + audit_path = _write_json(package_dir / "audit.json", audit) + + registry_path = None if hub_layout else (_update_registry(data_dir, package["metadata"]) if update_registry else None) + hub_files: List[Path] = [] + if hub_layout: + hub_files.extend(_write_hub_root_files(study_root, package["metadata"], extraction, audit)) + hub_files.extend(_write_hub_scripts(study_root, resolved_study_id)) + if pdf_path and Path(pdf_path).exists(): + pdf_dest = package_dir / Path(pdf_path).name + if Path(pdf_path).resolve() != pdf_dest.resolve(): + shutil.copy2(pdf_path, pdf_dest) + hub_files.append(pdf_dest) + + config_path = None + config_status = "skipped" + config_error = None + if generate_config: + if not use_llm: + config_status = "skipped_no_llm" + else: + config_path = (study_root / "scripts" / "config.py") if hub_layout else config_dir / f"{resolved_study_id}_config.py" + try: + print(f" Stage 4 generating StudyConfig adapter: {config_path}", flush=True) + ConfigGenerator(provider=provider, model=model, api_key=api_key, api_base=api_base).generate( + extraction, + resolved_study_id, + config_path, + pdf_path=pdf_path, + study_dir=package_dir, + standalone=hub_layout, + ) + config_status = "generated" + except Exception as exc: + config_path = None + config_status = "failed" + config_error = str(exc) + + generated_files = [ + source_path, + package_dir / "metadata.json", + package_dir / "specification.json", + package_dir / "ground_truth.json", + audit_path, + *package["material_paths"], + *hub_files, + ] + if registry_path: + generated_files.append(registry_path) + if config_path: + generated_files.append(config_path) + + return { + "stage4_version": "human-study-bench", + "study_id": resolved_study_id, + "source_json": str(json_path), + "data_dir": str(data_dir), + "study_dir": str(study_root), + "source_dir": str(package_dir), + "hub_layout": hub_layout, + "registry_path": str(registry_path) if registry_path else None, + "audit_path": str(audit_path), + "config_path": str(config_path) if config_path else None, + "config_status": config_status, + "config_error": config_error, + "json_generation": package["json_generation"], + "generated_files": [str(path) for path in generated_files], + } + + +def build_stage4_experiments( + json_path: Path, + *, + out_dir: Path, + client: Any = None, + max_repair_iters: int = 3, + use_llm: bool = False, + study_id: Optional[str] = None, + pdf_path: Optional[Path] = None, + provider: str = "openai", + model: Optional[str] = None, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + generate_config: bool = False, +) -> Dict[str, Any]: + """Compatibility wrapper for the old Stage 4 function name.""" + del max_repair_iters + if client is not None: + model = model or getattr(client, "model", None) + api_key = api_key or getattr(client, "api_key", None) + api_base = api_base or getattr(client, "api_base", None) + return build_human_study_package( + json_path, + data_dir=Path(out_dir), + study_id=study_id, + pdf_path=pdf_path, + provider=provider, + model=model or "gpt-4o-mini", + api_key=api_key, + api_base=api_base, + use_llm=use_llm, + selection_llm_client=client if client is not None else None, + generate_config=generate_config, + ) diff --git a/generation_pipeline/stage5.py b/generation_pipeline/stage5.py new file mode 100644 index 0000000..9c6e739 --- /dev/null +++ b/generation_pipeline/stage5.py @@ -0,0 +1,289 @@ +from __future__ import annotations + +import json +import time +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from src.agents.llm_participant_agent import ParticipantPool +from src.core.benchmark import HumanStudyBench +from src.core.study import Study +from src.core.study_config import get_study_config + + +@dataclass +class Stage5Options: + n_participants: Optional[int] = None + n_agents: Optional[int] = None + max_agents: Optional[int] = None + repeats: int = 1 + seed: int = 42 + temperature: float = 1.0 + num_workers: Optional[int] = None + mock: bool = False + dry_run: bool = False + use_cache: bool = False + cache_dir: Optional[Path] = None + profiles_json: Optional[Path] = None + system_prompt_file: Optional[Path] = None + system_prompt_preset: str = "v3_human_plus_demo" + reasoning: str = "default" + enable_reasoning: bool = False + api_key: Optional[str] = None + api_base: Optional[str] = None + + def requested_participants(self) -> Optional[int]: + value = self.n_participants if self.n_participants is not None else self.n_agents + if value is None: + return None + value = int(value) + if self.max_agents is not None: + value = min(value, int(self.max_agents)) + return max(value, 1) + + @property + def use_real_llm(self) -> bool: + return not (self.mock or self.dry_run) + + +def _safe_model_slug(model: str) -> str: + return str(model).replace("/", "_").replace("-", "_").replace(":", "_") + + +def _load_profiles(path: Optional[Path]) -> Optional[List[Dict[str, Any]]]: + if not path: + return None + data = json.loads(Path(path).read_text(encoding="utf-8")) + if isinstance(data, dict): + data = data.get("profiles") + if not isinstance(data, list): + raise ValueError(f"profiles_json must contain a list or {{'profiles': list}}: {path}") + return data + + +def _read_optional_text(path: Optional[Path]) -> Optional[str]: + if not path: + return None + return Path(path).read_text(encoding="utf-8") + + +def _is_study_package_dir(path: Path) -> bool: + flat = ( + path.is_dir() + and (path / "metadata.json").exists() + and (path / "specification.json").exists() + and (path / "ground_truth.json").exists() + and (path / "materials").is_dir() + ) + source = path / "source" + hub = ( + path.is_dir() + and source.is_dir() + and (source / "metadata.json").exists() + and (source / "specification.json").exists() + and (source / "ground_truth.json").exists() + and (source / "materials").is_dir() + ) + return flat or hub + + +def _resolve_study(study: str | Path, data_dir: Path) -> Tuple[str, Path, Optional[Path]]: + candidate = Path(study) + if candidate.exists() and candidate.is_dir(): + if _is_study_package_dir(candidate): + study_id = candidate.name + if candidate.parent.name == "studies": + return study_id, candidate.parent.parent, None + return study_id, data_dir, candidate + if (candidate / "registry.json").exists(): + raise ValueError("When passing a data directory to Stage 5, also pass --study-id.") + return str(study), data_dir, None + + +def _choose_trial_mode( + specification: Dict[str, Any], + requested_n: Optional[int], + trials: List[Dict[str, Any]], +) -> Tuple[int, bool]: + by_sub_study = specification.get("participants", {}).get("by_sub_study") + if requested_n is not None or by_sub_study: + return len(trials), True + n_default = specification.get("participants", {}).get("n") or 30 + return int(n_default), False + + +def _cache_path(options: Stage5Options, study_id: str, model: str, repeat_idx: int) -> Optional[Path]: + if not options.use_cache: + return None + cache_dir = Path(options.cache_dir or ".cache/stage5") + cache_dir.mkdir(parents=True, exist_ok=True) + return cache_dir / f"{study_id}_{_safe_model_slug(model)}_repeat_{repeat_idx}.json" + + +def _run_single_repeat( + *, + study_id: str, + study: Any, + study_config: Any, + model: str, + repeat_idx: int, + options: Stage5Options, + profiles: Optional[List[Dict[str, Any]]], + system_prompt_override: Optional[str], +) -> Dict[str, Any]: + cache_path = _cache_path(options, study_id, model, repeat_idx) + if cache_path and cache_path.exists(): + cached = json.loads(cache_path.read_text(encoding="utf-8")) + if isinstance(cached, dict) and cached.get("raw_results"): + return cached["raw_results"] + + requested_n = options.requested_participants() + trials = study_config.create_trials(n_trials=requested_n) + instructions = study_config.get_instructions() + builder = study_config.get_prompt_builder() + + participant_pool_kwargs = { + "study_specification": study.specification, + "use_real_llm": options.use_real_llm, + "model": model, + "api_key": options.api_key, + "api_base": options.api_base, + "random_seed": options.seed + repeat_idx, + "num_workers": int(options.num_workers) if options.num_workers is not None else None, + "profiles": profiles, + "prompt_builder": builder, + "system_prompt_override": system_prompt_override, + "system_prompt_preset": options.system_prompt_preset, + "reasoning": options.reasoning, + "enable_reasoning": options.enable_reasoning, + "study_id": study_id, + "temperature": options.temperature, + } + + if getattr(study_config, "REQUIRES_GROUP_TRIALS", False) and hasattr(study_config, "run_group_experiment"): + raw_results = study_config.run_group_experiment( + trials, + instructions, + participant_pool_kwargs, + prompt_builder=builder, + ) + else: + n_participants, one_to_one = _choose_trial_mode(study.specification, requested_n, trials) + participant_pool_kwargs["n_participants"] = n_participants + pool = ParticipantPool(**participant_pool_kwargs) + raw_results = pool.run_experiment( + trials, + instructions, + prompt_builder=builder, + one_to_one=one_to_one, + ) + + if cache_path: + cache_path.write_text( + json.dumps( + {"version": 1, "study_id": study_id, "model": model, "repeat_idx": repeat_idx, "raw_results": raw_results}, + indent=2, + ensure_ascii=False, + ), + encoding="utf-8", + ) + return raw_results + + +def run_stage5( + study: str | Path, + *, + runs_dir: Path = Path("results/benchmark"), + models: Optional[List[str]] = None, + client_factory: Any = None, + options: Optional[Stage5Options] = None, + data_dir: Path = Path("data"), +) -> Dict[str, Any]: + del client_factory + options = options or Stage5Options() + model_names = models or ["mistralai/mistral-nemo"] + study_id, resolved_data_dir, direct_study_path = _resolve_study(study, Path(data_dir)) + + if direct_study_path is not None: + loaded_study = Study.load(direct_study_path) + study_path = direct_study_path + else: + benchmark = HumanStudyBench(resolved_data_dir) + loaded_study = benchmark.load_study(study_id) + study_path = resolved_data_dir / "studies" / study_id + study_config = get_study_config(study_id, study_path, loaded_study.specification) + + profiles = _load_profiles(options.profiles_json) + system_prompt_override = _read_optional_text(options.system_prompt_file) + start_time = time.time() + all_model_runs: List[Dict[str, Any]] = [] + + for model in model_names: + raw_runs: List[Dict[str, Any]] = [] + for repeat_idx in range(max(int(options.repeats), 1)): + raw_runs.append( + _run_single_repeat( + study_id=study_id, + study=loaded_study, + study_config=study_config, + model=model, + repeat_idx=repeat_idx, + options=options, + profiles=profiles, + system_prompt_override=system_prompt_override, + ) + ) + + try: + aggregate = study_config.aggregate_results(raw_runs[0] if raw_runs else {"individual_data": []}) + except Exception as exc: + aggregate = {"descriptive_statistics": {}, "inferential_statistics": {}, "error": str(exc)} + + model_dir = Path(runs_dir) / study_id / _safe_model_slug(model) + model_dir.mkdir(parents=True, exist_ok=True) + output_path = model_dir / "full_benchmark.json" + save_data = { + "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"), + "stage5_version": "human-study-bench", + "study_id": study_id, + "title": loaded_study.metadata.get("title", ""), + "model": model, + "use_real_llm": options.use_real_llm, + "system_prompt_preset": options.system_prompt_preset, + "reasoning": options.reasoning, + "random_seed": options.seed, + "repeats": len(raw_runs), + "elapsed_time": time.time() - start_time, + "descriptive_statistics": aggregate.get("descriptive_statistics", {}), + "inferential_statistics": aggregate.get("inferential_statistics", {}), + "individual_data": raw_runs[0].get("individual_data", []) if raw_runs else [], + "all_runs_raw_results": [ + {"individual_data": run.get("individual_data", [])} + for run in raw_runs + ], + } + output_path.write_text(json.dumps(save_data, indent=2, ensure_ascii=False), encoding="utf-8") + all_model_runs.append( + { + "model": model, + "output_path": str(output_path), + "repeats": len(raw_runs), + "responses": sum(len(run.get("individual_data", [])) for run in raw_runs), + "aggregate_error": aggregate.get("error"), + } + ) + + return { + "stage5_version": "human-study-bench", + "study_id": study_id, + "data_dir": str(resolved_data_dir), + "study_path": str(study_path), + "runs_dir": str(runs_dir), + "models": model_names, + "run_count": sum(item["repeats"] for item in all_model_runs), + "completed": len(all_model_runs), + "use_real_llm": options.use_real_llm, + "runs": all_model_runs, + } diff --git a/generation_pipeline/study_selector.py b/generation_pipeline/study_selector.py new file mode 100644 index 0000000..39f2d9d --- /dev/null +++ b/generation_pipeline/study_selector.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import json +import re +from typing import Any, Dict, List, Optional + +from src.llm.helpers import call_with_timeout + +STUDY_SELECTION_RUBRIC = """You are curating the sub-studies of ONE research paper for an LLM-participant +simulation benchmark. A simulated participant can only READ self-contained written +materials and return a written response. Decide for EACH study: keep or drop. + +KEEP a study when an LLM participant can faithfully complete it: +- Vignette / scenario studies (read a scenario, give judgments or ratings). +- Message / article reading studies (read text, then rate attitudes/openness). +- Clean experimental manipulations delivered as written instructions / assignment + to a between-subjects condition, measured by ratings / Likert / a single choice. +- Keep BOTH an original and its replication when the replication adds a new + measure/construct/population and is NOT described as fixing the earlier one. + +DROP a study only when it cannot be faithfully simulated or is clearly secondary: +- Pilot / norming / validation / manipulation-check-only studies. +- Interactive behavioral paradigms: economic games, public-vs-private decisions + with real partners, real money/behavior - anything needing live interaction. +- Repetitive multi-trial psychophysical / discounting tasks whose result is a + derived curve / indifference point rather than a self-contained response. +- An EARLIER study when a later study is described as a "higher-powered" (or + better-powered / corrected) replication of essentially the same manipulation, + because that label means the earlier study was under-powered; keep the + higher-powered version and drop the earlier one. + +Return ONLY a JSON array, one object per study: +[{"experiment_id": "...", "keep": true, "reason": "<=15 words"}]""" + + +def build_brief(exp: Dict[str, Any]) -> str: + """Compact, decision-relevant brief for one Stage 1 experiment record.""" + return ( + "- experiment_id: %s\n" + " name: %s\n" + " design: %s\n" + " conditions/factors: %s\n" + " input (what participant does): %s\n" + " participant_task: %s\n" + " output (DV): %s\n" + " source_hints: %s" + % ( + exp.get("experiment_id"), + exp.get("experiment_name"), + _brief_value(exp.get("design_type")), + _brief_value(exp.get("conditions_or_factors")), + str(exp.get("input"))[:240], + str(exp.get("participant_task") or "")[:240], + str(exp.get("output"))[:160], + _brief_value(exp.get("candidate_source_hints"))[:240], + ) + ) + + +def _brief_value(value: Any) -> str: + if value is None: + return "" + if isinstance(value, list): + parts = [] + for item in value: + if isinstance(item, dict): + text = item.get("description") or item.get("kind") or item + else: + text = item + if str(text).strip(): + parts.append(str(text).strip()) + return "; ".join(parts) + return str(value).strip() + + +def build_prompt(experiments: List[Dict[str, Any]]) -> str: + briefs = "\n".join(build_brief(e) for e in experiments) + return f"{STUDY_SELECTION_RUBRIC}\n\nPaper studies:\n{briefs}" + + +def _parse_decisions(response: str) -> List[Dict[str, Any]]: + text = response if isinstance(response, str) else str(response) + m = re.search(r"\[.*\]", text, re.DOTALL) + if not m: + return [] + try: + data = json.loads(m.group()) + except json.JSONDecodeError: + return [] + return [d for d in data if isinstance(d, dict) and d.get("experiment_id")] + + +def _select_once( + experiments: List[Dict[str, Any]], + client: Any, + *, + timeout: Optional[float] = 60.0, +) -> Optional[List[Dict[str, Any]]]: + try: + response = _generate_selection_response(client, prompt=build_prompt(experiments), timeout=timeout) + except Exception: # pragma: no cover - network/LLM dependent + return None + return _parse_decisions(response) + + +def _generate_selection_response(client: Any, *, prompt: str, timeout: Optional[float]) -> str: + def _call() -> str: + try: + return client.generate_content(prompt=prompt, timeout=timeout, max_tokens=1000) + except TypeError: + try: + return client.generate_content(prompt=prompt, timeout=timeout) + except TypeError: + return client.generate_content(prompt=prompt) + + return call_with_timeout(_call, timeout=timeout) + + +_DROP_REASON_RE = re.compile( + r"\b(" + r"requires\s+real|no\s+self[- ]contained\s+scenario|field\s+survey|" + r"longitudinal|multi[- ]wave|supervisor[- ]rated|real[- ]world\s+interaction|" + r"real\s+employees|not\s+faithfully\s+simulat" + r")\b", + re.IGNORECASE, +) + +_FIELD_STUDY_RE = re.compile( + r"\b(field\s+survey|multi[- ]wave|longitudinal|time[- ]lagged|" + r"supervisor[- ]rated|real\s+employees|workplace\s+survey)\b", + re.IGNORECASE, +) + +_SELF_CONTAINED_EXPERIMENT_RE = re.compile( + r"\b(vignette|scenario|random\s+assignment|randomly\s+assigned|" + r"between[- ]subjects|within[- ]subjects|manipulat|written\s+instruction)\b", + re.IGNORECASE, +) + +_REAL_ACTION_RE = re.compile( + r"\b(" + r"behavioral\s+paradigm|economic\s+game|donat(?:e|ion)|bonus|30\s*¢|30\s*cents?|" + r"real[- ]?money|binary\s+donation|public\s+vs\.?\s+private|observability" + r")\b", + re.IGNORECASE, +) + + +def _deterministic_drop(exp: Dict[str, Any], reason: str = "") -> Optional[str]: + """Hard vetoes for cases the LLM sometimes labels keep despite its reason. + + These are not replacements for the LLM selector. They only catch obvious + non-simulatable field/longitudinal studies and contradictory reasons such as + "requires real employees; no self-contained scenario". + """ + reason_s = str(reason or "") + if _DROP_REASON_RE.search(reason_s): + return "deterministic-drop: " + reason_s[:180] + + text = " ".join( + str(exp.get(key) or "") + for key in ("experiment_name", "design_type", "conditions_or_factors", "input", "participant_task", "output") + ) + if _REAL_ACTION_RE.search(text): + return "deterministic-drop: real behavioral/money action not self-contained written response" + if _FIELD_STUDY_RE.search(text) and not _SELF_CONTAINED_EXPERIMENT_RE.search(text): + return "deterministic-drop: field/longitudinal study without self-contained scenario" + return None + + +def select_studies( + experiments: List[Dict[str, Any]], + client: Any, + *, + votes: int = 1, + timeout: Optional[float] = 60.0, +) -> Dict[str, Dict[str, Any]]: + """Return {experiment_id: {"keep": bool, "reason": str}} for a paper. + + On any LLM/parse failure a study defaults to keep=True (fail-open: never + silently drop a study the model could not score). With `votes` > 1 the judge + is polled several times and each study's decision is a majority vote, which + stabilizes the borderline calls (ties resolve to keep). + """ + ids = [str(e.get("experiment_id")) for e in experiments if e.get("experiment_id")] + decisions: Dict[str, Dict[str, Any]] = { + eid: {"keep": True, "reason": "default-keep (not scored)"} for eid in ids + } + + keep_votes: Dict[str, int] = {eid: 0 for eid in ids} + drop_votes: Dict[str, int] = {eid: 0 for eid in ids} + reasons: Dict[str, str] = {eid: "" for eid in ids} + rounds = 0 + for _ in range(max(1, votes)): + parsed = _select_once(experiments, client, timeout=timeout) + if parsed is None: + continue + rounds += 1 + for d in parsed: + eid = str(d.get("experiment_id")) + if eid not in keep_votes: + continue + if bool(d.get("keep", True)): + keep_votes[eid] += 1 + else: + drop_votes[eid] += 1 + if d.get("reason"): + reasons[eid] = str(d.get("reason"))[:200] + + if rounds == 0: + for eid in ids: + exp = next((item for item in experiments if str(item.get("experiment_id")) == eid), {}) + veto = _deterministic_drop(exp, "") + if veto: + decisions[eid] = {"keep": False, "reason": veto} + else: + decisions[eid]["reason"] = "selection skipped: LLM unavailable" + return decisions + + for eid in ids: + # tie or no votes -> keep (fail-open) + keep = keep_votes[eid] >= drop_votes[eid] + suffix = "" + if votes > 1: + suffix = f" (votes {keep_votes[eid]}K/{drop_votes[eid]}d)" + reason = (reasons[eid] + suffix).strip() + exp = next((item for item in experiments if str(item.get("experiment_id")) == eid), {}) + veto = _deterministic_drop(exp, reason) + if veto: + decisions[eid] = {"keep": False, "reason": veto} + else: + decisions[eid] = {"keep": keep, "reason": reason} + return decisions + + +def selection_summary(decisions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + kept = [eid for eid, d in decisions.items() if d["keep"]] + dropped = [eid for eid, d in decisions.items() if not d["keep"]] + return { + "total": len(decisions), + "kept": kept, + "dropped": dropped, + "n_kept": len(kept), + "n_dropped": len(dropped), + } diff --git a/generation_pipeline/utils/__init__.py b/generation_pipeline/utils/__init__.py new file mode 100644 index 0000000..cd58d6a --- /dev/null +++ b/generation_pipeline/utils/__init__.py @@ -0,0 +1,6 @@ +from generation_pipeline.utils.document_loader import DocumentLoader +from generation_pipeline.utils.pdf_extractor import extract_pdf_text +from generation_pipeline.utils.output_formatter import OutputFormatter +from generation_pipeline.utils.review_parser import ReviewParser + +__all__ = ["DocumentLoader", "extract_pdf_text", "OutputFormatter", "ReviewParser"] diff --git a/generation_pipeline/utils/document_loader.py b/generation_pipeline/utils/document_loader.py new file mode 100644 index 0000000..7533709 --- /dev/null +++ b/generation_pipeline/utils/document_loader.py @@ -0,0 +1,42 @@ +""" +Document Loader (ai-ethics): inlined — no `legacy/` dependency. +Only the methods used by the filter / extractor. +""" + +import json +from pathlib import Path +from typing import Any, Dict, List + + +class DocumentLoader: + """Loads PDF / JSON / markdown for the extraction pipeline.""" + + @staticmethod + def get_pdf_pages(pdf_path: Path) -> List[int]: + """Return list of 1-indexed page numbers in the PDF.""" + pdf_path = Path(pdf_path) + if not pdf_path.exists(): + raise FileNotFoundError(f"PDF file not found: {pdf_path}") + try: + from pypdf import PdfReader + except ImportError: + try: + from PyPDF2 import PdfReader # type: ignore + except ImportError: + raise ImportError("Install pypdf: pip install pypdf") + reader = PdfReader(str(pdf_path)) + return list(range(1, len(reader.pages) + 1)) + + @staticmethod + def load_json(json_path: Path) -> Dict[str, Any]: + json_path = Path(json_path) + if not json_path.exists(): + raise FileNotFoundError(f"JSON file not found: {json_path}") + return json.loads(json_path.read_text(encoding="utf-8")) + + @staticmethod + def load_markdown(md_path: Path) -> str: + md_path = Path(md_path) + if not md_path.exists(): + raise FileNotFoundError(f"Markdown file not found: {md_path}") + return md_path.read_text(encoding="utf-8") diff --git a/generation_pipeline/utils/json_generator.py b/generation_pipeline/utils/json_generator.py new file mode 100644 index 0000000..541a927 --- /dev/null +++ b/generation_pipeline/utils/json_generator.py @@ -0,0 +1,1032 @@ +""" +JSON Generator - Generates compatible JSON files (metadata.json, specification.json, ground_truth.json) +and materials files +""" + +import json +import sys +import re +from pathlib import Path +from typing import Dict, Any, List, Optional + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) +from src.llm.factory import get_client +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + +PDF_TEXT_MAX_CHARS = 400000 + + +class JSONGenerator: + """Generate JSON files compatible with existing study format""" + + def __init__( + self, + provider: str = "gemini", + model: str = "models/gemini-3-flash-preview", + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ): + """ + Initialize JSON generator. + + Args: + provider: One of gemini, openai, anthropic, xai, openrouter + model: Model name for LLM-based materials generation + api_key: Optional API key + api_base: Optional API base URL + """ + self.client = get_client(provider=provider, model=model, api_key=api_key, api_base=api_base) + + def generate_metadata( + self, + extraction_result: Dict[str, Any], + study_id: str, + pdf_path: Optional[Path] = None + ) -> Dict[str, Any]: + """ + Generate metadata.json using LLM to infer domain/subdomain and structure. + + Args: + extraction_result: Results from stage2 extraction + study_id: Study ID (e.g., "study_005") + pdf_path: Optional path to PDF file for context + + Returns: + Dictionary matching metadata.json format + """ + studies = extraction_result.get('studies', []) + if not studies: + raise ValueError("No studies found") + + # Collect all scenarios from all sub-studies + scenarios = [] + for study in studies: + sub_studies = study.get('sub_studies', []) + for sub_study in sub_studies: + sub_id = sub_study.get('sub_study_id', '') + if sub_id: + scenarios.append(sub_id) + + # Use LLM to infer domain/subdomain and keywords + extraction_summary = json.dumps(extraction_result, indent=2, ensure_ascii=False) + + prompt = f"""You are a metadata generator. Based on the extraction results, generate the domain, subdomain, keywords, and findings for this study. + +EXTRACTION RESULTS: +{extraction_summary[:120000]}... + +TASK: +Generate a JSON object with: +- domain: The psychological domain (e.g., "social_psychology", "cognitive_psychology", "developmental_psychology") +- subdomain: The specific subdomain (e.g., "social_cognition", "judgment_and_decision_making", "memory") +- keywords: List of relevant keywords (3-5 keywords) +- findings: List of findings extracted from the study, each with: + - finding_id: Unique identifier (e.g., "F1", "F2") + - main_hypothesis: The main hypothesis or claim for this finding + - weight: Default weight of 1.0 (can be adjusted later) + - tests: List of statistical tests for this finding, each with: + - test_name: Name of the statistical test + - weight: Default weight of 1.0 (can be adjusted later) + +OUTPUT FORMAT (JSON only): +{{ + "domain": "domain_name", + "subdomain": "subdomain_name", + "keywords": ["keyword1", "keyword2", "keyword3"], + "findings": [ + {{ + "finding_id": "F1", + "main_hypothesis": "Description of the finding/hypothesis", + "weight": 1.0, + "tests": [ + {{ + "test_name": "Name of statistical test", + "weight": 1.0 + }} + ] + }} + ] +}} + +STRICT REQUIREMENTS: +1. **NO HALLUCINATION**: Only infer metadata based on the provided EXTRACTION RESULTS. +2. **HONESTY**: If the domain or subdomain cannot be clearly determined from the text, use null. +3. **SPECIFICITY**: Use underscores instead of spaces for domain and subdomain names. +4. **FINDINGS**: Extract all findings mentioned in the extraction results. If no findings are explicitly mentioned, use an empty array []. + +Generate the JSON: +""" + + try: + if pdf_path and pdf_path.exists(): + pdf_text = extract_pdf_text(pdf_path, max_chars=PDF_TEXT_MAX_CHARS) + full_prompt = f"PDF content:\n\n{pdf_text}\n\n{prompt}" + response = self.client.generate_content(prompt=full_prompt, temperature=0.3) + else: + response = self.client.generate_content(prompt=prompt, temperature=0.3) + + if response: + # Parse LLM response + llm_metadata = self._parse_json_response(response) + domain = llm_metadata.get('domain') + subdomain = llm_metadata.get('subdomain') + keywords = llm_metadata.get('keywords', []) + llm_findings = llm_metadata.get('findings', []) + else: + domain = None + subdomain = None + keywords = [] + llm_findings = [] + except Exception as e: + print(f"Warning: Error generating metadata with LLM: {e}") + domain = None + subdomain = None + keywords = [] + llm_findings = [] + + # Extract keywords from phenomenon if LLM didn't provide + if not keywords: + study = studies[0] + phenomenon = study.get('phenomenon', '') + if phenomenon: + keywords = [phenomenon.replace(' ', '_')] + + # Use LLM-extracted findings if available, otherwise extract from extraction_result + findings = llm_findings if llm_findings else [] + + # Fallback: Extract findings from extraction_result if LLM didn't provide them + if not findings: + for study in studies: + sub_studies = study.get('sub_studies', []) + for sub_study in sub_studies: + # Look for findings in human_data or statistical_results + human_data = sub_study.get('human_data', {}) + statistical_results = human_data.get('statistical_results', []) + + # Group statistical tests by finding/hypothesis + finding_map = {} + for stat_result in statistical_results: + finding_key = stat_result.get('claim', '') or stat_result.get('hypothesis', '') or 'Unknown Finding' + if finding_key not in finding_map: + finding_map[finding_key] = { + "finding_id": f"F{len(findings) + 1}", + "main_hypothesis": finding_key, + "weight": 1.0, # Default weight + "tests": [] + } + finding_map[finding_key]["tests"].append({ + "test_name": stat_result.get('test_name', 'Unknown Test'), + "weight": 1.0 # Default weight for each test + }) + + findings.extend(list(finding_map.values())) + + # If no findings found, create empty findings list + # (will be populated later from ground_truth.json if available) + if not findings: + findings = [] + + return { + "id": study_id, + "title": extraction_result.get('paper_title', ''), + "authors": extraction_result.get('paper_authors', []), + "year": extraction_result.get('paper_year'), + "domain": domain, + "subdomain": subdomain, + "keywords": keywords, + "difficulty": "medium", # TODO: Determine from study complexity + "description": extraction_result.get('paper_abstract', '')[:2000] + "...", + "scenarios": scenarios, + "findings": findings + } + + def _parse_json_response(self, response: str) -> Dict[str, Any]: + """Parse JSON from LLM response""" + response_text = response.strip() + + # Remove markdown code blocks if present + if '```json' in response_text: + response_text = response_text.split('```json')[1].split('```')[0].strip() + elif '```' in response_text: + response_text = response_text.split('```')[1].split('```')[0].strip() + + try: + return json.loads(response_text) + except json.JSONDecodeError: + # Try to extract JSON object + import re + json_match = re.search(r'\{.*\}', response_text, re.DOTALL) + if json_match: + return json.loads(json_match.group()) + return {} + + def generate_specification(self, extraction_result: Dict[str, Any], study_id: str) -> Dict[str, Any]: + """ + Generate specification.json using LLM to ensure accuracy and avoid defaults. + """ + extraction_summary = json.dumps(extraction_result, indent=2, ensure_ascii=False) + + prompt = f"""You are an experimental design expert. Based on the extraction results from a research paper, generate a `specification.json` object. + +EXTRACTION RESULTS: +{extraction_summary[:120000]}... + +TASK: +Generate a JSON object that describes the experimental setup. +ONLY include information explicitly present in the extraction results. + +JSON STRUCTURE: +{{ + "study_id": "{study_id}", + "title": "Paper Title", + "participants": {{ + "n": 0, // Total N found in paper. 0 if not found. + "population": "string", // e.g. 'undergraduates', 'MTurk workers'. null if not found. + "recruitment_source": "string", // null if not found. + "demographics": {{}}, // Any age, gender, etc. reported. Empty if not found. + "by_sub_study": {{ + "sub_id": {{ "n": 0, ... }} // Breakdowns if available. + }} + }}, + "design": {{ + "type": "string", // e.g. 'Between-Subjects', 'Within-Subjects', 'Mixed' + "factors": [ + {{ + "name": "Factor Name", + "levels": ["level1", "level2"], + "type": "Between-Subjects|Within-Subjects" + }} + ] + }}, + "procedure": {{ + "steps": ["Step 1", "Step 2"] // Extract actual steps from the paper. + }} +}} + +STRICT REQUIREMENTS: +1. **NO HALLUCINATION**: If participant N is not mentioned, use 0. If population is not mentioned, use null. DO NOT guess. +2. **NO DEFAULTS**: Do not assume "undergraduates" or "0.05" or any other common value. +3. **ACCURACY**: Extract factors and levels precisely from the experimental conditions described. + +Generate the JSON: +""" + try: + response = self.client.generate_content(prompt=prompt, temperature=0.1) + if response: + return self._parse_json_response(response) + except Exception as e: + print(f"Warning: Error generating specification with LLM: {e}") + + return { + "study_id": study_id, + "title": extraction_result.get('paper_title', ''), + "participants": {"n": 0, "demographics": {}}, + "design": {"factors": []}, + "procedure": {"steps": []} + } + + def generate_ground_truth( + self, + extraction_result: Dict[str, Any], + study_id: str, + debug_dir: Optional[Path] = None, + ) -> Dict[str, Any]: + """ + Generate ground_truth.json using LLM to ensure validation criteria + match the paper's claims and statistical results. + """ + studies = extraction_result.get('studies', []) + if not studies: + raise ValueError("No studies found") + + extraction_summary = json.dumps(extraction_result, indent=2, ensure_ascii=False) + + prompt = f"""You are a scientific data extractor. Your goal is to FAITHFULLY extract ALL validation criteria and statistical results from a research paper based on the provided extraction results. + +EXTRACTION RESULTS: +{extraction_summary[:120000]}... + +TASK: +Generate a `ground_truth.json` object. + +**CRITICAL EXTRACTION REQUIREMENTS (READ CAREFULLY):** + +1. **EXTRACT ALL STATISTICAL TESTS - NO EXCEPTIONS**: + - If the extraction results list 17 item-level t-tests, you MUST output ALL 17 in the JSON. + - DO NOT summarize. DO NOT select "representative" items. Output EVERY SINGLE numerical finding. + - The number of items in your `original_data_points.data` MUST match the number of items in the input `item_level_results`. + +2. **PRIORITIZE FULL STATISTICS**: + - For each finding, you MUST first look for standard statistics: t-value, F-value, r, or chi2. + - If a standard statistic is available, use it (e.g., "t(79) = 2.66"). + - If ONLY a p-value or a proportion is reported (e.g., for Sign tests or descriptive claims), you MUST use the following format: `p=VALUE, n=N` (e.g., "p=0.01, n=92"). + - Strip all non-numeric symbols from the values (e.g., convert "p < .01" to "p=0.01"). + - These values are CRITICAL for calculating the Bayesian Alignment Score (PAS). + +3. **HIERARCHICAL GROUPING**: + - Findings MUST be nested under the correct Study/Experiment they belong to. + +4. **DATA TRACEABILITY**: + - `original_data_points` must show the exact numbers (Means, SDs, Ns, percentages) used to calculate the reported statistics for EVERY item. + +5. **NO HALLUCINATION**: + - Only include numbers and claims explicitly present in the EXTRACTION RESULTS. + +JSON STRUCTURE: +{{ + "study_id": "{study_id}", + "title": "Paper Title", + "authors": ["Author 1"], + "year": 19XX, + "studies": [ + {{ + "study_id": "Study 1", + "study_name": "Name of the study", + "findings": [ + {{ + "finding_id": "F1", + "main_hypothesis": "The core psychological claim (e.g., 'People perceive a false consensus').", + "statistical_tests": [ + {{ + "test_name": "Exact test name (e.g., 'Independent t-test').", + "statistical_hypothesis": "Formal comparison (e.g., 'Mean A > Mean B').", + "reported_statistics": "Verbatim statistics string (e.g., 't(79) = 5.4, p < .001').", + "significance_level": 0.05, + "expected_direction": "positive|negative|none" + }} + ], + "original_data_points": {{ + "description": "Description of data used for these statistics.", + "data": {{ + "item_or_condition_name": {{ + "mean": 0.0, + "sd": 0.0, + "n": 0, + "percentage": 0.0, + "t": 0.0, + "other_stat": 0.0 + }} + }} + }} + }} + ] + }} + ], + "overall_original_results": {{ + "description": "Structured repository of ALL numerical findings.", + "data_tables": [ + {{ + "table_name": "Table Name", + "headers": ["Col1", "Col2"], + "rows": [["Val1", "Val2"]] + }} + ], + "all_raw_data": {{}} + }} +}} + +VERIFICATION CHECKLIST (self-check before output): +- [ ] Count of items in input `item_level_results` == Count of items in your output `data`? +- [ ] Are ALL t-values/F-values preserved and stripped of symbols? +- [ ] Did you avoid summarizing or skipping items? + +CRITICAL: Output ONLY valid JSON. Do NOT include comments, markdown code blocks, or any text outside the JSON object. + +Generate the JSON: +""" + try: + response = self.client.generate_content(prompt=prompt, temperature=0.2) + if response: + # Save raw response BEFORE parsing (in case parsing fails) + debug_path = (Path(debug_dir) if debug_dir else Path(f"data/studies/{study_id}")) / "ground_truth_raw_response.txt" + debug_path.parent.mkdir(parents=True, exist_ok=True) + debug_path.write_text(str(response), encoding='utf-8') + + parsed = self._parse_json_response(response) + if parsed and len(parsed) > 0: # Check for non-empty dict + return parsed + # If parsing failed, raise error with debug file location + raise ValueError(f"Failed to parse JSON response. Raw response saved to {debug_path}") + except ValueError: + raise # Re-raise ValueError (our custom error) + except Exception as e: + print(f"Error generating ground_truth with LLM: {e}") + raise + + def generate_materials( + self, + extraction_result: Dict[str, Any], + study_dir: Path, + pdf_path: Optional[Path] = None + ) -> List[Path]: + """ + Generate materials files using LLM-based dynamic function generation. + + This method uses LLM to generate a study-specific generate_materials function + that knows how to extract complete materials from the extraction_result. + + Args: + extraction_result: Results from stage2 extraction + study_dir: Study directory path + pdf_path: Optional path to PDF file for context + + Returns: + List of generated material file paths + """ + # Load ground_truth.json if it exists (to ensure label consistency) + ground_truth = None + gt_path = study_dir / "ground_truth.json" + if gt_path.exists(): + try: + with open(gt_path, 'r', encoding='utf-8') as f: + ground_truth = json.load(f) + except Exception: + pass # If loading fails, continue without it + + # Use LLM to generate a study-specific materials generation function + materials_generator_code = self._generate_materials_function_with_llm( + extraction_result, + pdf_path, + ground_truth=ground_truth + ) + + # Execute the generated function + materials_dir = study_dir / "materials" + materials_dir.mkdir(parents=True, exist_ok=True) + + # Create a namespace for executing the generated code + namespace = { + 'json': json, + 'Path': Path, + 'materials_dir': materials_dir, + 'extraction_result': extraction_result, + 'study_dir': study_dir + } + + try: + exec(materials_generator_code, namespace) + # Call the generated function + if 'generate_materials' in namespace: + generated_files = namespace['generate_materials'](extraction_result, materials_dir) + else: + generated_files = namespace.get('generated_files', []) + + # Ensure all items are Path objects + generated_files = [Path(f) if not isinstance(f, Path) else f for f in generated_files] + + if not generated_files: + print(f"Warning: Generated function returned no files, falling back to basic generation") + generated_files = self._generate_materials_basic(extraction_result, study_dir) + except Exception as e: + print(f"Warning: Error executing generated materials function: {e}") + import traceback + traceback.print_exc() + # Fallback to basic generation + generated_files = self._generate_materials_basic(extraction_result, study_dir) + + return generated_files + + def _generate_materials_function_with_llm( + self, + extraction_result: Dict[str, Any], + pdf_path: Optional[Path], + ground_truth: Optional[Dict[str, Any]] = None + ) -> str: + """ + Use LLM to generate a study-specific generate_materials function. + + Args: + extraction_result: Results from stage2 extraction + pdf_path: Optional path to PDF file + + Returns: + Python code string for the generated function + """ + extraction_summary = json.dumps(extraction_result, indent=2, ensure_ascii=False) + + # Include ground_truth in prompt if available (for label consistency) + gt_context = "" + if ground_truth: + # Extract all scenario/item keys from ground_truth for reference + # Group by study for better organization + gt_keys_by_study = {} + all_gt_keys = [] + + for study_gt in ground_truth.get("studies", []): + study_id = study_gt.get("study_id", "") + study_keys = [] + + for finding in study_gt.get("findings", []): + data_points = finding.get("original_data_points", {}).get("data", {}) + if data_points: + keys = list(data_points.keys()) + study_keys.extend(keys) + all_gt_keys.extend(keys) + + if study_keys: + gt_keys_by_study[study_id] = list(set(study_keys)) + + if all_gt_keys: + # Create a comprehensive list organized by study + gt_keys_list = [] + for study_id, keys in gt_keys_by_study.items(): + gt_keys_list.append(f" {study_id}: {keys}") + + gt_context = f""" +**CRITICAL: GROUND TRUTH KEYS (MUST MATCH EXACTLY)** + +The following keys are used in ground_truth.json. You MUST use these EXACT same keys as `metadata.label` for corresponding items: + +ALL GROUND TRUTH KEYS (organized by study): +{chr(10).join(gt_keys_list)} + +COMPLETE LIST OF ALL KEYS: +{json.dumps(sorted(set(all_gt_keys)), indent=2)} + +**MANDATORY REQUIREMENTS:** +1. For EVERY item in materials that corresponds to a key in the list above, you MUST set `metadata.label` to the EXACT key from the list. +2. If an item corresponds to a ground truth key, use that exact key. +3. If an item does NOT correspond to any ground truth key (e.g., background/filler items), you may omit the label or use a descriptive name, but prioritize matching the keys above. +4. The label matching is case-sensitive and must match exactly (after normalization). + +**EXAMPLE:** +- If ground truth has key "shy", then materials item should have `"metadata": {{"label": "shy"}}` +- If ground truth has key "hometown_gt_200k", then materials item should have `"metadata": {{"label": "hometown_gt_200k"}}` +- DO NOT use "Shy (not shy)" or "Hometown > 200k" - use the EXACT key from the list above. + +""" + + # Build prompt + # Ensure ground truth context is included BEFORE truncating extraction_summary + # Truncate extraction_summary to leave room for gt_context + max_extraction_length = 100000 # Leave room for gt_context + truncated_extraction = extraction_summary[:max_extraction_length] + if len(extraction_summary) > max_extraction_length: + truncated_extraction += "...\n[Truncated for length]" + + prompt = f"""You are a Python code generator. Generate a complete `generate_materials` function that extracts materials for a SIMULATION AGENT into a UNIFIED JSON format. + +{gt_context} +EXTRACTION RESULTS: +{truncated_extraction} +TASK: +Generate a Python function that creates standardized JSON files for each sub-study. +1. Each replicable sub-study should have EXACTLY ONE JSON file in the `materials/` directory named `{{sub_study_id}}.json`. +2. The JSON structure for each file MUST follow this schema: +{{ + "sub_study_id": "string (the unique ID for this condition/task)", + "instructions": "string (the FULL text of scenario or instructions given to the participant)", + "items": [ + {{ + "id": "string (item number/id)", + "question": "string (the actual question or stimulus text)", + "options": ["string"], // Optional: include if the question has fixed choices or Likert scale levels + "type": "string (e.g., 'multiple_choice', 'open_ended', 'estimation', 'likert')", + "metadata": {{ + "label": "string" // CRITICAL: If this item has a corresponding key in ground_truth.json, use the EXACT same key as the label + }} // Optional: any item-specific data (anchors, true values, etc.) + }} + ] +}} + +REQUIREMENTS: +- UNIFIED FORMAT: All materials MUST be JSON. No .txt files. +- STRICT DIRECTORY: You MUST ONLY write files to the `materials_dir` provided in the namespace. +- NO OVERWRITE: DO NOT attempt to modify or overwrite any files in `generation_pipeline/outputs/` or the parent directory. +- SELF-CONTAINED: Instructions MUST be complete. Reconstruct them if they refer to other experiments. +- ITEMS: If a sub-study has multiple questions/items (e.g., 15 estimation tasks), they MUST all be listed in the "items" array of that sub-study's JSON. +- If a sub-study is just a single scenario with one choice, the "items" array will have one object. +- **LABEL CONSISTENCY (CRITICAL)**: If ground_truth keys are provided above, you MUST use those EXACT keys as `metadata.label` for corresponding items. This ensures automatic matching between materials and ground_truth. + +EXAMPLE CODE STRUCTURE: +```python +def generate_materials(extraction_result, materials_dir): + from pathlib import Path + import json + + generated_files = [] + studies = extraction_result.get('studies', []) + + for study in studies: + for sub in study.get('sub_studies', []): + sub_id = sub.get('sub_study_id', 'unknown') + + # Combine everything into one JSON object per sub-study + material_data = {{ + "sub_study_id": sub_id, + "instructions": sub.get('content', ''), + "items": sub.get('items', []) + }} + + # Ensure items have standard structure (id, question, etc.) + # [Logic to transform sub.get('items') into the required schema goes here] + + file_path = materials_dir / f"{{sub_id}}.json" + file_path.write_text(json.dumps(material_data, indent=2, ensure_ascii=False), encoding='utf-8') + generated_files.append(file_path) + + return generated_files +``` + +OUTPUT: +Provide ONLY the complete Python function code. The function should be production-ready. +""" + + # Call LLM (with PDF if available) + try: + if pdf_path and pdf_path.exists(): + pdf_text = extract_pdf_text(pdf_path, max_chars=PDF_TEXT_MAX_CHARS) + full_prompt = f"PDF content:\n\n{pdf_text}\n\n{prompt}" + response = self.client.generate_content(prompt=full_prompt) + else: + response = self.client.generate_content(prompt=prompt) + except Exception as e: + raise RuntimeError(f"Error calling LLM API: {e}. Check API key and provider env.") + + if response is None: + raise ValueError("LLM returned None response. Check API key and network connection.") + + # Extract code from response + code = self._extract_code_from_response(response) + + return code + + def _extract_code_from_response(self, response: str) -> str: + """Extract Python code from LLM response""" + response_text = response.strip() + + # Remove markdown code blocks if present + if '```python' in response_text: + response_text = response_text.split('```python')[1].split('```')[0].strip() + elif '```' in response_text: + response_text = response_text.split('```')[1].split('```')[0].strip() + + return response_text + + def _generate_materials_basic( + self, + extraction_result: Dict[str, Any], + study_dir: Path + ) -> List[Path]: + """ + Basic fallback materials generation (unifying to JSON). + + Args: + extraction_result: Results from stage2 extraction + study_dir: Study directory path + + Returns: + List of generated material file paths + """ + materials_dir = study_dir / "materials" + materials_dir.mkdir(parents=True, exist_ok=True) + + generated_files = [] + studies = extraction_result.get('studies', []) + + for study in studies: + sub_studies = study.get('sub_studies', []) + + for sub_study in sub_studies: + sub_id = sub_study.get('sub_study_id', '') + content = sub_study.get('content', '') + items = sub_study.get('items', []) + + if not sub_id: + continue + + # Unify to JSON + material_data = { + "sub_study_id": sub_id, + "instructions": content, + "items": items + } + + file_path = materials_dir / f"{sub_id}.json" + file_path.write_text( + json.dumps(material_data, indent=2, ensure_ascii=False), + encoding='utf-8' + ) + generated_files.append(file_path) + + return generated_files + + def inject_gt_keys_into_materials( + self, + study_dir: Path, + material_files: List[Path] + ) -> Dict[str, Any]: + """ + Inject gt_key into materials items by matching against ground_truth.json. + + This method: + 1. Loads ground_truth.json + 2. Extracts all scenario/item keys from ground_truth + 3. Matches each item's label to a gt_key + 4. Writes gt_key to item.metadata.gt_key + 5. Reports coverage statistics + + Args: + study_dir: Study directory path + material_files: List of material JSON file paths + + Returns: + Dictionary with coverage statistics: + { + "sub_studies": { + "sub_study_id": { + "gt_keys_total": int, + "items_total": int, + "matched": int, + "missing": int, + "missing_items": [item_ids] + } + } + } + """ + gt_path = study_dir / "ground_truth.json" + if not gt_path.exists(): + print(f"Warning: ground_truth.json not found at {gt_path}. Skipping gt_key injection.") + return {} + + # Load ground truth + with open(gt_path, 'r', encoding='utf-8') as f: + ground_truth = json.load(f) + + # Extract all gt_keys organized by sub_study_id + # First, collect all materials sub_study_ids + materials_sub_ids = set() + for material_file in material_files: + if material_file.exists(): + with open(material_file, 'r', encoding='utf-8') as f: + material_data = json.load(f) + sub_id = material_data.get("sub_study_id", "") + if sub_id: + materials_sub_ids.add(sub_id) + + # Build mapping from study labels to sub_study_ids + # Common patterns for study_001: + # "Study 1" -> "study_1_hypothetical_stories" + # "Study 2" -> "study_2_personal_description_items" + # "Study 3" -> "study_3_sandwich_board_hypothetical" + gt_keys_by_substudy = {} + all_gt_keys = set() # Fallback: all keys if sub-study matching fails + + for study_gt in ground_truth.get("studies", []): + study_label = study_gt.get("study_id", "") + + # Extract keys from all findings + all_keys_in_study = [] + for finding in study_gt.get("findings", []): + data_points = finding.get("original_data_points", {}).get("data", {}) + if data_points: + all_keys_in_study.extend(list(data_points.keys())) + + all_gt_keys.update(all_keys_in_study) + + # Try to match study to sub_study_id using common patterns + matched_sub_ids = [] + study_num_match = re.search(r'Study\s+(\d+)', study_label) + + if study_num_match: + study_num = study_num_match.group(1) + # Try common patterns + patterns = [ + f"study_{study_num}_", + f"_study_{study_num}_", + f"study_{study_num}", + ] + for sub_id in materials_sub_ids: + if any(pattern in sub_id for pattern in patterns): + matched_sub_ids.append(sub_id) + + # Also try keyword matching (e.g., "hypothetical" in study_label -> "hypothetical_stories" in sub_id) + if not matched_sub_ids: + study_keywords = [w.lower() for w in study_label.split() if len(w) > 4] + for sub_id in materials_sub_ids: + sub_id_lower = sub_id.lower() + if any(kw in sub_id_lower for kw in study_keywords): + matched_sub_ids.append(sub_id) + + # If we found matches, assign keys to those sub_studies + if matched_sub_ids: + for sub_id in matched_sub_ids: + if sub_id not in gt_keys_by_substudy: + gt_keys_by_substudy[sub_id] = [] + gt_keys_by_substudy[sub_id].extend(all_keys_in_study) + + # Process each material file + stats = {} + total_matched = 0 + total_items = 0 + total_missing = 0 + + for material_file in material_files: + if not material_file.exists(): + continue + + # Load material file + with open(material_file, 'r', encoding='utf-8') as f: + material_data = json.load(f) + + sub_study_id = material_data.get("sub_study_id", "") + items = material_data.get("items", []) + + if not sub_study_id: + continue + + # Get gt_keys for this sub-study + gt_keys = gt_keys_by_substudy.get(sub_study_id, []) + + # If not found, try to use all keys (fallback) + if not gt_keys: + # Use all unique keys from ground_truth as fallback + gt_keys = list(all_gt_keys) + + # Initialize stats for this sub-study + if sub_study_id not in stats: + stats[sub_study_id] = { + "gt_keys_total": len(gt_keys), + "items_total": len(items), + "matched": 0, + "missing": 0, + "missing_items": [] + } + + # Process each item + for item in items: + total_items += 1 + item_id = item.get("id", "") + label = item.get("metadata", {}).get("label", "") + + # Skip items without labels (they won't be evaluated against ground_truth) + if not label: + continue + + # Try to find matching gt_key + gt_key = self._find_gt_key_for_label(label, gt_keys, item_id) + + if gt_key: + # Ensure metadata exists + if "metadata" not in item: + item["metadata"] = {} + item["metadata"]["gt_key"] = gt_key + stats[sub_study_id]["matched"] += 1 + total_matched += 1 + else: + # Item has label but no matching gt_key + # This could mean: + # 1. The item doesn't have a corresponding ground_truth entry (not all items are evaluated) + # 2. The label format doesn't match ground_truth keys + # We'll count it as missing but only fail if we have gt_keys available (meaning we should have matched) + stats[sub_study_id]["missing"] += 1 + total_missing += 1 + stats[sub_study_id]["missing_items"].append(item_id) + + # Save updated material file + with open(material_file, 'w', encoding='utf-8') as f: + json.dump(material_data, f, indent=2, ensure_ascii=False) + + # Print coverage report + print(f"\n📊 gt_key Coverage Report:") + print(f" Total items processed: {total_items}") + print(f" Matched: {total_matched}") + print(f" Missing: {total_missing}") + + for sub_id, sub_stats in stats.items(): + print(f"\n {sub_id}:") + print(f" Items: {sub_stats['items_total']}, GT Keys: {sub_stats['gt_keys_total']}") + print(f" Matched: {sub_stats['matched']}, Missing: {sub_stats['missing']}") + if sub_stats['missing'] > 0: + print(f" Missing items: {', '.join(sub_stats['missing_items'][:5])}") + if len(sub_stats['missing_items']) > 5: + print(f" ... and {len(sub_stats['missing_items']) - 5} more") + + # Fail only if we have gt_keys available but couldn't match a significant portion + # Note: Not all items may have corresponding ground_truth entries (e.g., Study 2 has 34 items but only 17 have stats) + # So we only fail if: + # 1. We have gt_keys available (meaning there should be matches) + # 2. The missing rate is very high (>80% of items with labels, or >50% of available gt_keys) + items_with_labels = sum(s["items_total"] for s in stats.values()) + items_with_gt_keys_available = sum(s["gt_keys_total"] for s in stats.values() if s["gt_keys_total"] > 0) + + if total_missing > 0 and items_with_gt_keys_available > 0: + # Calculate missing rate relative to items with labels + missing_rate_vs_items = total_missing / items_with_labels if items_with_labels > 0 else 0 + # Calculate missing rate relative to available gt_keys (more meaningful) + missing_rate_vs_keys = total_missing / items_with_gt_keys_available if items_with_gt_keys_available > 0 else 0 + + # Fail if: + # - More than 80% of items with labels are missing, OR + # - More than 50% of available gt_keys couldn't be matched (meaning we're missing most of the evaluable items) + if missing_rate_vs_items > 0.8 or missing_rate_vs_keys > 0.5: + raise ValueError( + f"gt_key injection failed: {total_missing} items could not be matched to ground_truth keys. " + f"Missing rate: {missing_rate_vs_items*100:.1f}% of items with labels, {missing_rate_vs_keys*100:.1f}% of available gt_keys. " + f"Please check the mapping between materials labels and ground_truth keys. " + f"Note: Some items may not have corresponding ground_truth entries (not all items are evaluated)." + ) + else: + print(f" ⚠️ Warning: {total_missing} items could not be matched. " + f"This may be expected if not all items have ground_truth entries " + f"(e.g., Study 2 has 34 items but only 17 have statistical results in ground_truth).") + + return stats + + def _find_gt_key_for_label( + self, + label: str, + gt_keys: List[str], + item_id: str = "" + ) -> Optional[str]: + """ + Find the best matching gt_key for a given label. + + Uses multi-step matching: + 1. Exact match (normalized) + 2. Normalized exact match + 3. Token-based similarity (Jaccard) + 4. Keyword matching + + Args: + label: Item label from materials + gt_keys: List of available ground truth keys + item_id: Optional item ID for debugging + + Returns: + Best matching gt_key or None + """ + if not label or not gt_keys: + return None + + # Step 1: Normalize label + normalized_label = self._normalize_label(label) + + # Step 2: Exact match (normalized) + if normalized_label in gt_keys: + return normalized_label + + # Step 3: Try normalized versions of gt_keys + normalized_gt_map = {self._normalize_label(k): k for k in gt_keys} + if normalized_label in normalized_gt_map: + return normalized_gt_map[normalized_label] + + # Step 4: Token-based similarity (Jaccard) + label_words = set(normalized_label.lower().split()) + best_match = None + best_score = 0.0 + + for gt_key in gt_keys: + gt_words = set(self._normalize_label(gt_key).lower().split()) + if not label_words or not gt_words: + continue + + intersection = len(label_words & gt_words) + union = len(label_words | gt_words) + score = intersection / union if union > 0 else 0.0 + + if score > best_score: + best_score = score + best_match = gt_key + + # Require high similarity (>0.7) for token-based match + if best_score > 0.7: + return best_match + + # Step 5: Keyword matching (fallback for complex cases) + # Check if main keywords from label are in gt_key + for gt_key in gt_keys: + gt_words = set(self._normalize_label(gt_key).lower().split()) + # If at least 2 significant words match + significant_words = {w for w in label_words if len(w) > 3} + if len(significant_words & gt_words) >= 2: + return gt_key + + return None + + def _normalize_label(self, label: str) -> str: + """ + Normalize label for matching: + - Remove parentheses and their content + - Remove common suffixes (yes/no, on list provided, etc.) + - Normalize special symbols (more than -> >, less than -> <) + - Clean whitespace + """ + if not label: + return "" + + # Remove parentheses and their content + normalized = re.sub(r"\s*\([^)]*\)\s*", " ", label) + + # Remove common suffixes + normalized = re.sub(r"\s+(yes|no|on list provided|don't|do not).*$", "", normalized, flags=re.IGNORECASE) + + # Normalize special symbols + normalized = re.sub(r"\s+more\s+than\s+", " > ", normalized, flags=re.IGNORECASE) + normalized = re.sub(r"\s+less\s+than\s+", " < ", normalized, flags=re.IGNORECASE) + + # Clean whitespace + normalized = re.sub(r"\s+", " ", normalized).strip() + + return normalized diff --git a/generation_pipeline/utils/osf_crawler.py b/generation_pipeline/utils/osf_crawler.py new file mode 100644 index 0000000..c6200f4 --- /dev/null +++ b/generation_pipeline/utils/osf_crawler.py @@ -0,0 +1,544 @@ +""" +OSF crawler for ai-ethics stage4 corpus. + +Scans per-paper JSON files, finds effects whose experiment materials live on OSF +(not in the paper PDF), resolves OSF node/registration URLs, and downloads +matching supplementary files. +""" + +from __future__ import annotations + +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Iterable + +OSF_API = "https://api.osf.io/v2" +OSF_URL_RE = re.compile( + r"https?://(?:www\.)?osf\.io/(?P[a-z0-9]{5})/?(?:\?[^\s\"']*)?", + re.IGNORECASE, +) +STUDY_NUM_RE = re.compile(r"(?:study|experiment|exp\.?)\s*([0-9]+[a-z]?)", re.IGNORECASE) + +MATERIAL_KEYWORDS = ( + "manipulation", + "material", + "materials", + "stimulus", + "stimuli", + "scale", + "survey", + "questionnaire", + "items", + "instrument", + "measure", + "appendix", + "supplement", + "supplementary", + "instruction", + "scenario", + "script", + "vignette", + "task", + "protocol", +) + +SLOTS = ("materials", "manipulation", "items") +NEEDS_OSF_STATUSES = {"osf_only", "not_in_paper", "cited_scale"} + + +@dataclass +class EffectNeed: + study: str + study_key: str | None + effect_index: int + slot: str + status: str + iv: str + dv: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass +class OsfFile: + name: str + kind: str + path: str + size: int | None + download_url: str | None + api_url: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass +class PaperOsfPlan: + paper_folder: str + json_path: str + paper_title: str + osf_url: str + osf_id: str + view_only: str | None + entity_type: str | None = None + entity_title: str | None = None + effects_needing_osf: list[EffectNeed] = field(default_factory=list) + matched_files: list[OsfFile] = field(default_factory=list) + downloaded_files: list[str] = field(default_factory=list) + skipped_files: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + @property + def can_download(self) -> bool: + return bool(self.osf_id) + + @property + def needs_download(self) -> bool: + """True when missing materials AND an OSF id is present.""" + return bool(self.effects_needing_osf and self.osf_id) + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["effects_needing_osf"] = [e.to_dict() if isinstance(e, EffectNeed) else e for e in self.effects_needing_osf] + d["matched_files"] = [f.to_dict() if isinstance(f, OsfFile) else f for f in self.matched_files] + return d + + +class OsfClient: + """Minimal OSF API v2 client (public + view_only; optional bearer token).""" + + def __init__( + self, + token: str | None = None, + view_only: str | None = None, + delay_s: float = 0.15, + timeout_s: float = 120, + ): + self.token = token + self.view_only = view_only + self.delay_s = delay_s + self.timeout_s = timeout_s + + def _request(self, url: str) -> dict[str, Any]: + time.sleep(self.delay_s) + parsed = urllib.parse.urlparse(url) + query = dict(urllib.parse.parse_qsl(parsed.query)) + if self.view_only and "view_only" not in query: + query["view_only"] = self.view_only + url = urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode(query))) + + headers = {"Accept": "application/vnd.api+json"} + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=self.timeout_s) as resp: + return json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"OSF API {e.code} for {url}: {body[:300]}") from e + + def download(self, url: str, dest: Path) -> None: + time.sleep(self.delay_s) + parsed = urllib.parse.urlparse(url) + query = dict(urllib.parse.parse_qsl(parsed.query)) + if self.view_only and "view_only" not in query: + query["view_only"] = self.view_only + url = urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode(query))) + + headers: dict[str, str] = {} + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + + req = urllib.request.Request(url, headers=headers) + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_suffix(dest.suffix + ".tmp") + try: + with urllib.request.urlopen(req, timeout=self.timeout_s) as resp: + tmp.write_bytes(resp.read()) + tmp.replace(dest) + except Exception: + if tmp.exists(): + tmp.unlink(missing_ok=True) + raise + + def resolve_entity(self, osf_id: str) -> tuple[str, str, str]: + """Return (entity_type, title, files_index_url).""" + for entity_type in ("nodes", "registrations"): + url = f"{OSF_API}/{entity_type}/{osf_id}/" + try: + payload = self._request(url) + except RuntimeError as e: + if "404" in str(e): + continue + raise + data = payload.get("data") + if not data: + continue + title = data.get("attributes", {}).get("title", osf_id) + files_url = data.get("relationships", {}).get("files", {}).get("links", {}).get("related", {}).get("href") + if not files_url: + raise RuntimeError(f"No files relationship for {entity_type}/{osf_id}") + return entity_type, title, files_url + raise RuntimeError(f"Could not resolve OSF id '{osf_id}' as node or registration") + + def list_files_recursive(self, provider_url: str, prefix: str = "") -> list[OsfFile]: + payload = self._request(provider_url) + out: list[OsfFile] = [] + for item in payload.get("data", []): + attrs = item.get("attributes", {}) + name = attrs.get("name", "unknown") + rel_path = f"{prefix}/{name}".lstrip("/") + kind = attrs.get("kind", "file") + download_url = item.get("links", {}).get("download") + out.append( + OsfFile( + name=name, + kind=kind, + path=rel_path, + size=attrs.get("size"), + download_url=download_url if kind == "file" else None, + api_url=item.get("links", {}).get("info") or provider_url, + ) + ) + if kind == "folder": + child_url = item.get("relationships", {}).get("files", {}).get("links", {}).get("related", {}).get("href") + if child_url: + out.extend(self.list_files_recursive(child_url, rel_path)) + return out + + +def parse_osf_url(url: str) -> tuple[str, str | None]: + """Extract (osf_id, view_only_token) from an OSF URL.""" + if not url: + raise ValueError("Empty OSF URL") + m = OSF_URL_RE.search(url) + if not m: + raise ValueError(f"Not an OSF URL: {url}") + parsed = urllib.parse.urlparse(url) + view_only = urllib.parse.parse_qs(parsed.query).get("view_only", [None])[0] + return m.group("id").lower(), view_only + + +def extract_study_key(study_name: str) -> str | None: + m = STUDY_NUM_RE.search(study_name or "") + return m.group(1).lower() if m else None + + +def find_paper_jsons(stage4_dir: Path) -> list[Path]: + """Find main extraction JSON for each paper (supports flat or runnable/non_runnable layout).""" + jsons: list[Path] = [] + seen: set[str] = set() + + def consider(folder: Path) -> None: + if folder.name in {"runnable", "non_runnable", "replicable", "non_replicable"}: + return + candidates = [ + p for p in folder.glob("*.json") + if p.name not in {"sum.json", "osf_manifest.json"} + and "manifest" not in p.name.lower() + ] + if candidates and folder.name not in seen: + jsons.append(candidates[0]) + seen.add(folder.name) + + for folder in sorted(stage4_dir.iterdir()): + if not folder.is_dir(): + continue + if folder.name in {"runnable", "non_runnable"}: + for sub in sorted(folder.iterdir()): + if sub.is_dir(): + consider(sub) + else: + consider(folder) + return jsons + + +def analyze_paper_needs(data: dict[str, Any]) -> tuple[str | None, str | None, list[EffectNeed]]: + """Return (osf_url, view_only, effects_needing_osf).""" + meta = data.get("paper_metadata") or {} + link = meta.get("link") or "" + osf_url: str | None = None + view_only: str | None = None + if "osf.io" in link.lower(): + osf_url = link + try: + _, view_only = parse_osf_url(link) + except ValueError: + pass + + needs: list[EffectNeed] = [] + for study in data.get("eligible_studies", []): + study_name = study.get("study", "") + study_key = extract_study_key(study_name) + for ei, effect in enumerate(study.get("effects", [])): + notes = (effect.get("materials_notes") or "").lower() + notes_osf = "osf" in notes or "open science framework" in notes + for slot in SLOTS: + slot_obj = effect.get(slot) or {} + status = slot_obj.get("status") + content = slot_obj.get("content") + missing = content is None or (isinstance(content, str) and not content.strip()) + if status == "osf_only" or (status in NEEDS_OSF_STATUSES and missing) or (notes_osf and missing): + needs.append( + EffectNeed( + study=study_name, + study_key=study_key, + effect_index=ei, + slot=slot, + status=status or "unknown", + iv=effect.get("IV", "") or "", + dv=effect.get("DV", "") or "", + ) + ) + return osf_url, view_only, needs + + +def build_paper_plan(json_path: Path) -> PaperOsfPlan: + data = json.loads(json_path.read_text(encoding="utf-8")) + folder = json_path.parent.name + osf_url, view_only, needs = analyze_paper_needs(data) + + osf_id = "" + if osf_url: + try: + osf_id, parsed_view = parse_osf_url(osf_url) + view_only = view_only or parsed_view + except ValueError: + pass + + # Only download when experiments actually reference missing OSF materials. + if not needs: + osf_url = osf_url or "" + elif not osf_id: + return PaperOsfPlan( + paper_folder=folder, + json_path=str(json_path), + paper_title=data.get("paper_title", ""), + osf_url=osf_url or "", + osf_id="", + view_only=view_only, + effects_needing_osf=needs, + errors=["Paper has effects needing OSF materials but no osf.io link in paper_metadata.link"], + ) + + return PaperOsfPlan( + paper_folder=folder, + json_path=str(json_path), + paper_title=data.get("paper_title", ""), + osf_url=osf_url or "", + osf_id=osf_id, + view_only=view_only, + effects_needing_osf=needs, + ) + + +def _study_matches_path(study_key: str | None, study_name: str, path_lower: str) -> bool: + if study_key and re.search(rf"(?:study|experiment|exp\.?)\s*{re.escape(study_key)}\b", path_lower): + return True + slug = re.sub(r"[^a-z0-9]+", " ", (study_name or "").lower()).strip() + tokens = [t for t in slug.split() if len(t) > 3 and t not in {"study", "experiment"}] + return any(t in path_lower for t in tokens[:3]) + + +def score_file_relevance(path_lower: str, needs: list[EffectNeed]) -> float: + score = 0.0 + for need in needs: + if _study_matches_path(need.study_key, need.study, path_lower): + score += 5.0 + for kw in MATERIAL_KEYWORDS: + if kw in path_lower: + score += 1.0 + # Deprioritize raw data / syntax unless nothing else matches. + if any(x in path_lower for x in ("data", "syntax", "output", ".sav", ".r", ".spss")): + score -= 1.5 + if path_lower.endswith((".pdf", ".docx", ".doc", ".txt", ".html", ".htm")): + score += 0.5 + return score + + +HIGH_VALUE_KEYWORDS = ( + "appendix", + "supplement", + "supplementary", + "manipulation", + "material", + "scale", + "survey", + "questionnaire", + "instrument", + "measure", + "items", + "stimulus", + "scenario", + "instruction", +) + + +def select_files(all_files: list[OsfFile], needs: list[EffectNeed], download_all: bool = False) -> list[OsfFile]: + files = [f for f in all_files if f.kind == "file" and f.download_url] + if download_all: + return files + + selected: dict[str, OsfFile] = {} + scored: list[tuple[float, OsfFile]] = [] + for f in files: + path_lower = f.path.lower() + s = score_file_relevance(path_lower, needs) + if s > 0: + scored.append((s, f)) + if any(kw in path_lower for kw in HIGH_VALUE_KEYWORDS): + selected[f.path] = f + + if scored: + scored.sort(key=lambda x: (-x[0], x[1].path)) + max_score = scored[0][0] + threshold = max(1.0, max_score - 2.0) + for s, f in scored: + if s >= threshold: + selected[f.path] = f + + if selected: + return [selected[k] for k in sorted(selected.keys())] + + # Fallback: if we know materials are missing, grab everything except obvious data dumps. + fallback = [ + f for f in files + if not any(x in f.path.lower() for x in (".sav", ".csv", ".rdata", "syntax", "prereg")) + ] + return fallback or files + + +def fetch_osf_for_plan( + plan: PaperOsfPlan, + output_root: Path | None = None, + token: str | None = None, + download_all: bool = False, + dry_run: bool = False, +) -> PaperOsfPlan: + if not plan.can_download: + return plan + + # JSON already complete → grab full OSF storage for archival. + if not plan.effects_needing_osf: + download_all = True + + out_dir = (output_root or Path(plan.json_path).parent) / "osf" + files_dir = out_dir / "files" + client = OsfClient(token=token, view_only=plan.view_only) + + try: + entity_type, title, files_index = client.resolve_entity(plan.osf_id) + plan.entity_type = entity_type + plan.entity_title = title + + providers = client._request(files_index).get("data", []) + all_files: list[OsfFile] = [] + for provider in providers: + provider_files_url = ( + provider.get("relationships", {}) + .get("files", {}) + .get("links", {}) + .get("related", {}) + .get("href") + ) + if provider_files_url: + all_files.extend(client.list_files_recursive(provider_files_url)) + + plan.matched_files = select_files(all_files, plan.effects_needing_osf, download_all=download_all) + if not plan.matched_files: + if not all_files: + plan.errors.append("OSF project has no files in storage") + else: + plan.errors.append("No OSF files matched selection criteria") + return plan + + for f in plan.matched_files: + dest = files_dir / f.path + if dry_run: + plan.skipped_files.append(str(dest)) + continue + try: + client.download(f.download_url, dest) + plan.downloaded_files.append(str(dest)) + except Exception as e: + plan.errors.append(f"Failed to download {f.path}: {e}") + + if not dry_run: + manifest_path = out_dir / "osf_manifest.json" + manifest_path.write_text( + json.dumps( + {"plan": plan.to_dict(), "all_osf_files": [x.to_dict() for x in all_files]}, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + except Exception as e: + plan.errors.append(str(e)) + + return plan + + +def scan_stage4( + stage4_dir: Path, + only: Iterable[str] | None = None, + *, + all_linked: bool = False, + missing_only: bool = False, +) -> list[PaperOsfPlan]: + plans: list[PaperOsfPlan] = [] + for json_path in find_paper_jsons(stage4_dir): + if only and not any(sub in json_path.as_posix() for sub in only): + continue + plan = build_paper_plan(json_path) + if not plan.osf_id: + continue + if missing_only and not plan.effects_needing_osf: + continue + if all_linked or plan.effects_needing_osf: + plans.append(plan) + return plans + + +def run_batch( + stage4_dir: Path, + only: list[str] | None = None, + token: str | None = None, + download_all: bool = False, + dry_run: bool = False, + list_only: bool = False, + *, + all_linked: bool = True, + missing_only: bool = False, +) -> list[PaperOsfPlan]: + plans = scan_stage4( + stage4_dir, + only=only, + all_linked=all_linked, + missing_only=missing_only, + ) + if list_only: + return plans + results: list[PaperOsfPlan] = [] + for plan in plans: + if not plan.osf_id: + results.append(plan) + continue + results.append( + fetch_osf_for_plan( + plan, + token=token, + download_all=download_all, + dry_run=dry_run, + ) + ) + return results diff --git a/generation_pipeline/utils/osf_fill.py b/generation_pipeline/utils/osf_fill.py new file mode 100644 index 0000000..6d90965 --- /dev/null +++ b/generation_pipeline/utils/osf_fill.py @@ -0,0 +1,218 @@ +""" +Fill missing materials/manipulation/items slots from downloaded OSF files. + +Rule-based extraction (no LLM): maps OSF file text to JSON slots by study + DV. +""" + +from __future__ import annotations + +import json +import re +import shutil +import zipfile +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any + +from generation_pipeline.utils.osf_crawler import ( + NEEDS_OSF_STATUSES, + SLOTS, + build_paper_plan, + find_paper_jsons, +) +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + +# --- Pre-extracted scale blocks (from OSF Online Appendix / Qualtrics) --- + +THEBENEFIT_ITEMS = { + "moral rumination": """Rumination (Trapnell & Campbell, 1999; adapted for moral topics at work; Study 3, OSF Online Appendix): +• I ruminate about the morality of my behaviors. +• I spend a great amount of time thinking about whether my behavior at work is moral or not. +• I ruminate or dwell on whether I did the right or wrong thing at work.""", + "cognitive flexibility": """Cognitive Flexibility (Martin, 1995; Study 3, OSF Online Appendix): +At work: +• I consider alternatives when dealing with work problems. +• I can find workable solutions to seemingly unsolvable problems. +• I have the self-confidence necessary to try different ways of behaving.""", + "creativity": """Creativity (Zhou & George, 2001; Study 3, OSF Online Appendix): +Supervisor/rater items: +• Comes up with creative solutions to problems. +• Often has new and innovative ideas. +• Often has a fresh approach to problems. +(Study 3: creativity was based on trained raters' evaluations of participants' three park ideas.)""", +} + +THEPROBABI_LONGTERMISM_ITEMS = """Longtermism belief items (Study 4; OSF Qualtrics .qsf): +• We should act wisely because what we do today will influence an untold number of people in the future. +• It is important to consider the long-term consequences of our actions and decisions. +• Intergenerational cooperation is important for addressing long-term challenges. +• It is important that we reduce existential and extinction risks to humanity and promote sustainable development goals to ensure the long-term survival of future generations. +• We should always have in view not only the present but also future generations. +• There are things we can do to steer the long-term future to a better course. +• Positively influencing the long-term future is a key moral priority of our time. +(Each item was rated for multiple future timeframes: 1,000 / 10,000 / 100,000 years in the future.)""" + +THEPROBABI_REFORM_ITEMS = """Policy support measure (Study 4; OSF Qualtrics .qsf): +On a scale of 0–100, how much do you support reform for your country's legal system to protect the welfare (broadly understood as the rights, interests, and/or well-being) of the following groups? +• Humans living in the present +• Non-human animals +• Environment (e.g., rivers, trees or nature itself)""" + + +def _docx_text(path: Path) -> str: + with zipfile.ZipFile(path) as z: + root = ET.fromstring(z.read("word/document.xml")) + return "\n".join(t.text for t in root.iter() if t.tag.endswith("}t") and t.text) + + +def _load_osf_texts(osf_dir: Path) -> dict[str, str]: + """Map relative path -> extracted text.""" + out: dict[str, str] = {} + files_dir = osf_dir / "files" + if not files_dir.exists(): + return out + for path in files_dir.rglob("*"): + if not path.is_file(): + continue + rel = str(path.relative_to(files_dir)) + try: + if path.suffix.lower() == ".pdf": + out[rel] = extract_pdf_text(path, max_chars=80000) + elif path.suffix.lower() == ".docx": + out[rel] = _docx_text(path) + elif path.suffix.lower() in (".txt", ".html", ".htm"): + out[rel] = path.read_text(encoding="utf-8", errors="replace")[:80000] + elif path.suffix.lower() == ".qsf": + out[rel] = path.read_text(encoding="utf-8", errors="replace")[:200000] + except Exception as e: + out[rel] = f"[extract error: {e}]" + return out + + +def _match_items_by_dv(dv: str, paper_folder: str) -> str | None: + dv_l = (dv or "").lower() + if paper_folder == "2023_TheBenefit": + for key, block in THEBENEFIT_ITEMS.items(): + if key in dv_l: + return block + if paper_folder == "2026_TheProbabi": + if "longtermism" in dv_l and "reform" not in dv_l and "support" not in dv_l: + return THEPROBABI_LONGTERMISM_ITEMS + if "reform" in dv_l or "support" in dv_l or "policy" in dv_l: + return THEPROBABI_REFORM_ITEMS + return None + + +def _normalize_docx_text(raw: str) -> str: + """Collapse broken line breaks from DOCX extraction.""" + return re.sub(r"\s+", " ", raw).strip() + + +def _fill_leadersand(data: dict[str, Any], osf_texts: dict[str, str]) -> list[str]: + """Upgrade Study 3 slots from Study 3 Manipulations.docx.""" + changes: list[str] = [] + manip_path = next((k for k in osf_texts if "manipulation" in k.lower()), None) + if not manip_path: + return ["No Study 3 Manipulations file found on OSF"] + + raw = _normalize_docx_text(osf_texts[manip_path]) + + # Fast-food misconduct scenario (materials) + scenario = None + if "McDonald" in raw and "Charlie" in raw: + start = raw.find("McDonald") + end = raw.find("three times", start) + if end > start: + scenario = raw[start : end + len("three times")] + + # Full cognitive-load + typing instructions (manipulation) + manip_block = raw if "Cognitive Load" in raw else None + + for study in data.get("eligible_studies", []): + if "study 3" not in (study.get("study") or "").lower(): + continue + for effect in study.get("effects", []): + if scenario: + effect["materials"] = {"status": "verbatim", "content": scenario} + if "Study 3 materials" not in str(changes): + changes.append("Study 3 materials → verbatim from OSF") + + if manip_block: + effect["manipulation"] = {"status": "verbatim", "content": manip_block} + if "Study 3 manipulation" not in str(changes): + changes.append("Study 3 manipulation → verbatim from OSF") + + slot = effect.get("items", {}) + if slot.get("content") is None and slot.get("status") in NEEDS_OSF_STATUSES: + if "items still missing" not in str(changes): + changes.append( + "Study 3 items still missing (punishment scales not in OSF download)" + ) + + return changes + + +def fill_paper_json(json_path: Path, dry_run: bool = False) -> dict[str, Any]: + plan = build_paper_plan(json_path) + if not plan.osf_id: + return {"paper": plan.paper_folder, "skipped": "no OSF link"} + + osf_dir = json_path.parent / "osf" + if not (osf_dir / "files").exists(): + return {"paper": plan.paper_folder, "skipped": "no osf/files/ — run fetch_osf.py first"} + + data = json.loads(json_path.read_text(encoding="utf-8")) + osf_texts = _load_osf_texts(osf_dir) + changes: list[str] = [] + + if plan.paper_folder == "2023_Leadersand": + changes.extend(_fill_leadersand(data, osf_texts)) + + for study in data.get("eligible_studies", []): + for effect in study.get("effects", []): + for slot_name in SLOTS: + slot = effect.get(slot_name) or {} + status = slot.get("status") + content = slot.get("content") + missing = content is None or (isinstance(content, str) and not content.strip()) + if status not in NEEDS_OSF_STATUSES | {"osf_only"}: + continue + if not missing: + continue + + if slot_name == "items": + block = _match_items_by_dv(effect.get("DV", ""), plan.paper_folder) + if block: + effect["items"] = { + "status": "verbatim", + "content": block, + } + changes.append( + f"{study.get('study')} items filled for DV={effect.get('DV', '')[:40]}" + ) + + if changes and not dry_run: + bak = json_path.with_suffix(json_path.suffix + ".bak") + if not bak.exists(): + shutil.copy2(json_path, bak) + json_path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + return { + "paper": plan.paper_folder, + "changes": changes, + "dry_run": dry_run, + } + + +def run_fill(stage4_dir: Path, only: list[str] | None = None, dry_run: bool = False) -> list[dict]: + results = [] + for jp in find_paper_jsons(stage4_dir): + plan = build_paper_plan(jp) + if not plan.osf_id: + continue + if only and not any(s in jp.as_posix() for s in only): + continue + if not plan.effects_needing_osf: + continue + results.append(fill_paper_json(jp, dry_run=dry_run)) + return results diff --git a/generation_pipeline/utils/output_formatter.py b/generation_pipeline/utils/output_formatter.py new file mode 100644 index 0000000..231cc11 --- /dev/null +++ b/generation_pipeline/utils/output_formatter.py @@ -0,0 +1,314 @@ +""" +Output Formatter (ai-ethics): formats extraction results as markdown review files. +""" + +import json +from typing import Any, Dict + + +class OutputFormatter: + + @staticmethod + def format_stage1_review(filter_result: Dict[str, Any]) -> str: + md = f"""# Stage 1: Replicability Filter Review + +## Paper Information +- **Title**: {filter_result.get('paper_title', 'N/A')} +- **Authors**: {', '.join(filter_result.get('paper_authors', []))} +- **Abstract**: {filter_result.get('paper_abstract', 'N/A')} + +## Experiments Overview + +""" + quality = filter_result.get("stage1_quality") if isinstance(filter_result.get("stage1_quality"), dict) else {} + if quality: + md += f"""## Quality Audit +- **Experiments total**: {quality.get('experiments_total', 0)} +- **Eligible / uncertain**: {quality.get('eligible_or_uncertain', 0)} +- **Missing materials count**: {quality.get('missing_materials_count', 0)} +- **Study contract ready / needs review**: {quality.get('study_contract_ready', 'N/A')} / {quality.get('study_contract_needs_review', 'N/A')} +- **Study contract blocking issues**: {quality.get('study_contract_blocking_issues', 'N/A')} +- **Verifier status / overall**: {quality.get('verifier_status', 'N/A')} / {quality.get('verifier_overall', 'N/A')} +- **Verifier suggestions**: {quality.get('verifier_suggestions', 0)} +- **Needs human review**: {quality.get('needs_human_review', False)} + +""" + for issue in quality.get("issues", []): + if isinstance(issue, dict): + md += f"- `{issue.get('severity', 'warning')}` {issue.get('path', '$')}: {issue.get('message', '')}\n" + if quality.get("issues"): + md += "\n" + + contract = filter_result.get("stage1_study_contract") if isinstance(filter_result.get("stage1_study_contract"), dict) else {} + if contract: + md += f"""## Stage 1 Study Contract +- **Ready studies**: {contract.get('ready', 0)}/{contract.get('total_studies', 0)} +- **Eligible / uncertain**: {contract.get('eligible_or_uncertain', 0)} +- **Needs review**: {contract.get('needs_review', 0)} +- **Missing by field**: `{json.dumps(contract.get('missing_by_field', {}), ensure_ascii=False)}` + +""" + + refinement = filter_result.get("stage1_refinement") if isinstance(filter_result.get("stage1_refinement"), dict) else {} + history = filter_result.get("stage1_refinement_history") + history = history if isinstance(history, list) else [] + if refinement or history: + md += f"""## Auto-Refinement +- **Attempts used**: {refinement.get('attempts', len(history))} +- **Max attempts**: {refinement.get('max_attempts', 'N/A')} +- **Final verifier overall**: {refinement.get('final_verifier_overall', 'N/A')} + +""" + for item in history: + if not isinstance(item, dict): + continue + regen = item.get("regeneration_instructions") if isinstance(item.get("regeneration_instructions"), dict) else {} + md += ( + f"- Attempt {item.get('attempt')}: trigger={item.get('trigger')} " + f"verifier={item.get('verifier_status')}/{item.get('verifier_overall')} " + f"feedback_items={sum(len(v) for v in regen.values() if isinstance(v, list))}\n" + ) + md += "\n" + + verifier = filter_result.get("stage1_verification") if isinstance(filter_result.get("stage1_verification"), dict) else {} + if verifier: + md += f"""## Verifier Report +- **Status**: {verifier.get('status', 'N/A')} +- **Overall**: {verifier.get('overall', 'N/A')} +- **Confidence**: {verifier.get('confidence', 'N/A')} +- **Notes**: {verifier.get('notes', '')} + +""" + for check in verifier.get("inventory_checks", []): + if isinstance(check, dict): + md += ( + f"- Inventory `{check.get('study', 'N/A')}`: " + f"{check.get('verdict', 'needs_review')} - {check.get('issue', '')} " + f"{check.get('evidence', '')}\n" + ) + for check in verifier.get("eligibility_checks", []): + if isinstance(check, dict): + md += ( + f"- Eligibility `{check.get('study', 'N/A')}`: " + f"{check.get('verdict', 'needs_review')} " + f"expected={check.get('expected_label', 'N/A')} - {check.get('issue', '')} " + f"{check.get('evidence', '')}\n" + ) + if verifier.get("inventory_checks") or verifier.get("eligibility_checks"): + md += "\n" + + for i, exp in enumerate(filter_result.get("experiments", []), 1): + source_hints = exp.get("candidate_source_hints") if isinstance(exp.get("candidate_source_hints"), list) else [] + md += f"""### Experiment {i}: {exp.get('experiment_name', exp.get('experiment_id', 'Unknown'))} +- **Study ID**: {exp.get('study_id', 'N/A')} +- **Design type**: {exp.get('design_type', 'N/A')} +- **Conditions / factors**: {_format_stage1_conditions(exp.get('conditions_or_factors'))} +- **Input**: {exp.get('input', 'N/A')} +- **Participant task**: {exp.get('participant_task', 'N/A')} +- **Participants**: {exp.get('participants', 'N/A')} +- **Output**: {exp.get('output', 'N/A')} +- **Candidate source hints**: `{json.dumps(source_hints, ensure_ascii=False)}` +- **Replicable**: {exp.get('replicable', 'UNCERTAIN')} +- **Self-contained materials**: {exp.get('has_self_contained_materials', 'N/A')} +- **Missing materials**: {exp.get('missing_materials', '')} +- **Exclusion reasons**: {', '.join(exp.get('exclusion_reasons', [])) or 'None'} + +#### Checklist: +- [ ] Outcome is moral / ethical / prosocial +- [ ] Has quantitative statistic +- [ ] Sample / design recoverable +- [ ] Stimuli / scales accounted for (in-paper, OSF, or cited) + +#### Comments: +[填写] + +--- + +""" + md += f"""## Overall Assessment +- **Overall Replicable**: {'YES' if filter_result.get('overall_replicable', False) else 'NO'} +- **Confidence**: {filter_result.get('confidence', 0.0)} +- **Notes**: {filter_result.get('notes', 'N/A')} + +## Review Status +- **Reviewed By**: [填写] +- **Review Status**: [pending/approved/needs_refinement] +- **Action**: [continue_to_stage2/refine_stage1/exclude] +""" + return md + + @staticmethod + def format_stage2_review(extraction: Dict[str, Any]) -> str: + if not isinstance(extraction, dict): + raise ValueError(f"extraction is not a dict: {type(extraction)}") + + md = f"""# Stage 2: Ethics Effect Extraction Review + +## Paper +- **Title**: {extraction.get('paper_title', 'N/A')} +- **Authors**: {', '.join((extraction.get('paper_metadata') or {}).get('authors', []))} +- **Year**: {(extraction.get('paper_metadata') or {}).get('year', 'N/A')} +- **Journal**: {(extraction.get('paper_metadata') or {}).get('journal', 'N/A')} +- **DOI**: {(extraction.get('paper_metadata') or {}).get('doi', 'N/A')} + +""" + quality = extraction.get("stage2_quality") if isinstance(extraction.get("stage2_quality"), dict) else {} + if quality: + coverage = quality.get("stage1_stage2_coverage") if isinstance(quality.get("stage1_stage2_coverage"), dict) else {} + md += f"""## Quality Audit +- **Studies total**: {quality.get('studies_total', 0)} +- **Effects total**: {quality.get('effects_total', 0)} +- **Consolidated findings total**: {quality.get('findings_total', 0)} +- **Primary finding candidates**: {quality.get('primary_finding_candidates', 0)} +- **Effects with reported stats**: {quality.get('effects_with_reported_stats', 0)} +- **Legacy per-effect material slots observed**: {quality.get('legacy_effect_material_slot_hints', 0)} +- **Finding contract ready / needs review**: {quality.get('finding_contract_ready', 'N/A')} / {quality.get('finding_contract_needs_review', 'N/A')} +- **Finding contract blocking issues**: {quality.get('finding_contract_blocking_issues', 'N/A')} +- **Verifier status / overall**: {quality.get('verifier_status', 'N/A')} / {quality.get('verifier_overall', 'N/A')} +- **Verifier suggestions**: {quality.get('verifier_suggestions', 0)} +- **Stage1 eligible / uncertain**: {coverage.get('stage1_eligible_or_uncertain', 'N/A')} +- **Missing from Stage2**: {', '.join(coverage.get('missing_from_stage2', [])) or 'None'} +- **Extra in Stage2**: {', '.join(coverage.get('extra_in_stage2', [])) or 'None'} +- **Needs human review**: {quality.get('needs_human_review', False)} + +""" + for issue in quality.get("issues", []): + if isinstance(issue, dict): + md += f"- `{issue.get('severity', 'warning')}` {issue.get('path', '$')}: {issue.get('message', '')}\n" + if quality.get("issues"): + md += "\n" + legacy_details = [ + item + for item in quality.get("legacy_effect_material_slot_details", []) or [] + if isinstance(item, dict) + ] + if legacy_details: + md += "### Legacy Effect Material Slot Locations\n" + for item in legacy_details[:12]: + md += ( + f"- `{item.get('path', '$')}` " + f"{item.get('study', '')} effect={item.get('effect_index')} " + f"statuses=`{json.dumps(item.get('statuses', {}), ensure_ascii=False)}`\n" + ) + if len(legacy_details) > 12: + md += f"- ... {len(legacy_details) - 12} more in `stage2_quality.legacy_effect_material_slot_details`\n" + md += "\n" + + contract = extraction.get("stage2_finding_contract") if isinstance(extraction.get("stage2_finding_contract"), dict) else {} + if contract: + md += f"""## Stage 2 Finding Contract +- **Ready studies**: {contract.get('ready', 0)}/{contract.get('total_studies', 0)} +- **Needs review**: {contract.get('needs_review', 0)} +- **Total findings**: {contract.get('total_findings', 0)} +- **Primary simulation targets**: {contract.get('primary_simulation_targets', 0)} +- **Missing by field**: `{json.dumps(contract.get('missing_by_field', {}), ensure_ascii=False)}` + +""" + + refinement = extraction.get("stage2_refinement") if isinstance(extraction.get("stage2_refinement"), dict) else {} + history = extraction.get("stage2_refinement_history") + history = history if isinstance(history, list) else [] + if refinement or history: + md += f"""## Auto-Refinement +- **Attempts used**: {refinement.get('attempts', len(history))} +- **Max attempts**: {refinement.get('max_attempts', 'N/A')} +- **Final verifier overall**: {refinement.get('final_verifier_overall', 'N/A')} + +""" + for item in history: + if not isinstance(item, dict): + continue + regen = item.get("regeneration_instructions") if isinstance(item.get("regeneration_instructions"), dict) else {} + md += ( + f"- Attempt {item.get('attempt')}: trigger={item.get('trigger')} " + f"verifier={item.get('verifier_status')}/{item.get('verifier_overall')} " + f"feedback_items={sum(len(v) for v in regen.values() if isinstance(v, list))}\n" + ) + md += "\n" + + verification = extraction.get("stage2_verification") if isinstance(extraction.get("stage2_verification"), dict) else {} + if verification: + md += f"""## Verifier Report +- **Status**: {verification.get('status', 'N/A')} +- **Overall**: {verification.get('overall', 'N/A')} +- **Confidence**: {verification.get('confidence', 'N/A')} +- **Notes**: {verification.get('notes', '')} + +""" + for item in verification.get("study_coverage", []) or []: + if isinstance(item, dict): + md += ( + f"- Study `{item.get('study')}`: {item.get('verdict')} " + f"issue={item.get('issue', '')} evidence={item.get('evidence', '')}\n" + ) + for item in verification.get("finding_checks", []) or []: + if isinstance(item, dict): + md += ( + f"- Finding `{item.get('finding_id')}`: {item.get('verdict')} " + f"issue={item.get('issue', '')} evidence={item.get('evidence', '')}\n" + ) + md += "\n" + + for study in extraction.get("eligible_studies", []): + sample = study.get("sample") + sample_text = json.dumps(sample, ensure_ascii=False) if sample is not None else "N/A" + md += f"""## {study.get('study', 'Unknown')} +- **Eligibility rationale**: {study.get('eligibility_rationale', '')} +- **Study sample**: `{sample_text}` + +""" + findings = [finding for finding in study.get("findings", []) or [] if isinstance(finding, dict)] + if findings: + md += "### Consolidated Findings\n" + for finding in findings: + target = finding.get("simulation_target") if isinstance(finding.get("simulation_target"), dict) else {} + md += ( + f"- **{finding.get('finding_id')}** " + f"role={finding.get('role')} target={target.get('candidate')} " + f"effect={finding.get('representative_effect_index')} " + f"IV={finding.get('IV')} -> DV={finding.get('DV')} " + f"stats={finding.get('reported_statistics')}\n" + ) + md += "\n" + + for j, eff in enumerate(study.get("effects", []), 1): + stats = eff.get("stats", {}) or {} + md += f"""### Effect {j} +- **Platform**: {eff.get('platform', 'N/A')} +- **IV → DV**: {eff.get('IV', '?')} → {eff.get('DV', '?')} +- **Effect type / direction**: {eff.get('effecttype', '?')} / {eff.get('direction', '?')} +- **N**: {eff.get('size', '?')} +- **Analysis N / scope**: {eff.get('analysis_n', None)} / {eff.get('analysis_scope', None)} +- **Stats**: B={stats.get('B')} t={stats.get('t')} F={stats.get('f')} eta²={stats.get('eta_square')} p={stats.get('p_value')} ({stats.get('sig')}) +- **CI**: {stats.get('ci')} +- **Location**: {eff.get('table_or_page_location', 'N/A')} +- **Notes**: {eff.get('materials_notes', '')} + +#### Checklist: +- [ ] Stats numbers match paper +- [ ] N matches paper +- [ ] Finding role / simulation-target mapping is correct +- [ ] Material search should use Stage3 study-level evidence, not this effect row + +#### Comments: +[填写] + +""" + md += "---\n\n" + + md += """## Review Status +- **Reviewed By**: [填写] +- **Review Status**: [pending/approved/needs_refinement] +- **Action**: [accept/refine_stage2/back_to_stage1] +""" + return md + + +def _format_stage1_conditions(value: Any) -> str: + if value is None: + return "N/A" + if isinstance(value, list): + parts = [str(item).strip() for item in value if str(item).strip()] + return "; ".join(parts) if parts else "N/A" + text = str(value).strip() + return text or "N/A" diff --git a/generation_pipeline/utils/pdf_chunker.py b/generation_pipeline/utils/pdf_chunker.py new file mode 100644 index 0000000..c865481 --- /dev/null +++ b/generation_pipeline/utils/pdf_chunker.py @@ -0,0 +1,309 @@ +from __future__ import annotations + +import re +import sys +import unicodedata +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Optional + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from generation_pipeline.utils.pdf_extractor import extract_pdf_text + +PDF_TEXT_MAX_CHARS = 400000 +DEFAULT_CHUNK_SIZE = 2500 +DEFAULT_OVERLAP = 600 +_PAGE_MARKER = re.compile(r"---\s*Page\s+(\d+)\s*---", re.IGNORECASE) +_TOKEN = re.compile(r"[a-z0-9]+") +_STOPWORDS = frozenset( + """ + the a an and or of to in on for with without that this these those is are was were be been being + by as at from into over under between within across about not no yes it its their our your his her + they we you he she them us i me my study studies effect effects paper participants subject subjects + """.split() +) + + +@dataclass +class Chunk: + """A contiguous slice of text with page and optional source provenance.""" + + id: int + text: str + page_start: int + page_end: int + char_start: int + source_path: str | None = None + source_kind: str | None = None + study_keys: tuple[str, ...] = () + + def to_dict(self) -> dict: + return { + "id": self.id, + "page_start": self.page_start, + "page_end": self.page_end, + "char_start": self.char_start, + "source_path": self.source_path, + "source_kind": self.source_kind, + "study_keys": list(self.study_keys), + "text": self.text, + } + + +def parse_pages( + pdf_path: Path, + *, + max_chars: Optional[int] = PDF_TEXT_MAX_CHARS, +) -> list[tuple[int, str]]: + """Extract text and split into (page_no, page_text) using the page markers.""" + raw = extract_pdf_text(Path(pdf_path), max_chars=max_chars) + return split_pages(raw) + + +def split_pages(raw_text: str) -> list[tuple[int, str]]: + """Split text produced by extract_pdf_text into (page_no, text) pairs.""" + if not raw_text: + return [] + parts = _PAGE_MARKER.split(raw_text) + pages: list[tuple[int, str]] = [] + # parts = [pre, page_no, body, page_no, body, ...] + if parts and parts[0].strip() and not _PAGE_MARKER.match("--- Page 0 ---"): + # leading text before first marker (rare); attach as page 0 + lead = parts[0].strip() + if lead: + pages.append((0, lead)) + it = iter(parts[1:]) + for page_no, body in zip(it, it): + try: + num = int(page_no) + except (TypeError, ValueError): + num = len(pages) + 1 + pages.append((num, body)) + if not pages and raw_text.strip(): + pages.append((1, raw_text)) + return pages + + +def clean_text(text: str) -> str: + if not text: + return "" + text = unicodedata.normalize("NFKC", text) + text = ( + text.replace("‘", "'") + .replace("’", "'") + .replace("“", '"') + .replace("”", '"') + .replace("fi", "fi") + .replace("fl", "fl") + ) + text = re.sub(r"(\w)-\s*\n\s*(\w)", r"\1\2", text) # de-hyphenate across line breaks + text = re.sub(r"[ \t]*\n[ \t]*", " ", text) + text = re.sub(r"\s+", " ", text) + return text.strip() + + +def build_chunks( + pages: list[tuple[int, str]], + *, + chunk_size: int = DEFAULT_CHUNK_SIZE, + overlap: int = DEFAULT_OVERLAP, + source_path: str | None = None, + source_kind: str | None = None, + study_keys: Iterable[str] = (), +) -> list[Chunk]: + """Concatenate cleaned page text and slice into overlapping windows. + + Each chunk records the page range it spans so evidence can be cited. + """ + if chunk_size <= 0: + raise ValueError("chunk_size must be positive") + overlap = max(0, min(overlap, chunk_size - 1)) + + # Build one cleaned string + a per-character page map. + segments: list[str] = [] + page_marks: list[tuple[int, int]] = [] # (char_offset_in_joined, page_no) + offset = 0 + for page_no, body in pages: + cleaned = clean_text(body) + if not cleaned: + continue + if segments: + segments.append(" ") + offset += 1 + page_marks.append((offset, page_no)) + segments.append(cleaned) + offset += len(cleaned) + joined = "".join(segments) + if not joined: + return [] + + def page_at(char_index: int) -> int: + page = page_marks[0][1] if page_marks else 1 + for mark_offset, page_no in page_marks: + if mark_offset <= char_index: + page = page_no + else: + break + return page + + chunks: list[Chunk] = [] + step = max(1, chunk_size - overlap) + cid = 0 + for start in range(0, len(joined), step): + end = min(start + chunk_size, len(joined)) + body = joined[start:end].strip() + if body: + chunks.append( + Chunk( + id=cid, + text=body, + page_start=page_at(start), + page_end=page_at(end - 1), + char_start=start, + source_path=source_path, + source_kind=source_kind, + study_keys=tuple(study_keys), + ) + ) + cid += 1 + if end >= len(joined): + break + return chunks + + +def chunk_pdf( + pdf_path: Path, + *, + chunk_size: int = DEFAULT_CHUNK_SIZE, + overlap: int = DEFAULT_OVERLAP, + max_chars: Optional[int] = PDF_TEXT_MAX_CHARS, +) -> list[Chunk]: + """Convenience: parse + chunk a PDF in one call.""" + return build_chunks(parse_pages(pdf_path, max_chars=max_chars), chunk_size=chunk_size, overlap=overlap) + + +def _tokens(text: str) -> list[str]: + return _TOKEN.findall(unicodedata.normalize("NFKC", text or "").lower()) + + +def _content_tokens(text: str, *, min_len: int = 3) -> set[str]: + return {t for t in _tokens(text) if len(t) >= min_len and t not in _STOPWORDS} + + +def score_chunk(query: str, chunk_text: str, *, keywords: Iterable[str] = ()) -> float: + """Lexical relevance score for a chunk against a query. + + coverage of distinct query content-tokens (0..1) + bonus for slot keywords, + with a small fuzzy tie-breaker. Deterministic; no embeddings. + """ + q_tokens = _content_tokens(query) + if not q_tokens: + return 0.0 + chunk_tokens = set(_tokens(chunk_text)) + coverage = len(q_tokens & chunk_tokens) / len(q_tokens) + + low = chunk_text.lower() + kw_hits = sum(1 for kw in keywords if kw.lower() in low) + kw_bonus = min(kw_hits, 5) * 0.15 + + fuzzy = _fuzzy_ratio(query.lower(), low) / 100.0 * 0.1 + return coverage + kw_bonus + fuzzy + + +def retrieve( + chunks: list[Chunk], + query: str, + *, + k: int = 8, + keywords: Iterable[str] = (), + allowed_sources: set[str] | None = None, + use_bm25: bool = True, +) -> list[tuple[Chunk, float]]: + """Return the top-k chunks for a query, highest score first.""" + keywords = list(keywords) + if allowed_sources is not None: + chunks = [chunk for chunk in chunks if chunk.source_path in allowed_sources] + if use_bm25: + scored = _bm25_score_chunks(query, chunks, keywords=keywords) + else: + scored = [(chunk, score_chunk(query, chunk.text, keywords=keywords)) for chunk in chunks] + scored.sort(key=lambda item: item[1], reverse=True) + return [item for item in scored[:k] if item[1] > 0.0] + + +def format_evidence(scored_chunks: list[tuple[Chunk, float]]) -> str: + """Render retrieved chunks as a numbered evidence block for an LLM prompt.""" + blocks = [] + for chunk, score in scored_chunks: + pages = ( + f"p.{chunk.page_start}" + if chunk.page_start == chunk.page_end + else f"pp.{chunk.page_start}-{chunk.page_end}" + ) + source = f" | source={chunk.source_path}" if chunk.source_path else "" + blocks.append(f"[Evidence {chunk.id} | {pages}{source} | relevance={score:.2f}]\n{chunk.text}") + return "\n\n".join(blocks) + + +def _bm25_score_chunks( + query: str, + chunks: list[Chunk], + *, + keywords: Iterable[str] = (), + k1: float = 1.5, + b: float = 0.75, +) -> list[tuple[Chunk, float]]: + """Pure-Python BM25 over the candidate chunks, with existing keyword/fuzzy tiebreaks.""" + if not chunks: + return [] + query_tokens = [t for t in _tokens(query) if len(t) >= 3 and t not in _STOPWORDS] + if not query_tokens: + return [] + + docs = [[t for t in _tokens(chunk.text) if len(t) >= 3 and t not in _STOPWORDS] for chunk in chunks] + avg_len = sum(len(doc) for doc in docs) / max(len(docs), 1) + df: dict[str, int] = {} + for doc in docs: + for token in set(doc): + df[token] = df.get(token, 0) + 1 + + import math + + n_docs = len(docs) + scores: list[tuple[Chunk, float]] = [] + for chunk, doc in zip(chunks, docs): + if not doc: + continue + tf: dict[str, int] = {} + for token in doc: + tf[token] = tf.get(token, 0) + 1 + bm25 = 0.0 + doc_len = len(doc) + for token in query_tokens: + freq = tf.get(token, 0) + if not freq: + continue + idf = math.log(1 + (n_docs - df.get(token, 0) + 0.5) / (df.get(token, 0) + 0.5)) + denom = freq + k1 * (1 - b + b * doc_len / max(avg_len, 1)) + bm25 += idf * (freq * (k1 + 1)) / max(denom, 1e-9) + + low = chunk.text.lower() + kw_hits = sum(1 for kw in keywords if kw.lower() in low) + kw_bonus = min(kw_hits, 5) * 0.15 + fuzzy = _fuzzy_ratio(query.lower(), low) / 100.0 * 0.1 + scores.append((chunk, bm25 + kw_bonus + fuzzy)) + return scores + + +def _fuzzy_ratio(needle: str, haystack: str) -> float: + if not needle or not haystack: + return 0.0 + try: + from rapidfuzz import fuzz + + return float(fuzz.partial_ratio(needle, haystack)) + except ImportError: + import difflib + + return difflib.SequenceMatcher(None, needle, haystack[: max(len(needle) * 4, 1000)]).ratio() * 100.0 diff --git a/generation_pipeline/utils/pdf_extractor.py b/generation_pipeline/utils/pdf_extractor.py new file mode 100644 index 0000000..604ba60 --- /dev/null +++ b/generation_pipeline/utils/pdf_extractor.py @@ -0,0 +1,221 @@ +""" +PDF text extraction for multi-provider pipeline (no upload_file dependency). +""" + +from __future__ import annotations + +import io +import shutil +import subprocess +import tempfile +import warnings +from pathlib import Path +from typing import Optional + + +DEFAULT_OCR_MIN_CHARS = 800 + + +class OCRUnavailableError(RuntimeError): + """Raised internally when optional OCR dependencies are unavailable.""" + + +def extract_pdf_text( + pdf_path: Path, + max_chars: Optional[int] = None, + *, + ocr_fallback: bool = True, + ocr_min_chars: int = DEFAULT_OCR_MIN_CHARS, +) -> str: + """ + Extract text from PDF with page markers. + + Uses pypdf first. If the extracted text is too short or appears garbled, + tries optional OCR fallbacks (`ocrmypdf`, then `pytesseract` with PyMuPDF or + pdf2image). If OCR dependencies are absent, returns the pypdf text and emits + a warning rather than failing the pipeline. + + Args: + pdf_path: Path to PDF file + max_chars: If set, truncate to this many characters (for context limits) + + Returns: + Text with "--- Page N ---" markers between pages + """ + pdf_path = Path(pdf_path) + if not pdf_path.exists(): + raise FileNotFoundError(f"PDF not found: {pdf_path}") + + pypdf_text = "" + pypdf_error: Exception | None = None + try: + pypdf_text = _extract_with_pypdf(pdf_path, max_chars=max_chars) + except ImportError as exc: + pypdf_error = exc + + if pypdf_text and not _needs_ocr(pypdf_text, max_chars=max_chars, min_chars=ocr_min_chars): + return pypdf_text + + if ocr_fallback: + try: + ocr_text = _extract_with_ocr(pdf_path, max_chars=max_chars) + if ocr_text.strip(): + return ocr_text + except OCRUnavailableError as exc: + if pypdf_text: + warnings.warn( + f"PDF text looked unreliable for {pdf_path}; OCR fallback unavailable: {exc}", + RuntimeWarning, + ) + return pypdf_text + if pypdf_error is not None: + raise ImportError( + "pypdf is required for PDF extraction unless OCR dependencies are installed. " + "Install pypdf or install ocrmypdf/pytesseract OCR support." + ) from pypdf_error + raise + except Exception as exc: + if pypdf_text: + warnings.warn( + f"PDF text looked unreliable for {pdf_path}; OCR fallback failed: {exc}", + RuntimeWarning, + ) + return pypdf_text + raise + + if pypdf_text: + return pypdf_text + if pypdf_error is not None: + raise ImportError("pypdf required for PDF text extraction. Install with: pip install pypdf") from pypdf_error + return "" + + +def _extract_with_pypdf(pdf_path: Path, max_chars: Optional[int] = None) -> str: + try: + from pypdf import PdfReader + except ImportError: + try: + from PyPDF2 import PdfReader # type: ignore + except ImportError as exc: + raise ImportError("pypdf required for PDF text extraction. Install with: pip install pypdf") from exc + + reader = PdfReader(str(pdf_path)) + parts = [] + total = 0 + for i, page in enumerate(reader.pages, 1): + text = page.extract_text() or "" + block = f"--- Page {i} ---\n{text}" + if max_chars and total + len(block) > max_chars: + remaining = max_chars - total - 50 + if remaining > 0: + block = block[:remaining] + "\n[... truncated]" + parts.append(block) + break + parts.append(block) + total += len(block) + return "\n\n".join(parts) + + +def _needs_ocr(text: str, *, max_chars: Optional[int], min_chars: int) -> bool: + content = text.replace("\n", " ") + content = " ".join(part for part in content.split() if not part.startswith("---")) + stripped = content.strip() + if not stripped: + return True + threshold = min_chars if max_chars is None else min(min_chars, max_chars) + if len(stripped) < threshold: + return True + + replacement_ratio = stripped.count("\ufffd") / max(len(stripped), 1) + control_count = sum(1 for ch in stripped if ord(ch) < 32 and ch not in "\n\t\r") + control_ratio = control_count / max(len(stripped), 1) + alpha_num_ratio = sum(1 for ch in stripped if ch.isalnum()) / max(len(stripped), 1) + return replacement_ratio > 0.01 or control_ratio > 0.02 or alpha_num_ratio < 0.20 + + +def _extract_with_ocr(pdf_path: Path, max_chars: Optional[int]) -> str: + errors: list[str] = [] + try: + return _extract_with_ocrmypdf(pdf_path, max_chars=max_chars) + except OCRUnavailableError as exc: + errors.append(str(exc)) + except Exception as exc: + errors.append(f"ocrmypdf failed: {exc}") + + try: + return _extract_with_pytesseract(pdf_path, max_chars=max_chars) + except OCRUnavailableError as exc: + errors.append(str(exc)) + + raise OCRUnavailableError("; ".join(errors) if errors else "no OCR backend available") + + +def _extract_with_ocrmypdf(pdf_path: Path, max_chars: Optional[int]) -> str: + if shutil.which("ocrmypdf") is None: + raise OCRUnavailableError("ocrmypdf command not found") + + with tempfile.TemporaryDirectory() as tmpdir: + out_pdf = Path(tmpdir) / "ocr.pdf" + cmd = ["ocrmypdf", "--force-ocr", "--quiet", str(pdf_path), str(out_pdf)] + subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return _extract_with_pypdf(out_pdf, max_chars=max_chars) + + +def _extract_with_pytesseract(pdf_path: Path, max_chars: Optional[int]) -> str: + try: + import pytesseract + except ImportError as exc: + raise OCRUnavailableError("pytesseract package not installed") from exc + + try: + return _extract_with_pymupdf(pdf_path, pytesseract, max_chars=max_chars) + except OCRUnavailableError: + return _extract_with_pdf2image(pdf_path, pytesseract, max_chars=max_chars) + + +def _extract_with_pymupdf(pdf_path: Path, pytesseract, max_chars: Optional[int]) -> str: + try: + import fitz # PyMuPDF + from PIL import Image + except ImportError as exc: + raise OCRUnavailableError("PyMuPDF/Pillow not installed for pytesseract PDF rendering") from exc + + doc = fitz.open(str(pdf_path)) + parts: list[str] = [] + total = 0 + for index, page in enumerate(doc, 1): + pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False) + image = Image.open(io.BytesIO(pix.tobytes("png"))) + text = pytesseract.image_to_string(image) + block = f"--- Page {index} ---\n{text}" + if max_chars and total + len(block) > max_chars: + remaining = max_chars - total - 50 + if remaining > 0: + block = block[:remaining] + "\n[... truncated]" + parts.append(block) + break + parts.append(block) + total += len(block) + return "\n\n".join(parts) + + +def _extract_with_pdf2image(pdf_path: Path, pytesseract, max_chars: Optional[int]) -> str: + try: + from pdf2image import convert_from_path + except ImportError as exc: + raise OCRUnavailableError("pdf2image not installed for pytesseract PDF rendering") from exc + + parts: list[str] = [] + total = 0 + for index, image in enumerate(convert_from_path(str(pdf_path), dpi=200), 1): + text = pytesseract.image_to_string(image) + block = f"--- Page {index} ---\n{text}" + if max_chars and total + len(block) > max_chars: + remaining = max_chars - total - 50 + if remaining > 0: + block = block[:remaining] + "\n[... truncated]" + parts.append(block) + break + parts.append(block) + total += len(block) + return "\n\n".join(parts) diff --git a/generation_pipeline/utils/review_parser.py b/generation_pipeline/utils/review_parser.py new file mode 100644 index 0000000..87fdfb0 --- /dev/null +++ b/generation_pipeline/utils/review_parser.py @@ -0,0 +1,134 @@ +""" +Review Parser - Parses markdown review files to extract checklist and comments +""" + +import re +from pathlib import Path +from typing import Dict, Any, List, Optional + + +class ReviewParser: + """Parse markdown review files to extract human feedback""" + + @staticmethod + def parse(md_file: Path) -> Dict[str, Any]: + """ + Parse markdown review file. + + Args: + md_file: Path to markdown review file + + Returns: + Dictionary with parsed information: + { + "checklists": [...], # List of checklist items with status + "comments": {...}, # Comments by section + "review_status": {...} # Review status information + } + """ + if not md_file.exists(): + raise FileNotFoundError(f"Review file not found: {md_file}") + + content = md_file.read_text(encoding='utf-8') + + return { + "checklists": ReviewParser._extract_checklists(content), + "comments": ReviewParser._extract_comments(content), + "review_status": ReviewParser._extract_review_status(content) + } + + @staticmethod + def _extract_checklists(content: str) -> List[Dict[str, Any]]: + """Extract checklist items from markdown""" + checklists = [] + + # Pattern: - [x] or - [ ] followed by text + pattern = r'- \[([ x])\] (.+)' + + for match in re.finditer(pattern, content): + checked = match.group(1).strip() == 'x' + text = match.group(2).strip() + + # Find which section this checklist belongs to + section = ReviewParser._find_section(content, match.start()) + + checklists.append({ + "checked": checked, + "text": text, + "section": section + }) + + return checklists + + @staticmethod + def _extract_comments(content: str) -> Dict[str, str]: + """Extract comments from #### Comments: sections""" + comments = {} + + # Pattern: #### Comments: followed by content until next #### or --- + pattern = r'#### Comments:\s*\n(.*?)(?=\n####|\n---|$)' + + for match in re.finditer(pattern, content, re.DOTALL): + comment_text = match.group(1).strip() + + # Find which section this comment belongs to + section = ReviewParser._find_section(content, match.start()) + comments[section] = comment_text + + return comments + + @staticmethod + def _extract_review_status(content: str) -> Dict[str, str]: + """Extract review status section""" + status = {} + + # Look for Review Status section + status_match = re.search(r'## Review Status\s*\n(.*?)(?=\n##|$)', content, re.DOTALL) + if status_match: + status_text = status_match.group(1) + + # Extract key-value pairs + for line in status_text.split('\n'): + if ':' in line: + key, value = line.split(':', 1) + key = key.strip().strip('-*').strip() + value = value.strip() + if key and value: + status[key.lower().replace(' ', '_')] = value + + return status + + @staticmethod + def _find_section(content: str, position: int) -> str: + """Find which section a position belongs to""" + # Find the last ### or ## before this position + before_content = content[:position] + + # Find last section header + section_match = re.findall(r'^### (.+)$', before_content, re.MULTILINE) + if section_match: + return section_match[-1] + + section_match = re.findall(r'^## (.+)$', before_content, re.MULTILINE) + if section_match: + return section_match[-1] + + return "unknown" + + @staticmethod + def get_action(review_status: Dict[str, str]) -> Optional[str]: + """Extract action from review status""" + action = review_status.get('action', '').lower() + + if 'continue' in action or 'stage2' in action: + return 'continue_to_stage2' + elif 'refine' in action: + return 'refine' + elif 'exclude' in action: + return 'exclude' + elif 'back' in action: + return 'back_to_stage1' + elif 'final' in action: + return 'continue_to_final' + + return None diff --git a/generation_pipeline/verification/__init__.py b/generation_pipeline/verification/__init__.py new file mode 100644 index 0000000..5844094 --- /dev/null +++ b/generation_pipeline/verification/__init__.py @@ -0,0 +1,9 @@ +"""Verification helpers for Stage 2/3 outputs and Stage 4 gating.""" + +from generation_pipeline.verification.schema_validator import ( + SchemaValidationReport, + validate_paper, +) +from generation_pipeline.verification.verbatim_verifier import verify_paper + +__all__ = ["SchemaValidationReport", "validate_paper", "verify_paper"] diff --git a/generation_pipeline/verification/schema_validator.py b/generation_pipeline/verification/schema_validator.py new file mode 100644 index 0000000..212cb38 --- /dev/null +++ b/generation_pipeline/verification/schema_validator.py @@ -0,0 +1,1083 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import shutil +import sys +from copy import deepcopy +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + + +SLOT_NAMES = ("materials", "manipulation", "items") +# "source_missing": external sources (OSF, etc.) were fetched and searched, but +# the material for this slot is genuinely not present in them — distinct from +# "osf_only" (believed to be on OSF, not yet fetched) and "not_in_paper". +VALID_SLOT_STATUSES = {"verbatim", "paraphrased", "cited_scale", "osf_only", "not_in_paper", "source_missing"} +VALID_SIG = {"sig", "ns", "marginal"} +VALID_DIRECTION = {"pos", "neg"} +VALID_EFFECT_TYPES = {"main", "int", "simple", "mediation", "correlation"} + +STATS_NUMERIC_FIELDS = ("B", "b", "chi_square", "D", "eta_square", "f", "t", "z") +EFFECT_NUMERIC_FIELDS = ("mean_group1", "sd_group1", "mean_group2", "sd_group2") +EFFECT_REQUIRED_FIELDS = ( + # NOTE: "sample" is no longer an effect-level field — it lives at study level. + "platform", + "effecttype", + "IV", + "DV", + "size", + "direction", + "mean_group1", + "sd_group1", + "mean_group2", + "sd_group2", + "stats", + "materials_notes", + "table_or_page_location", + "materials", + "manipulation", + "items", +) + +# Controlled vocabulary for sample.platform. +# Any of these values (plus "Other") are accepted without further normalization. +# The verifier knows platform-specific aliases (e.g. "MTurk" → "mechanical turk"). +SAMPLE_PLATFORM_VOCAB = frozenset({ + "MTurk", "Prolific", "CloudResearch", "Undergraduate", "Graduate", + "Lab", "Organizational", "Online", "Field", "Archival", "Mixed", "Other", +}) + +# Controlled vocabulary for effect.analysis_scope. +# Describes the scope of the participants used in a specific effect analysis, +# which may differ from the study-level total_n. +VALID_ANALYSIS_SCOPE = frozenset({ + "full_sample", # all recruited participants were included + "subgroup", # a subset based on demographic/screening criterion + "condition", # participants in a specific condition only + "cell", # a single experimental cell (e.g., 2×2 design cell) + "simple_effect", # simple effect analysis within a level of a factor + "other", # any other analytical subset +}) + +STATS_REQUIRED_FIELDS = (*STATS_NUMERIC_FIELDS, "ci", "p_value", "sig") + +NULL_STRINGS = {"", "null", "none", "n/a", "na", "nan", "not reported", "not_reported", "-"} +_MISSING = object() + + +@dataclass +class ValidationIssue: + path: str + message: str + severity: str = "warning" + fixed: bool = False + before: Any = _MISSING + after: Any = _MISSING + + def to_dict(self) -> dict[str, Any]: + out: dict[str, Any] = { + "path": self.path, + "message": self.message, + "severity": self.severity, + "fixed": self.fixed, + } + if self.before is not _MISSING: + out["before"] = self.before + if self.after is not _MISSING: + out["after"] = self.after + return out + + +class SchemaValidationReport: + """JSON-serializable schema validation report.""" + + def __init__(self, *, path: str | None = None): + self.path = path + self.issues: list[ValidationIssue] = [] + self.changed = False + + @property + def valid(self) -> bool: + return not any(issue.severity == "error" and not issue.fixed for issue in self.issues) + + @property + def error_count(self) -> int: + return sum(1 for issue in self.issues if issue.severity == "error" and not issue.fixed) + + @property + def warning_count(self) -> int: + return sum(1 for issue in self.issues if issue.severity == "warning" and not issue.fixed) + + @property + def fixed_count(self) -> int: + return sum(1 for issue in self.issues if issue.fixed) + + def add( + self, + path: str, + message: str, + *, + severity: str = "warning", + fixed: bool = False, + before: Any = _MISSING, + after: Any = _MISSING, + ) -> None: + self.issues.append( + ValidationIssue( + path=path, + message=message, + severity=severity, + fixed=fixed, + before=_safe_value(before), + after=_safe_value(after), + ) + ) + if fixed: + self.changed = True + + def to_dict(self) -> dict[str, Any]: + return { + "path": self.path, + "valid": self.valid, + "changed": self.changed, + "summary": { + "errors": self.error_count, + "warnings": self.warning_count, + "fixed": self.fixed_count, + "issues": len(self.issues), + }, + "issues": [issue.to_dict() for issue in self.issues], + } + + +def validate_paper( + paper: dict[str, Any], + *, + repair: bool = True, + path: str | Path | None = None, +) -> tuple[dict[str, Any], SchemaValidationReport]: + """ + Validate and optionally normalize a Stage 2/3 paper JSON object. + + Returns a deep-copied paper object plus a report. Deterministic repairs are + applied only to the returned object, never to the caller's input. + """ + report = SchemaValidationReport(path=str(path) if path is not None else None) + data = deepcopy(paper) + + if not isinstance(data, dict): + report.add("$", f"paper JSON must be an object, got {type(data).__name__}", severity="error") + return paper, report + + _ensure_key(data, "paper_title", None, "$.paper_title", report, repair) + _ensure_key(data, "paper_metadata", {}, "$.paper_metadata", report, repair) + _ensure_key(data, "eligible_studies", [], "$.eligible_studies", report, repair) + + metadata = data.get("paper_metadata") + if not isinstance(metadata, dict): + if repair: + report.add( + "$.paper_metadata", + "paper_metadata must be an object; replaced with empty metadata", + fixed=True, + before=metadata, + after={}, + ) + metadata = {} + data["paper_metadata"] = metadata + else: + report.add("$.paper_metadata", "paper_metadata must be an object", severity="error", before=metadata) + metadata = {} + + _normalize_metadata(metadata, report, repair) + + studies = data.get("eligible_studies") + if not isinstance(studies, list): + report.add("$.eligible_studies", "eligible_studies must be a list", severity="error", before=studies) + return data, report + + for study_index, study in enumerate(studies): + study_path = f"$.eligible_studies[{study_index}]" + if not isinstance(study, dict): + report.add(study_path, "study entry must be an object", severity="error", before=study) + continue + _normalize_study(study, study_path, report, repair) + + return data, report + + +def validate_file( + path: Path, + *, + repair: bool = True, + write: bool = False, + backup: bool = True, +) -> tuple[dict[str, Any] | None, SchemaValidationReport]: + """Validate one JSON file and optionally write deterministic repairs.""" + try: + data = json.loads(Path(path).read_text(encoding="utf-8")) + except Exception as exc: + report = SchemaValidationReport(path=str(path)) + report.add("$", f"could not load JSON: {exc}", severity="error") + return None, report + + if not _looks_like_paper_json(data): + report = SchemaValidationReport(path=str(path)) + report.add("$", "skipped: JSON does not contain eligible_studies", severity="warning") + return data, report + + repaired, report = validate_paper(data, repair=repair, path=path) + if write and report.valid and report.changed: + _write_json_atomic(Path(path), repaired, backup=backup) + return repaired, report + + +def normalize_stage4_tree( + stage4_dir: Path, + *, + write: bool = False, + backup: bool = True, + report_path: Path | None = None, +) -> dict[str, Any]: + """Scan stage4/**/*.json, repair paper JSONs in memory, and write a report.""" + stage4_dir = Path(stage4_dir) + reports: list[dict[str, Any]] = [] + skipped: list[dict[str, Any]] = [] + + for json_path in sorted(stage4_dir.glob("**/*.json")): + rel = str(json_path.relative_to(stage4_dir)) + try: + data = json.loads(json_path.read_text(encoding="utf-8")) + except Exception as exc: + report = SchemaValidationReport(path=rel) + report.add("$", f"could not load JSON: {exc}", severity="error") + reports.append(report.to_dict()) + continue + + if not _looks_like_paper_json(data): + skipped.append({"path": rel, "reason": "not a per-paper corpus JSON"}) + continue + + repaired, report = validate_paper(data, repair=True, path=rel) + if write and report.valid and report.changed: + _write_json_atomic(json_path, repaired, backup=backup) + reports.append(report.to_dict()) + + aggregate = { + "stage4_dir": str(stage4_dir), + "write": write, + "summary": { + "paper_jsons": len(reports), + "skipped": len(skipped), + "changed": sum(1 for item in reports if item["changed"]), + "invalid": sum(1 for item in reports if not item["valid"]), + "fixed": sum(item["summary"]["fixed"] for item in reports), + "errors": sum(item["summary"]["errors"] for item in reports), + "warnings": sum(item["summary"]["warnings"] for item in reports), + }, + "files": reports, + "skipped": skipped, + } + + if report_path is not None: + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(json.dumps(aggregate, indent=2, ensure_ascii=False), encoding="utf-8") + + return aggregate + + +def summarize_report(report: SchemaValidationReport) -> str: + """Return a compact human-readable summary.""" + return ( + f"valid={report.valid}, changed={report.changed}, " + f"fixed={report.fixed_count}, errors={report.error_count}, warnings={report.warning_count}" + ) + + +def _normalize_metadata(metadata: dict[str, Any], report: SchemaValidationReport, repair: bool) -> None: + for key, default in {"authors": [], "year": None, "journal": None, "doi": None, "link": None}.items(): + _ensure_key(metadata, key, default, f"$.paper_metadata.{key}", report, repair) + + authors = metadata.get("authors") + if isinstance(authors, str): + _replace(metadata, "authors", [authors], "$.paper_metadata.authors", report, repair, "authors must be a list") + elif authors is None: + _replace(metadata, "authors", [], "$.paper_metadata.authors", report, repair, "authors must be a list") + elif not isinstance(authors, list): + report.add("$.paper_metadata.authors", "authors must be a list", severity="error", before=authors) + + _normalize_int(metadata, "year", "$.paper_metadata.year", report, repair, nullable=True) + for key in ("journal", "doi", "link"): + _normalize_nullable_string(metadata, key, f"$.paper_metadata.{key}", report, repair) + + +def _normalize_study(study: dict[str, Any], path: str, report: SchemaValidationReport, repair: bool) -> None: + _ensure_key(study, "study", None, f"{path}.study", report, repair) + _ensure_key(study, "eligibility_rationale", None, f"{path}.eligibility_rationale", report, repair) + _ensure_key(study, "effects", [], f"{path}.effects", report, repair) + _ensure_key(study, "sample", None, f"{path}.sample", report, repair) + _normalize_nullable_string(study, "study", f"{path}.study", report, repair) + _normalize_nullable_string(study, "eligibility_rationale", f"{path}.eligibility_rationale", report, repair) + + # Migration: lift effect-level sample to study level (one-time, repair=True only). + # Old format had `sample` inside each effect; new format has it at study level. + if repair and study.get("sample") is None: + effects_list = study.get("effects") or [] + old_sample = next( + (e.get("sample") for e in effects_list if isinstance(e, dict) and isinstance(e.get("sample"), dict)), + None, + ) + if old_sample is not None: + study["sample"] = _migrate_old_sample(old_sample) + report.add( + f"{path}.sample", + "migrated sample from effect[0] to study level", + fixed=True, + before=None, + after=study["sample"], + ) + + _normalize_study_sample(study, path, report, repair) + + effects = study.get("effects") + if not isinstance(effects, list): + report.add(f"{path}.effects", "effects must be a list", severity="error", before=effects) + return + + for effect_index, effect in enumerate(effects): + effect_path = f"{path}.effects[{effect_index}]" + if not isinstance(effect, dict): + report.add(effect_path, "effect entry must be an object", severity="error", before=effect) + continue + # Remove obsolete effect-level sample during migration. + if repair and isinstance(effect.get("sample"), dict): + del effect["sample"] + report.add(effect_path + ".sample", "removed effect-level sample (now at study level)", fixed=True) + _normalize_effect(effect, effect_path, report, repair) + + +def _migrate_old_sample(old: dict[str, Any]) -> dict[str, Any]: + """Convert old nested effect.sample → new flat study.sample.""" + new: dict[str, Any] = { + "total_n": None, "analyzed_n": None, + "mean_age": None, "female_percent": None, "male_percent": None, + "platform": None, "country": None, + "inclusion_criteria": None, "exclusion_criteria": None, "notes": None, + } + # N fields + total = old.get("total_n") + new["total_n"] = int(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else total + n_excl = old.get("n_excluded") + if isinstance(n_excl, (int, float)) and not isinstance(n_excl, bool) and isinstance(total, (int, float)): + new["analyzed_n"] = int(total - n_excl) + + # Age + age = old.get("age") or {} + mean = age.get("mean") if isinstance(age, dict) else None + if isinstance(mean, (int, float)) and not isinstance(mean, bool): + new["mean_age"] = float(mean) + + # Gender — normalize to 0-100 percent + gender = old.get("gender") or {} + if isinstance(gender, dict): + for old_key, new_key in (("female_pct", "female_percent"), ("male_pct", "male_percent")): + val = gender.get(old_key) + if isinstance(val, (int, float)) and not isinstance(val, bool): + # If stored as proportion (≤1.0) convert to percent + pct = float(val) * 100.0 if float(val) <= 1.0 else float(val) + new[new_key] = round(pct, 4) + + # Platform: prefer recruitment.source over recruitment.platform + rec = old.get("recruitment") or {} + if isinstance(rec, dict): + platform = rec.get("source") or rec.get("platform") + if isinstance(platform, str) and platform.strip(): + new["platform"] = platform.strip() + + # Exclusion criteria: list → single string (joined) + excl = old.get("exclusion_criteria") + if isinstance(excl, list): + parts = [str(x).strip() for x in excl if x and str(x).strip()] + new["exclusion_criteria"] = "; ".join(parts) if parts else None + elif isinstance(excl, str) and excl.strip(): + new["exclusion_criteria"] = excl.strip() + + # notes from old "other" (only if not an LLM summary — no heuristic, just pass through) + other = old.get("other") + if isinstance(other, str) and other.strip(): + new["notes"] = other.strip() + + return new + + +def _extract_platform_canonical(value: str) -> str | None: + """Try to extract a canonical SAMPLE_PLATFORM_VOCAB token from a freeform string. + + Examples: + "we recruited subjects via the platform CloudResearch." → "CloudResearch" + "Amazon MTurk" → "MTurk" + "Prolific Academic" → "Prolific" + "something completely unknown" → None + + Checks vocab tokens longest-first so e.g. "CloudResearch" beats "Online". + """ + v_lower = value.lower() + for token in sorted(SAMPLE_PLATFORM_VOCAB, key=len, reverse=True): + if token.lower() in v_lower: + return token + return None + + +def _normalize_study_sample(study: dict[str, Any], path: str, report: SchemaValidationReport, repair: bool) -> None: + """Validate and normalize the study-level sample object. + + The sample object is general enough for all social-science paper types: + online panels (MTurk/Prolific), undergraduate pools, lab experiments, + organizational/field studies, archival data, and multi-wave designs. + Most fields are nullable — only fill what the paper actually reports. + + Forward-compatibility: unknown keys in the sample dict are preserved with + a warning so future schema additions don't lose data. + """ + sample = study.get("sample") + sample_path = f"{path}.sample" + + if _is_null_string(sample): + _replace(study, "sample", None, sample_path, report, repair, "empty sample normalized to null") + return + if sample is None or isinstance(sample, str): + return + if not isinstance(sample, dict): + report.add(sample_path, "sample must be a dict, string, or null", severity="error", before=sample) + return + + # Ensure all expected keys exist (nullable defaults) + sample_defaults: dict[str, Any] = { + "total_n": None, + "analyzed_n": None, + "mean_age": None, + "female_percent": None, + "male_percent": None, + "platform": None, + "country": None, + "inclusion_criteria": None, + "exclusion_criteria": None, + "notes": None, + } + for key, default in sample_defaults.items(): + _ensure_key(sample, key, default, f"{sample_path}.{key}", report, repair) + + # Integer fields + for key in ("total_n", "analyzed_n"): + _normalize_int(sample, key, f"{sample_path}.{key}", report, repair, nullable=True) + + # Float fields (0-100 percent scale; null if not reported) + for key in ("mean_age", "female_percent", "male_percent"): + _normalize_number(sample, key, f"{sample_path}.{key}", report, repair, nullable=True) + val = sample.get(key) + if isinstance(val, (int, float)) and not isinstance(val, bool): + # Normalize proportion (≤1.0) → percent for gender fields + if key in ("female_percent", "male_percent") and float(val) <= 1.0: + corrected = round(float(val) * 100.0, 4) + _replace( + sample, key, corrected, f"{sample_path}.{key}", report, repair, + f"{key} normalized from proportion to percent: {val} → {corrected}" + ) + + # String fields (nullable) + for key in ("platform", "country", "inclusion_criteria", "exclusion_criteria", "notes"): + _normalize_nullable_string(sample, key, f"{sample_path}.{key}", report, repair) + + # Platform vocab enforcement: if value is set but not a canonical token, try auto-extraction. + platform = sample.get("platform") + if isinstance(platform, str) and platform not in SAMPLE_PLATFORM_VOCAB: + canonical = _extract_platform_canonical(platform) + if canonical is not None: + _replace( + sample, "platform", canonical, f"{sample_path}.platform", report, repair, + f"platform extracted from sentence to controlled vocab: {platform!r} → {canonical!r}", + ) + else: + report.add( + f"{sample_path}.platform", + f"platform value {platform!r} is not in SAMPLE_PLATFORM_VOCAB " + f"{sorted(SAMPLE_PLATFORM_VOCAB)}; cannot auto-canonicalize", + severity="warning", + ) + + # Forward-compat: warn about unknown keys so they are visible in reports but not deleted. + known_keys = set(sample_defaults.keys()) + for unknown_key in sorted(set(sample.keys()) - known_keys): + report.add( + f"{sample_path}.{unknown_key}", + f"unknown sample field {unknown_key!r} preserved for forward-compatibility; " + "consider using 'notes' for free-form text that does not fit the standard fields", + severity="warning", + ) + + +def _normalize_effect(effect: dict[str, Any], path: str, report: SchemaValidationReport, repair: bool) -> None: + defaults = { + # "sample" intentionally absent — sample now lives at study level. + "platform": None, + "effecttype": None, + "IV": None, + "DV": None, + "size": None, + "direction": None, + "mean_group1": None, + "sd_group1": None, + "mean_group2": None, + "sd_group2": None, + "stats": {}, + "materials_notes": None, + "table_or_page_location": None, + "materials": {"status": None, "content": None}, + "manipulation": {"status": None, "content": None}, + "items": {"status": None, "content": None}, + } + for key in EFFECT_REQUIRED_FIELDS: + _ensure_key(effect, key, deepcopy(defaults[key]), f"{path}.{key}", report, repair) + + for key in ("platform", "IV", "DV", "materials_notes", "table_or_page_location"): + _normalize_nullable_string(effect, key, f"{path}.{key}", report, repair) + _normalize_effecttype(effect, f"{path}.effecttype", report, repair) + _normalize_direction(effect, f"{path}.direction", report, repair) + _normalize_effect_sample_size(effect, f"{path}.size", report, repair) + for key in EFFECT_NUMERIC_FIELDS: + _normalize_number(effect, key, f"{path}.{key}", report, repair, nullable=True) + + stats = effect.get("stats") + if not isinstance(stats, dict): + if repair: + report.add(f"{path}.stats", "stats must be an object; replaced with empty stats", fixed=True, before=stats, after={}) + stats = {} + effect["stats"] = stats + else: + report.add(f"{path}.stats", "stats must be an object", severity="error", before=stats) + stats = {} + _normalize_stats(stats, f"{path}.stats", report, repair) + + for slot_name in SLOT_NAMES: + _normalize_slot(effect, slot_name, f"{path}.{slot_name}", report, repair) + + # Optional effect-level analysis sample fields (not in EFFECT_REQUIRED_FIELDS — they are + # supplementary). Only normalize when the key is present so existing JSONs are not polluted + # with extra null fields unless explicitly written by the extractor. + if "analysis_n" in effect: + _normalize_int(effect, "analysis_n", f"{path}.analysis_n", report, repair, nullable=True) + if "analysis_scope" in effect: + _normalize_nullable_string(effect, "analysis_scope", f"{path}.analysis_scope", report, repair) + scope_val = effect.get("analysis_scope") + if isinstance(scope_val, str) and scope_val not in VALID_ANALYSIS_SCOPE: + scope_norm = re.sub(r"[\s\-]+", "_", scope_val.lower().strip()) + if scope_norm in VALID_ANALYSIS_SCOPE: + _replace( + effect, "analysis_scope", scope_norm, f"{path}.analysis_scope", report, repair, + f"analysis_scope normalized: {scope_val!r} → {scope_norm!r}", + ) + else: + report.add( + f"{path}.analysis_scope", + f"analysis_scope {scope_val!r} not in {sorted(VALID_ANALYSIS_SCOPE)}", + severity="warning", + ) + + +# _normalize_sample (old effect-level) removed — replaced by _normalize_study_sample above. + + +def _normalize_stats(stats: dict[str, Any], path: str, report: SchemaValidationReport, repair: bool) -> None: + for key in STATS_REQUIRED_FIELDS: + default = [None, None] if key == "ci" else None + _ensure_key(stats, key, deepcopy(default), f"{path}.{key}", report, repair) + for key in STATS_NUMERIC_FIELDS: + _normalize_number(stats, key, f"{path}.{key}", report, repair, nullable=True) + _normalize_ci_like(stats, "ci", f"{path}.ci", report, repair, label="ci") + _normalize_p_value(stats, path, report, repair) + _normalize_sig(stats, f"{path}.sig", report, repair) + + +def _normalize_slot(effect: dict[str, Any], key: str, path: str, report: SchemaValidationReport, repair: bool) -> None: + slot = effect.get(key) + if isinstance(slot, str): + replacement = {"status": None, "content": None if _is_null_string(slot) else slot} + _replace(effect, key, replacement, path, report, repair, "slot string converted to status/content object") + slot = effect.get(key) + elif slot is None: + replacement = {"status": None, "content": None} + _replace(effect, key, replacement, path, report, repair, "missing slot converted to status/content object") + slot = effect.get(key) + + if not isinstance(slot, dict): + report.add(path, "slot must be an object with status/content", severity="error", before=slot) + return + + _ensure_key(slot, "status", None, f"{path}.status", report, repair) + _ensure_key(slot, "content", None, f"{path}.content", report, repair) + _normalize_slot_status(slot, f"{path}.status", report, repair) + content = slot.get("content") + if _is_null_string(content): + _replace(slot, "content", None, f"{path}.content", report, repair, "empty slot content normalized to null") + elif content is not None and not isinstance(content, str): + _replace(slot, "content", str(content), f"{path}.content", report, repair, "slot content converted to string") + + if slot.get("status") == "verbatim" and slot.get("content") is None: + report.add(f"{path}.content", "verbatim slot must have non-empty content", severity="error") + + +def _normalize_p_value(stats: dict[str, Any], path: str, report: SchemaValidationReport, repair: bool) -> None: + key = "p_value" + value = stats.get(key) + field_path = f"{path}.p_value" + if _is_null_string(value): + _replace(stats, key, None, field_path, report, repair, "empty p_value normalized to null") + elif value is None: + return + elif not isinstance(value, str): + _replace(stats, key, str(value), field_path, report, repair, "p_value must be stored as a string") + + +def _normalize_sig(stats: dict[str, Any], path: str, report: SchemaValidationReport, repair: bool) -> None: + mapping = { + "significant": "sig", + "statistically significant": "sig", + "non significant": "ns", + "non-significant": "ns", + "nonsig": "ns", + "non_sig": "ns", + "not significant": "ns", + "not_significant": "ns", + "insignificant": "ns", + "marginally significant": "marginal", + "marginal significant": "marginal", + "unknown": None, + "mixed": None, + "not tested": None, + "not reported": None, + } + _normalize_enum(stats, "sig", VALID_SIG, mapping, path, report, repair, nullable=True) + + +def _normalize_direction(effect: dict[str, Any], path: str, report: SchemaValidationReport, repair: bool) -> None: + mapping = { + "positive": "pos", + "marginal positive": "pos", + "marginal pos": "pos", + "positive marginal": "pos", + "+": "pos", + "negative": "neg", + "marginal negative": "neg", + "marginal neg": "neg", + "negative marginal": "neg", + "-": "neg", + "none": None, + "no effect": None, + "no_effect": None, + "null": None, + "zero": None, + "equal": None, + "same": None, + "multi group": None, + "multi-group": None, + "mixed": None, + "complex": None, + } + _normalize_enum(effect, "direction", VALID_DIRECTION, mapping, path, report, repair, nullable=True) + + +def _normalize_effecttype(effect: dict[str, Any], path: str, report: SchemaValidationReport, repair: bool) -> None: + mapping = { + "interaction": "int", + "interactions": "int", + "moderation": "int", + "main effect": "main", + "main_effect": "main", + "simple effect": "simple", + "simple_effect": "simple", + "pearson": "correlation", + "corr": "correlation", + "r": "correlation", + } + _normalize_enum(effect, "effecttype", VALID_EFFECT_TYPES, mapping, path, report, repair, nullable=False) + + +def _normalize_slot_status(slot: dict[str, Any], path: str, report: SchemaValidationReport, repair: bool) -> None: + mapping = { + "quoted": "verbatim", + "quote": "verbatim", + "paraphrase": "paraphrased", + "paraphrase_summary": "paraphrased", + "scale": "cited_scale", + "cited scale": "cited_scale", + "osf": "osf_only", + "supplement": "osf_only", + "supplementary": "osf_only", + "missing": "not_in_paper", + "not found": "not_in_paper", + } + _normalize_enum(slot, "status", VALID_SLOT_STATUSES, mapping, path, report, repair, nullable=True) + + +def _normalize_enum( + obj: dict[str, Any], + key: str, + valid: set[str], + mapping: dict[str, str | None], + path: str, + report: SchemaValidationReport, + repair: bool, + *, + nullable: bool, +) -> None: + value = obj.get(key) + if _is_null_string(value): + if nullable: + _replace(obj, key, None, path, report, repair, f"{key} empty value normalized to null") + return + report.add(path, f"{key} must be one of {sorted(valid)}", severity="error", before=value) + return + if value is None: + if nullable: + return + report.add(path, f"{key} must be one of {sorted(valid)}", severity="error", before=value) + return + if not isinstance(value, str): + value_text = str(value) + else: + value_text = value + normalized_key = _enum_key(value_text) + if normalized_key in mapping: + _replace(obj, key, mapping[normalized_key], path, report, repair, f"{key} enum normalized") + return + if value_text in valid: + return + report.add(path, f"{key} must be one of {sorted(valid)}", severity="error", before=value) + + +def _normalize_int( + obj: dict[str, Any], + key: str, + path: str, + report: SchemaValidationReport, + repair: bool, + *, + nullable: bool, +) -> None: + value = obj.get(key) + if _is_null_string(value) or value is None: + if nullable: + _replace(obj, key, None, path, report, repair, f"{key} empty value normalized to null") + return + report.add(path, f"{key} must be an integer", severity="error", before=value) + return + if isinstance(value, bool): + report.add(path, f"{key} must be an integer, not boolean", severity="error", before=value) + return + if isinstance(value, int): + return + if isinstance(value, float) and value.is_integer(): + _replace(obj, key, int(value), path, report, repair, f"{key} converted to integer") + return + parsed = _parse_number(value) + if parsed is not None and parsed.is_integer(): + _replace(obj, key, int(parsed), path, report, repair, f"{key} converted to integer") + return + report.add(path, f"{key} must be an integer or null", severity="error", before=value) + + +def _normalize_effect_sample_size( + effect: dict[str, Any], + path: str, + report: SchemaValidationReport, + repair: bool, +) -> None: + """Normalize effect.size, whose schema meaning is analysis/sample N.""" + value = effect.get("size") + if _is_null_string(value) or value is None: + _replace(effect, "size", None, path, report, repair, "size empty value normalized to null") + return + if isinstance(value, bool): + report.add(path, "size must be an integer, not boolean", severity="error", before=value) + return + if isinstance(value, int): + return + if isinstance(value, float) and value.is_integer(): + _replace(effect, "size", int(value), path, report, repair, "size converted to integer") + return + if isinstance(value, float) and repair: + _preserve_noninteger_size(effect, value) + _replace( + effect, + "size", + None, + path, + report, + repair, + "non-integer size looked like an effect-size statistic; preserved in effect_size_notes and normalized to null", + ) + return + parsed = _parse_number(value) + if parsed is not None and parsed.is_integer(): + _replace(effect, "size", int(parsed), path, report, repair, "size converted to integer") + return + if parsed is not None and repair: + _preserve_noninteger_size(effect, value) + _replace( + effect, + "size", + None, + path, + report, + repair, + "non-integer size looked like an effect-size statistic; preserved in effect_size_notes and normalized to null", + ) + return + report.add(path, "size must be an integer or null", severity="error", before=value) + + +def _preserve_noninteger_size(effect: dict[str, Any], value: Any) -> None: + note = ( + f"Non-integer size field value preserved from extraction: {value!r}. " + "In this schema, size is sample/analysis N; effect-size statistics belong " + "in stats or reported_statistics_text." + ) + existing = effect.get("effect_size_notes") + if isinstance(existing, str) and existing.strip(): + if note not in existing: + effect["effect_size_notes"] = f"{existing.rstrip()} {note}" + else: + effect["effect_size_notes"] = note + + +def _normalize_number( + obj: dict[str, Any], + key: str, + path: str, + report: SchemaValidationReport, + repair: bool, + *, + nullable: bool, +) -> None: + value = obj.get(key) + if _is_null_string(value) or value is None: + if nullable: + _replace(obj, key, None, path, report, repair, f"{key} empty value normalized to null") + return + report.add(path, f"{key} must be numeric", severity="error", before=value) + return + if isinstance(value, bool): + report.add(path, f"{key} must be numeric, not boolean", severity="error", before=value) + return + if isinstance(value, (int, float)): + return + parsed = _parse_number(value) + if parsed is not None: + final: int | float = int(parsed) if parsed.is_integer() else parsed + _replace(obj, key, final, path, report, repair, f"{key} converted to number") + return + report.add(path, f"{key} must be numeric or null", severity="error", before=value) + + +def _normalize_ci_like( + obj: dict[str, Any], + key: str, + path: str, + report: SchemaValidationReport, + repair: bool, + *, + label: str, +) -> None: + value = obj.get(key) + if _is_null_string(value) or value is None: + _replace(obj, key, [None, None], path, report, repair, f"{label} normalized to [null, null]") + value = obj.get(key) + + if not isinstance(value, list): + parsed = _parse_number_list(value) + if parsed is not None: + _replace(obj, key, parsed, path, report, repair, f"{label} parsed into two-element array") + value = obj.get(key) + else: + _replace(obj, key, [None, None], path, report, repair, f"{label} must be a two-element array") + value = obj.get(key) + + if not isinstance(value, list): + report.add(path, f"{label} must be a two-element array", severity="error", before=value) + return + + if len(value) != 2: + replacement = (value + [None, None])[:2] + _replace(obj, key, replacement, path, report, repair, f"{label} resized to two elements") + value = obj.get(key) + + if isinstance(value, list) and len(value) == 2: + for index in range(2): + wrapper = {"value": value[index]} + _normalize_number(wrapper, "value", f"{path}[{index}]", report, repair, nullable=True) + if repair: + value[index] = wrapper["value"] + + +def _normalize_nullable_string( + obj: dict[str, Any], + key: str, + path: str, + report: SchemaValidationReport, + repair: bool, +) -> None: + value = obj.get(key) + if _is_null_string(value): + _replace(obj, key, None, path, report, repair, f"{key} empty value normalized to null") + elif value is not None and not isinstance(value, str): + _replace(obj, key, str(value), path, report, repair, f"{key} converted to string") + + +def _ensure_key( + obj: dict[str, Any], + key: str, + default: Any, + path: str, + report: SchemaValidationReport, + repair: bool, +) -> None: + if key in obj: + return + if repair: + obj[key] = deepcopy(default) + report.add(path, "missing required key added", fixed=True, before="", after=default) + else: + report.add(path, "missing required key", severity="error") + + +def _replace( + obj: dict[str, Any], + key: str, + value: Any, + path: str, + report: SchemaValidationReport, + repair: bool, + message: str, +) -> None: + before = obj.get(key) + if before == value: + return + if repair: + obj[key] = value + report.add(path, message, fixed=True, before=before, after=value) + else: + report.add(path, message, severity="error", before=before, after=value) + + +def _parse_number(value: Any) -> float | None: + if isinstance(value, str): + text = value.strip().replace(",", "") + if _is_null_string(text): + return None + if re.fullmatch(r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)", text): + try: + return float(text) + except ValueError: + return None + if re.match(r"^\s*bs?\s*=", text, flags=re.I): + return None + statistic_match = re.search(r"\)\s*=\s*([-+]?(?:\d+(?:\.\d*)?|\.\d+))", text) + if statistic_match: + return float(statistic_match.group(1)) + leading_match = re.match(r"\s*([-+]?(?:\d+(?:\.\d*)?|\.\d+))\b", text) + if leading_match: + return float(leading_match.group(1)) + equals_matches = re.findall(r"=\s*([-+]?(?:\d+(?:\.\d*)?|\.\d+))", text) + if len(equals_matches) == 1: + return float(equals_matches[0]) + return None + + +def _parse_number_list(value: Any) -> list[Any] | None: + if not isinstance(value, str): + return None + numbers = re.findall(r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)", value.replace(",", "")) + if len(numbers) >= 2: + parsed = [float(numbers[0]), float(numbers[1])] + return [int(x) if x.is_integer() else x for x in parsed] + return None + + +def _is_null_string(value: Any) -> bool: + return isinstance(value, str) and value.strip().lower() in NULL_STRINGS + + +def _enum_key(value: str) -> str: + return re.sub(r"\s+", " ", value.strip().lower().replace("_", " ")) + + +def _safe_value(value: Any) -> Any: + if value is _MISSING: + return _MISSING + try: + json.dumps(value, ensure_ascii=False) + return value + except TypeError: + return repr(value) + + +def _looks_like_paper_json(data: Any) -> bool: + return isinstance(data, dict) and "eligible_studies" in data + + +def _write_json_atomic(path: Path, data: dict[str, Any], *, backup: bool) -> None: + if backup and path.exists(): + shutil.copy2(path, path.with_suffix(path.suffix + ".bak")) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") + tmp.replace(path) + + +def _paths_from_args(paths: Iterable[str]) -> list[Path]: + return [Path(item) for item in paths] + + +def main() -> None: + parser = argparse.ArgumentParser(description="Validate and normalize ai-ethics paper JSON schema") + parser.add_argument("json", nargs="*", help="One or more per-paper JSON files") + parser.add_argument("--stage4-dir", type=Path, help="Scan a stage4 tree instead of explicit JSON files") + parser.add_argument("--write", action="store_true", help="Write deterministic repairs back to JSON files") + parser.add_argument("--no-backup", action="store_true", help="Do not create .bak files when --write is used") + parser.add_argument( + "--report", + type=Path, + default=Path("stage4/schema_normalization_report.json"), + help="Report path for --stage4-dir scans", + ) + args = parser.parse_args() + + if args.stage4_dir: + aggregate = normalize_stage4_tree( + args.stage4_dir, + write=args.write, + backup=not args.no_backup, + report_path=args.report, + ) + print(json.dumps(aggregate["summary"], indent=2, ensure_ascii=False)) + print(f"Report: {args.report}") + raise SystemExit(1 if aggregate["summary"]["invalid"] else 0) + + if not args.json: + parser.error("pass JSON files or --stage4-dir") + + any_invalid = False + for json_path in _paths_from_args(args.json): + _, report = validate_file(json_path, repair=True, write=args.write, backup=not args.no_backup) + print(f"{json_path}: {summarize_report(report)}") + any_invalid = any_invalid or not report.valid + raise SystemExit(1 if any_invalid else 0) + + +if __name__ == "__main__": + main() diff --git a/generation_pipeline/verification/stage_quality.py b/generation_pipeline/verification/stage_quality.py new file mode 100644 index 0000000..0913f49 --- /dev/null +++ b/generation_pipeline/verification/stage_quality.py @@ -0,0 +1,324 @@ +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional, Set + + +_VALID_REPLICABLE = {"YES", "NO", "UNCERTAIN"} + + +def _norm(value: Any) -> str: + text = str(value or "").strip().lower() + text = re.sub(r"[^a-z0-9]+", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def _study_key(value: Any) -> str: + text = _norm(value) + match = re.search(r"\b(?:study|experiment|exp)\s+(\d+[a-z]?)\b", text) + if match: + return match.group(1) + match = re.search(r"\b(pilot|validation)\b", text) + if match: + return match.group(1) + text = re.sub(r"\b(study|experiment|exp)\b", "", text) + return re.sub(r"\s+", " ", text).strip() + + +def _has_text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _has_stats(effect: Dict[str, Any]) -> bool: + stats = effect.get("stats") if isinstance(effect.get("stats"), dict) else {} + if effect.get("reported_statistics_text"): + return True + for key in ("B", "b", "t", "f", "z", "chi_square", "D", "eta_square", "p_value", "sig"): + value = stats.get(key) + if value not in (None, "", [], [None, None]): + return True + ci = stats.get("ci") + return isinstance(ci, list) and any(item is not None for item in ci) + + +def _slot_status(effect: Dict[str, Any], slot: str) -> Optional[str]: + value = effect.get(slot) + if isinstance(value, dict): + status = value.get("status") + return str(status).strip() if status is not None else None + return None + + +def _regen_item_count(verification: Dict[str, Any]) -> int: + regen = verification.get("regeneration_instructions") + if not isinstance(regen, dict): + return 0 + count = 0 + for value in regen.values(): + if isinstance(value, list): + count += len(value) + elif value not in (None, "", {}): + count += 1 + return count + + +def _needs_human_review( + issues: List[Dict[str, Any]], + *, + verifier_status: Any, + verifier_overall: Any, + verifier_suggestions: int, +) -> bool: + if any(issue.get("severity") == "error" for issue in issues): + return True + if verifier_status == "error": + return True + if verifier_overall in {"needs_review", "fail"}: + return True + return verifier_suggestions > 0 + + +def _stage1_eligible_keys(stage1_json: Dict[str, Any]) -> Set[str]: + keys: Set[str] = set() + for exp in stage1_json.get("experiments", []) or []: + if not isinstance(exp, dict): + continue + replicable = str(exp.get("replicable") or "").strip().upper() + if replicable in {"YES", "UNCERTAIN"}: + key = _study_key(exp.get("experiment_id") or exp.get("experiment_name")) + if key: + keys.add(key) + return keys + + +def build_stage1_quality(stage1_json: Dict[str, Any]) -> Dict[str, Any]: + """Return a compact quality report for the Stage 1 study inventory.""" + experiments = [exp for exp in stage1_json.get("experiments", []) or [] if isinstance(exp, dict)] + issues: List[Dict[str, Any]] = [] + eligible = 0 + material_missing = 0 + + if not experiments: + issues.append({"severity": "error", "path": "$.experiments", "message": "no experiments extracted"}) + + for idx, exp in enumerate(experiments): + path = f"$.experiments[{idx}]" + replicable = str(exp.get("replicable") or "").strip().upper() + if replicable not in _VALID_REPLICABLE: + issues.append({"severity": "warning", "path": f"{path}.replicable", "message": "invalid replicable label"}) + if replicable in {"YES", "UNCERTAIN"}: + eligible += 1 + for field in ("experiment_id", "input", "participants", "output"): + if not _has_text(exp.get(field)): + issues.append({"severity": "warning", "path": f"{path}.{field}", "message": "missing eligible-study anchor"}) + if not _has_text(exp.get("design_type")): + issues.append({"severity": "warning", "path": f"{path}.design_type", "message": "missing design type"}) + if not exp.get("conditions_or_factors"): + issues.append({"severity": "warning", "path": f"{path}.conditions_or_factors", "message": "missing factors or conditions"}) + if exp.get("has_self_contained_materials") is False or _has_text(exp.get("missing_materials")): + material_missing += 1 + + if eligible == 0 and experiments: + issues.append({"severity": "error", "path": "$.experiments", "message": "no eligible or uncertain experiments"}) + + study_contract = stage1_json.get("stage1_study_contract") if isinstance(stage1_json.get("stage1_study_contract"), dict) else {} + contract_blocking = int(study_contract.get("blocking_issue_count") or 0) if study_contract else 0 + if contract_blocking: + issues.append( + { + "severity": "error", + "path": "$.stage1_study_contract", + "message": f"Stage 1 study contract has {contract_blocking} blocking issue(s)", + } + ) + + verification = stage1_json.get("stage1_verification") if isinstance(stage1_json.get("stage1_verification"), dict) else {} + verifier_status = verification.get("status") + verifier_overall = verification.get("overall") + if verifier_status == "error": + issues.append({"severity": "warning", "path": "$.stage1_verification", "message": "Stage 1 verifier failed"}) + elif verifier_overall in {"needs_review", "fail"}: + severity = "error" if verifier_overall == "fail" else "warning" + issues.append({"severity": severity, "path": "$.stage1_verification.overall", "message": f"Stage 1 verifier overall={verifier_overall}"}) + verifier_suggestions = _regen_item_count(verification) + if verifier_suggestions: + issues.append({"severity": "warning", "path": "$.stage1_verification.regeneration_instructions", "message": f"Stage 1 verifier returned {verifier_suggestions} regeneration suggestion(s)"}) + + return { + "version": "stage1_quality_v1", + "experiments_total": len(experiments), + "eligible_or_uncertain": eligible, + "missing_materials_count": material_missing, + "study_contract_ready": study_contract.get("ready") if study_contract else None, + "study_contract_needs_review": study_contract.get("needs_review") if study_contract else None, + "study_contract_blocking_issues": contract_blocking if study_contract else None, + "verifier_status": verifier_status, + "verifier_overall": verifier_overall, + "verifier_suggestions": verifier_suggestions, + "needs_human_review": _needs_human_review( + issues, + verifier_status=verifier_status, + verifier_overall=verifier_overall, + verifier_suggestions=verifier_suggestions, + ), + "issues": issues, + } + + +def build_stage2_quality(stage2_json: Dict[str, Any], stage1_json: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Return quality checks for Stage 2 study/effect extraction.""" + studies = [study for study in stage2_json.get("eligible_studies", []) or [] if isinstance(study, dict)] + issues: List[Dict[str, Any]] = [] + study_reports: List[Dict[str, Any]] = [] + effects_total = 0 + effects_with_stats = 0 + effects_with_legacy_material_slots = 0 + legacy_material_slot_details: List[Dict[str, Any]] = [] + findings_total = 0 + primary_finding_candidates = 0 + stage2_keys: Set[str] = set() + + if not studies: + issues.append({"severity": "error", "path": "$.eligible_studies", "message": "no eligible studies extracted"}) + + for sidx, study in enumerate(studies): + path = f"$.eligible_studies[{sidx}]" + study_name = study.get("study") or study.get("name") or f"Study {sidx + 1}" + key = _study_key(study_name) + if key: + stage2_keys.add(key) + sample = study.get("sample") if isinstance(study.get("sample"), dict) else {} + study_issues: List[str] = [] + effects = [effect for effect in study.get("effects", []) or [] if isinstance(effect, dict)] + findings = [finding for finding in study.get("findings", []) or [] if isinstance(finding, dict)] + effects_total += len(effects) + findings_total += len(findings) + + if not effects: + study_issues.append("no_effects") + issues.append({"severity": "error", "path": f"{path}.effects", "message": "study has no effects"}) + if not findings: + study_issues.append("no_findings") + issues.append({"severity": "warning", "path": f"{path}.findings", "message": "study has no consolidated findings"}) + if not sample.get("total_n") and not sample.get("analyzed_n"): + study_issues.append("missing_sample_n") + issues.append({"severity": "warning", "path": f"{path}.sample", "message": "missing study sample N"}) + + for fidx, finding in enumerate(findings): + fpath = f"{path}.findings[{fidx}]" + if finding.get("simulation_target", {}).get("candidate"): + primary_finding_candidates += 1 + for field in ("finding_id", "representative_effect_index", "role", "IV", "DV", "reported_statistics"): + if finding.get(field) in (None, "", []): + issues.append({"severity": "warning", "path": f"{fpath}.{field}", "message": "finding missing required mapping field"}) + + for eidx, effect in enumerate(effects): + epath = f"{path}.effects[{eidx}]" + if not _has_text(effect.get("IV")): + issues.append({"severity": "warning", "path": f"{epath}.IV", "message": "missing IV"}) + if not _has_text(effect.get("DV")): + issues.append({"severity": "warning", "path": f"{epath}.DV", "message": "missing DV"}) + if _has_stats(effect): + effects_with_stats += 1 + else: + issues.append({"severity": "error", "path": f"{epath}.stats", "message": "effect has no reported statistic"}) + if not _has_text(effect.get("table_or_page_location")): + issues.append({"severity": "warning", "path": f"{epath}.table_or_page_location", "message": "missing paper location"}) + active_legacy_slots = { + slot: status + for slot in ("materials", "manipulation", "items") + if (status := _slot_status(effect, slot)) in {"verbatim", "paraphrased", "cited_scale", "osf_only"} + } + if active_legacy_slots: + effects_with_legacy_material_slots += 1 + legacy_material_slot_details.append( + { + "path": epath, + "study": study_name, + "effect_index": eidx, + "statuses": active_legacy_slots, + } + ) + + study_reports.append( + { + "study": study_name, + "effects": len(effects), + "findings": len(findings), + "sample_n_present": bool(sample.get("total_n") or sample.get("analyzed_n")), + "issues": sorted(set(study_issues)), + } + ) + + coverage: Dict[str, Any] = {} + if isinstance(stage1_json, dict): + stage1_keys = _stage1_eligible_keys(stage1_json) + coverage = { + "stage1_eligible_or_uncertain": len(stage1_keys), + "stage2_studies": len(stage2_keys), + "missing_from_stage2": sorted(stage1_keys - stage2_keys), + "extra_in_stage2": sorted(stage2_keys - stage1_keys), + } + for key in coverage["missing_from_stage2"]: + issues.append({"severity": "warning", "path": "$.eligible_studies", "message": f"Stage1 study not covered in Stage2: {key}"}) + + finding_contract = stage2_json.get("stage2_finding_contract") if isinstance(stage2_json.get("stage2_finding_contract"), dict) else {} + contract_blocking = int(finding_contract.get("blocking_issue_count") or 0) if finding_contract else 0 + if contract_blocking: + issues.append( + { + "severity": "error", + "path": "$.stage2_finding_contract", + "message": f"Stage 2 finding contract has {contract_blocking} blocking issue(s)", + } + ) + + verification = stage2_json.get("stage2_verification") if isinstance(stage2_json.get("stage2_verification"), dict) else {} + verifier_status = verification.get("status") + verifier_overall = verification.get("overall") + if verifier_status == "error": + issues.append({"severity": "warning", "path": "$.stage2_verification", "message": "Stage 2 verifier failed"}) + elif verifier_overall in {"needs_review", "fail"}: + severity = "error" if verifier_overall == "fail" else "warning" + issues.append({"severity": severity, "path": "$.stage2_verification.overall", "message": f"Stage 2 verifier overall={verifier_overall}"}) + verifier_suggestions = _regen_item_count(verification) + if verifier_suggestions: + issues.append({"severity": "warning", "path": "$.stage2_verification.regeneration_instructions", "message": f"Stage 2 verifier returned {verifier_suggestions} regeneration suggestion(s)"}) + + if effects_with_legacy_material_slots: + issues.append( + { + "severity": "warning", + "path": "$.eligible_studies[*].effects[*].materials/manipulation/items", + "message": ( + f"{effects_with_legacy_material_slots} effect(s) still contain legacy per-effect material " + "slot hints; Stage 3 must use study-level PDF/OSF evidence for final materials" + ), + } + ) + + return { + "version": "stage2_quality_v1", + "studies_total": len(studies), + "effects_total": effects_total, + "findings_total": findings_total, + "primary_finding_candidates": primary_finding_candidates, + "effects_with_reported_stats": effects_with_stats, + "legacy_effect_material_slot_hints": effects_with_legacy_material_slots, + "legacy_effect_material_slot_details": legacy_material_slot_details, + "stage1_stage2_coverage": coverage, + "finding_contract_ready": finding_contract.get("ready") if finding_contract else None, + "finding_contract_needs_review": finding_contract.get("needs_review") if finding_contract else None, + "finding_contract_blocking_issues": contract_blocking if finding_contract else None, + "verifier_status": verifier_status, + "verifier_overall": verifier_overall, + "verifier_suggestions": verifier_suggestions, + "study_reports": study_reports, + "needs_human_review": _needs_human_review( + issues, + verifier_status=verifier_status, + verifier_overall=verifier_overall, + verifier_suggestions=verifier_suggestions, + ), + "issues": issues, + } diff --git a/generation_pipeline/verification/verbatim_verifier.py b/generation_pipeline/verification/verbatim_verifier.py new file mode 100644 index 0000000..12d258c --- /dev/null +++ b/generation_pipeline/verification/verbatim_verifier.py @@ -0,0 +1,1294 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import sys +import unicodedata +from copy import deepcopy +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from generation_pipeline.utils.pdf_extractor import extract_pdf_text +from generation_pipeline.verification.schema_validator import SLOT_NAMES, validate_paper + + +DEFAULT_THRESHOLD = 90.0 +PDF_TEXT_MAX_CHARS = 400000 +# Aliases used when looking for platform names in source text. +# Keys are normalized (lowercase, stripped); values are tuples of substrings to search. +SAMPLE_PLATFORM_ALIASES: dict[str, tuple[str, ...]] = { + # Online crowdsourcing + "mturk": ("mturk", "mechanical turk", "amazon turk"), + "mechanical turk": ("mturk", "mechanical turk", "amazon turk"), + "prolific": ("prolific",), + "cloudresearch": ("cloudresearch", "cloud research", "turkprime"), + "turkprime": ("turkprime", "cloudresearch"), + # University pools + "undergraduate": ("undergraduate", "undergrad", "sona", "subject pool", "student pool", "college student"), + "graduate": ("graduate student", "grad student"), + # Lab / field + "lab": ("lab", "laboratory", "in-person", "in person"), + "field": ("field experiment", "field study", "naturalistic"), + # Organizational + "organizational": ("organizational", "employees", "workers", "company", "firm", "organization"), + # Other online + "online": ("online", "internet", "web-based", "web based"), + # Archival + "archival": ("archival", "administrative data", "secondary data", "existing data", "database"), +} + + +@dataclass +class VerbatimRecord: + path: str + study: str | None + effect_index: int + slot: str + score: float | None + threshold: float + action: str + content_preview: str + span_scores: list[dict[str, Any]] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + data = { + "path": self.path, + "study": self.study, + "effect_index": self.effect_index, + "slot": self.slot, + "score": self.score, + "threshold": self.threshold, + "action": self.action, + "content_preview": self.content_preview, + } + if self.span_scores: + data["span_scores"] = self.span_scores + return data + + +@dataclass +class SampleRecord: + path: str + study: str | None + effect_index: int + field: str + value_preview: str + evidence_kind: str + action: str + evidence_preview: str | None + + def to_dict(self) -> dict[str, Any]: + return { + "path": self.path, + "study": self.study, + "effect_index": self.effect_index, + "field": self.field, + "value_preview": self.value_preview, + "evidence_kind": self.evidence_kind, + "action": self.action, + "evidence_preview": self.evidence_preview, + } + + +def verify_paper( + paper: dict[str, Any], + source_text: str, + *, + threshold: float = DEFAULT_THRESHOLD, + repair: bool = True, + path: str | Path | None = None, +) -> tuple[dict[str, Any], dict[str, Any]]: + """ + Verify all verbatim slots in a paper object against source_text. + + Returns a repaired deep copy and a JSON-serializable report. + """ + data, schema_report = validate_paper(deepcopy(paper), repair=True, path=path) + records: list[VerbatimRecord] = [] + normalized_source = normalize_for_match(source_text) + unverified = 0 + downgraded = 0 + passed = 0 + + for slot_ref in _iter_slots(data): + slot = slot_ref["slot_obj"] + if slot.get("status") != "verbatim": + continue + + content = slot.get("content") + if not isinstance(content, str) or not content.strip(): + action = "downgraded_missing_content" if repair else "missing_content" + score: float | None = 0.0 + if repair: + slot["status"] = "paraphrased" + downgraded += 1 + else: + unverified += 1 + records.append(_record(slot_ref, score, threshold, action, content)) + continue + + if not normalized_source: + action = "unverified_no_source_text" + score = None + unverified += 1 + records.append(_record(slot_ref, score, threshold, action, content)) + continue + + score, span_scores = _best_verbatim_score(content, normalized_source) + if score >= threshold: + passed += 1 + records.append(_record(slot_ref, score, threshold, "passed", content, span_scores=span_scores)) + else: + action = "downgraded_low_match" if repair else "would_downgrade_low_match" + if repair: + slot["status"] = "paraphrased" + downgraded += 1 + else: + unverified += 1 + records.append(_record(slot_ref, score, threshold, action, content, span_scores=span_scores)) + + sample_repairs: list[dict[str, Any]] = [] + repair_schema_report = None + if repair and source_text: + sample_repairs = _repair_sample_values_from_source(data, source_text) + if sample_repairs: + data, repair_schema_report = validate_paper(data, repair=True, path=path) + + sample_records = verify_sample_fields(data, source_text) + sample_passed = sum(1 for record in sample_records if record.action == "passed") + sample_unverified = sum(1 for record in sample_records if record.action != "passed") + + # Hard sample failures: numeric facts (N, age, gender %) and platform not found in source. + # These indicate real data integrity issues and block the gate. + # Soft failures: verbatim text fields (inclusion/exclusion_criteria, notes) that are + # paraphrases. In new extractions these will be null (and not checked); in migrated + # old data they may be paraphrases, which is a quality warning but not a hard blocker. + _HARD_SAMPLE_FIELDS = frozenset({"total_n", "analyzed_n", "mean_age", "female_percent", + "male_percent", "platform", "country"}) + sample_hard_failures = sum( + 1 for r in sample_records + if r.action != "passed" and r.field in _HARD_SAMPLE_FIELDS + ) + + data, final_schema_report = validate_paper(data, repair=True, path=path) + empty_slot_records = _empty_required_slot_records(data) + empty_slots_by_status = _count_empty_slots_by_status(empty_slot_records) + report = { + "path": str(path) if path is not None else None, + "threshold": threshold, + "passed": final_schema_report.valid and unverified == 0 and sample_hard_failures == 0, + "summary": { + "verbatim_slots": len(records), + "passed": passed, + "downgraded": downgraded, + "unverified": unverified, + "sample_fields": len(sample_records), + "sample_passed": sample_passed, + "sample_unverified": sample_unverified, + "sample_hard_failures": sample_hard_failures, + "sample_repairs": len(sample_repairs), + "empty_required_slots": len(empty_slot_records), + "empty_required_slots_by_status": empty_slots_by_status, + "stage4_ready": len(empty_slot_records) == 0, + "schema_valid": final_schema_report.valid, + "schema_fixed": ( + schema_report.fixed_count + + (repair_schema_report.fixed_count if repair_schema_report is not None else 0) + + final_schema_report.fixed_count + ), + }, + "records": [record.to_dict() for record in records], + "sample_records": [record.to_dict() for record in sample_records], + "sample_repairs": sample_repairs, + "empty_required_slot_records": empty_slot_records, + "schema_report": final_schema_report.to_dict(), + } + return data, report + + +def verify_json_file( + json_path: Path, + source_paths: list[Path], + *, + threshold: float = DEFAULT_THRESHOLD, + write: bool = True, + backup: bool = True, + report_path: Path | None = None, + write_report: bool = True, + repair: bool = True, +) -> dict[str, Any]: + """Verify one JSON file against one or more PDF/text source files.""" + json_path = Path(json_path) + data = json.loads(json_path.read_text(encoding="utf-8")) + if not isinstance(data, dict) or "eligible_studies" not in data: + report = { + "json_path": str(json_path), + "source_paths": [], + "skipped": True, + "passed": True, + "reason": "not a per-paper corpus JSON", + "summary": { + "verbatim_slots": 0, + "passed": 0, + "downgraded": 0, + "unverified": 0, + "sample_fields": 0, + "sample_passed": 0, + "sample_unverified": 0, + "empty_required_slots": 0, + "empty_required_slots_by_status": {}, + "stage4_ready": True, + "schema_valid": True, + "schema_fixed": 0, + }, + "records": [], + "sample_records": [], + "empty_required_slot_records": [], + } + if write_report: + if report_path is None: + report_path = json_path.parent / f"{json_path.stem}_verification_report.json" + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + report["report_path"] = str(report_path) + return report + + source_text, loaded_sources = load_source_text(source_paths) + repaired, report = verify_paper( + data, + source_text, + threshold=threshold, + repair=repair, + path=json_path, + ) + report["json_path"] = str(json_path) + report["source_paths"] = [str(path) for path in loaded_sources] + + if write and report["passed"] and _json_changed(data, repaired): + _write_json_atomic(json_path, repaired, backup=backup) + elif write and not report["passed"]: + report["write_skipped_reason"] = "verification did not pass" + + if write_report: + if report_path is None: + report_path = json_path.parent / "verification_report.json" + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + report["report_path"] = str(report_path) + return report + + +def verify_files( + json_paths: list[Path], + *, + pdf_paths: list[Path] | None = None, + source_paths: list[Path] | None = None, + source_dirs: list[Path] | None = None, + threshold: float = DEFAULT_THRESHOLD, + write: bool = True, + backup: bool = True, + aggregate_report_path: Path | None = None, + repair: bool = True, +) -> dict[str, Any]: + """Verify multiple JSON files and write per-paper plus optional aggregate reports.""" + json_paths = [Path(path) for path in json_paths] + pdf_paths = [Path(path) for path in (pdf_paths or [])] + base_sources = [Path(path) for path in (source_paths or [])] + for source_dir in source_dirs or []: + base_sources.extend(_files_under(Path(source_dir))) + + reports: list[dict[str, Any]] = [] + for index, json_path in enumerate(json_paths): + paired_sources = list(base_sources) + if len(pdf_paths) == len(json_paths): + paired_sources.append(pdf_paths[index]) + else: + paired_sources.extend(pdf_paths) + report = verify_json_file( + json_path, + paired_sources, + threshold=threshold, + write=write, + backup=backup, + write_report=aggregate_report_path is None, + repair=repair, + ) + reports.append(report) + + aggregate = { + "threshold": threshold, + "write": write, + "summary": { + "files": len(reports), + "skipped": sum(1 for item in reports if item.get("skipped")), + "passed": sum(1 for item in reports if item.get("passed")), + "failed": sum(1 for item in reports if not item.get("passed")), + "verbatim_slots": sum(item["summary"]["verbatim_slots"] for item in reports), + "downgraded": sum(item["summary"]["downgraded"] for item in reports), + "unverified": sum(item["summary"]["unverified"] for item in reports), + "sample_fields": sum(item["summary"].get("sample_fields", 0) for item in reports), + "sample_passed": sum(item["summary"].get("sample_passed", 0) for item in reports), + "sample_unverified": sum(item["summary"].get("sample_unverified", 0) for item in reports), + "empty_required_slots": sum(item["summary"].get("empty_required_slots", 0) for item in reports), + "stage4_ready_files": sum( + 1 for item in reports + if not item.get("skipped") and item["summary"].get("stage4_ready", False) + ), + "stage4_not_ready_files": sum( + 1 for item in reports + if not item.get("skipped") and not item["summary"].get("stage4_ready", False) + ), + }, + "files": reports, + } + if aggregate_report_path is not None: + aggregate_report_path.parent.mkdir(parents=True, exist_ok=True) + aggregate_report_path.write_text(json.dumps(aggregate, indent=2, ensure_ascii=False), encoding="utf-8") + return aggregate + + +def load_source_text(source_paths: Iterable[Path]) -> tuple[str, list[Path]]: + """Load text from PDF and text-like source files.""" + chunks: list[str] = [] + loaded: list[Path] = [] + for source_path in source_paths: + source_path = Path(source_path) + if not source_path.exists() or source_path.is_dir(): + continue + try: + if source_path.suffix.lower() == ".pdf": + text = extract_pdf_text(source_path, max_chars=PDF_TEXT_MAX_CHARS) + else: + text = _read_text_like_file(source_path) + except Exception as exc: + chunks.append(f"\n[Could not read {source_path}: {exc}]\n") + continue + if text.strip(): + loaded.append(source_path) + chunks.append(f"\n--- Source: {source_path} ---\n{text}") + return "\n\n".join(chunks), loaded + + +def normalize_for_match(text: str) -> str: + """Normalize text before fuzzy matching.""" + text = unicodedata.normalize("NFKC", text or "") + text = text.replace("\u2018", "'").replace("\u2019", "'").replace("\u201c", '"').replace("\u201d", '"') + text = re.sub(r"(\w)-\s+(\w)", r"\1\2", text) + text = re.sub(r"--- Page \d+ ---", " ", text, flags=re.I) + text = re.sub(r"\s+", " ", text) + return text.strip().lower() + + +def best_partial_ratio(needle: str, haystack: str) -> float: + """Return a 0-100 partial fuzzy match score.""" + if not needle or not haystack: + return 0.0 + try: + from rapidfuzz import fuzz + + return float(fuzz.partial_ratio(needle, haystack)) + except ImportError: + return _difflib_partial_ratio(needle, haystack) + + +def verify_sample_fields(data: dict[str, Any], source_text: str) -> list[SampleRecord]: + """Verify structured sample fields without fuzzy matching or LLM calls. + + Study-level sample string/numeric fields are checked with field-specific rules. + Effect-level analysis_n is also checked here (number-in-source, no context). + + This is intentionally high-precision: paraphrases are reported as unverified + rather than treated as passing. + """ + source_windows = _source_windows(source_text) + normalized_source = normalize_for_match(source_text) + records: list[SampleRecord] = [] + + # Study-level sample claims (total_n requires context; others do not) + for claim in _iter_sample_claims(data): + value = claim["value"] + if _empty_sample_value(value): + continue + if isinstance(value, str): + record = _verify_sample_string_claim(claim, value, source_windows, normalized_source) + elif isinstance(value, (int, float)) and not isinstance(value, bool): + record = _verify_sample_numeric_claim(claim, value, source_windows) + else: + continue + records.append(record) + + # Effect-level analysis_n claims (no context required — these appear near stats) + for claim in _iter_effect_analysis_claims(data): + value = claim["value"] + if _empty_sample_value(value) or not isinstance(value, (int, float)): + continue + records.append(_verify_sample_numeric_claim(claim, value, source_windows)) + + return records + + +def _repair_sample_values_from_source(data: dict[str, Any], source_text: str) -> list[dict[str, Any]]: + """Repair old study.sample values using source-backed evidence. + + This is intentionally conservative and source-aware. It only fills or + replaces study.sample.total_n when the paper reports a study-level final N + near the study label (e.g. a sample-information table row). Analysis-table + Ns that were previously stored in study.sample.total_n are copied to matching + effect.analysis_n, then the study-level total_n is corrected. + """ + windows = _source_windows(source_text) + repairs: list[dict[str, Any]] = [] + + for study_index, study in enumerate(data.get("eligible_studies", [])): + if not isinstance(study, dict): + continue + sample = study.get("sample") + if not isinstance(sample, dict): + continue + + study_name = study.get("study") or f"study[{study_index}]" + current_total = _coerce_int(sample.get("total_n")) + source_total = _find_study_level_total_n(study_name, windows) + if source_total is not None and source_total != current_total: + if current_total is not None: + _copy_matching_effect_size_to_analysis_n( + study, + current_total, + source_total, + repairs, + reason="old study.sample.total_n matched an effect-level analysis N", + ) + before = sample.get("total_n") + sample["total_n"] = source_total + repairs.append( + { + "path": f"$.eligible_studies[{study_index}].sample.total_n", + "study": study_name, + "field": "total_n", + "before": before, + "after": source_total, + "action": "repaired_from_study_level_source", + } + ) + + repaired_total = _coerce_int(sample.get("total_n")) + if repaired_total is not None: + _fill_effect_analysis_n_from_size(study, repaired_total, repairs) + + return repairs + + +def _find_study_level_total_n(study_name: Any, source_windows: list[str]) -> int | None: + """Find a study-level final sample N from source windows. + + Prefer explicit final-N / final-sample language tied to the study label. + This intentionally ignores generic analysis table rows such as Table 4 + simple-effect Ns. + """ + study_id = _study_id(study_name) + if not study_id: + return None + + patterns = _study_total_n_patterns(study_id) + for window in source_windows: + normalized = normalize_for_match(window) + if not _has_study_level_sample_language(normalized): + continue + prose_total = _find_final_sample_prose_total(study_id, normalized) + if prose_total is not None: + return prose_total + for pattern in patterns: + match = re.search(pattern, normalized, flags=re.I) + if match: + return _parse_int(match.group("n")) + return None + + +def _find_final_sample_prose_total(study_id: str, normalized_window: str) -> int | None: + """Parse prose like "final samples consisted of n=... in Study 1a and n=... in Study 1b".""" + escaped = re.escape(study_id) + for marker in re.finditer(r"\bfinal\s+samples?\b", normalized_window, flags=re.I): + segment = normalized_window[marker.start() : marker.start() + 1200] + segment = re.split( + r"\.\s+(?=(?:in\s+study|preregistration|we\s+|table|for\s+|our\s+|studies?\b))", + segment, + maxsplit=1, + flags=re.I, + )[0] + match = re.search( + rf"\bn\s*=\s*(?P[\d,]+)(?:(?!\bn\s*=).){{0,300}}?\bstudy\s*{escaped}\b", + segment, + flags=re.I | re.S, + ) + if match: + return _parse_int(match.group("n")) + match = re.search( + rf"\bstudy\s*{escaped}\b(?:(?!\bn\s*=).){{0,300}}?" + rf"\bn\s*=\s*(?P[\d,]+)", + segment, + flags=re.I | re.S, + ) + if match: + return _parse_int(match.group("n")) + return None + + +def _study_total_n_patterns(study_id: str) -> list[str]: + escaped = re.escape(study_id) + token = rf"(?[\d,]+)", + # Explicit prose: "Study 3 ... final n = 4,001" + rf"\bstudy\s*{escaped}\b[^.{{}}]{{0,500}}?\bfinal\s+n\s*=\s*(?P[\d,]+)", + ] + + +def _has_study_level_sample_language(normalized_window: str) -> bool: + return any( + phrase in normalized_window + for phrase in ( + "final n", + "final sample", + "sample information", + "sample demographics", + "target n", + "we recruited", + "recruited a target", + "our final sample", + "final samples consisted", + ) + ) + + +def _study_id(study_name: Any) -> str | None: + text = normalize_for_match(str(study_name or "")) + match = re.search(r"\b(?:study|experiment)\s*(\d+[a-z]?)\b", text) + if match: + return match.group(1) + match = re.search(r"\b(\d+[a-z]?)\b", text) + return match.group(1) if match else None + + +def _copy_matching_effect_size_to_analysis_n( + study: dict[str, Any], + old_total_n: int, + study_total_n: int | None, + repairs: list[dict[str, Any]], + *, + reason: str, +) -> None: + for effect_index, effect in enumerate(study.get("effects", [])): + if not isinstance(effect, dict): + continue + size = _coerce_int(effect.get("size")) + if size != old_total_n or effect.get("analysis_n") is not None: + continue + effect["analysis_n"] = old_total_n + effect.setdefault("analysis_scope", _infer_analysis_scope(effect, old_total_n, study_total_n)) + repairs.append( + { + "path": f"{study.get('study', 'study')}.effects[{effect_index}].analysis_n", + "study": study.get("study"), + "effect_index": effect_index, + "field": "analysis_n", + "before": None, + "after": old_total_n, + "action": "moved_from_study_total_n", + "reason": reason, + } + ) + + +def _fill_effect_analysis_n_from_size( + study: dict[str, Any], + study_total_n: int, + repairs: list[dict[str, Any]], +) -> None: + for effect_index, effect in enumerate(study.get("effects", [])): + if not isinstance(effect, dict): + continue + size = _coerce_int(effect.get("size")) + existing_analysis_n = _coerce_int(effect.get("analysis_n")) + if existing_analysis_n == study_total_n: + before_n = effect.pop("analysis_n", None) + before_scope = effect.pop("analysis_scope", None) + repairs.append( + { + "path": f"{study.get('study', 'study')}.effects[{effect_index}].analysis_n", + "study": study.get("study"), + "effect_index": effect_index, + "field": "analysis_n", + "before": before_n, + "after": None, + "action": "removed_redundant_full_sample_analysis_n", + "analysis_scope_before": before_scope, + } + ) + continue + if size is None or size == study_total_n or effect.get("analysis_n") is not None: + continue + effect["analysis_n"] = size + effect.setdefault("analysis_scope", _infer_analysis_scope(effect, size, study_total_n)) + repairs.append( + { + "path": f"{study.get('study', 'study')}.effects[{effect_index}].analysis_n", + "study": study.get("study"), + "effect_index": effect_index, + "field": "analysis_n", + "before": None, + "after": size, + "action": "filled_from_effect_size", + } + ) + + +def _infer_analysis_scope(effect: dict[str, Any], analysis_n: int, study_total_n: int | None) -> str: + effecttype = normalize_for_match(effect.get("effecttype") or "") + if effecttype == "simple": + return "simple_effect" + if study_total_n is not None and analysis_n < study_total_n: + return "subgroup" + if effecttype == "int": + return "subgroup" + return "other" + + +def _coerce_int(value: Any) -> int | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, str): + text = value.replace(",", "").strip() + if re.fullmatch(r"\d+", text): + return int(text) + return None + + +def _parse_int(value: str) -> int: + return int(value.replace(",", "")) + + +def _iter_effect_analysis_claims(data: dict[str, Any]) -> Iterable[dict[str, Any]]: + """Yield analysis_n claims from effect-level fields. + + effect.analysis_n is the N used in a *specific* analysis (e.g. a subgroup, + a simple-effect cell, or a condition subset). It is separate from + study.sample.total_n (the full recruited study N). + """ + for study_index, study in enumerate(data.get("eligible_studies", [])): + if not isinstance(study, dict): + continue + study_name = study.get("study") + for effect_index, effect in enumerate(study.get("effects", [])): + if not isinstance(effect, dict): + continue + analysis_n = effect.get("analysis_n") + if analysis_n is not None and isinstance(analysis_n, (int, float)) and not isinstance(analysis_n, bool): + yield _sample_claim( + f"$.eligible_studies[{study_index}].effects[{effect_index}].analysis_n", + study_name, + effect_index, + "analysis_n", + int(analysis_n), + ) + + +def _difflib_partial_ratio(needle: str, haystack: str) -> float: + import difflib + + if needle in haystack: + return 100.0 + window = min(max(len(needle) * 2, 500), max(len(haystack), 1)) + step = max(len(needle) // 2, 200) + best = 0.0 + for start in range(0, max(len(haystack) - window + 1, 1), step): + chunk = haystack[start : start + window] + best = max(best, difflib.SequenceMatcher(None, needle, chunk).ratio() * 100.0) + if best >= 99.0: + break + return best + + +def _iter_slots(data: dict[str, Any]) -> Iterable[dict[str, Any]]: + for study_index, study in enumerate(data.get("eligible_studies", [])): + if not isinstance(study, dict): + continue + study_name = study.get("study") + for effect_index, effect in enumerate(study.get("effects", [])): + if not isinstance(effect, dict): + continue + for slot_name in SLOT_NAMES: + slot = effect.get(slot_name) + if not isinstance(slot, dict): + continue + yield { + "path": f"$.eligible_studies[{study_index}].effects[{effect_index}].{slot_name}", + "study": study_name, + "effect_index": effect_index, + "slot": slot_name, + "slot_obj": slot, + } + + +def _empty_required_slot_records(data: dict[str, Any]) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for slot_ref in _iter_slots(data): + slot = slot_ref["slot_obj"] + status = slot.get("status") + content = slot.get("content") + if isinstance(content, str) and content.strip(): + continue + if content is not None and not isinstance(content, str): + continue + records.append( + { + "path": slot_ref["path"], + "study": slot_ref["study"], + "effect_index": slot_ref["effect_index"], + "slot": slot_ref["slot"], + "status": status, + "reason": "required slot has no content", + } + ) + return records + + +def _count_empty_slots_by_status(records: list[dict[str, Any]]) -> dict[str, int]: + counts: dict[str, int] = {} + for record in records: + status = record.get("status") + key = "null" if status is None else str(status) + counts[key] = counts.get(key, 0) + 1 + return dict(sorted(counts.items())) + + +def _iter_sample_claims(data: dict[str, Any]) -> Iterable[dict[str, Any]]: + """Iterate verifiable claims from study-level sample objects. + + New schema: sample lives at study level, not per-effect. + Falls back to effect[0].sample for backward-compat with old JSONs. + """ + for study_index, study in enumerate(data.get("eligible_studies", [])): + if not isinstance(study, dict): + continue + study_name = study.get("study") + + # New format: study.sample + sample = study.get("sample") + if isinstance(sample, (dict, str)): + base_path = f"$.eligible_studies[{study_index}].sample" + if isinstance(sample, str): + yield _sample_claim(base_path, study_name, 0, "sample", sample) + else: + yield from _iter_study_sample_claims(sample, base_path, study_name) + continue + + # Backward compat: old effect-level sample (effect[0] only to avoid duplication) + effects = study.get("effects") or [] + if effects and isinstance(effects[0], dict): + old_sample = effects[0].get("sample") + if isinstance(old_sample, dict): + base_path = f"$.eligible_studies[{study_index}].effects[0].sample" + yield from _iter_study_sample_claims( + _migrate_old_sample_for_verify(old_sample), base_path, study_name + ) + + +def _migrate_old_sample_for_verify(old: dict[str, Any]) -> dict[str, Any]: + """Best-effort convert old nested sample → flat for verification pass.""" + from generation_pipeline.verification.schema_validator import _migrate_old_sample + try: + return _migrate_old_sample(old) + except Exception: + return {} + + +def _iter_study_sample_claims( + sample: dict[str, Any], + base_path: str, + study_name: str | None, +) -> Iterable[dict[str, Any]]: + """Yield one claim per non-null sample field for the new flat study.sample schema.""" + # Numeric fields — verified as number-in-source (no context requirement) + for field in ("total_n", "analyzed_n"): + val = sample.get(field) + if val is not None: + yield _sample_claim(f"{base_path}.{field}", study_name, 0, field, val) + + # Float fields — age in years, gender in 0-100 percent + for field in ("mean_age", "female_percent", "male_percent"): + val = sample.get(field) + if val is not None: + yield _sample_claim(f"{base_path}.{field}", study_name, 0, field, val) + + # Categorical / short string fields — platform alias match or substring + for field in ("platform", "country"): + val = sample.get(field) + if val is not None: + yield _sample_claim(f"{base_path}.{field}", study_name, 0, field, val) + + # Verbatim text fields — fuzzy match ≥ 90% required + for field in ("inclusion_criteria", "exclusion_criteria", "notes"): + val = sample.get(field) + if val is not None: + yield _sample_claim(f"{base_path}.{field}", study_name, 0, field, val) + + +def _sample_claim( + path: str, + study_name: str | None, + effect_index: int, + field: str, + value: Any, +) -> dict[str, Any]: + return { + "path": path, + "study": study_name, + "effect_index": effect_index, + "field": field, + "value": value, + } + + +def _empty_sample_value(value: Any) -> bool: + if value is None: + return True + if isinstance(value, str) and not value.strip(): + return True + return False + + +def _verify_sample_string_claim( + claim: dict[str, Any], + value: str, + source_windows: list[str], + normalized_source: str, +) -> SampleRecord: + field = claim["field"] + + # Platform: check against known aliases (substring match) + if field == "platform": + evidence = _find_platform_alias_window(value, source_windows) + return _sample_record( + claim, value, + evidence_kind="platform_alias_substring", + action="passed" if evidence else "unverified_platform_not_found", + evidence=evidence, + ) + + # Country: simple case-insensitive substring match + if field == "country": + evidence = _find_substring_window(value, source_windows) + return _sample_record( + claim, value, + evidence_kind="country_substring", + action="passed" if evidence else "unverified_country_not_found", + evidence=evidence, + ) + + # Verbatim text fields (inclusion_criteria, exclusion_criteria, notes): + # require fuzzy match ≥ threshold against full source. + if field in ("inclusion_criteria", "exclusion_criteria", "notes"): + score = best_partial_ratio(normalize_for_match(value), normalized_source) + passed = score >= DEFAULT_THRESHOLD + return _sample_record( + claim, value, + evidence_kind="fuzzy_verbatim", + action="passed" if passed else "unverified_verbatim_below_threshold", + evidence=f"score={score:.1f}" if passed else None, + ) + + # Legacy fallback: normalized exact text + evidence = _find_normalized_exact_window(value, source_windows, normalized_source) + return _sample_record( + claim, value, + evidence_kind="normalized_exact_text", + action="passed" if evidence else "unverified_no_exact_source_span", + evidence=evidence, + ) + + +def _verify_sample_numeric_claim( + claim: dict[str, Any], + value: int | float, + source_windows: list[str], +) -> SampleRecord: + field = claim["field"] + if field == "total_n": + source_total = _find_study_level_total_n(claim.get("study"), source_windows) + value_int = _coerce_int(value) + if source_total is not None: + evidence = None + action = "unverified_not_study_final_n" + if value_int == source_total: + evidence = _find_numeric_evidence( + _numeric_variants(source_total, field), + _numeric_context_terms(field), + source_windows, + ) + action = "passed" if evidence else "unverified_number_not_found" + return _sample_record( + claim, value, + evidence_kind="study_final_n_exact", + action=action, + evidence=evidence, + ) + + variants = _numeric_variants(value, field) + # N fields: just find the number anywhere — no context required. + # Age/percent fields: same — the number itself is distinctive enough. + require_context = _numeric_requires_context(field) + context_terms = _numeric_context_terms(field) if require_context else () + evidence = _find_numeric_evidence(variants, context_terms, source_windows) + return _sample_record( + claim, value, + evidence_kind="numeric_in_source" if not require_context else "numeric_context_exact", + action="passed" if evidence else "unverified_number_not_found", + evidence=evidence, + ) + + +def _sample_record( + claim: dict[str, Any], + value: Any, + *, + evidence_kind: str, + action: str, + evidence: str | None, +) -> SampleRecord: + return SampleRecord( + path=claim["path"], + study=claim["study"], + effect_index=claim["effect_index"], + field=claim["field"], + value_preview=_preview(value), + evidence_kind=evidence_kind, + action=action, + evidence_preview=_preview(evidence, limit=320) if evidence else None, + ) + + +def _source_windows(source_text: str) -> list[str]: + text = source_text or "" + candidates: list[str] = [] + candidates.extend(part.strip() for part in re.split(r"\n\s*\n+", text) if part.strip()) + candidates.extend(part.strip() for part in re.split(r"(?<=[.!?])\s+", text) if part.strip()) + if not candidates and text.strip(): + candidates.append(text.strip()) + + windows: list[str] = [] + for candidate in candidates: + if len(candidate) <= 1200: + windows.append(candidate) + continue + for start in range(0, len(candidate), 900): + chunk = candidate[start : start + 1200].strip() + if chunk: + windows.append(chunk) + return windows + + +def _find_normalized_exact_window( + value: str, + source_windows: list[str], + normalized_source: str, +) -> str | None: + normalized_value = normalize_for_match(value) + if not normalized_value: + return None + for window in source_windows: + if normalized_value in normalize_for_match(window): + return window + if normalized_value in normalized_source: + return "Exact normalized text is present in the full source corpus, but no compact source window was isolated." + return None + + +def _find_substring_window(value: str, source_windows: list[str]) -> str | None: + """Case-insensitive substring search across source windows.""" + needle = value.lower().strip() + if not needle: + return None + for window in source_windows: + if needle in window.lower(): + return window + return None + + +def _find_platform_alias_window(value: str, source_windows: list[str]) -> str | None: + aliases = SAMPLE_PLATFORM_ALIASES.get(normalize_for_match(value), (normalize_for_match(value),)) + aliases = tuple(alias for alias in aliases if alias) + if not aliases: + return None + for window in source_windows: + normalized_window = normalize_for_match(window) + if any(alias in normalized_window for alias in aliases): + return window + return None + + +def _numeric_variants(value: int | float, field: str) -> list[str]: + """Return candidate string forms of a numeric value to search for in source text. + + Handles integer N values, age means, and gender percentages (always 0-100 in + the new schema but we also search for the 0-1 proportion form as a fallback + since papers sometimes report either). + """ + variants: set[str] = set() + if isinstance(value, int) or (isinstance(value, float) and float(value).is_integer()): + integer = int(value) + variants.add(str(integer)) + variants.add(f"{integer:,}") # thousand separator + variants.add(f"n = {integer}") + variants.add(f"n={integer}") + variants.add(f"n = {integer:,}") + variants.add(f"n={integer:,}") + variants.add(f"n= {integer}") + else: + number = float(value) + variants.add(_format_float(number)) + variants.add(f"{number:.2f}") + variants.add(f"{number:.1f}") + + # Gender percentages: new schema stores 0-100; paper may write as % or proportion. + if field in ("female_percent", "male_percent"): + # 0-100 forms + variants.add(f"{number:.2f}%") + variants.add(f"{number:.1f}%") + variants.add(f"{number:.0f}%") + # proportion form (÷100) + prop = number / 100.0 + variants.add(_format_float(prop)) + variants.add(f"{prop:.4f}") + variants.add(f"{prop:.3f}") + variants.add(f"{prop:.2f}") + + # Age mean: common formats + if field == "mean_age": + variants.add(f"m = {number:.2f}") + variants.add(f"m={number:.2f}") + variants.add(f"mage = {number:.2f}") + variants.add(f"mage={number:.2f}") + variants.add(f"mean age = {number:.2f}") + + return sorted(variants, key=len, reverse=True) + + +def _format_float(value: float) -> str: + return f"{value:.6f}".rstrip("0").rstrip(".") + + +def _numeric_requires_context(field: str) -> bool: + """Return True when the bare number is ambiguous and a context word is needed. + + `total_n` REQUIRES context to avoid confusing the study-level recruited N + with a subgroup / analysis / cell N that appears elsewhere in the paper + (e.g. in a Table row reporting a simple-effect with n = 958). + + `analyzed_n` also requires participant-language context for the same reason. + + All other numeric fields (age, gender %) are distinctive enough that the + bare number is sufficient. + + `analysis_n` (effect-level) intentionally uses NO context — it is supposed + to match a stats-adjacent subgroup number; see _iter_effect_analysis_claims. + """ + return field in ("total_n", "analyzed_n") + + +def _numeric_context_terms(field: str) -> tuple[str, ...]: + """Context keywords that must appear in the same source window as the number.""" + if field in ("total_n", "analyzed_n"): + # Study-level N must be near participant/recruitment/final-N language. + # Do not use bare "sample": analysis tables often have a "Sample" column + # header next to subgroup Ns, which are effect.analysis_n, not total_n. + return ( + "participant", "participants", "subject", "subjects", + "recruited", "enrolled", "respondent", "respondents", + "final n", "final sample", "our final sample", + "completed the study", + ) + if field == "mean_age": + return ("age", "mage", "mean age", "years") + if field == "female_percent": + return ("female", "women", "woman") + if field == "male_percent": + return ("male", "men", "man") + return () + + +def _find_numeric_evidence( + variants: list[str], + context_terms: tuple[str, ...], + source_windows: list[str], +) -> str | None: + for window in source_windows: + normalized_window = normalize_for_match(window) + if not any(_contains_numeric_variant(normalized_window, variant) for variant in variants): + continue + if context_terms and not any(term in normalized_window for term in context_terms): + continue + return window + return None + + +def _contains_numeric_variant(normalized_window: str, variant: str) -> bool: + normalized_variant = normalize_for_match(variant) + if not normalized_variant: + return False + # Use \d only (not [\d.]) so that end-of-sentence periods (e.g., "n = 958.") don't block + # a match. We still prevent matches inside longer numbers ("1958" won't match "958"). + return re.search(rf"(? tuple[float, list[dict[str, Any]]]: + """Score a verbatim slot. + + Multi-span Stage 3 content is stored as labeled blocks: + + [intro] + exact quote A + + [manipulation: public] + exact quote B + + Those labels and joins are not a contiguous source quote, so verifying the + whole string creates false negatives. If labeled spans are present, score + each quote separately and use the weakest span as the slot score. + """ + spans = _labeled_verbatim_spans(content) + if not spans: + return best_partial_ratio(normalize_for_match(content), normalized_source), [] + + span_scores: list[dict[str, Any]] = [] + weakest = 100.0 + for label, quote in spans: + score = best_partial_ratio(normalize_for_match(quote), normalized_source) + weakest = min(weakest, score) + span_scores.append( + { + "label": label, + "score": round(score, 2), + "chars": len(quote), + "content_preview": _preview(quote), + } + ) + return weakest, span_scores + + +def _labeled_verbatim_spans(content: str) -> list[tuple[str, str]]: + pattern = re.compile( + r"(?:^|\n)\[(?P
HumanStudy-Bench Logo -

HumanStudy-Bench: Community Edition

-

Open community-driven expansion of the HumanStudy-Bench benchmark

- - Read the Paper - License: MIT - Contributions Welcome - Docs +

HumanStudy-Bench

+

A benchmark for evaluating AI agents through runnable human studies

+ Read the Paper + MIT License + Contributions welcome
--- -> LLMs are increasingly used to simulate human participants in social science research, but existing evaluations conflate base model capabilities with agent design choices, making it unclear whether results reflect the model or the configuration. - -## Overview - -HumanStudy-Bench treats participant simulation as an *agent design problem* and provides a **standardized testbed** — combining an **Execution Engine** that reconstructs full experimental protocols from published studies and a **Benchmark** with standardized evaluation metrics — for *replaying human-subject experiments end-to-end* with alignment evaluation at the level of scientific inference. - -## Automated Study Generation Pipeline - -This repo also includes a staged generation pipeline for drafting a Hub study -folder from a paper PDF and optional OSF/supplementary sources. - -The pipeline is intentionally human-in-the-loop: - -- Stage 1 compiles a source-grounded inventory of empirical units and - source-explicit comparison groups from any social-science topic. -- Stage 2 extracts study-level findings, effects, samples, and statistics. -- Stage 3 assembles and audits participant-facing materials from - OSF/QSF/SAV/PDF sources. -- Stage 4 writes a Hub study folder under `studies//`. -- Stage 5 can run simulation after `scripts/config.py` exists and the materials - pass the package-readiness gate. - -Install the LLM and layout-aware PDF dependencies for generation: - -```bash -python -m pip install -e ".[llm,pdf]" -``` +HumanStudy-Bench is a standardized testbed for replaying human-subject +experiments with AI agents. It combines: -Docling is the primary PDF parser. It preserves page/block provenance and table -structure; image-dominant PDFs additionally use Docling's RapidOCR path. The -first parse may download Docling model weights. If Docling is unavailable, the -pipeline records a degraded pypdf fallback and will not silently treat flat text -as layout-complete evidence. +- an execution engine that reconstructs published experimental protocols; +- a growing collection of runnable social-science studies; and +- evaluation at the level of behavior, effects, and scientific findings. -Copy the settings template and configure API credentials locally: +The 12 foundational studies cover cognition, strategic interaction, and social +psychology. Community studies live alongside them under [`studies/`](studies/). -```bash -cp config/settings.example.yaml config/settings.yaml -``` - -Do not commit `config/settings.yaml`. +## Add a study -### Stage 1: evidence-grounded study inventory +There are two contribution paths. -Stage 1 is domain-independent. It inventories every reported empirical unit in -a social-science paper, including experiments, surveys, pilots, validation -samples, field studies, and observational studies. It then determines whether -each human-participant task can be represented in HumanStudy-Bench. Psychology, -behavioral economics, organizational behavior, political science, sociology, -communication, marketing, education, and HCI use the same task definition; no -ethics-specific topic or outcome filter remains. +| Path | Best for | Current status | +|---|---|---| +| **Build Study** | Researchers starting from a paper PDF and optional open materials | **Private beta.** HumanStudy-Hub runs this repository's pipeline, persists review decisions, and produces a ZIP or GitHub pull request. | +| **Direct pull request** | Contributors who want full control over study files | **Available now.** Build the folder locally, run validation, and open a PR. | -The complete PDF is never placed in one LLM prompt. Stage 1 instead uses this -evidence compiler: +The intended Build Study workflow is: ```text -Docling layout-aware parse (once) - -> targeted second OCR only for ambiguous numeric/statistical tokens - -> bounded overlapping windows covering every parsed block - -> parallel high-recall empirical-unit and relationship discovery - -> deterministic reconciliation by source labels and assigned variants - -> bounded global candidate-ledger adjudication of simulation task families - -> shared sample/procedure evidence attached without merging different tasks - -> one bounded, evidence-anchored extraction per task family - -> provenance and empirical-support gate - -> task families + nested material variants + source-explicit comparison groups - -> bounded-window coverage audit; global ledger remains boundary authority - -> complete cited-evidence verifier for each study's fields and eligibility - -> deterministic numeric/OCR conflict hints that every study audit must adjudicate - -> typed field and eligibility repair with boundary-audit reuse - -> full recompilation only for grounded missing-unit or boundary changes - -> accept a refinement only when deterministic audit quality improves -``` - -The three output levels are intentional. `experiments[]` contains coherent -simulation task families, `experiments[].material_variants[]` preserves stories, -forms, orders, stimuli, and conditions that Stage 3 may need to reconstruct -separately, and `comparison_groups[]` records task families that the paper -explicitly combines in one contrast or finding. Different tasks remain separate -even when they share one questionnaire and participant pool; repeated or -parameterized items remain together when they use one task template and response -format. Questionnaire-wide sample/procedure evidence is retained under -`shared_contexts` instead of becoming a fake study or comparison group. Table -rows, outcome groups, and individual effects are never promoted to studies. A -material variant must be a genuine alternative version assigned across -participants or occasions; response options shown together inside one question -remain part of that question rather than becoming separate variants. - -Every accepted unit records PDF block/page evidence, `unit_provenance`, whether -it is a distinct empirical unit, and direct support for its sample, participant -task, and quantitative target. Cited prior work and isolated result fragments -are rejected from the current-paper inventory. Missing exact questionnaire -wording or supplementary files is recorded as a Stage 3 material gap and does -not by itself exclude an otherwise simulatable unit. A real task with no exact -numeric response result remains in the inventory but is labeled `NO`; stimulus -amounts, response-scale endpoints, and qualitative majority/significance claims -do not count as quantitative target results. `replicable` remains the legacy -JSON field name; it means simulation eligibility, not a guarantee that the -original result will reproduce. - -`simulation_barriers` separately records source-grounded execution requirements -such as observed physical action or live interaction. A barrier only forces a -`NO` label when it affects the paper's original quantitative target; it cannot -be bypassed by silently replacing that target with a hypothetical proxy. - -Run Stage 1 with the default four bounded-call workers: - -```bash -python generation_pipeline/run.py \ - --settings config/settings.yaml \ - --stage 1 \ - --pdf /absolute/path/to/paper.pdf \ - --provider openai \ - --model gpt-5-mini \ - --stage1-workers 4 \ - --stage1-timeout 300 \ - --stage1-verifier-timeout 300 -``` - -Inspect `generation_pipeline/outputs//stage1.json` and `stage1.md`. -The JSON is ready to pass to Stage 2 only when -`stage1_evidence.extraction_complete` and -`stage1_evidence.all_comparison_relations_resolved` are true, -`stage1_study_contract.blocking_issue_count` is zero, and -`stage1_verification.study_audit.all_cited_evidence_included` is true, and -`stage1_verification.overall` is `pass`. A paper window is never allowed to -invalidate a study field merely because that field's evidence appears in a -different section; field checks run against the study's complete cited-evidence -bundle instead. Field-only refinement never reruns discovery or changes the -accepted boundary audit, and a failed refinement candidate cannot replace the -last valid result. - -### PDF-only Stage 3 - -OSF and supplementary material handling remains the preferred path when those -sources exist. For PDF-only papers, Stage 3 uses this fail-closed flow: - -```text -Docling/RapidOCR parse and page images - -> page-grounded evidence index and study-specific retrieval - -> participant blocks vs non-runtime source structures - -> semantic table linking and visual cell recovery/adjudication - -> source-axis runtime plan with an independent plan audit - -> bounded parallel unit compilation and per-unit source audit - -> whole-instrument semantic verification and constrained repair - -> coverage ledger and ready/not-ready decision -``` - -Tables used to construct conditions are retained as non-runtime evidence. Only -source-supported questions, options, scales, matrices, and instructions become -participant-facing material. Missing questionnaire wording is reported as a -blocking source absence; construct names, results summaries, and placeholders -cannot be substituted merely to make a package runnable. - -Example using `gpt-5-mini` after Stages 1 and 2 have produced `stage2.json`: - -```bash -python generation_pipeline/run.py \ - --settings config/settings.yaml \ - --stage 3 \ - --json generation_pipeline/outputs//stage2.json \ - --pdf /absolute/path/to/paper.pdf \ - --provider openai \ - --model gpt-5-mini \ - --stage3-select-votes 1 \ - --stage3-select-timeout 120 \ - --stage3-pdf-timeout 300 \ - --no-backup +paper PDF + optional OSF/open materials + -> evidence and study extraction + -> researcher review at uncertain decisions + -> runnable HumanStudy-Bench folder + -> download ZIP and/or save to GitHub ``` -Stage 3 writes `stage3.json`, `stage3.md`, and reusable parser/compiler evidence -under `generation_pipeline/outputs//pdf_artifacts/`, including -`blocks.json`, `parser_report.json`, page images, table-link/recovery caches, and -runtime compiler plans/units. A selected material is not necessarily ready: -inspect `study_materials..readiness`, `coverage_ledger`, and -`source_trace.semantic_verifier` before Stage 4. +Build Study is currently operated through the private HumanStudy-Hub web +application. The direct contribution documentation remains supported for +researchers who prefer to author or inspect every file themselves. -### Stage 4 Hub package +### Direct pull request -Example Stage 4 Hub package generation from an existing `stage3.json`: +1. Fork this repository and clone your fork. ```bash -python generation_pipeline/run.py \ - --settings config/settings.yaml \ - --stage 4 \ - --json generation_pipeline/outputs//stage3.json \ - --pdf /absolute/path/to/paper.pdf \ - --study-id study_my_generated_study \ - --provider openai \ - --model gpt-5-mini \ - --hub-layout \ - --hub-studies-dir studies +git clone https://github.com//HumanStudy-Bench.git +cd HumanStudy-Bench +git checkout -b contrib-- ``` -In Hub layout, Stage 4 writes: +2. Add a study folder. ```text -studies// +studies/_/ index.json README.md source/ - metadata.json - specification.json - ground_truth.json - source_extraction.json - audit.json - materials/*.json scripts/ - study_utils.py - config.py # generated by the Stage 4 LLM config generator when configured - evaluator.py # placeholder; fill after inspecting Stage 5 responses ``` -Stage 4 compiles `source/materials/*.json` deterministically from Stage 3 -`study_materials`. It also derives metadata findings, specification coverage, -and ground-truth references deterministically from the complete structured Stage -3 payload, so a long extraction cannot be truncated to one sub-study. The LLM may -enrich metadata domain/keywords and generates adapter code, but it does not -regenerate participant-facing items or decide finding/material coverage. - -`scripts/config.py` is Stage 5 input, not Stage 5 output. It is generated by -the Stage 4 config generator when a Stage 4 LLM is configured; otherwise Stage 4 -marks config generation as skipped. `scripts/evaluator.py` is deliberately a -placeholder because reliable evaluator code usually needs response samples from -a Stage 5 smoke run plus human review. - -Stage 5 enforces `source/audit.json` for pipeline-generated packages and refuses -to run while `ready_for_simulation` is false. For an explicit debugging-only -smoke run, pass `--allow-unready`; the override is recorded in the Stage 5 JSON. -Stage 4 also imports and validates the generated config adapter. Missing or -invalid adapter code makes the package unready even when its JSON files exist. - -## How to Contribute a Study - -### 1. Fork and clone +3. Validate it locally. ```bash -git clone https://github.com//HumanStudy-Bench.git -cd HumanStudy-Bench -git checkout -b contrib--013 +bash scripts/verify_study.sh _ ``` -### 2. Create your study folder +4. Commit the study, push your branch, and open a pull request targeting + `main`. CI validates the package and contributor attribution; maintainers + complete the human review and assign the final study ID. -Add a new directory under `studies/` with the required folders: +Contributor guides: -``` -studies/_013/ - ├── index.json - ├── source/ - ├── scripts/ - └── README.md -``` - -See the docs below for what goes inside each folder and the exact schemas: +- [What should I submit?](docs/what_to_submit.md) +- [How to extract data from a paper](docs/extract_from_paper.md) +- [How to build study files](docs/build_study_files.md) +- [How to submit a study](docs/submit_study.md) -| # | Guide | Description | -|---|-------|-------------| -| 1 | [What Should I Submit?](https://www.hs-bench.clawder.ai/docs/what_to_submit) | Overview of contribution, required folders and files | -| 2 | [How to Extract Data from a Paper](https://www.hs-bench.clawder.ai/docs/extract_from_paper) | Paper hierarchy, AI extraction prompt, walkthrough example | -| 3 | [How to Build Your Study Files](https://www.hs-bench.clawder.ai/docs/build_study_files) | Schemas, code examples, and contracts for each file | -| 4 | [How to Submit Your Study](https://www.hs-bench.clawder.ai/docs/submit_study) | Fork, verify, push, and open a PR | +## Generation pipeline -### 3. Verify locally +This repository includes a human-in-the-loop pipeline for drafting a study +folder from a paper PDF and optional OSF or supplementary sources. -```bash -bash scripts/verify_study.sh _013 -``` +1. Inventory empirical units and comparison groups. +2. Extract findings, effects, samples, and statistics. +3. Assemble and audit participant-facing materials. +4. Write a package under `studies//`. +5. Run a simulation only after the package passes readiness review. -### 4. Commit and push +Install the generation dependencies: ```bash -git add studies/_013/ -git commit -m "Add study: " -git push origin contrib--013 +python -m pip install -e ".[llm,pdf]" +cp config/settings.example.yaml config/settings.yaml ``` -### 5. Open a Pull Request +Do not commit `config/settings.yaml`. -Open a PR on GitHub targeting the `main` branch. Maintainers assign final `study_XXX` numbering by merge order. CI runs validation automatically; confirmation is by human review. +Run a stage with: -You can also submit a study via **web upload** at [hs-bench.clawder.ai/contribute](https://www.hs-bench.clawder.ai/contribute). +```bash +python generation_pipeline/run.py \ + --settings config/settings.yaml \ + --stage 1 \ + --pdf /absolute/path/to/paper.pdf \ + --provider openai \ + --model gpt-5-mini +``` -## Existing Studies +Generated evidence and review artifacts are written under +`generation_pipeline/outputs//`. Pipeline-generated packages remain +blocked from simulation until `source/audit.json` marks them ready. The pipeline +entry point is [`generation_pipeline/run.py`](generation_pipeline/run.py); run +it with `--help` for the full stage and review options. -The 12 foundational studies (cognition, strategic interaction, social psychology) serve as reference examples. Browse them on the [website](https://www.hs-bench.clawder.ai/contribute#studies) or locally under `studies/`. +## Repository structure -## Citation +```text +studies/ Runnable benchmark studies +src/ Execution and evaluation code +generation_pipeline/ PDF/material extraction and package generation +scripts/ Study validation and index utilities +docs/ Manual contribution guides +tests/ Pipeline and benchmark tests +``` -If you use HumanStudy-Bench, please cite: +## Citation ```bibtex @misc{liu2026humanstudybenchaiagentdesign, @@ -316,6 +143,6 @@ If you use HumanStudy-Bench, please cite: } ``` -## License +## License -MIT License. See [LICENSE](LICENSE) for details. +MIT License. See [LICENSE](LICENSE). diff --git a/co_website/.gitignore b/co_website/.gitignore deleted file mode 100644 index 5ef6a52..0000000 --- a/co_website/.gitignore +++ /dev/null @@ -1,41 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -/node_modules -/.pnp -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/versions - -# testing -/coverage - -# next.js -/.next/ -/out/ - -# production -/build - -# misc -.DS_Store -*.pem - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.pnpm-debug.log* - -# env files (can opt-in for committing if needed) -.env* - -# vercel -.vercel - -# typescript -*.tsbuildinfo -next-env.d.ts diff --git a/co_website/README.md b/co_website/README.md deleted file mode 100644 index 9b95270..0000000 --- a/co_website/README.md +++ /dev/null @@ -1,41 +0,0 @@ -## co_website (HumanStudy-Bench web app) - -Next.js app for: -- Homepage sections (`Overview`, `Leaderboard`, `Dataset`) -- Study catalog and study detail pages -- Contribute flow (upload + PR creation) -- Docs tab (`/docs`) - -## Local preview - -```bash -cd co_website -npm install -npm run dev -``` - -Open `http://localhost:3000`. - -## Build checks - -```bash -cd co_website -npm run lint -npm run build -``` - -## Study ZIP download - -ZIP downloads are served by: -- `GET /api/studies/:studyId/zip` - -This endpoint packages `studies/:studyId` directly from the repository. - -## Refreshing effect-size plot data - -Homepage effect-size scatter reads: -- `public/data/effects/gemini_flash_v4_effect_data.json` - -To refresh: -1. From repo root: `python scripts/plot_single_fig3_effects.py --export-data /absolute/path/to/co_website/public/data/effects/gemini_flash_v4_effect_data.json` -2. Rebuild/redeploy the site. diff --git a/co_website/adm-zip.d.ts b/co_website/adm-zip.d.ts deleted file mode 100644 index 1caa077..0000000 --- a/co_website/adm-zip.d.ts +++ /dev/null @@ -1,14 +0,0 @@ -declare module "adm-zip" { - interface ZipEntry { - isDirectory: boolean; - entryName: string; - getData(): Buffer; - } - - class AdmZip { - constructor(buffer?: Buffer); - getEntries(): ZipEntry[]; - } - - export default AdmZip; -} diff --git a/co_website/app/api/contribute/upload/route.ts b/co_website/app/api/contribute/upload/route.ts deleted file mode 100644 index 3aa42a0..0000000 --- a/co_website/app/api/contribute/upload/route.ts +++ /dev/null @@ -1,288 +0,0 @@ -import { NextRequest, NextResponse } from "next/server"; - -const MAX_ZIP_BYTES = 50 * 1024 * 1024; // 50MB - -function getStudyIdFromPath(entryName: string): string | null { - const m = entryName.match(/^(?:data\/studies\/)?(study_\d{3})\//) || entryName.match(/^(study_\d{3})\//); - return m ? m[1] : null; -} - -function parseJson(buf: Buffer): unknown { - return JSON.parse(buf.toString("utf8")); -} - -function normalizeGithub(input: string): string { - const raw = input.trim(); - if (!raw) return ""; - if (raw.startsWith("http://") || raw.startsWith("https://")) return raw; - return `https://github.com/${raw.replace(/^@/, "")}`; -} - -type ValidationResult = { ok: true; studyId: string; files: Map } | { ok: false; errors: string[] }; - -function validateZip(entries: { name: string; data: Buffer }[]): ValidationResult { - const errors: string[] = []; - const fileMap = new Map(); - for (const e of entries) { - if (e.data.length) fileMap.set(e.name, e.data); - } - - let studyId: string | null = null; - for (const name of fileMap.keys()) { - const id = getStudyIdFromPath(name) || (name.startsWith("study_") && name.match(/^study_\d{3}/)?.[0]); - if (id) { - studyId = id; - break; - } - } - if (!studyId) { - const dataStudy = [...fileMap.keys()].find((k) => k.includes("studies/study_")); - const match = dataStudy?.match(/study_\d{3}/); - studyId = match ? match[0] : null; - } - if (!studyId) { - errors.push("Zip must contain a study_XXX directory (e.g. studies/study_014/ or study_014/)."); - return { ok: false, errors }; - } - - const dataPrefix = `studies/${studyId}/`; - const dataPrefixAlt = `${studyId}/`; - const getData = (path: string) => - fileMap.get(dataPrefix + path) || fileMap.get(dataPrefixAlt + path); - - const specBuf = getData("specification.json"); - const metaBuf = getData("metadata.json"); - const gtBuf = getData("ground_truth.json"); - if (!specBuf) errors.push("Missing specification.json"); - if (!metaBuf) errors.push("Missing metadata.json"); - if (!gtBuf) errors.push("Missing ground_truth.json"); - - let spec: Record; - let meta: Record; - let gt: Record; - try { - spec = specBuf ? (parseJson(specBuf) as Record) : {}; - meta = metaBuf ? (parseJson(metaBuf) as Record) : {}; - gt = gtBuf ? (parseJson(gtBuf) as Record) : {}; - } catch { - errors.push("Invalid JSON in specification, metadata, or ground_truth."); - return { ok: false, errors }; - } - - if (spec) { - if (!spec.participants || typeof (spec.participants as Record)?.n !== "number") - errors.push("specification.json must have participants.n (number)."); - if (!spec.design) errors.push("specification.json must have design."); - if (!spec.procedure) errors.push("specification.json must have procedure."); - } - if (meta) { - if (!meta.id && !meta.title) errors.push("metadata.json must have id or title."); - if (!meta.authors) errors.push("metadata.json must have authors."); - if (!meta.domain) errors.push("metadata.json must have domain."); - if (!meta.findings) errors.push("metadata.json must have findings."); - } - const studies = gt?.studies as Array> | undefined; - if (!Array.isArray(studies) || studies.length === 0) { - errors.push("ground_truth.json must have a non-empty studies array."); - } - - const materialsEntries = [...fileMap.entries()].filter( - ([n]) => n.startsWith(dataPrefix + "materials/") || n.startsWith(dataPrefixAlt + "materials/") - ); - if (materialsEntries.length === 0) errors.push("At least one materials/*.json file is required."); - const gtKeys = new Set(); - if (Array.isArray(studies)) { - for (const s of studies) { - const findings = s.findings as Array> | undefined; - if (!Array.isArray(findings)) continue; - for (const f of findings) { - const odp = f.original_data_points as Record | undefined; - const data = odp?.data as Record | undefined; - if (data && typeof data === "object") Object.keys(data).forEach((k) => gtKeys.add(k)); - } - } - } - for (const [name, buf] of materialsEntries) { - if (!name.endsWith(".json")) continue; - try { - const mat = parseJson(buf) as Record; - if (!mat.sub_study_id) errors.push(`Materials file ${name} must have sub_study_id.`); - if (!mat.instructions) errors.push(`Materials file ${name} must have instructions.`); - const items = mat.items as Array> | undefined; - if (!Array.isArray(items)) { - errors.push(`Materials file ${name} must have items array.`); - } else { - for (const item of items) { - const metaItem = item.metadata as Record | undefined; - const gk = metaItem?.gt_key as string | undefined; - if (gk && !gtKeys.has(gk)) errors.push(`gt_key "${gk}" in ${name} not found in ground_truth data.`); - } - } - } catch { - errors.push(`Invalid JSON in materials file ${name}.`); - } - } - - const configName = `src/studies/${studyId}_config.py`; - const evaluatorName = `src/studies/${studyId}_evaluator.py`; - const hasConfig = fileMap.has(configName) || [...fileMap.keys()].some((k) => k.endsWith(`${studyId}_config.py`)); - const hasEvaluator = fileMap.has(evaluatorName) || [...fileMap.keys()].some((k) => k.endsWith(`${studyId}_evaluator.py`)); - if (!hasConfig) errors.push(`Missing ${studyId}_config.py in src/studies/ or zip root.`); - if (!hasEvaluator) errors.push(`Missing ${studyId}_evaluator.py in src/studies/ or zip root.`); - - if (errors.length > 0) return { ok: false, errors }; - return { ok: true, studyId, files: fileMap }; -} - -export async function POST(request: NextRequest) { - try { - const formData = await request.formData(); - const file = formData.get("file") as File | null; - const contributorGithub = normalizeGithub(String(formData.get("contributor_github") || "")); - const contributorName = String(formData.get("contributor_name") || "").trim(); - const contributorInstitution = String(formData.get("contributor_institution") || "Independent Researcher").trim(); - if (!file || !(file instanceof File)) { - return NextResponse.json({ success: false, errors: ["No file provided."] }, { status: 400 }); - } - if (!contributorGithub) { - return NextResponse.json( - { success: false, errors: ["GitHub ID or profile URL is required."] }, - { status: 400 } - ); - } - if (!file.name.endsWith(".zip")) { - return NextResponse.json({ success: false, errors: ["File must be a .zip archive."] }, { status: 400 }); - } - const buf = Buffer.from(await file.arrayBuffer()); - if (buf.length > MAX_ZIP_BYTES) { - return NextResponse.json({ success: false, errors: ["Zip file exceeds 50MB limit."] }, { status: 400 }); - } - - const AdmZip = (await import("adm-zip")).default; - const zip = new AdmZip(buf); - const entries = zip.getEntries(); - const extracted: { name: string; data: Buffer }[] = []; - for (const e of entries) { - if (e.isDirectory) continue; - extracted.push({ name: e.entryName.replace(/\/$/, ""), data: e.getData() }); - } - const result = validateZip(extracted); - if (!result.ok) { - return NextResponse.json({ success: false, errors: result.errors }, { status: 400 }); - } - const { studyId, files: validatedFiles } = result; - - const token = process.env.GITHUB_TOKEN; - const repo = process.env.GITHUB_REPO || "HumanStudy-Hub/HumanStudy-Bench"; - if (!token) { - return NextResponse.json( - { success: false, errors: ["Server is not configured with GITHUB_TOKEN for creating PRs."] }, - { status: 503 } - ); - } - - const [owner, repoName] = repo.split("/"); - const { Octokit } = await import("@octokit/rest"); - const octokit = new Octokit({ auth: token }); - - const { data: defaultBranch } = await octokit.repos.get({ owner, repo: repoName }); - const baseRef = defaultBranch.default_branch || "main"; - const branchName = `contribute/${studyId}-${Date.now()}`; - - const { data: baseRefData } = await octokit.git.getRef({ owner, repo: repoName, ref: `heads/${baseRef}` }); - const baseCommitSha = baseRefData.object.sha; - const { data: commitData } = await octokit.git.getCommit({ owner, repo: repoName, commit_sha: baseCommitSha }); - const baseTreeSha = commitData.tree.sha; - - function zipEntryToRepoPath(entryName: string): string | null { - const sid = studyId; - if (entryName.startsWith(`studies/${sid}/`)) return entryName; - if (entryName.startsWith(`${sid}/`)) return `studies/${entryName}`; - if (entryName === `src/studies/${sid}_config.py` || entryName.endsWith(`/${sid}_config.py`)) return `src/studies/${sid}_config.py`; - if (entryName === `src/studies/${sid}_evaluator.py` || entryName.endsWith(`/${sid}_evaluator.py`)) return `src/studies/${sid}_evaluator.py`; - if (entryName.includes("specification.json")) return `studies/${sid}/specification.json`; - if (entryName.includes("metadata.json")) return `studies/${sid}/metadata.json`; - if (entryName.includes("ground_truth.json")) return `studies/${sid}/ground_truth.json`; - if (entryName.includes("materials/")) return `studies/${sid}/materials/${entryName.split("materials/")[1]}`; - return null; - } - - const pathToContent = new Map(); - for (const [name, data] of validatedFiles.entries()) { - if (!data.length) continue; - const repoPath = zipEntryToRepoPath(name); - if (repoPath && (repoPath.startsWith("studies/") || repoPath.startsWith("src/"))) { - pathToContent.set(repoPath, data); - } - } - - const metaBuf = pathToContent.get(`studies/${studyId}/metadata.json`); - const meta = metaBuf ? (parseJson(metaBuf) as Record) : {}; - const title = (meta.title as string) || studyId; - - const generatedIndex = { - title: title, - authors: Array.isArray(meta.authors) ? meta.authors : [], - year: typeof meta.year === "number" ? meta.year : null, - description: typeof meta.summary === "string" ? meta.summary : typeof meta.abstract === "string" ? meta.abstract : "", - contributors: [{ name: contributorName || undefined, github: contributorGithub, institution: contributorInstitution }], - }; - pathToContent.set( - `studies/${studyId}/index.json`, - Buffer.from(JSON.stringify(generatedIndex, null, 2), "utf8") - ); - - const treeItems: { path: string; mode: "100644"; type: "blob"; sha: string }[] = []; - for (const [path, data] of pathToContent.entries()) { - const { data: blob } = await octokit.git.createBlob({ - owner, - repo: repoName, - content: data.toString("base64"), - encoding: "base64", - }); - treeItems.push({ path, mode: "100644", type: "blob", sha: blob.sha }); - } - - const { data: newTree } = await octokit.git.createTree({ - owner, - repo: repoName, - base_tree: baseTreeSha, - tree: treeItems, - }); - - const { data: newCommit } = await octokit.git.createCommit({ - owner, - repo: repoName, - message: `chore: add ${studyId} (contribute upload)`, - tree: newTree.sha, - parents: [baseCommitSha], - }); - - await octokit.git.createRef({ - owner, - repo: repoName, - ref: `refs/heads/${branchName}`, - sha: newCommit.sha, - }); - - const prBody = `## Contributed study (website upload)\n\n- **Study ID:** ${studyId}\n- **Title:** ${title}\n- **GitHub:** ${contributorGithub}\n- **Domain:** ${meta.domain || "—"}\n\nPlease run \`python scripts/validate_study.py --study ${studyId}\` and review.`; - - const pr = await octokit.pulls.create({ - owner, - repo: repoName, - title: `Add ${studyId}: ${title}`, - head: branchName, - base: baseRef, - body: prBody, - }); - - return NextResponse.json({ success: true, pr_url: pr.data.html_url }); - } catch (err) { - console.error("Contribute upload error:", err); - const message = err instanceof Error ? err.message : "Unknown error"; - return NextResponse.json( - { success: false, errors: [message] }, - { status: 500 } - ); - } -} diff --git a/co_website/app/benchmark/page.tsx b/co_website/app/benchmark/page.tsx deleted file mode 100644 index 2068084..0000000 --- a/co_website/app/benchmark/page.tsx +++ /dev/null @@ -1,14 +0,0 @@ -import fs from "fs"; -import path from "path"; -import ReactMarkdown from "react-markdown"; - -export default function BenchmarkPage() { - const filePath = path.join(process.cwd(), "content/benchmark.md"); - const content = fs.readFileSync(filePath, "utf-8"); - - return ( -
- {content} -
- ); -} diff --git a/co_website/app/citation/page.tsx b/co_website/app/citation/page.tsx deleted file mode 100644 index 7195b6a..0000000 --- a/co_website/app/citation/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -export default function CitationPage() { - return ( -
-

Citation

-

Citation content will appear here.

-
- ); -} diff --git a/co_website/app/contribute/page.tsx b/co_website/app/contribute/page.tsx deleted file mode 100644 index e6bc088..0000000 --- a/co_website/app/contribute/page.tsx +++ /dev/null @@ -1,238 +0,0 @@ -import { promises as fs } from "fs"; -import path from "path"; -import Link from "next/link"; -import ContributeUploadForm from "@/components/ContributeUploadForm"; - -type Contributor = { name?: string; github?: string }; -type StudySummary = { - study_id: string; - title: string; - authors: string[]; - year: number | null; - contributors?: Contributor[]; -}; - -export default async function ContributePage() { - const filePath = path.join(process.cwd(), "data/studies_index.json"); - let studies: StudySummary[] = []; - try { - const raw = await fs.readFile(filePath, "utf8"); - const data = JSON.parse(raw); - studies = data.studies ?? []; - } catch (e) { - console.error("Could not read studies index", e); - } - - const foundational = studies.slice(0, 12); - const community = studies.slice(12); - - return ( -
- {/* Hero */} -
-
-

- Contribute -

-

- Add a new study to HumanStudy-Bench. -

-
-

HumanStudy-Bench is a non-profit and open-source platform.

-

We welcome collaborators from all backgrounds!

-

All accepted contributions will be acknowledged and credited.

-
- {/* Section nav */} -
- - Studies - - - How to Contribute - - - - - - - - - Step-by-step Tutorial → - - -
-
-
- - {/* Studies Catalog */} -
-
-

- Study Catalog -

- -
- {foundational.map((study, idx) => ( -
-
- - {(idx + 1).toString().padStart(3, "0")} - - - {study.title} - -
- {study.contributors?.[0]?.github && ( - - @{study.contributors[0].github.replace(/^https?:\/\/github\.com\//, "")} - - )} -
- ))} -
- - {community.length > 0 && ( - <> -
-
- - Community Contributions - -
-
- -
- {community.map((study, idx) => ( -
-
- - {(idx + 13).toString().padStart(3, "0")} - - - {study.title} - -
- {study.contributors?.[0]?.github && ( - - @{study.contributors[0].github.replace(/^https?:\/\/github\.com\//, "")} - - )} -
- ))} -
- - )} -
-
- - {/* How to Contribute */} -
-
-

- How to Contribute -

-

- There are two ways to submit a study. We recommend contributing via GitHub. -

- -
- {/* GitHub PR path */} -
-
- - Recommended - -

GitHub Pull Request

-
-
    -
  1. - 1 - Fork and clone the repository -
    2. -
  2. - 2 - Create a branch (e.g. contrib-yourgithubid-013) -
    3. -
  3. - 3 - Add your study under studies/ with index.json, source/, scripts/, and README.md -
    4. -
  4. - 4 - Run bash scripts/verify_study.sh study_XXX and open a PR -
    5. -
-

- Maintainers assign final study numbering by merge order. Include your GitHub ID in index.json contributors. -

-
- - {/* Web upload path */} -
-

Web Upload

-

- Upload a .zip of your study and we create a GitHub PR for you. -

- -
-
- - {/* Docs link */} -
- - - - - Not sure where to start? Read the detailed walkthrough - - - - -
-
-
- - {/* Footer note */} -
-

- Questions? Open an issue on{" "} - - GitHub - - . For other inquiries, contact{" "} - - xul049@ucsd.edu - - . -

-
-
- ); -} diff --git a/co_website/app/dataset/page.tsx b/co_website/app/dataset/page.tsx deleted file mode 100644 index a408697..0000000 --- a/co_website/app/dataset/page.tsx +++ /dev/null @@ -1,5 +0,0 @@ -import { redirect } from "next/navigation"; - -export default function DatasetPage() { - redirect("/#dataset"); -} diff --git a/co_website/app/docs/[slug]/page.tsx b/co_website/app/docs/[slug]/page.tsx deleted file mode 100644 index efe3089..0000000 --- a/co_website/app/docs/[slug]/page.tsx +++ /dev/null @@ -1,266 +0,0 @@ -import { promises as fs } from "fs"; -import path from "path"; -import { notFound } from "next/navigation"; -import Link from "next/link"; -import Markdown from "react-markdown"; -import remarkGfm from "remark-gfm"; -import rehypeHighlight from "rehype-highlight"; -import rehypeRaw from "rehype-raw"; - -const DOCS_DIR = path.join(process.cwd(), "..", "docs"); - -const docsMeta: Record< - string, - { title: string; description: string; order: number } -> = { - what_to_submit: { - title: "What Should I Submit?", - description: "The big picture — what goes in a study folder", - order: 1, - }, - extract_from_paper: { - title: "How to Extract Data from a Paper", - description: "Turn a PDF into structured, replicable data", - order: 2, - }, - build_study_files: { - title: "How to Build Your Study Files", - description: "Schemas, code, and contracts for every file", - order: 3, - }, - submit_study: { - title: "How to Submit Your Study", - description: "Fork, verify, push, and ship it", - order: 4, - }, -}; - -const CALLOUT_ICONS = { - tip: '', - info: '', - warning: '', - note: '', -}; - -function stripAdmonitions(md: string): string { - return md.replace( - /^!!! (\w+)(?: "([^"]*)")?\n((?: .*\n?)*)/gm, - (_match, type, title, body) => { - const content = body.replace(/^ /gm, "").trim(); - const normalizedType = type === "note" ? "info" : type; - const label = title || type.charAt(0).toUpperCase() + type.slice(1); - const icon = - CALLOUT_ICONS[normalizedType as keyof typeof CALLOUT_ICONS] || - CALLOUT_ICONS.info; - - return `\n
\n\n
\n${label}\n\n${content}\n\n
\n
\n\n`; - } - ); -} - -function fixInternalLinks(md: string): string { - return md.replace( - /\[([^\]]+)\]\((?!https?:\/\/)([a-z_]+)\.md(?:#([a-z_-]+))?\)/g, - (_match, text, slug, anchor) => { - const href = anchor ? `/docs/${slug}#${anchor}` : `/docs/${slug}`; - return `[${text}](${href})`; - } - ); -} - -export async function generateStaticParams() { - return Object.keys(docsMeta).map((slug) => ({ slug })); -} - -export default async function DocPage({ - params, -}: { - params: Promise<{ slug: string }>; -}) { - const { slug } = await params; - const meta = docsMeta[slug]; - if (!meta) notFound(); - - let content = ""; - try { - const raw = await fs.readFile(path.join(DOCS_DIR, `${slug}.md`), "utf8"); - const withoutH1 = raw.replace(/^# .+\n/, ""); - content = fixInternalLinks(stripAdmonitions(withoutH1)); - } catch { - notFound(); - } - - const allDocs = Object.entries(docsMeta).sort((a, b) => a[1].order - b[1].order); - const currentIdx = allDocs.findIndex(([s]) => s === slug); - const prev = currentIdx > 0 ? allDocs[currentIdx - 1] : null; - const next = - currentIdx < allDocs.length - 1 ? allDocs[currentIdx + 1] : null; - - return ( -
-
-
- {/* Sidebar — GitHub Docs style */} - - - {/* Main content */} -
-
- {/* Breadcrumb + title + description (inside content column) */} -
-

- - Contribute - - {" / "} - Docs -

-

- {meta.title} -

-

- {meta.description} -

-
- - {/* Mobile nav — horizontal scroll */} -
-
- {allDocs.map(([s, m]) => ( - - {m.title} - - ))} -
-
- - {/* Markdown content — GitHub Docs typography */} -
- - {content} - -
- - {/* Prev/Next — flat GitHub Docs style */} -
- {prev ? ( - - - - -
-

Previous

-

- {prev[1].title} -

-
- - ) : ( -
- )} - {next ? ( - -
-

Next

-

- {next[1].title} -

-
- - - - - ) : ( -
- )} -
-
-
-
-
-
- ); -} diff --git a/co_website/app/docs/page.tsx b/co_website/app/docs/page.tsx deleted file mode 100644 index 82fa24b..0000000 --- a/co_website/app/docs/page.tsx +++ /dev/null @@ -1,5 +0,0 @@ -import { redirect } from "next/navigation"; - -export default function DocsPage() { - redirect("/docs/what_to_submit"); -} diff --git a/co_website/app/globals.css b/co_website/app/globals.css deleted file mode 100644 index a31e8af..0000000 --- a/co_website/app/globals.css +++ /dev/null @@ -1,134 +0,0 @@ -@import "tailwindcss"; -@plugin "@tailwindcss/typography"; - -:root { - --background: #ffffff; - --foreground: #171717; -} - -@theme { - --font-serif: var(--font-serif), ui-serif, Georgia, Cambria, "Times New Roman", Times, serif; - --font-sans: var(--font-sans), ui-sans-serif, system-ui, sans-serif; -} - -@media (prefers-color-scheme: dark) { - :root { - /* Keeping it light/academic as per request even in dark mode preference */ - --background: #ffffff; - --foreground: #171717; - } -} - -body { - background: var(--background); - color: var(--foreground); -} - -/* GitHub-style callouts (Note / Tip / Warning) */ -.callout { - border-left: 3px solid; - border-radius: 6px; - padding: 16px; - margin: 24px 0; - display: flex; - gap: 12px; - align-items: flex-start; -} -.callout-icon { - flex-shrink: 0; - display: inline-flex; - align-items: center; - justify-content: center; - margin-top: 2px; -} -.callout-svg { - width: 1.25rem; - height: 1.25rem; -} -.callout-body { - flex: 1; - min-width: 0; -} -.callout-body strong { - display: block; - font-family: var(--font-sans); - font-size: 0.95rem; - margin-bottom: 0.5rem; -} -.callout-body > p:first-of-type { - margin-top: 0; -} -.callout-body > p:last-of-type { - margin-bottom: 0; -} -.callout-tip { - border-color: #22c55e; - background: #f0fdf4; -} -.callout-tip .callout-svg { - color: #16a34a; -} -.callout-info { - border-color: #0ea5e9; - background: #f0f9ff; -} -.callout-info .callout-svg { - color: #0284c7; -} -.callout-warning { - border-color: #eab308; - background: #fefce8; -} -.callout-warning .callout-svg { - color: #ca8a04; -} - -/* Collapsible details/summary sections */ -.prose details { - border: 1px solid #d0d7de; - border-radius: 6px; - padding: 0; - margin: 16px 0; -} -.prose details > summary { - padding: 12px 16px; - cursor: pointer; - list-style: none; - user-select: none; - background: #f6f8fa; - border-radius: 6px; - font-size: 0.925rem; - color: #1F2328; - transition: background-color 0.15s; -} -.prose details[open] > summary { - border-bottom: 1px solid #d0d7de; - border-radius: 6px 6px 0 0; -} -.prose details > summary:hover { - background: #eef1f5; -} -.prose details > summary::-webkit-details-marker { - display: none; -} -.prose details > summary::before { - content: "▶"; - display: inline-block; - margin-right: 8px; - font-size: 0.7em; - transition: transform 0.15s; - color: #656d76; -} -.prose details[open] > summary::before { - transform: rotate(90deg); -} -.prose details > :not(summary) { - padding-left: 16px; - padding-right: 16px; -} -.prose details > :last-child { - padding-bottom: 16px; -} -.prose details > :nth-child(2) { - padding-top: 16px; -} diff --git a/co_website/app/icon.svg b/co_website/app/icon.svg deleted file mode 100644 index d83ce00..0000000 --- a/co_website/app/icon.svg +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - H - diff --git a/co_website/app/layout.tsx b/co_website/app/layout.tsx deleted file mode 100644 index 2a30410..0000000 --- a/co_website/app/layout.tsx +++ /dev/null @@ -1,49 +0,0 @@ -import type { Metadata } from "next"; -import { Merriweather, Inter } from "next/font/google"; -import { SpeedInsights } from "@vercel/speed-insights/next"; -import { Analytics } from "@vercel/analytics/react"; -import "./globals.css"; -import "highlight.js/styles/github.min.css"; -import Navbar from "@/components/Navbar"; -import Footer from "@/components/Footer"; - -const merriweather = Merriweather({ - subsets: ["latin"], - weight: ["300", "400", "700", "900"], - variable: "--font-serif", -}); - -const inter = Inter({ - subsets: ["latin"], - variable: "--font-sans", -}); - -export const metadata: Metadata = { - title: "HumanStudy-Bench: Towards AI Agent Design for Participant Simulation", - description: "A reusable platform for replaying human-subject experiments end-to-end, evaluating agent design alignment with human ground-truth outcomes at the level of scientific inference.", - icons: { - icon: [{ url: "/favicon.svg?v=2", type: "image/svg+xml" }], - shortcut: "/favicon.svg?v=2", - apple: "/favicon.svg?v=2", - }, -}; - -export default function RootLayout({ - children, -}: Readonly<{ - children: React.ReactNode; -}>) { - return ( - - - -
{children}
-