From 4d9463364f174b6c232da006427ab1a5977e317f Mon Sep 17 00:00:00 2001 From: Gabriel Spadon Date: Fri, 21 Aug 2026 16:21:29 -0300 Subject: [PATCH] fix: fail closed on incoherent monthly citations --- citeforge/bibtex_build.py | 6 +- citeforge/io_utils.py | 51 ++++++++++++++++- citeforge/merge_utils.py | 23 ++++++++ citeforge/pipeline/article.py | 22 ++++---- citeforge/pipeline/postrun.py | 11 +++- citeforge/pipeline/scheduler.py | 10 +++- citeforge/text_utils.py | 62 +++++++++++++-------- tests/test_bibtex_build.py | 1 + tests/test_finalize_run.py | 35 ++++++++++++ tests/test_io_csv.py | 95 ++++++++++++++++++++++++++++++++ tests/test_pipeline.py | 69 +++++++++++++++++++++++ tests/test_regression.py | 43 +++++++++++++++ tests/test_scheduler_parallel.py | 38 +++++++++++++ tests/test_text_utils.py | 15 +++++ 14 files changed, 439 insertions(+), 42 deletions(-) diff --git a/citeforge/bibtex_build.py b/citeforge/bibtex_build.py index 1b8a8463..fbbeb419 100644 --- a/citeforge/bibtex_build.py +++ b/citeforge/bibtex_build.py @@ -29,7 +29,7 @@ _ARTICLE_TYPES = {"journal-article", "journal_article", "article"} _CONFERENCE_TYPES = {"proceedings-article", "paper-conference", "inproceedings", "conference"} _CHAPTER_TYPES = {"book-chapter", "book_chapter", "incollection"} -_BOOK_TYPES = {"book", "edited-book", "monograph", "reference-book"} +_BOOK_TYPES = {"book", "edited-book", "monograph", "proceedings", "reference-book"} _BOOK_SERIES_KEYWORDS = ( "lecture notes", @@ -169,12 +169,12 @@ def _classify_type_string(typ: str) -> str | None: match is found.""" if "journal" in typ or typ in _ARTICLE_TYPES: return "article" + if typ in _BOOK_TYPES: + return "book" if "proceed" in typ or typ in _CONFERENCE_TYPES: return "inproceedings" if "chapter" in typ or typ in _CHAPTER_TYPES: return "incollection" - if typ in _BOOK_TYPES: - return "book" return None diff --git a/citeforge/io_utils.py b/citeforge/io_utils.py index d3b14d27..51bd47cc 100644 --- a/citeforge/io_utils.py +++ b/citeforge/io_utils.py @@ -33,11 +33,18 @@ get_min_year, ) from .exceptions import CSV_ERRORS, FILE_READ_ERRORS -from .fsscan import iter_author_bibs, iter_output_dirs +from .fsscan import iter_author_bibs, iter_output_dirs, iter_parsed_author_bibs from .id_utils import doi_bases_match, normalize_doi from .models import Record from .refresh.census import load_census -from .text_utils import format_author_dirname, normalize_person_name, title_similarity +from .text_utils import ( + format_author_dirname, + name_signature, + normalize_person_name, + normalize_title, + parse_authors_any, + title_similarity, +) _SUMMARY_CSV_FIELDNAMES = [ "file_path", @@ -58,6 +65,42 @@ _SUMMARY_CSV_FLAG_FIELDS = [f for f in _SUMMARY_CSV_FIELDNAMES if f not in ("file_path", "trust_hits")] _CSV_LOCK = threading.Lock() +_CitationCoherenceSignature = tuple[str, str, tuple[tuple[str, str], ...], str] + + +def _citation_coherence_signature(entry: dict[str, Any]) -> _CitationCoherenceSignature: + """Return material citation fields that every copy of one DOI must agree on.""" + fields = entry.get("fields") or {} + authors: list[tuple[str, str]] = [] + for author in parse_authors_any(fields.get("author")): + signature = name_signature(author) + if signature and signature.get("last"): + authors.append((str(signature["last"]), str(signature.get("initials") or "")[:1])) + return ( + str(entry.get("type") or "").casefold(), + normalize_title(str(fields.get("title") or "")), + tuple(authors), + str(fields.get("year") or "").strip(), + ) + + +def find_incoherent_doi_author_dirs(out_dir: str) -> frozenset[str]: + """Return author directories containing materially conflicting copies of one DOI.""" + grouped: dict[str, list[tuple[str, _CitationCoherenceSignature]]] = {} + for dirname in iter_output_dirs(out_dir): + if dirname == A2I2_OUTPUT_DIR: + continue + author_dir = os.path.join(out_dir, dirname) + for _filename, _path, entry in iter_parsed_author_bibs(author_dir): + doi = normalize_doi((entry.get("fields") or {}).get("doi")) + if doi: + grouped.setdefault(doi, []).append((dirname, _citation_coherence_signature(entry))) + + affected: set[str] = set() + for copies in grouped.values(): + if len(copies) > 1 and len({signature for _, signature in copies}) > 1: + affected.update(dirname for dirname, _ in copies) + return frozenset(affected) def _project_root() -> str: @@ -571,6 +614,10 @@ def _pick_richer( if matched_key is not None: ki = doi_to_idx[matched_key] + if _citation_coherence_signature(kept[ki][0]) != _citation_coherence_signature(entry): + raise ValueError( + f"conflicting citation metadata for DOI {matched_key}: {kept[ki][1]} and {fpath}" + ) kept[ki] = _pick_richer(kept[ki], (entry, fpath)) else: doi_to_idx[doi] = len(kept) diff --git a/citeforge/merge_utils.py b/citeforge/merge_utils.py index 81d847b1..89ca2c0d 100644 --- a/citeforge/merge_utils.py +++ b/citeforge/merge_utils.py @@ -611,6 +611,29 @@ def value_ok(val: str | None) -> bool: merged = normalize_arxiv_metadata(merged, log=logger) _pop_fields(merged, {"keywords", "copyright"}, "unwanted_removed", logger) + # A Scholar baseline can combine one arXiv work with container metadata + # from a different publication by overlapping authors. When the baseline + # had no DOI and a validated secondary DOI identifies the record as a + # preprint, an unconfirmed Scholar-only journal is not publication + # evidence. Keep the validated preprint and discard the foreign container. + resolved_doi = _norm_doi(merged.get("doi")) + if ( + resolved_doi + and _is_preprint_doi(resolved_doi) + and not primary_doi + and etype == "article" + and merged.get("journal") + and field_sources.get("journal") == "scholar_min" + ): + stale_journal = merged.get("journal") + _pop_fields(merged, {"journal", "publisher", "volume", "number", "pages"}, "unconfirmed_container", logger) + etype = "misc" + logger.debug( + f"preprint_container_removed | journal={stale_journal} | doi={resolved_doi} " + "| reason=scholar_container_unconfirmed", + category=LogCategory.CLEANUP, + ) + # Strip trailing digit suffixes leaked from Scholar/DBLP author # disambiguation markers (e.g., "Das1" becomes "Das"). author_val = merged.get("author", "") diff --git a/citeforge/pipeline/article.py b/citeforge/pipeline/article.py index 933a3e1c..135129f1 100644 --- a/citeforge/pipeline/article.py +++ b/citeforge/pipeline/article.py @@ -117,6 +117,9 @@ def _entry_is_complete(entry: dict[str, Any]) -> bool: has_essentials = all(fields.get(k) and not has_placeholder(str(fields.get(k))) for k in ("title", "author", "year")) has_doi = bool(doi) and not has_placeholder(str(doi)) + normalized_doi = idu.normalize_doi(str(doi)) if has_doi else None + url_doi = idu.find_doi_in_text(str(fields.get("url") or "")) + url_doi_mismatch = bool(normalized_doi and url_doi and normalized_doi != url_doi) if has_essentials and has_venue and has_doi: doi_is_preprint = idu.is_secondary_doi(str(doi)) @@ -143,6 +146,7 @@ def _entry_is_complete(entry: dict[str, Any]) -> bool: and not journal_is_preprint and not venue_is_generic and not author_is_abbreviated + and not url_doi_mismatch ) logger.debug( @@ -150,7 +154,7 @@ def _entry_is_complete(entry: dict[str, Any]) -> bool: f"| has_author={bool(author)} | has_year={bool(year)} " f"| has_venue={has_venue} | has_doi={has_doi} " f"| doi_is_preprint={doi_is_preprint} | journal_is_preprint={journal_is_preprint} " - f"| author_is_abbreviated={author_is_abbreviated} " + f"| author_is_abbreviated={author_is_abbreviated} | url_doi_mismatch={url_doi_mismatch} " f"| result={result}", category=LogCategory.AUDIT, ) @@ -202,10 +206,8 @@ def _try_multiple_candidates( ) -> tuple[bool, Any | None]: """Try candidates from an API source in relevance order until one matches the baseline. - When *seen_dois* is provided, every DOI encountered across all candidates - (matched or not) is collected. This enables downstream duplicate detection - against files already on disk even when the candidate was rejected by the - matching gate. + When *seen_dois* is provided, the DOI from an identity-validated candidate + is collected for downstream duplicate detection against files on disk. Returns a (matched, matched_candidate) tuple. """ @@ -224,12 +226,6 @@ def _try_multiple_candidates( if not candidate_dict: continue - # Collect DOI from every parsed candidate for dedup - if seen_dois is not None: - cand_doi = idu.normalize_doi((candidate_dict.get("fields") or {}).get("doi", "")) - if cand_doi: - seen_dois.add(cand_doi) - evidence = evaluate_identity(baseline_entry, candidate_dict, context=IdentityContext.ENRICHMENT) match = evidence.verdict logger.debug( @@ -237,6 +233,10 @@ def _try_multiple_candidates( category=LogCategory.DEDUP, ) if match: + if seen_dois is not None: + cand_doi = idu.normalize_doi((candidate_dict.get("fields") or {}).get("doi", "")) + if cand_doi: + seen_dois.add(cand_doi) enr_list.append((flag_key, candidate_dict)) flags[flag_key] = True logger.success( diff --git a/citeforge/pipeline/postrun.py b/citeforge/pipeline/postrun.py index 9eef67f4..baffaae9 100644 --- a/citeforge/pipeline/postrun.py +++ b/citeforge/pipeline/postrun.py @@ -37,6 +37,7 @@ from citeforge.io_utils import ( build_a2i2_folder, collect_orphan_files, + find_incoherent_doi_author_dirs, flush_summary_csv, reconcile_summary_csv, retarget_summary_csv_paths, @@ -359,7 +360,15 @@ def finalize_run( category=LogCategory.CLEANUP, ) - a2i2_count = build_a2i2_folder(DEFAULT_A2I2_INPUT, records, out_dir) + unresolved_doi_conflicts = find_incoherent_doi_author_dirs(out_dir) + if unresolved_doi_conflicts: + affected = ", ".join(sorted(unresolved_doi_conflicts)) + raise FinalizationError(f"conflicting copies of the same DOI remain in author directories: {affected}") + + try: + a2i2_count = build_a2i2_folder(DEFAULT_A2I2_INPUT, records, out_dir) + except ValueError as exc: + raise FinalizationError("a2i2 rebuild rejected incoherent citation metadata") from exc if a2i2_count: logger.info( f"Built a2i2 folder: {a2i2_count} deduplicated files", diff --git a/citeforge/pipeline/scheduler.py b/citeforge/pipeline/scheduler.py index c914eb1d..0253bb5b 100644 --- a/citeforge/pipeline/scheduler.py +++ b/citeforge/pipeline/scheduler.py @@ -40,6 +40,7 @@ FULL_OPERATION_ERRORS, ) from citeforge.fsscan import iter_author_bibs, iter_parsed_author_bibs +from citeforge.io_utils import find_incoherent_doi_author_dirs from citeforge.log_utils import LogCategory, LogSource, logger from citeforge.models import Record from citeforge.pipeline.article import process_article @@ -350,6 +351,13 @@ def run_all( total_saved = 0 processed = 0 accounted: set[Future[int]] = set() + incoherent_author_dirs = find_incoherent_doi_author_dirs(out_dir) + if incoherent_author_dirs: + logger.warn( + f"Forcing enrichment for {len(incoherent_author_dirs)} author output directories " + "with conflicting copies of the same DOI", + category=LogCategory.PLAN, + ) def _account_result(future: Future[int], rec: Record) -> None: nonlocal processed, total_saved @@ -420,7 +428,7 @@ def _thread_excepthook(args: Any) -> None: or_creds=or_creds, gemini_api_key=gemini_api_key, summary_csv_path=summary_csv_path, - force_enrich=force_enrich, + force_enrich=force_enrich or _author_dirname(rec) in incoherent_author_dirs, ) future_to_author[future] = rec diff --git a/citeforge/text_utils.py b/citeforge/text_utils.py index 49803a56..804f220e 100644 --- a/citeforge/text_utils.py +++ b/citeforge/text_utils.py @@ -257,6 +257,7 @@ def normalize_person_name(n: Any | None) -> str: return "" n_str = to_text(n) n2 = strip_accents(n_str).lower() + n2 = n2.replace("'", "").replace("\u2019", "").replace("\u02bc", "") n2 = _PERSON_PUNCT_RE.sub(" ", n2) return " ".join(n2.split()) @@ -555,19 +556,8 @@ def authors_overlap(authors_a: str | None, authors_b: str | None) -> bool: return False -def _author_sig_key(sig: dict[str, Any]) -> str: - """Build a set key from a name signature, including initials when available.""" - last = sig.get("last", "") - initials = sig.get("initials", "") - return f"{last}_{initials}" if initials else last - - def author_overlap_ratio(authors_a: str | None, authors_b: str | None) -> float: - """Jaccard coefficient on normalized author signatures between two author lists. - - Uses last_name + initials when both sides have initials, falling back to - last-name-only matching otherwise. - """ + """Jaccard coefficient on compatible normalized author signatures.""" names_a = parse_authors_any(authors_a or "") names_b = parse_authors_any(authors_b or "") if not names_a or not names_b: @@ -576,18 +566,42 @@ def author_overlap_ratio(authors_a: str | None, authors_b: str | None) -> float: sigs_b_raw = [sig for nm in names_b if (sig := name_signature(nm)) and sig.get("last")] if not sigs_a_raw or not sigs_b_raw: return 0.0 - a_has_initials = all(s.get("initials") for s in sigs_a_raw) - b_has_initials = all(s.get("initials") for s in sigs_b_raw) - if a_has_initials and b_has_initials: - sigs_a = {_author_sig_key(s) for s in sigs_a_raw} - sigs_b = {_author_sig_key(s) for s in sigs_b_raw} - else: - # Fall back to last-name-only when one side lacks initials - sigs_a = {s["last"] for s in sigs_a_raw} - sigs_b = {s["last"] for s in sigs_b_raw} - intersection = len(sigs_a & sigs_b) - union = len(sigs_a | sigs_b) - return intersection / union if union > 0 else 0.0 + sigs_a = {(str(sig["last"]), str(sig.get("initials") or "")) for sig in sigs_a_raw} + unmatched_b = {(str(sig["last"]), str(sig.get("initials") or "")) for sig in sigs_b_raw} + b_count = len(unmatched_b) + matches = 0 + # Match the most specific signatures first and prefer exact initials. This + # produces a maximal one-to-one match when the same surname occurs more + # than once and one source abbreviates a given name to a prefix. + ordered_a = sorted(sigs_a, key=lambda item: (bool(item[1]), len(item[1]), item), reverse=True) + for last_a, initials_a in ordered_a: + candidates = [ + candidate + for candidate in unmatched_b + if candidate[0] == last_a + and ( + not initials_a + or not candidate[1] + or initials_a == candidate[1] + or initials_a.startswith(candidate[1]) + or candidate[1].startswith(initials_a) + ) + ] + compatible = min( + candidates, + key=lambda candidate: ( + candidate[1] != initials_a, + not candidate[1], + -len(candidate[1]), + candidate, + ), + default=None, + ) + if compatible is not None: + unmatched_b.remove(compatible) + matches += 1 + union = len(sigs_a) + b_count - matches + return matches / union if union > 0 else 0.0 def venue_similarity(fields_a: dict[str, Any], fields_b: dict[str, Any]) -> float: diff --git a/tests/test_bibtex_build.py b/tests/test_bibtex_build.py index 9f62d527..7e4e6964 100644 --- a/tests/test_bibtex_build.py +++ b/tests/test_bibtex_build.py @@ -230,6 +230,7 @@ def test_determine_entry_type_publication_types(pub_types: list[str], expected: [ ("journal-article", "article"), ("proceedings-article", "inproceedings"), + ("proceedings", "book"), ("book-chapter", "incollection"), ("book", "book"), ("something-weird", "misc"), diff --git a/tests/test_finalize_run.py b/tests/test_finalize_run.py index b8a3da92..79785813 100644 --- a/tests/test_finalize_run.py +++ b/tests/test_finalize_run.py @@ -751,6 +751,8 @@ def test_unreadable_year_window_candidate_raises(tmp_path: Path, no_a2i2: None) with pytest.raises(FinalizationError, match="year-window check"): finalize_run(str(out_dir), _records(), total_saved=1, processed=1, summary_csv_path=str(csv_path)) + assert broken.exists(), "aborting must not have deleted the file it could not read" + def test_unreadable_candidate_raises(tmp_path: Path, no_a2i2: None) -> None: """A tracked file that cannot be read aborts the run rather than being skipped. @@ -773,6 +775,39 @@ def test_unreadable_candidate_raises(tmp_path: Path, no_a2i2: None) -> None: assert broken.exists(), "aborting must not have deleted the file it could not read" +def test_incoherent_a2i2_source_raises_finalization_error( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, no_a2i2: None +) -> None: + """Conflicting member citations abort through the public finalization contract.""" + out_dir = tmp_path / "out" + _author_dir(out_dir) + + def reject_conflict(*_args: object, **_kwargs: object) -> int: + raise ValueError("conflicting citation metadata for DOI 10.5555/example") + + monkeypatch.setattr(postrun, "build_a2i2_folder", reject_conflict) + + with pytest.raises(FinalizationError, match="a2i2 rebuild"): + finalize_run(str(out_dir), _records(), total_saved=0, processed=1, summary_csv_path=None) + + +def test_unresolved_same_doi_conflict_aborts_finalization(tmp_path: Path, no_a2i2: None) -> None: + out_dir = tmp_path / "out" + first_dir = _author_dir(out_dir, "Doe (abc123)") + second_dir = _author_dir(out_dir, "Roe (def456)") + common = { + "title": "A Shared Network Study", + "year": str(_IN_WINDOW_YEAR), + "journal": "IEEE Transactions on Machine Learning in Communications and Networking", + "doi": "10.1109/example", + } + write_bib(first_dir, article(key="DoeStudy", author="Jane Doe and Richard Roe", **common), "Doe-Study.bib") + write_bib(second_dir, article(key="DoeStudy", author="Jane Doe and Alice Poe", **common), "Doe-Study.bib") + + with pytest.raises(FinalizationError, match="conflicting copies of the same DOI"): + finalize_run(str(out_dir), _records(), total_saved=2, processed=1, summary_csv_path=None) + + def test_failed_baseline_write_raises(tmp_path: Path, no_a2i2: None, monkeypatch: pytest.MonkeyPatch) -> None: """A baseline.json write that does not land aborts the run. The baseline is the run's own record of what it produced, so a discarded failure there makes diff --git a/tests/test_io_csv.py b/tests/test_io_csv.py index 8ca7411c..89565557 100644 --- a/tests/test_io_csv.py +++ b/tests/test_io_csv.py @@ -229,6 +229,60 @@ def _make_records(names_and_ids: list[tuple[str, str]]) -> list[Record]: return [Record(name=n, scholar_id=sid) for n, sid in names_and_ids] +def test_incoherent_doi_scan_returns_every_affected_author_directory(tmp_path: Path) -> None: + out = tmp_path / "output" + dir_a = out / "Oore (A1)" + dir_b = out / "Trappenberg (B1)" + dir_a.mkdir(parents=True) + dir_b.mkdir(parents=True) + common = { + "title": "Logical Activation Functions", + "author": "Scott Lowe and Robert Earle and Jason D'Eon and Thomas Trappenberg and Sageev Oore", + "doi": "10.52202/068431-2156", + "booktitle": "NeurIPS", + } + _write_bib(dir_a / "Lowe2021.bib", "inproceedings", "Lowe2021", {**common, "year": "2021"}) + _write_bib(dir_b / "Lowe2022.bib", "inproceedings", "Lowe2022", {**common, "year": "2022"}) + + assert io_utils.find_incoherent_doi_author_dirs(str(out)) == frozenset({"Oore (A1)", "Trappenberg (B1)"}) + + +def test_incoherent_doi_scan_compares_author_identity_not_only_count(tmp_path: Path) -> None: + out = tmp_path / "output" + dir_a = out / "Hasan (A1)" + dir_b = out / "Haque (B1)" + dir_a.mkdir(parents=True) + dir_b.mkdir(parents=True) + common = { + "title": "A Shared Network Study", + "year": "2025", + "doi": "10.1109/example", + "journal": "IEEE Transactions on Machine Learning in Communications and Networking", + } + _write_bib(dir_a / "Hasan2025.bib", "article", "Hasan2025", {**common, "author": "Tariq Hasan and Evan Frick"}) + _write_bib(dir_b / "Hasan2025.bib", "article", "Hasan2025", {**common, "author": "Tariq Hasan and Mahdi Haque"}) + + assert io_utils.find_incoherent_doi_author_dirs(str(out)) == frozenset({"Hasan (A1)", "Haque (B1)"}) + + +def test_incoherent_doi_scan_accepts_equivalent_author_renderings(tmp_path: Path) -> None: + out = tmp_path / "output" + dir_a = out / "Oore (A1)" + dir_b = out / "Rudzicz (B1)" + dir_a.mkdir(parents=True) + dir_b.mkdir(parents=True) + common = { + "title": "Measurement of Personal Rhythm From Speech and Movement", + "year": "2025", + "doi": "10.1016/j.biopsych.2025.02.114", + "journal": "Biological Psychiatry", + } + _write_bib(dir_a / "Oore2025.bib", "article", "Oore2025", {**common, "author": "Sageev Oore and Jason D'eon"}) + _write_bib(dir_b / "Oore2025.bib", "article", "Oore2025", {**common, "author": "Oore, Sageev and D'eon, Jason"}) + + assert io_utils.find_incoherent_doi_author_dirs(str(out)) == frozenset() + + class TestBuildA2i2Folder: """Tests for the automated a2i2 build step.""" @@ -339,6 +393,47 @@ def test_dedup_by_doi(self, tmp_path: Path) -> None: content = (out / "a2i2" / "Smith2024-SharedPaper.bib").read_text() assert "Nature" in content + def test_conflicting_same_doi_metadata_aborts_before_rebuild(self, tmp_path: Path) -> None: + out = tmp_path / "output" + dir_a = out / "Smith (A1)" + dir_b = out / "Jones (B1)" + dir_a.mkdir(parents=True) + dir_b.mkdir(parents=True) + _write_bib( + dir_a / "Lowe2021-LogicalActivation.bib", + "inproceedings", + "Lowe2021:LogicalActivation", + { + "title": "Logical Activation Functions", + "author": "Scott Lowe and Robert Earle and Jason D'Eon", + "year": "2021", + "doi": "10.52202/068431-2156", + }, + ) + _write_bib( + dir_b / "Lowe2022-LogicalActivation.bib", + "inproceedings", + "Lowe2022:LogicalActivation", + { + "title": "Logical Activation Functions", + "author": "Scott Lowe and Robert Earle and Jason D'Eon", + "year": "2022", + "doi": "10.52202/068431-2156", + }, + ) + a2i2_dir = out / "a2i2" + a2i2_dir.mkdir() + stale = a2i2_dir / "Existing2024-Preserved.bib" + stale.write_text("@misc{Existing2024, title={Preserved}, year={2024}}\n", encoding="utf-8") + csv_path = tmp_path / "a2i2.csv" + _make_a2i2_csv(csv_path, ["Alice Smith", "Bob Jones"]) + records = _make_records([("Alice Smith", "A1"), ("Bob Jones", "B1")]) + + with pytest.raises(ValueError, match=r"10\.52202/068431-2156"): + io_utils.build_a2i2_folder(str(csv_path), records, str(out)) + + assert stale.exists() + def test_dedup_by_title(self, tmp_path: Path) -> None: out = tmp_path / "output" dir_a = out / "Smith (A1)" diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 350c6c4e..ad768922 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -82,6 +82,23 @@ def test_initials_only_author_list_is_not_complete() -> None: assert article_mod._entry_is_complete(entry) is False +def test_published_doi_with_preprint_url_is_not_complete() -> None: + entry = { + "type": "article", + "key": "Kafaie2023", + "fields": { + "title": "Sarand: exploring antimicrobial resistance gene neighbourhoods", + "author": "Somayeh Kafaie and Robert G. Beiko and Finlay Maguire", + "year": "2023", + "journal": "NAR Genomics and Bioinformatics", + "doi": "10.1093/nargab/lqag066", + "url": "https://doi.org/10.1101/2023.10.29.564611", + }, + } + + assert article_mod._entry_is_complete(entry) is False + + @pytest.mark.parametrize(("title", "reaches_baseline"), [("Games", False), ("Good Title", True)]) def test_process_article_title_word_boundary_stops_before_or_reaches_baseline( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, title: str, reaches_baseline: bool @@ -169,6 +186,58 @@ def collect_version_candidate(*args: Any, **kwargs: Any) -> None: assert "10.20944/preprints202304.0409.v2" not in output +def test_rejected_candidate_doi_never_enters_disk_deduplication_net() -> None: + """A different preprint with a similar title must remain invisible to disk deduplication.""" + baseline = { + "type": "misc", + "key": "Strom2021", + "fields": { + "title": "Genome-wide association study identifies new locus associated with OCD", + "author": "Nora I. Strom and Dongmei Yu and Zachary F. Gerring", + "year": "2021", + "doi": "10.1101/2021.10.13.21261078", + }, + } + candidates = [ + { + "bibtex": dedent("""\ + @misc{Strom2024, + title = {Genome-wide association study identifies new loci associated with OCD}, + author = {Nora I. Strom and Matthew W. Halvorsen and Chao Tian}, + year = {2024}, + doi = {10.1101/2024.03.06.24303776} + }"""), + }, + { + "bibtex": dedent("""\ + @misc{Strom2021, + title = {Genome-wide association study identifies new locus associated with OCD}, + author = {Nora I. Strom and Dongmei Yu and Zachary F. Gerring}, + year = {2021}, + doi = {10.1101/2021.10.13.21261078} + }"""), + }, + ] + enrichers: list[tuple[str, dict[str, Any]]] = [] + flags = {"crossref": False} + seen_dois: set[str] = set() + + matched, _ = article_mod._try_multiple_candidates( + "Crossref", + candidates, + lambda candidate, **_kwargs: candidate["bibtex"], + baseline, + "strom", + enrichers, + flags, + "crossref", + seen_dois=seen_dois, + ) + + assert matched is True + assert seen_dois == {"10.1101/2021.10.13.21261078"} + + def test_validate_doi_candidate_both_formats_match() -> None: """ Verify that DOI validation succeeds when both CSL and BibTeX metadata diff --git a/tests/test_regression.py b/tests/test_regression.py index 85c0d816..f8f5a9f7 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -1669,6 +1669,49 @@ def test_journal_backfilled_after_phantom_removal(self) -> None: assert merged["fields"].get("journal") == "Sensors" +def test_validated_arxiv_doi_drops_unconfirmed_scholar_container() -> None: + primary = { + "type": "article", + "key": "Rodriguez2024", + "fields": { + "title": "Predicting Individual Depression Symptoms from Acoustic Features During Speech", + "author": ( + "Sebastian Rodriguez and Sri Harsha Dumpala and Katerina Dikaios and " + "Sheri Rempel and Rudolf Uher and Sageev Oore" + ), + "year": "2024", + "journal": "Scientific Reports", + "publisher": "Springer Science and Business Media LLC", + "volume": "13", + "number": "1", + "pages": "11155", + "eprint": "2406.16000", + "archiveprefix": "arXiv", + }, + } + arxiv = { + "type": "misc", + "fields": { + "title": primary["fields"]["title"], + "author": primary["fields"]["author"], + "year": "2024", + "howpublished": "arXiv", + "doi": "10.48550/arxiv.2406.16000", + "url": "https://arxiv.org/abs/2406.16000", + "eprint": "2406.16000", + "archiveprefix": "arXiv", + }, + } + + result = merge_utils.merge_with_policy(primary, [("csl", arxiv)]) + + assert result["type"] == "misc" + assert result["fields"]["doi"] == "10.48550/arxiv.2406.16000" + assert result["fields"]["url"] == "https://arxiv.org/abs/2406.16000" + for field in ("journal", "publisher", "volume", "number", "pages"): + assert field not in result["fields"] + + class TestIncollectionPromotionRestricted: """incollection→inproceedings should only fire for GENERIC_SERIES_NAMES.""" diff --git a/tests/test_scheduler_parallel.py b/tests/test_scheduler_parallel.py index 8331b83c..febac3e2 100644 --- a/tests/test_scheduler_parallel.py +++ b/tests/test_scheduler_parallel.py @@ -141,3 +141,41 @@ def capture(_rec: Record, art: dict[str, Any], *_args: object, **_kwargs: object assert saved == 1 assert [item["title"] for item in seen] == ["An Orphaned Publication Record"] assert seen[0]["source"] == "existing_corpus" + + +def test_run_all_forces_only_authors_with_incoherent_doi_copies( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + forced: dict[str, bool] = {} + + def capture_record( + _serpapi_key: str, + _serply_key: str | None, + rec: Record, + _out_dir: str, + **kwargs: Any, + ) -> int: + forced[rec.name] = bool(kwargs["force_enrich"]) + return 0 + + monkeypatch.setattr( + scheduler, + "find_incoherent_doi_author_dirs", + lambda _out_dir: frozenset({"Lovelace (a1)"}), + raising=False, + ) + monkeypatch.setattr(scheduler, "process_record", capture_record) + + scheduler.run_all( + "key", + None, + None, + None, + None, + [Record("Ada Lovelace", scholar_id="a1"), Record("Grace Hopper", scholar_id="g1")], + str(tmp_path), + None, + False, + ) + + assert forced == {"Ada Lovelace": True, "Grace Hopper": False} diff --git a/tests/test_text_utils.py b/tests/test_text_utils.py index 8c28fa5f..9138d2c7 100644 --- a/tests/test_text_utils.py +++ b/tests/test_text_utils.py @@ -127,6 +127,21 @@ def test_author_overlap_distinguishes_same_surname_different_initials() -> None: assert different < identical +def test_author_overlap_accepts_abbreviated_prefix_initials() -> None: + abbreviated = "Etienne D and Archambault P and Witteman HO" + full = "Doriane Etienne and Patrick M Archambault and Holly O Witteman" + + assert author_overlap_ratio(abbreviated, full) == 1.0 + + +def test_author_overlap_uses_maximal_one_to_one_initial_matching() -> None: + assert author_overlap_ratio("Smith, P and Smith, P M", "Smith, P M and Smith, P X") == 1.0 + + +def test_author_overlap_preserves_apostrophes_inside_surnames() -> None: + assert author_overlap_ratio("Sageev Oore and Jason D'eon", "Oore, Sageev and D'eon, Jason") == 1.0 + + def _preprint_side() -> dict[str, str]: return {"title": "T", "author": "Smith, John", "year": "2021", "journal": "arXiv"}