修复四位 arXiv 编号识别与过长数字误截断

fastinsight/article.py

@@ -29,11 +29,11 @@
 DOI_RE = re.compile(r"\b10\.\d{4,9}/[^\s<>\"']+", re.IGNORECASE)
 ARXIV_LABEL_RE = re.compile(
     r"(?:https?://)?arxiv\.org/(?:abs|pdf)/"
-    r"([a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?|\d{4}\.\d{5}(?:v\d+)?)|"
-    r"arXiv:\s*([a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?|\d{4}\.\d{5}(?:v\d+)?)",
+    r"([a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?|\d{4}\.\d{4,5}(?!\d)(?:v\d+)?)|"
+    r"arXiv:\s*([a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?|\d{4}\.\d{4,5}(?!\d)(?:v\d+)?)",
     re.IGNORECASE,
 )
-ARXIV_BARE_RE = re.compile(r"(?<![\w./])(\d{4}\.\d{5}(?:v\d+)?)(?![\w.])")
+ARXIV_BARE_RE = re.compile(r"(?<![\w./])(\d{4}\.\d{4,5}(?:v\d+)?)(?![\w.])")
 WECHAT_BLOCKED = ("环境异常", "当前环境异常", "完成验证后即可继续", "security verify")
 SHARE_PATTERNS = (
     re.compile(r"复制后打开【小红书】查看笔记[！!]?", re.I),

scripts/verify_paper.py

@@ -110,12 +110,12 @@ def _load_env_file(path):
 DOI_RE = re.compile(r"\b10\.\d{4,9}/[^\s<>\"']+", re.IGNORECASE)
 ARXIV_LABEL_RE = re.compile(
     r"(?:https?://)?arxiv\.org/(?:abs|pdf)/"
-    r"([a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?|\d{4}\.\d{5}(?:v\d+)?)|"
-    r"arXiv:\s*([a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?|\d{4}\.\d{5}(?:v\d+)?)",
+    r"([a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?|\d{4}\.\d{4,5}(?!\d)(?:v\d+)?)|"
+    r"arXiv:\s*([a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?|\d{4}\.\d{4,5}(?!\d)(?:v\d+)?)",
     re.IGNORECASE,
 )
 ARXIV_BARE_RE = re.compile(
-    r"(?<![\w./])(\d{4}\.\d{5}(?:v\d+)?)(?![\w.])"
+    r"(?<![\w./])(\d{4}\.\d{4,5}(?:v\d+)?)(?![\w.])"
 )
 
 

tests/test_article.py

@@ -2,7 +2,7 @@
 from __future__ import annotations
 
 import unittest
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 from fastinsight.article import (
     clean_share_text,
@@ -83,6 +83,51 @@ def test_identifiers_from_blob(self) -> None:
         self.assertEqual(found["arxiv_id"], "1706.03762")
         self.assertEqual(found["doi"], "10.1000/xyz")
 
+    def test_body_identifiers_produce_canonical_arxiv_url(self) -> None:
+        for arxiv_id in ("1409.0473", "1409.0473v2", "1706.03762", "1706.03762v2"):
+            for label in ("", "arXiv:"):
+                with self.subTest(arxiv_id=arxiv_id, label=label):
+                    html = (
+                        '<html><title>论文介绍</title><div id="js_content">'
+                        f"本文介绍预印本 {label}{arxiv_id} 的研究成果。"
+                        "</div></html>"
+                    )
+                    llm = Mock(return_value=None)
+                    with patch("fastinsight.article.fetch_html") as fetch:
+                        paper = extract_paper_from_article(
+                            "https://mp.weixin.qq.com/s/abc", html=html, llm=llm
+                        )
+                    self.assertEqual(paper["arxiv_id"], arxiv_id)
+                    self.assertEqual(paper["url"], f"https://arxiv.org/abs/{arxiv_id}")
+                    self.assertTrue(paper["paper_found"])
+                    fetch.assert_not_called()
+                    llm.assert_not_called()
+
+    def test_body_doi_still_takes_priority_over_arxiv(self) -> None:
+        html = (
+            '<html><title>论文介绍</title><div id="js_content">'
+            "arXiv:1409.0473v2 DOI: 10.1000/ABC;"
+            "</div></html>"
+        )
+        paper = extract_paper_from_article("https://mp.weixin.qq.com/s/abc", html=html)
+        self.assertEqual(paper["arxiv_id"], "1409.0473v2")
+        self.assertEqual(paper["doi"], "10.1000/abc")
+        self.assertEqual(paper["url"], "https://doi.org/10.1000/abc")
+        self.assertTrue(paper["paper_found"])
+
+    def test_existing_pdf_url_still_takes_priority_over_doi(self) -> None:
+        url = "https://arxiv.org/pdf/1409.0473v2.pdf"
+        html = (
+            '<html><title>论文介绍</title><div id="js_content">'
+            f'<a href="{url}">论文</a> DOI: 10.1000/ABC;'
+            "</div></html>"
+        )
+        paper = extract_paper_from_article("https://mp.weixin.qq.com/s/abc", html=html)
+        self.assertEqual(paper["arxiv_id"], "1409.0473v2")
+        self.assertEqual(paper["doi"], "10.1000/abc")
+        self.assertEqual(paper["url"], url)
+        self.assertTrue(paper["paper_found"])
+
 
 class PaperFromUrlTests(unittest.TestCase):
     def test_arxiv_url_is_unchanged(self) -> None:

tests/test_arxiv_identifiers.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import unittest
+
+from fastinsight.article import extract_identifiers
+from scripts.verify_paper import _find_identifiers
+
+
+class ArxivIdentifierTests(unittest.TestCase):
+    def assert_identifiers(self, text: str, expected: str) -> None:
+        for extractor in (extract_identifiers, _find_identifiers):
+            with self.subTest(extractor=extractor.__name__, text=text):
+                self.assertEqual(extractor(text).get("arxiv_id", ""), expected)
+
+    def assert_modern_forms(self, arxiv_id: str) -> None:
+        for text in (
+            arxiv_id,
+            f"see ({arxiv_id})",
+            f"arXiv: {arxiv_id}",
+            f"https://arxiv.org/abs/{arxiv_id}",
+            f"https://arxiv.org/pdf/{arxiv_id}",
+            f"https://arxiv.org/pdf/{arxiv_id}.pdf",
+            f"http://arxiv.org/abs/{arxiv_id}",
+            f"arxiv.org/abs/{arxiv_id}",
+        ):
+            self.assert_identifiers(text, arxiv_id)
+
+    def test_four_digit_identifiers(self) -> None:
+        self.assert_modern_forms("1409.0473")
+
+    def test_four_digit_identifiers_preserve_version(self) -> None:
+        self.assert_modern_forms("1409.0473v12")
+
+    def test_five_digit_identifiers(self) -> None:
+        self.assert_modern_forms("1706.03762")
+
+    def test_five_digit_identifiers_preserve_version(self) -> None:
+        self.assert_modern_forms("1706.03762v12")
+
+    def test_long_numeric_identifiers_are_not_truncated(self) -> None:
+        for arxiv_id in (
+            "1409.047312",
+            "1706.0376201",
+            "1409.04731234567890123456",
+            "11409.0473",
+            "11706.03762",
+        ):
+            for version in ("", "v12"):
+                for template in (
+                    "{}",
+                    "arXiv: {}",
+                    "https://arxiv.org/abs/{}",
+                    "https://arxiv.org/pdf/{}",
+                    "https://arxiv.org/pdf/{}.pdf",
+                ):
+                    self.assert_identifiers(template.format(arxiv_id + version), "")
+
+    def test_bare_identifiers_keep_existing_boundaries(self) -> None:
+        for arxiv_id in ("1409.0473", "1706.03762"):
+            for template in ("prefix{}", "folder/{}", "1.{}", "{}suffix", "{}.1"):
+                self.assert_identifiers(template.format(arxiv_id), "")
+
+    def test_legacy_identifiers_are_unchanged(self) -> None:
+        for arxiv_id in ("hep-th/9901001", "math.GT/0309136v2"):
+            for template in (
+                "arXiv: {}",
+                "https://arxiv.org/abs/{}",
+                "https://arxiv.org/pdf/{}.pdf",
+            ):
+                self.assert_identifiers(template.format(arxiv_id), arxiv_id)
+
+    def test_labeled_identifiers_still_take_priority_over_bare(self) -> None:
+        self.assert_identifiers("1706.03762 arXiv:1409.0473v2", "1409.0473v2")
+
+    def test_doi_normalization_and_first_match_are_unchanged(self) -> None:
+        text = "DOI: 10.1000/ABC; arXiv:1409.0473v2 DOI: 10.1000/DEF"
+        for extractor in (extract_identifiers, _find_identifiers):
+            with self.subTest(extractor=extractor.__name__):
+                found = extractor(text)
+                self.assertEqual(found["doi"], "10.1000/abc")
+                self.assertEqual(found.get("arxiv_id", ""), "1409.0473v2")
+
+
+if __name__ == "__main__":
+    unittest.main()