实现图片附件的文字识别

crawler/pyproject.toml

@@ -7,11 +7,13 @@ dependencies = [
     "httpx>=0.28",
     "legacy-doc>=0.2.1",
     "lxml>=6.1.3",
+    "onnxruntime>=1.31.1",
     "openpyxl>=3.1.5",
     "playwright>=1.64.1",
     "pydantic>=2.14.1",
     "pypdf>=7.0.0",
     "python-docx>=1.2.0",
+    "rapidocr>=3.10.0",
     "xlrd>=2.0.2",
 ]
 
@@ -21,6 +23,7 @@ crawler = "crawler.main:main"
 [dependency-groups]
 dev = [
     "datamodel-code-generator>=0.86.0",
+    "pillow>=13.0.0",
     "pyright>=1.1.400",
     "pytest>=9.1",
     "ruff>=0.17",

crawler/src/crawler/parse/files.py

@@ -1,5 +1,5 @@
-"""The words in the files notices attach (design §5.1): PDF, Word and Excel, the old binary formats
-too. Images wait for text recognition (#47); archives and other files have none to read."""
+"""The words in the files notices attach (design §5.1): images, PDF, Word and Excel, including
+the old binary formats. Archives and other files have none to read."""
 
 import io
 import unicodedata
@@ -14,9 +14,20 @@
 from legacy_doc import extract_text
 from pypdf import PdfReader
 
+from crawler.parse.image import image_text
+
 _PDF = b"%PDF"
 _ZIP = b"PK\x03\x04"
 _OLE = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+_IMAGE_HEADERS = (
+    b"\x89PNG\r\n\x1a\n",
+    b"\xff\xd8\xff",
+    b"GIF87a",
+    b"GIF89a",
+    b"BM",
+    b"II*\x00",
+    b"MM\x00*",
+)
 # Old Word files keep their text in a stream of this name; old Excel files have none.
 _WORD = "WordDocument".encode("utf-16-le")
 # Some PDFs map common characters to the Kangxi radicals that look like them, ⽣ for 生.
@@ -26,6 +37,8 @@
 def file_text(data: bytes) -> str:
     """Raises ValueError when the file is broken."""
     try:
+        if data.startswith(_IMAGE_HEADERS) or (data.startswith(b"RIFF") and data[8:12] == b"WEBP"):
+            return image_text(data)
         if data.startswith(_PDF):
             pages = PdfReader(io.BytesIO(data)).pages
             return "\n".join(page.extract_text() for page in pages).translate(_RADICALS)

crawler/src/crawler/parse/files_test.py

@@ -1,13 +1,39 @@
 import io
 import zipfile
+from pathlib import Path
 
 import docx
 import openpyxl
 import pytest
+from PIL import Image, ImageDraw, ImageFont
 
 from crawler.parse.files import file_text
 
 
+def test_image_reads_generated_chinese_attachment() -> None:
+    image = Image.new("RGB", (1600, 500), "white")
+    draw = ImageDraw.Draw(image)
+    font = ImageFont.truetype(Path(__file__).with_name("fixtures") / "NotoSansSC-test.ttf", 64)
+    draw.text((60, 50), "夏令营报名时间", fill="black", font=font)
+    draw.text((60, 170), "7月6日10:00至8月10日17:00", fill="black", font=font)
+    draw.text((60, 290), "请填写申请表", fill="black", font=font)
+    buffer = io.BytesIO()
+    image.save(buffer, format="PNG")
+
+    text = file_text(buffer.getvalue())
+
+    assert "夏令营报名时间" in text
+    assert "7月6日10:00至8月10日17:00" in text
+    assert "请填写申请表" in text
+
+
+def test_blank_image_has_no_words_to_read() -> None:
+    buffer = io.BytesIO()
+    Image.new("RGB", (600, 300), "white").save(buffer, format="JPEG")
+
+    assert file_text(buffer.getvalue()) == ""
+
+
 def test_word_keeps_paragraphs_and_tables_in_order_with_merged_cells_once() -> None:
     document = docx.Document()
     document.add_paragraph("报名截止6月30日。")
@@ -80,9 +106,10 @@ def test_files_with_no_words_to_read_give_none() -> None:
     with zipfile.ZipFile(archive, "w") as files:
         files.writestr("说明.txt", "自编的说明")
     assert file_text(archive.getvalue()) == ""
-    assert file_text(b"\x89PNG\r\n\x1a\n") == ""
 
 
 def test_a_broken_file_is_an_error() -> None:
     with pytest.raises(ValueError, match="BadZipFile"):
         file_text(b"PK\x03\x04" + b"\x00" * 30)
+    with pytest.raises(ValueError, match="UnidentifiedImageError"):
+        file_text(b"\x89PNG\r\n\x1a\n")

crawler/src/crawler/parse/fixtures/NotoSansSC-OFL.txt

@@ -0,0 +1,93 @@
+Copyright 2014-2021 Adobe (http://www.adobe.com/), with Reserved Font Name 'Source'
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+https://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

crawler/src/crawler/parse/image.py

@@ -0,0 +1,43 @@
+"""Read text from image attachments."""
+
+import logging
+import resource
+import threading
+import time
+
+from rapidocr import RapidOCR
+from rapidocr.utils.output import RapidOCROutput
+
+logger = logging.getLogger(__name__)
+_lock = threading.Lock()
+_engine: RapidOCR | None = None
+
+
+def image_text(data: bytes) -> str:
+    """Return recognized lines in reading order, with one OCR call at a time."""
+    global _engine
+
+    with _lock:
+        start = time.monotonic()
+        if _engine is None:
+            _engine = RapidOCR(
+                params={
+                    # Find lines on a smaller copy, then crop the legible text from the original.
+                    "Global.use_preprocess_img": False,
+                    "Det.limit_type": "max",
+                    "Det.limit_side_len": 2000,
+                    # A batch is padded to its widest line; one line at a time keeps the peak low.
+                    "Rec.rec_batch_num": 1,
+                }
+            )
+        result = _engine(data)
+        if not isinstance(result, RapidOCROutput):
+            raise TypeError("OCR did not return recognized text")
+        logger.info(
+            "Image OCR completed",
+            extra={
+                "elapsed": time.monotonic() - start,
+                "peak_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
+            },
+        )
+        return "\n".join(result.txts or [])