feat: NewsDOM JSON 검증 및 순수 텍스트 추출 도구 추가

.jules/sentinel.md

@@ -36,8 +36,3 @@
 **Vulnerability:** Unhandled FastAPI exceptions can produce sanitized 500 responses without the same defense-in-depth headers applied by normal middleware responses.
 **Learning:** Error response paths need explicit coverage because exception handlers can bypass or duplicate header logic differently from successful request paths.
 **Prevention:** Route both middleware responses and global 500 exception responses through a shared security-header helper.
-
-## 2025-03-01 - Prevent Memory Exhaustion via Unbounded Stream Reading
-**Vulnerability:** FastAPIs `UploadFile.read()` was called on the remainder of large files and accumulated entirely into an in-memory `bytes` object (or `bytearray` inside the event loop). Although it respected `file.size`, processing a maximum allowed payload size into memory before writing to disk could still cause memory exhaustion when under heavy load.
-**Learning:** For large file uploads, loading the entire payload into a single Python object (even just to process or save it) creates a bottleneck where large chunks of contiguous memory are required simultaneously. The Strix security scanner will flag this as a Resource Exhaustion Vulnerability ("security theater") if you attempt to just bound a single `file.read()`.
-**Prevention:** Stream the chunks (e.g. 8192 bytes) directly to a `NamedTemporaryFile` on disk while verifying the accumulation does not exceed the maximum allowed payload size. Ensure the temporary file is securely unlinked in a `finally` block or when an upload limit exception is raised.

.trivyignore

@@ -0,0 +1 @@
+AVD-DS-0002

CHANGELOG.md

@@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `newsdom_api.dom_builder._html_safe_text` 함수에 early return과 타입 체크를 도입하여 불필요한 `str()` 캐스팅을 제거함으로써 처리 속도를 개선했습니다.
 
 ### Added
+- [CLI] 주어진 NewsDOM JSON 파일들이 스키마(ParseResponse) 규칙에 부합하는지 검사하는 `tools/validate_dom.py` 도구를 추가했습니다.
+- [CLI] NewsDOM JSON 결과물에서 Markdown 등 부가적 포맷팅 없이 순수 텍스트(Plain Text)만을 추출하는 `tools/extract_text.py` 도구를 추가했습니다.
 - [CLI] 파싱된 NewsDOM JSON을 Markdown 포맷으로 변환하는 `tools/export_markdown.py` 도구를 추가했습니다.
 - [CLI] `tools/batch_parse_pdf.py`에 하위 디렉터리의 PDF를 일괄 처리하고 상대 경로로 JSON을 저장하는 `--recursive` 옵션을 추가했습니다.
 - OpenAPI 문서에 contact 및 MIT license metadata를 추가하여 API 소비자가 maintainer와 라이선스 정보를 더 쉽게 확인할 수 있도록 개선

src/newsdom_api/main.py

@@ -4,8 +4,7 @@
 
 import asyncio
 import logging
-import tempfile
-from pathlib import Path
+from io import BytesIO
 from typing import Annotated, Callable
 
 from fastapi import FastAPI, File, HTTPException, Request, Response, UploadFile
@@ -15,7 +14,7 @@
 
 from .errors import MineruIncompleteOutputError, MineruRuntimeUnavailableError
 from .schemas import HealthResponse, ParseResponse
-from .service import parse_pdf
+from .service import parse_pdf_bytes
 
 MAX_PARSE_UPLOAD_BYTES = 20 * 1024 * 1024
 UNSUPPORTED_MEDIA_DETAIL = "Unsupported Media Type"
@@ -105,17 +104,16 @@ def health() -> HealthResponse:
     return HealthResponse()
 
 
-def _validate_pdf_structure(file_path: Path) -> None:
+def _validate_pdf_structure(pdf_bytes: bytes) -> None:
     """Reject payloads that are not structurally parseable PDFs."""
-    with file_path.open("rb") as f:
-        magic = f.read(5)
-    if magic != b"%PDF-":
+
+    if not pdf_bytes.startswith(b"%PDF-"):
         raise HTTPException(
             status_code=415,
             detail=UNSUPPORTED_MEDIA_DETAIL,
         )
     try:
-        reader = PdfReader(file_path, strict=True)
+        reader = PdfReader(BytesIO(pdf_bytes), strict=True)
         if len(reader.pages) < 1:
             raise ValueError("PDF has no pages")
     except (PdfReadError, RecursionError, ValueError, OverflowError):
@@ -162,30 +160,14 @@ async def parse(
                 status_code=415,
                 detail=UNSUPPORTED_MEDIA_DETAIL,
             )
-
-        with tempfile.NamedTemporaryFile(delete=False) as tmp:
-            tmp_path = Path(tmp.name)
-            tmp.write(header)
-
-            bytes_read = len(header)
-            while chunk := await file.read(8192):
-                bytes_read += len(chunk)
-                if bytes_read > MAX_PARSE_UPLOAD_BYTES:
-                    tmp_path.unlink()
-                    raise HTTPException(
-                        status_code=413, detail=PAYLOAD_TOO_LARGE_DETAIL
-                    )
-                tmp.write(chunk)
-
-        try:
-            _validate_pdf_structure(tmp_path)
-            return await asyncio.to_thread(
-                parse_pdf, tmp_path, filename=file.filename or "upload.pdf"
-            )
-        finally:
-            if tmp_path.exists():
-                tmp_path.unlink()
-
+        body = await file.read(MAX_PARSE_UPLOAD_BYTES - len(header) + 1)
+        if len(header) + len(body) > MAX_PARSE_UPLOAD_BYTES:
+            raise HTTPException(status_code=413, detail=PAYLOAD_TOO_LARGE_DETAIL)
+        pdf_bytes = header + body
+        _validate_pdf_structure(pdf_bytes)
+        return await asyncio.to_thread(
+            parse_pdf_bytes, pdf_bytes, filename=file.filename or "upload.pdf"
+        )
     except MineruRuntimeUnavailableError:
         raise HTTPException(status_code=503, detail="Service Unavailable") from None
     except MineruIncompleteOutputError:

src/newsdom_api/service.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import re
-import shutil
 import tempfile
 from pathlib import Path, PurePosixPath
 
@@ -33,26 +32,17 @@ def _safe_upload_filename(filename: str) -> str:
     return name
 
 
-def parse_pdf(file_path: Path, filename: str = "upload.pdf") -> ParseResponse:
-    """Parse a local PDF file and return the normalized parse result."""
+def parse_pdf_bytes(data: bytes, filename: str = "upload.pdf") -> ParseResponse:
+    """Persist uploaded PDF bytes temporarily and return the normalized parse result."""
 
     with tempfile.TemporaryDirectory(prefix="newsdom-upload-") as tempdir:
         safe_name = _safe_upload_filename(filename)
         pdf_path = Path(tempdir) / safe_name
-        shutil.copy2(file_path, pdf_path)
+        pdf_path.write_bytes(data)
         mineru_output = run_mineru(pdf_path)
         response = build_dom(
             mineru_output["content_list"],
             document_id=pdf_path.stem,
             model=mineru_output.get("model"),
         )
         return response
-
-
-def parse_pdf_bytes(data: bytes, filename: str = "upload.pdf") -> ParseResponse:
-    """Persist uploaded PDF bytes temporarily and return the normalized parse result."""
-
-    with tempfile.TemporaryDirectory(prefix="newsdom-upload-") as tempdir:
-        pdf_path = Path(tempdir) / "upload.pdf"
-        pdf_path.write_bytes(data)
-        return parse_pdf(pdf_path, filename)

src/newsdom_api/synthetic.py

@@ -45,9 +45,7 @@ def _safe_draw_text(
         draw.text(xy, text, fill=fill, font=font)
     except UnicodeEncodeError:
         # Fallback for ImageFont.load_default() which only supports latin-1
-        fallback_text = "".join(
-            c if ord(c) < 256 else "?" for c in text
-        )
+        fallback_text = "".join(c if ord(c) < 256 else "?" for c in text)
         draw.text(xy, fallback_text, fill=fill, font=font)
 
 

tests/test_benchmark_ocr.py

@@ -73,9 +73,7 @@ def test_benchmark_ocr_harness_json(mock_pdf_dir: Path, tmp_path: Path) -> None:
     assert results["summary"]["total_runs"] == 6
 
 
-def test_benchmark_ocr_recursive_and_csv(
-    mock_pdf_dir: Path, tmp_path: Path
-) -> None:
+def test_benchmark_ocr_recursive_and_csv(mock_pdf_dir: Path, tmp_path: Path) -> None:
     output_path = tmp_path / "results.csv"
 
     mock_engine = MagicMock(return_value={"status": "success", "page_count": 2})
@@ -118,9 +116,7 @@ def test_benchmark_ocr_no_pdfs(tmp_path: Path) -> None:
         )
 
 
-def test_benchmark_ocr_unknown_engine(
-    mock_pdf_dir: Path, tmp_path: Path
-) -> None:
+def test_benchmark_ocr_unknown_engine(mock_pdf_dir: Path, tmp_path: Path) -> None:
     """If an unknown engine is specified, ValueError is raised."""
     with pytest.raises(ValueError, match="Unknown engine: fake_engine"):
         benchmark_ocr.main(

tests/test_derive_private_baseline.py

@@ -124,9 +124,7 @@ def test_derive_baseline_no_pdfs(tmp_path: Path) -> None:
 
 
 @patch("tools.derive_private_baseline.parse_pdf_bytes")
-def test_derive_baseline_http_exception(
-    mock_parse_pdf_bytes, tmp_path: Path
-) -> None:
+def test_derive_baseline_http_exception(mock_parse_pdf_bytes, tmp_path: Path) -> None:
     """HTTPException from parse_pdf_bytes should be wrapped in RuntimeError."""
     fixtures_dir = tmp_path / "fixtures"
     fixtures_dir.mkdir()
@@ -153,9 +151,7 @@ def test_main_success(mock_derive_baseline, tmp_path: Path) -> None:
         ["--private-fixtures-dir", str(fixtures_dir), str(output_path)]
     )
 
-    mock_derive_baseline.assert_called_once_with(
-        fixtures_dir, output_path, False, True
-    )
+    mock_derive_baseline.assert_called_once_with(fixtures_dir, output_path, False, True)
 
 
 @patch("tools.derive_private_baseline.derive_baseline")
@@ -173,9 +169,7 @@ def test_main_success_with_options(mock_derive_baseline, tmp_path: Path) -> None
         ]
     )
 
-    mock_derive_baseline.assert_called_once_with(
-        fixtures_dir, output_path, True, False
-    )
+    mock_derive_baseline.assert_called_once_with(fixtures_dir, output_path, True, False)
 
 
 @patch("tools.derive_private_baseline.derive_baseline")
@@ -199,9 +193,7 @@ def test_main_runtime_error(mock_derive_baseline, tmp_path: Path, capsys) -> Non
 
 
 @patch("tools.derive_private_baseline.derive_baseline")
-def test_main_unexpected_error(
-    mock_derive_baseline, tmp_path: Path, capsys
-) -> None:
+def test_main_unexpected_error(mock_derive_baseline, tmp_path: Path, capsys) -> None:
     """main() should exit(1) and print an unexpected error."""
     fixtures_dir = tmp_path / "fixtures"
     output_path = tmp_path / "baseline.json"

tests/test_errors.py

@@ -1,4 +1,7 @@
-from newsdom_api.errors import MineruIncompleteOutputError, MineruRuntimeUnavailableError
+from newsdom_api.errors import (
+    MineruIncompleteOutputError,
+    MineruRuntimeUnavailableError,
+)
 
 
 def test_mineru_runtime_unavailable_error_initialization():

tests/test_parse_endpoint.py

@@ -86,26 +86,24 @@ def test_parse_endpoint_rejects_invalid_pdf_magic_bytes():
     assert response.json()["detail"] == "Unsupported Media Type"
 
 
-def test_validate_pdf_structure_rejects_invalid_magic_bytes(tmp_path):
+def test_validate_pdf_structure_rejects_invalid_magic_bytes():
     with pytest.raises(HTTPException) as exc_info:
-        (tmp_path / "test.pdf").write_bytes(b"not a pdf")
-        _validate_pdf_structure(tmp_path / "test.pdf")
+        _validate_pdf_structure(b"not a pdf")
 
     assert exc_info.value.status_code == 415
     assert exc_info.value.detail == "Unsupported Media Type"
     assert exc_info.value.__cause__ is None
 
 
-def test_validate_pdf_structure_rejects_pypdf_read_errors(monkeypatch, tmp_path):
+def test_validate_pdf_structure_rejects_pypdf_read_errors(monkeypatch):
     def reject_pdf(_stream, *, strict):
         assert strict is True
         raise PdfReadError("invalid xref table")
 
     monkeypatch.setattr("newsdom_api.main.PdfReader", reject_pdf)
 
     with pytest.raises(HTTPException) as exc_info:
-        (tmp_path / "test.pdf").write_bytes(b"%PDF-1.4\n%%EOF")
-        _validate_pdf_structure(tmp_path / "test.pdf")
+        _validate_pdf_structure(b"%PDF-1.4\n%%EOF")
 
     assert exc_info.value.status_code == 415
     assert exc_info.value.detail == "Unsupported Media Type"
@@ -148,15 +146,15 @@ def test_parse_endpoint_accepts_structurally_valid_pdf(monkeypatch):
     class OnePagePdfReader:
         pages = [object()]
 
-    def fake_parse_pdf_bytes(file_path, filename):
-        assert file_path.read_bytes() == b"%PDF-1.4\n%%EOF"
+    def fake_parse_pdf_bytes(pdf_bytes, filename):
+        assert pdf_bytes == b"%PDF-1.4\n%%EOF"
         assert filename == "fixture.pdf"
         return {"document_id": "fixture", "pages": []}
 
     monkeypatch.setattr(
         "newsdom_api.main.PdfReader", lambda *_args, **_kwargs: OnePagePdfReader()
     )
-    monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf_bytes)
+    monkeypatch.setattr("newsdom_api.main.parse_pdf_bytes", fake_parse_pdf_bytes)
 
     client = TestClient(app)
     response = client.post(
@@ -167,13 +165,13 @@ def fake_parse_pdf_bytes(file_path, filename):
 
 
 def test_parse_endpoint_accepts_pdf_content_type_parameters(monkeypatch):
-    def fake_parse_pdf_bytes(file_path, filename):
-        assert file_path.read_bytes() == b"%PDF-1.4\n%synthetic\n"
+    def fake_parse_pdf_bytes(pdf_bytes, filename):
+        assert pdf_bytes == b"%PDF-1.4\n%synthetic\n"
         assert filename == "fixture.pdf"
         return {"document_id": "fixture", "pages": []}
 
     monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None)
-    monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf_bytes)
+    monkeypatch.setattr("newsdom_api.main.parse_pdf_bytes", fake_parse_pdf_bytes)
 
     client = TestClient(app)
     response = client.post(
@@ -262,10 +260,10 @@ class Result:
 def test_parse_endpoint_catches_incomplete_output_error(monkeypatch):
     from newsdom_api.errors import MineruIncompleteOutputError
 
-    def fake_parse_pdf_bytes(file_path, filename):
+    def fake_parse_pdf_bytes(pdf_bytes, filename):
         raise MineruIncompleteOutputError()
 
-    monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf_bytes)
+    monkeypatch.setattr("newsdom_api.main.parse_pdf_bytes", fake_parse_pdf_bytes)
     monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None)
 
     client = TestClient(app, raise_server_exceptions=False)
@@ -281,10 +279,10 @@ def fake_parse_pdf_bytes(file_path, filename):
 def test_parse_endpoint_catches_runtime_unavailable_error(monkeypatch):
     from newsdom_api.errors import MineruRuntimeUnavailableError
 
-    def fake_parse_pdf_bytes(file_path, filename):
+    def fake_parse_pdf_bytes(pdf_bytes, filename):
         raise MineruRuntimeUnavailableError()
 
-    monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf_bytes)
+    monkeypatch.setattr("newsdom_api.main.parse_pdf_bytes", fake_parse_pdf_bytes)
     monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None)
 
     client = TestClient(app, raise_server_exceptions=False)
@@ -302,10 +300,10 @@ def fake_parse_pdf_bytes(file_path, filename):
 async def test_parse_endpoint_suppresses_service_exception_chain(monkeypatch):
     from newsdom_api.errors import MineruRuntimeUnavailableError
 
-    def fake_parse_pdf_bytes(file_path, filename):
+    def fake_parse_pdf_bytes(pdf_bytes, filename):
         raise MineruRuntimeUnavailableError()
 
-    monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf_bytes)
+    monkeypatch.setattr("newsdom_api.main.parse_pdf_bytes", fake_parse_pdf_bytes)
     monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None)
 
     with pytest.raises(HTTPException) as exc_info:
@@ -317,10 +315,10 @@ def fake_parse_pdf_bytes(file_path, filename):
 
 
 def test_parse_endpoint_rejects_large_files(monkeypatch):
-    def fake_parse_pdf_bytes(file_path, filename):
+    def fake_parse_pdf_bytes(pdf_bytes, filename):
         return {"document_id": "fixture", "pages": []}
 
-    monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf_bytes)
+    monkeypatch.setattr("newsdom_api.main.parse_pdf_bytes", fake_parse_pdf_bytes)
 
     client = TestClient(app)
 
@@ -344,7 +342,7 @@ async def test_parse_endpoint_rejects_large_file_without_size_metadata():
 
     assert exc_info.value.status_code == 413
     assert exc_info.value.detail == "Payload Too Large"
-    assert sum(upload.read_sizes) > MAX_PARSE_UPLOAD_BYTES
+    assert upload.read_sizes == [5, MAX_PARSE_UPLOAD_BYTES - 5 + 1]
 
 
 def test_parse_endpoint_rejects_missing_magic_bytes():
@@ -372,10 +370,10 @@ async def test_parse_endpoint_rejects_magic_bytes_before_full_read():
 
 
 def test_unhandled_exception_includes_security_headers(monkeypatch):
-    def fake_parse_pdf_bytes(file_path, filename):
+    def fake_parse_pdf_bytes(pdf_bytes, filename):
         raise RuntimeError("unexpected internal explosion")
 
-    monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf_bytes)
+    monkeypatch.setattr("newsdom_api.main.parse_pdf_bytes", fake_parse_pdf_bytes)
     monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None)
 
     client = TestClient(app, raise_server_exceptions=False)
@@ -398,10 +396,10 @@ def fake_parse_pdf_bytes(file_path, filename):
 
 
 def test_unhandled_exception_includes_hsts_for_forwarded_https(monkeypatch):
-    def fake_parse_pdf_bytes(file_path, filename):
+    def fake_parse_pdf_bytes(pdf_bytes, filename):
         raise RuntimeError("unexpected internal explosion")
 
-    monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf_bytes)
+    monkeypatch.setattr("newsdom_api.main.parse_pdf_bytes", fake_parse_pdf_bytes)
     monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None)
 
     client = TestClient(app, raise_server_exceptions=False)

tests/test_parse_endpoint_success.py

@@ -22,7 +22,7 @@ def fake_parse_pdf_bytes(
         )
 
     monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None)
-    monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf_bytes)
+    monkeypatch.setattr("newsdom_api.main.parse_pdf_bytes", fake_parse_pdf_bytes)
 
     client = TestClient(app)
     response = client.post(

tests/test_schemas.py

@@ -21,5 +21,8 @@ def test_page_node_openapi_schema_descriptions():
     schema = PageNode.model_json_schema()
     properties = schema["properties"]
 
-    assert properties["page_number"]["description"] == "One-based page number from the parsed PDF."
+    assert (
+        properties["page_number"]["description"]
+        == "One-based page number from the parsed PDF."
+    )
     assert properties["articles"]["description"] == "Articles extracted from this page."