diff --git a/CHANGELOG.md b/CHANGELOG.md index 46fa37c..201db01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `newsdom_api.dom_builder._html_safe_text` 함수에 early return과 타입 체크를 도입하여 불필요한 `str()` 캐스팅을 제거함으로써 처리 속도를 개선했습니다. ### Added +- [CLI] 파싱된 NewsDOM JSON이 Pydantic 스키마(`ParseResponse`)와 일치하는지 엄격하게 검증하는 `tools/validate_dom.py` 도구 추가 +- [CLI] 파싱된 NewsDOM JSON의 기사 제목(headline)과 본문(body_blocks)에서 텍스트를 검색하여 위치를 반환하는 `tools/search_dom.py` 도구 추가 - [CLI] 파싱된 NewsDOM JSON을 Markdown 포맷으로 변환하는 `tools/export_markdown.py` 도구를 추가했습니다. - [CLI] `tools/batch_parse_pdf.py`에 하위 디렉터리의 PDF를 일괄 처리하고 상대 경로로 JSON을 저장하는 `--recursive` 옵션을 추가했습니다. - OpenAPI 문서에 contact 및 MIT license metadata를 추가하여 API 소비자가 maintainer와 라이선스 정보를 더 쉽게 확인할 수 있도록 개선 diff --git a/tests/test_tools_search_dom.py b/tests/test_tools_search_dom.py new file mode 100644 index 0000000..897d431 --- /dev/null +++ b/tests/test_tools_search_dom.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import json +from pathlib import Path +import pytest + +from tools.search_dom import main, search_dom + + +def create_sample_dom(path: Path) -> None: + data = { + "pages": [ + { + "page_number": 1, + "articles": [ + { + "article_id": "art-1", + "headline": "Breaking News Today", + "body_blocks": [ + "This is a test block.", + "We found something amazing.", + "The keyword is hidden here.", + ], + } + ], + }, + { + "page_number": 2, + "articles": [ + { + "article_id": "art-2", + "headline": "Another Keyword headline", + "body_blocks": ["Just random text."], + } + ], + }, + ] + } + path.write_text(json.dumps(data), encoding="utf-8") + + +def test_search_dom_found(tmp_path: Path) -> None: + json_path = tmp_path / "test.json" + create_sample_dom(json_path) + + results = search_dom(json_path, "keyword") + + assert len(results) == 2 + assert results[0]["type"] == "body_block" + assert results[0]["page"] == 1 + assert results[0]["article_id"] == "art-1" + + assert results[1]["type"] == "headline" + assert results[1]["page"] == 2 + assert results[1]["article_id"] == "art-2" + + +def test_search_dom_not_found(tmp_path: Path) -> None: + json_path = tmp_path / "test.json" + create_sample_dom(json_path) + + results = search_dom(json_path, "missingword") + assert len(results) == 0 + + +def test_search_dom_file_not_found(tmp_path: Path) -> None: + non_existent = tmp_path / "missing.json" + with pytest.raises(FileNotFoundError, match="File not found or is not a file"): + search_dom(non_existent, "query") + + +def test_search_dom_invalid_extension(tmp_path: Path) -> None: + txt_path = tmp_path / "test.txt" + txt_path.write_text("not json", encoding="utf-8") + with pytest.raises(ValueError, match="File must be a .json file"): + search_dom(txt_path, "query") + + +def test_search_dom_invalid_json(tmp_path: Path) -> None: + json_path = tmp_path / "invalid_json.json" + json_path.write_text("invalid{json", encoding="utf-8") + with pytest.raises(ValueError, match="Invalid JSON file"): + search_dom(json_path, "query") + + +def test_main_success(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + json_path = tmp_path / "test.json" + create_sample_dom(json_path) + + main([str(json_path), "keyword"]) + + captured = capsys.readouterr() + assert "Found 2 results for query: 'keyword'" in captured.out + assert ( + "- Page 1, Article art-1 [Body Block 2]: The keyword is hidden here." + in captured.out + ) + assert ( + "- Page 2, Article art-2 [Headline]: Another Keyword headline" in captured.out + ) + + +def test_main_no_results(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + json_path = tmp_path / "test.json" + create_sample_dom(json_path) + + main([str(json_path), "nonexistent"]) + + captured = capsys.readouterr() + assert "No results found for query: 'nonexistent'" in captured.out + + +def test_main_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + non_existent = tmp_path / "missing.json" + + with pytest.raises(SystemExit) as exc_info: + main([str(non_existent), "query"]) + + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "Error:" in captured.err + assert "File not found" in captured.err + + +def test_search_dom_unknown_type( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + # Test the unexpected else branch in main() for line 86 + + json_path = tmp_path / "test.json" + data = {"pages": []} + json_path.write_text(json.dumps(data), encoding="utf-8") + + # We mock search_dom to return a result with an unknown type + import tools.search_dom + + def mock_search_dom(*args, **kwargs): + return [ + {"type": "unknown_type", "page": 1, "article_id": "art-1", "text": "text"} + ] + + monkeypatch.setattr(tools.search_dom, "search_dom", mock_search_dom) + + tools.search_dom.main([str(json_path), "query"]) + + captured = capsys.readouterr() + assert "Found 1 results" in captured.out + assert "Headline" not in captured.out + assert "Body Block" not in captured.out diff --git a/tests/test_tools_validate_dom.py b/tests/test_tools_validate_dom.py new file mode 100644 index 0000000..6f5fce6 --- /dev/null +++ b/tests/test_tools_validate_dom.py @@ -0,0 +1,119 @@ +from __future__ import annotations +import runpy +import sys + +import json +from pathlib import Path +import pytest + +from pydantic import ValidationError +from tools.validate_dom import main, validate_dom + + +def test_validate_dom_success(tmp_path: Path) -> None: + json_path = tmp_path / "valid.json" + valid_data = { + "document_id": "test-doc-123", + "pages": [], + "quality": {"status": "success", "parser": "mineru", "warnings": []}, + } + json_path.write_text(json.dumps(valid_data), encoding="utf-8") + + validate_dom(json_path) + + +def test_validate_dom_file_not_found(tmp_path: Path) -> None: + non_existent = tmp_path / "missing.json" + with pytest.raises(FileNotFoundError, match="File not found or is not a file"): + validate_dom(non_existent) + + +def test_validate_dom_invalid_extension(tmp_path: Path) -> None: + txt_path = tmp_path / "test.txt" + txt_path.write_text("not json", encoding="utf-8") + with pytest.raises(ValueError, match="File must be a .json file"): + validate_dom(txt_path) + + +def test_validate_dom_invalid_json(tmp_path: Path) -> None: + json_path = tmp_path / "invalid_json.json" + json_path.write_text("invalid{json", encoding="utf-8") + with pytest.raises(ValueError, match="Invalid JSON file"): + validate_dom(json_path) + + +def test_validate_dom_validation_error(tmp_path: Path) -> None: + json_path = tmp_path / "invalid_schema.json" + invalid_data = { + # missing document_id + "pages": [] + } + json_path.write_text(json.dumps(invalid_data), encoding="utf-8") + + with pytest.raises(ValidationError): + validate_dom(json_path) + + +def test_main_success(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + json_path = tmp_path / "valid.json" + valid_data = { + "document_id": "test-doc-123", + "pages": [], + "quality": {"status": "success", "parser": "mineru", "warnings": []}, + } + json_path.write_text(json.dumps(valid_data), encoding="utf-8") + + main([str(json_path)]) + + captured = capsys.readouterr() + assert "Validation successful: JSON matches ParseResponse schema." in captured.out + + +def test_main_validation_error( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + json_path = tmp_path / "invalid_schema.json" + invalid_data = {"pages": []} + json_path.write_text(json.dumps(invalid_data), encoding="utf-8") + + with pytest.raises(SystemExit) as exc_info: + main([str(json_path)]) + + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "Validation Error" in captured.err + + +def test_main_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + non_existent = tmp_path / "missing.json" + + with pytest.raises(SystemExit) as exc_info: + main([str(non_existent)]) + + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "Error:" in captured.err + assert "File not found" in captured.err + + +def test_sys_path_injection(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # Clear sys.path to force the injection block to run + monkeypatch.setattr(sys, "path", []) + + json_path = tmp_path / "valid.json" + valid_data = { + "document_id": "test-doc-123", + "pages": [], + "quality": {"status": "success", "parser": "mineru", "warnings": []}, + } + json_path.write_text(json.dumps(valid_data), encoding="utf-8") + + # Mock sys.argv + monkeypatch.setattr(sys, "argv", ["validate_dom.py", str(json_path)]) + + # Run the module to cover the import condition and __main__ block + import warnings + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=RuntimeWarning) + runpy.run_module("tools.validate_dom", run_name="__main__") diff --git a/tools/search_dom.py b/tools/search_dom.py new file mode 100644 index 0000000..069a26f --- /dev/null +++ b/tools/search_dom.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + + +def search_dom(json_path: Path, query: str) -> list[dict[str, str | int]]: + """Search for a text query in DOM JSON (headlines and body blocks).""" + if not json_path.is_file(): + raise FileNotFoundError(f"File not found or is not a file: {json_path}") + if json_path.suffix.lower() != ".json": + raise ValueError("File must be a .json file.") + + try: + data = json.loads(json_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON file: {exc}") from exc + + pages = data.get("pages", []) + results = [] + + # Pre-compile regex for performance + pattern = re.compile(re.escape(query), re.IGNORECASE) + + for page in pages: + page_num = page.get("page_number", -1) + articles = page.get("articles", []) + + for article in articles: + article_id = article.get("article_id", "unknown") + headline = article.get("headline", "") + + # Search in headline + if pattern.search(headline): + results.append( + { + "page": page_num, + "article_id": article_id, + "type": "headline", + "text": headline, + } + ) + + # Search in body blocks + body_blocks = article.get("body_blocks", []) + for i, block in enumerate(body_blocks): + if pattern.search(block): + results.append( + { + "page": page_num, + "article_id": article_id, + "type": "body_block", + "index": i, + "text": block, + } + ) + + return results + + +def main(argv: list[str] | None = None) -> None: + """Run search_dom main entry point.""" + parser = argparse.ArgumentParser( + description="Search for text in NewsDOM JSON output." + ) + parser.add_argument("input", type=Path, help="Path to the JSON DOM file.") + parser.add_argument("query", type=str, help="Text to search for.") + + args = parser.parse_args(argv) + + try: + results = search_dom(args.input, args.query) + if not results: + print(f"No results found for query: '{args.query}'") + return + + print(f"Found {len(results)} results for query: '{args.query}'") + for res in results: + if res["type"] == "headline": + print( + f"- Page {res['page']}, Article {res['article_id']} [Headline]: {res['text']}" + ) + elif res["type"] == "body_block": + print( + f"- Page {res['page']}, Article {res['article_id']} [Body Block {res['index']}]: {res['text']}" + ) + + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": # pragma: no cover + main() diff --git a/tools/validate_dom.py b/tools/validate_dom.py new file mode 100644 index 0000000..216b8dc --- /dev/null +++ b/tools/validate_dom.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_SRC_ROOT = _REPO_ROOT / "src" +if str(_SRC_ROOT) not in sys.path: + sys.path.insert(0, str(_SRC_ROOT)) + +from pydantic import ValidationError # noqa: E402 +from newsdom_api.schemas import ParseResponse # noqa: E402 + + +def validate_dom(json_path: Path) -> None: + """Validate DOM JSON against ParseResponse schema.""" + if not json_path.is_file(): + raise FileNotFoundError(f"File not found or is not a file: {json_path}") + if json_path.suffix.lower() != ".json": + raise ValueError("File must be a .json file.") + + try: + data = json.loads(json_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON file: {exc}") from exc + + # This will raise ValidationError if the data doesn't match the schema + ParseResponse.model_validate(data) + + +def main(argv: list[str] | None = None) -> None: + """Run validate_dom main entry point.""" + parser = argparse.ArgumentParser( + description="Validate NewsDOM JSON output against schema." + ) + parser.add_argument("input", type=Path, help="Path to the JSON DOM file.") + + args = parser.parse_args(argv) + + try: + validate_dom(args.input) + print("Validation successful: JSON matches ParseResponse schema.") + except ValidationError as e: + print(f"Validation Error: {e}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": # pragma: no cover + main()