diff --git a/app/render.cpp b/app/render.cpp index f2a91a66..7db4e5cb 100644 --- a/app/render.cpp +++ b/app/render.cpp @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #include "parse.h" #include "render.h" @@ -476,7 +477,7 @@ int main(int argc, char* argv[]) // --- Initialize fonts (shared by both -i and -d modes) --- { nlohmann::json data; - std::string resource_dir = resource_utils::get_resources_dir(false).string(); + std::string resource_dir = utils::filesystem::to_utf8(resource_utils::get_resources_dir(false)); data[pdflib::pdf_resource::RESOURCE_DIR_KEY] = resource_dir; std::unordered_map font_timings; pdflib::pdf_resource::initialise(data, font_timings); diff --git a/app/run_scaling.cpp b/app/run_scaling.cpp index b301fa0b..868613b1 100644 --- a/app/run_scaling.cpp +++ b/app/run_scaling.cpp @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #include "parse.h" #include "render.h" @@ -1183,7 +1184,7 @@ namespace void initialise_fonts() { - std::string resource_dir = resource_utils::get_resources_dir(false).string(); + std::string resource_dir = utils::filesystem::to_utf8(resource_utils::get_resources_dir(false)); nlohmann::json data = nlohmann::json::object({}); data[pdflib::pdf_resource::RESOURCE_DIR_KEY] = resource_dir; std::unordered_map font_timings; diff --git a/src/parse.h b/src/parse.h index 2d00f763..c8ddb734 100644 --- a/src/parse.h +++ b/src/parse.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT // std libraries #include @@ -39,6 +40,7 @@ #include // code to locate pdf-resources (eg fonts) +#include #include // third-party includes diff --git a/src/parse/pdf_resources/page_font/base_font.h b/src/parse/pdf_resources/page_font/base_font.h index a3c29849..330a77a0 100644 --- a/src/parse/pdf_resources/page_font/base_font.h +++ b/src/parse/pdf_resources/page_font/base_font.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_PAGE_FONT_BASE_FONT_H #define PDF_PAGE_FONT_BASE_FONT_H @@ -357,7 +358,7 @@ namespace pdflib LOG_S(WARNING) << "initialising base-font: " << filename; - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); std::string line; while (std::getline(file, line)) diff --git a/src/parse/pdf_resources/page_font/base_fonts.h b/src/parse/pdf_resources/page_font/base_fonts.h index face0113..3a2efd51 100644 --- a/src/parse/pdf_resources/page_font/base_fonts.h +++ b/src/parse/pdf_resources/page_font/base_fonts.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_PAGE_FONT_BASE_FONTS_H #define PDF_PAGE_FONT_BASE_FONTS_H @@ -275,7 +276,7 @@ namespace pdflib { std::string fontname = "unknown"; - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); std::string line; while (std::getline(file, line)) diff --git a/src/parse/pdf_resources/page_font/encoding.h b/src/parse/pdf_resources/page_font/encoding.h index 03f51ab2..de7f3c3c 100644 --- a/src/parse/pdf_resources/page_font/encoding.h +++ b/src/parse/pdf_resources/page_font/encoding.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_PAGE_FONT_ENCODING_H #define PDF_PAGE_FONT_ENCODING_H @@ -63,7 +64,7 @@ namespace pdflib name = name_; - std::ifstream file(file_name.c_str()); + std::ifstream file(utils::filesystem::from_utf8(file_name)); std::string line; while (std::getline(file, line)) diff --git a/src/parse/pdf_resources/page_font/font_cid.h b/src/parse/pdf_resources/page_font/font_cid.h index 98a14b71..fdac2ef6 100644 --- a/src/parse/pdf_resources/page_font/font_cid.h +++ b/src/parse/pdf_resources/page_font/font_cid.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_PAGE_FONT_CID_H #define PDF_PAGE_FONT_CID_H @@ -460,7 +461,7 @@ namespace pdflib } visited.insert(filename); - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); if(file.fail()) { @@ -505,7 +506,7 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__ << "\t filename: " << filename; - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); if(file.fail()) { diff --git a/src/parse/pdf_resources/page_font/glyphs.h b/src/parse/pdf_resources/page_font/glyphs.h index efd09487..58872464 100644 --- a/src/parse/pdf_resources/page_font/glyphs.h +++ b/src/parse/pdf_resources/page_font/glyphs.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_PAGE_FONT_GLYPHS_H #define PDF_PAGE_FONT_GLYPHS_H @@ -163,7 +164,7 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__ << ": " << filename; - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); if(file.fail()) { @@ -258,7 +259,7 @@ namespace pdflib { LOG_S(WARNING) << __FUNCTION__ << ": " << filename; - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); if(file.fail()) { diff --git a/src/parse/utils/files.h b/src/parse/utils/files.h index d17b3f2d..1c70ec7c 100644 --- a/src/parse/utils/files.h +++ b/src/parse/utils/files.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_UTILS_FILES_H #define PDF_UTILS_FILES_H @@ -25,7 +26,7 @@ namespace utils } */ bool is_dir(const std::string& file_name) { - return std::filesystem::is_directory(file_name); + return std::filesystem::is_directory(from_utf8(file_name)); } /* @@ -87,17 +88,17 @@ namespace utils try { - for (const auto& entry : std::filesystem::directory_iterator(directory)) + for (const auto& entry : std::filesystem::directory_iterator(from_utf8(directory))) { if (std::filesystem::is_regular_file(entry.path())) { if (basename) { - files.push_back(entry.path().filename().string()); // Get the file name only + files.push_back(to_utf8(entry.path().filename())); // Get the file name only } else { - files.push_back(entry.path().string()); // Full path + files.push_back(to_utf8(entry.path())); // Full path } } } diff --git a/src/parse/utils/utf8_path.h b/src/parse/utils/utf8_path.h new file mode 100644 index 00000000..096b32d4 --- /dev/null +++ b/src/parse/utils/utf8_path.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MIT + +#ifndef PDF_UTILS_UTF8_PATH_H +#define PDF_UTILS_UTF8_PATH_H + +#include +#include + +namespace utils +{ + namespace filesystem + { + // Python and the parser's string paths use UTF-8. On Windows a native + // filesystem path is wide, and narrow file opens use the ANSI codepage. + inline std::filesystem::path from_utf8(const std::string& value) + { + return std::filesystem::path(std::u8string(value.begin(), value.end())); + } + + inline std::string to_utf8(const std::filesystem::path& value) + { + const auto utf8 = value.u8string(); + return std::string(utf8.begin(), utf8.end()); + } + } +} + +#endif diff --git a/src/pybind/docling_parser.h b/src/pybind/docling_parser.h index 04bcb0f1..5423dffa 100644 --- a/src/pybind/docling_parser.h +++ b/src/pybind/docling_parser.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PYBIND_PDF_PARSER_H #define PYBIND_PDF_PARSER_H @@ -14,11 +15,11 @@ #include #endif +#include + #include #include -#include - namespace docling { class docling_parser: public docling_resources @@ -114,7 +115,7 @@ namespace docling docling_parser::docling_parser(std::string level, int max_concurrent_results): docling_resources(), - pdf_resources_dir(resource_utils::get_resources_dir(true).string()), + pdf_resources_dir(utils::filesystem::to_utf8(resource_utils::get_resources_dir(true))), doc_decoders({}), page_decoders({}), max_concurrent_results(max_concurrent_results) diff --git a/src/pybind/docling_resources.h b/src/pybind/docling_resources.h index 36733fa8..f65d2bbc 100644 --- a/src/pybind/docling_resources.h +++ b/src/pybind/docling_resources.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PYBIND_DOCLING_RESOURCES_H #define PYBIND_DOCLING_RESOURCES_H @@ -30,7 +31,7 @@ namespace docling std::string docling_resources::get_resources_path() { - return resource_utils::get_resources_dir(true).string(); + return utils::filesystem::to_utf8(resource_utils::get_resources_dir(true)); } bool docling_resources::set_resources_path() @@ -44,7 +45,7 @@ namespace docling // Extract the string value of the filename const char* filename = PyUnicode_AsUTF8(filenameObj); - std::filesystem::path __init__path(filename); + std::filesystem::path __init__path = utils::filesystem::from_utf8(filename); std::filesystem::path package_path = __init__path.parent_path(); std::filesystem::path resources_path = package_path / resource_utils::resources_relative_path; diff --git a/src/pybind/docling_threaded_base.h b/src/pybind/docling_threaded_base.h index 23ca5ed1..acc67e9e 100644 --- a/src/pybind/docling_threaded_base.h +++ b/src/pybind/docling_threaded_base.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PYBIND_THREADED_PDF_BASE_H #define PYBIND_THREADED_PDF_BASE_H @@ -23,12 +24,12 @@ #include #endif +#include + #include #include #include -#include - namespace docling { // --------------------------------------------------------------------------- @@ -174,7 +175,7 @@ namespace docling // Force thread-safe mode since we're doing parallel decoding this->config.do_thread_safe = true; - std::string pdf_resources_dir = resource_utils::get_resources_dir(true).string(); + std::string pdf_resources_dir = utils::filesystem::to_utf8(resource_utils::get_resources_dir(true)); LOG_S(WARNING) << "pdf_resources_dir: " << pdf_resources_dir; auto RESOURCE_DIR_KEY = pdflib::pdf_resource::RESOURCE_DIR_KEY; diff --git a/src/resources.h b/src/resources.h index 427d3ce1..b8f4107b 100644 --- a/src/resources.h +++ b/src/resources.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef RESOURCE_UTILS_H_ #define RESOURCE_UTILS_H_ @@ -21,8 +22,9 @@ class resource_utils private: - static inline std::filesystem::path ROOT_DIR = ROOT_PATH; // ROOT_PATH is the default provided during compilation - static inline std::filesystem::path PACKAGE_DIR = ROOT_PATH / package_name; + // ROOT_PATH is a UTF-8 source literal supplied by the build. + static inline std::filesystem::path ROOT_DIR = utils::filesystem::from_utf8(ROOT_PATH); + static inline std::filesystem::path PACKAGE_DIR = ROOT_DIR / package_name; static inline std::filesystem::path RESOURCES_DIR = PACKAGE_DIR / resources_relative_path; }; diff --git a/tests/cpp/test_memory_on_lines.cpp b/tests/cpp/test_memory_on_lines.cpp index abf2b67a..1d281986 100644 --- a/tests/cpp/test_memory_on_lines.cpp +++ b/tests/cpp/test_memory_on_lines.cpp @@ -507,7 +507,7 @@ void measure(std::string mode, std::string filename, int pause_seconds, const st } json font_config = {{pdflib::pdf_resource::RESOURCE_DIR_KEY, - resource_utils::get_resources_dir(true).string()}}; + utils::filesystem::to_utf8(resource_utils::get_resources_dir(true))}}; std::unordered_map font_timings; pdflib::pdf_resource::initialise(font_config, font_timings); pdflib::pdf_decoder document; diff --git a/tests/cpp/test_structure_concurrency.cpp b/tests/cpp/test_structure_concurrency.cpp index 0012e3f3..4015cee8 100644 --- a/tests/cpp/test_structure_concurrency.cpp +++ b/tests/cpp/test_structure_concurrency.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MIT // Run against a multipage tagged PDF, optionally compiled with ThreadSanitizer. // This exercises parser-owned synchronization; linked third-party libraries // must also be instrumented for a complete audit of their internals. @@ -12,7 +13,7 @@ int main(int argc, char** argv) loguru::g_stderr_verbosity = loguru::Verbosity_FATAL; nlohmann::json font_config = {{ pdflib::pdf_resource::RESOURCE_DIR_KEY, - resource_utils::get_resources_dir(true).string()}}; + utils::filesystem::to_utf8(resource_utils::get_resources_dir(true))}}; std::unordered_map font_timings; pdflib::pdf_resource::initialise(font_config, font_timings); pdflib::pdf_decoder document; diff --git a/tests/test_unit_unicode_paths.py b/tests/test_unit_unicode_paths.py index af4ab310..e8ede9ce 100644 --- a/tests/test_unit_unicode_paths.py +++ b/tests/test_unit_unicode_paths.py @@ -1,7 +1,8 @@ #!/usr/bin/env python +# SPDX-License-Identifier: MIT """Paths that contain non-ASCII characters. -Two routes carry such paths into the native code, and on Windows both go wrong +Three routes carry such paths into the native code, and on Windows they go wrong in the ANSI codepage while POSIX passes the bytes through unchanged -- so these tests guard intent everywhere but only bite on Windows runners. @@ -13,9 +14,13 @@ The renderer's font resolver takes its scan directories from the environment (LOCALAPPDATA, DOCLING_PARSE_*_FALLBACK_FONT); a narrow getenv() there broke every threaded render under a non-ASCII user profile (docling-parse#354). + +The installed package can also live under a non-ASCII directory. Its glyph, +encoding, font metric and CMap resources must open there (docling-parse#115). """ import os +import shutil import subprocess import sys import textwrap @@ -35,6 +40,51 @@ # '?' under cp1252, the codepage of GitHub's Windows runners, and would pass. NON_ASCII_DIR_NAME = "docling_é_한글_Привет" +_PARSE_WITH_RELOCATED_PACKAGE = textwrap.dedent( + """ + import sys + from io import BytesIO + from pathlib import Path + + import docling_parse + from docling_parse.pdf_parser import ( + DoclingPdfParser, + DoclingThreadedPdfParser, + RenderConfig, + ThreadedPdfParserConfig, + ) + + assert Path(docling_parse.__file__).resolve().parent.parent == Path(sys.argv[1]).resolve() + source = BytesIO(sys.stdin.buffer.read()) + if sys.argv[2] == "serial": + parser = DoclingPdfParser(loglevel="fatal") + doc = parser.load(source) + assert doc.number_of_pages() == 1 + page = doc.get_page(1) + else: + parser = DoclingThreadedPdfParser( + parser_config=ThreadedPdfParserConfig( + loglevel="fatal", threads=1, + render_config=RenderConfig() if sys.argv[2] == "rendered" else None, + ) + ) + parser.load(source) + results = list(parser.iterate_results()) + assert len(results) == 1 + assert results[0].success, results[0].error_message + page = results[0].get_page() + if sys.argv[2] == "rendered": + assert results[0].get_image().width > 0 + text = "".join(cell.text for cell in page.textline_cells) + assert "Title Case Workers" in text, repr(text) + assert "\\u4e2d\\u6587" in text, repr(text) + if sys.argv[2] == "serial": + doc.unload() + else: + parser.unload_all() + """ +) + _RENDER_ONE_PAGE = textwrap.dedent( """ import sys @@ -94,6 +144,64 @@ def test_load_from_bytesio_is_path_independent(): assert doc.number_of_pages() == 1 +def _package_resource_pdf() -> bytes: + """Latin base-font metrics and a predefined CJK CMap both read resources.""" + content = CONTENT + "\nBT /F2 12 Tf 72 700 Td Tj ET" + return build_pdf( + [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + "/Resources << /Font << /F1 4 0 R /F2 6 0 R >> >> /Contents 5 0 R >>", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + f"<< /Length {len(content)} >>\nstream\n{content}\nendstream", + "<< /Type /Font /Subtype /Type0 /BaseFont /SimSun " + "/Encoding /GBK-EUC-H /DescendantFonts [7 0 R] >>", + "<< /Type /Font /Subtype /CIDFontType2 /BaseFont /SimSun " + "/CIDSystemInfo << /Registry (Adobe) /Ordering (GB1) /Supplement 2 >> " + "/FontDescriptor 8 0 R /DW 1000 >>", + "<< /Type /FontDescriptor /FontName /SimSun /Flags 4 " + "/FontBBox [0 0 1000 1000] /ItalicAngle 0 /Ascent 859 /Descent -141 " + "/CapHeight 674 /StemV 91 >>", + ] + ) + + +@pytest.mark.parametrize("parser_kind", ["serial", "threaded", "rendered"]) +@pytest.mark.parametrize("dir_name", ["ascii_install", "安装目录", NON_ASCII_DIR_NAME]) +def test_parse_from_non_ascii_package_directory( + tmp_path: Path, dir_name: str, parser_kind: str +) -> None: + """An ordinary installation path must not prevent reading bundled resources.""" + package_root = tmp_path / dir_name + package_dir = Path(docling_parse.__file__).resolve().parent + shutil.copytree( + package_dir, + package_root / "docling_parse", + ignore=shutil.ignore_patterns("__pycache__"), + ) + # A fresh process avoids reusing the original extension's global resources. + # Its cwd and first import directory both select the relocated installation. + env = {**os.environ, "PYTHONPATH": str(package_root)} + proc = subprocess.run( + [ + sys.executable, + "-X", + "utf8", + "-c", + _PARSE_WITH_RELOCATED_PACKAGE, + str(package_root), + parser_kind, + ], + input=_package_resource_pdf(), + env=env, + cwd=tmp_path, + capture_output=True, + timeout=300, + ) + assert proc.returncode == 0, proc.stderr.decode("utf-8", "replace") + + def _render_in_fresh_process( cwd: Path, **env_overrides: str ) -> subprocess.CompletedProcess: