From aa0a4d3f7f35f47e1d881fc65bfca88c9b7f5ee3 Mon Sep 17 00:00:00 2001 From: ainuo-AI <2470447645@qq.com> Date: Sun, 15 Nov 2026 19:58:49 +0800 Subject: [PATCH 1/2] fix: load bundled PDF resources from non-ASCII Windows paths Keep resource paths UTF-8 at string boundaries and use native filesystem paths when locating and opening glyph, encoding, AFM and CMap resources. Use the same conversions in the serial/threaded parsers and native tools. Add subprocess regressions for ASCII, Chinese and mixed-script package locations through the serial, threaded and rendered public APIs. Fixes #115. Signed-off-by: ainuo-AI <2470447645@qq.com> --- app/render.cpp | 3 +- app/run_scaling.cpp | 3 +- src/parse/pdf_resources/page_font/base_font.h | 3 +- .../pdf_resources/page_font/base_fonts.h | 3 +- src/parse/pdf_resources/page_font/encoding.h | 3 +- src/parse/pdf_resources/page_font/font_cid.h | 5 +- src/parse/pdf_resources/page_font/glyphs.h | 5 +- src/parse/utils/files.h | 11 +- src/parse/utils/utf8_path.h | 28 +++++ src/pybind/docling_parser.h | 3 +- src/pybind/docling_resources.h | 6 +- src/pybind/docling_threaded_base.h | 3 +- src/resources.h | 7 +- tests/cpp/test_memory_on_lines.cpp | 2 +- tests/cpp/test_structure_concurrency.cpp | 3 +- tests/test_unit_unicode_paths.py | 110 +++++++++++++++++- 16 files changed, 176 insertions(+), 22 deletions(-) create mode 100644 src/parse/utils/utf8_path.h diff --git a/app/render.cpp b/app/render.cpp index f2a91a668..7db4e5cb5 100644 --- a/app/render.cpp +++ b/app/render.cpp @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #include "parse.h" #include "render.h" @@ -476,7 +477,7 @@ int main(int argc, char* argv[]) // --- Initialize fonts (shared by both -i and -d modes) --- { nlohmann::json data; - std::string resource_dir = resource_utils::get_resources_dir(false).string(); + std::string resource_dir = utils::filesystem::to_utf8(resource_utils::get_resources_dir(false)); data[pdflib::pdf_resource::RESOURCE_DIR_KEY] = resource_dir; std::unordered_map font_timings; pdflib::pdf_resource::initialise(data, font_timings); diff --git a/app/run_scaling.cpp b/app/run_scaling.cpp index b301fa0b5..868613b11 100644 --- a/app/run_scaling.cpp +++ b/app/run_scaling.cpp @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #include "parse.h" #include "render.h" @@ -1183,7 +1184,7 @@ namespace void initialise_fonts() { - std::string resource_dir = resource_utils::get_resources_dir(false).string(); + std::string resource_dir = utils::filesystem::to_utf8(resource_utils::get_resources_dir(false)); nlohmann::json data = nlohmann::json::object({}); data[pdflib::pdf_resource::RESOURCE_DIR_KEY] = resource_dir; std::unordered_map font_timings; diff --git a/src/parse/pdf_resources/page_font/base_font.h b/src/parse/pdf_resources/page_font/base_font.h index a3c29849b..330a77a0d 100644 --- a/src/parse/pdf_resources/page_font/base_font.h +++ b/src/parse/pdf_resources/page_font/base_font.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_PAGE_FONT_BASE_FONT_H #define PDF_PAGE_FONT_BASE_FONT_H @@ -357,7 +358,7 @@ namespace pdflib LOG_S(WARNING) << "initialising base-font: " << filename; - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); std::string line; while (std::getline(file, line)) diff --git a/src/parse/pdf_resources/page_font/base_fonts.h b/src/parse/pdf_resources/page_font/base_fonts.h index face0113e..3a2efd514 100644 --- a/src/parse/pdf_resources/page_font/base_fonts.h +++ b/src/parse/pdf_resources/page_font/base_fonts.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_PAGE_FONT_BASE_FONTS_H #define PDF_PAGE_FONT_BASE_FONTS_H @@ -275,7 +276,7 @@ namespace pdflib { std::string fontname = "unknown"; - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); std::string line; while (std::getline(file, line)) diff --git a/src/parse/pdf_resources/page_font/encoding.h b/src/parse/pdf_resources/page_font/encoding.h index 03f51ab26..de7f3c3ca 100644 --- a/src/parse/pdf_resources/page_font/encoding.h +++ b/src/parse/pdf_resources/page_font/encoding.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_PAGE_FONT_ENCODING_H #define PDF_PAGE_FONT_ENCODING_H @@ -63,7 +64,7 @@ namespace pdflib name = name_; - std::ifstream file(file_name.c_str()); + std::ifstream file(utils::filesystem::from_utf8(file_name)); std::string line; while (std::getline(file, line)) diff --git a/src/parse/pdf_resources/page_font/font_cid.h b/src/parse/pdf_resources/page_font/font_cid.h index 98a14b712..fdac2ef61 100644 --- a/src/parse/pdf_resources/page_font/font_cid.h +++ b/src/parse/pdf_resources/page_font/font_cid.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_PAGE_FONT_CID_H #define PDF_PAGE_FONT_CID_H @@ -460,7 +461,7 @@ namespace pdflib } visited.insert(filename); - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); if(file.fail()) { @@ -505,7 +506,7 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__ << "\t filename: " << filename; - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); if(file.fail()) { diff --git a/src/parse/pdf_resources/page_font/glyphs.h b/src/parse/pdf_resources/page_font/glyphs.h index efd094878..58872464f 100644 --- a/src/parse/pdf_resources/page_font/glyphs.h +++ b/src/parse/pdf_resources/page_font/glyphs.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_PAGE_FONT_GLYPHS_H #define PDF_PAGE_FONT_GLYPHS_H @@ -163,7 +164,7 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__ << ": " << filename; - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); if(file.fail()) { @@ -258,7 +259,7 @@ namespace pdflib { LOG_S(WARNING) << __FUNCTION__ << ": " << filename; - std::ifstream file(filename.c_str()); + std::ifstream file(utils::filesystem::from_utf8(filename)); if(file.fail()) { diff --git a/src/parse/utils/files.h b/src/parse/utils/files.h index d17b3f2d1..76d91b750 100644 --- a/src/parse/utils/files.h +++ b/src/parse/utils/files.h @@ -1,8 +1,11 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PDF_UTILS_FILES_H #define PDF_UTILS_FILES_H +#include "utf8_path.h" + //#include //#include //#include @@ -25,7 +28,7 @@ namespace utils } */ bool is_dir(const std::string& file_name) { - return std::filesystem::is_directory(file_name); + return std::filesystem::is_directory(from_utf8(file_name)); } /* @@ -87,17 +90,17 @@ namespace utils try { - for (const auto& entry : std::filesystem::directory_iterator(directory)) + for (const auto& entry : std::filesystem::directory_iterator(from_utf8(directory))) { if (std::filesystem::is_regular_file(entry.path())) { if (basename) { - files.push_back(entry.path().filename().string()); // Get the file name only + files.push_back(to_utf8(entry.path().filename())); // Get the file name only } else { - files.push_back(entry.path().string()); // Full path + files.push_back(to_utf8(entry.path())); // Full path } } } diff --git a/src/parse/utils/utf8_path.h b/src/parse/utils/utf8_path.h new file mode 100644 index 000000000..096b32d4a --- /dev/null +++ b/src/parse/utils/utf8_path.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MIT + +#ifndef PDF_UTILS_UTF8_PATH_H +#define PDF_UTILS_UTF8_PATH_H + +#include +#include + +namespace utils +{ + namespace filesystem + { + // Python and the parser's string paths use UTF-8. On Windows a native + // filesystem path is wide, and narrow file opens use the ANSI codepage. + inline std::filesystem::path from_utf8(const std::string& value) + { + return std::filesystem::path(std::u8string(value.begin(), value.end())); + } + + inline std::string to_utf8(const std::filesystem::path& value) + { + const auto utf8 = value.u8string(); + return std::string(utf8.begin(), utf8.end()); + } + } +} + +#endif diff --git a/src/pybind/docling_parser.h b/src/pybind/docling_parser.h index 04bcb0f13..936fb0855 100644 --- a/src/pybind/docling_parser.h +++ b/src/pybind/docling_parser.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PYBIND_PDF_PARSER_H #define PYBIND_PDF_PARSER_H @@ -114,7 +115,7 @@ namespace docling docling_parser::docling_parser(std::string level, int max_concurrent_results): docling_resources(), - pdf_resources_dir(resource_utils::get_resources_dir(true).string()), + pdf_resources_dir(utils::filesystem::to_utf8(resource_utils::get_resources_dir(true))), doc_decoders({}), page_decoders({}), max_concurrent_results(max_concurrent_results) diff --git a/src/pybind/docling_resources.h b/src/pybind/docling_resources.h index 36733fa85..bb7e8673f 100644 --- a/src/pybind/docling_resources.h +++ b/src/pybind/docling_resources.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PYBIND_DOCLING_RESOURCES_H #define PYBIND_DOCLING_RESOURCES_H @@ -6,6 +7,7 @@ #include #include +#include namespace docling { @@ -30,7 +32,7 @@ namespace docling std::string docling_resources::get_resources_path() { - return resource_utils::get_resources_dir(true).string(); + return utils::filesystem::to_utf8(resource_utils::get_resources_dir(true)); } bool docling_resources::set_resources_path() @@ -44,7 +46,7 @@ namespace docling // Extract the string value of the filename const char* filename = PyUnicode_AsUTF8(filenameObj); - std::filesystem::path __init__path(filename); + std::filesystem::path __init__path = utils::filesystem::from_utf8(filename); std::filesystem::path package_path = __init__path.parent_path(); std::filesystem::path resources_path = package_path / resource_utils::resources_relative_path; diff --git a/src/pybind/docling_threaded_base.h b/src/pybind/docling_threaded_base.h index 23ca5ed19..62b630694 100644 --- a/src/pybind/docling_threaded_base.h +++ b/src/pybind/docling_threaded_base.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef PYBIND_THREADED_PDF_BASE_H #define PYBIND_THREADED_PDF_BASE_H @@ -174,7 +175,7 @@ namespace docling // Force thread-safe mode since we're doing parallel decoding this->config.do_thread_safe = true; - std::string pdf_resources_dir = resource_utils::get_resources_dir(true).string(); + std::string pdf_resources_dir = utils::filesystem::to_utf8(resource_utils::get_resources_dir(true)); LOG_S(WARNING) << "pdf_resources_dir: " << pdf_resources_dir; auto RESOURCE_DIR_KEY = pdflib::pdf_resource::RESOURCE_DIR_KEY; diff --git a/src/resources.h b/src/resources.h index 427d3ce1c..f78c3b256 100644 --- a/src/resources.h +++ b/src/resources.h @@ -1,10 +1,12 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT #ifndef RESOURCE_UTILS_H_ #define RESOURCE_UTILS_H_ #define LOGURU_WITH_STREAMS 1 #include +#include class resource_utils { @@ -21,8 +23,9 @@ class resource_utils private: - static inline std::filesystem::path ROOT_DIR = ROOT_PATH; // ROOT_PATH is the default provided during compilation - static inline std::filesystem::path PACKAGE_DIR = ROOT_PATH / package_name; + // ROOT_PATH is a UTF-8 source literal supplied by the build. + static inline std::filesystem::path ROOT_DIR = utils::filesystem::from_utf8(ROOT_PATH); + static inline std::filesystem::path PACKAGE_DIR = ROOT_DIR / package_name; static inline std::filesystem::path RESOURCES_DIR = PACKAGE_DIR / resources_relative_path; }; diff --git a/tests/cpp/test_memory_on_lines.cpp b/tests/cpp/test_memory_on_lines.cpp index abf2b67a2..1d2819865 100644 --- a/tests/cpp/test_memory_on_lines.cpp +++ b/tests/cpp/test_memory_on_lines.cpp @@ -507,7 +507,7 @@ void measure(std::string mode, std::string filename, int pause_seconds, const st } json font_config = {{pdflib::pdf_resource::RESOURCE_DIR_KEY, - resource_utils::get_resources_dir(true).string()}}; + utils::filesystem::to_utf8(resource_utils::get_resources_dir(true))}}; std::unordered_map font_timings; pdflib::pdf_resource::initialise(font_config, font_timings); pdflib::pdf_decoder document; diff --git a/tests/cpp/test_structure_concurrency.cpp b/tests/cpp/test_structure_concurrency.cpp index 0012e3f35..4015cee85 100644 --- a/tests/cpp/test_structure_concurrency.cpp +++ b/tests/cpp/test_structure_concurrency.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MIT // Run against a multipage tagged PDF, optionally compiled with ThreadSanitizer. // This exercises parser-owned synchronization; linked third-party libraries // must also be instrumented for a complete audit of their internals. @@ -12,7 +13,7 @@ int main(int argc, char** argv) loguru::g_stderr_verbosity = loguru::Verbosity_FATAL; nlohmann::json font_config = {{ pdflib::pdf_resource::RESOURCE_DIR_KEY, - resource_utils::get_resources_dir(true).string()}}; + utils::filesystem::to_utf8(resource_utils::get_resources_dir(true))}}; std::unordered_map font_timings; pdflib::pdf_resource::initialise(font_config, font_timings); pdflib::pdf_decoder document; diff --git a/tests/test_unit_unicode_paths.py b/tests/test_unit_unicode_paths.py index af4ab310c..e8ede9ce9 100644 --- a/tests/test_unit_unicode_paths.py +++ b/tests/test_unit_unicode_paths.py @@ -1,7 +1,8 @@ #!/usr/bin/env python +# SPDX-License-Identifier: MIT """Paths that contain non-ASCII characters. -Two routes carry such paths into the native code, and on Windows both go wrong +Three routes carry such paths into the native code, and on Windows they go wrong in the ANSI codepage while POSIX passes the bytes through unchanged -- so these tests guard intent everywhere but only bite on Windows runners. @@ -13,9 +14,13 @@ The renderer's font resolver takes its scan directories from the environment (LOCALAPPDATA, DOCLING_PARSE_*_FALLBACK_FONT); a narrow getenv() there broke every threaded render under a non-ASCII user profile (docling-parse#354). + +The installed package can also live under a non-ASCII directory. Its glyph, +encoding, font metric and CMap resources must open there (docling-parse#115). """ import os +import shutil import subprocess import sys import textwrap @@ -35,6 +40,51 @@ # '?' under cp1252, the codepage of GitHub's Windows runners, and would pass. NON_ASCII_DIR_NAME = "docling_é_한글_Привет" +_PARSE_WITH_RELOCATED_PACKAGE = textwrap.dedent( + """ + import sys + from io import BytesIO + from pathlib import Path + + import docling_parse + from docling_parse.pdf_parser import ( + DoclingPdfParser, + DoclingThreadedPdfParser, + RenderConfig, + ThreadedPdfParserConfig, + ) + + assert Path(docling_parse.__file__).resolve().parent.parent == Path(sys.argv[1]).resolve() + source = BytesIO(sys.stdin.buffer.read()) + if sys.argv[2] == "serial": + parser = DoclingPdfParser(loglevel="fatal") + doc = parser.load(source) + assert doc.number_of_pages() == 1 + page = doc.get_page(1) + else: + parser = DoclingThreadedPdfParser( + parser_config=ThreadedPdfParserConfig( + loglevel="fatal", threads=1, + render_config=RenderConfig() if sys.argv[2] == "rendered" else None, + ) + ) + parser.load(source) + results = list(parser.iterate_results()) + assert len(results) == 1 + assert results[0].success, results[0].error_message + page = results[0].get_page() + if sys.argv[2] == "rendered": + assert results[0].get_image().width > 0 + text = "".join(cell.text for cell in page.textline_cells) + assert "Title Case Workers" in text, repr(text) + assert "\\u4e2d\\u6587" in text, repr(text) + if sys.argv[2] == "serial": + doc.unload() + else: + parser.unload_all() + """ +) + _RENDER_ONE_PAGE = textwrap.dedent( """ import sys @@ -94,6 +144,64 @@ def test_load_from_bytesio_is_path_independent(): assert doc.number_of_pages() == 1 +def _package_resource_pdf() -> bytes: + """Latin base-font metrics and a predefined CJK CMap both read resources.""" + content = CONTENT + "\nBT /F2 12 Tf 72 700 Td Tj ET" + return build_pdf( + [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + "/Resources << /Font << /F1 4 0 R /F2 6 0 R >> >> /Contents 5 0 R >>", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + f"<< /Length {len(content)} >>\nstream\n{content}\nendstream", + "<< /Type /Font /Subtype /Type0 /BaseFont /SimSun " + "/Encoding /GBK-EUC-H /DescendantFonts [7 0 R] >>", + "<< /Type /Font /Subtype /CIDFontType2 /BaseFont /SimSun " + "/CIDSystemInfo << /Registry (Adobe) /Ordering (GB1) /Supplement 2 >> " + "/FontDescriptor 8 0 R /DW 1000 >>", + "<< /Type /FontDescriptor /FontName /SimSun /Flags 4 " + "/FontBBox [0 0 1000 1000] /ItalicAngle 0 /Ascent 859 /Descent -141 " + "/CapHeight 674 /StemV 91 >>", + ] + ) + + +@pytest.mark.parametrize("parser_kind", ["serial", "threaded", "rendered"]) +@pytest.mark.parametrize("dir_name", ["ascii_install", "安装目录", NON_ASCII_DIR_NAME]) +def test_parse_from_non_ascii_package_directory( + tmp_path: Path, dir_name: str, parser_kind: str +) -> None: + """An ordinary installation path must not prevent reading bundled resources.""" + package_root = tmp_path / dir_name + package_dir = Path(docling_parse.__file__).resolve().parent + shutil.copytree( + package_dir, + package_root / "docling_parse", + ignore=shutil.ignore_patterns("__pycache__"), + ) + # A fresh process avoids reusing the original extension's global resources. + # Its cwd and first import directory both select the relocated installation. + env = {**os.environ, "PYTHONPATH": str(package_root)} + proc = subprocess.run( + [ + sys.executable, + "-X", + "utf8", + "-c", + _PARSE_WITH_RELOCATED_PACKAGE, + str(package_root), + parser_kind, + ], + input=_package_resource_pdf(), + env=env, + cwd=tmp_path, + capture_output=True, + timeout=300, + ) + assert proc.returncode == 0, proc.stderr.decode("utf-8", "replace") + + def _render_in_fresh_process( cwd: Path, **env_overrides: str ) -> subprocess.CompletedProcess: From a94d62fdd6c63a1021a80079fb623c06cee141e8 Mon Sep 17 00:00:00 2001 From: ainuo-AI <2470447645@qq.com> Date: Mon, 16 Nov 2026 20:08:08 +0800 Subject: [PATCH 2/2] fix: centralize UTF-8 path includes in parse header Signed-off-by: ainuo-AI <2470447645@qq.com> --- src/parse.h | 2 ++ src/parse/utils/files.h | 2 -- src/pybind/docling_parser.h | 4 ++-- src/pybind/docling_resources.h | 1 - src/pybind/docling_threaded_base.h | 4 ++-- src/resources.h | 1 - 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/parse.h b/src/parse.h index 2d00f7635..c8ddb734e 100644 --- a/src/parse.h +++ b/src/parse.h @@ -1,4 +1,5 @@ //-*-C++-*- +// SPDX-License-Identifier: MIT // std libraries #include @@ -39,6 +40,7 @@ #include // code to locate pdf-resources (eg fonts) +#include #include // third-party includes diff --git a/src/parse/utils/files.h b/src/parse/utils/files.h index 76d91b750..1c70ec7cf 100644 --- a/src/parse/utils/files.h +++ b/src/parse/utils/files.h @@ -4,8 +4,6 @@ #ifndef PDF_UTILS_FILES_H #define PDF_UTILS_FILES_H -#include "utf8_path.h" - //#include //#include //#include diff --git a/src/pybind/docling_parser.h b/src/pybind/docling_parser.h index 936fb0855..5423dffa3 100644 --- a/src/pybind/docling_parser.h +++ b/src/pybind/docling_parser.h @@ -15,11 +15,11 @@ #include #endif +#include + #include #include -#include - namespace docling { class docling_parser: public docling_resources diff --git a/src/pybind/docling_resources.h b/src/pybind/docling_resources.h index bb7e8673f..f65d2bbc2 100644 --- a/src/pybind/docling_resources.h +++ b/src/pybind/docling_resources.h @@ -7,7 +7,6 @@ #include #include -#include namespace docling { diff --git a/src/pybind/docling_threaded_base.h b/src/pybind/docling_threaded_base.h index 62b630694..acc67e9ef 100644 --- a/src/pybind/docling_threaded_base.h +++ b/src/pybind/docling_threaded_base.h @@ -24,12 +24,12 @@ #include #endif +#include + #include #include #include -#include - namespace docling { // --------------------------------------------------------------------------- diff --git a/src/resources.h b/src/resources.h index f78c3b256..b8f4107b0 100644 --- a/src/resources.h +++ b/src/resources.h @@ -6,7 +6,6 @@ #define LOGURU_WITH_STREAMS 1 #include -#include class resource_utils {