Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion app/render.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#include "parse.h"
#include "render.h"
Expand Down Expand Up @@ -476,7 +477,7 @@ int main(int argc, char* argv[])
// --- Initialize fonts (shared by both -i and -d modes) ---
{
nlohmann::json data;
std::string resource_dir = resource_utils::get_resources_dir(false).string();
std::string resource_dir = utils::filesystem::to_utf8(resource_utils::get_resources_dir(false));
data[pdflib::pdf_resource<pdflib::PAGE_FONT>::RESOURCE_DIR_KEY] = resource_dir;
std::unordered_map<std::string, double> font_timings;
pdflib::pdf_resource<pdflib::PAGE_FONT>::initialise(data, font_timings);
Expand Down
3 changes: 2 additions & 1 deletion app/run_scaling.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#include "parse.h"
#include "render.h"
Expand Down Expand Up @@ -1183,7 +1184,7 @@ namespace

void initialise_fonts()
{
std::string resource_dir = resource_utils::get_resources_dir(false).string();
std::string resource_dir = utils::filesystem::to_utf8(resource_utils::get_resources_dir(false));
nlohmann::json data = nlohmann::json::object({});
data[pdflib::pdf_resource<pdflib::PAGE_FONT>::RESOURCE_DIR_KEY] = resource_dir;
std::unordered_map<std::string, double> font_timings;
Expand Down
2 changes: 2 additions & 0 deletions src/parse.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

// std libraries
#include <set>
Expand Down Expand Up @@ -39,6 +40,7 @@
#include <qpdf/Buffer.hh>

// code to locate pdf-resources (eg fonts)
#include <parse/utils/utf8_path.h>
#include <resources.h>

// third-party includes
Expand Down
3 changes: 2 additions & 1 deletion src/parse/pdf_resources/page_font/base_font.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#ifndef PDF_PAGE_FONT_BASE_FONT_H
#define PDF_PAGE_FONT_BASE_FONT_H
Expand Down Expand Up @@ -357,7 +358,7 @@ namespace pdflib

LOG_S(WARNING) << "initialising base-font: " << filename;

std::ifstream file(filename.c_str());
std::ifstream file(utils::filesystem::from_utf8(filename));

std::string line;
while (std::getline(file, line))
Expand Down
3 changes: 2 additions & 1 deletion src/parse/pdf_resources/page_font/base_fonts.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#ifndef PDF_PAGE_FONT_BASE_FONTS_H
#define PDF_PAGE_FONT_BASE_FONTS_H
Expand Down Expand Up @@ -275,7 +276,7 @@ namespace pdflib
{
std::string fontname = "unknown";

std::ifstream file(filename.c_str());
std::ifstream file(utils::filesystem::from_utf8(filename));

std::string line;
while (std::getline(file, line))
Expand Down
3 changes: 2 additions & 1 deletion src/parse/pdf_resources/page_font/encoding.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#ifndef PDF_PAGE_FONT_ENCODING_H
#define PDF_PAGE_FONT_ENCODING_H
Expand Down Expand Up @@ -63,7 +64,7 @@ namespace pdflib

name = name_;

std::ifstream file(file_name.c_str());
std::ifstream file(utils::filesystem::from_utf8(file_name));

std::string line;
while (std::getline(file, line))
Expand Down
5 changes: 3 additions & 2 deletions src/parse/pdf_resources/page_font/font_cid.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#ifndef PDF_PAGE_FONT_CID_H
#define PDF_PAGE_FONT_CID_H
Expand Down Expand Up @@ -460,7 +461,7 @@ namespace pdflib
}
visited.insert(filename);

std::ifstream file(filename.c_str());
std::ifstream file(utils::filesystem::from_utf8(filename));

if(file.fail())
{
Expand Down Expand Up @@ -505,7 +506,7 @@ namespace pdflib
{
LOG_S(INFO) << __FUNCTION__ << "\t filename: " << filename;

std::ifstream file(filename.c_str());
std::ifstream file(utils::filesystem::from_utf8(filename));

if(file.fail())
{
Expand Down
5 changes: 3 additions & 2 deletions src/parse/pdf_resources/page_font/glyphs.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#ifndef PDF_PAGE_FONT_GLYPHS_H
#define PDF_PAGE_FONT_GLYPHS_H
Expand Down Expand Up @@ -163,7 +164,7 @@ namespace pdflib
{
LOG_S(INFO) << __FUNCTION__ << ": " << filename;

std::ifstream file(filename.c_str());
std::ifstream file(utils::filesystem::from_utf8(filename));

if(file.fail())
{
Expand Down Expand Up @@ -258,7 +259,7 @@ namespace pdflib
{
LOG_S(WARNING) << __FUNCTION__ << ": " << filename;

std::ifstream file(filename.c_str());
std::ifstream file(utils::filesystem::from_utf8(filename));

if(file.fail())
{
Expand Down
9 changes: 5 additions & 4 deletions src/parse/utils/files.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#ifndef PDF_UTILS_FILES_H
#define PDF_UTILS_FILES_H
Expand All @@ -25,7 +26,7 @@ namespace utils
}
*/
bool is_dir(const std::string& file_name) {
return std::filesystem::is_directory(file_name);
return std::filesystem::is_directory(from_utf8(file_name));
}

/*
Expand Down Expand Up @@ -87,17 +88,17 @@ namespace utils

try
{
for (const auto& entry : std::filesystem::directory_iterator(directory))
for (const auto& entry : std::filesystem::directory_iterator(from_utf8(directory)))
{
if (std::filesystem::is_regular_file(entry.path()))
{
if (basename)
{
files.push_back(entry.path().filename().string()); // Get the file name only
files.push_back(to_utf8(entry.path().filename())); // Get the file name only
}
else
{
files.push_back(entry.path().string()); // Full path
files.push_back(to_utf8(entry.path())); // Full path
}
}
}
Expand Down
28 changes: 28 additions & 0 deletions src/parse/utils/utf8_path.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// SPDX-License-Identifier: MIT

#ifndef PDF_UTILS_UTF8_PATH_H
#define PDF_UTILS_UTF8_PATH_H

#include <filesystem>
#include <string>

namespace utils
{
namespace filesystem
{
// Python and the parser's string paths use UTF-8. On Windows a native
// filesystem path is wide, and narrow file opens use the ANSI codepage.
inline std::filesystem::path from_utf8(const std::string& value)
{
return std::filesystem::path(std::u8string(value.begin(), value.end()));
}

inline std::string to_utf8(const std::filesystem::path& value)
{
const auto utf8 = value.u8string();
return std::string(utf8.begin(), utf8.end());
}
}
}

#endif
7 changes: 4 additions & 3 deletions src/pybind/docling_parser.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#ifndef PYBIND_PDF_PARSER_H
#define PYBIND_PDF_PARSER_H
Expand All @@ -14,11 +15,11 @@
#include <codecvt>
#endif

#include <parse.h>

#include <pybind/native_memory.h>
#include <pybind/docling_resources.h>

#include <parse.h>

namespace docling
{
class docling_parser: public docling_resources
Expand Down Expand Up @@ -114,7 +115,7 @@ namespace docling

docling_parser::docling_parser(std::string level, int max_concurrent_results):
docling_resources(),
pdf_resources_dir(resource_utils::get_resources_dir(true).string()),
pdf_resources_dir(utils::filesystem::to_utf8(resource_utils::get_resources_dir(true))),
doc_decoders({}),
page_decoders({}),
max_concurrent_results(max_concurrent_results)
Expand Down
5 changes: 3 additions & 2 deletions src/pybind/docling_resources.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#ifndef PYBIND_DOCLING_RESOURCES_H
#define PYBIND_DOCLING_RESOURCES_H
Expand Down Expand Up @@ -30,7 +31,7 @@ namespace docling

std::string docling_resources::get_resources_path()
{
return resource_utils::get_resources_dir(true).string();
return utils::filesystem::to_utf8(resource_utils::get_resources_dir(true));
}

bool docling_resources::set_resources_path()
Expand All @@ -44,7 +45,7 @@ namespace docling
// Extract the string value of the filename
const char* filename = PyUnicode_AsUTF8(filenameObj);

std::filesystem::path __init__path(filename);
std::filesystem::path __init__path = utils::filesystem::from_utf8(filename);

std::filesystem::path package_path = __init__path.parent_path();
std::filesystem::path resources_path = package_path / resource_utils::resources_relative_path;
Expand Down
7 changes: 4 additions & 3 deletions src/pybind/docling_threaded_base.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#ifndef PYBIND_THREADED_PDF_BASE_H
#define PYBIND_THREADED_PDF_BASE_H
Expand All @@ -23,12 +24,12 @@
#include <codecvt>
#endif

#include <parse.h>

#include <pybind/native_memory.h>
#include <pybind/docling_resources.h>
#include <pybind/docling_threaded_results.h>

#include <parse.h>

namespace docling
{
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -174,7 +175,7 @@ namespace docling
// Force thread-safe mode since we're doing parallel decoding
this->config.do_thread_safe = true;

std::string pdf_resources_dir = resource_utils::get_resources_dir(true).string();
std::string pdf_resources_dir = utils::filesystem::to_utf8(resource_utils::get_resources_dir(true));
LOG_S(WARNING) << "pdf_resources_dir: " << pdf_resources_dir;

auto RESOURCE_DIR_KEY = pdflib::pdf_resource<pdflib::PAGE_FONT>::RESOURCE_DIR_KEY;
Expand Down
6 changes: 4 additions & 2 deletions src/resources.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//-*-C++-*-
// SPDX-License-Identifier: MIT

#ifndef RESOURCE_UTILS_H_
#define RESOURCE_UTILS_H_
Expand All @@ -21,8 +22,9 @@ class resource_utils

private:

static inline std::filesystem::path ROOT_DIR = ROOT_PATH; // ROOT_PATH is the default provided during compilation
static inline std::filesystem::path PACKAGE_DIR = ROOT_PATH / package_name;
// ROOT_PATH is a UTF-8 source literal supplied by the build.
static inline std::filesystem::path ROOT_DIR = utils::filesystem::from_utf8(ROOT_PATH);
static inline std::filesystem::path PACKAGE_DIR = ROOT_DIR / package_name;
static inline std::filesystem::path RESOURCES_DIR = PACKAGE_DIR / resources_relative_path;
};

Expand Down
2 changes: 1 addition & 1 deletion tests/cpp/test_memory_on_lines.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -507,7 +507,7 @@ void measure(std::string mode, std::string filename, int pause_seconds, const st
}

json font_config = {{pdflib::pdf_resource<pdflib::PAGE_FONT>::RESOURCE_DIR_KEY,
resource_utils::get_resources_dir(true).string()}};
utils::filesystem::to_utf8(resource_utils::get_resources_dir(true))}};
std::unordered_map<std::string, double> font_timings;
pdflib::pdf_resource<pdflib::PAGE_FONT>::initialise(font_config, font_timings);
pdflib::pdf_decoder<pdflib::DOCUMENT> document;
Expand Down
3 changes: 2 additions & 1 deletion tests/cpp/test_structure_concurrency.cpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// SPDX-License-Identifier: MIT
// Run against a multipage tagged PDF, optionally compiled with ThreadSanitizer.
// This exercises parser-owned synchronization; linked third-party libraries
// must also be instrumented for a complete audit of their internals.
Expand All @@ -12,7 +13,7 @@ int main(int argc, char** argv)
loguru::g_stderr_verbosity = loguru::Verbosity_FATAL;
nlohmann::json font_config = {{
pdflib::pdf_resource<pdflib::PAGE_FONT>::RESOURCE_DIR_KEY,
resource_utils::get_resources_dir(true).string()}};
utils::filesystem::to_utf8(resource_utils::get_resources_dir(true))}};
std::unordered_map<std::string, double> font_timings;
pdflib::pdf_resource<pdflib::PAGE_FONT>::initialise(font_config, font_timings);
pdflib::pdf_decoder<pdflib::DOCUMENT> document;
Expand Down
Loading
Loading