diff --git a/src/parse/pdf_decoders/stream.h b/src/parse/pdf_decoders/stream.h index 8946f182..561f4f56 100644 --- a/src/parse/pdf_decoders/stream.h +++ b/src/parse/pdf_decoders/stream.h @@ -317,6 +317,7 @@ namespace pdflib int bits_per_component = 1; std::string color_space = ""; std::vector filters; + QPDFObjectHandle decode_parms = QPDFObjectHandle::newNull(); bool decode_present = false; std::vector decode_array; bool image_mask = false; @@ -955,10 +956,10 @@ namespace pdflib } else if(key == "/DecodeParms") { - // Only /CCITTFaxDecode carries parameters we act on here. Written - // as an array they line up with the filters; the first dictionary - // is the one that can hold them, since every filter that may - // precede the codec takes none. + // Preserve the complete filter-aligned parameters for transport + // decoding, including Flate/LZW predictors. Extract CCITT's + // codec-specific settings separately below. + inline_image.decode_parms = value; QPDFObjectHandle parms = value; if(parms.isArray()) { @@ -1104,6 +1105,7 @@ namespace pdflib inline_image.bits_per_component, inline_image.color_space, inline_image.filters, + inline_image.decode_parms, inline_image.decode_present, inline_image.decode_array, inline_image.image_mask, diff --git a/src/parse/pdf_resources/page_font.h b/src/parse/pdf_resources/page_font.h index 346f7645..20163197 100644 --- a/src/parse/pdf_resources/page_font.h +++ b/src/parse/pdf_resources/page_font.h @@ -533,7 +533,15 @@ namespace pdflib auto& bfont = *(matched_base_font().font); - if(bfont.has(c)) + // AFM metrics are numbered in the font's built-in encoding, so a + // font that declares an /Encoding (or /Differences) is looked up by + // the glyph name that encoding gives the code (docling-parse#403). + const std::string glyph_name = get_glyph_name(c); + if(not glyph_name.empty() and bfont.has_glyph_name(glyph_name)) + { + return bfont.get_glyph_width(glyph_name); + } + else if(bfont.has(c)) { return bfont.get_width(c); } @@ -655,7 +663,13 @@ namespace pdflib { auto& bfont = *(matched_base_font().font); - if(bfont.has(c) and bfont.get_width(c)>0) + const std::string glyph_name = get_glyph_name(c); + if(not glyph_name.empty() and bfont.has_glyph_name(glyph_name) and + bfont.get_glyph_width(glyph_name)>0) + { + return bfont.get_glyph_width(glyph_name); + } + else if(bfont.has(c) and bfont.get_width(c)>0) { return bfont.get_width(c); } @@ -725,7 +739,20 @@ namespace pdflib { if(matched_base_font().font) { - return matched_base_font().font->has_char_bbox(c); + auto& bfont = *(matched_base_font().font); + + const std::string glyph_name = get_glyph_name(c); + if(not glyph_name.empty() and bfont.has_glyph_bbox(glyph_name)) + { + // Some AFM entries, including Euro, have placeholder bounds. + // Preserve blank spaces; use font metrics for other empty bounds. + const auto bbox = bfont.get_glyph_bbox(glyph_name); + return glyph_name == "space" or bbox[3] > bbox[1]; + } + + // Builtin subset codes do not identify standard-font characters: + // code 32, for example, may draw "m" rather than a space. + return not uses_builtin_encoding() and bfont.has_char_bbox(c); } return false; @@ -740,7 +767,15 @@ namespace pdflib return bfonts.get("Unknown")->get_char_bbox(c); } - return matched_base_font().font->get_char_bbox(c); + auto& bfont = *(matched_base_font().font); + + const std::string glyph_name = get_glyph_name(c); + if(not glyph_name.empty() and bfont.has_glyph_bbox(glyph_name)) + { + return bfont.get_glyph_bbox(glyph_name); + } + + return bfont.get_char_bbox(c); } bool pdf_resource::has_char_bbox(const std::string& c) diff --git a/src/parse/pdf_resources/page_font/base_font.h b/src/parse/pdf_resources/page_font/base_font.h index 758c8f3c..a3c29849 100644 --- a/src/parse/pdf_resources/page_font/base_font.h +++ b/src/parse/pdf_resources/page_font/base_font.h @@ -35,6 +35,12 @@ namespace pdflib std::array get_char_bbox(const uint32_t& numb); std::array get_char_bbox(const std::string& c); + // Lookups by glyph name, covering encoded and unencoded (`C -1`) glyphs; + // has_glyph_name() above guards get_glyph_width(). + double get_glyph_width(const std::string& glyph_name); + bool has_glyph_bbox(const std::string& glyph_name); + std::array get_glyph_bbox(const std::string& glyph_name); + std::string get_string(uint32_t numb); std::string to_utf8(uint32_t numb); @@ -145,6 +151,24 @@ namespace pdflib return 500; } + double base_font::get_glyph_width(const std::string& glyph_name) + { + initialise(); + return name_to_width.at(glyph_name); + } + + bool base_font::has_glyph_bbox(const std::string& glyph_name) + { + initialise(); + return (name_to_bbox.count(glyph_name)==1); + } + + std::array base_font::get_glyph_bbox(const std::string& glyph_name) + { + initialise(); + return name_to_bbox.at(glyph_name); + } + bool base_font::has_char_bbox(const uint32_t& numb) { initialise(); @@ -406,6 +430,10 @@ namespace pdflib if(not name.empty()) { name_to_width[name] = wval; + if(have_bbox) + { + name_to_bbox[name] = bbox; + } } if(numb>=0 and name!="") @@ -420,7 +448,6 @@ namespace pdflib if(have_bbox) { numb_to_bbox[uc] = bbox; - name_to_bbox[name] = bbox; LOG_S(INFO) << "char-bbox: " << uc << "\t" << name << "\t[" << bbox[0] << ", " << bbox[1] << ", " diff --git a/src/parse/pdf_states/bitmap.h b/src/parse/pdf_states/bitmap.h index 2a6089b1..5089fa15 100644 --- a/src/parse/pdf_states/bitmap.h +++ b/src/parse/pdf_states/bitmap.h @@ -182,6 +182,7 @@ namespace pdflib int bits_per_component, const std::string& color_space, const std::vector& filters, + QPDFObjectHandle decode_parms, bool decode_present, const std::vector& decode_array, bool image_mask, @@ -438,6 +439,7 @@ namespace pdflib int bits_per_component, const std::string& color_space, const std::vector& filters, + QPDFObjectHandle decode_parms, bool decode_present, const std::vector& decode_array, bool image_mask, @@ -485,7 +487,7 @@ namespace pdflib auto encoded = stream_filters::apply_filters(stream_data, filters, codec_index, - QPDFObjectHandle::newNull()); + decode_parms); if(not encoded) { LOG_S(WARNING) << "failed to undo the transport filters of inline image " diff --git a/tests/constants.py b/tests/constants.py index c8188ec0..242ba491 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,7 +1,7 @@ from tests.regression_page_selection import REGRESSION_PAGE_SELECTION HF_DATASET_REPO_ID = "docling-project/regression-dataset-for-docling-parse" -HF_DATASET_REVISION = "b09d0574ca75199dd56371ddbf93528a140dea55" +HF_DATASET_REVISION = "12531342c926708dcb1387ea25213db4603a7255" REGRESSION_DIR = "tests/data/regression" diff --git a/tests/test_regression_pypdfium_cited_pages.py b/tests/test_regression_pypdfium_cited_pages.py index 2f90bf31..131528a7 100644 --- a/tests/test_regression_pypdfium_cited_pages.py +++ b/tests/test_regression_pypdfium_cited_pages.py @@ -129,6 +129,19 @@ ), id="soft-masked-drop-shadow", ), + # #403: Helvetica-Bold without /Widths, whose /Differences maps 193 to Aacute. + # The width came from the AFM's Standard encoding, where 193 is grave, so the + # heading's "Á" was squeezed to less than half its width (mean_abs_error 3.92, + # changed_pixels_ratio 0.083 before the fix; 3.25 and 0.079 after). + pytest.param( + "7413869934284023443-1.pdf", + 2, + PYPDFIUM_RENDER_SCALE, + ImageTolerance( + pixel_threshold=12, mean_abs_error=3.6, changed_pixels_ratio=0.081 + ), + id="standard-font-differences-width", + ), ] diff --git a/tests/test_unit_builtin_encoding.py b/tests/test_unit_builtin_encoding.py index 13160803..5129182e 100644 --- a/tests/test_unit_builtin_encoding.py +++ b/tests/test_unit_builtin_encoding.py @@ -20,7 +20,8 @@ import pytest -from tests.pdf_builder import parse_page, simple_page_pdf, stream_object +from tests.pdf_builder import parse_page, render_page, simple_page_pdf, stream_object +from tests.rendering_regression import coverage_ratio, region_image pytest.importorskip("fontTools", reason="building the test fonts needs fontTools") @@ -229,6 +230,32 @@ def test_builtin_encoding_outranks_a_core_14_alias(): assert text == "Č2" +@pytest.mark.parametrize("base_font", ["/ABCDEF+ArialNarrow,Bold", "/ABCDEF+Helvetica"]) +def test_visible_glyph_at_space_code_keeps_its_render_height(base_font: str): + # The reported ArialNarrow subset places "m" at code 32. A base-font + # space bbox collapses its height and makes the renderer discard it. + from fontTools.ttLib import TTFont + + font = TTFont(BytesIO(_symbol_truetype(glyph_names=True))) + font["cmap"].tables[0].cmap[0xF020] = "two" + program = BytesIO() + font.save(program) + pdf = _page( + program.getvalue(), + " ", + subtype="/TrueType", + font_file="/FontFile2", + base_font=base_font, + ) + result = render_page(pdf) + instructions = result._export_render_instructions_json()["instructions"] + text = [item for item in instructions if item["type"] == "text"] + assert len(text) == 1 + assert text[0]["text"] == "2" + assert text[0]["quad"]["r_y2"] > text[0]["quad"]["r_y0"] + assert coverage_ratio(region_image(result, (20, 83, 32, 100))) > 0.2 + + def test_declared_base_encoding_still_wins(): # A symbolic-flagged font that declares /WinAnsiEncoding keeps it. text = _text(_truetype_page("#2", encoding="/Encoding /WinAnsiEncoding ")) diff --git a/tests/test_unit_font_fallback.py b/tests/test_unit_font_fallback.py index d6773f7f..62e01111 100644 --- a/tests/test_unit_font_fallback.py +++ b/tests/test_unit_font_fallback.py @@ -63,6 +63,46 @@ def test_latin_text_without_an_embedded_font_is_drawn(): ) +@pytest.mark.parametrize( + "base_font", + [ + "Helvetica", + "Helvetica-Bold", + "Helvetica-Oblique", + "Helvetica-BoldOblique", + "Times-Roman", + "Times-Bold", + "Times-Italic", + "Times-BoldItalic", + "Courier", + "Courier-Bold", + "Courier-Oblique", + "Courier-BoldOblique", + ], +) +@pytest.mark.parametrize("text", ["\\200", " "], ids=["euro", "space"]) +def test_empty_afm_bounds_do_not_hide_euro_or_paint_spaces(base_font: str, text: str): + # Core-14 AFMs give Euro and space the same empty bounds. A Euro must + # still draw using the font's vertical metrics; a space stays blank. + pdf = _non_embedded_font_pdf( + f"BT /F1 36 Tf 20 100 Td ({text}) Tj ET", + f"<< /Type /Font /Subtype /Type1 /BaseFont /{base_font} " + "/Encoding /WinAnsiEncoding >>", + ) + result = render_page(pdf) + instructions = result._export_render_instructions_json()["instructions"] + glyph = next(item for item in instructions if item["type"] == "text") + coverage = coverage_ratio(region_image(result, TEXT_BOX)) + if text == " ": + assert glyph["text"] == " " + assert glyph["quad"]["r_y2"] == glyph["quad"]["r_y0"] + assert coverage == 0 + else: + assert glyph["text"] == "€" + assert glyph["quad"]["r_y2"] > glyph["quad"]["r_y0"] + assert coverage > 0.001 + + def test_arabic_text_without_an_embedded_font_is_drawn(): """An Arabic run needs a face chosen for its script, not for its name. diff --git a/tests/test_unit_image_decode.py b/tests/test_unit_image_decode.py index 2fcec88a..3cf4eaf0 100644 --- a/tests/test_unit_image_decode.py +++ b/tests/test_unit_image_decode.py @@ -61,3 +61,44 @@ def test_single_component_decode_samples( assert bytes(artifacts[0]["raw_data"]) == bytes( expected_mapping[value] for value in samples ) + + +@pytest.mark.parametrize("height", [1, 2]) +@pytest.mark.parametrize("filter_chain", ["single", "array", "hex-flate"]) +def test_inline_rgb_png_predictors_restore_samples(height: int, filter_chain: str): + # These tiny predictor-encoded RGB strips make up the gradient bullets in + # 7413869934284023443-1.pdf. Ignoring /DP leaves deltas as pixel samples. + rows = [ + bytes([0, 0, 255, 32, 48, 255, 96, 112, 255, 192, 224, 255]), + bytes([8, 16, 240, 40, 64, 240, 104, 128, 240, 200, 240, 240]), + ][:height] + # PNG Sub on the first row, Up on the second; /Predictor 15 uses the + # leading per-row filter byte to choose the reconstruction operation. + predicted = bytes([1]) + bytes( + (value - (rows[0][i - 3] if i >= 3 else 0)) % 256 + for i, value in enumerate(rows[0]) + ) + if height == 2: + predicted += bytes([2]) + bytes( + (value - rows[0][i]) % 256 for i, value in enumerate(rows[1]) + ) + payload = zlib.compress(predicted) + parms = "<< /Predictor 15 /Columns 4 /Colors 3 /BitsPerComponent 8 >>" + if filter_chain == "single": + filters = f"/F /Fl /DP {parms}" + elif filter_chain == "array": + filters = f"/F [/Fl] /DP [{parms}]" + else: + filters = f"/F [/AHx /Fl] /DP [null {parms}]" + payload = payload.hex().encode("ascii") + b">" + content = ( + ( + f"q 100 0 0 100 50 50 cm BI /W 4 /H {height} /BPC 8 /CS /RGB {filters} ID\n" + ).encode("ascii") + + payload + + b"\nEI Q\n" + ) + result = render_page(simple_page_pdf(content.decode("latin-1"))) + artifacts = result._export_bitmap_artifacts() + assert len(artifacts) == 1 + assert bytes(artifacts[0]["raw_data"]) == b"".join(rows) diff --git a/tests/test_unit_standard_font_widths.py b/tests/test_unit_standard_font_widths.py index 67c114ac..0dae5826 100644 --- a/tests/test_unit_standard_font_widths.py +++ b/tests/test_unit_standard_font_widths.py @@ -120,3 +120,23 @@ def test_explicit_widths_take_precedence_over_base_metrics(): assert advances for advance in advances: assert abs(advance - 10.0) < 0.01 + + +def test_non_ascii_widths_follow_the_declared_encoding(): + """Codes are resolved through the declared /WinAnsiEncoding before the AFM. + + The AFM numbers glyphs in the font's built-in (Standard) encoding, so the + raw code gave WinAnsi `©` (0xA9) the width of Standard's `quotesingle` and + `é` (0xE9) that of `Oslash`; `í` (0xED), unassigned in Standard and listed + in the AFM only as an unencoded `C -1` glyph, fell back to 500 units + (docling-parse#403). At 100 pt the advance in points is units / 10. + """ + for code, afm_width in (("\251", 760), ("\351", 444), ("\355", 278)): + advances = _char_advances("Times-Roman", f"X{code}X", font_size=100) + assert advances == [72.2, afm_width / 10], code + + +def test_apostrophe_width_follows_the_declared_encoding(): + """WinAnsi 0x27 is `quotesingle` (180), not Standard's `quoteright` (333).""" + advances = _char_advances("Times-Roman", "X'X", font_size=100) + assert advances == [72.2, 18.0]