diff --git a/packages/markitdown/src/markitdown/converters/_markdownify.py b/packages/markitdown/src/markitdown/converters/_markdownify.py index 19e8a2984..4631d8a78 100644 --- a/packages/markitdown/src/markitdown/converters/_markdownify.py +++ b/packages/markitdown/src/markitdown/converters/_markdownify.py @@ -2,7 +2,18 @@ import markdownify from typing import Any, Optional -from urllib.parse import quote, unquote, urlparse, urlunparse +from urllib.parse import quote, urlparse, urlunparse + + +def _quote_path_preserving_percent_encoded_octets(path: str) -> str: + parts = [] + start = 0 + for match in re.finditer(r"%[0-9A-Fa-f]{2}", path): + parts.append(quote(path[start : match.start()], safe="/")) + parts.append(match.group(0)) + start = match.end() + parts.append(quote(path[start:], safe="/")) + return "".join(parts) class _CustomMarkdownify(markdownify.MarkdownConverter): @@ -60,7 +71,13 @@ def convert_a( parsed_url = urlparse(href) # type: ignore if parsed_url.scheme and parsed_url.scheme.lower() not in ["http", "https", "file"]: # type: ignore return "%s%s%s" % (prefix, text, suffix) - href = urlunparse(parsed_url._replace(path=quote(unquote(parsed_url.path)))) # type: ignore + href = urlunparse( + parsed_url._replace( + path=_quote_path_preserving_percent_encoded_octets( + parsed_url.path + ) + ) + ) # type: ignore except ValueError: # It's not clear if this ever gets thrown return "%s%s%s" % (prefix, text, suffix) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 4d62e4919..be20757ef 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -288,6 +288,30 @@ def test_input_as_strings() -> None: assert "# Test" in result.text_content +def test_html_preserves_percent_encoded_href_octets() -> None: + markitdown = MarkItDown() + href = "https://abc.com/hist/%a5%c8%a5%c3%a5%d7%a5%da%a1%bc%a5%b8" + input_data = f'
example' + + result = markitdown.convert_stream( + io.BytesIO(input_data.encode("utf-8")), + file_extension=".html", + ) + + assert f"[example]({href})" in result.text_content + assert "%EF%BF%BD" not in result.text_content + + raw_percent_href = "https://abc.com/hist/100% ready.html" + input_data = f'raw' + + result = markitdown.convert_stream( + io.BytesIO(input_data.encode("utf-8")), + file_extension=".html", + ) + + assert "[raw](https://abc.com/hist/100%25%20ready.html)" in result.text_content + + def test_deeply_nested_html_fallback() -> None: """Large, deeply nested HTML should fall back to plain-text extraction instead of silently returning unconverted HTML (issue #1636).