From 896c7f5042d00e89f4cc2b3ef8d4db6e87b06862 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sat, 14 Nov 2026 13:25:02 +0000 Subject: [PATCH 1/4] fix(catalog): a <...> URL keeps its pipe and a destination its spaces; pin the bracket counter Settles two review points of #459 (T11.2 and T13.2). The end-of-URL scan cut every URL at a `|`, so `` was probed as `https://acme.dev/a`. A delimited URL (an autolink or a `<...>` destination) now ends at its `>` only; the undelimited case still ends at a pipe. In a table row (a line that starts with `|`) an unescaped pipe still ends the URL, because a GFM table splits its row at it before it reads links. A `<...>` destination may hold spaces under CommonMark: `[a]()` was cut at the space and `/x` was probed. It is now read up to its `>` and httpx requests `/x%20y`. An autolink holds no space and is unchanged. Tests pin both and the table-row choice, and `[see https://acme.dev/a[b]c] for more` kills the mutant that drops `brackets -= 1`. Signed-off-by: Anton Volkov --- ...0261114_100000_delimited-url-pipe-space.md | 3 +++ dokimi/dokimi/external/online_checks.py | 19 +++++++++++--- dokimi/tests/test_catalog_online.py | 25 +++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261114_100000_delimited-url-pipe-space.md diff --git a/dokimi/changelog.d/20261114_100000_delimited-url-pipe-space.md b/dokimi/changelog.d/20261114_100000_delimited-url-pipe-space.md new file mode 100644 index 000000000..5552586ac --- /dev/null +++ b/dokimi/changelog.d/20261114_100000_delimited-url-pipe-space.md @@ -0,0 +1,3 @@ +### Fixed + +- `catalog check --online` reads a `<...>` link destination up to its `>`: `[a]()` is requested as `https://acme.dev/x%20y` (it was cut at the space, so `/x` was probed), and a `|` inside an autolink or a `<...>` destination stays in the URL (`` was cut at the pipe). In a table row (a line that starts with `|`) an unescaped pipe still ends the URL. (#459) diff --git a/dokimi/dokimi/external/online_checks.py b/dokimi/dokimi/external/online_checks.py index 71fa90572..6e36e551c 100644 --- a/dokimi/dokimi/external/online_checks.py +++ b/dokimi/dokimi/external/online_checks.py @@ -99,6 +99,7 @@ _URL_RE = re.compile(r"https?://(?:[^\s\"`<>'“”‘’—，。、；：！？（）「」]|'(?=\w))+", re.IGNORECASE) # an apostrophe in a word is in a URL ("Moore's_law") # Where a literal URL starts: after a markdown destination's ``](``, or a reference definition's ``[id]: ``. _LITERAL_START_RE = re.compile(r"\]\(|^[ \t]*\[(?!\^)[^\]\n]*\]:[ \t]*", re.MULTILINE) # not a footnote "[^1]:" +_ANGLE_REST_RE = re.compile(r"([^<>\n]*)>") # a "<...>" link destination: the text up to its ">", blanks included _ESCAPE_RE = re.compile(r"\\([!-/:-@\[-`{-~])") # a backslash escape of ASCII punctuation, as CommonMark reads it # An escape or a character reference of a destination, in one pass so that "\&" is the text "&". _DECODE_RE = re.compile(r"\\([!-/:-@\[-`{-~])|&(#[xX][0-9a-fA-F]{1,6}|#[0-9]{1,7}|[A-Za-z][A-Za-z0-9]{1,31});") @@ -388,7 +389,8 @@ def _decode(match: re.Match[str]) -> str: return html.unescape(text) if text[1] == "#" or text[1:] in html5 else text -def _clean_url(raw: str, literal: bool = False, decode: bool = False, delimited: bool = False) -> Optional[str]: +def _clean_url(raw: str, literal: bool = False, decode: bool = False, delimited: bool = False, + table_row: bool = False) -> Optional[str]: """``raw`` without what a sentence, a table cell, emphasis or a markdown link puts after it: the URL ends before the first ``)`` or ``]`` that closes no ``(`` or ``[`` of the URL, and before a ``|``; then trailing punctuation and ``*`` emphasis go. A balanced bracket stays ("f[0]=1", "[2001:db8::1]"). None for an empty result. @@ -398,7 +400,8 @@ def _clean_url(raw: str, literal: bool = False, decode: bool = False, delimited: decodes its backslash escapes (``guide\\_notes`` is ``guide_notes``) and its character references (``a&b`` is ``a&b``), and an escaped character never ends it (``a\\)b`` is ``a)b``). ``delimited`` is a destination or autolink in ``<...>``: it ends at ``>``, so an - unmatched ``)`` or ``]`` is part of it. + unmatched ``)`` or ``]`` and a ``|`` are part of it, unless ``table_row``: a GFM table splits its row at an + unescaped ``|`` before it reads the inline text, so the pipe ends the URL there. """ parens = brackets = 0 end = len(raw) @@ -407,7 +410,8 @@ def _clean_url(raw: str, literal: bool = False, decode: bool = False, delimited: parens += 1 elif ch == "[": brackets += 1 - elif ch == "|" or (not delimited and ((ch == ")" and parens == 0) or (ch == "]" and brackets == 0))): + elif (ch == "|" and (table_row or not delimited)) or ( + not delimited and ((ch == ")" and parens == 0) or (ch == "]" and brackets == 0))): end = i break elif ch == ")": @@ -492,7 +496,14 @@ def _distinct_urls(body: str) -> tuple[list[str], list[str]]: continue destination = _starts_destination(body, m.start(), literal) autolink = m.start() > 0 and body[m.start() - 1] == "<" and body[m.end():m.end() + 1] == ">" - url = _clean_url(m.group(), literal=destination or autolink, decode=destination, delimited=autolink) + raw, angled = m.group(), False + if destination and body[m.start() - 1] == "<": # a "<...>" destination may hold spaces; it ends at ">" + close = _ANGLE_REST_RE.match(body, m.start()) + if close: + raw, angled = close.group(1).rstrip(), True + line_start = body.rfind("\n", 0, m.start()) + 1 + url = _clean_url(raw, literal=destination or autolink, decode=destination, delimited=autolink or angled, + table_row=body[line_start:m.start()].lstrip().startswith("|")) if url is not None: url = url.partition("#")[0] # the fragment is never sent: anchors of one page are one request if url is not None and not _placeholder_url(url): diff --git a/dokimi/tests/test_catalog_online.py b/dokimi/tests/test_catalog_online.py index a561be9a4..f0eef4102 100644 --- a/dokimi/tests/test_catalog_online.py +++ b/dokimi/tests/test_catalog_online.py @@ -348,6 +348,31 @@ def test_an_escaped_paren_bracket_or_pipe_stays_in_a_destination(): assert online_checks.extract_urls("[x](https://x.dev/p\\\\)") == ["https://x.dev/p\\"] +def test_a_pipe_in_an_angle_delimited_url_is_part_of_it_outside_a_table_row(): + assert online_checks.extract_urls("") == ["https://acme.dev/a|b"] + assert online_checks.extract_urls("[a]()") == ["https://acme.dev/a|b"] + assert online_checks.extract_urls("see https://acme.dev/a|b now") == ["https://acme.dev/a"] + # A GFM table splits its row at an unescaped "|" before it reads links: the pipe ends the URL there. + assert online_checks.extract_urls("| | x |") == ["https://acme.dev/a"] + assert online_checks.extract_urls(" | [a]() | x |") == ["https://acme.dev/a"] + + +def test_an_angle_destination_keeps_its_spaces_and_is_probed_with_the_path_encoded(tmp_path): + assert online_checks.extract_urls("[a]()") == ["https://acme.dev/x y"] + assert online_checks.extract_urls("[a]()") == ["https://acme.dev/x y)z"] + assert online_checks.extract_urls("") == ["https://acme.dev/x"] # an autolink holds no space + web = Web({"https://acme.dev/x%20y": 404}) + root = _one_skill(tmp_path, "See [a]().\n") + + [review] = _reviews(root, web) + assert _codes(review) == ["link-dead"] + assert {seen for _, seen in web.requests} == {"https://acme.dev/x%20y"} + + +def test_an_unbalanced_closing_bracket_ends_the_url_after_a_balanced_pair(): + assert online_checks.extract_urls("[see https://acme.dev/a[b]c] for more") == ["https://acme.dev/a[b]c"] + + def test_a_backslash_in_an_autolink_is_not_decoded_and_a_hash_after_a_command_word_is_prose(): # Self-review: CommonMark has no escapes inside an autolink; and "npm (see #setup)" is prose, not an install. assert online_checks.extract_urls("") == ["https://x.dev/a\\_b"] From c552bbba97e592946e6dd1ee6a6f89125d9d4efb Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sat, 14 Nov 2026 19:46:37 +0000 Subject: [PATCH 2/4] fix(catalog): a <...> destination keeps its trailing blanks Settles the review point that `.rstrip()` cut the trailing space of `[a]()`, so `/x` was probed instead of `/x%20`. The destination is now taken up to its `>` as written. Tests cover the extraction, the request, and the unclosed `<` fallback branch that codecov reported as partial. Signed-off-by: Anton Volkov --- dokimi/dokimi/external/online_checks.py | 2 +- dokimi/tests/test_catalog_online.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/external/online_checks.py b/dokimi/dokimi/external/online_checks.py index 6e36e551c..b043442fa 100644 --- a/dokimi/dokimi/external/online_checks.py +++ b/dokimi/dokimi/external/online_checks.py @@ -500,7 +500,7 @@ def _distinct_urls(body: str) -> tuple[list[str], list[str]]: if destination and body[m.start() - 1] == "<": # a "<...>" destination may hold spaces; it ends at ">" close = _ANGLE_REST_RE.match(body, m.start()) if close: - raw, angled = close.group(1).rstrip(), True + raw, angled = close.group(1), True line_start = body.rfind("\n", 0, m.start()) + 1 url = _clean_url(raw, literal=destination or autolink, decode=destination, delimited=autolink or angled, table_row=body[line_start:m.start()].lstrip().startswith("|")) diff --git a/dokimi/tests/test_catalog_online.py b/dokimi/tests/test_catalog_online.py index f0eef4102..db9e138a6 100644 --- a/dokimi/tests/test_catalog_online.py +++ b/dokimi/tests/test_catalog_online.py @@ -360,6 +360,8 @@ def test_a_pipe_in_an_angle_delimited_url_is_part_of_it_outside_a_table_row(): def test_an_angle_destination_keeps_its_spaces_and_is_probed_with_the_path_encoded(tmp_path): assert online_checks.extract_urls("[a]()") == ["https://acme.dev/x y"] assert online_checks.extract_urls("[a]()") == ["https://acme.dev/x y)z"] + assert online_checks.extract_urls("[a]()") == ["https://acme.dev/x "] # a trailing blank is part of it + assert online_checks.extract_urls("[a](" closes it: read as a bare URL assert online_checks.extract_urls("") == ["https://acme.dev/x"] # an autolink holds no space web = Web({"https://acme.dev/x%20y": 404}) root = _one_skill(tmp_path, "See [a]().\n") @@ -368,6 +370,11 @@ def test_an_angle_destination_keeps_its_spaces_and_is_probed_with_the_path_encod assert _codes(review) == ["link-dead"] assert {seen for _, seen in web.requests} == {"https://acme.dev/x%20y"} + trailing = Web({"https://acme.dev/x%20": 404}) + [review] = _reviews(_one_skill(tmp_path / "t", "See [a]().\n"), trailing) + assert _codes(review) == ["link-dead"] + assert {seen for _, seen in trailing.requests} == {"https://acme.dev/x%20"} + def test_an_unbalanced_closing_bracket_ends_the_url_after_a_balanced_pair(): assert online_checks.extract_urls("[see https://acme.dev/a[b]c] for more") == ["https://acme.dev/a[b]c"] From 213e7572ce616cb5edf76ecb66e270bb7b27a34e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sat, 14 Nov 2026 20:04:27 +0000 Subject: [PATCH 3/4] fix(catalog): a backslash-escaped > or < does not end or stop a <...> destination Review point 4575431969: the scan for a "<...>" destination stopped at any ">" and refused any "<", so "[a](z>)" was probed as "/x y\" instead of "/x%20y%3Ez", and an escaped "<" left the destination unscanned. The scan now reads a backslash escape of ASCII punctuation as one unit, keeping the original text so that _clean_url decodes it. The test covers extraction of both escapes and the request the transport sees. Signed-off-by: Anton Volkov --- dokimi/changelog.d/20261114_100000_delimited-url-pipe-space.md | 2 +- dokimi/dokimi/external/online_checks.py | 3 ++- dokimi/tests/test_catalog_online.py | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dokimi/changelog.d/20261114_100000_delimited-url-pipe-space.md b/dokimi/changelog.d/20261114_100000_delimited-url-pipe-space.md index 5552586ac..24c1329e8 100644 --- a/dokimi/changelog.d/20261114_100000_delimited-url-pipe-space.md +++ b/dokimi/changelog.d/20261114_100000_delimited-url-pipe-space.md @@ -1,3 +1,3 @@ ### Fixed -- `catalog check --online` reads a `<...>` link destination up to its `>`: `[a]()` is requested as `https://acme.dev/x%20y` (it was cut at the space, so `/x` was probed), and a `|` inside an autolink or a `<...>` destination stays in the URL (`` was cut at the pipe). In a table row (a line that starts with `|`) an unescaped pipe still ends the URL. (#459) +- `catalog check --online` reads a `<...>` link destination up to its `>`: `[a]()` is requested as `https://acme.dev/x%20y` (it was cut at the space, so `/x` was probed), and a `|` inside an autolink or a `<...>` destination stays in the URL (`` was cut at the pipe). In a table row (a line that starts with `|`) an unescaped pipe still ends the URL. A backslash-escaped `>` or `<` inside the `<...>` does not end it: `[a](z>)` is requested as `https://acme.dev/x%20y%3Ez`. (#459) diff --git a/dokimi/dokimi/external/online_checks.py b/dokimi/dokimi/external/online_checks.py index b043442fa..eeb4de8e4 100644 --- a/dokimi/dokimi/external/online_checks.py +++ b/dokimi/dokimi/external/online_checks.py @@ -99,7 +99,8 @@ _URL_RE = re.compile(r"https?://(?:[^\s\"`<>'“”‘’—，。、；：！？（）「」]|'(?=\w))+", re.IGNORECASE) # an apostrophe in a word is in a URL ("Moore's_law") # Where a literal URL starts: after a markdown destination's ``](``, or a reference definition's ``[id]: ``. _LITERAL_START_RE = re.compile(r"\]\(|^[ \t]*\[(?!\^)[^\]\n]*\]:[ \t]*", re.MULTILINE) # not a footnote "[^1]:" -_ANGLE_REST_RE = re.compile(r"([^<>\n]*)>") # a "<...>" link destination: the text up to its ">", blanks included +# a "<...>" link destination: the text up to its unescaped ">", blanks and escaped "<" ">" included +_ANGLE_REST_RE = re.compile(r"((?:\\[!-/:-@\[-`{-~]|[^<>\n\\]|\\(?![!-/:-@\[-`{-~]))*)>") _ESCAPE_RE = re.compile(r"\\([!-/:-@\[-`{-~])") # a backslash escape of ASCII punctuation, as CommonMark reads it # An escape or a character reference of a destination, in one pass so that "\&" is the text "&". _DECODE_RE = re.compile(r"\\([!-/:-@\[-`{-~])|&(#[xX][0-9a-fA-F]{1,6}|#[0-9]{1,7}|[A-Za-z][A-Za-z0-9]{1,31});") diff --git a/dokimi/tests/test_catalog_online.py b/dokimi/tests/test_catalog_online.py index db9e138a6..52005c28d 100644 --- a/dokimi/tests/test_catalog_online.py +++ b/dokimi/tests/test_catalog_online.py @@ -363,6 +363,7 @@ def test_an_angle_destination_keeps_its_spaces_and_is_probed_with_the_path_encod assert online_checks.extract_urls("[a]()") == ["https://acme.dev/x "] # a trailing blank is part of it assert online_checks.extract_urls("[a](" closes it: read as a bare URL assert online_checks.extract_urls("") == ["https://acme.dev/x"] # an autolink holds no space + assert [online_checks.extract_urls(f"[a]()") for c in "<>"] == [["https://acme.dev/x yz"]] # an escaped bracket neither ends nor stops it web = Web({"https://acme.dev/x%20y": 404}) root = _one_skill(tmp_path, "See [a]().\n") @@ -374,6 +375,8 @@ def test_an_angle_destination_keeps_its_spaces_and_is_probed_with_the_path_encod [review] = _reviews(_one_skill(tmp_path / "t", "See [a]().\n"), trailing) assert _codes(review) == ["link-dead"] assert {seen for _, seen in trailing.requests} == {"https://acme.dev/x%20"} + [review] = _reviews(_one_skill(tmp_path / "e", "See [a](z>).\n"), escaped := Web({"https://acme.dev/x%20y%3Ez": 404})) + assert _codes(review) == ["link-dead"] and {seen for _, seen in escaped.requests} == {"https://acme.dev/x%20y%3Ez"} def test_an_unbalanced_closing_bracket_ends_the_url_after_a_balanced_pair(): From 7777982742d98a684f95b4b604d9eebc72e12ba1 Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:05:23 -0600 Subject: [PATCH 4/4] fix(catalog): an escaped < in a <...> destination is not a template `[a]()` extracted nothing: the template guard saw the `<` after the match before the angle scan ran. The guard now runs after the scan and ignores a destination the scan closed at its `>`. Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- dokimi/dokimi/external/online_checks.py | 4 ++-- dokimi/tests/test_catalog_online.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/external/online_checks.py b/dokimi/dokimi/external/online_checks.py index eeb4de8e4..a010afe72 100644 --- a/dokimi/dokimi/external/online_checks.py +++ b/dokimi/dokimi/external/online_checks.py @@ -491,8 +491,6 @@ def _distinct_urls(body: str) -> tuple[list[str], list[str]]: body = body.replace("](", " ") # a markdown link's text and its target are both URLs; the length is kept code = [m.span() for m in _URL_CODE_SPAN_RE.finditer(body)] # a URL in a code span is an example, as in a fence for m in _URL_RE.finditer(body): - if _TEMPLATE_RE.search(m.group()) or body[m.end():m.end() + 1] in ("<", "{", "$"): - continue if any(start <= m.start() < end for start, end in code): continue destination = _starts_destination(body, m.start(), literal) @@ -502,6 +500,8 @@ def _distinct_urls(body: str) -> tuple[list[str], list[str]]: close = _ANGLE_REST_RE.match(body, m.start()) if close: raw, angled = close.group(1), True + if _TEMPLATE_RE.search(raw) or (not angled and body[m.end():m.end() + 1] in ("<", "{", "$")): + continue line_start = body.rfind("\n", 0, m.start()) + 1 url = _clean_url(raw, literal=destination or autolink, decode=destination, delimited=autolink or angled, table_row=body[line_start:m.start()].lstrip().startswith("|")) diff --git a/dokimi/tests/test_catalog_online.py b/dokimi/tests/test_catalog_online.py index 52005c28d..a1594bfea 100644 --- a/dokimi/tests/test_catalog_online.py +++ b/dokimi/tests/test_catalog_online.py @@ -363,14 +363,14 @@ def test_an_angle_destination_keeps_its_spaces_and_is_probed_with_the_path_encod assert online_checks.extract_urls("[a]()") == ["https://acme.dev/x "] # a trailing blank is part of it assert online_checks.extract_urls("[a](" closes it: read as a bare URL assert online_checks.extract_urls("") == ["https://acme.dev/x"] # an autolink holds no space - assert [online_checks.extract_urls(f"[a]()") for c in "<>"] == [["https://acme.dev/x yz"]] # an escaped bracket neither ends nor stops it + assert [online_checks.extract_urls(f"[a]()") for b in ("", " y") for c in "<>"] == [["https://acme.dev/xz"], ["https://acme.dev/x yz"]] # an escaped bracket neither ends nor stops it, blank or not + assert online_checks.extract_urls("[a]()") == [] # an unescaped "<" is a template web = Web({"https://acme.dev/x%20y": 404}) root = _one_skill(tmp_path, "See [a]().\n") [review] = _reviews(root, web) assert _codes(review) == ["link-dead"] assert {seen for _, seen in web.requests} == {"https://acme.dev/x%20y"} - trailing = Web({"https://acme.dev/x%20": 404}) [review] = _reviews(_one_skill(tmp_path / "t", "See [a]().\n"), trailing) assert _codes(review) == ["link-dead"]