From b4fe6bac04216c5de72ca9711872bb73d5ab4678 Mon Sep 17 00:00:00 2001 From: Praveena Hennadige Date: Fri, 7 Aug 2026 12:03:09 +0530 Subject: [PATCH 1/2] Correct link resolution and add a step-by-step link fixer --- .claude/skills/wso2-doc-frontmatter/SKILL.md | 94 ++++-- .../references/conventions.md | 9 + .../scripts/check_links.py | 117 ++++++- .../scripts/check_style.py | 47 +-- .../wso2-doc-frontmatter/scripts/fix_links.py | 298 ++++++++++++++++++ .../wso2-doc-frontmatter/scripts/fm_fix.py | 41 ++- .../wso2-doc-frontmatter/scripts/fm_lib.py | 80 +++++ .../scripts/report_links.py | 181 ++++++++--- 8 files changed, 738 insertions(+), 129 deletions(-) create mode 100644 .claude/skills/wso2-doc-frontmatter/scripts/fix_links.py diff --git a/.claude/skills/wso2-doc-frontmatter/SKILL.md b/.claude/skills/wso2-doc-frontmatter/SKILL.md index a05adbd8c..8f7616d2c 100644 --- a/.claude/skills/wso2-doc-frontmatter/SKILL.md +++ b/.claude/skills/wso2-doc-frontmatter/SKILL.md @@ -89,19 +89,19 @@ can tell which parts are theirs. The report classifies every finding by **cause**, because the causes have completely different fixes and completely different risk: -| Tier | Cause | Fix | -|---|---|---| -| 0 | `{{base_path}}` and the resource exists | Exact rewrite to a relative path | -| — | `{{base_path}}` and it does not | **Leave alone.** May be served by a redirect | -| 0 | Malformed link syntax | Exact rewrite. No judgement. Safe in bulk. | -| 1 | Wrong relative depth | Exact rewrite. No judgement. Safe in bulk. | -| 2 | Renamed or moved target | A file of that name exists elsewhere; proposed, with confidence | -| 3 | Pre-migration domain | Needs the new equivalent page — human | -| 4 | Missing anchor | Heading was reworded — human | -| 5 | No target anywhere | Was it dropped, missed, or merged? — human | - -It ends with a **ready-to-paste prompt for an AI coding agent**, deliberately -scoped to tiers 0 and 1 and the high-confidence half of tier 2. Do not widen that scope. +| Tier name | Cause | Fix | Applied by `fix_links.py`? | +|---|---|---|---| +| `templated_fixable` | `{{base_path}}` and the resource exists | Exact rewrite to a relative path | Yes | +| `malformed` | Malformed link syntax | Exact rewrite. No judgement. | Yes | +| `depth` | Wrong relative depth | Exact rewrite. No judgement. | Yes | +| `renamed` | Renamed or moved target | A file of that name exists elsewhere; proposed, with confidence | Yes, `high` confidence only by default | +| `templated` | `{{base_path}}` and it does not exist | **Leave alone.** May be served by a redirect | No — refused | +| `stale` | Pre-migration domain | Needs the new equivalent page | No — refused | +| `anchor` | Missing anchor | Heading was reworded | No — refused | +| `gone` | No target anywhere | Was it dropped, missed, or merged? | No — refused | + +The report also ends with a **ready-to-paste prompt for an AI coding agent**, for +when someone wants to hand the work off rather than run step 5 here. Two rules the reporter enforces, and you must not work around: @@ -111,15 +111,69 @@ Two rules the reporter enforces, and you must not work around: - **Never propose a target in a different version.** If a page under one version links to something missing, the replacement must live under that same version. A cross-version link silently sends a reader to a different release. -Tiers 3 to 5 need information that is not in the repo, and an agent asked to fix -them produces confident links to the wrong pages — worse than a visibly broken -link, because a plausible wrong link never gets re-checked. +`stale`, `anchor` and `gone` need information that is not in the repo, and an agent +asked to fix them produces confident links to the wrong pages — worse than a +visibly broken link, because a plausible wrong link never gets re-checked. When you report back, give the tier counts and say plainly how many need a human. A raw total is alarming and useless on its own; "N have an exact fix, M need a decision" is what someone can act on. -### 5. Re-audit, and verify against a real build +### 5. Fix the links one tier at a time, and ask before each tier + +The report is a plan, not a change. `scripts/fix_links.py` is the only thing that +applies it, and it takes **one tier per run**: + +```bash +# show what this tier would do — nothing is written +python3 scripts/fix_links.py en/docs --plan BROKEN-LINKS-.json --tier malformed +# apply it, and record what changed +python3 scripts/fix_links.py en/docs --plan BROKEN-LINKS-.json --tier malformed \ + --apply --journal /tmp/fixed-malformed.json +``` + +Work the tiers in this order, easiest and safest first: + +1. `malformed` — link syntax +2. `depth` — wrong number of `../` +3. `renamed` — target moved (`high` confidence only; `--min-confidence` widens it) +4. `templated_fixable` — if the scope has any + +**The rule for each tier: dry-run it, show the person a sample, tell them how many +would change and how many the verifier refused, and wait for an explicit yes before +`--apply`.** Never chain tiers in one go, and never apply a tier the person has not +seen. Their answer for one tier is not their answer for the next — `depth` is +arithmetic, but `renamed` is a proposal, and someone may want every one of those +eyeballed. + +After each applied tier, report what actually changed and stop: + +``` +tier `depth`: 667 verified, 3 refused, 132 files changed, 667 links rewritten. +Refused: 3 where the anchor no longer exists. Next tier is `renamed` (448, 406 +verified). Apply it? +``` + +**Regenerate the plan between tiers.** Fixing one tier changes what the others +resolve to, so a plan written before the last tier was applied is stale — and +`fix_links.py` will skip entries whose link text it can no longer find rather than +guess. + +Two things the script does that you should not work around: + +- **It verifies every rewrite against the disk before writing anything.** A + proposal whose target does not resolve, or whose anchor does not exist on the new + page, is refused rather than applied. Those refusals are the useful output — they + are the cases where the report was optimistic. +- **It refuses `templated`, `stale`, `anchor` and `gone` outright.** Those need + information that is not in the repo. Do not hand-apply them in bulk to save + time; a plausible wrong link is worse than a visibly broken one, because nobody + re-checks it. + +Finish by re-running the link checker and quoting the before/after, since that — +not the number of files touched — is what says the fix worked. + +### 6. Re-audit, and verify against a real build ```bash python3 scripts/fm_audit.py en/docs --gate @@ -132,7 +186,11 @@ Re-auditing is not optional: it's the only thing that proves the fix worked rath Where the repo can be built, `mkdocs build` is the authoritative check on links — the link checker is calibrated against it and finds a superset of what it reports. If you have the dependencies, run it and reconcile any difference rather than assuming the script is right. -## The other checkers +## The other scripts + +`scripts/fix_links.py` applies one tier of a `report_links.py` plan. Dry run by +default; `--apply` writes; `--journal` records every rewrite so a tier can be +reviewed or undone. It is the only script here that edits link text. `scripts/check_redirects.py` validates `redirect_maps` in `mkdocs.yml` — targets exist, no source shadowed by a real file, no chains (the plugin doesn't follow them), no map left pointing at a superseded version after a version bump. diff --git a/.claude/skills/wso2-doc-frontmatter/references/conventions.md b/.claude/skills/wso2-doc-frontmatter/references/conventions.md index efe1c81f6..1d9e16e2a 100644 --- a/.claude/skills/wso2-doc-frontmatter/references/conventions.md +++ b/.claude/skills/wso2-doc-frontmatter/references/conventions.md @@ -118,6 +118,15 @@ fixable, and `report_links.py` splits them accordingly: Redirects themselves belong either in a `redirects.yml` file or in a `redirects` block inside `mkdocs.yml`. +## Applying a link plan + +`report_links.py` proposes; `fix_links.py` is the only script that rewrites link +text. It applies one tier per run, verifies each rewrite against the disk first, +and refuses the tiers that need a person (`templated`, `stale`, `anchor`, `gone`). + +Regenerate the plan between tiers: fixing one tier changes what the others resolve +to. Entries whose link text can no longer be found are skipped rather than guessed. + ## Adding another source of documentation Nothing in the scripts is tied to a particular product or version. Versions are diff --git a/.claude/skills/wso2-doc-frontmatter/scripts/check_links.py b/.claude/skills/wso2-doc-frontmatter/scripts/check_links.py index 52a280166..f861dcbad 100644 --- a/.claude/skills/wso2-doc-frontmatter/scripts/check_links.py +++ b/.claude/skills/wso2-doc-frontmatter/scripts/check_links.py @@ -11,6 +11,8 @@ _ap.add_argument("--json", dest="json_out", default=None, help="Write full findings to this path. Omitted = summary only.") _ap.add_argument("--gate", action="store_true", help="Exit 1 if any blocking finding.") +_ap.add_argument("--mkdocs-yml", dest="mkdocs_yml", default=None, + help="Where to read `toc_depth` from. Defaults to /../mkdocs.yml.") _args = _ap.parse_args() DOCS = _args.docs_root.rstrip("/") SITE = "https://wso2.com/api-platform/docs" @@ -33,24 +35,77 @@ def slug(h): h = re.sub(r"[^\w\s-]", "", h).strip().lower() return re.sub(r"[-\s]+", "-", h) +def toc_depth(mkdocs_yml): + """The `toc_depth` configured for the python-markdown toc extension. + + This is load-bearing, not cosmetic. With `toc_depth: 3`, python-markdown + assigns NO `id` to h4 and deeper — so `#some-h4-heading` links resolve to + nothing in the built site even though the heading is right there in the + Markdown. Neither this checker nor `mkdocs build` on 1.4.x flags that on its + own, so a page can look clean and still have dead in-page links. + """ + if not os.path.isfile(mkdocs_yml): + return 6 + text = open(mkdocs_yml, encoding="utf-8", errors="replace").read() + m = re.search(r"^\s*toc_depth:\s*['\"]?(\d)", text, re.M) + return int(m.group(1)) if m else 6 + + +MKDOCS_YML = _args.mkdocs_yml or os.path.join(os.path.dirname(DOCS) or ".", "mkdocs.yml") +TOC_DEPTH = toc_depth(MKDOCS_YML) + # harvest anchors per file (headings + explicit /{#id}) -anchors = {} +# +# `anchors` is what the build actually produces. `deep_anchors` holds the ids a +# heading WOULD have if toc_depth allowed it — kept separately so a link to one +# can be reported as its own cause rather than as a generic missing anchor. +# +# Do NOT suggest `{#id}` as the fix for a deep heading here, even though +# `attr_list` would honour it: the `markdownextradata` plugin runs every page +# through Jinja BEFORE Markdown, and `{#` opens a Jinja comment. An unterminated +# one fails the whole build with "Missing end of comment tag". The safe additive +# fix is `` immediately above the heading — inert HTML, already used +# in ~300 pages here, and it leaves the heading level and the TOC untouched. +anchors, deep_anchors = {}, {} for p in md_files: txt = open(os.path.join(DOCS, p), encoding="utf-8", errors="replace").read() txt = re.sub(r"", "", txt, flags=re.S) txt = re.sub(r"```.*?```", "", txt, flags=re.S) - a = set() - for m in re.finditer(r"^#{1,6}\s+(.+?)\s*$", txt, re.M): - h = m.group(1) + a, deep = set(), set() + for m in re.finditer(r"^(#{1,6})\s+(.+?)\s*$", txt, re.M): + level, h = len(m.group(1)), m.group(2) exp = re.search(r"\{#([\w-]+)\}", h) - if exp: a.add(exp.group(1)); h = h[:exp.start()] - a.add(slug(h)) + if exp: + a.add(exp.group(1)) # explicit id survives any toc_depth + h = h[:exp.start()] + (a if level <= TOC_DEPTH else deep).add(slug(h)) for m in re.finditer(r']+(?:name|id)="([^"]+)"', txt): a.add(m.group(1)) for m in re.finditer(r'\{#([\w-]+)\}', txt): a.add(m.group(1)) anchors[p] = a + deep_anchors[p] = deep - a LINK = re.compile(r'(!?)\[([^\]]*)\]\(\s*]+)>?(?:\s+"[^"]*")?\s*\)') -HTML_SRC = re.compile(r']+src="([^"]+)"') +# Tag name is captured so an `` is reported as a broken LINK and an +# `` as a missing IMAGE. Lumping them together mislabels every raw-HTML +# link as an image, which sends whoever reads the report looking for the wrong thing. +HTML_SRC = re.compile(r'<(img|a|source|iframe)[^>]+(?:src|href)="([^"]+)"') + + +def url_base(rel): + """Directory the RENDERED page sits in, under `use_directory_urls: true`. + + `a/b/page.md` is served at `/a/b/page/` — one level deeper than the source — + while `a/b/index.md` is served at `/a/b/`, the same level. + + mkdocs rewrites relative targets written in Markdown syntax, resolving them + against the source file, but passes raw HTML through untouched, so the browser + resolves an `` against the rendered URL instead. The identical string + is therefore correct in one syntax and broken in the other. Resolving both the + same way is how a working image gets "fixed" into a broken one. + """ + d = os.path.dirname(rel) + stem = os.path.basename(rel)[:-3] if rel.endswith(".md") else os.path.basename(rel) + return d if stem in ("index", "README") else (f"{d}/{stem}" if d else stem) findings = [] def add(f, sev, code, msg): @@ -64,10 +119,10 @@ def add(f, sev, code, msg): body = re.sub(r"`[^`\n]*`", "", body) d = os.path.dirname(p) - targets = [(m.group(1) == "!", m.group(3)) for m in LINK.finditer(body)] - targets += [(True, m.group(1)) for m in HTML_SRC.finditer(body)] + targets = [(m.group(1) == "!", m.group(3), False) for m in LINK.finditer(body)] + targets += [(m.group(1).lower() != "a", m.group(2), True) for m in HTML_SRC.finditer(body)] - for is_img, t in targets: + for is_img, t, is_html in targets: if t.startswith(("mailto:", "tel:", "#!")): continue # Build-time template variables (e.g. `{{base_path}}`) are not paths. They @@ -92,7 +147,14 @@ def add(f, sev, code, msg): if t.startswith("#"): frag = urllib.parse.unquote(t[1:]) if frag and frag not in anchors.get(p, set()): - add(p, "should-fix", "ANCHOR_MISSING", f"In-page anchor `{t}` has no matching heading.") + if frag in deep_anchors.get(p, set()): + add(p, "blocking", "ANCHOR_TOO_DEEP", + f"`{t}` names a heading deeper than h{TOC_DEPTH}, and `toc_depth: {TOC_DEPTH}` " + f"means the build gives it no id — so the link goes nowhere. Add " + f"`` just above the heading, or promote the heading " + f"to h{TOC_DEPTH}.") + else: + add(p, "should-fix", "ANCHOR_MISSING", f"In-page anchor `{t}` has no matching heading.") continue path, _, frag = t.partition("#") @@ -100,7 +162,29 @@ def add(f, sev, code, msg): frag = urllib.parse.unquote(frag) if not path: continue - cand = os.path.normpath(os.path.join(d, path)) if not path.startswith("/") else path.lstrip("/") + # WHICH BASE APPLIES — verified against a real mkdocs build, not inferred. + # + # mkdocs rewrites a Markdown target only when the literal path names a file + # that exists in docs_dir (`../c/target.md`, `../img.png`). Then, and only + # then, is the target resolved against the SOURCE directory. + # + # Everything else is passed through verbatim and resolved by the browser + # against the RENDERED URL, which sits one level deeper for a non-index page: + # * raw HTML (``, ``) + # * directory-style Markdown links (`../c/target/`) + # * extensionless Markdown links (`../c/target`) — passed through even + # when `target.md` exists right there + # + # Judging a passed-through link source-relative is how a link that is broken + # in the browser gets reported as clean. + if path.startswith("/"): + cand = path.lstrip("/") + rewritten = False + else: + literal = os.path.normpath(os.path.join(d, path)).replace("\\", "/") + rewritten = (not is_html) and literal in all_files + rel_base = d if rewritten else url_base(p) + cand = os.path.normpath(os.path.join(rel_base, path)) if cand.startswith(".."): add(p, "blocking", "LINK_ESCAPES_ROOT", f"Link `{t}` resolves outside the docs root.") continue @@ -118,8 +202,13 @@ def add(f, sev, code, msg): continue if frag and resolved.endswith(".md"): if frag not in anchors.get(resolved, set()): - add(p, "should-fix", "ANCHOR_MISSING", - f"Anchor `#{frag}` not found in `{resolved}` (link was `{t}`).") + if frag in deep_anchors.get(resolved, set()): + add(p, "blocking", "ANCHOR_TOO_DEEP", + f"`{t}` names a heading in `{resolved}` deeper than h{TOC_DEPTH}, which " + f"`toc_depth: {TOC_DEPTH}` leaves without an id, so the link goes nowhere.") + else: + add(p, "should-fix", "ANCHOR_MISSING", + f"Anchor `#{frag}` not found in `{resolved}` (link was `{t}`).") # Alt text. The style guide is specific here and it is easy to get wrong: # - alt="" is CORRECT for purely decorative images or screenshots that diff --git a/.claude/skills/wso2-doc-frontmatter/scripts/check_style.py b/.claude/skills/wso2-doc-frontmatter/scripts/check_style.py index c0bccc5ff..ec2ce4b74 100644 --- a/.claude/skills/wso2-doc-frontmatter/scripts/check_style.py +++ b/.claude/skills/wso2-doc-frontmatter/scripts/check_style.py @@ -16,48 +16,11 @@ SMALL = {"a","an","and","as","at","but","by","for","from","in","into","nor","of","on","onto","or", "over","per","the","to","up","via","with","vs","near","off","out"} -# Product/proper nouns and acronyms that legitimately stay capitalised mid-heading. -PROPER = {"WSO2","API","APIs","AI","LLM","LLMs","MCP","REST","gRPC","GraphQL","JSON","YAML","XML", - "HTTP","HTTPS","OAuth","OAuth2","JWT","SSO","IdP","OIDC","SAML","TLS","mTLS","SSL","CORS", - "URL","URI","URLs","ID","IDs","UI","CLI","SDK","IDE","CI","CD","VM","VMs","K8s","Kubernetes", - "Docker","Helm","Istio","Envoy","Redis","PostgreSQL","MySQL","Oracle","Grafana","Prometheus", - "Jaeger","Zipkin","OpenTelemetry","OpenSearch","Elasticsearch","Moesif","Stripe","AWS","Azure", - "GCP","Bedrock","OpenAI","Anthropic","Gemini","Mistral","Ollama","Choreo","Bijira","Ballerina", - "Git","GitHub","GitLab","Linux","Windows","macOS","Java","Python","Go","Node","npm","Maven", - "Gradle","Swagger","OpenAPI","AsyncAPI","Postman","Portal","Gateway","Platform","Manager", - "Developer","Control","Plane","Hub","Workspace","Analytics","PII","RBAC","ACL","SLA","TPS", - "QPS","DNS","IP","TCP","UDP","gRPC-Web","Kafka","RabbitMQ","NGINX","Terraform","Ansible", - "Prometheus-compatible","Bitbucket","Vault","Keycloak","Okta","Auth0","Asgardeo","I","Step", - "Table","Contents","Note","Tip","Warning","Example","Appendix","FAQ","README","Enumerated","Values"} - -# Phrase-level allowlist: matched case-sensitively and masked out BEFORE per-word -# scanning, because multi-word product names cannot be expressed as single words. -PROPER_PHRASES = { - # Third-party products / cloud services - "Google Cloud Trace","Google Cloud Monitoring","Google Cloud","Azure AI Content Safety", - "Azure Content Safety Content Moderation","Azure Content Safety Guardrail", - "Azure Content Safety","Azure OpenAI","AWS Bedrock Guardrails","AWS Bedrock Guardrail", - "Docker Compose","OpenSearch Dashboards","VS Code","Server-Sent Events", - # Kubernetes API kinds / concepts - "Horizontal Pod Autoscaler","Pod Disruption Budget","Custom Resource Definition", - "Service Account","ConfigMap","StatefulSet","DaemonSet","HTTPRoute","Gateway API", - # Standards - "JSON Schema Draft 7","JSON Schema","JSONPath", - # WSO2 components (capitalised-dominant in the corpus) - "Gateway Controller","Developer Portal","Control Plane","Policy Hub","Policy Engine", - "Gateway Builder","Event Gateway","API Platform Console","API Platform","MCP Proxy","API Proxy", - # --- Policy Hub policy names ------------------------------------------------- - # Treated as proper nouns, matching the Policy Hub catalogue. Remove this block - # if policy names should instead follow sentence case. - "Model Weighted Round Robin","Model Round Robin","Sentence Count Guardrail", - "Word Count Guardrail","Content Length Guardrail","JSON Schema Guardrail", - "Regex Guardrail","URL Guardrail","Semantic Prompt Guard","Semantic Tool Filtering", - "PII Masking","Analytics Header Filter","Subscription Validation", - "API Key Auth","Basic Auth","JWT Auth","Rate Limit", -} -# Conventional release-stage / status labels never count as Title Case evidence. -STATUS_LABELS = {"beta","alpha","ga","preview","deprecated","optional","experimental", - "recommended","required","default","new","legacy"} +# PROPER, PROPER_PHRASES and STATUS_LABELS live in fm_lib.py so that this checker +# and fm_fix.py's sentence_case() cannot disagree about which capitals are correct. +# Add new product names there, not here. +from fm_lib import PROPER, PROPER_PHRASES, STATUS_LABELS # noqa: E402 + _PHRASES = sorted(PROPER_PHRASES, key=len, reverse=True) RULES = [ diff --git a/.claude/skills/wso2-doc-frontmatter/scripts/fix_links.py b/.claude/skills/wso2-doc-frontmatter/scripts/fix_links.py new file mode 100644 index 000000000..8a072d82a --- /dev/null +++ b/.claude/skills/wso2-doc-frontmatter/scripts/fix_links.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Apply the link fixes proposed in a `report_links.py` plan, one tier at a time. + + # look, change nothing (the default) + python3 scripts/fix_links.py en/docs --plan BROKEN-LINKS-4.6.0.json --tier malformed + # apply that tier + python3 scripts/fix_links.py en/docs --plan BROKEN-LINKS-4.6.0.json --tier malformed --apply + +The split from `report_links.py` is deliberate. The reporter reads and proposes; +this script is the only thing that writes. So a plan can be reviewed, committed, +and handed to someone else, and applying it later is a separate, auditable act. + +Tiers are applied ONE AT A TIME on purpose. Each tier has a different failure +mode, so each deserves its own look and its own commit — a bad `renamed` +proposal is a link to a real page about the wrong thing, which no build catches. + +Only tiers whose entries carry a `suggested` value can be applied at all: + + malformed link syntax. Exact. + dir_style written in URL shape; add `.md` so mkdocs resolves it. Exact. + depth wrong number of `../`. Exact. + renamed target moved. Proposed, with a confidence — high only by default. + templated_fixable `{{base_path}}` where the resource exists. Exact. + +`templated`, `stale`, `anchor` and `gone` are refused. They need information that +is not in the repo, and a guess there produces a confident link to the wrong page. + +Every rewrite is verified against the disk BEFORE it is written, and the exact +link text must still be present in the file — so a plan that has gone stale +skips rather than corrupts. +""" +import os +import re +import sys +import json +import argparse +import collections + +APPLICABLE = { + "malformed": "link syntax, exact", + "dir_style": "written as a URL; adding `.md` lets mkdocs resolve it", + "depth": "wrong relative depth, exact", + "templated_fixable": "build-time variable where the resource exists, exact", + "renamed": "target moved, proposed with a confidence", +} +REFUSED = { + "templated": "the resource does not exist at that path — may be served by a redirect", + "stale": "points at a pre-migration domain — needs the new equivalent page", + "anchor": "the heading was reworded — needs someone to pick the new one", + "gone": "no target anywhere — was it dropped, missed, or merged?", +} + + +def slug(h): + """python-markdown's toc slugify: strip non-word chars, then collapse runs of + whitespace AND hyphens into one hyphen. Kept identical to check_links.py.""" + h = re.sub(r"`|\*", "", h) + h = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", h) + h = re.sub(r"<[^>]+>", "", h) + h = re.sub(r"[^\w\s-]", "", h).strip().lower() + return re.sub(r"[-\s]+", "-", h) + + +def anchors_of(path): + txt = open(path, encoding="utf-8", errors="replace").read() + txt = re.sub(r"", "", txt, flags=re.S) + txt = re.sub(r"```.*?```", "", txt, flags=re.S) + out = set() + for m in re.finditer(r"^#{1,6}\s+(.+?)\s*$", txt, re.M): + h = m.group(1) + exp = re.search(r"\{#([\w-]+)\}", h) + if exp: + out.add(exp.group(1)) + h = h[:exp.start()] + out.add(slug(h)) + for m in re.finditer(r']+(?:name|id)="([^"]+)"', txt): + out.add(m.group(1)) + for m in re.finditer(r"\{#([\w-]+)\}", txt): + out.add(m.group(1)) + return out + + +def page_id(p): + """Collapse the ways one page can be spelled into a single identity, so + `foo/bar`, `foo/bar.md` and `foo/bar/index.md` compare equal.""" + p = (p or "").rstrip("/") + for suffix in ("/index.md", "/README.md"): + if p.endswith(suffix): + return p[: -len(suffix)] + return p[:-3] if p.endswith(".md") else p + + +def resolve(root, src_rel, target): + """Where does `target`, written in the page `src_rel`, land on disk? + + Returns a repo-relative path that exists, or None. Directory URLs mean a + bare path may be a file, the same file with `.md`, or a directory's index. + """ + target = target.split("#")[0].split("?")[0] + if not target: + return src_rel + base = os.path.dirname(src_rel) + cand = os.path.normpath(os.path.join(base, target)).replace("\\", "/") + for c in (cand, cand + ".md", cand + "/index.md", cand + "/README.md"): + if os.path.isfile(os.path.join(root, c)): + return c + return None + + +def verify(root, entry): + """Can this entry's `suggested` value be trusted? Returns (ok, reason).""" + src = entry["file"] + suggested = entry.get("suggested") + if not suggested: + return False, "no suggested value in the plan" + if not os.path.isfile(os.path.join(root, src)): + return False, "source page no longer exists" + + # An absolute URL or mail link has no on-disk target to check. That is not a + # reason to skip: these appear in `malformed`, where the fix is purely + # syntactic (unwrapping backticks, removing a stray quote) and the address + # itself is untouched. + if re.match(r"^(?:[a-z][a-z0-9+.-]*:)?//|^mailto:", suggested, re.I): + return True, "external address, syntax-only fix" + + frag = suggested.partition("#")[2] + if suggested.startswith("#"): + landed = src # same-page anchor + else: + landed = resolve(root, src, suggested) + if landed is None: + return False, f"suggested target does not resolve: {suggested}" + + # Where the plan already recorded the on-disk target, the two must agree — on + # the *page*, not the spelling. The reporter records some targets in their + # directory-URL form (`foo/bar`) and others as the file (`foo/bar.md`), and + # those name the same page. + recorded = entry.get("resolves_to") or entry.get("found_at") + if recorded and page_id(landed) != page_id(recorded): + return False, f"resolves to {landed}, but the plan recorded {recorded}" + + if frag and landed.endswith(".md"): + if frag not in anchors_of(os.path.join(root, landed)): + return False, f"anchor #{frag} not found in {landed}" + return True, "" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("docs_root", nargs="?", default="en/docs") + ap.add_argument("--plan", required=True, help="The JSON written by report_links.py.") + ap.add_argument("--tier", required=True, + help="One tier name. Run --list to see what a plan holds.") + ap.add_argument("--apply", action="store_true", help="Write changes. Default is a dry run.") + ap.add_argument("--sample", type=int, default=5, + help="How many before/after examples to print (dry run only).") + ap.add_argument("--min-confidence", default="high", choices=["high", "medium", "low"], + help="For `renamed`: the lowest confidence to act on. Default high.") + ap.add_argument("--files", nargs="*", default=None, + help="Limit to these pages (paths as they appear in the plan, or " + "prefixed with the docs root). Use this to try a tier on one " + "page before running it across the scope.") + ap.add_argument("--journal", default=None, + help="Write what changed to this JSON, for review or reverting.") + args = ap.parse_args() + + root = args.docs_root.rstrip("/") + plan = json.load(open(args.plan, encoding="utf-8")) + tiers = plan.get("tiers", {}) + + if args.tier not in tiers: + print(f"No tier {args.tier!r} in {args.plan}. This plan holds:") + for t, items in tiers.items(): + print(f" {t:<20} {len(items):>5}") + return 2 + + if args.tier in REFUSED: + print(f"Refusing to apply `{args.tier}`: {REFUSED[args.tier]}.") + print("These need a person. Work them from the report by hand.") + return 2 + if args.tier not in APPLICABLE: + print(f"`{args.tier}` has no proposed replacements, so there is nothing to apply.") + return 2 + + entries = tiers[args.tier] + + if args.files: + # Accept either form: `en/docs/a/b.md` or the plan's `a/b.md`. + wanted = {f[len(root):].lstrip("/") if f.startswith(root) else f for f in args.files} + before = len(entries) + entries = [e for e in entries if e.get("file") in wanted] + unmatched = wanted - {e.get("file") for e in entries} + print(f"--files: {len(entries)} of {before} entries in this tier match " + f"{len(wanted)} path(s).") + if unmatched: + print(" no entries in this tier for: " + ", ".join(sorted(unmatched))) + if not entries: + return 0 + + rank = {"high": 3, "medium": 2, "low": 1} + if args.tier == "renamed": + floor = rank[args.min_confidence] + held = [e for e in entries if rank.get(e.get("confidence", "low"), 1) < floor] + entries = [e for e in entries if rank.get(e.get("confidence", "low"), 1) >= floor] + else: + held = [] + + # Verify everything first. Nothing is written until the whole tier is checked. + ready, skipped = [], [] + for e in entries: + ok, why = verify(root, e) + (ready if ok else skipped).append(e if ok else (e, why)) + + print("=" * 70) + print(f"LINK FIX tier: {args.tier} ({APPLICABLE[args.tier]})") + print("=" * 70) + print(f"scope in plan : {plan.get('scope', '?')}") + print(f"entries in tier : {len(entries) if args.files else len(tiers[args.tier])}") + if held: + print(f"below confidence : {len(held)} (raise with --min-confidence)") + print(f"verified fixable : {len(ready)}") + print(f"skipped : {len(skipped)}") + + if skipped: + print("\nSkipped, with reasons:") + for e, why in skipped[:10]: + print(f" {e['file']}\n {e.get('link')} -> {why}") + if len(skipped) > 10: + print(f" ... and {len(skipped) - 10} more") + + if not args.apply: + n = min(args.sample, len(ready)) + if n: + print(f"\nSample of {n} of {len(ready)} fixes (nothing written):\n") + for e in ready[:n]: + print(f" {e['file']}") + print(f" - {e['link']}") + print(f" + {e['suggested']}") + if e.get("confidence"): + print(f" confidence: {e['confidence']}") + print() + print(f"To apply these {len(ready)}, re-run with --apply.") + return 0 + + # Group by file so each file is read and written once. + by_file = collections.defaultdict(list) + for e in ready: + by_file[e["file"]].append(e) + + changed_files = 0 + rewrites = 0 + missing, already = [], 0 + journal = [] + for rel, es in sorted(by_file.items()): + path = os.path.join(root, rel) + txt = open(path, encoding="utf-8", errors="replace").read() + orig = txt + done = set() + for e in es: + if e["link"] in done: + # The plan lists one entry per occurrence, and the first rewrite + # replaced every copy of that exact string. Not a problem. + already += 1 + continue + if e["link"] not in txt: + missing.append((rel, e["link"])) + continue + n = txt.count(e["link"]) + txt = txt.replace(e["link"], e["suggested"]) + done.add(e["link"]) + rewrites += n + journal.append({"file": rel, "from": e["link"], + "to": e["suggested"], "occurrences": n}) + if txt != orig: + open(path, "w", encoding="utf-8").write(txt) + changed_files += 1 + + print(f"\nAPPLIED files changed: {changed_files} link occurrences rewritten: {rewrites}") + if already: + print(f" {already} further entries were duplicates of a link already " + f"rewritten in the same file") + if missing: + print(f"\nNot found in the file, so left alone ({len(missing)}) — the plan is " + f"older than the page:") + for rel, link in missing[:10]: + print(f" {rel}: {link}") + + if args.journal: + json.dump(journal, open(args.journal, "w"), indent=1, ensure_ascii=False) + print(f"\njournal -> {args.journal}") + + print("\nReview with `git diff --stat`, then regenerate the plan before the next " + "tier — fixing one tier changes what the others resolve to.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/skills/wso2-doc-frontmatter/scripts/fm_fix.py b/.claude/skills/wso2-doc-frontmatter/scripts/fm_fix.py index f20a75e91..c91dfa490 100644 --- a/.claude/skills/wso2-doc-frontmatter/scripts/fm_fix.py +++ b/.claude/skills/wso2-doc-frontmatter/scripts/fm_fix.py @@ -43,16 +43,45 @@ def sentence_case(s): - """Lowercase a Title Case heading, preserving acronyms and the first word.""" - words = s.split() + """Lowercase a Title Case heading into sentence case. + + Preserved: the first word, acronyms, anything in `TITLE_PROPER`, and any + multi-word product name in `PROPER_PHRASES` ("API Manager", "Developer + Portal"). Those allowlists are shared with `check_style.py` — without them + this function lowercases the very capitals that checker calls correct, and + an H1 of "API Manager Configuration Catalog" becomes the wrong + "API manager configuration catalog". + """ + from fm_lib import TITLE_PROPER, PROPER_PHRASES + + # Mask phrases first: they cannot be decided one word at a time. Match on word + # boundaries — a bare `in` test lets "Rate Limit" swallow "Rate Limiting" and + # preserve a capital that belongs in lower case. + # A trailing "s" is kept, so "Developer Portals" survives as a plural rather than + # being restored as the singular the allowlist happens to spell. + holes = {} + for n, ph in enumerate(sorted(PROPER_PHRASES, key=len, reverse=True)): + pat = r"\b" + re.escape(ph) + r"(s?)\b" + + def _hole(m, n=n): + token = "\x00%d.%d\x00" % (n, len(holes)) + holes[token] = m.group(0) + return token + + s = re.sub(pat, _hole, s) + out = [] - for i, w in enumerate(words): - core = w.strip("()[],.:;") - if i == 0 or core.upper() == core or not re.match(r"^[A-Z][a-z]+$", core): + for i, w in enumerate(s.split()): + core = w.strip("()[],.:;\"'") + if (i == 0 or core in TITLE_PROPER or core.upper() == core + or not re.match(r"^[A-Z][a-z]+$", core)): out.append(w) else: out.append(w[0].lower() + w[1:]) - return " ".join(out) + joined = " ".join(out) + for token, ph in holes.items(): + joined = joined.replace(token, ph) + return joined def derive_tags(rel): diff --git a/.claude/skills/wso2-doc-frontmatter/scripts/fm_lib.py b/.claude/skills/wso2-doc-frontmatter/scripts/fm_lib.py index 0b7b87491..bdbb28262 100644 --- a/.claude/skills/wso2-doc-frontmatter/scripts/fm_lib.py +++ b/.claude/skills/wso2-doc-frontmatter/scripts/fm_lib.py @@ -170,6 +170,86 @@ def version_index(rel): } +# --------------------------------------------------------------------------- +# Capitalisation allowlists. Shared, because two scripts need the same answer: +# `check_style.py` decides whether a capital in a heading is a violation, and +# `fm_fix.py` decides whether to lowercase a word when it derives a `title` from +# an H1. If those disagree, one script lowercases what the other says is correct. +# --------------------------------------------------------------------------- + +SMALL = {"a", "an", "and", "as", "at", "but", "by", "for", "from", "in", "into", "nor", + "of", "on", "onto", "or", "over", "per", "the", "to", "up", "via", "with", + "vs", "near", "off", "out"} + +# Product/proper nouns and acronyms that legitimately stay capitalised mid-heading. +PROPER = {"WSO2", "API", "APIs", "AI", "LLM", "LLMs", "MCP", "REST", "gRPC", "GraphQL", + "JSON", "YAML", "XML", "HTTP", "HTTPS", "OAuth", "OAuth2", "JWT", "SSO", "IdP", + "OIDC", "SAML", "TLS", "mTLS", "SSL", "CORS", "URL", "URI", "URLs", "ID", "IDs", + "UI", "CLI", "SDK", "IDE", "CI", "CD", "VM", "VMs", "K8s", "Kubernetes", + "Docker", "Helm", "Istio", "Envoy", "Redis", "PostgreSQL", "MySQL", "Oracle", + "Grafana", "Prometheus", "Jaeger", "Zipkin", "OpenTelemetry", "OpenSearch", + "Elasticsearch", "Moesif", "Stripe", "AWS", "Azure", "GCP", "Bedrock", "OpenAI", + "Anthropic", "Gemini", "Mistral", "Ollama", "Choreo", "Bijira", "Ballerina", + "Git", "GitHub", "GitLab", "Linux", "Windows", "macOS", "Java", "Python", "Go", + "Node", "npm", "Maven", "Gradle", "Swagger", "OpenAPI", "AsyncAPI", "Postman", + "Portal", "Gateway", "Platform", "Manager", "Publisher", "Developer", + "Control", "Plane", + "Hub", "Workspace", "Analytics", "PII", "RBAC", "ACL", "SLA", "TPS", "QPS", + "DNS", "IP", "TCP", "UDP", "gRPC-Web", "Kafka", "RabbitMQ", "NGINX", "Terraform", + "Ansible", "Prometheus-compatible", "Bitbucket", "Vault", "Keycloak", "Okta", + "Auth0", "Asgardeo", "I", "Step", "Table", "Contents", "Note", "Tip", "Warning", + "Example", "Appendix", "FAQ", "README", "Enumerated", "Values"} + +# Phrase-level allowlist: matched case-sensitively and masked out BEFORE per-word +# scanning, because multi-word product names cannot be expressed as single words. +PROPER_PHRASES = { + # Third-party products / cloud services + "Google Cloud Trace", "Google Cloud Monitoring", "Google Cloud", + "Azure AI Content Safety", "Azure Content Safety Content Moderation", + "Azure Content Safety Guardrail", "Azure Content Safety", "Azure OpenAI", + "AWS Bedrock Guardrails", "AWS Bedrock Guardrail", "Docker Compose", + "OpenSearch Dashboards", "VS Code", "Server-Sent Events", + # Kubernetes API kinds / concepts + "Horizontal Pod Autoscaler", "Pod Disruption Budget", "Custom Resource Definition", + "Service Account", "ConfigMap", "StatefulSet", "DaemonSet", "HTTPRoute", "Gateway API", + # Standards + "JSON Schema Draft 7", "JSON Schema", "JSONPath", + # WSO2 components (capitalised-dominant in the corpus) + "Gateway Controller", "Developer Portal", "Control Plane", "Policy Hub", + "Policy Engine", "Gateway Builder", "Event Gateway", "API Platform Console", + "API Platform", "MCP Proxy", "API Proxy", "Publisher Portal", "Admin Portal", + "API Gateway", + "API Manager", "AI Gateway", "Universal Gateway", "Micro Integrator", + "Service Catalog", "API Product", "API Products", + # --- Policy Hub policy names ------------------------------------------------- + # Treated as proper nouns, matching the Policy Hub catalogue. Remove this block + # if policy names should instead follow sentence case. + "Model Weighted Round Robin", "Model Round Robin", "Sentence Count Guardrail", + "Word Count Guardrail", "Content Length Guardrail", "JSON Schema Guardrail", + "Regex Guardrail", "URL Guardrail", "Semantic Prompt Guard", + "Semantic Tool Filtering", "PII Masking", "Analytics Header Filter", + "Subscription Validation", "API Key Auth", "Basic Auth", "JWT Auth", "Rate Limit", +} + +# Conventional release-stage / status labels never count as Title Case evidence. +STATUS_LABELS = {"beta", "alpha", "ga", "preview", "deprecated", "optional", + "experimental", "recommended", "required", "default", "new", "legacy"} + +# Words in PROPER that are NOT product names on their own. Two kinds: +# document furniture ("Table of Contents", "Step 3"), and generic component words +# that are only capitalised inside a longer name — "Gateway" in "AI Gateway", +# "Portal" in "Developer Portal". `check_style.py` needs them in PROPER so it does +# not flag those longer names word by word, but a title derived from an H1 should +# lowercase them unless the full phrase is present in PROPER_PHRASES. +TITLE_GENERIC = {"Table", "Contents", "Note", "Tip", "Warning", "Example", "Appendix", + "Step", "Values", "Enumerated", "I", + "Gateway", "Portal", "Platform", "Manager", "Publisher", "Developer", + "Control", "Plane", "Hub", "Workspace", "Analytics"} + +# What `fm_fix.py` protects when it sentence-cases an H1 into a `title`. +TITLE_PROPER = PROPER - TITLE_GENERIC + + # Domains the documentation has migrated away from. A link or a frontmatter URL # still pointing at one of these is migration debt regardless of which source repo # it came from, so add to this list rather than editing the checkers when another diff --git a/.claude/skills/wso2-doc-frontmatter/scripts/report_links.py b/.claude/skills/wso2-doc-frontmatter/scripts/report_links.py index b8861b6cb..8dbb3b359 100644 --- a/.claude/skills/wso2-doc-frontmatter/scripts/report_links.py +++ b/.claude/skills/wso2-doc-frontmatter/scripts/report_links.py @@ -33,6 +33,23 @@ HTML_SRC = re.compile(r'<(?:img|a)[^>]+(?:src|href)="([^"]+)"') +def url_base(rel): + """Directory the RENDERED page sits in, under `use_directory_urls: true`. + + `a/b/page.md` is served at `/a/b/page/`, so it is one level DEEPER than the + source file, while `a/b/index.md` is served at `/a/b/` — the same level. + + This matters because mkdocs rewrites relative targets in Markdown syntax + (resolving them against the source file) but passes raw HTML through + untouched, so an `` is resolved by the browser against the rendered + URL instead. The same string is therefore correct in one syntax and broken in + the other, and the two must not be checked the same way. + """ + d = os.path.dirname(rel) + stem = os.path.basename(rel)[:-3] if rel.endswith(".md") else os.path.basename(rel) + return d if stem in ("index", "README") else (f"{d}/{stem}" if d else stem) + + def version_root(rel): """`//a/b.md` -> `/`, else ''.""" ver, _ = split_version(rel) @@ -96,8 +113,8 @@ def slug(h): a.add(m.group(1)) anchors[p] = a - tiers = {k: [] for k in ("templated_fixable", "templated", "malformed", "depth", - "renamed", "gone", "stale", "anchor")} + tiers = {k: [] for k in ("templated_fixable", "templated", "malformed", "dir_style", + "depth", "renamed", "gone", "stale", "anchor")} targets = [p for p in md_list if not args.scope or p.startswith(args.scope)] for p in targets: @@ -108,10 +125,13 @@ def slug(h): stem = os.path.basename(p)[:-3] vroot = version_root(p) - raw = [(m.group(1) == "!", m.group(3)) for m in LINK.finditer(body)] - raw += [(False, m.group(1)) for m in HTML_SRC.finditer(body)] + # The third element records whether the target was written in raw HTML. + # mkdocs rewrites Markdown targets and leaves HTML alone, so the two need + # different bases — see url_base() above. + raw = [(m.group(1) == "!", m.group(3), False) for m in LINK.finditer(body)] + raw += [(False, m.group(1), True) for m in HTML_SRC.finditer(body)] - for is_img, t in raw: + for is_img, t, is_html in raw: if t.startswith(("mailto:", "tel:", "//", "#!")): continue @@ -177,24 +197,56 @@ def slug(h): tiers["anchor"].append({"file": p, "link": t, "target_file": p, "anchor": frag}) continue - src_rel = os.path.normpath(os.path.join(d, path)).replace("\\", "/") - if resolves(src_rel): + # WHICH BASE APPLIES — verified against a real mkdocs build. + # + # mkdocs rewrites a Markdown target only when the literal path names a + # file that exists; then it is resolved against the SOURCE directory. + # Everything else — raw HTML, directory-style links (`../foo/bar/`), + # extensionless links (`../foo/bar`) — is passed through verbatim and + # resolved by the browser against the RENDERED URL, one level deeper. + if is_html: + own_base, alt_base = url_base(p), d + rewritten = False + else: + literal = os.path.normpath(os.path.join(d, path)).replace("\\", "/") + rewritten = literal in all_files + own_base, alt_base = (d, url_base(p)) if rewritten else (url_base(p), d) + + own_rel = os.path.normpath(os.path.join(own_base, path)).replace("\\", "/") + if resolves(own_rel): if frag: - tf = next((c for c in (src_rel, src_rel + ".md", src_rel + "/index.md", - src_rel + "/README.md") if c in all_files), None) + tf = next((c for c in (own_rel, own_rel + ".md", own_rel + "/index.md", + own_rel + "/README.md") if c in all_files), None) if tf and tf.endswith(".md") and frag not in anchors.get(tf, set()): tiers["anchor"].append({"file": p, "link": t, "target_file": tf, "anchor": frag}) continue - # Directory-URL semantics: the rendered page sits one level deeper - # than the source file, so a link written against the *URL* needs one - # fewer `../` to be correct against the source. - url_rel = os.path.normpath(os.path.join(d, stem, path)).replace("\\", "/") - if resolves(url_rel): - fixed = os.path.relpath(url_rel, d).replace("\\", "/") - if url_rel + ".md" in all_files: + # A Markdown link written in URL shape, whose `.md` file sits exactly + # where the link already points. Adding the extension is the real fix, + # not adding a `../`: mkdocs then owns the depth calculation and the link + # keeps working when the page moves. Checked BEFORE the depth tier so the + # brittle fix never wins. + if not is_html and not rewritten: + src_guess = os.path.normpath(os.path.join(d, path)).replace("\\", "/") + md_target = next((c for c in (src_guess + ".md", src_guess + "/index.md", + src_guess + "/README.md") if c in all_files), None) + if md_target: + fixed = os.path.relpath(md_target, d).replace("\\", "/") + tiers["dir_style"].append({ + "file": p, "link": t, "resolves_to": md_target, + "suggested": fixed + (("#" + frag) if frag else ""), + "why": "written as a URL, so mkdocs passes it through unresolved"}) + continue + + # Resolves against the OTHER base — so the depth is wrong by exactly the + # one level between a source file and its rendered directory. + alt_rel = os.path.normpath(os.path.join(alt_base, path)).replace("\\", "/") + if resolves(alt_rel): + fixed = os.path.relpath(alt_rel, own_base).replace("\\", "/") + if not is_html and alt_rel + ".md" in all_files: fixed += ".md" - tiers["depth"].append({"file": p, "link": t, "resolves_to": url_rel, + tiers["depth"].append({"file": p, "link": t, "resolves_to": alt_rel, + "is_html": is_html, "suggested": fixed + (("#" + frag) if frag else "")}) continue @@ -242,7 +294,7 @@ def slug(h): n = {k: len(v) for k, v in tiers.items()} total = sum(n.values()) scope_label = args.scope or f"all of {root}" - auto = (n["templated_fixable"] + n["malformed"] + n["depth"] + auto = (n["templated_fixable"] + n["malformed"] + n["dir_style"] + n["depth"] + len([x for x in tiers["renamed"] if x["confidence"] == "high"])) L = [] @@ -253,19 +305,34 @@ def slug(h): f"**{auto}** have an exact or high-confidence mechanical fix; " f"**{n['gone']}** need a human decision.") w("") - w("| Tier | Cause | Count | Fixable how |") - w("|---|---|---|---|") - w(f"| 0 | `{{{{base_path}}}}` where the resource exists | {n['templated_fixable']} | Exact rewrite to a relative path |") - w(f"| — | `{{{{base_path}}}}` where it does not | {n['templated']} | **Leave alone** — may be a redirect |") - w(f"| 0 | Malformed link syntax | {n['malformed']} | Exact rewrite — no judgement |") - w(f"| 1 | Wrong relative depth | {n['depth']} | Exact rewrite — no judgement |") - w(f"| 2 | Renamed or moved target | {n['renamed']} | Proposed target, check confidence |") - w(f"| 3 | Pre-migration domain | {n['stale']} | Map to new site, or drop |") - w(f"| 4 | Missing anchor | {n['anchor']} | Heading was reworded |") - w(f"| 5 | No target anywhere | {n['gone']} | Human decision — cannot automate |") + # Groups are named, not numbered. The name is the value `fix_links.py --tier` + # takes, so a row in this table is directly runnable. Numbering them invited + # the obvious question of why two groups shared a number and one had none. + w("### Fixable by script") + w("") + w("Run in this order. Each is a separate `fix_links.py --tier` run, and every " + "rewrite is verified against the files on disk before it is written.") + w("") + w("| Order | Group | Cause | Count | Fix |") + w("|---|---|---|---|---|") + w(f"| 1 | `malformed` | Malformed link syntax | {n['malformed']} | Exact — no judgement |") + w(f"| 2 | `dir_style` | Written as a URL, so mkdocs never resolves it | {n['dir_style']} | Add `.md` — mkdocs then owns the depth |") + w(f"| 3 | `depth` | Wrong relative depth | {n['depth']} | Exact — no judgement |") + w(f"| 4 | `renamed` | Renamed or moved target | {n['renamed']} | Proposed; `high` confidence applied by default |") + w(f"| 5 | `templated_fixable` | `{{{{base_path}}}}` where the resource exists | {n['templated_fixable']} | Exact rewrite to a relative path |") w("") - w("Work the tiers in order. Tier 1 is safe to apply in bulk; tier 5 is the only " - "one that needs someone who knows what the page was supposed to say.") + w("### Needs a person") + w("") + w("`fix_links.py` refuses these. The information needed is not in the repository, " + "and a guess produces a confident link to the wrong page — worse than a visibly " + "broken one, because nobody re-checks it.") + w("") + w("| Group | Cause | Count | Why it cannot be automated |") + w("|---|---|---|---|") + w(f"| `templated` | `{{{{base_path}}}}` where the resource does not exist | {n['templated']} | May be served by a redirect |") + w(f"| `stale` | Pre-migration domain | {n['stale']} | Needs the equivalent page on the new site |") + w(f"| `anchor` | Missing anchor | {n['anchor']} | The heading was reworded — which one now? |") + w(f"| `gone` | No target anywhere | {n['gone']} | Was it dropped, missed, or merged? |") w("") def table(rows, cols, keys, limit): @@ -280,7 +347,7 @@ def table(rows, cols, keys, limit): w("") if n["templated_fixable"]: - w("## Tier 0 — `{{base_path}}` where the resource exists") + w("## `templated_fixable` — `{{base_path}}` where the resource exists") w("") w("`{{base_path}}` stands for the root of the version's site, so the rest of the " "target is a path within that version's directory. For these, the resource is " @@ -301,7 +368,7 @@ def table(rows, cols, keys, limit): ["file", "link", "variable"], args.max_rows) if n["malformed"]: - w("## Tier 0 — Malformed link syntax") + w("## `malformed` — Malformed link syntax") w("") w("The link target is not a valid path or URL, so it renders as literal broken text " "regardless of whether the destination exists. Carried over from the old wiki. " @@ -310,8 +377,21 @@ def table(rows, cols, keys, limit): table(tiers["malformed"], ["Page", "Currently", "Change to", "Why"], ["file", "link", "suggested", "why"], args.max_rows) + if n["dir_style"]: + w("## `dir_style` — Written as a URL, so mkdocs never resolves it") + w("") + w("These point at the right page already. Because the target is written in URL " + "shape rather than naming the `.md` file, mkdocs passes it through untouched and " + "the browser resolves it against the rendered page URL — one directory deeper " + "than the source file, so it lands one level short. Adding the extension hands " + "the depth calculation back to mkdocs, permanently.") + w("") + table(tiers["dir_style"], ["Page", "Currently", "Change to"], + ["file", "link", "suggested"], args.max_rows) + w("") + if n["depth"]: - w("## Tier 1 — Wrong relative depth") + w("## `depth` — Wrong relative depth") w("") w("The target exists; the path has one `../` too many. These render correctly in a " "browser (the published URL sits one directory deeper than the source file), so they " @@ -324,7 +404,7 @@ def table(rows, cols, keys, limit): if n["renamed"]: hi = [x for x in tiers["renamed"] if x["confidence"] == "high"] lo = [x for x in tiers["renamed"] if x["confidence"] != "high"] - w("## Tier 2 — Renamed or moved target") + w("## `renamed` — Renamed or moved target") w("") w("The target does not exist at the path written, but a file of the same name exists " "elsewhere under the same version. This is the restructure: directories were renamed " @@ -341,7 +421,7 @@ def table(rows, cols, keys, limit): ["file", "link", "suggested", "confidence"], args.max_rows) if n["stale"]: - w("## Tier 3 — Links to the pre-migration site") + w("## `stale` — Links to the pre-migration site") w("") w("These point at a location the documentation has migrated away from. For each " "one: find the equivalent page on the new site and link to it relatively, or if the " @@ -351,7 +431,7 @@ def table(rows, cols, keys, limit): table(tiers["stale"], ["Page", "Link"], ["file", "link"], args.max_rows) if n["anchor"]: - w("## Tier 4 — Missing anchor") + w("## `anchor` — Missing anchor") w("") w("The page resolves but the `#fragment` matches no heading, so the reader lands at the " "top instead of the section. Usually the heading was reworded. Open the target, find " @@ -361,7 +441,7 @@ def table(rows, cols, keys, limit): ["file", "link", "target_file", "anchor"], args.max_rows) if n["gone"]: - w("## Tier 5 — No target anywhere") + w("## `gone` — No target anywhere") w("") w("No file of this name exists anywhere under the docs root, so there is nothing to " "point at. Each needs a decision: was the page meant to be migrated and missed, was it " @@ -377,8 +457,11 @@ def table(rows, cols, keys, limit): w("## Prompt for an AI coding agent") w("") w("Paste the block below to an agent working in the repo root. It is deliberately scoped to " - "tiers 1 and 2-high — the tiers with a defensible mechanical answer. Tiers 3 to 5 need " - "judgement and are left out on purpose.") + "the four groups with a defensible mechanical answer. `templated`, `stale`, `anchor` and " + "`gone` need judgement and are left out on purpose.") + w("") + w("Alternatively, run `fix_links.py --tier ` yourself — same scope, one group at a " + "time, and every rewrite verified against the files on disk before it is written.") w("") w("````text") w(f"You are fixing broken links in the WSO2 API Platform docs, scope: {scope_label}.") @@ -386,11 +469,12 @@ def table(rows, cols, keys, limit): w(f"Read the fix plan in `{args.out}`" + (f" and the machine-readable list in `{args.json_out}`." if args.json_out else ".")) w("") - w("Apply ONLY these tiers:") - w(" - Tier 0 (`{{base_path}}` where the resource exists): apply every row as given.") - w(" - Tier 0 (Malformed link syntax): apply every row exactly as given.") - w(" - Tier 1 (Wrong relative depth): apply every row exactly as given.") - w(" - Tier 2, high-confidence subsection only: apply every row as given.") + w("Apply ONLY these groups:") + w(" - `templated_fixable`: apply every row as given.") + w(" - `malformed`: apply every row exactly as given.") + w(" - `dir_style`: apply every row exactly as given (adds `.md`).") + w(" - `depth`: apply every row exactly as given.") + w(" - `renamed`, the high-confidence subsection only: apply every row as given.") w("") w("Rules:") w(" 1. Replace only the link target inside the parentheses. Never change the link TEXT,") @@ -398,9 +482,8 @@ def table(rows, cols, keys, limit): w(" 2. A target may appear more than once in a file — replace every occurrence of that") w(" exact target in that file.") w(" 3. Preserve any `#fragment` already on the link unless the plan says otherwise.") - w(" 4. Do NOT touch tiers 3, 4, or 5, and do NOT touch anything in the") - w(" \"`{{base_path}}` where the resource does not exist\" section. Do not") - w(" invent a target that is not in the plan.") + w(" 4. Do NOT touch `templated`, `stale`, `anchor` or `gone`. Do not invent a") + w(" target that is not in the plan.") w(" 5. Do not reformat, reflow, or reorder anything. Minimal diffs only.") w("") w("Verify when done, from the repo root:") @@ -410,10 +493,10 @@ def table(rows, cols, keys, limit): w("The blocking count must go DOWN and no new codes may appear. If any count rises, stop") w("and report what you changed rather than continuing.") w("") - w("Then report: rows applied per tier, files touched, and the before/after blocking counts.") + w("Then report: rows applied per group, files touched, and the before/after blocking counts.") w("````") w("") - w("### Why tiers 3 to 5 are excluded") + w("### Why `templated`, `stale`, `anchor` and `gone` are excluded") w("") w("Each needs information that isn't in the repo: which new page replaces an old-site link, " "which reworded heading was meant, whether a missing page was dropped on purpose. An agent " From 658a0ff96171493081be0dc184d13a84e198f344 Mon Sep 17 00:00:00 2001 From: Praveena Hennadige Date: Fri, 7 Aug 2026 17:07:19 +0530 Subject: [PATCH 2/2] Match single-quoted HTML attributes --- .../wso2-doc-frontmatter/scripts/check_links.py | 11 ++++++++--- .../skills/wso2-doc-frontmatter/scripts/fix_links.py | 4 ++-- .../wso2-doc-frontmatter/scripts/report_links.py | 11 +++++++---- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.claude/skills/wso2-doc-frontmatter/scripts/check_links.py b/.claude/skills/wso2-doc-frontmatter/scripts/check_links.py index f861dcbad..26d90312b 100644 --- a/.claude/skills/wso2-doc-frontmatter/scripts/check_links.py +++ b/.claude/skills/wso2-doc-frontmatter/scripts/check_links.py @@ -79,7 +79,7 @@ def toc_depth(mkdocs_yml): a.add(exp.group(1)) # explicit id survives any toc_depth h = h[:exp.start()] (a if level <= TOC_DEPTH else deep).add(slug(h)) - for m in re.finditer(r']+(?:name|id)="([^"]+)"', txt): a.add(m.group(1)) + for m in re.finditer(r"""]+(?:name|id)=(["'])(.*?)\1""", txt): a.add(m.group(2)) for m in re.finditer(r'\{#([\w-]+)\}', txt): a.add(m.group(1)) anchors[p] = a deep_anchors[p] = deep - a @@ -88,7 +88,12 @@ def toc_depth(mkdocs_yml): # Tag name is captured so an `` is reported as a broken LINK and an # `` as a missing IMAGE. Lumping them together mislabels every raw-HTML # link as an image, which sends whoever reads the report looking for the wrong thing. -HTML_SRC = re.compile(r'<(img|a|source|iframe)[^>]+(?:src|href)="([^"]+)"') +# +# The quote character is captured and back-referenced, so single-quoted attributes +# are matched too. HTML allows either, the migrated pages use both, and a +# double-quote-only pattern skips the single-quoted ones silently — they look +# checked when they were never read. +HTML_SRC = re.compile(r"""<(img|a|source|iframe)[^>]+(?:src|href)=(["'])(.*?)\2""") def url_base(rel): @@ -120,7 +125,7 @@ def add(f, sev, code, msg): d = os.path.dirname(p) targets = [(m.group(1) == "!", m.group(3), False) for m in LINK.finditer(body)] - targets += [(m.group(1).lower() != "a", m.group(2), True) for m in HTML_SRC.finditer(body)] + targets += [(m.group(1).lower() != "a", m.group(3), True) for m in HTML_SRC.finditer(body)] for is_img, t, is_html in targets: if t.startswith(("mailto:", "tel:", "#!")): diff --git a/.claude/skills/wso2-doc-frontmatter/scripts/fix_links.py b/.claude/skills/wso2-doc-frontmatter/scripts/fix_links.py index 8a072d82a..5478b158f 100644 --- a/.claude/skills/wso2-doc-frontmatter/scripts/fix_links.py +++ b/.claude/skills/wso2-doc-frontmatter/scripts/fix_links.py @@ -73,8 +73,8 @@ def anchors_of(path): out.add(exp.group(1)) h = h[:exp.start()] out.add(slug(h)) - for m in re.finditer(r']+(?:name|id)="([^"]+)"', txt): - out.add(m.group(1)) + for m in re.finditer(r"""]+(?:name|id)=(["'])(.*?)\1""", txt): + out.add(m.group(2)) for m in re.finditer(r"\{#([\w-]+)\}", txt): out.add(m.group(1)) return out diff --git a/.claude/skills/wso2-doc-frontmatter/scripts/report_links.py b/.claude/skills/wso2-doc-frontmatter/scripts/report_links.py index 8dbb3b359..652e10ec4 100644 --- a/.claude/skills/wso2-doc-frontmatter/scripts/report_links.py +++ b/.claude/skills/wso2-doc-frontmatter/scripts/report_links.py @@ -30,7 +30,10 @@ from fm_lib import split_version, is_legacy_url # noqa: E402 LINK = re.compile(r'(!?)\[([^\]]*)\]\(\s*]+)>?(?:\s+"[^"]*")?\s*\)') -HTML_SRC = re.compile(r'<(?:img|a)[^>]+(?:src|href)="([^"]+)"') +# Quote character captured and back-referenced: HTML allows single or double +# quotes and these pages use both, so a double-quote-only pattern silently skips +# whatever is single-quoted. +HTML_SRC = re.compile(r"""<(img|a|source|iframe)[^>]+(?:src|href)=(["'])(.*?)\2""") def url_base(rel): @@ -109,8 +112,8 @@ def slug(h): a.add(exp.group(1)) h = h[: exp.start()] a.add(slug(h)) - for m in re.finditer(r']+(?:name|id)="([^"]+)"', txt): - a.add(m.group(1)) + for m in re.finditer(r"""]+(?:name|id)=(["'])(.*?)\1""", txt): + a.add(m.group(2)) anchors[p] = a tiers = {k: [] for k in ("templated_fixable", "templated", "malformed", "dir_style", @@ -129,7 +132,7 @@ def slug(h): # mkdocs rewrites Markdown targets and leaves HTML alone, so the two need # different bases — see url_base() above. raw = [(m.group(1) == "!", m.group(3), False) for m in LINK.finditer(body)] - raw += [(False, m.group(1), True) for m in HTML_SRC.finditer(body)] + raw += [(m.group(1).lower() != "a", m.group(3), True) for m in HTML_SRC.finditer(body)] for is_img, t, is_html in raw: if t.startswith(("mailto:", "tel:", "//", "#!")):