Skip to content

Use rocm-docs-core's llms.txt generator #128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
j-stephan wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Use rocm-docs-core's llms.txt generator #128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 4 additions & 222 deletions docs/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
"""Configuration file for the Sphinx documentation builder."""
import os
import re
from pathlib import Path

external_projects_remote_repository = ""
external_projects_current_project = "dcgpu"
Expand Down Expand Up @@ -50,225 +48,10 @@

exclude_patterns = ['.venv']

html_extra_path = ["llms.txt"]

EXCLUDED_DIRS = {
"_build",
"_templates",
"_static",
".git",
".venv",
}

MARKUP_PREFIXES = (
":::",
"```{",
"```",
":img-top:",
":class",
":link:",
":link-type:",
":shadow:",
":columns:",
":padding:",
":gutter:",
":open:",
":name:",
":header-rows:",
":alt:",
"+++",
"-->",
"{bdg-",
)

# Matches lines like "align: center", "alt:", "name: foo" (directive options
# not starting with a colon, common in MyST figure/table fences)
_BARE_DIRECTIVE_RE = re.compile(r"^[a-z][a-z_-]*:\s*\S*$")

# Matches MyST/RST anchor labels like "(gpu-arch-documentation)="
_ANCHOR_LABEL_RE = re.compile(r"^\(\w[\w-]*\)=$")

# Matches RST section underlines (e.g. "====", "----", "~~~~")
_RST_UNDERLINE_RE = re.compile(r"^[=\-~^\"\'#*+]{3,}$")

# Matches RST code block directives (e.g. ".. code-block:: cpp", ".. code:: sh")
_RST_CODE_BLOCK_RE = re.compile(r"^\.\.\s+(code-block|code|sourcecode)::")

# Matches markdown table separator rows (e.g. "|---|---|", "| :--- | ---: |").
_MD_TABLE_SEP_RE = re.compile(r"^\|[\s|:\-]+\|$")

# Matches RST directives whose indented body should be discarded (e.g. raw HTML).
_RST_SKIP_BLOCK_RE = re.compile(r"^\.\.\s+raw::")

# Matches HTML tags (e.g. "<div>", "</p>", "<!--") but NOT RST hyperlink URL
# continuation lines (e.g. "<https://...>`_"). The negative lookahead excludes
# URL schemes so that multi-line RST inline hyperlinks are preserved.
_HTML_TAG_RE = re.compile(r"^<(?!https?://|ftp://|mailto:)[a-zA-Z/!]")

# Matches trailing HTML close tags at the end of a prose line
# (e.g. "Browse blogs.</p>", "See the guide.</li></ul>").
_TRAILING_HTML_CLOSE_RE = re.compile(r"(</[a-zA-Z]+>)+\s*$")

MIN_PROSE_LINES = 10


def should_skip(path: Path) -> bool:
return any(part in EXCLUDED_DIRS for part in path.parts)


def is_prose_line(line: str) -> bool:
stripped = line.strip()
if not stripped:
return False
if stripped.startswith(MARKUP_PREFIXES):
return False
# Drop bare directive-option lines (e.g. "align: center", "alt:")
if _BARE_DIRECTIVE_RE.match(stripped):
return False
# Drop MyST/RST anchor labels (e.g. "(gpu-arch-documentation)=")
if _ANCHOR_LABEL_RE.match(stripped):
return False
# Drop markdown table separator rows (e.g. "|---|---|", "| :--- | ---: |")
if _MD_TABLE_SEP_RE.match(stripped):
return False
# Drop HTML tags (e.g. "<div>", "</p>") but keep RST hyperlink URL
# continuation lines (e.g. "<https://rocm.docs.amd.com/...>`_")
if _HTML_TAG_RE.match(stripped):
return False
# Drop RST directives, comments, hyperlink targets, and substitution definitions
if stripped.startswith(".."):
return False
# Drop YAML frontmatter key-value pairs (e.g. "description lang=en": "text")
if stripped.startswith('"') and re.match(r'^"[^"]+"\s*:', stripped):
return False
# Drop RST field list items (e.g. ":type: int") and extended RST meta
# options (e.g. ":description lang=en: text"). Excludes inline roles at line
# start (e.g. ":cpp:func:`hipMalloc` returns..." or ":ref:`foo <bar>` describes...")
# because those are followed by a backtick, not a space or end-of-line.
if re.match(r"^:[A-Za-z][A-Za-z0-9_ =-]*:(\s|$)", stripped):
return False
# Drop RST section underlines (e.g. "====", "----", "~~~~")
if _RST_UNDERLINE_RE.match(stripped):
return False
return True


def generate_combined_markdown(app, exception):
if exception:
return

docs_root = Path(app.srcdir)
output_file = Path(app.outdir) / "llms-full.txt"
base_file = docs_root / "llms.txt"

combined = []

if base_file.exists():
base_text = base_file.read_text(encoding="utf-8").rstrip().rstrip("-").rstrip()
combined.append(base_text)
else:
combined.append("# AMD Instinct Data Center GPU Documentation")

all_files = sorted(
list(docs_root.rglob("*.md")) + list(docs_root.rglob("*.rst"))
)

for doc_file in all_files:
if should_skip(doc_file):
continue

if doc_file == base_file:
continue

try:
content = doc_file.read_text(encoding="utf-8")
except Exception:
continue

lines = content.splitlines()
prose_lines = [line for line in lines if is_prose_line(line)]

if len(prose_lines) < MIN_PROSE_LINES:
continue

relative = doc_file.relative_to(docs_root)
in_backtick_fence = False
in_rst_code_block = False
in_rst_skip_block = False
in_html_comment = False # inside <!-- ... --> block
in_html_open_tag = False # inside a multi-line HTML opening tag
kept = []
for line in lines:
stripped = line.strip()
# Backtick fences (MyST/Markdown)
if stripped.startswith("```"):
in_backtick_fence = not in_backtick_fence
kept.append(line)
continue
if in_backtick_fence:
kept.append(line)
continue
# HTML comment block (<!-- ... -->): discard all content until -->
if in_html_comment:
if "-->" in stripped:
in_html_comment = False
continue
# RST skip block (e.g. .. raw::): discard all indented content
if in_rst_skip_block:
if not stripped or line[0] in (" ", "\t"):
continue
in_rst_skip_block = False
# RST code block: exit when a non-blank, non-indented line appears
if in_rst_code_block:
if not stripped or line[0] in (" ", "\t"):
kept.append(line)
continue
in_rst_code_block = False
# RST raw block: enter and discard both the directive and its body
if _RST_SKIP_BLOCK_RE.match(stripped):
in_rst_skip_block = True
continue
# RST code block: enter on directive line (directive itself is dropped)
if _RST_CODE_BLOCK_RE.match(stripped):
in_rst_code_block = True
continue
# HTML comment open (<!-- ... -->): discard opener and enter state
if stripped.startswith("<!--"):
if "-->" not in stripped:
in_html_comment = True
continue
# Multi-line HTML opening tag: skip continuation lines until >
if in_html_open_tag:
if ">" in stripped:
in_html_open_tag = False
continue
# Detect HTML opening tags that wrap across lines (no > on this line)
if _HTML_TAG_RE.match(stripped) and ">" not in stripped:
in_html_open_tag = True
continue
if not stripped:
kept.append(line)
elif is_prose_line(line):
# Strip trailing HTML close tags (e.g. "See the guide.</p>")
cleaned = _TRAILING_HTML_CLOSE_RE.sub("", line).rstrip()
cleaned_stripped = cleaned.strip()
if not cleaned_stripped:
# Entire line was HTML close tags — keep original (shouldn't
# normally reach here since _is_prose_line filters HTML).
kept.append(line)
elif re.search(r"\w", cleaned_stripped):
# Line has real word content after stripping close tags.
kept.append(cleaned)
# else: only punctuation remains (e.g. bare ".") — discard.
cleaned = "\n".join(kept)

combined.append(f"\n\n---\n\n# {relative}\n")
combined.append(cleaned.strip())

output_file.write_text(
"\n".join(combined) + "\n",
encoding="utf-8",
)
# Generate llms.txt and llms-full.txt after each build (the llms.txt standard,
# https://llmstxt.org/). See the rocm-docs-core guide:
# https://rocm.docs.amd.com/projects/rocm-docs-core/en/latest/user_guide/llms.html
rocm_docs_generate_llms = True

# Pages (by docname) whose primary (left) sidebar should be collapsed by default.
# #pst-primary-sidebar-checkbox restores the sidebar.
Expand Down Expand Up @@ -302,4 +85,3 @@ def collapse_primary_sidebar(app, pagename, templatename, context, doctree):
def setup(app):
app.add_css_file("css/index.css")
app.connect("html-page-context", collapse_primary_sidebar)
app.connect("build-finished", generate_combined_markdown)
83 changes: 0 additions & 83 deletions docs/llms.txt
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion docs/sphinx/requirements.in
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
rocm-docs-core==1.38.0
rocm-docs-core[llms]==1.38.0
sphinx-reredirects
Loading
Loading