From 4a6944fbf2549585a00c50773681895813634729 Mon Sep 17 00:00:00 2001
From: Istvan Kiss <istvan.kiss@amd.com>
Date: Fri, 31 Jul 2026 11:17:22 +0200
Subject: [PATCH] Use rocm-docs-core's llms.txt generator

Signed-off-by: Istvan Kiss <istvan.kiss@amd.com>
---
 docs/conf.py                 | 226 +----------------------------------
 docs/llms.txt                |  83 -------------
 docs/sphinx/requirements.in  |   2 +-
 docs/sphinx/requirements.txt |  38 +++---
 4 files changed, 27 insertions(+), 322 deletions(-)
 delete mode 100644 docs/llms.txt

diff --git a/docs/conf.py b/docs/conf.py
index a3ff4bc..ef87cae 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,7 +1,5 @@
 """Configuration file for the Sphinx documentation builder."""
 import os
-import re
-from pathlib import Path
 
 external_projects_remote_repository = ""
 external_projects_current_project = "dcgpu"
@@ -50,225 +48,10 @@
 
 exclude_patterns = ['.venv']
 
-html_extra_path = ["llms.txt"]
-
-EXCLUDED_DIRS = {
-    "_build",
-    "_templates",
-    "_static",
-    ".git",
-    ".venv",
-}
-
-MARKUP_PREFIXES = (
-    ":::",
-    "```{",
-    "```",
-    ":img-top:",
-    ":class",
-    ":link:",
-    ":link-type:",
-    ":shadow:",
-    ":columns:",
-    ":padding:",
-    ":gutter:",
-    ":open:",
-    ":name:",
-    ":header-rows:",
-    ":alt:",
-    "+++",
-    "-->",
-    "{bdg-",
-)
-
-# Matches lines like "align: center", "alt:", "name: foo" (directive options
-# not starting with a colon, common in MyST figure/table fences)
-_BARE_DIRECTIVE_RE = re.compile(r"^[a-z][a-z_-]*:\s*\S*$")
-
-# Matches MyST/RST anchor labels like "(gpu-arch-documentation)="
-_ANCHOR_LABEL_RE = re.compile(r"^\(\w[\w-]*\)=$")
-
-# Matches RST section underlines (e.g. "====", "----", "~~~~")
-_RST_UNDERLINE_RE = re.compile(r"^[=\-~^\"\'#*+]{3,}$")
-
-# Matches RST code block directives (e.g. ".. code-block:: cpp", ".. code:: sh")
-_RST_CODE_BLOCK_RE = re.compile(r"^\.\.\s+(code-block|code|sourcecode)::")
-
-# Matches markdown table separator rows (e.g. "|---|---|", "| :--- | ---: |").
-_MD_TABLE_SEP_RE = re.compile(r"^\|[\s|:\-]+\|$")
-
-# Matches RST directives whose indented body should be discarded (e.g. raw HTML).
-_RST_SKIP_BLOCK_RE = re.compile(r"^\.\.\s+raw::")
-
-# Matches HTML tags (e.g. "<div>", "</p>", "<!--") but NOT RST hyperlink URL
-# continuation lines (e.g. "<https://...>`_").  The negative lookahead excludes
-# URL schemes so that multi-line RST inline hyperlinks are preserved.
-_HTML_TAG_RE = re.compile(r"^<(?!https?://|ftp://|mailto:)[a-zA-Z/!]")
-
-# Matches trailing HTML close tags at the end of a prose line
-# (e.g. "Browse blogs.</p>", "See the guide.</li></ul>").
-_TRAILING_HTML_CLOSE_RE = re.compile(r"(</[a-zA-Z]+>)+\s*$")
-
-MIN_PROSE_LINES = 10
-
-
-def should_skip(path: Path) -> bool:
-    return any(part in EXCLUDED_DIRS for part in path.parts)
-
-
-def is_prose_line(line: str) -> bool:
-    stripped = line.strip()
-    if not stripped:
-        return False
-    if stripped.startswith(MARKUP_PREFIXES):
-        return False
-    # Drop bare directive-option lines (e.g. "align: center", "alt:")
-    if _BARE_DIRECTIVE_RE.match(stripped):
-        return False
-    # Drop MyST/RST anchor labels (e.g. "(gpu-arch-documentation)=")
-    if _ANCHOR_LABEL_RE.match(stripped):
-        return False
-    # Drop markdown table separator rows (e.g. "|---|---|", "| :--- | ---: |")
-    if _MD_TABLE_SEP_RE.match(stripped):
-        return False
-    # Drop HTML tags (e.g. "<div>", "</p>") but keep RST hyperlink URL
-    # continuation lines (e.g. "<https://rocm.docs.amd.com/...>`_")
-    if _HTML_TAG_RE.match(stripped):
-        return False
-    # Drop RST directives, comments, hyperlink targets, and substitution definitions
-    if stripped.startswith(".."):
-        return False
-    # Drop YAML frontmatter key-value pairs (e.g. "description lang=en": "text")
-    if stripped.startswith('"') and re.match(r'^"[^"]+"\s*:', stripped):
-        return False
-    # Drop RST field list items (e.g. ":type: int") and extended RST meta
-    # options (e.g. ":description lang=en: text"). Excludes inline roles at line
-    # start (e.g. ":cpp:func:`hipMalloc` returns..." or ":ref:`foo <bar>` describes...")
-    # because those are followed by a backtick, not a space or end-of-line.
-    if re.match(r"^:[A-Za-z][A-Za-z0-9_ =-]*:(\s|$)", stripped):
-        return False
-    # Drop RST section underlines (e.g. "====", "----", "~~~~")
-    if _RST_UNDERLINE_RE.match(stripped):
-        return False
-    return True
-
-
-def generate_combined_markdown(app, exception):
-    if exception:
-        return
-
-    docs_root = Path(app.srcdir)
-    output_file = Path(app.outdir) / "llms-full.txt"
-    base_file = docs_root / "llms.txt"
-
-    combined = []
-
-    if base_file.exists():
-        base_text = base_file.read_text(encoding="utf-8").rstrip().rstrip("-").rstrip()
-        combined.append(base_text)
-    else:
-        combined.append("# AMD Instinct Data Center GPU Documentation")
-
-    all_files = sorted(
-        list(docs_root.rglob("*.md")) + list(docs_root.rglob("*.rst"))
-    )
-
-    for doc_file in all_files:
-        if should_skip(doc_file):
-            continue
-
-        if doc_file == base_file:
-            continue
-
-        try:
-            content = doc_file.read_text(encoding="utf-8")
-        except Exception:
-            continue
-
-        lines = content.splitlines()
-        prose_lines = [line for line in lines if is_prose_line(line)]
-
-        if len(prose_lines) < MIN_PROSE_LINES:
-            continue
-
-        relative = doc_file.relative_to(docs_root)
-        in_backtick_fence = False
-        in_rst_code_block = False
-        in_rst_skip_block = False
-        in_html_comment = False  # inside <!-- ... --> block
-        in_html_open_tag = False  # inside a multi-line HTML opening tag
-        kept = []
-        for line in lines:
-            stripped = line.strip()
-            # Backtick fences (MyST/Markdown)
-            if stripped.startswith("```"):
-                in_backtick_fence = not in_backtick_fence
-                kept.append(line)
-                continue
-            if in_backtick_fence:
-                kept.append(line)
-                continue
-            # HTML comment block (<!-- ... -->): discard all content until -->
-            if in_html_comment:
-                if "-->" in stripped:
-                    in_html_comment = False
-                continue
-            # RST skip block (e.g. .. raw::): discard all indented content
-            if in_rst_skip_block:
-                if not stripped or line[0] in (" ", "\t"):
-                    continue
-                in_rst_skip_block = False
-            # RST code block: exit when a non-blank, non-indented line appears
-            if in_rst_code_block:
-                if not stripped or line[0] in (" ", "\t"):
-                    kept.append(line)
-                    continue
-                in_rst_code_block = False
-            # RST raw block: enter and discard both the directive and its body
-            if _RST_SKIP_BLOCK_RE.match(stripped):
-                in_rst_skip_block = True
-                continue
-            # RST code block: enter on directive line (directive itself is dropped)
-            if _RST_CODE_BLOCK_RE.match(stripped):
-                in_rst_code_block = True
-                continue
-            # HTML comment open (<!-- ... -->): discard opener and enter state
-            if stripped.startswith("<!--"):
-                if "-->" not in stripped:
-                    in_html_comment = True
-                continue
-            # Multi-line HTML opening tag: skip continuation lines until >
-            if in_html_open_tag:
-                if ">" in stripped:
-                    in_html_open_tag = False
-                continue
-            # Detect HTML opening tags that wrap across lines (no > on this line)
-            if _HTML_TAG_RE.match(stripped) and ">" not in stripped:
-                in_html_open_tag = True
-                continue
-            if not stripped:
-                kept.append(line)
-            elif is_prose_line(line):
-                # Strip trailing HTML close tags (e.g. "See the guide.</p>")
-                cleaned = _TRAILING_HTML_CLOSE_RE.sub("", line).rstrip()
-                cleaned_stripped = cleaned.strip()
-                if not cleaned_stripped:
-                    # Entire line was HTML close tags — keep original (shouldn't
-                    # normally reach here since _is_prose_line filters HTML).
-                    kept.append(line)
-                elif re.search(r"\w", cleaned_stripped):
-                    # Line has real word content after stripping close tags.
-                    kept.append(cleaned)
-                # else: only punctuation remains (e.g. bare ".") — discard.
-        cleaned = "\n".join(kept)
-
-        combined.append(f"\n\n---\n\n# {relative}\n")
-        combined.append(cleaned.strip())
-
-    output_file.write_text(
-        "\n".join(combined) + "\n",
-        encoding="utf-8",
-    )
+# Generate llms.txt and llms-full.txt after each build (the llms.txt standard,
+# https://llmstxt.org/). See the rocm-docs-core guide:
+# https://rocm.docs.amd.com/projects/rocm-docs-core/en/latest/user_guide/llms.html
+rocm_docs_generate_llms = True
 
 # Pages (by docname) whose primary (left) sidebar should be collapsed by default.
 # #pst-primary-sidebar-checkbox restores the sidebar.
@@ -302,4 +85,3 @@ def collapse_primary_sidebar(app, pagename, templatename, context, doctree):
 def setup(app):
     app.add_css_file("css/index.css")
     app.connect("html-page-context", collapse_primary_sidebar)
-    app.connect("build-finished", generate_combined_markdown)
diff --git a/docs/llms.txt b/docs/llms.txt
deleted file mode 100644
index 5a1ab7c..0000000
--- a/docs/llms.txt
+++ /dev/null
@@ -1,83 +0,0 @@
-# AMD Instinct Data Center GPU Documentation
-
-> Comprehensive guides and technical documentation for deploying AMD Instinct Data Center GPUs in enterprise environments. Covers system administration, cluster management, monitoring, and operational best practices for HPC and AI workloads. For API documentation and the ROCm software stack, see https://rocm.docs.amd.com.
-
-## GPU architecture
-
-- [AMD Instinct GPU architecture overview](https://instinct.docs.amd.com/en/latest/gpu-arch/gpu-arch.html): Microarchitecture overview of AMD Instinct MI350, MI300, MI250, and MI100 GPU accelerators.
-- [AMD Instinct MI350](https://instinct.docs.amd.com/en/latest/gpu-arch/mi350.html): Architecture and specifications for the MI350 series.
-- [AMD Instinct MI300](https://instinct.docs.amd.com/en/latest/gpu-arch/mi300.html): Architecture and specifications for the MI300 series, including MI300X and MI300A.
-- [AMD Instinct MI250](https://instinct.docs.amd.com/en/latest/gpu-arch/mi250.html): Architecture and specifications for the MI250 series.
-- [AMD Instinct MI100](https://instinct.docs.amd.com/en/latest/gpu-arch/mi100.html): Architecture and specifications for the MI100 series.
-
-## System administration
-
-### Bare metal
-
-- [AMD GPU Driver (amdgpu)](https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/): Install and configure the AMD GPU driver. Covers logging and error codes.
-- [AMD Instinct Customer Acceptance Guide](https://instinct.docs.amd.com/projects/system-acceptance/en/latest/): Configure, validate, benchmark, and baseline AMD Instinct GPUs.
-- [AMD SMI](https://rocm.docs.amd.com/projects/amdsmi/en/latest/): System management interface for monitoring and managing AMD GPUs.
-- [ROCm Validation Suite](https://rocm.docs.amd.com/projects/ROCmValidationSuite/en/latest/): System validation and diagnostics.
-- [Cluster Validation Suite](https://rocm.docs.amd.com/projects/cvs/en/latest/): Test scripts to validate AMD AI clusters.
-
-### Containers and orchestration
-
-- [AMD GPU Operator](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/): Deploy and manage AMD Instinct GPUs within Kubernetes clusters.
-- [Network Operator](https://instinct.docs.amd.com/projects/network-operator/en/main/): Use AMD AINICs in Kubernetes environments.
-- [Kubernetes Device Plugin](https://instinct.docs.amd.com/projects/k8s-device-plugin/en/latest/): Register AMD GPUs to Kubernetes container clusters.
-- [Device Metrics Exporter](https://instinct.docs.amd.com/projects/device-metrics-exporter/en/latest/): Prometheus-format GPU metrics collection.
-- [AMD Container Toolkit](https://instinct.docs.amd.com/projects/container-toolkit/en/latest/): Integrate AMD Instinct GPUs with Docker.
-
-### Cluster management
-
-- [Cluster Networking](https://instinct.docs.amd.com/projects/gpu-cluster-networking/en/latest/): Optimize networking for Instinct GPU applications.
-
-### Cloud
-
-- [AMD Instinct GPUs on Azure](https://instinct.docs.amd.com/projects/instinct-azure/latest/): Get started with AMD Instinct on Microsoft Azure.
-
-### Virtualization
-
-- [AMD Instinct Virtualization Driver](https://instinct.docs.amd.com/projects/virt-drv/en/latest/): Virtualization driver for AMD Instinct GPUs.
-- [AMD SMI for Virtualization](https://instinct.docs.amd.com/projects/amd-smi-virt/en/latest/): Manage and monitor AMD virtualization-enabled GPUs.
-
-## Workloads
-
-### Computer vision
-
-- [Models and applications](https://instinct.docs.amd.com/en/latest/vision/ai.html): Design, train, and infer computer vision models on AMD Instinct GPUs.
-- [Image and video decoding](https://instinct.docs.amd.com/en/latest/vision/decode.html): Decode image and video formats using AMD Instinct GPUs.
-- [Image processing](https://instinct.docs.amd.com/en/latest/vision/preprocess.html): Eliminate image preprocessing bottlenecks with AMD Instinct GPUs.
-
-### Data science
-
-- [ROCm Data Science Toolkit (ROCm-DS)](https://instinct.docs.amd.com/en/latest/data-science/index.html): Accelerate data science workloads on AMD Instinct GPUs.
-- [hipDF](https://instinct.docs.amd.com/en/latest/data-science/hipDF.html): GPU-accelerated DataFrames for data manipulation and analysis.
-- [hipGRAPH](https://instinct.docs.amd.com/en/latest/data-science/hipGRAPH.html): Create and analyze graphs and complex networks on AMD Instinct GPUs.
-- [hipVS](https://instinct.docs.amd.com/en/latest/data-science/hipVS.html): Vector search operations on AMD Instinct GPUs.
-- [hipMM](https://instinct.docs.amd.com/en/latest/data-science/hipMM.html): Advanced memory management for ROCm-DS libraries.
-- [hipRAFT](https://instinct.docs.amd.com/en/latest/data-science/hipRAFT.html): Fundamental algorithms and primitives for ROCm-DS libraries.
-
-### Life science
-
-- [ROCm Toolkit for Life Science (ROCm-LS)](https://instinct.docs.amd.com/en/latest/life-science/index.html): Accelerate life science workloads on AMD Instinct GPUs.
-- [hipCIM](https://instinct.docs.amd.com/en/latest/life-science/hipCIM.html): N-dimensional image processing for medical imaging on AMD Instinct GPUs.
-- [MONAI](https://instinct.docs.amd.com/en/latest/life-science/MONAI.html): Train and deploy AI models for medical imaging on AMD Instinct GPUs.
-
-## ISV applications
-
-- [Ansys Fluent](https://instinct.docs.amd.com/en/latest/isv-apps/ansys-fluent.html): Computational fluid dynamics with AMD GPU acceleration.
-- [Ansys Mechanical](https://instinct.docs.amd.com/en/latest/isv-apps/ansys-mechanical.html): Structural simulation on AMD Instinct GPUs.
-- [Cadence Fidelity LES Solver](https://instinct.docs.amd.com/en/latest/isv-apps/cadence-fidelity.html): Computational fluid dynamics on AMD Instinct GPUs.
-- [Devito Codes DevitoPRO](https://instinct.docs.amd.com/en/latest/isv-apps/devito.html): Seismic imaging on AMD Instinct GPUs.
-- [Siemens Simcenter STAR-CCM+](https://instinct.docs.amd.com/en/latest/isv-apps/siemens.html): Multiphysics CFD simulation on AMD Instinct GPUs.
-- [Stone Ridge Technology ECHELON](https://instinct.docs.amd.com/en/latest/isv-apps/stone-ridge.html): Reservoir simulation on AMD Instinct GPUs.
-- [GSplat](https://instinct.docs.amd.com/en/latest/simulation/gsplat.html): Gaussian splatting on AMD Instinct MI300X.
-
-## Programming reference
-
-- [HIP C++](https://rocm.docs.amd.com/projects/HIP/en/latest/): HIP programming model and API for AMD GPUs.
-- [OpenMP](https://rocm.docs.amd.com/projects/llvm-project/en/latest/conceptual/openmp.html): OpenMP programming model for AMD GPUs.
-- [AMD SMI API](https://rocm.docs.amd.com/projects/amdsmi/en/latest/): Full AMD SMI API reference.
-
----
diff --git a/docs/sphinx/requirements.in b/docs/sphinx/requirements.in
index 85b0a49..ef633e8 100644
--- a/docs/sphinx/requirements.in
+++ b/docs/sphinx/requirements.in
@@ -1,2 +1,2 @@
-rocm-docs-core==1.38.0
+rocm-docs-core[llms]==1.38.0
 sphinx-reredirects
diff --git a/docs/sphinx/requirements.txt b/docs/sphinx/requirements.txt
index fe9a3eb..fcc92d9 100644
--- a/docs/sphinx/requirements.txt
+++ b/docs/sphinx/requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile docs/sphinx/requirements.in
+#    pip-compile --output-file=requirements.txt requirements.in
 #
 accessible-pygments==0.0.5
     # via pydata-sphinx-theme
@@ -23,7 +23,7 @@ beautifulsoup4==4.15.0
     # via pydata-sphinx-theme
 breathe==4.36.0
     # via rocm-docs-core
-certifi==2026.6.17
+certifi==2026.7.22
     # via requests
 cffi==2.1.0
     # via
@@ -41,24 +41,23 @@ cryptography==49.0.0
     # via pyjwt
 debugpy==1.8.21
     # via ipykernel
-decorator==5.3.1
-    # via ipython
 docutils==0.22.4
     # via
     #   myst-parser
     #   pydata-sphinx-theme
     #   sphinx
+    #   sphinx-markdown-builder
 executing==2.2.1
     # via stack-data
-fastjsonschema==2.21.2
+fastjsonschema==2.22.1
     # via
     #   nbformat
     #   rocm-docs-core
 gitdb==4.0.12
     # via gitpython
-gitpython==3.1.52
+gitpython==3.1.57
     # via rocm-docs-core
-greenlet==3.5.3
+greenlet==3.5.4
     # via sqlalchemy
 idna==3.18
     # via requests
@@ -70,7 +69,7 @@ importlib-metadata==9.0.0
     #   myst-nb
 ipykernel==7.3.0
     # via myst-nb
-ipython==9.15.0
+ipython==9.16.0
     # via
     #   ipykernel
     #   myst-nb
@@ -136,9 +135,9 @@ parso==0.8.7
     # via jedi
 pexpect==4.9.0
     # via ipython
-platformdirs==4.10.0
+platformdirs==4.11.0
     # via jupyter-core
-prompt-toolkit==3.0.52
+prompt-toolkit==3.0.53
     # via ipython
 psutil==7.2.2
     # via
@@ -188,19 +187,21 @@ requests==2.34.2
     # via
     #   pygithub
     #   sphinx
-rocm-docs-core==1.38.0
-    # via -r docs/sphinx/requirements.in
+rocm-docs-core[llms]==1.38.0
+    # via -r requirements.in
 roman-numerals==4.1.0
     # via sphinx
 rpds-py==2026.6.3
-    # via -r requirements.in
+    # via
+    #   jsonschema
+    #   referencing
 six==1.17.0
     # via python-dateutil
 smmap==5.0.3
     # via gitdb
 snowballstemmer==3.1.1
     # via sphinx
-soupsieve==2.8.4
+soupsieve==2.9.1
     # via beautifulsoup4
 sphinx==9.1.0
     # via
@@ -213,6 +214,7 @@ sphinx==9.1.0
     #   sphinx-copybutton
     #   sphinx-design
     #   sphinx-external-toc
+    #   sphinx-markdown-builder
     #   sphinx-multitoc-numbering
     #   sphinx-notfound-page
     #   sphinx-reredirects
@@ -224,12 +226,14 @@ sphinx-design==0.7.0
     # via rocm-docs-core
 sphinx-external-toc==1.1.0
     # via rocm-docs-core
+sphinx-markdown-builder==0.6.10
+    # via rocm-docs-core
 sphinx-multitoc-numbering==0.1.3
     # via sphinx-external-toc
 sphinx-notfound-page==1.1.0
     # via rocm-docs-core
 sphinx-reredirects==1.1.0
-    # via -r docs/sphinx/requirements.in
+    # via -r requirements.in
 sphinxcontrib-applehelp==2.0.0
     # via sphinx
 sphinxcontrib-devhelp==2.0.0
@@ -247,7 +251,9 @@ sqlalchemy==2.0.51
 stack-data==0.6.3
     # via ipython
 tabulate==0.10.0
-    # via jupyter-cache
+    # via
+    #   jupyter-cache
+    #   sphinx-markdown-builder
 tornado==6.5.7
     # via
     #   ipykernel