Add Sphinx support to documentation

.github/workflows/build-ais.yml

@@ -137,6 +137,7 @@ jobs:
             -w /ais/hipFile/build \
             "${AIS_CONTAINER_NAME}" \
             /bin/bash -c '
+              source /opt/venv/bin/activate
               export CPACK_DEBIAN_PACKAGE_RELEASE="${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}~$(source /etc/os-release && echo ${VERSION_ID})"
               export CPACK_RPM_PACKAGE_RELEASE="${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}"
               cmake \

.github/workflows/pylint.yml

@@ -18,11 +18,12 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 #v6.2.0
 
-      - name: Set up pylint and black
+      - name: Install packages
         run: |
           python -m pip install --upgrade pip
           python -m pip install pylint
           python -m pip install black
+          python -m pip install rocm-docs-core
 
       - name: Lint all Python files
         run: |

.readthedocs.yaml

@@ -0,0 +1,18 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+formats: []
+
+python:
+  install:
+    - requirements: docs/sphinx/requirements.txt
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"

CHANGELOG.md

@@ -16,6 +16,7 @@
 * Added check in the Fastpath/AIS backend to ensure the HIP Runtime is initialized. This avoids causing a segfault in the HIP Runtime.
 * The default CMake build type was changed from `Debug` to `RelWithDebInfo`
 * Added file type and file system validation in Fastpath. Fastpath will only accept IO targeting block devices or regular files backed by xfs or ext4 with ordered journaling mode. Other file systems can be explicitly allowed via the `HIPFILE_UNSUPPORTED_FILE_SYSTEMS` environment variable.
+* The docs directory has been reorganized and `AIS_BUILD_DOCS` will now generate Sphinx output for the web.
 
 ### Removed
 * The rocFile library has been completely removed and the code is now a part of hipFile.

INSTALL.md

@@ -116,6 +116,9 @@ sudo systemctl reboot
     * `libmount-dev` (Debian/Ubuntu)
     * `libmount-devel` (RHEL/Fedora/openSUSE)
 
+Please the the README.md file in the docs directory for requirements
+for building the documentation.
+
 ### Configure
 
 You do not need to set the `HIP_PLATFORM` environment variable, as
@@ -126,7 +129,7 @@ that will be set by CMake.
 Options
 |Option|Default|Purpose|
 |------|-------|-------|
-|AIS\_BUILD\_DOCS|OFF|Build API documentation (requires Doxygen)|
+|AIS\_BUILD\_DOCS|OFF|Build API documentation (see `docs/README.md`)|
 |AIS\_INSTALL\_EXAMPLES|ON|Install example programs|
 |AIS\_USE\_CLANG\_TIDY|OFF|Run the `clang-tidy` tool (clang only)|
 |AIS\_USE\_CODE\_COVERAGE|OFF|Generate code coverage information when tests are run (clang only)|
@@ -191,18 +194,5 @@ The results will be wrote to `<path/to/repo>/build`, in the
 `coverage-report.txt` and `coverage-lines.txt` files.
 
 ### Documentation
-The API documentation is built using Doxygen. To build it, use the
-`AIS_BUILD_DOCS` option. This will build the documentation for any
-libraries that have been configured. As a special case, configuring
-the documentation without any library will build the documentation
-for BOTH libraries, allowing for a docs-only build.
-
-The documentation will be built with the libraries and appear in
-`docs/(hip|roc)file`. We build HTML, XML, and LaTeX docs. If you
-want a pdf, run `make pdf` in the `latex` directory, which will
-create a file named refman.pdf that you can rename.
-
-If you want to build the docs without compiling the libraries,
-you can just build the `doc` target (if you've set `AIS_BUILD_DOCS`):
 
-    `cmake --build . --target doc`
+See `docs/README.md`

cmake/AISDocumentation.cmake

@@ -13,27 +13,88 @@ option(AIS_BUILD_DOCS "Build the hipFile docs (requires Doxygen, Sphinx, and Bre
 
 if(AIS_BUILD_DOCS)
     find_package(Doxygen REQUIRED)
+    find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+    # Verify Sphinx + rocm_docs are available
+    execute_process(
+        COMMAND "${Python3_EXECUTABLE}" -c "import rocm_docs"
+        RESULT_VARIABLE _rocm_docs_found
+        OUTPUT_QUIET ERROR_QUIET
+    )
+    if(NOT _rocm_docs_found EQUAL 0)
+        message(FATAL_ERROR "rocm_docs Python package not found. Install it with: pip install rocm-docs-core")
+    endif()
 
     # Set Doxygen input (pasted into Doxyfile.in)
     set(AIS_DOXYFILE_INPUT "${HIPFILE_ROOT_PATH}/include")
 
     # Set the path to the documentation
     set(AIS_DOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/docs")
 
+    ###########
+    # Doxygen #
+    ###########
+
+    # The Doxygen HTML, XML, etc. goes in docs/doxygen/
+
     # Set the Doxyfile install location
-    set(AIS_DOXYFILE ${AIS_DOC_PATH}/Doxyfile)
+    set(AIS_DOXYFILE "${AIS_DOC_PATH}/doxygen/Doxyfile")
+
+    # Doxygen XML output dir (consumed by Breathe inside Sphinx)
+    set(AIS_DOXYGEN_XML_DIR "${AIS_DOC_PATH}/doxygen/xml")
 
     # Create the Doxyfile from the input file
-    configure_file("docs/Doxyfile.in" ${AIS_DOXYFILE})
+    configure_file("docs/doxygen/Doxyfile.in" ${AIS_DOXYFILE})
+
+    ##########
+    # Sphinx #
+    ##########
+
+    # The Sphinx HTML, XML, etc. goes in docs/sphinx/
+
+    # Set the target for the config file and toc file
+    set(AIS_SPHINX_CONF_FILE "${AIS_DOC_PATH}/sphinx/conf.py")
+    set(AIS_SPHINX_TOC_FILE "${AIS_DOC_PATH}/sphinx/_toc.yml.in")
+
+    # Sphinx HTML output dir
+    set(AIS_SPHINX_BUILD_DIR "${AIS_DOC_PATH}/sphinx/html")
 
-    # Set the output directory
-    set(DOXYGEN_OUT ${AIS_DOC_PATH})
+    # Copy conf.py and the table of contents file
+    # Need to do this so the toc file doesn't dirty the repo
+    # _toc.yml.in is transformed by rocm_docs, not CMake
+    configure_file("docs/sphinx/conf.py" ${AIS_SPHINX_CONF_FILE} COPYONLY)
+    configure_file("docs/sphinx/_toc.yml.in" ${AIS_SPHINX_TOC_FILE} COPYONLY)
 
-    # Configure the documentation build
-    add_custom_target("doc"
-        COMMAND ${DOXYGEN_EXECUTABLE} ${AIS_DOXYFILE}
-        WORKING_DIRECTORY ${AIS_DOC_PATH}
-        COMMENT "Generating hipFile API documentation with Doxygen"
+    # Build docs: Doxygen first, then Sphinx (which pulls in rocm_docs via conf.py)
+    add_custom_target(doc
+        # Step 1: Run Doxygen to produce XML for Breathe
+        COMMAND "${DOXYGEN_EXECUTABLE}" "${AIS_DOXYFILE}"
+        # Step 2: Run Sphinx (html), injecting the Doxygen paths via environment
+        COMMAND
+            ${CMAKE_COMMAND} -E env
+                "DOXYFILE_PATH=${AIS_DOXYFILE}"
+                "DOXYGEN_ROOT=${AIS_DOC_PATH}/doxygen"
+                "DOXYGEN_XML_DIR=${AIS_DOXYGEN_XML_DIR}"
+            "${Python3_EXECUTABLE}" -m sphinx
+                -b html
+                -c "${AIS_DOC_PATH}/sphinx"
+                "${HIPFILE_ROOT_PATH}/docs"
+                "${AIS_SPHINX_BUILD_DIR}"
+                -v
+        # Step 3: Run Sphinx (LaTeX), injecting the Doxygen paths via environment
+        COMMAND
+            ${CMAKE_COMMAND} -E env
+                "DOXYFILE_PATH=${AIS_DOXYFILE}"
+                "DOXYGEN_ROOT=${AIS_DOC_PATH}/doxygen"
+                "DOXYGEN_XML_DIR=${AIS_DOXYGEN_XML_DIR}"
+            "${Python3_EXECUTABLE}" -m sphinx
+                -b latex
+                -c "${AIS_DOC_PATH}/sphinx"
+                "${HIPFILE_ROOT_PATH}/docs"
+                "${AIS_SPHINX_BUILD_DIR}"
+                -v
+        WORKING_DIRECTORY "${AIS_DOC_PATH}"
+        COMMENT "Generating hipFile API documentation with Doxygen + Sphinx (rocm_docs)"
         VERBATIM
     )
 

docs/README.md

@@ -0,0 +1,79 @@
+# hipFile Documentation
+
+hipFile is documented using Doxygen markup in the public header
+files and Sphinx .rst files.
+
+## Building the Documentation
+
+hipFile can generate two documentation products:
+
+1. API documentation
+2. Sphinx HTML documentation (includes #1)
+
+API documentation is generated from Doxygen and limited to markup
+in the header files. This will generate HTML, XML, and LaTeX output.
+The LaTeX can be used to create a pdf document.
+
+The Sphinx documentation incorporates the Doxygen API markup and
+adds reStructured text file content to generate HTML documentation
+for the web.
+
+The documentation CMake target currently builds both. There is
+no way to select "just Doxygen".
+
+### Requirements
+
+* CMake >= 3.21
+* Doxygen (we use 1.9.8, other versions are untested)
+* Python 3.12 (earlier versions might work but are untested)
+* The following Python packages:
+    * breathe
+    * rocm-docs-core
+    * sphinx
+    * sphinx-rtd-theme
+
+If you want to build a pdf, you will need LaTeX and pdflatex.
+
+### Generation
+
+Building the documentation is done via CMake by turning on the docs
+option (which is off by default).
+
+    `cmake -DAIS_BUILD_DOCS=ON <options> <path/to/source>`
+
+You can then build the `doc` target.
+
+    `cmake --build . --target doc`
+
+The documentation will be in the `docs` subdirectory of the build
+directory:
+
+```
+    ├── doxygen
+    │   ├── html
+    │   ├── latex
+    │   └── xml
+    └── sphinx
+        └── html
+```
+
+To generate a pdf for the API docs, navigate to `doxygen/latex` and run `make pdf` to
+generate a pdf named `refman.pdf`.
+
+To generate a pdf for the Sphinx docs, navigate to `sphinx/html` and run `make` in
+the directory. This will generate a pdf named `rocshmem.pdf` (we are stealing their
+configs until `rocm-docs-core` is updated).
+
+## Adding to the Documentation
+
+### API Documentation (Doxygen)
+
+* Make your markup look like the rest of the file
+* Doxygen macros can be found in `docs/doxygen/Doxyfile.in`
+* All public API types, functions, etc. MUST have Doxygen markup
+
+### Other Documentation (Sphinx)
+
+All other documentation goes in Sphinx .rst documents. API-specific documentation
+belongs in `docs/API` and other documentation goes in `docs`. Be sure to update
+`sphinx/_toc.yml.in` if you add any new files.

docs/Doxyfile.in → docs/doxygen/Doxyfile.in

@@ -68,7 +68,7 @@ PROJECT_LOGO           =
 # entered, it will be relative to the location where doxygen was started. If
 # left blank the current directory will be used.
 
-OUTPUT_DIRECTORY       =
+OUTPUT_DIRECTORY       = @AIS_DOC_PATH@/doxygen
 
 # If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096
 # sub-directories (in 2 levels) under the output directory of each output format

docs/hipify.rst

@@ -0,0 +1,2 @@
+hipify
+======

docs/index.rst

@@ -0,0 +1,43 @@
+.. meta::
+  :description: hipFile does hipFile stuff
+  :keywords: hipFile, ROCm, storage, library, API, DMA
+
+*********************
+hipFile documentation
+*********************
+
+hipFile introduction goes here. For more information, see :doc:`introduction`.
+
+The hipFile public repository is located at `<https://github.com/ROCm/hipFile>`_.
+
+.. grid:: 2
+  :gutter: 3
+
+  .. grid-item-card:: Build and install
+
+      * :doc:`Configure and build <./config_build>`
+      * :doc:`Sanitizers <./sanitizers>`
+
+  .. grid-item-card:: Using hipFile
+
+      * :doc:`hipify <./hipify>`
+      * :doc:`NVIDIA compatibility <./nvidia_compat>`
+      * :doc:`fio <./fio>`
+      * :doc:`Collecting statistics <./stats_collection>`
+
+  .. grid-item-card:: API reference
+
+      * :doc:`Errors <./api/errors>`
+      * :doc:`Core API <./api/core>`
+      * :doc:`Drivers <./api/driver>`
+      * :doc:`File operations <./api/file>`
+      * :doc:`Asynchronous operations <./api/async>`
+      * :doc:`Batch operations <./api/batch>`
+      * :doc:`RDMA <./api/rdma>`
+      * :doc:`Other <./api/misc_api>`
+
+To contribute to the documentation, refer to
+`Contributing to ROCm <https://rocm.docs.amd.com/en/latest/contribute/contributing.html>`_.
+
+You can find licensing information on the
+`Licensing <https://rocm.docs.amd.com/en/latest/about/license.html>`_ page.

docs/sphinx/_toc.yml.in

@@ -0,0 +1,45 @@
+defaults:
+  numbered: False
+root: index
+subtrees:
+
+- entries:
+  - file: introduction.rst
+    title: Introduction
+
+- caption: Build and install
+  entries:
+  - file: config_build.rst
+    title: Configure and build
+  - file: sanitizers.rst
+    title: Sanitizers
+
+- caption: Using hipFile
+  entries:
+  - file: hipify.rst
+    title: hipify
+  - file: nvidia_compat.rst
+    title: cuFile compatibility
+  - file: fio.rst
+    title: fio support
+  - file: stats_collection.rst
+    title: Collecting statistics
+
+- caption: API reference
+  entries:
+  - file: api/errors.rst
+    title: Errors
+  - file: api/core.rst
+    title: Core
+  - file: api/driver.rst
+    title: Drivers
+  - file: api/file.rst
+    title: File operations
+  - file: api/async.rst
+    title: Asynchronous operations
+  - file: api/batch.rst
+    title: Batch operations
+  - file: api/rdma.rst
+    title: RDMA
+  - file: api/misc_api.rst
+    title: Other