diff --git a/.github/workflows/build-ais.yml b/.github/workflows/build-ais.yml index c86a2e9a..a9a677de 100644 --- a/.github/workflows/build-ais.yml +++ b/.github/workflows/build-ais.yml @@ -137,6 +137,7 @@ jobs: -w /ais/hipFile/build \ "${AIS_CONTAINER_NAME}" \ /bin/bash -c ' + source /opt/venv/bin/activate export CPACK_DEBIAN_PACKAGE_RELEASE="${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}~$(source /etc/os-release && echo ${VERSION_ID})" export CPACK_RPM_PACKAGE_RELEASE="${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}" cmake \ diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 27311aed..e0606ca1 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -18,11 +18,12 @@ jobs: - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 #v6.2.0 - - name: Set up pylint and black + - name: Install packages run: | python -m pip install --upgrade pip python -m pip install pylint python -m pip install black + python -m pip install rocm-docs-core - name: Lint all Python files run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..f35d5c2f --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,18 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +sphinx: + configuration: docs/conf.py + +formats: [] + +python: + install: + - requirements: docs/sphinx/requirements.txt + +build: + os: ubuntu-22.04 + tools: + python: "3.10" diff --git a/CHANGELOG.md b/CHANGELOG.md index a13a12b6..b979017e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * Added check in the Fastpath/AIS backend to ensure the HIP Runtime is initialized. This avoids causing a segfault in the HIP Runtime. * The default CMake build type was changed from `Debug` to `RelWithDebInfo` * Added file type and file system validation in Fastpath. Fastpath will only accept IO targeting block devices or regular files backed by xfs or ext4 with ordered journaling mode. Other file systems can be explicitly allowed via the `HIPFILE_UNSUPPORTED_FILE_SYSTEMS` environment variable. +* The docs directory has been reorganized and `AIS_BUILD_DOCS` will now generate Sphinx output for the web. ### Removed * The rocFile library has been completely removed and the code is now a part of hipFile. diff --git a/INSTALL.md b/INSTALL.md index b2e9cc97..35485392 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -116,6 +116,9 @@ sudo systemctl reboot * `libmount-dev` (Debian/Ubuntu) * `libmount-devel` (RHEL/Fedora/openSUSE) +Please the the README.md file in the docs directory for requirements +for building the documentation. + ### Configure You do not need to set the `HIP_PLATFORM` environment variable, as @@ -126,7 +129,7 @@ that will be set by CMake. Options |Option|Default|Purpose| |------|-------|-------| -|AIS\_BUILD\_DOCS|OFF|Build API documentation (requires Doxygen)| +|AIS\_BUILD\_DOCS|OFF|Build API documentation (see `docs/README.md`)| |AIS\_INSTALL\_EXAMPLES|ON|Install example programs| |AIS\_USE\_CLANG\_TIDY|OFF|Run the `clang-tidy` tool (clang only)| |AIS\_USE\_CODE\_COVERAGE|OFF|Generate code coverage information when tests are run (clang only)| @@ -191,18 +194,5 @@ The results will be wrote to `/build`, in the `coverage-report.txt` and `coverage-lines.txt` files. ### Documentation -The API documentation is built using Doxygen. To build it, use the -`AIS_BUILD_DOCS` option. This will build the documentation for any -libraries that have been configured. As a special case, configuring -the documentation without any library will build the documentation -for BOTH libraries, allowing for a docs-only build. - -The documentation will be built with the libraries and appear in -`docs/(hip|roc)file`. We build HTML, XML, and LaTeX docs. If you -want a pdf, run `make pdf` in the `latex` directory, which will -create a file named refman.pdf that you can rename. - -If you want to build the docs without compiling the libraries, -you can just build the `doc` target (if you've set `AIS_BUILD_DOCS`): - `cmake --build . --target doc` +See `docs/README.md` diff --git a/cmake/AISDocumentation.cmake b/cmake/AISDocumentation.cmake index 2407978f..2bb9596d 100644 --- a/cmake/AISDocumentation.cmake +++ b/cmake/AISDocumentation.cmake @@ -13,6 +13,17 @@ option(AIS_BUILD_DOCS "Build the hipFile docs (requires Doxygen, Sphinx, and Bre if(AIS_BUILD_DOCS) find_package(Doxygen REQUIRED) + find_package(Python3 REQUIRED COMPONENTS Interpreter) + + # Verify Sphinx + rocm_docs are available + execute_process( + COMMAND "${Python3_EXECUTABLE}" -c "import rocm_docs" + RESULT_VARIABLE _rocm_docs_found + OUTPUT_QUIET ERROR_QUIET + ) + if(NOT _rocm_docs_found EQUAL 0) + message(FATAL_ERROR "rocm_docs Python package not found. Install it with: pip install rocm-docs-core") + endif() # Set Doxygen input (pasted into Doxyfile.in) set(AIS_DOXYFILE_INPUT "${HIPFILE_ROOT_PATH}/include") @@ -20,20 +31,70 @@ if(AIS_BUILD_DOCS) # Set the path to the documentation set(AIS_DOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/docs") + ########### + # Doxygen # + ########### + + # The Doxygen HTML, XML, etc. goes in docs/doxygen/ + # Set the Doxyfile install location - set(AIS_DOXYFILE ${AIS_DOC_PATH}/Doxyfile) + set(AIS_DOXYFILE "${AIS_DOC_PATH}/doxygen/Doxyfile") + + # Doxygen XML output dir (consumed by Breathe inside Sphinx) + set(AIS_DOXYGEN_XML_DIR "${AIS_DOC_PATH}/doxygen/xml") # Create the Doxyfile from the input file - configure_file("docs/Doxyfile.in" ${AIS_DOXYFILE}) + configure_file("docs/doxygen/Doxyfile.in" ${AIS_DOXYFILE}) + + ########## + # Sphinx # + ########## + + # The Sphinx HTML, XML, etc. goes in docs/sphinx/ + + # Set the target for the config file and toc file + set(AIS_SPHINX_CONF_FILE "${AIS_DOC_PATH}/sphinx/conf.py") + set(AIS_SPHINX_TOC_FILE "${AIS_DOC_PATH}/sphinx/_toc.yml.in") + + # Sphinx HTML output dir + set(AIS_SPHINX_BUILD_DIR "${AIS_DOC_PATH}/sphinx/html") - # Set the output directory - set(DOXYGEN_OUT ${AIS_DOC_PATH}) + # Copy conf.py and the table of contents file + # Need to do this so the toc file doesn't dirty the repo + # _toc.yml.in is transformed by rocm_docs, not CMake + configure_file("docs/sphinx/conf.py" ${AIS_SPHINX_CONF_FILE} COPYONLY) + configure_file("docs/sphinx/_toc.yml.in" ${AIS_SPHINX_TOC_FILE} COPYONLY) - # Configure the documentation build - add_custom_target("doc" - COMMAND ${DOXYGEN_EXECUTABLE} ${AIS_DOXYFILE} - WORKING_DIRECTORY ${AIS_DOC_PATH} - COMMENT "Generating hipFile API documentation with Doxygen" + # Build docs: Doxygen first, then Sphinx (which pulls in rocm_docs via conf.py) + add_custom_target(doc + # Step 1: Run Doxygen to produce XML for Breathe + COMMAND "${DOXYGEN_EXECUTABLE}" "${AIS_DOXYFILE}" + # Step 2: Run Sphinx (html), injecting the Doxygen paths via environment + COMMAND + ${CMAKE_COMMAND} -E env + "DOXYFILE_PATH=${AIS_DOXYFILE}" + "DOXYGEN_ROOT=${AIS_DOC_PATH}/doxygen" + "DOXYGEN_XML_DIR=${AIS_DOXYGEN_XML_DIR}" + "${Python3_EXECUTABLE}" -m sphinx + -b html + -c "${AIS_DOC_PATH}/sphinx" + "${HIPFILE_ROOT_PATH}/docs" + "${AIS_SPHINX_BUILD_DIR}" + -v + # Step 3: Run Sphinx (LaTeX), injecting the Doxygen paths via environment + COMMAND + ${CMAKE_COMMAND} -E env + "DOXYFILE_PATH=${AIS_DOXYFILE}" + "DOXYGEN_ROOT=${AIS_DOC_PATH}/doxygen" + "DOXYGEN_XML_DIR=${AIS_DOXYGEN_XML_DIR}" + "${Python3_EXECUTABLE}" -m sphinx + -b latex + -c "${AIS_DOC_PATH}/sphinx" + "${HIPFILE_ROOT_PATH}/docs" + "${AIS_SPHINX_BUILD_DIR}" + -v + WORKING_DIRECTORY "${AIS_DOC_PATH}" + COMMENT "Generating hipFile API documentation with Doxygen + Sphinx (rocm_docs)" VERBATIM ) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..6b156561 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,79 @@ +# hipFile Documentation + +hipFile is documented using Doxygen markup in the public header +files and Sphinx .rst files. + +## Building the Documentation + +hipFile can generate two documentation products: + +1. API documentation +2. Sphinx HTML documentation (includes #1) + +API documentation is generated from Doxygen and limited to markup +in the header files. This will generate HTML, XML, and LaTeX output. +The LaTeX can be used to create a pdf document. + +The Sphinx documentation incorporates the Doxygen API markup and +adds reStructured text file content to generate HTML documentation +for the web. + +The documentation CMake target currently builds both. There is +no way to select "just Doxygen". + +### Requirements + +* CMake >= 3.21 +* Doxygen (we use 1.9.8, other versions are untested) +* Python 3.12 (earlier versions might work but are untested) +* The following Python packages: + * breathe + * rocm-docs-core + * sphinx + * sphinx-rtd-theme + +If you want to build a pdf, you will need LaTeX and pdflatex. + +### Generation + +Building the documentation is done via CMake by turning on the docs +option (which is off by default). + + `cmake -DAIS_BUILD_DOCS=ON ` + +You can then build the `doc` target. + + `cmake --build . --target doc` + +The documentation will be in the `docs` subdirectory of the build +directory: + +``` + ├── doxygen + │   ├── html + │   ├── latex + │   └── xml + └── sphinx + └── html +``` + +To generate a pdf for the API docs, navigate to `doxygen/latex` and run `make pdf` to +generate a pdf named `refman.pdf`. + +To generate a pdf for the Sphinx docs, navigate to `sphinx/html` and run `make` in +the directory. This will generate a pdf named `rocshmem.pdf` (we are stealing their +configs until `rocm-docs-core` is updated). + +## Adding to the Documentation + +### API Documentation (Doxygen) + +* Make your markup look like the rest of the file +* Doxygen macros can be found in `docs/doxygen/Doxyfile.in` +* All public API types, functions, etc. MUST have Doxygen markup + +### Other Documentation (Sphinx) + +All other documentation goes in Sphinx .rst documents. API-specific documentation +belongs in `docs/API` and other documentation goes in `docs`. Be sure to update +`sphinx/_toc.yml.in` if you add any new files. diff --git a/docs/async.rst b/docs/api/async.rst similarity index 100% rename from docs/async.rst rename to docs/api/async.rst diff --git a/docs/batch.rst b/docs/api/batch.rst similarity index 100% rename from docs/batch.rst rename to docs/api/batch.rst diff --git a/docs/core.rst b/docs/api/core.rst similarity index 100% rename from docs/core.rst rename to docs/api/core.rst diff --git a/docs/driver.rst b/docs/api/driver.rst similarity index 100% rename from docs/driver.rst rename to docs/api/driver.rst diff --git a/docs/errors.rst b/docs/api/errors.rst similarity index 100% rename from docs/errors.rst rename to docs/api/errors.rst diff --git a/docs/file.rst b/docs/api/file.rst similarity index 100% rename from docs/file.rst rename to docs/api/file.rst diff --git a/docs/misc_api.rst b/docs/api/misc_api.rst similarity index 100% rename from docs/misc_api.rst rename to docs/api/misc_api.rst diff --git a/docs/rdma.rst b/docs/api/rdma.rst similarity index 100% rename from docs/rdma.rst rename to docs/api/rdma.rst diff --git a/docs/Doxyfile.in b/docs/doxygen/Doxyfile.in similarity index 99% rename from docs/Doxyfile.in rename to docs/doxygen/Doxyfile.in index 70058fb9..b9a60f43 100644 --- a/docs/Doxyfile.in +++ b/docs/doxygen/Doxyfile.in @@ -68,7 +68,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = +OUTPUT_DIRECTORY = @AIS_DOC_PATH@/doxygen # If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 # sub-directories (in 2 levels) under the output directory of each output format diff --git a/docs/hipify.rst b/docs/hipify.rst new file mode 100644 index 00000000..f47bdfee --- /dev/null +++ b/docs/hipify.rst @@ -0,0 +1,2 @@ +hipify +====== diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..73c3ef1f --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,43 @@ +.. meta:: + :description: hipFile does hipFile stuff + :keywords: hipFile, ROCm, storage, library, API, DMA + +********************* +hipFile documentation +********************* + +hipFile introduction goes here. For more information, see :doc:`introduction`. + +The hipFile public repository is located at ``_. + +.. grid:: 2 + :gutter: 3 + + .. grid-item-card:: Build and install + + * :doc:`Configure and build <./config_build>` + * :doc:`Sanitizers <./sanitizers>` + + .. grid-item-card:: Using hipFile + + * :doc:`hipify <./hipify>` + * :doc:`NVIDIA compatibility <./nvidia_compat>` + * :doc:`fio <./fio>` + * :doc:`Collecting statistics <./stats_collection>` + + .. grid-item-card:: API reference + + * :doc:`Errors <./api/errors>` + * :doc:`Core API <./api/core>` + * :doc:`Drivers <./api/driver>` + * :doc:`File operations <./api/file>` + * :doc:`Asynchronous operations <./api/async>` + * :doc:`Batch operations <./api/batch>` + * :doc:`RDMA <./api/rdma>` + * :doc:`Other <./api/misc_api>` + +To contribute to the documentation, refer to +`Contributing to ROCm `_. + +You can find licensing information on the +`Licensing `_ page. diff --git a/docs/sphinx/_toc.yml.in b/docs/sphinx/_toc.yml.in new file mode 100644 index 00000000..c4d32d37 --- /dev/null +++ b/docs/sphinx/_toc.yml.in @@ -0,0 +1,45 @@ +defaults: + numbered: False +root: index +subtrees: + +- entries: + - file: introduction.rst + title: Introduction + +- caption: Build and install + entries: + - file: config_build.rst + title: Configure and build + - file: sanitizers.rst + title: Sanitizers + +- caption: Using hipFile + entries: + - file: hipify.rst + title: hipify + - file: nvidia_compat.rst + title: cuFile compatibility + - file: fio.rst + title: fio support + - file: stats_collection.rst + title: Collecting statistics + +- caption: API reference + entries: + - file: api/errors.rst + title: Errors + - file: api/core.rst + title: Core + - file: api/driver.rst + title: Drivers + - file: api/file.rst + title: File operations + - file: api/async.rst + title: Asynchronous operations + - file: api/batch.rst + title: Batch operations + - file: api/rdma.rst + title: RDMA + - file: api/misc_api.rst + title: Other diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py new file mode 100644 index 00000000..52c5f78e --- /dev/null +++ b/docs/sphinx/conf.py @@ -0,0 +1,47 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# Pylint is NOT happy with the naming scheme ROCm chose +# pylint: disable=invalid-name + +"""This file is for configuring hipFile documentation""" + +import os +from rocm_docs import ROCmDocs + +version_number = "0.2.0" +left_nav_title = f"hipFile {version_number} documentation" + +# for PDF output on Read the Docs +project = "rocSHMEM" +author = "Advanced Micro Devices, Inc." +# pylint: disable=redefined-builtin +copyright = "Copyright (c) Advanced Micro Devices, Inc. All rights reserved." +# pylint: enable=redefined-builtin +version = version_number +release = version_number +external_projects_current_project = "rocshmem" +extensions = ["sphinx_design"] +external_toc_path = "./sphinx/_toc.yml" + +docs_core = ROCmDocs(left_nav_title) + +doxygen_root = os.environ.get("DOXYGEN_ROOT") +doxygen_xml_dir = os.environ.get("DOXYGEN_XML_DIR") + +if not doxygen_root or not doxygen_xml_dir: + raise RuntimeError( + "DOXYGEN_ROOT and DOXYGEN_XML_DIR must be set. " + "Build this documentation via CMake, not directly with sphinx-build." + ) + +docs_core.run_doxygen(doxygen_root=doxygen_root, doxygen_path=doxygen_xml_dir) + +docs_core.setup() + +# Transfer all Sphinx config variables into this module's global scope +for sphinx_var in ROCmDocs.SPHINX_VARS: + globals()[sphinx_var] = getattr(docs_core, sphinx_var) diff --git a/docs/stats_collection.rst b/docs/stats_collection.rst index e9d5e597..f8a9782b 100644 --- a/docs/stats_collection.rst +++ b/docs/stats_collection.rst @@ -4,10 +4,9 @@ Stats Collection Tool Command-line Tool ----------------- ``ais-stats`` can be run two ways: -* ``$ ais-stats -p [-i]`` will collect stats from a running process. - ``-i`` will report immediately rather than wait for the process to exit. -* ``$ ais-stats [args...]`` will launch ```` with the - provided arguments and report the collected stats when it exits. + +* ``$ ais-stats -p [-i]`` will collect stats from a running process. ``-i`` will report immediately rather than wait for the process to exit. +* ``$ ais-stats [args...]`` will launch ```` with the provided arguments and report the collected stats when it exits. Configuration ------------- @@ -24,4 +23,4 @@ Value Description Stats Collected --------------- * Basic: Bytes read/written on the fastpath backend -* Basic: Bytes read/written on the fallback backend \ No newline at end of file +* Basic: Bytes read/written on the fallback backend diff --git a/util/docker/DOCKERFILE.ais_ci_rocky b/util/docker/DOCKERFILE.ais_ci_rocky index 37a5f391..9d9d38f4 100644 --- a/util/docker/DOCKERFILE.ais_ci_rocky +++ b/util/docker/DOCKERFILE.ais_ci_rocky @@ -71,8 +71,18 @@ RUN dnf makecache && \ hip-devel \ libmount-devel \ llvm-devel \ + python3 \ + python3-pip \ rpm-build +# Install Python packages (don't source venv/bin/activate) +RUN python3 -m venv /opt/venv +RUN /opt/venv/bin/pip3 install \ + sphinx \ + breathe \ + sphinx-rtd-theme \ + rocm-docs-core + # Configure system linker for ROCm RUN cat < /etc/ld.so.conf.d/rocm.conf /opt/rocm/lib diff --git a/util/docker/DOCKERFILE.ais_ci_suse b/util/docker/DOCKERFILE.ais_ci_suse index 7e766415..c654c9e6 100644 --- a/util/docker/DOCKERFILE.ais_ci_suse +++ b/util/docker/DOCKERFILE.ais_ci_suse @@ -68,9 +68,21 @@ RUN zypper install -y \ hip-devel \ libmount-devel \ llvm19-devel \ + python312 \ + python312-pip \ rocm-device-libs \ rpm-build +# Install Python packages (don't source venv/bin/activate) +RUN python3.12 --version +RUN python3.12 -m venv /opt/venv +RUN /opt/venv/bin/pip3 install --upgrade pip +RUN /opt/venv/bin/pip3 install \ + sphinx \ + breathe \ + sphinx-rtd-theme \ + rocm-docs-core + # Configure system linker for ROCm RUN cat < /etc/ld.so.conf.d/rocm.conf /opt/rocm/lib diff --git a/util/docker/DOCKERFILE.ais_ci_ubuntu b/util/docker/DOCKERFILE.ais_ci_ubuntu index 0696bedb..288cfe20 100644 --- a/util/docker/DOCKERFILE.ais_ci_ubuntu +++ b/util/docker/DOCKERFILE.ais_ci_ubuntu @@ -51,8 +51,19 @@ RUN apt update && \ libboost-program-options-dev \ libmount-dev \ llvm-dev \ + python3 \ + python3-pip \ + python3-venv \ rocm-llvm-dev +# Install Python packages (don't source venv/bin/activate) +RUN python3 -m venv /opt/venv +RUN /opt/venv/bin/pip3 install \ + sphinx \ + breathe \ + sphinx-rtd-theme \ + rocm-docs-core + # Configure system linker for ROCm RUN cat < /etc/ld.so.conf.d/rocm.conf /opt/rocm/lib