Intel(R) SHMEM (ISHMEM) 1.5.0 (#11)

Signed-off-by: sys_shmem <sys_shmem@intel.com>

Author: sys-shmem

CMakeLists.txt

@@ -12,7 +12,7 @@ set(PROJECT_NAME "ishmem")
 set(PROJECT_FULL_NAME "Intel® SHMEM")
 
 set(ISHMEM_MAJOR_VERSION "1")
-set(ISHMEM_MINOR_VERSION "4")
+set(ISHMEM_MINOR_VERSION "5")
 set(ISHMEM_PATCH_VERSION "0")
 set(PROJECT_VERSION "${ISHMEM_MAJOR_VERSION}.${ISHMEM_MINOR_VERSION}.${ISHMEM_PATCH_VERSION}")
 
@@ -40,7 +40,9 @@ option(ENABLE_AOT_COMPILATION "Enables AOT compilation for GPU kernels" TRUE)
 
 # Set default device type(s) for AOT compilation
 if (NOT ISHMEM_AOT_DEVICE_TYPES)
-    set(ISHMEM_AOT_DEVICE_TYPES "pvc")
+    # xe-hpc: Intel(R) Data Center GPU Max Series
+    # xe2: Intel(R) Arc(TM) B-Series GPU Family
+    set(ISHMEM_AOT_DEVICE_TYPES "xe-hpc,xe2")
 endif()
 
 # -------------------------------------------------------------------
@@ -87,6 +89,7 @@ if (ENABLE_AOT_COMPILATION)
 endif()
 message(STATUS "Enable OpenSHMEM support:    ${ENABLE_OPENSHMEM}")
 message(STATUS "Enable MPI support:          ${ENABLE_MPI}")
+message(STATUS "Default Runtime:             ${ISHMEM_DEFAULT_RUNTIME_STR}")
 message(STATUS "===================================\n")
 
 # -------------------------------------------------------------------
@@ -116,7 +119,7 @@ set(ISHMEM_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/src")
 
 # -------------------------------------------------------------------
 # Generate and install files
-configure_file(${PROJECT_SOURCE_DIR}/src/ishmem_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/ishmem/config.h)
+configure_file(${PROJECT_SOURCE_DIR}/src/ishmem_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/ishmem/config.h @ONLY)
 configure_file(${PROJECT_SOURCE_DIR}/cmake/ishmem ${CMAKE_CURRENT_BINARY_DIR}/ishmem @ONLY)
 configure_file(${PROJECT_SOURCE_DIR}/cmake/vars.sh.in ${CMAKE_CURRENT_BINARY_DIR}/vars.sh @ONLY)
 configure_file(${PROJECT_SOURCE_DIR}/pkgconfig/ishmem.pc.in ${CMAKE_CURRENT_BINARY_DIR}/pkgconfig/ishmem.pc @ONLY)

CONTRIBUTING.md

@@ -34,14 +34,12 @@ Contributed code must be:
 
 ### Coding style
 
-The code style and consistency is maintained using `clang-format`. When submitting a contribution, please make sure that it adheres to the existing coding style by using the following command:
+The code style and consistency is maintained using `clang-format`. When submitting a contribution, please ensure it adheres to the existing coding style by using the following command from the top-level directory of the repository:
 
 ```
 clang-format -style=file -i <FILE>
 ```
 
-This will format the code using the `.clang-format` file found in the top-level directory of this repository.
-
 ### Unit tests
 
 Be sure to extend the existing tests when fixing an issue.
@@ -90,7 +88,9 @@ By making a contribution to this project, I certify that:
 
 Then add a line to every git commit message:
 
-    Signed-off-by: Kris Smith <kris.smith@email.com>
+```
+Signed-off-by: Kris Smith <kris.smith@email.com>
+```
 
 **Note**: Use your real name.
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2024 Intel Corporation.
+Copyright (c) 2025 Intel Corporation.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -7,24 +7,24 @@ met:
 1. Redistributions of source code must retain the above copyright
 notice, this list of conditions and the following disclaimer.
 
-2. Redistributions in binary form must reproduce the above copyright 
-notice, this list of conditions and the following disclaimer in the 
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
 documentation and/or other materials provided with the distribution.
 
-3. Neither the name of the copyright holder nor the names of its 
-contributors may be used to endorse or promote products derived from 
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
 this software without specific prior written permission.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
-“AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+“AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 SPDX-License-Identifier: BSD-3-Clause

README.md

@@ -17,7 +17,7 @@ Intel® SHMEM provides an efficient implementation of GPU-initiated communicatio
 ## Prerequisites
 
 - Linux OS
-- Intel® oneAPI DPC++/C++ Compiler 2024.0 or higher.
+- Intel® oneAPI DPC++/C++ Compiler 2025.0 or higher.
 
 ### SYCL support <!-- omit in toc -->
 Intel® oneAPI DPC++/C++ Compiler with Level Zero support.
@@ -34,7 +34,7 @@ To install, download the oneAPI Level Zero from the repository.
 git clone https://github.com/oneapi-src/level-zero.git
 ```
 
-Build Level Zero following instructions below. 
+Build Level Zero following instructions below.
 
 ```
 cd level-zero
@@ -44,16 +44,16 @@ cmake -DCMAKE_INSTALL_PREFIX=<level_zero_dir> ..
 make -j
 make install
 ```
-### The Host Back-End Library
-Intel® SHMEM requires a host OpenSHMEM or MPI back-end to be used for host-sided operations support. In particular, the OpenSHMEM back-end relies on a collection of extension APIs (`shmemx_heap_create`, `shmemx_heap_preinit`, and `shmemx_heap_postinit`) to coordinate the Intel® SHMEM and OpenSHMEM heaps. We recommend [Sandia OpenSHMEM v1.5.3rc1](https://github.com/Sandia-OpenSHMEM/SOS/releases/tag/v1.5.3rc1) or newer for this purpose. A [work-in-progress branch](https://github.com/davidozog/oshmpi/tree/wip/ishmem) of [OSHMPI](https://github.com/pmodels/oshmpi.git) is also supported but is currently considered experimental.  See the [Building OSHMPI](#building-oshmpi-optional-and-experimental) section before for more details.
+### The Host Backend Library
+Intel® SHMEM requires a host OpenSHMEM or MPI backend to be used for scale-out communication. In particular, the OpenSHMEM backend relies on a collection of extension APIs (`shmemx_heap_create`, `shmemx_heap_preinit`, and `shmemx_heap_postinit`) to coordinate the Intel® SHMEM and OpenSHMEM heaps. We recommend [Sandia OpenSHMEM v1.5.3](https://github.com/Sandia-OpenSHMEM/SOS/releases/tag/v1.5.3) or newer for this purpose. A [work-in-progress branch](https://github.com/davidozog/oshmpi/tree/wip/ishmem) of [OSHMPI](https://github.com/pmodels/oshmpi.git) is also supported but is currently considered experimental. See the [Building OSHMPI](#building-oshmpi-optional-and-experimental) section before for more details.
 
-We recommend the Intel® MPI Library as the MPI back-end option for the current version of Intel® SHMEM. See the [Building Intel® SHMEM](#building-intel-shmem) section below for more details.
+We recommend the Intel® MPI Library as the MPI backend option for the current version of Intel® SHMEM. See the [Building Intel® SHMEM](#building-intel-shmem) section below for more details.
 
 ### Building Sandia OpenSHMEM (SOS)
-Download the SOS repo to be configured as a back-end for Intel® SHMEM.
+Download the SOS repo to be configured as a backend for Intel® SHMEM.
 
 ```
-git clone --recurse-submodules https://github.com/Sandia-OpenSHMEM/SOS.git SOS
+git clone -b v1.5.3 --recurse-submodules https://github.com/Sandia-OpenSHMEM/SOS.git SOS
 ```
 
 Build SOS following instructions below. `FI_HMEM` support in the provider is required for use with Intel® SHMEM. To enable `FI_HMEM` with a supported provider, we recommend a specific set of config flags. Below are two examples for configuring and building SOS with two providers supporting `FI_HMEM`. To configure SOS with the `verbs;ofi_rxm` provider, use the following instructions:
@@ -93,8 +93,7 @@ To download the OSHMPI repository:
 ```
 git clone -b wip/ishmem --recurse-submodules https://github.com/davidozog/oshmpi.git oshmpi
 ```
-After ensuring Intel® MPI Library is enabled (for example, by sourcing the `/opt/intel/oneapi/setvars.sh` script),
-please build OSHMPI following the instructions below.
+After ensuring the Intel® MPI Library is present in the environment, please build OSHMPI following the instructions below.
 
 ```
 cd oshmpi
@@ -107,57 +106,48 @@ make install
 ### Building Intel® SHMEM
 Check that the SOS build process has successfully created a `<shmem_dir>` directory with `include` and `lib` as subdirectories. Please find `shmem.h` and `shmemx.h` in `include`.
 
-Build Intel® SHMEM with an OpenSHMEM back-end using the following instructions:
+Build Intel® SHMEM with an OpenSHMEM backend using the following instructions:
 
 ```
 cd ishmem
 mkdir build
 cd build
-CC=icx CXX=icpx cmake .. -DENABLE_OPENSHMEM=ON -DSHMEM_DIR=<shmem_dir> -DCMAKE_INSTALL_PREFIX=<ishmem_install_dir>
+cmake .. -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DENABLE_OPENSHMEM=ON -DSHMEM_DIR=<shmem_dir> -DCMAKE_INSTALL_PREFIX=<ishmem_install_dir>
 make -j
 ```
-Alternatively, Intel® SHMEM can be built by enabling an Intel® MPI Library back-end.
+Alternatively, Intel® SHMEM can be built by enabling an Intel® MPI Library backend.
 Here is information on how to [Get Started with Intel® MPI Library on Linux](https://www.intel.com/content/www/us/en/docs/mpi-library/get-started-guide-linux/2021-11/overview.html).
 
 ```
-CC=icx CXX=icpx cmake .. -DENABLE_OPENSHMEM=OFF -DENABLE_MPI=ON -DMPI_DIR=<impi_dir> -DCMAKE_INSTALL_PREFIX=<ishmem_install_dir>
+cmake .. -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DENABLE_MPI=ON -DMPI_DIR=<impi_dir> -DCMAKE_INSTALL_PREFIX=<ishmem_install_dir>
 ```
 where `<impi_dir>` is the path to the Intel® MPI Library installation.
 
-Enabling both the OpenSHMEM and MPI back-ends is also supported.  In this case,
-the desired backend can be selected via the environment variable,
-`ISHMEM_RUNTIME`, which can be set to either "OpenSHMEM" or "MPI".
-The default value for `ISHMEM_RUNTIME` is "OpenSHMEM".
+Enabling both the OpenSHMEM and MPI backends is also supported. You may specify the default runtime at configure time with `-DISHMEM_DEFAULT_RUNTIME=<backend>` where `<backend>` is `MPI` or `OPENSHMEM` (case-insensitive). Furthermore, the desired backend can be selected at runtime via the environment variable `ISHMEM_RUNTIME=<backend>` where is `MPI` or `OPENSHMEM` (case-insensitive). If a default runtime is not specified, it will be automatically selected from the enabled backends in the following order: `OPENSHMEM` then `MPI`.
 
 ## Usage
 
 ### Launching Example Application
 
 Validate that Intel® SHMEM was built correctly by running an example program.
 
-1. Add the path for the back-end library to the environment, for example:
+1. Add the path for the backend library to the environment, for example:
 
 ```
 export LD_LIBRARY_PATH=<shmem_dir>/lib:$LD_LIBRARY_PATH
+source <impi_dir>/env/vars.sh
 ```
 
-When enabling only the Intel® MPI Library back-end, simply source the appropriate
-`setvars.sh` script. When enabling both OpenSHMEM and MPI back-ends, first
-source the `setvars.sh` script, then configure the dynamic linker to load the
-OpenSHMEM library (for example by prepending `<shmem_dir>/lib` to
-`LD_LIBRARY_PATH`).
-
 2. Run the example program or test on an allocated node using a process launcher:
 
 ```
-ISHMEM_RUNTIME=<back-end> mpiexec.hydra -n 2 -hosts <allocated_node_id> ./scripts/ishmrun ./test/unit/int_get_device
+ISHMEM_RUNTIME=<backend> mpiexec.hydra -n 2 -hosts <allocated_node_id> ./scripts/ishmrun ./test/unit/int_get_device
 ```
-where `<back-end>` is the selected host back-end library.
 
 - *Note:* Current supported launchers include: MPI process launchers (i.e. `mpiexec`, `mpiexec.hydra`, `mpirun`, etc.), Slurm (i.e. `srun`, `salloc`, etc.), and PBS (i.e. `qsub`).
 
-- *Note:* Intel® SHMEM execution model requires applications to use a 1:1 mapping between PEs and GPU devices. Attempting to run an application without the ishmrun launch script may result in undefined behavior if this mapping is not maintained.
-  - For further details on the device selection, please see [the ONEAPI_DEVICE_SELECTOR](https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md#oneapi_device_selector).
+- *Note:* Intel® SHMEM execution model requires applications to use a 1:1 mapping between PEs and GPU devices. Attempting to run an application without the `ishmrun` launch script may result in failure if this mapping is not maintained.
+  - For further details on device selection, please see [the ONEAPI_DEVICE_SELECTOR](https://github.com/intel/llvm/blob/sycl/sycl/doc/EnvironmentVariables.md#oneapi_device_selector).
 
 3. Validate the application ran successfully; example output:
 
@@ -211,6 +201,18 @@ The following values may be assigned to `CTEST_LAUNCHER` at configure-time (ex.
    ```
    export JOB_QUEUE=<queue>
    ```
+### Hardware-specific Environment Settings
+The following environment settings are either **required** or *recommended* when running Intel® SHMEM on the specified hardware. For GPU-specific environment settings, the launch script `ishmrun` will automatically detect and set the appropriate environment. For interconnect-specific environment settings, it is up to the user to ensure the appropriate environment is set:
+
+- HPE Slingshot Interconnect
+  - `FI_CXI_OPTIMIZED_MRS=0` is **required** when running with an OpenSHMEM backend.
+  - `FI_CXI_DEFAULT_CQ_SIZE=131072` is *recommended* for all backends.
+- Mellanox ConnectX® Interconnects
+  - `MLX5_SCATTER_TO_CQE=0` is **required** when running with an OpenSHMEM backend.
+- Intel® Data Center GPU Max Series
+  - `EnableImplicitScaling=0` is **required**. *Note:* you will also need to ensure `NEOReadDebugKeys=1` in case it is not already set.
+- Intel® Arc™ B-Series GPUs
+  - `RenderCompressedBuffersEnabled=0` is **required**. *Note:* you will also need to ensure `NEOReadDebugKeys=1` in case it is not already set.
 
 ## Additional Resources
 

RELEASE_NOTES.md

@@ -1,6 +1,26 @@
 # Release Notes <!-- omit in toc -->
 This document contains a list of new features and known limitations of Intel® SHMEM releases.
 
+## Release 1.5.0
+
+### New Features and Enhancements
+- Support for new collectives: inclusive and exclusive scan.
+- Improved affinity assignment through launcher script `ishmrun`.
+- Preliminary support for Intel® Arc™ B-Series GPUs.
+- Bug fixes improving functionality.
+
+### Known Limitations
+- Only [Sandia OpenSHMEM](https://github.com/Sandia-OpenSHMEM/SOS) and [Intel® MPI Library](https://www.intel.com/content/www/us/en/developer/tools/oneapi/mpi-library.html) are currently supported as the host back-end.
+- Not all APIs from OpenSHMEM standard are supported. Please refer to [Supported/Unsupported Features](https://oneapi-src.github.io/ishmem/supported_features.html) to get a complete view.
+- Intel® SHMEM requires a one-to-one mapping of PEs to SYCL devices. This implies that Intel® SHMEM executions must launch with a number of processes on each compute node that is no more than the number of available SYCL devices on each one of those nodes. By default, the Intel® SHMEM runtime considers each individual device tile to make up a single SYCL device and assigns a tile per PE.
+- All collective operations within a kernel must complete before invoking subsequent kernel-initiated collective operation.
+- To run Intel® SHMEM with SOS enabling the Slingshot provider in OFI, environment variable `FI_CXI_OPTIMIZED_MRS=0` must be used. It is also recommended to use `FI_CXI_DEFAULT_CQ_SIZE=131072`.
+- To run Intel® SHMEM with SOS enabling the verbs provider, environment variable `MLX5_SCATTER_TO_CQE=0` must be used.
+- To run Intel® SHMEM with Intel® MPI Library, environment variable `I_MPI_OFFLOAD=1` must be used. Additionally, `I_MPI_OFFLOAD_RDMA=1` may be necessary for GPU RDMA depending on the OFI provider. Please refer to the [reference guide](https://www.intel.com/content/www/us/en/docs/mpi-library/developer-reference-linux/2021-16/gpu-buffers-support.html) for further details.
+- Inter-node communication in Intel® SHMEM requires [dma-buf](https://www.kernel.org/doc/html/latest/driver-api/dma-buf.html) support in the Linux kernel. Inter-node functionality in Intel® SHMEM Release 1.5.0 is tested with SUSE Linux Enterprise Server 15 SP4.
+- Support for Intel® Arc™ B-Series GPUs is preliminary. As such, not all APIs are currently supported.
+- When using Intel® Arc™ B-Series GPUs, environment variable `RenderCompressedBuffersEnabled=0` is required. This is automatically set when running with the launcher script `ishmrun`.
+
 ## Release 1.4.0
 
 ### New Features and Enhancements

SECURITY.md

@@ -1,5 +1,5 @@
 # Security Policy
-Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. 
+Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation.
 
 ## Reporting a Vulnerability
 Please report any security vulnerabilities in this project [utilizing the guidelines here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html).

cmake/ishmem

@@ -1,7 +1,7 @@
 #%Module1.0
 ##############################################################################
 
-# Copyright (c) 2025, Intel Corporation
+# Copyright (c) 2024, Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause
 
 ##############################################################################
@@ -79,5 +79,3 @@ prepend-path    LD_LIBRARY_PATH     "$topdir/lib"
 prepend-path    LIBRARY_PATH        "$topdir/lib"
 prepend-path    C_INCLUDE_PATH      "$topdir/include"
 prepend-path    CPLUS_INCLUDE_PATH  "$topdir/include"
-prepend-path    PKG_CONFIG_PATH     "$topdir/lib/pkgconfig"
-prepend-path    CMAKE_PREFIX_PATH   "$topdir/lib/cmake/ishmem"