diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 6b81b4f3..778226b7 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -130,7 +130,7 @@ jobs: fetch-depth: 0 - uses: cachix/install-nix-action@v13 with: - nix_path: nixpkgs=channel:nixos-20.09 + nix_path: nixpkgs=channel:nixos-24.05 - run: nix-build -A aml distcheck: runs-on: ubuntu-latest @@ -190,7 +190,7 @@ jobs: run: shell: bash container: - image: nvcr.io/nvidia/nvhpc:24.7-devel-cuda12.5-ubuntu22.04 + image: nvcr.io/nvidia/nvhpc:24.7-devel-cuda12.5-ubuntu24.04 steps: - uses: actions/checkout@v2 with: @@ -202,11 +202,11 @@ jobs: echo "0.8.0" > .tarball-version - name: build run: | - source /usr/share/lmod/6.6/init/bash + source /usr/share/lmod/lmod/init/bash module load nvhpc ./autogen.sh mkdir build - ./configure --prefix=`pwd`/build --with-cuda CUDA_HOME=$NVHPC_ROOT/cuda + ./configure --prefix=`pwd`/build --without-hwloc --with-cuda CUDA_HOME=$NVHPC_ROOT/cuda make make check make install diff --git a/configure.ac b/configure.ac index f6aef156..45837bab 100644 --- a/configure.ac +++ b/configure.ac @@ -46,7 +46,6 @@ AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects 1.12]) AC_LANG([C]) AC_USE_SYSTEM_EXTENSIONS AC_PROG_CC -AC_PROG_CC_C99 AM_PROG_CC_C_O AC_PROG_CPP AC_TYPE_SIZE_T @@ -141,6 +140,57 @@ else fi AC_SUBST(OPENMP_LIBS) +# MPI support +############# + +AC_ARG_WITH([mpi], + [AS_HELP_STRING([--with-mpi], + [support mpi inside the library (default is check)])], + [ + if test "x$withval" = "xno"; then + want_mpi="no" + else + want_mpi="yes" + fi + ], + [ + want_mpi="check" + ]) + +MPI_CFLAGS="" +MPI_LIBS="" +if test "x$want_mpi" != "xno"; then + AC_MSG_NOTICE([starting checks for MPI]) + AC_CHECK_PROGS(MPICC, mpicc hcc mpxlc_r mpxlc mpcc cmpicc, no) + + if test "x$MPICC" = "xno"; then + AC_MSG_NOTICE([No MPI compiler found]) + if test "x$want_mpi" = "xyes"; then + AC_MSG_ERROR([MPI compiler (mpicc) not found but --with-mpi was requested]) + fi + + have_mpi=0 + else + MPI_CC_SHOW=`$MPICC -show 2>/dev/null` + for arg in $MPI_CC_SHOW; do + case "$arg" in + -I*|-D*|-std=*|-f*) MPI_CFLAGS="$MPI_CFLAGS $arg" ;; + -L*|-l*|-Wl,*|-W*) MPI_LIBS="$MPI_LIBS $arg" ;; + *) ;; # Skip compiler name or unknown args + esac + done + have_mpi=1 + CC="$MPICC" + fi +fi + +AC_SUBST([MPI_CFLAGS]) +AC_SUBST([MPI_LIBS]) +AC_SUBST([MPICC]) +AC_SUBST([HAVE_MPI],[$have_mpi]) +AC_DEFINE_UNQUOTED([HAVE_MPI], [$have_mpi], [Whether aml supports MPI.]) +AM_CONDITIONAL([HAVE_MPI], [test "$have_mpi" == "1"]) + # NUMA support ############## @@ -450,6 +500,14 @@ Active: $HAVE_OPENMP CFLAGS: $OPENMP_CFLAGS LDFLAGS: $OPENMP_LIBS +MPI: +====== + +Active: $HAVE_MPI +MPICC: $MPICC +CFLAGS: $MPI_CFLAGS +LDFLAGS: $MPI_LIBS + HWLOC: ====== diff --git a/include/Makefile.am b/include/Makefile.am index d7ca0b54..4fbb3f69 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -71,6 +71,11 @@ include_aml_area_HEADERS += aml/area/hip.h include_aml_dma_HEADERS += aml/dma/hip.h endif +if HAVE_MPI +include_aml_area_HEADERS += aml/area/mpi.h +include_aml_backend_HEADERS += aml/utils/backend/mpi.h +endif + if HAVE_OPENCL include_aml_area_HEADERS += aml/area/opencl.h endif diff --git a/include/aml/area/mpi.h b/include/aml/area/mpi.h new file mode 100644 index 00000000..2eeca428 --- /dev/null +++ b/include/aml/area/mpi.h @@ -0,0 +1,139 @@ +/******************************************************************************* + * Copyright 2019 UChicago Argonne, LLC. + * (c.f. AUTHORS, LICENSE) + * + * This file is part of the AML project. + * For more info, see https://github.com/anlsys/aml + * + * SPDX-License-Identifier: BSD-3-Clause + ******************************************************************************/ + +#ifndef AML_AREA_MPI_H +#define AML_AREA_MPI_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/** + * @defgroup aml_area_mpi "AML MPI Areas" + * @brief MPI RDMA Implementation of AML Areas. + * + * This building block relies on the MPI One-sided operations to implement + * allocations. + * + * @code + * #include + * @endcode + * @{ + **/ + +/** + * Options that can eventually be passed to mmap + * call. + **/ +struct aml_area_mpi_mmap_options { + /** + * Specify the communicator for an allocation. + **/ + MPI_Comm comm; + /** + * Specify the info for an allocation. + **/ + MPI_Info info; + /** + * Return variable for the underlying window. + **/ + MPI_Win win; + /** + * Unit displacement (in bytes) + **/ + int disp; +}; + +/** + * Contains area operations implementation + * for the MPI area. + **/ +extern struct aml_area_ops aml_area_mpi_ops; + +/** + * Default MPI area: + * Uses COMM_WORLD and allocates every time. + * Can be used out-of-the-box with aml_area_*() functions. + **/ +extern struct aml_area aml_area_mpi; + +/** + * Implementation of aml_area_data for MPI areas. + **/ +struct aml_area_mpi_data { + /** hash table keeping track of windows **/ + struct aml_area_mpi_window *windows; +}; + +/** + * \brief MPI area creation. + * + * Allocates and initializes struct aml_area implemented by aml_area_mpi + * operations. + * @param[out] area pointer to an uninitialized struct aml_area pointer to + * receive the new area. + * @return On success, returns 0 and fills "area" with a pointer to the new + * aml_area. + * @return On failure, fills "area" with NULL and returns one of AML error + * codes: + * - AML_ENOMEM if there wasn't enough memory available. + **/ +int aml_area_mpi_create(struct aml_area **area); + +/** + * \brief MPI area destruction. + * + * Destroys (finalizes and frees resources) struct aml_area created by + * aml_area_mpi_create(). + * + * @param area address of an initialized struct aml_area pointer, which will be + * reset to NULL on return from this call. + **/ +void aml_area_mpi_destroy(struct aml_area **area); + +/** + * \brief mmap block for AML area. + * + * This function is a wrapper around the MPI_Win_allocate call using arguments + * set in opts. + * @param area_data: An aml_area_mpi_data. + * @param size: The size to allocate. + * @param opts: See "aml_area_mpi_mmap_options". + * @return a valid memory pointer, or NULL on failure. + * On failure, "errno" should be checked for further information. + **/ +void *aml_area_mpi_mmap(const struct aml_area_data *area_data, + size_t size, + struct aml_area_mmap_options *opts); + +/** + * \brief munmap hook for AML area. + * + * Unmaps memory mapped with aml_area_mpi_mmap(). + * @param area_data: unused + * @param ptr: The virtual memory to unmap. + * @param size: The size of the virtual memory to unmap. + * @return AML_SUCCESS on success, else AML_FAILURE. + * On failure, "errno" should be checked for further information. + **/ +int aml_area_mpi_munmap(const struct aml_area_data *area_data, + void *ptr, + const size_t size); + +/** + * @} + **/ + +#ifdef __cplusplus +} +#endif +#endif // AML_AREA_MPI_H diff --git a/include/aml/utils/backend/mpi.h b/include/aml/utils/backend/mpi.h new file mode 100644 index 00000000..a43f860a --- /dev/null +++ b/include/aml/utils/backend/mpi.h @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright 2019 UChicago Argonne, LLC. + * (c.f. AUTHORS, LICENSE) + * + * This file is part of the AML project. + * For more info, see https://github.com/anlsys/aml + * + * SPDX-License-Identifier: BSD-3-Clause + ******************************************************************************/ + +#ifndef AML_UTILS_BACKEND_MPI_H +#define AML_UTILS_BACKEND_MPI_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @defgroup aml_backend_mpi "AML MPI Utils" + * @brief Boilerplate Code and Initialization for MPI Backend. + * @code + * #include + * @endcode + * + * @{ + **/ + +/** + * MPI backend initialization function. + * This function should only be called once. + * This function should not fail unless the system is out of memory. + * + * @return AML_SUCCESS on success. + * @return -AML_ENOMEM on error. + */ +int aml_backend_mpi_init(void); + +/** + * linux backend initialization function. + */ +int aml_backend_mpi_finalize(void); + +/** + * @} + **/ + +#ifdef __cplusplus +} +#endif +#endif // AML_UTILS_BACKEND_MPI_H diff --git a/include/aml/utils/features.h.in b/include/aml/utils/features.h.in index 6587ed4d..472956da 100644 --- a/include/aml/utils/features.h.in +++ b/include/aml/utils/features.h.in @@ -35,19 +35,23 @@ #define AML_HAVE_BACKEND_ZE @HAVE_ZE@ /** Whether aml had hip capabilities at compile time **/ #define AML_HAVE_BACKEND_HIP @HAVE_HIP@ +/** Whether aml had mpi capabilities at compile time **/ +#define AML_HAVE_BACKEND_MPI @HAVE_MPI@ -/** Flag for checking runtime suport for libnuma **/ +/** Flag for checking runtime support for libnuma **/ #define AML_BACKEND_LIBNUMA (1UL<<1) -/** Flag for checking runtime suport for cuda **/ +/** Flag for checking runtime support for cuda **/ #define AML_BACKEND_CUDA (1UL<<2) -/** Flag for checking runtime suport for hwloc **/ +/** Flag for checking runtime support for hwloc **/ #define AML_BACKEND_HWLOC (1UL<<3) -/** Flag for checking runtime suport for opencl **/ +/** Flag for checking runtime support for opencl **/ #define AML_BACKEND_OPENCL (1UL<<4) -/** Flag for checking runtime suport for level zero **/ +/** Flag for checking runtime support for level zero **/ #define AML_BACKEND_ZE (1UL<<5) -/** Flag for checking runtime suport for hip **/ +/** Flag for checking runtime support for hip **/ #define AML_BACKEND_HIP (1UL<<6) +/** Flag for checking runtime support for mpi **/ +#define AML_BACKEND_MPI (1UL<<7) /** * Check if a set of backends can be used at runtime. diff --git a/m4/ax_mpi.m4 b/m4/ax_mpi.m4 new file mode 100644 index 00000000..f6db3d0c --- /dev/null +++ b/m4/ax_mpi.m4 @@ -0,0 +1,176 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_mpi.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_MPI([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]]) +# +# DESCRIPTION +# +# This macro tries to find out how to compile programs that use MPI +# (Message Passing Interface), a standard API for parallel process +# communication (see http://www-unix.mcs.anl.gov/mpi/) +# +# On success, it sets the MPICC, MPICXX, MPIF77, or MPIFC output variable +# to the name of the MPI compiler, depending upon the current language. +# (This may just be $CC/$CXX/$F77/$FC, but is more often something like +# mpicc/mpiCC/mpif77/mpif90.) It also sets MPILIBS to any libraries that +# are needed for linking MPI (e.g. -lmpi or -lfmpi, if a special +# MPICC/MPICXX/MPIF77/MPIFC was not found). +# +# Note that this macro should be used only if you just have a few source +# files that need to be compiled using MPI. In particular, you should +# neither overwrite CC/CXX/F77/FC with the values of +# MPICC/MPICXX/MPIF77/MPIFC, nor assume that you can use the same flags +# etc. as the standard compilers. If you want to compile a whole program +# using the MPI compiler commands, use one of the macros +# AX_PROG_{CC,CXX,FC}_MPI. +# +# ACTION-IF-FOUND is a list of shell commands to run if an MPI library is +# found, and ACTION-IF-NOT-FOUND is a list of commands to run if it is not +# found. If ACTION-IF-FOUND is not specified, the default action will +# define HAVE_MPI. +# +# LICENSE +# +# Copyright (c) 2008 Steven G. Johnson +# Copyright (c) 2008 Julian C. Cummings +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 9 + +AU_ALIAS([ACX_MPI], [AX_MPI]) +AC_DEFUN([AX_MPI], [ +AC_PREREQ(2.50) dnl for AC_LANG_CASE + +AC_LANG_CASE([C], [ + AC_REQUIRE([AC_PROG_CC]) + AC_ARG_VAR(MPICC,[MPI C compiler command]) + AC_CHECK_PROGS(MPICC, mpicc hcc mpxlc_r mpxlc mpcc cmpicc, $CC) + ax_mpi_save_CC="$CC" + CC="$MPICC" + AC_SUBST(MPICC) +], +[C++], [ + AC_REQUIRE([AC_PROG_CXX]) + AC_ARG_VAR(MPICXX,[MPI C++ compiler command]) + AC_CHECK_PROGS(MPICXX, mpic++ mpicxx mpiCC hcp mpxlC_r mpxlC mpCC cmpic++, $CXX) + ax_mpi_save_CXX="$CXX" + CXX="$MPICXX" + AC_SUBST(MPICXX) +], +[Fortran 77], [ + AC_REQUIRE([AC_PROG_F77]) + AC_ARG_VAR(MPIF77,[MPI Fortran 77 compiler command]) + AC_CHECK_PROGS(MPIF77, mpif77 hf77 mpxlf_r mpxlf mpf77 cmpifc, $F77) + ax_mpi_save_F77="$F77" + F77="$MPIF77" + AC_SUBST(MPIF77) +], +[Fortran], [ + AC_REQUIRE([AC_PROG_FC]) + AC_ARG_VAR(MPIFC,[MPI Fortran compiler command]) + AC_CHECK_PROGS(MPIFC, mpif90 mpxlf95_r mpxlf90_r mpxlf95 mpxlf90 mpf90 cmpif90c, $FC) + ax_mpi_save_FC="$FC" + FC="$MPIFC" + AC_SUBST(MPIFC) +]) + +if test x = x"$MPILIBS"; then + AC_LANG_CASE([C], [AC_CHECK_FUNC(MPI_Init, [MPILIBS=" "])], + [C++], [AC_CHECK_FUNC(MPI_Init, [MPILIBS=" "])], + [Fortran 77], [AC_MSG_CHECKING([for MPI_Init]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[ call MPI_Init])],[MPILIBS=" " + AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])], + [Fortran], [AC_MSG_CHECKING([for MPI_Init]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[ call MPI_Init])],[MPILIBS=" " + AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])]) +fi +AC_LANG_CASE([Fortran 77], [ + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(fmpi, MPI_Init, [MPILIBS="-lfmpi"]) + fi + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(fmpich, MPI_Init, [MPILIBS="-lfmpich"]) + fi +], +[Fortran], [ + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(fmpi, MPI_Init, [MPILIBS="-lfmpi"]) + fi + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(mpichf90, MPI_Init, [MPILIBS="-lmpichf90"]) + fi +]) +if test x = x"$MPILIBS"; then + AC_CHECK_LIB(mpi, MPI_Init, [MPILIBS="-lmpi"]) +fi +if test x = x"$MPILIBS"; then + AC_CHECK_LIB(mpich, MPI_Init, [MPILIBS="-lmpich"]) +fi + +dnl We have to use AC_TRY_COMPILE and not AC_CHECK_HEADER because the +dnl latter uses $CPP, not $CC (which may be mpicc). +AC_LANG_CASE([C], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpi.h]) + AC_TRY_COMPILE([#include ],[],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi], +[C++], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpi.h]) + AC_TRY_COMPILE([#include ],[],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi], +[Fortran 77], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpif.h]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[ include 'mpif.h'])],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi], +[Fortran], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpif.h]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[ include 'mpif.h'])],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi]) + +AC_LANG_CASE([C], [CC="$ax_mpi_save_CC"], + [C++], [CXX="$ax_mpi_save_CXX"], + [Fortran 77], [F77="$ax_mpi_save_F77"], + [Fortran], [FC="$ax_mpi_save_FC"]) + +AC_SUBST(MPILIBS) + +# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: +if test x = x"$MPILIBS"; then + $2 + : +else + ifelse([$1],,[AC_DEFINE(HAVE_MPI,1,[Define if you have the MPI library.])],[$1]) + : +fi +])dnl AX_MPI diff --git a/nix/aml.nix b/nix/aml.nix index b66b2c7b..9f48a24a 100644 --- a/nix/aml.nix +++ b/nix/aml.nix @@ -1,7 +1,7 @@ -{ stdenv, autoreconfHook, git, pkgconf, numactl, hwloc}: +{ stdenv, autoreconfHook, git, pkgconf, numactl, hwloc, mpich}: stdenv.mkDerivation { src = ../.; name = "aml"; nativeBuildInputs = [ autoreconfHook pkgconf git ]; - buildInputs = [ hwloc numactl ]; + buildInputs = [ hwloc numactl mpich ]; } diff --git a/src/Makefile.am b/src/Makefile.am index 44994bad..0f59f1b1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -2,6 +2,7 @@ SUFFIXES=.c .cu AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/excit/src $(PTHREAD_CFLAGS) $(OPENMP_CFLAGS) AM_LDFLAGS = $(PTHREAD_LIBS) $(OPENMP_LIBS) noinst_LTLIBRARIES= +SUBDIRS = ############################################# # valgrind support @@ -115,3 +116,15 @@ libaml_la_SOURCES+=area/ze.c libaml_la_SOURCES+=dma/ze.c libaml_la_SOURCES+=backend/ze.c endif + +############################################# +# MPI sources + +if HAVE_MPI +# have to make sure current gets compiled before mpi +AM_LDFLAGS += $(MPI_LIBS) +AM_CPPFLAGS += $(MPI_CFLAGS) +libaml_la_SOURCES+=area/mpi.c +libaml_la_SOURCES+=backend/mpi.c +endif + diff --git a/src/aml.c b/src/aml.c index d0c67ed0..400eb001 100644 --- a/src/aml.c +++ b/src/aml.c @@ -18,6 +18,9 @@ #if HAVE_HWLOC == 1 #include "aml/utils/backend/hwloc.h" #endif +#if HAVE_MPI == 1 +#include "aml/utils/backend/mpi.h" +#endif #if HAVE_ZE == 1 #include "aml/utils/backend/ze.h" #endif @@ -42,10 +45,16 @@ int aml_init(int *argc, char **argv[]) if (err != AML_SUCCESS) return err; +#if HAVE_MPI == 1 + err = aml_backend_mpi_init(); + if (err != AML_SUCCESS) + goto err_with_linux; +#endif + #if HAVE_ZE == 1 err = aml_backend_ze_init(); if (err != AML_SUCCESS) - goto error; + goto err_with_mpi; #endif #if HAVE_HWLOC == 1 @@ -56,12 +65,23 @@ int aml_init(int *argc, char **argv[]) return AML_SUCCESS; +// bit of ugly code here: labels can't be unused so we need to only define them +// if their caller is defined, but we only need to do something if the feature +// before them exists. #if HAVE_HWLOC == 1 -err_with_ze:; +err_with_ze: #endif #if HAVE_ZE == 1 aml_backend_ze_finalize(); -error: +#endif +#if HAVE_ZE == 1 +err_with_mpi: +#endif +#if HAVE_MPI == 1 + aml_backend_mpi_finalize(); +#endif +#if HAVE_MPI == 1 +err_with_linux: #endif aml_backend_linux_finalize(); return err; @@ -77,5 +97,9 @@ int aml_finalize(void) #if HAVE_ZE == 1 aml_backend_ze_finalize(); #endif + +#if HAVE_MPI == 1 + aml_backend_mpi_finalize(); +#endif return 0; } diff --git a/src/area/mpi.c b/src/area/mpi.c new file mode 100644 index 00000000..5a3a5644 --- /dev/null +++ b/src/area/mpi.c @@ -0,0 +1,135 @@ +/******************************************************************************* + * Copyright 2019 UChicago Argonne, LLC. + * (c.f. AUTHORS, LICENSE) + * + * This file is part of the AML project. + * For more info, see https://github.com/anlsys/aml + * + * SPDX-License-Identifier: BSD-3-Clause + ******************************************************************************/ + +#include + +#include "aml.h" + +#include "aml/area/mpi.h" + +#include "internal/uthash.h" + +/******************************************************************************* + * Implementation + ******************************************************************************/ + +struct aml_area_mpi_window { + void *ptr; + MPI_Win win; + UT_hash_handle hh; +}; + +void *aml_area_mpi_mmap(const struct aml_area_data *area_data, + size_t size, + struct aml_area_mmap_options *options) +{ + int err; + struct aml_area_mpi_data *data; + struct aml_area_mpi_mmap_options *opts; + + data = (struct aml_area_mpi_data *)area_data; + opts = (struct aml_area_mpi_mmap_options *)options; + + MPI_Comm comm; + MPI_Info info; + int disp; + + if (opts != NULL) { + comm = opts->comm; + info = opts->info; + disp = opts->disp; + } else { + comm = MPI_COMM_WORLD; + MPI_Comm_get_info(comm, &info); + disp = 1; + } + struct aml_area_mpi_window *w = calloc(1, sizeof(*w)); + assert(w != NULL); + err = MPI_Win_allocate(size, disp, info, comm, &w->ptr, &w->win); + /* TODO error conversion */ + if (err != MPI_SUCCESS) { + free(w); + aml_errno = -AML_FAILURE; + return MAP_FAILED; + } + + /* store the window for munmap */ + HASH_ADD_PTR(data->windows, ptr, w); + if (opts != NULL) { + opts->win = w->win; + } + return w->ptr; +} + +int aml_area_mpi_munmap(const struct aml_area_data *area_data, + void *ptr, + const size_t size) +{ + (void)size; + struct aml_area_mpi_data *data; + struct aml_area_mpi_window *w = NULL; + + data = (struct aml_area_mpi_data *)area_data; + + HASH_FIND_PTR(data->windows, &ptr, w); + MPI_Win_free(&w->win); + HASH_DEL(data->windows, w); + free(w); + return AML_SUCCESS; +} + +/******************************************************************************* + * Areas Initialization + ******************************************************************************/ + +int aml_area_mpi_create(struct aml_area **area) +{ + struct aml_area *ret; + struct aml_area_mpi_data *data; + + ret = AML_INNER_MALLOC(struct aml_area, struct aml_area_mpi_data); + if (ret == NULL) + return -AML_ENOMEM; + + data = AML_INNER_MALLOC_GET_FIELD(ret, 2, struct aml_area, + struct aml_area_mpi_data); + + ret->ops = &aml_area_mpi_ops; + ret->data = (struct aml_area_data *)data; + + *area = ret; + return AML_SUCCESS; +} + +void aml_area_mpi_destroy(struct aml_area **area) +{ + if (*area == NULL) + return; + + free(*area); + *area = NULL; +} + +/******************************************************************************* + * Areas declaration + ******************************************************************************/ + +struct aml_area_mpi_data aml_area_mpi_data_default = { + .windows = NULL, +}; + +struct aml_area_ops aml_area_mpi_ops = { + .mmap = aml_area_mpi_mmap, + .munmap = aml_area_mpi_munmap, +}; + +struct aml_area aml_area_mpi = { + .ops = &aml_area_mpi_ops, + .data = (struct aml_area_data *)(&aml_area_mpi_data_default)}; diff --git a/src/backend/mpi.c b/src/backend/mpi.c new file mode 100644 index 00000000..e9ae3b2f --- /dev/null +++ b/src/backend/mpi.c @@ -0,0 +1,21 @@ +/******************************************************************************* + * Copyright 2019 UChicago Argonne, LLC. + * (c.f. AUTHORS, LICENSE) + * + * This file is part of the AML project. + * For more info, see https://github.com/anlsys/aml + * + * SPDX-License-Identifier: BSD-3-Clause + ******************************************************************************/ + +#include "aml.h" + +int aml_backend_mpi_init(void) +{ + return AML_SUCCESS; +} + +int aml_backend_mpi_finalize(void) +{ + return AML_SUCCESS; +} diff --git a/src/utils/features.c b/src/utils/features.c index 922bce51..1388d6dd 100644 --- a/src/utils/features.c +++ b/src/utils/features.c @@ -22,6 +22,9 @@ #include extern hwloc_topology_t aml_topology; #endif +#if HAVE_MPI == 1 +#include +#endif #if HAVE_ZE == 1 #include #endif @@ -116,6 +119,20 @@ static int aml_support_hwloc(void) #endif } +static int aml_support_mpi(void) +{ +#if HAVE_MPI == 0 + return 0; +#else + int initialized; + MPI_Initialized(&initialized); + if (initialized) + return 1; + else + return 0; +#endif +} + int aml_support_backends(const unsigned long backends) { // Cuda check: compilation support and runtime support must be present. @@ -128,6 +145,11 @@ int aml_support_backends(const unsigned long backends) !(AML_HAVE_BACKEND_HIP && aml_support_hip())) return 0; + // mpi check: compilation support and runtime support must be present. + if ((backends & AML_BACKEND_MPI) && + !(AML_HAVE_BACKEND_MPI && aml_support_mpi())) + return 0; + // OpenCL check: compilation support and runtime support must be // present. if ((backends & AML_BACKEND_OPENCL) && diff --git a/tests/Makefile.am b/tests/Makefile.am index 8e99c572..8b971a98 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -3,6 +3,7 @@ AM_COLOR_TESTS = yes AM_CFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/excit/src $(PTHREAD_CFLAGS) $(OPENMP_CFLAGS) AM_LDFLAGS = ../src/libaml.la $(top_builddir)/excit/src/libexcit.la $(PTHREAD_LIBS) $(OPENMP_LIBS) +SUBDIRS = if HAVE_CUDA LIBS += $(CUDA_LIBS) @@ -34,6 +35,16 @@ AM_LDFLAGS += $(HIP_LIBS) AM_CFLAGS += $(HIP_CFLAGS) endif +if HAVE_MPI +LIBS += $(MPI_LIBS) +AM_LDFLAGS += $(MPI_LIBS) +AM_CFLAGS += $(MPI_CFLAGS) +endif + +# if the env doesn't include CI vars, default to no +CI ?=no +TESTS_ENVIRONMENT=$(TEST_ENVIRONMENT) CI=$(CI) + # valgrind support @VALGRIND_CHECK_RULES@ VALGRIND_SUPPRESSIONS_FILES=../.valgrind.supp @@ -103,6 +114,10 @@ AREA_TESTS+=area/test_cuda AREA_TESTS+=dma/test_cuda endif +if HAVE_MPI +AREA_TESTS += area/test_mpi +endif + if HAVE_OPENCL AREA_TESTS += area/test_opencl endif @@ -134,6 +149,7 @@ UNIT_TESTS = $(UTILS_TESTS) \ # all tests TST_PROGS = $(UNIT_TESTS) +#TODO: odd, figure out if it's actually required if HAVE_CUDA test_area_cuda_SOURCES=area/test_cuda.c test_area_cuda_CPPFLAGS= $(AM_CFLAGS) $(CUDA_CFLAGS) diff --git a/tests/area/test_mpi.c b/tests/area/test_mpi.c new file mode 100644 index 00000000..c6eb8ed0 --- /dev/null +++ b/tests/area/test_mpi.c @@ -0,0 +1,55 @@ +/******************************************************************************* + * Copyright 2019 UChicago Argonne, LLC. + * (c.f. AUTHORS, LICENSE) + * + * This file is part of the AML project. + * For more info, see https://github.com/anlsys/aml + * + * SPDX-License-Identifier: BSD-3-Clause + ******************************************************************************/ + +#include "config.h" + +#include + +#include "aml.h" + +#include "aml/area/mpi.h" + +void test_map(const struct aml_area *area) +{ + assert(area != NULL); + assert(area->ops->mmap != NULL); + assert(area->ops->munmap != NULL); + + void *ptr; + size_t s; + const size_t sizes[4] = {1, 32, 4096, 1 << 20}; + + for (s = 0; s < sizeof(sizes) / sizeof(*sizes); s++) { + aml_errno = AML_SUCCESS; + ptr = aml_area_mmap(area, sizes[s], NULL); + assert(aml_errno == AML_SUCCESS); + assert(ptr != NULL); + assert(aml_area_munmap(area, ptr, sizes[s]) == AML_SUCCESS); + } +} + +void test_aml_area(struct aml_area *area) +{ + test_map(area); +} + +int main(int argc, char **argv) +{ + /* impossible to do those check in a CI environment consistently */ + if (!strcmp(getenv("CI"), "true")) + exit(77); + + MPI_Init(&argc, &argv); + aml_init(&argc, &argv); + test_map(&aml_area_mpi); + aml_finalize(); + MPI_Finalize(); + return 0; +}