From c39d63961330a1d39606a4c251fada44be5bf6fd Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Tue, 16 Sep 2025 09:36:02 -0500
Subject: [PATCH 01/11] [feature] Add MPI One-sided area

Add an area that can use one-sided window allocation to allocate memory.
The interface is limited by the required arguments on communicators,
requiring us to keep track of the created windows internally to be able
to cleanly destroy them.

Could probably also design a DMA to use puts and gets and fence, even if
that would be a bit strange in terms of API, since they don't really
match the "stream of copy request" model.
---
 configure.ac           |  18 ++++-
 include/aml/area/mpi.h | 143 +++++++++++++++++++++++++++++++++
 m4/ax_mpi.m4           | 176 +++++++++++++++++++++++++++++++++++++++++
 nix/aml.nix            |   4 +-
 src/Makefile.am        |   9 +++
 src/area/mpi.c         | 133 +++++++++++++++++++++++++++++++
 tests/Makefile.am      |   7 ++
 tests/area/test_mpi.c  |  48 +++++++++++
 8 files changed, 535 insertions(+), 3 deletions(-)
 create mode 100644 include/aml/area/mpi.h
 create mode 100644 m4/ax_mpi.m4
 create mode 100644 src/area/mpi.c
 create mode 100644 tests/area/test_mpi.c

diff --git a/configure.ac b/configure.ac
index f6aef156..dc0c4de8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -46,7 +46,6 @@ AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects 1.12])
 AC_LANG([C])
 AC_USE_SYSTEM_EXTENSIONS
 AC_PROG_CC
-AC_PROG_CC_C99
 AM_PROG_CC_C_O
 AC_PROG_CPP
 AC_TYPE_SIZE_T
@@ -141,6 +140,16 @@ else
 fi
 AC_SUBST(OPENMP_LIBS)
 
+# MPI support
+#############
+
+AX_MPI(have_mpi=1,have_mpi=0)
+TESTCC="$MPICC"
+AC_SUBST(TESTCC)
+AC_SUBST([HAVE_MPI],[$have_mpi])
+AC_DEFINE_UNQUOTED([HAVE_MPI], [$have_mpi], [Whether aml supports MPI.])
+AM_CONDITIONAL([HAVE_MPI], [test "$have_mpi" == "1"])
+
 # NUMA support
 ##############
 
@@ -450,6 +459,13 @@ Active:  $HAVE_OPENMP
 CFLAGS:  $OPENMP_CFLAGS
 LDFLAGS: $OPENMP_LIBS
 
+MPI:
+======
+
+Active:  $HAVE_MPI
+TESTCC: $TESTCC
+LDFLAGS: $MPILIBS
+
 HWLOC:
 ======
 
diff --git a/include/aml/area/mpi.h b/include/aml/area/mpi.h
new file mode 100644
index 00000000..82e2b111
--- /dev/null
+++ b/include/aml/area/mpi.h
@@ -0,0 +1,143 @@
+/*******************************************************************************
+ * Copyright 2019 UChicago Argonne, LLC.
+ * (c.f. AUTHORS, LICENSE)
+ *
+ * This file is part of the AML project.
+ * For more info, see https://github.com/anlsys/aml
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ ******************************************************************************/
+
+#ifndef AML_AREA_MPI_H
+#define AML_AREA_MPI_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <mpi.h>
+
+/**
+ * @defgroup aml_area_mpi "AML MPI Areas"
+ * @brief MPI RDMA Implementation of AML Areas.
+ *
+ * This building block relies on the MPI One-sided operations to implement
+ * allocations.
+ *
+ * @code
+ * #include <aml/area/mpi.h>
+ * @endcode
+ * @{
+ **/
+
+/**
+ * Options that can eventually be passed to mmap
+ * call.
+ **/
+struct aml_area_mpi_mmap_options {
+	/**
+	 * Specify the communicator for an allocation.
+	 **/
+	MPI_Comm comm;
+	/**
+	 * Specify the info for an allocation.
+	 **/
+	MPI_Info info;
+	/**
+	 * Return variable for the underlying window.
+	 **/
+	MPI_Win win;
+	/**
+	 * Unit displacement (in bytes)
+	 **/
+	int disp;
+	
+};
+
+/**
+ * Contains area operations implementation
+ * for the MPI area.
+ **/
+extern struct aml_area_ops aml_area_mpi_ops;
+
+/**
+ * Default MPI area:
+ * Uses COMM_WORLD and allocates every time.
+ * Can be used out-of-the-box with aml_area_*() functions.
+ **/
+extern struct aml_area aml_area_mpi;
+
+/**
+ * Implementation of aml_area_data for MPI areas.
+ **/
+struct aml_area_mpi_data {
+	/** hash table keeping track of windows **/
+	struct aml_area_mpi_window *windows;
+};
+
+/**
+ * \brief MPI area creation.
+ *
+ * Allocates and initializes struct aml_area implemented by aml_area_mpi
+ * operations.
+ * @param[out] area pointer to an uninitialized struct aml_area pointer to
+ *       receive the new area.
+ * @return On success, returns 0 and fills "area" with a pointer to the new
+ *       aml_area.
+ * @return On failure, fills "area" with NULL and returns one of AML error
+ * codes:
+ * - AML_ENOMEM if there wasn't enough memory available.
+ **/
+int aml_area_mpi_create(struct aml_area **area);
+
+
+/**
+ * \brief MPI area destruction.
+ *
+ * Destroys (finalizes and frees resources) struct aml_area created by
+ * aml_area_mpi_create().
+ *
+ * @param area address of an initialized struct aml_area pointer, which will be
+ * reset to NULL on return from this call.
+ **/
+void aml_area_mpi_destroy(struct aml_area **area);
+
+/**
+ * \brief mmap block for AML area.
+ *
+ * This function is a wrapper around the MPI_Win_allocate call using arguments
+ * set in opts.
+ * @param area_data: An aml_area_mpi_data.
+ * @param size: The size to allocate.
+ * @param opts: See "aml_area_mpi_mmap_options".
+ * @return a valid memory pointer, or NULL on failure.
+ * On failure, "errno" should be checked for further information.
+ **/
+void*
+aml_area_mpi_mmap(const struct aml_area_data  *area_data,
+		    size_t                       size,
+		    struct aml_area_mmap_options *opts);
+
+/**
+ * \brief munmap hook for AML area.
+ *
+ * Unmaps memory mapped with aml_area_mpi_mmap().
+ * @param area_data: unused
+ * @param ptr: The virtual memory to unmap.
+ * @param size: The size of the virtual memory to unmap.
+ * @return AML_SUCCESS on success, else AML_FAILURE.
+ * On failure, "errno" should be checked for further information.
+ **/
+int
+aml_area_mpi_munmap(const struct aml_area_data *area_data,
+		      void *ptr,
+		      const size_t size);
+
+/**
+ * @}
+ **/
+
+#ifdef __cplusplus
+}
+#endif
+#endif //AML_AREA_MPI_H
diff --git a/m4/ax_mpi.m4 b/m4/ax_mpi.m4
new file mode 100644
index 00000000..f6db3d0c
--- /dev/null
+++ b/m4/ax_mpi.m4
@@ -0,0 +1,176 @@
+# ===========================================================================
+#          https://www.gnu.org/software/autoconf-archive/ax_mpi.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_MPI([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]])
+#
+# DESCRIPTION
+#
+#   This macro tries to find out how to compile programs that use MPI
+#   (Message Passing Interface), a standard API for parallel process
+#   communication (see http://www-unix.mcs.anl.gov/mpi/)
+#
+#   On success, it sets the MPICC, MPICXX, MPIF77, or MPIFC output variable
+#   to the name of the MPI compiler, depending upon the current language.
+#   (This may just be $CC/$CXX/$F77/$FC, but is more often something like
+#   mpicc/mpiCC/mpif77/mpif90.) It also sets MPILIBS to any libraries that
+#   are needed for linking MPI (e.g. -lmpi or -lfmpi, if a special
+#   MPICC/MPICXX/MPIF77/MPIFC was not found).
+#
+#   Note that this macro should be used only if you just have a few source
+#   files that need to be compiled using MPI. In particular, you should
+#   neither overwrite CC/CXX/F77/FC with the values of
+#   MPICC/MPICXX/MPIF77/MPIFC, nor assume that you can use the same flags
+#   etc. as the standard compilers. If you want to compile a whole program
+#   using the MPI compiler commands, use one of the macros
+#   AX_PROG_{CC,CXX,FC}_MPI.
+#
+#   ACTION-IF-FOUND is a list of shell commands to run if an MPI library is
+#   found, and ACTION-IF-NOT-FOUND is a list of commands to run if it is not
+#   found. If ACTION-IF-FOUND is not specified, the default action will
+#   define HAVE_MPI.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Steven G. Johnson <stevenj@alum.mit.edu>
+#   Copyright (c) 2008 Julian C. Cummings <cummings@cacr.caltech.edu>
+#
+#   This program is free software: you can redistribute it and/or modify it
+#   under the terms of the GNU General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+#   Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+#   As a special exception, the respective Autoconf Macro's copyright owner
+#   gives unlimited permission to copy, distribute and modify the configure
+#   scripts that are the output of Autoconf when processing the Macro. You
+#   need not follow the terms of the GNU General Public License when using
+#   or distributing such scripts, even though portions of the text of the
+#   Macro appear in them. The GNU General Public License (GPL) does govern
+#   all other use of the material that constitutes the Autoconf Macro.
+#
+#   This special exception to the GPL applies to versions of the Autoconf
+#   Macro released by the Autoconf Archive. When you make and distribute a
+#   modified version of the Autoconf Macro, you may extend this special
+#   exception to the GPL to apply to your modified version as well.
+
+#serial 9
+
+AU_ALIAS([ACX_MPI], [AX_MPI])
+AC_DEFUN([AX_MPI], [
+AC_PREREQ(2.50) dnl for AC_LANG_CASE
+
+AC_LANG_CASE([C], [
+	AC_REQUIRE([AC_PROG_CC])
+	AC_ARG_VAR(MPICC,[MPI C compiler command])
+	AC_CHECK_PROGS(MPICC, mpicc hcc mpxlc_r mpxlc mpcc cmpicc, $CC)
+	ax_mpi_save_CC="$CC"
+	CC="$MPICC"
+	AC_SUBST(MPICC)
+],
+[C++], [
+	AC_REQUIRE([AC_PROG_CXX])
+	AC_ARG_VAR(MPICXX,[MPI C++ compiler command])
+	AC_CHECK_PROGS(MPICXX, mpic++ mpicxx mpiCC hcp mpxlC_r mpxlC mpCC cmpic++, $CXX)
+	ax_mpi_save_CXX="$CXX"
+	CXX="$MPICXX"
+	AC_SUBST(MPICXX)
+],
+[Fortran 77], [
+	AC_REQUIRE([AC_PROG_F77])
+	AC_ARG_VAR(MPIF77,[MPI Fortran 77 compiler command])
+	AC_CHECK_PROGS(MPIF77, mpif77 hf77 mpxlf_r mpxlf mpf77 cmpifc, $F77)
+	ax_mpi_save_F77="$F77"
+	F77="$MPIF77"
+	AC_SUBST(MPIF77)
+],
+[Fortran], [
+	AC_REQUIRE([AC_PROG_FC])
+	AC_ARG_VAR(MPIFC,[MPI Fortran compiler command])
+	AC_CHECK_PROGS(MPIFC, mpif90 mpxlf95_r mpxlf90_r mpxlf95 mpxlf90 mpf90 cmpif90c, $FC)
+	ax_mpi_save_FC="$FC"
+	FC="$MPIFC"
+	AC_SUBST(MPIFC)
+])
+
+if test x = x"$MPILIBS"; then
+	AC_LANG_CASE([C], [AC_CHECK_FUNC(MPI_Init, [MPILIBS=" "])],
+		[C++], [AC_CHECK_FUNC(MPI_Init, [MPILIBS=" "])],
+		[Fortran 77], [AC_MSG_CHECKING([for MPI_Init])
+			AC_LINK_IFELSE([AC_LANG_PROGRAM([],[      call MPI_Init])],[MPILIBS=" "
+				AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])],
+		[Fortran], [AC_MSG_CHECKING([for MPI_Init])
+			AC_LINK_IFELSE([AC_LANG_PROGRAM([],[      call MPI_Init])],[MPILIBS=" "
+				AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])])
+fi
+AC_LANG_CASE([Fortran 77], [
+	if test x = x"$MPILIBS"; then
+		AC_CHECK_LIB(fmpi, MPI_Init, [MPILIBS="-lfmpi"])
+	fi
+	if test x = x"$MPILIBS"; then
+		AC_CHECK_LIB(fmpich, MPI_Init, [MPILIBS="-lfmpich"])
+	fi
+],
+[Fortran], [
+	if test x = x"$MPILIBS"; then
+		AC_CHECK_LIB(fmpi, MPI_Init, [MPILIBS="-lfmpi"])
+	fi
+	if test x = x"$MPILIBS"; then
+		AC_CHECK_LIB(mpichf90, MPI_Init, [MPILIBS="-lmpichf90"])
+	fi
+])
+if test x = x"$MPILIBS"; then
+	AC_CHECK_LIB(mpi, MPI_Init, [MPILIBS="-lmpi"])
+fi
+if test x = x"$MPILIBS"; then
+	AC_CHECK_LIB(mpich, MPI_Init, [MPILIBS="-lmpich"])
+fi
+
+dnl We have to use AC_TRY_COMPILE and not AC_CHECK_HEADER because the
+dnl latter uses $CPP, not $CC (which may be mpicc).
+AC_LANG_CASE([C], [if test x != x"$MPILIBS"; then
+	AC_MSG_CHECKING([for mpi.h])
+	AC_TRY_COMPILE([#include <mpi.h>],[],[AC_MSG_RESULT(yes)], [MPILIBS=""
+		AC_MSG_RESULT(no)])
+fi],
+[C++], [if test x != x"$MPILIBS"; then
+	AC_MSG_CHECKING([for mpi.h])
+	AC_TRY_COMPILE([#include <mpi.h>],[],[AC_MSG_RESULT(yes)], [MPILIBS=""
+		AC_MSG_RESULT(no)])
+fi],
+[Fortran 77], [if test x != x"$MPILIBS"; then
+	AC_MSG_CHECKING([for mpif.h])
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[      include 'mpif.h'])],[AC_MSG_RESULT(yes)], [MPILIBS=""
+		AC_MSG_RESULT(no)])
+fi],
+[Fortran], [if test x != x"$MPILIBS"; then
+	AC_MSG_CHECKING([for mpif.h])
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[      include 'mpif.h'])],[AC_MSG_RESULT(yes)], [MPILIBS=""
+		AC_MSG_RESULT(no)])
+fi])
+
+AC_LANG_CASE([C], [CC="$ax_mpi_save_CC"],
+	[C++], [CXX="$ax_mpi_save_CXX"],
+	[Fortran 77], [F77="$ax_mpi_save_F77"],
+	[Fortran], [FC="$ax_mpi_save_FC"])
+
+AC_SUBST(MPILIBS)
+
+# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND:
+if test x = x"$MPILIBS"; then
+        $2
+        :
+else
+        ifelse([$1],,[AC_DEFINE(HAVE_MPI,1,[Define if you have the MPI library.])],[$1])
+        :
+fi
+])dnl AX_MPI
diff --git a/nix/aml.nix b/nix/aml.nix
index b66b2c7b..9f48a24a 100644
--- a/nix/aml.nix
+++ b/nix/aml.nix
@@ -1,7 +1,7 @@
-{ stdenv, autoreconfHook, git, pkgconf, numactl, hwloc}:
+{ stdenv, autoreconfHook, git, pkgconf, numactl, hwloc, mpich}:
 stdenv.mkDerivation {
   src = ../.;
   name = "aml";
   nativeBuildInputs = [ autoreconfHook pkgconf git ];
-  buildInputs = [ hwloc numactl ];
+  buildInputs = [ hwloc numactl mpich ];
 }
diff --git a/src/Makefile.am b/src/Makefile.am
index 44994bad..0501e409 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -115,3 +115,12 @@ libaml_la_SOURCES+=area/ze.c
 libaml_la_SOURCES+=dma/ze.c
 libaml_la_SOURCES+=backend/ze.c
 endif
+
+#############################################
+# MPI sources
+
+if HAVE_MPI
+AM_LDFLAGS += $(MPILIBS)
+libaml_la_SOURCES+=area/mpi.c
+endif
+
diff --git a/src/area/mpi.c b/src/area/mpi.c
new file mode 100644
index 00000000..54c3b5fe
--- /dev/null
+++ b/src/area/mpi.c
@@ -0,0 +1,133 @@
+/*******************************************************************************
+ * Copyright 2019 UChicago Argonne, LLC.
+ * (c.f. AUTHORS, LICENSE)
+ *
+ * This file is part of the AML project.
+ * For more info, see https://github.com/anlsys/aml
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ ******************************************************************************/
+
+#include <mpi.h>
+#include "aml.h"
+#include "aml/area/mpi.h"
+
+#include "internal/uthash.h"
+
+/*******************************************************************************
+ * Implementation
+ ******************************************************************************/
+
+struct aml_area_mpi_window {
+	void *ptr;
+	MPI_Win win;
+	UT_hash_handle hh;
+};
+
+void *aml_area_mpi_mmap(const struct aml_area_data *area_data,
+			 size_t size, struct aml_area_mmap_options *options)
+{
+	int err;
+	struct aml_area_mpi_data *data;
+	struct aml_area_mpi_mmap_options *opts;
+
+	data = (struct aml_area_mpi_data *)area_data;
+	opts = (struct aml_area_mpi_mmap_options *)options;
+
+	MPI_Comm comm;
+	MPI_Info info;
+	int disp;
+
+	if (opts != NULL) {
+		comm = opts->comm;
+		info = opts->info;
+		disp = opts->disp;
+	} else {
+		comm = MPI_COMM_WORLD;
+		MPI_Comm_get_info(comm, &info);
+		disp = 1;
+	}
+	struct aml_area_mpi_window *w = calloc(1, sizeof(*w));
+	assert(w != NULL);
+	err = MPI_Win_allocate(size, disp, info, comm, &w->ptr, &w->win);
+	/* TODO error conversion */
+	if (err != MPI_SUCCESS) {
+		free(w);
+		aml_errno = -AML_FAILURE;
+		return MAP_FAILED;
+	}
+
+	/* store the window for munmap */
+	HASH_ADD_PTR(data->windows, ptr, w);
+	if (opts != NULL) {
+		opts->win = w->win;
+	}
+	return w->ptr;
+}
+
+int aml_area_mpi_munmap(const struct aml_area_data *area_data,
+			 void *ptr, const size_t size)
+{
+	(void)size;
+	struct aml_area_mpi_data *data;
+	struct aml_area_mpi_window *w = NULL;
+
+	data = (struct aml_area_mpi_data *)area_data;
+
+	HASH_FIND_PTR(data->windows, &ptr, w);
+	MPI_Win_free(&w->win);
+	HASH_DEL(data->windows, w);
+	free(w);
+	return AML_SUCCESS;
+}
+
+/*******************************************************************************
+ * Areas Initialization
+ ******************************************************************************/
+
+int aml_area_mpi_create(struct aml_area **area)
+{
+	struct aml_area *ret;
+	struct aml_area_mpi_data *data;
+
+	ret = AML_INNER_MALLOC(struct aml_area,
+				      struct aml_area_mpi_data);
+	if (ret == NULL)
+		return -AML_ENOMEM;
+
+	data = AML_INNER_MALLOC_GET_FIELD(ret, 2, struct aml_area,
+					  struct aml_area_mpi_data);
+
+	ret->ops = &aml_area_mpi_ops;
+	ret->data = (struct aml_area_data *)data;
+
+	*area = ret;
+	return AML_SUCCESS;
+}
+
+void aml_area_mpi_destroy(struct aml_area **area)
+{
+	if (*area == NULL)
+		return;
+
+	free(*area);
+	*area = NULL;
+}
+
+/*******************************************************************************
+ * Areas declaration
+ ******************************************************************************/
+
+struct aml_area_mpi_data aml_area_mpi_data_default = {
+	.windows = NULL,
+};
+
+struct aml_area_ops aml_area_mpi_ops = {
+	.mmap = aml_area_mpi_mmap,
+	.munmap = aml_area_mpi_munmap,
+};
+
+struct aml_area aml_area_mpi = {
+	.ops = &aml_area_mpi_ops,
+	.data = (struct aml_area_data *)(&aml_area_mpi_data_default)
+};
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 8e99c572..d613c327 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -3,6 +3,7 @@ AM_COLOR_TESTS = yes
 
 AM_CFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/excit/src $(PTHREAD_CFLAGS) $(OPENMP_CFLAGS)
 AM_LDFLAGS = ../src/libaml.la $(top_builddir)/excit/src/libexcit.la $(PTHREAD_LIBS) $(OPENMP_LIBS)
+CC=$(TESTCC)
 
 if HAVE_CUDA
 LIBS += $(CUDA_LIBS)
@@ -122,6 +123,10 @@ AREA_TESTS+=area/test_hip
 AREA_TESTS+=dma/test_hip
 endif
 
+if HAVE_MPI
+AREA_TESTS+=area/test_mpi
+endif
+
 # unit tests
 UNIT_TESTS = $(UTILS_TESTS) \
 	     $(TILING_TESTS) \
@@ -134,6 +139,7 @@ UNIT_TESTS = $(UTILS_TESTS) \
 # all tests
 TST_PROGS = $(UNIT_TESTS)
 
+#TODO: odd, figure out if it's actually required
 if HAVE_CUDA
 test_area_cuda_SOURCES=area/test_cuda.c
 test_area_cuda_CPPFLAGS= $(AM_CFLAGS) $(CUDA_CFLAGS)
@@ -145,6 +151,7 @@ test_dma_cuda_LDFLAGS=$(AM_LDFLAGS) $(CUDA_LIBS)
 TST_PROGS+=test_dma_cuda test_area_cuda
 endif
 
+
 check_PROGRAMS = $(TST_PROGS)
 check_LTLIBRARIES = \
 	libdma_test.la \
diff --git a/tests/area/test_mpi.c b/tests/area/test_mpi.c
new file mode 100644
index 00000000..fc35f3a4
--- /dev/null
+++ b/tests/area/test_mpi.c
@@ -0,0 +1,48 @@
+/*******************************************************************************
+ * Copyright 2019 UChicago Argonne, LLC.
+ * (c.f. AUTHORS, LICENSE)
+ *
+ * This file is part of the AML project.
+ * For more info, see https://github.com/anlsys/aml
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ ******************************************************************************/
+
+#include "aml.h"
+#include "config.h"
+#include "aml/area/mpi.h"
+#include <assert.h>
+
+void test_map(const struct aml_area *area)
+{
+	assert(area != NULL);
+	assert(area->ops->mmap != NULL);
+	assert(area->ops->munmap != NULL);
+
+	void *ptr;
+	size_t s;
+	const size_t sizes[4] = {1, 32, 4096, 1<<20};
+
+	for (s = 0; s < sizeof(sizes)/sizeof(*sizes); s++) {
+		aml_errno = AML_SUCCESS;
+		ptr = aml_area_mmap(area, sizes[s], NULL);
+		assert(aml_errno == AML_SUCCESS);
+		assert(ptr != NULL);
+		assert(aml_area_munmap(area, ptr, sizes[s]) == AML_SUCCESS);
+	}
+}
+
+void test_aml_area(struct aml_area *area)
+{
+	test_map(area);
+}
+
+int main(int argc, char **argv)
+{
+	MPI_Init(&argc, &argv);
+	aml_init(&argc, &argv);
+	test_map(&aml_area_mpi);
+	aml_finalize();
+	MPI_Finalize();
+	return 0;
+}

From b29a0dabed571da351754a5542d5c275e1f63ab0 Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Tue, 16 Sep 2025 10:39:09 -0500
Subject: [PATCH 02/11] [fix] move mpi files to separate dirs

MPICC doesn't work for the entire aml, and changing the CC for just a
file is difficult, so let's put all mpi logic in subdirs, and create a
libaml-mpi.so for the mpi support.
---
 configure.ac                               |  2 ++
 include/Makefile.am                        |  4 ++++
 include/aml/area/mpi.h                     | 18 +++++++-----------
 src/Makefile.am                            |  4 ++--
 src/mpi/Makefile.am                        | 13 +++++++++++++
 src/{area/mpi.c => mpi/area.c}             |  0
 tests/Makefile.am                          | 10 +++++-----
 tests/mpi/Makefile.am                      | 16 ++++++++++++++++
 tests/{area/test_mpi.c => mpi/test_area.c} | 11 +++++++----
 9 files changed, 56 insertions(+), 22 deletions(-)
 create mode 100644 src/mpi/Makefile.am
 rename src/{area/mpi.c => mpi/area.c} (100%)
 create mode 100644 tests/mpi/Makefile.am
 rename tests/{area/test_mpi.c => mpi/test_area.c} (90%)

diff --git a/configure.ac b/configure.ac
index dc0c4de8..aa7826f9 100644
--- a/configure.ac
+++ b/configure.ac
@@ -426,8 +426,10 @@ AC_CONFIG_HEADERS([include/internal/config.h])
 
 AC_CONFIG_FILES([Makefile
 		 src/Makefile
+		 src/mpi/Makefile
 		 include/Makefile
 		 tests/Makefile
+		 tests/mpi/Makefile
 		 doc/tutorials/Makefile
 		 doc/Makefile
 		 doc/tutorials/hello_world/Makefile
diff --git a/include/Makefile.am b/include/Makefile.am
index d7ca0b54..929478a3 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -81,6 +81,10 @@ include_aml_dma_HEADERS += aml/dma/ze.h
 include_aml_backend_HEADERS += aml/utils/backend/ze.h
 endif
 
+if HAVE_MPI
+include_aml_area_HEADERS += aml/area/mpi.h
+endif
+
 noinst_include_internaldir = $(includedir)/internal
 noinst_include_internal_HEADERS = \
 				  internal/utarray.h \
diff --git a/include/aml/area/mpi.h b/include/aml/area/mpi.h
index 82e2b111..2eeca428 100644
--- a/include/aml/area/mpi.h
+++ b/include/aml/area/mpi.h
@@ -51,7 +51,6 @@ struct aml_area_mpi_mmap_options {
 	 * Unit displacement (in bytes)
 	 **/
 	int disp;
-	
 };
 
 /**
@@ -90,7 +89,6 @@ struct aml_area_mpi_data {
  **/
 int aml_area_mpi_create(struct aml_area **area);
 
-
 /**
  * \brief MPI area destruction.
  *
@@ -113,10 +111,9 @@ void aml_area_mpi_destroy(struct aml_area **area);
  * @return a valid memory pointer, or NULL on failure.
  * On failure, "errno" should be checked for further information.
  **/
-void*
-aml_area_mpi_mmap(const struct aml_area_data  *area_data,
-		    size_t                       size,
-		    struct aml_area_mmap_options *opts);
+void *aml_area_mpi_mmap(const struct aml_area_data *area_data,
+                        size_t size,
+                        struct aml_area_mmap_options *opts);
 
 /**
  * \brief munmap hook for AML area.
@@ -128,10 +125,9 @@ aml_area_mpi_mmap(const struct aml_area_data  *area_data,
  * @return AML_SUCCESS on success, else AML_FAILURE.
  * On failure, "errno" should be checked for further information.
  **/
-int
-aml_area_mpi_munmap(const struct aml_area_data *area_data,
-		      void *ptr,
-		      const size_t size);
+int aml_area_mpi_munmap(const struct aml_area_data *area_data,
+                        void *ptr,
+                        const size_t size);
 
 /**
  * @}
@@ -140,4 +136,4 @@ aml_area_mpi_munmap(const struct aml_area_data *area_data,
 #ifdef __cplusplus
 }
 #endif
-#endif //AML_AREA_MPI_H
+#endif // AML_AREA_MPI_H
diff --git a/src/Makefile.am b/src/Makefile.am
index 0501e409..2046d3bc 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -2,6 +2,7 @@ SUFFIXES=.c .cu
 AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/excit/src $(PTHREAD_CFLAGS) $(OPENMP_CFLAGS)
 AM_LDFLAGS = $(PTHREAD_LIBS) $(OPENMP_LIBS)
 noinst_LTLIBRARIES=
+SUBDIRS =
 
 #############################################
 # valgrind support
@@ -120,7 +121,6 @@ endif
 # MPI sources
 
 if HAVE_MPI
-AM_LDFLAGS += $(MPILIBS)
-libaml_la_SOURCES+=area/mpi.c
+SUBDIRS += mpi
 endif
 
diff --git a/src/mpi/Makefile.am b/src/mpi/Makefile.am
new file mode 100644
index 00000000..4d7c5afc
--- /dev/null
+++ b/src/mpi/Makefile.am
@@ -0,0 +1,13 @@
+AM_CPPFLAGS = \
+	      -I$(top_srcdir)/include  \
+	      -I$(top_builddir)/include \
+	      -I$(top_srcdir)/excit/src \
+	      $(PTHREAD_CFLAGS) $(OPENMP_CFLAGS)
+AM_LDFLAGS = $(PTHREAD_LIBS) $(OPENMP_LIBS)
+CC=@MPICC@
+
+lib_LTLIBRARIES = libaml-mpi.la
+
+libaml_mpi_la_SOURCES = area.c
+
+libaml_mpi_la_LIBADD = $(top_builddir)/src/libaml.la
diff --git a/src/area/mpi.c b/src/mpi/area.c
similarity index 100%
rename from src/area/mpi.c
rename to src/mpi/area.c
diff --git a/tests/Makefile.am b/tests/Makefile.am
index d613c327..aa0014bf 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -3,7 +3,7 @@ AM_COLOR_TESTS = yes
 
 AM_CFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/excit/src $(PTHREAD_CFLAGS) $(OPENMP_CFLAGS)
 AM_LDFLAGS = ../src/libaml.la $(top_builddir)/excit/src/libexcit.la $(PTHREAD_LIBS) $(OPENMP_LIBS)
-CC=$(TESTCC)
+SUBDIRS =
 
 if HAVE_CUDA
 LIBS += $(CUDA_LIBS)
@@ -123,10 +123,6 @@ AREA_TESTS+=area/test_hip
 AREA_TESTS+=dma/test_hip
 endif
 
-if HAVE_MPI
-AREA_TESTS+=area/test_mpi
-endif
-
 # unit tests
 UNIT_TESTS = $(UTILS_TESTS) \
 	     $(TILING_TESTS) \
@@ -151,6 +147,10 @@ test_dma_cuda_LDFLAGS=$(AM_LDFLAGS) $(CUDA_LIBS)
 TST_PROGS+=test_dma_cuda test_area_cuda
 endif
 
+if HAVE_MPI
+SUBDIRS += mpi
+endif
+
 
 check_PROGRAMS = $(TST_PROGS)
 check_LTLIBRARIES = \
diff --git a/tests/mpi/Makefile.am b/tests/mpi/Makefile.am
new file mode 100644
index 00000000..1f343cba
--- /dev/null
+++ b/tests/mpi/Makefile.am
@@ -0,0 +1,16 @@
+AM_COLOR_TESTS = yes
+AM_CFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/excit/src $(PTHREAD_CFLAGS) $(OPENMP_CFLAGS)
+AM_LDFLAGS = ../../src/mpi/libaml-mpi.la $(top_builddir)/excit/src/libexcit.la $(PTHREAD_LIBS) $(OPENMP_LIBS) $(MPILIBS)
+CC=$(MPICC)
+
+test_area_mpi_SOURCES=test_area.c
+test_area_mpi_CPPFLAGS= $(AM_CFLAGS)
+
+TST_PROGS = test_area_mpi
+
+check_PROGRAMS = $(TST_PROGS)
+TESTS = $(TST_PROGS)
+
+# phony target to allow us to compile the check programs without running the
+# tests. Required for any environment where we are cross-compiling
+check-programs-local: $(TST_PROGS)
diff --git a/tests/area/test_mpi.c b/tests/mpi/test_area.c
similarity index 90%
rename from tests/area/test_mpi.c
rename to tests/mpi/test_area.c
index fc35f3a4..7e446bde 100644
--- a/tests/area/test_mpi.c
+++ b/tests/mpi/test_area.c
@@ -8,11 +8,14 @@
  * SPDX-License-Identifier: BSD-3-Clause
  ******************************************************************************/
 
-#include "aml.h"
 #include "config.h"
-#include "aml/area/mpi.h"
+
 #include <assert.h>
 
+#include "aml.h"
+
+#include "aml/area/mpi.h"
+
 void test_map(const struct aml_area *area)
 {
 	assert(area != NULL);
@@ -21,9 +24,9 @@ void test_map(const struct aml_area *area)
 
 	void *ptr;
 	size_t s;
-	const size_t sizes[4] = {1, 32, 4096, 1<<20};
+	const size_t sizes[4] = {1, 32, 4096, 1 << 20};
 
-	for (s = 0; s < sizeof(sizes)/sizeof(*sizes); s++) {
+	for (s = 0; s < sizeof(sizes) / sizeof(*sizes); s++) {
 		aml_errno = AML_SUCCESS;
 		ptr = aml_area_mmap(area, sizes[s], NULL);
 		assert(aml_errno == AML_SUCCESS);

From 2fae0180e1748a560e314bd5b245634e47c8af18 Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Tue, 16 Sep 2025 12:45:45 -0500
Subject: [PATCH 03/11] ci fix

---
 src/Makefile.am |  3 ++-
 src/mpi/area.c  | 24 +++++++++++++-----------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/src/Makefile.am b/src/Makefile.am
index 2046d3bc..18aa0f04 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -121,6 +121,7 @@ endif
 # MPI sources
 
 if HAVE_MPI
-SUBDIRS += mpi
+# have to make sure current gets compiled before mpi
+SUBDIRS += . mpi
 endif
 
diff --git a/src/mpi/area.c b/src/mpi/area.c
index 54c3b5fe..5a3a5644 100644
--- a/src/mpi/area.c
+++ b/src/mpi/area.c
@@ -9,7 +9,9 @@
  ******************************************************************************/
 
 #include <mpi.h>
+
 #include "aml.h"
+
 #include "aml/area/mpi.h"
 
 #include "internal/uthash.h"
@@ -25,7 +27,8 @@ struct aml_area_mpi_window {
 };
 
 void *aml_area_mpi_mmap(const struct aml_area_data *area_data,
-			 size_t size, struct aml_area_mmap_options *options)
+                        size_t size,
+                        struct aml_area_mmap_options *options)
 {
 	int err;
 	struct aml_area_mpi_data *data;
@@ -66,7 +69,8 @@ void *aml_area_mpi_mmap(const struct aml_area_data *area_data,
 }
 
 int aml_area_mpi_munmap(const struct aml_area_data *area_data,
-			 void *ptr, const size_t size)
+                        void *ptr,
+                        const size_t size)
 {
 	(void)size;
 	struct aml_area_mpi_data *data;
@@ -90,13 +94,12 @@ int aml_area_mpi_create(struct aml_area **area)
 	struct aml_area *ret;
 	struct aml_area_mpi_data *data;
 
-	ret = AML_INNER_MALLOC(struct aml_area,
-				      struct aml_area_mpi_data);
+	ret = AML_INNER_MALLOC(struct aml_area, struct aml_area_mpi_data);
 	if (ret == NULL)
 		return -AML_ENOMEM;
 
 	data = AML_INNER_MALLOC_GET_FIELD(ret, 2, struct aml_area,
-					  struct aml_area_mpi_data);
+	                                  struct aml_area_mpi_data);
 
 	ret->ops = &aml_area_mpi_ops;
 	ret->data = (struct aml_area_data *)data;
@@ -119,15 +122,14 @@ void aml_area_mpi_destroy(struct aml_area **area)
  ******************************************************************************/
 
 struct aml_area_mpi_data aml_area_mpi_data_default = {
-	.windows = NULL,
+        .windows = NULL,
 };
 
 struct aml_area_ops aml_area_mpi_ops = {
-	.mmap = aml_area_mpi_mmap,
-	.munmap = aml_area_mpi_munmap,
+        .mmap = aml_area_mpi_mmap,
+        .munmap = aml_area_mpi_munmap,
 };
 
 struct aml_area aml_area_mpi = {
-	.ops = &aml_area_mpi_ops,
-	.data = (struct aml_area_data *)(&aml_area_mpi_data_default)
-};
+        .ops = &aml_area_mpi_ops,
+        .data = (struct aml_area_data *)(&aml_area_mpi_data_default)};

From 9657136ba7d63c7743e163f02790df7fcf830594 Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Tue, 16 Sep 2025 12:58:41 -0500
Subject: [PATCH 04/11] ci fix

---
 tests/mpi/Makefile.am | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/mpi/Makefile.am b/tests/mpi/Makefile.am
index 1f343cba..ffe658d7 100644
--- a/tests/mpi/Makefile.am
+++ b/tests/mpi/Makefile.am
@@ -1,6 +1,6 @@
 AM_COLOR_TESTS = yes
 AM_CFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/excit/src $(PTHREAD_CFLAGS) $(OPENMP_CFLAGS)
-AM_LDFLAGS = ../../src/mpi/libaml-mpi.la $(top_builddir)/excit/src/libexcit.la $(PTHREAD_LIBS) $(OPENMP_LIBS) $(MPILIBS)
+AM_LDFLAGS = ../../src/libaml.la ../../src/mpi/libaml-mpi.la $(top_builddir)/excit/src/libexcit.la $(PTHREAD_LIBS) $(OPENMP_LIBS) $(MPILIBS)
 CC=$(MPICC)
 
 test_area_mpi_SOURCES=test_area.c

From 7385a747f23c1087327504f4dc867036aa2b2bca Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Tue, 16 Sep 2025 13:18:44 -0500
Subject: [PATCH 05/11] ci fix

---
 tests/mpi/test_area.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/mpi/test_area.c b/tests/mpi/test_area.c
index 7e446bde..c6eb8ed0 100644
--- a/tests/mpi/test_area.c
+++ b/tests/mpi/test_area.c
@@ -42,6 +42,10 @@ void test_aml_area(struct aml_area *area)
 
 int main(int argc, char **argv)
 {
+	/* impossible to do those check in a CI environment consistently */
+	if (!strcmp(getenv("CI"), "true"))
+		exit(77);
+
 	MPI_Init(&argc, &argv);
 	aml_init(&argc, &argv);
 	test_map(&aml_area_mpi);

From fdde969da3db2a613c00b29d268e7c85295ea7ec Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Thu, 18 Sep 2025 14:54:04 -0500
Subject: [PATCH 06/11] ongoing fixes

---
 configure.ac                               | 52 +++++++++++++++++++---
 include/Makefile.am                        |  5 +++
 include/aml/utils/backend/mpi.h            | 50 +++++++++++++++++++++
 include/aml/utils/features.h.in            | 16 ++++---
 src/Makefile.am                            |  5 ++-
 src/aml.c                                  | 30 +++++++++++--
 src/{mpi/area.c => area/mpi.c}             |  0
 src/backend/mpi.c                          | 21 +++++++++
 src/mpi/Makefile.am                        | 13 ------
 src/utils/features.c                       | 22 +++++++++
 tests/Makefile.am                          | 19 +++++---
 tests/{mpi/test_area.c => area/test_mpi.c} |  0
 tests/mpi/Makefile.am                      | 16 -------
 tests/replicaset/test_hwloc.c              |  2 +-
 14 files changed, 201 insertions(+), 50 deletions(-)
 create mode 100644 include/aml/utils/backend/mpi.h
 rename src/{mpi/area.c => area/mpi.c} (100%)
 create mode 100644 src/backend/mpi.c
 delete mode 100644 src/mpi/Makefile.am
 rename tests/{mpi/test_area.c => area/test_mpi.c} (100%)
 delete mode 100644 tests/mpi/Makefile.am

diff --git a/configure.ac b/configure.ac
index aa7826f9..7a2327fd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -143,9 +143,50 @@ AC_SUBST(OPENMP_LIBS)
 # MPI support
 #############
 
-AX_MPI(have_mpi=1,have_mpi=0)
-TESTCC="$MPICC"
-AC_SUBST(TESTCC)
+AC_ARG_WITH([mpi],
+	[AS_HELP_STRING([--with-mpi],
+		[support mpi inside the library (default is check)])],
+	[
+	if test "x$withval" = "xno"; then
+		want_mpi="no"
+	else
+		want_mpi="yes"
+	fi
+	],
+	[
+		want_mpi="check"
+	])
+
+MPI_CFLAGS=""
+MPI_LIBS=""
+if test "x$want_mpi" != "xno"; then
+	AC_MSG_NOTICE([starting checks for MPI])
+	AC_CHECK_PROGS(MPICC, mpicc hcc mpxlc_r mpxlc mpcc cmpicc, no)
+
+	if test "x$MPICC" = "xno"; then
+		AC_MSG_NOTICE([No MPI compiler found])
+		if test "x$want_mpi" = "xyes"; then
+        		AC_MSG_ERROR([MPI compiler (mpicc) not found but --with-mpi was requested])
+		fi
+		
+		have_mpi=0
+	else
+		MPI_CC_SHOW=`$MPICC -show 2>/dev/null`
+		for arg in $MPI_CC_SHOW; do
+			case "$arg" in
+				-I*|-D*|-std=*|-f*) MPI_CFLAGS="$MPI_CFLAGS $arg" ;;
+				-L*|-l*|-Wl,*|-W*)  MPI_LIBS="$MPI_LIBS $arg" ;;
+				*) ;;  # Skip compiler name or unknown args
+			esac
+		done
+		have_mpi=1
+		CC="$MPICC"
+	fi
+fi
+
+AC_SUBST([MPI_CFLAGS])
+AC_SUBST([MPI_LIBS])
+AC_SUBST([MPICC])
 AC_SUBST([HAVE_MPI],[$have_mpi])
 AC_DEFINE_UNQUOTED([HAVE_MPI], [$have_mpi], [Whether aml supports MPI.])
 AM_CONDITIONAL([HAVE_MPI], [test "$have_mpi" == "1"])
@@ -465,8 +506,9 @@ MPI:
 ======
 
 Active:  $HAVE_MPI
-TESTCC: $TESTCC
-LDFLAGS: $MPILIBS
+MPICC:   $MPICC
+CFLAGS:  $MPI_CFLAGS
+LDFLAGS: $MPI_LIBS
 
 HWLOC:
 ======
diff --git a/include/Makefile.am b/include/Makefile.am
index 929478a3..81a069ee 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -71,6 +71,11 @@ include_aml_area_HEADERS += aml/area/hip.h
 include_aml_dma_HEADERS += aml/dma/hip.h
 endif
 
+if HAVE_MPI
+include_aml_area_HEADERS += aml/area/mpi.h
+include_aml_backend_HEADERS += aml/utils/backend/mpi.h
+endif
+
 if HAVE_OPENCL
 include_aml_area_HEADERS += aml/area/opencl.h
 endif
diff --git a/include/aml/utils/backend/mpi.h b/include/aml/utils/backend/mpi.h
new file mode 100644
index 00000000..a43f860a
--- /dev/null
+++ b/include/aml/utils/backend/mpi.h
@@ -0,0 +1,50 @@
+/*******************************************************************************
+ * Copyright 2019 UChicago Argonne, LLC.
+ * (c.f. AUTHORS, LICENSE)
+ *
+ * This file is part of the AML project.
+ * For more info, see https://github.com/anlsys/aml
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ ******************************************************************************/
+
+#ifndef AML_UTILS_BACKEND_MPI_H
+#define AML_UTILS_BACKEND_MPI_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @defgroup aml_backend_mpi "AML MPI Utils"
+ * @brief Boilerplate Code and Initialization for MPI Backend.
+ * @code
+ * #include <aml/utils/backend/mpi.h>
+ * @endcode
+ *
+ * @{
+ **/
+
+/**
+ * MPI backend initialization function.
+ * This function should only be called once.
+ * This function should not fail unless the system is out of memory.
+ *
+ * @return AML_SUCCESS on success.
+ * @return -AML_ENOMEM on error.
+ */
+int aml_backend_mpi_init(void);
+
+/**
+ * linux backend initialization function.
+ */
+int aml_backend_mpi_finalize(void);
+
+/**
+ * @}
+ **/
+
+#ifdef __cplusplus
+}
+#endif
+#endif // AML_UTILS_BACKEND_MPI_H
diff --git a/include/aml/utils/features.h.in b/include/aml/utils/features.h.in
index 6587ed4d..472956da 100644
--- a/include/aml/utils/features.h.in
+++ b/include/aml/utils/features.h.in
@@ -35,19 +35,23 @@
 #define AML_HAVE_BACKEND_ZE @HAVE_ZE@
 /** Whether aml had hip capabilities at compile time **/
 #define AML_HAVE_BACKEND_HIP @HAVE_HIP@
+/** Whether aml had mpi capabilities at compile time **/
+#define AML_HAVE_BACKEND_MPI @HAVE_MPI@
 
-/** Flag for checking runtime suport for libnuma **/
+/** Flag for checking runtime support for libnuma **/
 #define AML_BACKEND_LIBNUMA (1UL<<1)
-/** Flag for checking runtime suport for cuda **/
+/** Flag for checking runtime support for cuda **/
 #define AML_BACKEND_CUDA (1UL<<2)
-/** Flag for checking runtime suport for hwloc **/
+/** Flag for checking runtime support for hwloc **/
 #define AML_BACKEND_HWLOC (1UL<<3)
-/** Flag for checking runtime suport for opencl **/
+/** Flag for checking runtime support for opencl **/
 #define AML_BACKEND_OPENCL (1UL<<4)
-/** Flag for checking runtime suport for level zero **/
+/** Flag for checking runtime support for level zero **/
 #define AML_BACKEND_ZE (1UL<<5)
-/** Flag for checking runtime suport for hip **/
+/** Flag for checking runtime support for hip **/
 #define AML_BACKEND_HIP (1UL<<6)
+/** Flag for checking runtime support for mpi **/
+#define AML_BACKEND_MPI (1UL<<7)
 
 /**
  * Check if a set of backends can be used at runtime.
diff --git a/src/Makefile.am b/src/Makefile.am
index 18aa0f04..0f59f1b1 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -122,6 +122,9 @@ endif
 
 if HAVE_MPI
 # have to make sure current gets compiled before mpi
-SUBDIRS += . mpi
+AM_LDFLAGS += $(MPI_LIBS)
+AM_CPPFLAGS += $(MPI_CFLAGS)
+libaml_la_SOURCES+=area/mpi.c
+libaml_la_SOURCES+=backend/mpi.c
 endif
 
diff --git a/src/aml.c b/src/aml.c
index d0c67ed0..400eb001 100644
--- a/src/aml.c
+++ b/src/aml.c
@@ -18,6 +18,9 @@
 #if HAVE_HWLOC == 1
 #include "aml/utils/backend/hwloc.h"
 #endif
+#if HAVE_MPI == 1
+#include "aml/utils/backend/mpi.h"
+#endif
 #if HAVE_ZE == 1
 #include "aml/utils/backend/ze.h"
 #endif
@@ -42,10 +45,16 @@ int aml_init(int *argc, char **argv[])
 	if (err != AML_SUCCESS)
 		return err;
 
+#if HAVE_MPI == 1
+	err = aml_backend_mpi_init();
+	if (err != AML_SUCCESS)
+		goto err_with_linux;
+#endif
+
 #if HAVE_ZE == 1
 	err = aml_backend_ze_init();
 	if (err != AML_SUCCESS)
-		goto error;
+		goto err_with_mpi;
 #endif
 
 #if HAVE_HWLOC == 1
@@ -56,12 +65,23 @@ int aml_init(int *argc, char **argv[])
 
 	return AML_SUCCESS;
 
+// bit of ugly code here: labels can't be unused so we need to only define them
+// if their caller is defined, but we only need to do something if the feature
+// before them exists.
 #if HAVE_HWLOC == 1
-err_with_ze:;
+err_with_ze:
 #endif
 #if HAVE_ZE == 1
 	aml_backend_ze_finalize();
-error:
+#endif
+#if HAVE_ZE == 1
+err_with_mpi:
+#endif
+#if HAVE_MPI == 1
+	aml_backend_mpi_finalize();
+#endif
+#if HAVE_MPI == 1
+err_with_linux:
 #endif
 	aml_backend_linux_finalize();
 	return err;
@@ -77,5 +97,9 @@ int aml_finalize(void)
 #if HAVE_ZE == 1
 	aml_backend_ze_finalize();
 #endif
+
+#if HAVE_MPI == 1
+	aml_backend_mpi_finalize();
+#endif
 	return 0;
 }
diff --git a/src/mpi/area.c b/src/area/mpi.c
similarity index 100%
rename from src/mpi/area.c
rename to src/area/mpi.c
diff --git a/src/backend/mpi.c b/src/backend/mpi.c
new file mode 100644
index 00000000..e9ae3b2f
--- /dev/null
+++ b/src/backend/mpi.c
@@ -0,0 +1,21 @@
+/*******************************************************************************
+ * Copyright 2019 UChicago Argonne, LLC.
+ * (c.f. AUTHORS, LICENSE)
+ *
+ * This file is part of the AML project.
+ * For more info, see https://github.com/anlsys/aml
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ ******************************************************************************/
+
+#include "aml.h"
+
+int aml_backend_mpi_init(void)
+{
+	return AML_SUCCESS;
+}
+
+int aml_backend_mpi_finalize(void)
+{
+	return AML_SUCCESS;
+}
diff --git a/src/mpi/Makefile.am b/src/mpi/Makefile.am
deleted file mode 100644
index 4d7c5afc..00000000
--- a/src/mpi/Makefile.am
+++ /dev/null
@@ -1,13 +0,0 @@
-AM_CPPFLAGS = \
-	      -I$(top_srcdir)/include  \
-	      -I$(top_builddir)/include \
-	      -I$(top_srcdir)/excit/src \
-	      $(PTHREAD_CFLAGS) $(OPENMP_CFLAGS)
-AM_LDFLAGS = $(PTHREAD_LIBS) $(OPENMP_LIBS)
-CC=@MPICC@
-
-lib_LTLIBRARIES = libaml-mpi.la
-
-libaml_mpi_la_SOURCES = area.c
-
-libaml_mpi_la_LIBADD = $(top_builddir)/src/libaml.la
diff --git a/src/utils/features.c b/src/utils/features.c
index 922bce51..1388d6dd 100644
--- a/src/utils/features.c
+++ b/src/utils/features.c
@@ -22,6 +22,9 @@
 #include <hwloc.h>
 extern hwloc_topology_t aml_topology;
 #endif
+#if HAVE_MPI == 1
+#include <mpi.h>
+#endif
 #if HAVE_ZE == 1
 #include <level_zero/ze_api.h>
 #endif
@@ -116,6 +119,20 @@ static int aml_support_hwloc(void)
 #endif
 }
 
+static int aml_support_mpi(void)
+{
+#if HAVE_MPI == 0
+	return 0;
+#else
+	int initialized;
+	MPI_Initialized(&initialized);
+	if (initialized)
+		return 1;
+	else
+		return 0;
+#endif
+}
+
 int aml_support_backends(const unsigned long backends)
 {
 	// Cuda check: compilation support and runtime support must be present.
@@ -128,6 +145,11 @@ int aml_support_backends(const unsigned long backends)
 	    !(AML_HAVE_BACKEND_HIP && aml_support_hip()))
 		return 0;
 
+	// mpi check: compilation support and runtime support must be present.
+	if ((backends & AML_BACKEND_MPI) &&
+	    !(AML_HAVE_BACKEND_MPI && aml_support_mpi()))
+		return 0;
+
 	// OpenCL check: compilation support and runtime support must be
 	// present.
 	if ((backends & AML_BACKEND_OPENCL) &&
diff --git a/tests/Makefile.am b/tests/Makefile.am
index aa0014bf..8b971a98 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -35,6 +35,16 @@ AM_LDFLAGS += $(HIP_LIBS)
 AM_CFLAGS += $(HIP_CFLAGS)
 endif
 
+if HAVE_MPI
+LIBS += $(MPI_LIBS)
+AM_LDFLAGS += $(MPI_LIBS)
+AM_CFLAGS += $(MPI_CFLAGS)
+endif
+
+# if the env doesn't include CI vars, default to no
+CI ?=no
+TESTS_ENVIRONMENT=$(TEST_ENVIRONMENT) CI=$(CI)
+
 # valgrind support
 @VALGRIND_CHECK_RULES@
 VALGRIND_SUPPRESSIONS_FILES=../.valgrind.supp
@@ -104,6 +114,10 @@ AREA_TESTS+=area/test_cuda
 AREA_TESTS+=dma/test_cuda
 endif
 
+if HAVE_MPI
+AREA_TESTS += area/test_mpi
+endif
+
 if HAVE_OPENCL
 AREA_TESTS += area/test_opencl
 endif
@@ -147,11 +161,6 @@ test_dma_cuda_LDFLAGS=$(AM_LDFLAGS) $(CUDA_LIBS)
 TST_PROGS+=test_dma_cuda test_area_cuda
 endif
 
-if HAVE_MPI
-SUBDIRS += mpi
-endif
-
-
 check_PROGRAMS = $(TST_PROGS)
 check_LTLIBRARIES = \
 	libdma_test.la \
diff --git a/tests/mpi/test_area.c b/tests/area/test_mpi.c
similarity index 100%
rename from tests/mpi/test_area.c
rename to tests/area/test_mpi.c
diff --git a/tests/mpi/Makefile.am b/tests/mpi/Makefile.am
deleted file mode 100644
index ffe658d7..00000000
--- a/tests/mpi/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-AM_COLOR_TESTS = yes
-AM_CFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/excit/src $(PTHREAD_CFLAGS) $(OPENMP_CFLAGS)
-AM_LDFLAGS = ../../src/libaml.la ../../src/mpi/libaml-mpi.la $(top_builddir)/excit/src/libexcit.la $(PTHREAD_LIBS) $(OPENMP_LIBS) $(MPILIBS)
-CC=$(MPICC)
-
-test_area_mpi_SOURCES=test_area.c
-test_area_mpi_CPPFLAGS= $(AM_CFLAGS)
-
-TST_PROGS = test_area_mpi
-
-check_PROGRAMS = $(TST_PROGS)
-TESTS = $(TST_PROGS)
-
-# phony target to allow us to compile the check programs without running the
-# tests. Required for any environment where we are cross-compiling
-check-programs-local: $(TST_PROGS)
diff --git a/tests/replicaset/test_hwloc.c b/tests/replicaset/test_hwloc.c
index 95b30e7a..f07196c4 100644
--- a/tests/replicaset/test_hwloc.c
+++ b/tests/replicaset/test_hwloc.c
@@ -30,7 +30,7 @@ void test_case(const hwloc_obj_type_t initiator_type,
 int main(int argc, char **argv)
 {
 	/* impossible to do those check in a CI environment consistently */
-	if (!strcmp(getenv("CI"), "true"))
+	if (!strcmp(getenv("CI") , "true"))
 		exit(77);
 
 	// Setup

From d40b3cf9ea14f5d9483d4c373ea198f4eb588b57 Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Thu, 18 Sep 2025 15:30:11 -0500
Subject: [PATCH 07/11] ci fixes

---
 configure.ac                  | 2 --
 tests/replicaset/test_hwloc.c | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index 7a2327fd..45837bab 100644
--- a/configure.ac
+++ b/configure.ac
@@ -467,10 +467,8 @@ AC_CONFIG_HEADERS([include/internal/config.h])
 
 AC_CONFIG_FILES([Makefile
 		 src/Makefile
-		 src/mpi/Makefile
 		 include/Makefile
 		 tests/Makefile
-		 tests/mpi/Makefile
 		 doc/tutorials/Makefile
 		 doc/Makefile
 		 doc/tutorials/hello_world/Makefile
diff --git a/tests/replicaset/test_hwloc.c b/tests/replicaset/test_hwloc.c
index f07196c4..95b30e7a 100644
--- a/tests/replicaset/test_hwloc.c
+++ b/tests/replicaset/test_hwloc.c
@@ -30,7 +30,7 @@ void test_case(const hwloc_obj_type_t initiator_type,
 int main(int argc, char **argv)
 {
 	/* impossible to do those check in a CI environment consistently */
-	if (!strcmp(getenv("CI") , "true"))
+	if (!strcmp(getenv("CI"), "true"))
 		exit(77);
 
 	// Setup

From d7a334e1049b9996adcad64da6c41d2cde5c017d Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Thu, 18 Sep 2025 15:46:22 -0500
Subject: [PATCH 08/11] ci fixes

---
 include/Makefile.am | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/include/Makefile.am b/include/Makefile.am
index 81a069ee..4fbb3f69 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -86,10 +86,6 @@ include_aml_dma_HEADERS += aml/dma/ze.h
 include_aml_backend_HEADERS += aml/utils/backend/ze.h
 endif
 
-if HAVE_MPI
-include_aml_area_HEADERS += aml/area/mpi.h
-endif
-
 noinst_include_internaldir = $(includedir)/internal
 noinst_include_internal_HEADERS = \
 				  internal/utarray.h \

From 6b93788fae6563ea18b536261f8aa05b94ce75dc Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Thu, 18 Sep 2025 16:03:04 -0500
Subject: [PATCH 09/11] nvhpc fix

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 6b81b4f3..10356a59 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -190,7 +190,7 @@ jobs:
       run:
         shell: bash
     container:
-      image: nvcr.io/nvidia/nvhpc:24.7-devel-cuda12.5-ubuntu22.04
+      image: nvcr.io/nvidia/nvhpc:25.7-devel-cuda12.9-ubuntu24.04
     steps:
       - uses: actions/checkout@v2
         with:

From 0d5a62729a444971631ed9eea698ea68cfd69abf Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Fri, 19 Sep 2025 15:49:08 -0500
Subject: [PATCH 10/11] ci fix

---
 .github/workflows/builds.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 10356a59..3f48cea4 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -130,7 +130,7 @@ jobs:
           fetch-depth: 0
       - uses: cachix/install-nix-action@v13
         with:
-          nix_path: nixpkgs=channel:nixos-20.09
+          nix_path: nixpkgs=channel:nixos-24.05
       - run: nix-build -A aml
   distcheck:
     runs-on: ubuntu-latest
@@ -190,7 +190,7 @@ jobs:
       run:
         shell: bash
     container:
-      image: nvcr.io/nvidia/nvhpc:25.7-devel-cuda12.9-ubuntu24.04
+      image: nvcr.io/nvidia/nvhpc:24.7-devel-cuda12.5-ubuntu24.04
     steps:
       - uses: actions/checkout@v2
         with:
@@ -206,7 +206,7 @@ jobs:
           module load nvhpc
           ./autogen.sh
           mkdir build
-          ./configure --prefix=`pwd`/build --with-cuda CUDA_HOME=$NVHPC_ROOT/cuda
+          ./configure --prefix=`pwd`/build --without-hwloc --with-cuda CUDA_HOME=$NVHPC_ROOT/cuda
           make
           make check
           make install

From 4b01cfab3d640ee91f547c96e56bac8dae1a089e Mon Sep 17 00:00:00 2001
From: Swann Perarnau <swann@anl.gov>
Date: Fri, 26 Sep 2025 15:51:57 -0500
Subject: [PATCH 11/11] fix ci

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 3f48cea4..778226b7 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -202,7 +202,7 @@ jobs:
           echo "0.8.0" > .tarball-version
       - name: build
         run: |
-          source /usr/share/lmod/6.6/init/bash
+          source /usr/share/lmod/lmod/init/bash
           module load nvhpc
           ./autogen.sh
           mkdir build