From f05d471d071778e65975a2c85a7a7ab7776ed98e Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <anastasios.zouzias@huawei.com>
Date: Thu, 25 May 2023 11:56:24 +0200
Subject: [PATCH 1/2] wip

---
 .env                                   |  4 ++
 docker-compose.yml                     | 19 +++++++
 docker/Dockerfile.devcontainer.openmpi | 55 +++++++++++++++++++
 docker/Dockerfile.openmpi              | 76 ++++++++++++++++++++++++++
 4 files changed, 154 insertions(+)
 create mode 100644 .env
 create mode 100644 docker-compose.yml
 create mode 100644 docker/Dockerfile.devcontainer.openmpi
 create mode 100644 docker/Dockerfile.openmpi

diff --git a/.env b/.env
new file mode 100644
index 0000000..5a698a8
--- /dev/null
+++ b/.env
@@ -0,0 +1,4 @@
+DOCKER_REGISTRY=ghcr.io
+DOCKER_ORG=Algebraic-Programming
+PYTORCH_VERSION=1.13.1
+OPENMPI_VERSION=4.1.5
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..5f158ce
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,19 @@
+version: "3.3"
+
+services:
+  pytorch-mpi:
+    image: "${DOCKER_REGISTRY}/${DOCKER_ORG}/pytorch-mpi:v${PYTORCH_VERSION}"
+    build:
+      context: .
+      args:
+        PYTORCH_VER: ${PYTORCH_VERSION}
+        MPI_VERSION: ${OPENMPI_VERSION}
+      dockerfile: docker/Dockerfile.openmpi
+  devcontainers-pytorch-mpi:
+    image: "${DOCKER_REGISTRY}/${DOCKER_ORG}/pytorch-mpi:v${PYTORCH_VERSION}-vscode-dev"
+    build:
+      context: .
+      args:
+        PYTORCH_VER: ${PYTORCH_VERSION}
+        MPI_VERSION: ${OPENMPI_VERSION}
+      dockerfile: docker/Dockerfile.devcontainer.openmpi
diff --git a/docker/Dockerfile.devcontainer.openmpi b/docker/Dockerfile.devcontainer.openmpi
new file mode 100644
index 0000000..cea0d42
--- /dev/null
+++ b/docker/Dockerfile.devcontainer.openmpi
@@ -0,0 +1,55 @@
+# hadolint global ignore=DL3003,DL3008,DL3013,DL3015,SC1091
+FROM mcr.microsoft.com/devcontainers/base:jammy
+
+LABEL maintainer="Anastasios Zouzias"
+
+# Installs OpenMPI with PyTorch having distributed with MPI backend enabled
+
+# Install dependencies
+RUN export DEBIAN_FRONTEND=noninteractive && apt-get update -qy && \
+    apt-get install -yq g++-9 make cmake \
+    autoconf libibverbs-dev hwloc lam-runtime python3-pip python3-venv python-is-python3 git vim wget bzip2 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 100
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-9 100
+
+# Build OpenMPI from source
+ARG MPI_VERSION="4.1.5"
+# See https://github.com/openucx/ucx/wiki/OpenMPI-and-OpenSHMEM-installation-with-UCX#running-open-mpi-with-ucx
+ARG MPI_CONFIGURE_OPTIONS="--without-pmi --enable-mpi1-compatibility  --enable-mpi-thread-multiple --without-slurm --without-verbs --disable-oshmem --disable-dependency-tracking --disable-mpi-fortran --disable-wrapper-rpath --disable-wrapper-runpath"
+ARG MPI_MAKE_OPTIONS="-j10"
+
+RUN mkdir -p /tmp/openmpi-build \
+    && cd /tmp/openmpi-build \
+    && wget --quiet "http://www.openmpi.org/software/ompi/v${MPI_VERSION%.*}/downloads/openmpi-${MPI_VERSION}.tar.bz2" \
+    && tar xjf openmpi-${MPI_VERSION}.tar.bz2 \
+    && cd openmpi-${MPI_VERSION}  \
+    && ./configure ${MPI_CONFIGURE_OPTIONS} \
+    && make ${MPI_MAKE_OPTIONS} \
+    && make install \
+    && ldconfig \
+    && cd / \
+    && rm -rf /tmp/openmpi-build
+
+USER vscode
+WORKDIR /home/vscode
+
+# Install PyTorch
+ARG PYTORCH_VER=1.13.1
+RUN python -m venv venv && venv/bin/pip install --upgrade pip && \
+    . venv/bin/activate && \
+    pip install --no-cache-dir astunparse numpy ninja pyyaml setuptools cmake \
+    cffi typing_extensions future six requests dataclasses sympy
+
+RUN git clone --depth 2 --recursive --single-branch --branch v${PYTORCH_VER}  https://github.com/pytorch/pytorch && \
+    . venv/bin/activate && cd pytorch && \
+    PATH=${PATH}:${HOME}/opt/openmpi/bin LD_LIBRARY_PATH=${HOME}/opt/openmpi/lib USE_CUDA=OFF CMAKE_C_COMPILER=$(which mpicc) CMAKE_CXX_COMPILER=$(which mpicxx) python setup.py build bdist_wheel && \
+    cd .. && \
+    cp pytorch/dist/*.whl . && \
+    rm -rf pytorch/
+
+RUN pip install --no-cache-dir  torch*.whl
+
+COPY test.py .
+COPY pytorch_gloo_bug_30723.py .
\ No newline at end of file
diff --git a/docker/Dockerfile.openmpi b/docker/Dockerfile.openmpi
new file mode 100644
index 0000000..d67ddd2
--- /dev/null
+++ b/docker/Dockerfile.openmpi
@@ -0,0 +1,76 @@
+# hadolint global ignore=DL3003,DL3008,DL3013,DL3015,SC1091
+FROM ubuntu:22.04
+
+LABEL maintainer="Anastasios Zouzias"
+
+# Installs OpenMPI with PyTorch having distributed with MPI backend enabled
+
+# Add 'ubuntu' sudoer
+RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u 1001 ubuntu
+
+# Install dependencies
+RUN export DEBIAN_FRONTEND=noninteractive && apt-get update -qy && \
+    apt-get install -yq g++-9 make cmake \
+    autoconf libibverbs-dev hwloc lam-runtime python3-pip python3-venv python-is-python3 git vim wget bzip2 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 100
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-9 100
+
+# Build OpenMPI from source
+ARG MPI_VERSION="4.1.5"
+# See https://github.com/openucx/ucx/wiki/OpenMPI-and-OpenSHMEM-installation-with-UCX#running-open-mpi-with-ucx
+ARG MPI_CONFIGURE_OPTIONS="--without-pmi --enable-mpi1-compatibility  --enable-mpi-thread-multiple --without-slurm --without-verbs --disable-oshmem --disable-dependency-tracking --disable-mpi-fortran --disable-wrapper-rpath --disable-wrapper-runpath"
+ARG MPI_MAKE_OPTIONS="-j10"
+
+RUN mkdir -p /tmp/openmpi-build \
+    && cd /tmp/openmpi-build \
+    && wget --quiet "http://www.openmpi.org/software/ompi/v${MPI_VERSION%.*}/downloads/openmpi-${MPI_VERSION}.tar.bz2" \
+    && tar xjf openmpi-${MPI_VERSION}.tar.bz2 \
+    && cd openmpi-${MPI_VERSION}  \
+    && ./configure ${MPI_CONFIGURE_OPTIONS} \
+    && make ${MPI_MAKE_OPTIONS} \
+    && make install \
+    && ldconfig \
+    && cd / \
+    && rm -rf /tmp/openmpi-build
+
+
+# Build OSU Benchmarks
+ARG OSU_VERSION="7.0.1"
+ARG OSU_CONFIGURE_OPTIONS="--prefix=/usr/local CC=mpicc CXX=mpicxx CFLAGS=-O3"
+ARG OSU_MAKE_OPTIONS="-j4"
+
+RUN mkdir -p /tmp/osu-benchmark-build \
+    && cd /tmp/osu-benchmark-build \
+    && wget --quiet "http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-${OSU_VERSION}.tar.gz" \
+    && tar xzvf osu-micro-benchmarks-${OSU_VERSION}.tar.gz \
+    && cd osu-micro-benchmarks-${OSU_VERSION} \
+    && ./configure ${OSU_CONFIGURE_OPTIONS} \
+    && make ${OSU_MAKE_OPTIONS} \
+    && make install \
+    && cd / \
+    && rm -rf /tmp/osu-benchmark-build
+ENV PATH="/usr/local/libexec/osu-micro-benchmarks/mpi/collective:/usr/local/libexec/osu-micro-benchmarks/mpi/one-sided:/usr/local/libexec/osu-micro-benchmarks/mpi/pt2pt:/usr/local/libexec/osu-micro-benchmarks/mpi/startup:$PATH"
+
+USER ubuntu
+WORKDIR /home/ubuntu
+
+# Install PyTorch
+ARG PYTORCH_VER=1.13.1
+RUN python -m venv venv && venv/bin/pip install --upgrade pip && \
+    . venv/bin/activate && \
+    pip install --no-cache-dir astunparse numpy ninja pyyaml setuptools cmake \
+    cffi typing_extensions future six requests dataclasses sympy
+
+RUN git clone --depth 2 --recursive --single-branch --branch v${PYTORCH_VER}  https://github.com/pytorch/pytorch && \
+    . venv/bin/activate && cd pytorch && \
+    PATH="${PATH}:${HOME}/opt/openmpi/bin" LD_LIBRARY_PATH="${HOME}/opt/openmpi/lib" USE_CUDA=OFF CMAKE_C_COMPILER=$(which mpicc) CMAKE_CXX_COMPILER=$(which mpicxx) python setup.py build bdist_wheel && \
+    cd .. && \
+    cp pytorch/dist/*.whl . && \
+    rm -rf pytorch/
+
+RUN pip install --no-cache-dir torch*.whl
+
+COPY test.py .
+COPY pytorch_gloo_bug_30723.py .

From 214228105333ae32844b6116cf73424346676a7a Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <anastasios.zouzias@huawei.com>
Date: Tue, 30 May 2023 13:32:40 +0200
Subject: [PATCH 2/2] wip

---
 .env                                   | 2 +-
 docker-compose.yml                     | 2 +-
 docker/Dockerfile.devcontainer.openmpi | 3 ---
 docker/Dockerfile.openmpi              | 5 +----
 4 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/.env b/.env
index 5a698a8..87da1f8 100644
--- a/.env
+++ b/.env
@@ -1,4 +1,4 @@
 DOCKER_REGISTRY=ghcr.io
-DOCKER_ORG=Algebraic-Programming
+DOCKER_ORG=algebraic-programming
 PYTORCH_VERSION=1.13.1
 OPENMPI_VERSION=4.1.5
diff --git a/docker-compose.yml b/docker-compose.yml
index 5f158ce..db4a436 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,7 +9,7 @@ services:
         PYTORCH_VER: ${PYTORCH_VERSION}
         MPI_VERSION: ${OPENMPI_VERSION}
       dockerfile: docker/Dockerfile.openmpi
-  devcontainers-pytorch-mpi:
+  vscode:
     image: "${DOCKER_REGISTRY}/${DOCKER_ORG}/pytorch-mpi:v${PYTORCH_VERSION}-vscode-dev"
     build:
       context: .
diff --git a/docker/Dockerfile.devcontainer.openmpi b/docker/Dockerfile.devcontainer.openmpi
index cea0d42..d891cf6 100644
--- a/docker/Dockerfile.devcontainer.openmpi
+++ b/docker/Dockerfile.devcontainer.openmpi
@@ -50,6 +50,3 @@ RUN git clone --depth 2 --recursive --single-branch --branch v${PYTORCH_VER}  ht
     rm -rf pytorch/
 
 RUN pip install --no-cache-dir  torch*.whl
-
-COPY test.py .
-COPY pytorch_gloo_bug_30723.py .
\ No newline at end of file
diff --git a/docker/Dockerfile.openmpi b/docker/Dockerfile.openmpi
index d67ddd2..95bf053 100644
--- a/docker/Dockerfile.openmpi
+++ b/docker/Dockerfile.openmpi
@@ -6,7 +6,7 @@ LABEL maintainer="Anastasios Zouzias"
 # Installs OpenMPI with PyTorch having distributed with MPI backend enabled
 
 # Add 'ubuntu' sudoer
-RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u 1001 ubuntu
+RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u 1000 ubuntu
 
 # Install dependencies
 RUN export DEBIAN_FRONTEND=noninteractive && apt-get update -qy && \
@@ -71,6 +71,3 @@ RUN git clone --depth 2 --recursive --single-branch --branch v${PYTORCH_VER}  ht
     rm -rf pytorch/
 
 RUN pip install --no-cache-dir torch*.whl
-
-COPY test.py .
-COPY pytorch_gloo_bug_30723.py .