diff --git a/.env b/.env
new file mode 100644
index 0000000..87da1f8
--- /dev/null
+++ b/.env
@@ -0,0 +1,4 @@
+DOCKER_REGISTRY=ghcr.io
+DOCKER_ORG=algebraic-programming
+PYTORCH_VERSION=1.13.1
+OPENMPI_VERSION=4.1.5
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..db4a436
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,19 @@
+version: "3.3"
+
+services:
+  pytorch-mpi:
+    image: "${DOCKER_REGISTRY}/${DOCKER_ORG}/pytorch-mpi:v${PYTORCH_VERSION}"
+    build:
+      context: .
+      args:
+        PYTORCH_VER: ${PYTORCH_VERSION}
+        MPI_VERSION: ${OPENMPI_VERSION}
+      dockerfile: docker/Dockerfile.openmpi
+  vscode:
+    image: "${DOCKER_REGISTRY}/${DOCKER_ORG}/pytorch-mpi:v${PYTORCH_VERSION}-vscode-dev"
+    build:
+      context: .
+      args:
+        PYTORCH_VER: ${PYTORCH_VERSION}
+        MPI_VERSION: ${OPENMPI_VERSION}
+      dockerfile: docker/Dockerfile.devcontainer.openmpi
diff --git a/docker/Dockerfile.devcontainer.openmpi b/docker/Dockerfile.devcontainer.openmpi
new file mode 100644
index 0000000..d891cf6
--- /dev/null
+++ b/docker/Dockerfile.devcontainer.openmpi
@@ -0,0 +1,52 @@
+# hadolint global ignore=DL3003,DL3008,DL3013,DL3015,SC1091
+FROM mcr.microsoft.com/devcontainers/base:jammy
+
+LABEL maintainer="Anastasios Zouzias"
+
+# Installs OpenMPI with PyTorch having distributed with MPI backend enabled
+
+# Install dependencies
+RUN export DEBIAN_FRONTEND=noninteractive && apt-get update -qy && \
+    apt-get install -yq g++-9 make cmake \
+    autoconf libibverbs-dev hwloc lam-runtime python3-pip python3-venv python-is-python3 git vim wget bzip2 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 100
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-9 100
+
+# Build OpenMPI from source
+ARG MPI_VERSION="4.1.5"
+# See https://github.com/openucx/ucx/wiki/OpenMPI-and-OpenSHMEM-installation-with-UCX#running-open-mpi-with-ucx
+ARG MPI_CONFIGURE_OPTIONS="--without-pmi --enable-mpi1-compatibility  --enable-mpi-thread-multiple --without-slurm --without-verbs --disable-oshmem --disable-dependency-tracking --disable-mpi-fortran --disable-wrapper-rpath --disable-wrapper-runpath"
+ARG MPI_MAKE_OPTIONS="-j10"
+
+RUN mkdir -p /tmp/openmpi-build \
+    && cd /tmp/openmpi-build \
+    && wget --quiet "http://www.openmpi.org/software/ompi/v${MPI_VERSION%.*}/downloads/openmpi-${MPI_VERSION}.tar.bz2" \
+    && tar xjf openmpi-${MPI_VERSION}.tar.bz2 \
+    && cd openmpi-${MPI_VERSION}  \
+    && ./configure ${MPI_CONFIGURE_OPTIONS} \
+    && make ${MPI_MAKE_OPTIONS} \
+    && make install \
+    && ldconfig \
+    && cd / \
+    && rm -rf /tmp/openmpi-build
+
+USER vscode
+WORKDIR /home/vscode
+
+# Install PyTorch
+ARG PYTORCH_VER=1.13.1
+RUN python -m venv venv && venv/bin/pip install --upgrade pip && \
+    . venv/bin/activate && \
+    pip install --no-cache-dir astunparse numpy ninja pyyaml setuptools cmake \
+    cffi typing_extensions future six requests dataclasses sympy
+
+RUN git clone --depth 2 --recursive --single-branch --branch v${PYTORCH_VER}  https://github.com/pytorch/pytorch && \
+    . venv/bin/activate && cd pytorch && \
+    PATH=${PATH}:${HOME}/opt/openmpi/bin LD_LIBRARY_PATH=${HOME}/opt/openmpi/lib USE_CUDA=OFF CMAKE_C_COMPILER=$(which mpicc) CMAKE_CXX_COMPILER=$(which mpicxx) python setup.py build bdist_wheel && \
+    cd .. && \
+    cp pytorch/dist/*.whl . && \
+    rm -rf pytorch/
+
+RUN pip install --no-cache-dir  torch*.whl
diff --git a/docker/Dockerfile.openmpi b/docker/Dockerfile.openmpi
new file mode 100644
index 0000000..95bf053
--- /dev/null
+++ b/docker/Dockerfile.openmpi
@@ -0,0 +1,73 @@
+# hadolint global ignore=DL3003,DL3008,DL3013,DL3015,SC1091
+FROM ubuntu:22.04
+
+LABEL maintainer="Anastasios Zouzias"
+
+# Installs OpenMPI with PyTorch having distributed with MPI backend enabled
+
+# Add 'ubuntu' sudoer
+RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u 1000 ubuntu
+
+# Install dependencies
+RUN export DEBIAN_FRONTEND=noninteractive && apt-get update -qy && \
+    apt-get install -yq g++-9 make cmake \
+    autoconf libibverbs-dev hwloc lam-runtime python3-pip python3-venv python-is-python3 git vim wget bzip2 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 100
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-9 100
+
+# Build OpenMPI from source
+ARG MPI_VERSION="4.1.5"
+# See https://github.com/openucx/ucx/wiki/OpenMPI-and-OpenSHMEM-installation-with-UCX#running-open-mpi-with-ucx
+ARG MPI_CONFIGURE_OPTIONS="--without-pmi --enable-mpi1-compatibility  --enable-mpi-thread-multiple --without-slurm --without-verbs --disable-oshmem --disable-dependency-tracking --disable-mpi-fortran --disable-wrapper-rpath --disable-wrapper-runpath"
+ARG MPI_MAKE_OPTIONS="-j10"
+
+RUN mkdir -p /tmp/openmpi-build \
+    && cd /tmp/openmpi-build \
+    && wget --quiet "http://www.openmpi.org/software/ompi/v${MPI_VERSION%.*}/downloads/openmpi-${MPI_VERSION}.tar.bz2" \
+    && tar xjf openmpi-${MPI_VERSION}.tar.bz2 \
+    && cd openmpi-${MPI_VERSION}  \
+    && ./configure ${MPI_CONFIGURE_OPTIONS} \
+    && make ${MPI_MAKE_OPTIONS} \
+    && make install \
+    && ldconfig \
+    && cd / \
+    && rm -rf /tmp/openmpi-build
+
+
+# Build OSU Benchmarks
+ARG OSU_VERSION="7.0.1"
+ARG OSU_CONFIGURE_OPTIONS="--prefix=/usr/local CC=mpicc CXX=mpicxx CFLAGS=-O3"
+ARG OSU_MAKE_OPTIONS="-j4"
+
+RUN mkdir -p /tmp/osu-benchmark-build \
+    && cd /tmp/osu-benchmark-build \
+    && wget --quiet "http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-${OSU_VERSION}.tar.gz" \
+    && tar xzvf osu-micro-benchmarks-${OSU_VERSION}.tar.gz \
+    && cd osu-micro-benchmarks-${OSU_VERSION} \
+    && ./configure ${OSU_CONFIGURE_OPTIONS} \
+    && make ${OSU_MAKE_OPTIONS} \
+    && make install \
+    && cd / \
+    && rm -rf /tmp/osu-benchmark-build
+ENV PATH="/usr/local/libexec/osu-micro-benchmarks/mpi/collective:/usr/local/libexec/osu-micro-benchmarks/mpi/one-sided:/usr/local/libexec/osu-micro-benchmarks/mpi/pt2pt:/usr/local/libexec/osu-micro-benchmarks/mpi/startup:$PATH"
+
+USER ubuntu
+WORKDIR /home/ubuntu
+
+# Install PyTorch
+ARG PYTORCH_VER=1.13.1
+RUN python -m venv venv && venv/bin/pip install --upgrade pip && \
+    . venv/bin/activate && \
+    pip install --no-cache-dir astunparse numpy ninja pyyaml setuptools cmake \
+    cffi typing_extensions future six requests dataclasses sympy
+
+RUN git clone --depth 2 --recursive --single-branch --branch v${PYTORCH_VER}  https://github.com/pytorch/pytorch && \
+    . venv/bin/activate && cd pytorch && \
+    PATH="${PATH}:${HOME}/opt/openmpi/bin" LD_LIBRARY_PATH="${HOME}/opt/openmpi/lib" USE_CUDA=OFF CMAKE_C_COMPILER=$(which mpicc) CMAKE_CXX_COMPILER=$(which mpicxx) python setup.py build bdist_wheel && \
+    cd .. && \
+    cp pytorch/dist/*.whl . && \
+    rm -rf pytorch/
+
+RUN pip install --no-cache-dir torch*.whl