Skip to content

Parallel Programming Models

New Examples

This is a WIP section.
In the future, all code examples will be available from a gitlab repository and will be tested continuously.

Code

Download: hello-world_mpi+omp.cpp, Makefile

#include <mpi.h>
#include <omp.h>
#include <iostream>
#include <sched.h>
#include <unistd.h> 
#include <hwloc.h>

using namespace std;

int main (int argc, char *argv[])
{
    int size, rank, name_len;
    char processor_name[MPI_MAX_PROCESSOR_NAME];

    hwloc_topology_t topology;
    hwloc_topology_init(&topology);
    hwloc_topology_load(topology);

    MPI_Init(NULL, NULL);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Get_processor_name(processor_name, &name_len);

    #pragma omp parallel
    {
        int tid = omp_get_thread_num();
        int nthreads = omp_get_num_threads();
        int logical_core = sched_getcpu();

        hwloc_obj_t obj = hwloc_get_pu_obj_by_os_index(topology, logical_core);
        int physical_core = obj->parent->logical_index;

        usleep(rank*100 + logical_core*1000 + tid*10000);

        #pragma omp critical
        cout<<"node: "<<processor_name
            <<" | MPI-task: "<<rank<<" / "<<size
            <<" | OpenMP-thread: "<<tid <<" / "<<nthreads
            <<" | physical core: " << physical_core
            <<" | logical core: " << logical_core
            <<endl;

    }

    hwloc_topology_destroy(topology);
    MPI_Finalize();
    return 0;
}

Submit scripts

OpenMPI

Download: batch_one-mpi-task-per-socket.sh 

#!/bin/bash
#SBATCH --partition=dev_cpuonly
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=38
#SBATCH --threads-per-core=1
#SBATCH --time=00:02
#SBATCH --mem=5000
#SBATCH --output="hello-world_mpi+omp_openmpi_%j.out"

# Load modules
module load compiler/gnu/14
module load mpi/openmpi/5.0

# Compile source code
make hello-world_mpi+omp_openmpi

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=cores
export OMP_PROC_BIND=TRUE

mpirun --map-by package --bind-to package ./hello-world_mpi+omp_openmpi

Download: batch_one-mpi-task-per-socket_hyperthreading.sh 

#!/bin/bash
#SBATCH --partition=dev_cpuonly
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=76
#SBATCH --threads-per-core=2
#SBATCH --time=00:02
#SBATCH --mem=5000
#SBATCH --output="hello-world_mpi+omp_openmpi_%j.out"

# Load modules
module load compiler/gnu/14
module load mpi/openmpi/5.0

# Compile source code
make hello-world_mpi+omp_openmpi

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=threads
export OMP_PROC_BIND=TRUE

mpirun --map-by package --bind-to package ./hello-world_mpi+omp_openmpi

Intel MPI

Download: batch_one-mpi-task-per-socket_intel.sh 

#!/bin/bash
#SBATCH --partition=dev_cpuonly
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=38
#SBATCH --threads-per-core=1
#SBATCH --time=00:02
#SBATCH --mem=5000
#SBATCH --output="hello-world_mpi+omp_intelmpi_%j.out"

# Load modules
module load compiler/intel/2025.1_llvm
module load mpi/impi/2021.11

# Compile source code
make hello-world_mpi+omp_intelmpi

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=cores
export OMP_PROC_BIND=TRUE
export I_MPI_PIN_DOMAIN=socket

mpirun ./hello-world_mpi+omp_intelmpi

Download: batch_one-mpi-task-per-socket_hyperthreading_intel.sh 

#!/bin/bash
#SBATCH --partition=dev_cpuonly
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=76
#SBATCH --threads-per-core=2
#SBATCH --time=00:02
#SBATCH --mem=5000
#SBATCH --output="hello-world_mpi+omp_intelmpi_%j.out"

# Load modules
module load compiler/intel/2025.1_llvm
module load mpi/impi/2021.11

# Compile source code
make hello-world_mpi+omp_intelmpi

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=threads
export OMP_PROC_BIND=TRUE
export I_MPI_PIN_DOMAIN=socket

mpirun ./hello-world_mpi+omp_intelmpi