Slurm Examples¶

Examples on how to submit jobs, depending on the parallelization paradigm.

Single-threaded Jobs¶

Serial code.

Code

C++ source code

#include <iostream>
#include <sched.h>
#include <unistd.h> 
#include <hwloc.h>

using namespace std;

int main (int argc, char *argv[])
{
    hwloc_topology_t topology;
    hwloc_topology_init(&topology);
    hwloc_topology_load(topology);


    int logical_core = sched_getcpu();
    char hostname[HOST_NAME_MAX + 1];

    hwloc_obj_t obj = hwloc_get_pu_obj_by_os_index(topology, logical_core);
    int physical_core = obj->parent->logical_index;

    if (gethostname(hostname, sizeof(hostname)) == 0) {
        std::cout << "Knotenname: " << hostname << std::endl;
    } else {
        std::cerr << "Fehler beim Abrufen des Knotennamens!" << std::endl;
    }

    cout<<"physical core: " << physical_core
        <<" | logical core: " << logical_core <<endl;






    hwloc_topology_destroy(topology);
    return 0;
}

Download: hello-world_serial.cpp

Submit script

Serial GCC

Download: hk2_serial.sh

#!/bin/bash
#SBATCH --partition=cpu
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=1
#SBATCH --threads-per-core=1
#SBATCH --time=00:02
#SBATCH --output="hello-world_serial_%j.out"

# Load modules
module purge
module load GCC

# Compile source code
g++ -lhwloc hello-world_serial.cpp -o hello-world_serial

./hello-world_serial

Multi-threaded (OpenMP) Jobs¶

Multi-threaded code, uses all cores of the HoreKa 2 CPU partition, with or without using Hyperthreading.

Code

C++ source code

#include <omp.h>
#include <iostream>
#include <sched.h>
#include <unistd.h> 
#include <hwloc.h>

using namespace std;

int main (int argc, char *argv[])
{

    char hostname[HOST_NAME_MAX + 1];

    hwloc_topology_t topology;
    hwloc_topology_init(&topology);
    hwloc_topology_load(topology);

    if (gethostname(hostname, sizeof(hostname)) == 0) {
        std::cout << "Knotenname: " << hostname << std::endl;
    } else {
        std::cerr << "Fehler beim Abrufen des Knotennamens!" << std::endl;
    }

    #pragma omp parallel
    {
        int tid = omp_get_thread_num();
        int nthreads = omp_get_num_threads();
        int logical_core = sched_getcpu();

        hwloc_obj_t obj = hwloc_get_pu_obj_by_os_index(topology, logical_core);
        int physical_core = obj->parent->logical_index;

        usleep(logical_core*1000 + tid*10000);

        #pragma omp critical
        cout<<"OpenMP-thread: "<<tid <<" / "<<nthreads
            <<" | physical core: " << physical_core
            <<" | logical core: " << logical_core
            <<endl;

    }

    hwloc_topology_destroy(topology);
    return 0;
}

Download: hello-world_omp.cpp

Submit scripts

GCC

No HyperthreadingHyperthreading

Download: c

#!/bin/bash
#SBATCH --partition=cpu
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=96
#SBATCH --threads-per-core=1
#SBATCH --time=00:02
#SBATCH --output="hello-world_omp_%j.out"

# Load modules
module purge
module load GCC

# Compile source code
g++ -fopenmp -lhwloc hello-world_omp.cpp -o hello-world_omp

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=cores
export OMP_PROC_BIND=TRUE

./hello-world_omp

Download: hk2_openmp_hyperthreading.sh

#!/bin/bash
#SBATCH --partition=cpu
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=192
#SBATCH --threads-per-core=2
#SBATCH --time=00:02
#SBATCH --output="hello-world_omp_%j.out"

# Load modules
module purge
module load GCC

# Compile source code
g++ -fopenmp -lhwloc hello-world_omp.cpp -o hello-world_omp

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=threads
export OMP_PROC_BIND=TRUE

./hello-world_omp

Intel

No HyperthreadingHyperthreading

Download: hk2_openmp_intel.sh

#!/bin/bash
#SBATCH --partition=cpu
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=96
#SBATCH --threads-per-core=1
#SBATCH --time=00:02
#SBATCH --output="hello-world_omp_intel_%j.out"

# Load modules
module purge
module load intel

# Compile source code
icpx -qopenmp -lhwloc hello-world_omp.cpp -o hello-world_omp_intel

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=cores
export OMP_PROC_BIND=TRUE


./hello-world_omp_intel

Download: hk2_openmp_hyperthreading_intel.sh

#!/bin/bash
#SBATCH --partition=cpu
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=192
#SBATCH --threads-per-core=2
#SBATCH --time=00:02
#SBATCH --output="hello-world_omp_intel_%j.out"

# Load modules
module purge
module load intel

# Compile source code
icpx -qopenmp -lhwloc hello-world_omp.cpp -o hello-world_omp_intel

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=threads
export OMP_PROC_BIND=TRUE
export I_MPI_PIN_DOMAIN=socket

./hello-world_omp_intel

MPI Jobs¶

Hybrid MPI + OpenMP Jobs¶

Multi-node, multi-threaded code, spawns one MPI-task per socket and 96 physical cores per socket of the HoreKa 2 CPU partition, with or without using Hyperthreading.

Code

C++ source code

#include <mpi.h>
#include <omp.h>
#include <iostream>
#include <sched.h>
#include <unistd.h> 
#include <hwloc.h>

using namespace std;

int main (int argc, char *argv[])
{
    int size, rank, name_len;
    char processor_name[MPI_MAX_PROCESSOR_NAME];

    hwloc_topology_t topology;
    hwloc_topology_init(&topology);
    hwloc_topology_load(topology);

    MPI_Init(NULL, NULL);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Get_processor_name(processor_name, &name_len);

    #pragma omp parallel
    {
        int tid = omp_get_thread_num();
        int nthreads = omp_get_num_threads();
        int logical_core = sched_getcpu();

        hwloc_obj_t obj = hwloc_get_pu_obj_by_os_index(topology, logical_core);
        int physical_core = obj->parent->logical_index;

        usleep(rank*100 + logical_core*1000 + tid*10000);

        #pragma omp critical
        cout<<"node: "<<processor_name
            <<" | MPI-task: "<<rank<<" / "<<size
            <<" | OpenMP-thread: "<<tid <<" / "<<nthreads
            <<" | physical core: " << physical_core
            <<" | logical core: " << logical_core
            <<endl;

    }

    hwloc_topology_destroy(topology);
    MPI_Finalize();
    return 0;
}

Download: hello-world_mpi+omp.cpp

Submit scripts

OpenMPI

One MPI task per socketOne MPI task per socket + Hyperthreading

Download: batch_one-mpi-task-per-socket.sh

#!/bin/bash
#SBATCH --partition=cpu
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=96
#SBATCH --threads-per-core=1
#SBATCH --time=00:02
#SBATCH --mem=5000
#SBATCH --output="hello-world_mpi+omp_openmpi_%j.out"

# Load modules
module load OpenMPI

# Compile source code
# make hello-world_mpi+omp_openmpi
mpic++ -fopenmp -lhwloc hello-world_mpi+omp.cpp -o hello-world_mpi+omp_openmpi

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=cores
export OMP_PROC_BIND=TRUE

mpirun --map-by package --bind-to package ./hello-world_mpi+omp_openmpi

Download: batch_one-mpi-task-per-socket_hyperthreading.sh

#!/bin/bash
#SBATCH --partition=cpu
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=192
#SBATCH --threads-per-core=2
#SBATCH --time=00:02
#SBATCH --mem=5000
#SBATCH --output="hello-world_mpi+omp_openmpi_%j.out"

# Load modules
module load OpenMPI

# Compile source code
# make hello-world_mpi+omp_openmpi
mpic++ -fopenmp -lhwloc hello-world_mpi+omp.cpp -o hello-world_mpi+omp_openmpi

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=threads
export OMP_PROC_BIND=TRUE

mpirun --map-by package --bind-to package ./hello-world_mpi+omp_openmpi

Intel MPI

One MPI task per socketOne MPI task per socket + Hyperthreading

Download: batch_one-mpi-task-per-socket_intel.sh

#!/bin/bash
#SBATCH --partition=cpu
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=96
#SBATCH --threads-per-core=1
#SBATCH --time=00:02
#SBATCH --mem=5000
#SBATCH --output="hello-world_mpi+omp_intelmpi_%j.out"

# Load modules
module load intel
# module load impi

# Compile source code
# make hello-world_mpi+omp_intelmpi
mpiicpx -qopenmp -lhwloc hello-world_mpi+omp.cpp -o hello-world_mpi+omp_intelmpi

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=cores
export OMP_PROC_BIND=TRUE
export I_MPI_PIN_DOMAIN=socket

mpirun ./hello-world_mpi+omp_intelmpi

Download: batch_one-mpi-task-per-socket_hyperthreading_intel.sh

#!/bin/bash
#SBATCH --partition=cpu
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --ntasks-per-socket=1
#SBATCH --cpus-per-task=192
#SBATCH --threads-per-core=2
#SBATCH --time=00:02
#SBATCH --mem=5000
#SBATCH --output="hello-world_mpi+omp_intelmpi_%j.out"

# Load modules
module load intel

# Compile source code
# make hello-world_mpi+omp_intelmpi
mpic++ -fopenmp -lhwloc hello-world_mpi+omp.cpp -o hello-world_mpi+omp_openmpi

export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
export OMP_PLACES=threads
export OMP_PROC_BIND=TRUE
export I_MPI_PIN_DOMAIN=socket

mpirun ./hello-world_mpi+omp_intelmpi

Slurm Examples¶

Single-threaded Jobs¶

Multi-threaded (OpenMP) Jobs¶

MPI Jobs¶

Hybrid MPI + OpenMP Jobs¶

GPU Jobs¶

Interactive Jobs¶