Sample batch scripts

If possible, batch jobs on more than 8 compute nodes should use the Intel Ivy Bridge processors which provide 20 physical cores per node. So you should specify 20 MPI tasks or OpenMP threads / ConsumableCpus in your job script (or 40 in hyperthreading mode, respectively).
Small batch jobs on 1 or 2 nodes will only run on the Intel Sandy Bridge nodes with 16 physical cores per node (or 32 in hyperthreading mode, respectively.)

 

Sample MPI batch job (IBM MPI/POE) without hyperthreading


# @ shell=/bin/bash
#
# Sample script for LoadLeveler
#
# @ error = job.err.$(jobid)
# @ output = job.out.$(jobid)
# @ job_type = parallel
# @ node_usage= not_shared
# @ node =32
# @ tasks_per_node = 20
# @ resources = ConsumableCpus(1)
# @ network.MPI = sn_all,not_shared,us
# @ wall_clock_limit = 24:00:00
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue
# run the program
cd /ptmp/${USER}/ poe /u/${USER}/myprog > prog.out

 

The following submit script runs a job with 512 MPI tasks. First example for Sandy Bridge nodes, second example for Ivy Bridge nodes (recommended).

# @ shell=/bin/bash
#
# Sample script for LoadLeveler
#
# @ error = job.err.$(jobid)
# @ output = job.out.$(jobid)
# @ job_type = parallel
# @ node_usage= not_shared
# @ node = 32
# @ tasks_per_node = 16
# @ resources = ConsumableCpus(1)
# @ network.MPI = sn_all,not_shared,us
# @ wall_clock_limit = 24:00:00
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue
# run the program
cd /ptmp/${USER}/ poe /u/${USER}/myprog > prog.out

To run the same job on a HYDRA Ivy Bridge partition with 20 cores per node, the changes are as follows. Note the new parameter "@ first_node_tasks" and the reduced number of nodes. In total, 512=1*12+25*20 MPI processes are used.

# @ shell=/bin/bash
#
# Sample script for LoadLeveler
#
# @ error = job.err.$(jobid)
# @ output = job.out.$(jobid)
# @ job_type = parallel
# @ node_usage= not_shared
# @ node = 26
# @ tasks_per_node = 20
# @ first_node_tasks = 12
# @ resources = ConsumableCpus(1)
# @ network.MPI = sn_all,not_shared,us
# @ wall_clock_limit = 24:00:00
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue

# run the program
cd /ptmp/${USER}/ poe /u/${USER}/myprog > prog.out

 

Sample hybrid MPI/OpenMP batch job without hyperthreading (using IBM POE)


# @ shell=/bin/bash
#
# Sample script for LoadLeveler
#
# @ error = job.err.$(jobid)
# @ output = job.out.$(jobid)
# @ job_type = parallel
# @ node_usage= not_shared
# @ node =32
# @ tasks_per_node = 1
# @ resources = ConsumableCpus(20)
# @ network.MPI = sn_all,not_shared,us
# @ wall_clock_limit = 12:00:00
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue
export OMP_NUM_THREADS=20
# run the program
cd /ptmp/${USER}/ poe /u/${USER}/myprog > prog.out

 

Sample MPI batch job (IBM MPI/POE) in hyperthreading mode using 120 GB of memory per node

 

# @ shell=/bin/bash
#
# Sample script for LoadLeveler
#
# @ error = job.err.$(jobid)
# @ output = job.out.$(jobid)
# @ job_type = parallel
# @ node_usage= not_shared
# @ node = 32
# @ tasks_per_node = 40
# @ resources = ConsumableCpus(1)
# @ node_resources = ConsumableMemory(120gb)
# @ network.MPI = sn_all,not_shared,us
# @ wall_clock_limit = 12:00:00
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue
# run the program
cd /ptmp/${USER}/ poe /u/${USER}/myprog > prog.out

 

Sample hybrid MPI/OpenMP batch job using 120 GB of memory per node in hyperthreading mode (using IBM POE)

 

# @ shell=/bin/bash
#
# Sample script for LoadLeveler
#
# @ error = job.err.$(jobid)
# @ output = job.out.$(jobid)
# @ job_type = parallel
# @ node_usage= not_shared
# @ node = 32
# @ tasks_per_node = 4
# @ resources = ConsumableCpus(10)
# @ node_resources = ConsumableMemory(120gb)
# @ network.MPI = sn_all,not_shared,us
# @ wall_clock_limit = 12:00:00
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue
export OMP_NUM_THREADS=10
# run the program
cd /ptmp/${USER}/ poe /u/${USER}/myprog > prog.out

 

Sample batch job (IBM MPI/POE) using job steps

 

# @ shell=/bin/bash
#
# Sample script for LoadLeveler job steps
#
# @ error = job.err.$(jobid).$(stepid)
# @ output = job.out.$(jobid).$(stepid)
# @ job_type = parallel
# @ node_usage= not_shared
#
# @ step_name = iteration1
# @ node = 8
# @ tasks_per_node = 32
# @ resources = ConsumableCpus(1)
# @ wall_clock_limit = 24:00:00
# @ network.MPI = sn_all,not_shared,us
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue
#
# @ step_name = iteration2
# @ dependency = (iteration1 == 0)
# @ node = 8
# @ tasks_per_node = 32
# @ resources = ConsumableCpus(1)
# @ wall_clock_limit = 24:00:00
# @ network.MPI = sn_all,not_shared,us
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue
#
# @ step_name = iteration3
# @ dependency = (iteration2 == 0)
# @ node = 8
# @ tasks_per_node = 32
# @ resources = ConsumableCpus(1)
# @ wall_clock_limit = 10:00:00
# @ network.MPI = sn_all,not_shared,us
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue
cd /ptmp/${USER}/
echo "Running step $LOADL_STEP_ID ..."
poe /u/${USER}/myprog > prog.out.$LOADL_STEP_ID

 

Sample MPI batch job for NVIDIA GPUs (using IBM MPI/POE)

 

# @ shell=/bin/bash
#
# Sample script for LoadLeveler
#
# @ error = job.err.$(jobid)
# @ output = job.out.$(jobid)
# @ job_type = parallel
# @ requirements = (Feature=="gpu")
# @ node_usage= not_shared
# @ node = 8
# @ tasks_per_node = 20
# @ resources = ConsumableCpus(1)
# @ network.MPI = sn_all,not_shared,us
# @ wall_clock_limit = 24:00:00
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue

module load cuda

# run the program
cd /ptmp/${USER}/ poe /u/${USER}/my_gpu_prog > prog.out

 

Sample batch job for Intel Xeon Phi accelerator cards ("offload mode", using IBM MPI/POE)

The following LoadLeveler sample script submits a job to a node equipped with Xeon Phi accelerators. The example uses 8 threads on the host and 236 threads on the first Xeon Phi card (ID 0).

# @ shell=/bin/bash
# @ error = job.err.$(jobid)
# @ output = job.out.$(jobid)
# @ job_type = parallel
# @ requirements = (Feature=="mic")
# @ node_usage= not_shared
# @ node = 1
# @ tasks_per_node = 1
# @ resources = ConsumableCpus(8)
# @ network.MPI = sn_all,not_shared,us
# @ wall_clock_limit = 0:01:00
# @ notification = complete
# @ notify_user = $(user)@rzg.mpg.de
# @ queue

# load runtime libraries necessary to offload code to the Xeon Phi device
module load intel/13.1 mkl/11.0 xeon-phi-rt/13.1

export OMP_NUM_THREADS=8
export KMP_AFFINITY=verbose,compact
export OFFLOAD_DEVICES=0 export OFFLOAD_REPORT=2 export MIC_ENV_PREFIX=MIC_ export MIC_OMP_NUM_THREADS=236 export MIC_KMP_AFFINITY=explicit,granularity=fine,proclist=[1-236:1] # run the program cd /ptmp/${USER}/ poe /u/${USER}/mictest.exe


Document Actions