OFP: mpimcexec wrapper around mpirun for OFP users

Makefile.in

@@ -1,5 +1,6 @@
 TARGET = @TARGET@
 SBINDIR = @SBINDIR@
+BINDIR = @BINDIR@
 INCDIR =  @INCDIR@
 ETCDIR = @ETCDIR@
 MANDIR = @MANDIR@
@@ -47,6 +48,7 @@ install:
 		mkdir -p -m 755 $(SBINDIR); \
 		install -m 755 arch/x86_64/tools/mcreboot-smp-x86.sh $(SBINDIR)/mcreboot.sh; \
 		install -m 755 arch/x86_64/tools/mcstop+release-smp-x86.sh $(SBINDIR)/mcstop+release.sh; \
+		install -m 755 arch/x86_64/tools/mpimcexec $(BINDIR)/mpimcexec; \
 		install -m 755 arch/x86_64/tools/mcoverlay-destroy-smp-x86.sh $(SBINDIR)/mcoverlay-destroy.sh; \
 		install -m 755 arch/x86_64/tools/mcoverlay-create-smp-x86.sh $(SBINDIR)/mcoverlay-create.sh; \
 		install -m 755 arch/x86_64/tools/eclair-dump-backtrace.exp $(SBINDIR)/eclair-dump-backtrace.exp;\

arch/x86_64/tools/mpimcexec.in

@@ -0,0 +1,112 @@
+#!/bin/bash
+#
+# OFP McKernel MPI wrapper script
+# author: Balazs Gerofi <bgerofi@riken.jp>
+#      Copyright (C) 2018  RIKEN R-CCS
+#
+
+prefix="@prefix@"
+BINDIR="${prefix}/bin"
+
+if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then
+	echo "You need at least bash-4.0 to run this script." >&2
+	exit 1
+fi
+
+RANKS=""
+NODES=""
+PPN=""
+MPI_ENV=""
+COMMAND=""
+NUMA=""
+HOSTFILE=""
+
+help_exit() {
+	echo ""
+	echo "Spawn an McKernel MPI job on Oakforest-PACS."
+	echo "usage: `basename $0` -ppn ranks_per_node [--nodes nodes] [-n ranks] [--env additional_environment]... command"
+	echo ""
+	echo "   -ppn | --ppn | --ranks-per-node         Number of MPI ranks per node (required)"
+	echo "   -n | --n | --ranks                      Total number of MPI ranks in the job"
+	echo "   --nodes                                 Number of nodes to be used"
+	echo "   --env | -env                            Pass an additional environment variable"
+	echo "   -m | --numa                             Preferred NUMA node(s)"
+	echo "   -h | --hostfile                         Host file for MPI"
+	echo "   --help                                  Show help message"
+	exit 1
+}
+
+# Parse options
+while true; do
+	case $1 in
+		-ppn | --ppn | --ranks-per-node )
+			PPN=$2
+			shift 2
+			;;
+		-n | --n | --ranks )
+			RANKS=$2
+			shift 2
+			;;
+		-m | --numa )
+			NUMA="-m $2"
+			shift 2
+			;;
+		--nodes )
+			NODES=$2
+			shift 2
+			;;
+		--env | -env )
+			if [ -z "`echo $2 | grep I_MPI_PIN`" ]; then
+				MPI_ENV=`echo "${MPI_ENV} -env $2" | xargs`
+			fi
+			shift 2
+			;;
+		-h | --hostfile )
+			HOSTFILE="-hostfile $2"
+			shift 2
+			;;
+		--help )
+			help_exit
+			;;
+		* )
+			COMMAND=$@
+			break
+			;;
+	esac
+done
+
+if [ -z ${PPN} ]; then
+	echo "error: please specify the number of ranks per node"
+	help_exit
+fi
+
+# Unless explicitly specified, use Fujitsu inherited value
+if [ -z ${NODES} ]; then
+	NODES=${PJM_VNODES}
+fi
+
+if [ -z ${RANKS} ] && [ -z ${NODES} ]; then
+	echo "error: please specify the total number of ranks or the number of nodes"
+	help_exit
+fi
+
+# Calculate total job size if not specified
+if [ -z ${RANKS} ]; then
+	let RANKS=(${PPN}*${NODES})
+fi
+
+# Support direct SSH when not executed from Fujitsu job system
+if [ -z ${PJM_VNODES} ]; then
+	HOSTFILE="-launcher-exec ssh ${HOSTFILE}"
+fi
+
+export I_MPI_PIN=off
+export PSM2_RCVTHREAD=0
+export HFI_NO_CPUAFFINITY=1
+export I_MPI_COLL_INTRANODE_SHM_THRESHOLD=4194304
+export PSM2_MQ_RNDV_HFI_WINDOW=4194304
+export PSM2_MQ_EAGER_SDMA_SZ=65536
+export PSM2_MQ_RNDV_HFI_THRESH=200000
+
+mpirun ${HOSTFILE} -n ${RANKS} -ppn ${PPN} ${MPI_ENV} ${BINDIR}/mcexec -n ${PPN} ${NUMA} --enable-hfi1 --mpol-threshold=1M --stack-premap=4M,4G --extend-heap-by=8M --disable-sched-yield --mpol-shm-premap ${COMMAND}
+

configure

@@ -5060,7 +5060,7 @@ ac_config_headers="$ac_config_headers config.h"
 
 # POSTK_DEBUG_ARCH_DEP_37
 # AC_CONFIG_FILES arch dependfiles separate
-ac_config_files="$ac_config_files Makefile executer/user/Makefile executer/user/mcexec.1:executer/user/mcexec.1in executer/user/vmcore2mckdump executer/user/arch/$ARCH/Makefile executer/user/arch/x86_64/Makefile executer/kernel/mcctrl/Makefile executer/kernel/mcctrl/arch/$ARCH/Makefile executer/kernel/mcoverlayfs/Makefile executer/kernel/mcoverlayfs/linux-3.10.0-327.36.1.el7/Makefile executer/kernel/mcoverlayfs/linux-4.0.9/Makefile executer/kernel/mcoverlayfs/linux-4.6.7/Makefile executer/include/qlmpilib.h kernel/Makefile kernel/Makefile.build kernel/include/swapfmt.h arch/x86_64/tools/mcreboot-attached-mic.sh arch/x86_64/tools/mcshutdown-attached-mic.sh arch/x86_64/tools/mcreboot-builtin-x86.sh arch/x86_64/tools/mcreboot-smp-x86.sh arch/x86_64/tools/mcstop+release-smp-x86.sh arch/x86_64/tools/mcoverlay-destroy-smp-x86.sh arch/x86_64/tools/mcoverlay-create-smp-x86.sh arch/x86_64/tools/eclair-dump-backtrace.exp arch/x86_64/tools/mcshutdown-builtin-x86.sh arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in arch/x86_64/tools/irqbalance_mck.service arch/x86_64/tools/irqbalance_mck.in tools/mcstat/Makefile"
+ac_config_files="$ac_config_files Makefile executer/user/Makefile executer/user/mcexec.1:executer/user/mcexec.1in executer/user/vmcore2mckdump executer/user/arch/$ARCH/Makefile executer/user/arch/x86_64/Makefile executer/kernel/mcctrl/Makefile executer/kernel/mcctrl/arch/$ARCH/Makefile executer/kernel/mcoverlayfs/Makefile executer/kernel/mcoverlayfs/linux-3.10.0-327.36.1.el7/Makefile executer/kernel/mcoverlayfs/linux-4.0.9/Makefile executer/kernel/mcoverlayfs/linux-4.6.7/Makefile executer/include/qlmpilib.h kernel/Makefile kernel/Makefile.build kernel/include/swapfmt.h arch/x86_64/tools/mcreboot-attached-mic.sh arch/x86_64/tools/mcshutdown-attached-mic.sh arch/x86_64/tools/mcreboot-builtin-x86.sh arch/x86_64/tools/mcreboot-smp-x86.sh arch/x86_64/tools/mcstop+release-smp-x86.sh arch/x86_64/tools/mcoverlay-destroy-smp-x86.sh arch/x86_64/tools/mcoverlay-create-smp-x86.sh arch/x86_64/tools/eclair-dump-backtrace.exp arch/x86_64/tools/mcshutdown-builtin-x86.sh arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in arch/x86_64/tools/mpimcexec arch/x86_64/tools/irqbalance_mck.service arch/x86_64/tools/irqbalance_mck.in tools/mcstat/Makefile"
 
 
 if test "$TARGET" = "smp-x86"; then
@@ -5795,6 +5795,7 @@ do
     "arch/x86_64/tools/eclair-dump-backtrace.exp") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/eclair-dump-backtrace.exp" ;;
     "arch/x86_64/tools/mcshutdown-builtin-x86.sh") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/mcshutdown-builtin-x86.sh" ;;
     "arch/x86_64/tools/mcreboot.1") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in" ;;
+    "arch/x86_64/tools/mpimcexec") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/mpimcexec" ;;
     "arch/x86_64/tools/irqbalance_mck.service") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/irqbalance_mck.service" ;;
     "arch/x86_64/tools/irqbalance_mck.in") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/irqbalance_mck.in" ;;
     "tools/mcstat/Makefile") CONFIG_FILES="$CONFIG_FILES tools/mcstat/Makefile" ;;

configure.ac

@@ -568,6 +568,7 @@ AC_CONFIG_FILES([
 	arch/x86_64/tools/eclair-dump-backtrace.exp
 	arch/x86_64/tools/mcshutdown-builtin-x86.sh
 	arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in
+	arch/x86_64/tools/mpimcexec
 	arch/x86_64/tools/irqbalance_mck.service
 	arch/x86_64/tools/irqbalance_mck.in
 	tools/mcstat/Makefile