make checking write-combine arch-dependent

Change-Id: I4c0fca7d34e69b4774141e115b8ebc03c5c1e8b3
Fujitsu: POSTK_DEBUG_ARCH_DEP_12
Refs: #1355

arch/x86_64/kernel/include/arch-memory.h

@@ -184,12 +184,10 @@ enum ihk_mc_pt_attribute {
 
 enum ihk_mc_pt_attribute attr_mask;
 
-#ifdef POSTK_DEBUG_ARCH_DEP_12
 static inline int pfn_is_write_combined(uintptr_t pfn)
 {
 	return ((pfn & PFL1_PWT) && !(pfn & PFL1_PCD));
 }
-#endif /* #ifdef POSTK_DEBUG_ARCH_DEP_12 */
 
 static inline int pte_is_null(pte_t *ptep)
 {

executer/kernel/mcctrl/arch/arm64/include/archdeps.h

@@ -9,8 +9,6 @@
 extern int translate_rva_to_rpa(ihk_os_t os, unsigned long rpt, unsigned long rva,
 				unsigned long *rpap, unsigned long *pgsizep);
 
-#ifdef POSTK_DEBUG_ARCH_DEP_12
-
 #define PFN_WRITE_COMBINED PTE_ATTRINDX(MT_NORMAL_NC)
 static inline bool pte_is_write_combined(pte_t pte)
 {
@@ -31,7 +29,6 @@ static inline bool pte_is_write_combined(pte_t pte)
 #endif
 	return ((pte_val(pte) & PTE_ATTRINDX_MASK) == PFN_WRITE_COMBINED);
 }
-#endif /* POSTK_DEBUG_ARCH_DEP_12 */
 
 #define ARMV8_IDX_COUNTER0	0
 #define ARCH_PERF_COUNTER_START	ARMV8_IDX_COUNTER0

executer/kernel/mcctrl/arch/x86_64/archdeps.c

@@ -370,13 +370,11 @@ out:
 	return error;
 }
 
-#ifdef POSTK_DEBUG_ARCH_DEP_12
 #define PFN_WRITE_COMBINED _PAGE_PWT
 static inline bool pte_is_write_combined(pte_t pte)
 {
 	return ((pte_flags(pte) & _PAGE_PWT) && !(pte_flags(pte) & _PAGE_PCD));
 }
-#endif /* POSTK_DEBUG_ARCH_DEP_12 */
 
 /*
  * The assembler switch_ctx is save/load registers in the context.

executer/kernel/mcctrl/arch/x86_64/include/archdeps.h

@@ -9,14 +9,12 @@
 extern int translate_rva_to_rpa(ihk_os_t os, unsigned long rpt, unsigned long rva,
 				unsigned long *rpap, unsigned long *pgsizep);
 
-#ifdef POSTK_DEBUG_ARCH_DEP_12
 #define PFN_WRITE_COMBINED _PAGE_PWT
 
 static inline bool pte_is_write_combined(pte_t pte)
 {
 	return ((pte_flags(pte) & _PAGE_PWT) && !(pte_flags(pte) & _PAGE_PCD));
 }
-#endif /* POSTK_DEBUG_ARCH_DEP_12 */
 
 #define ARCH_PERF_COUNTER_START	0
 

executer/kernel/mcctrl/syscall.c

@@ -1510,16 +1510,9 @@ retry:
 						pfn |= PFN_VALID | PFN_PRESENT;
 
 						/* Check if mapping is write-combined */
-#ifdef POSTK_DEBUG_ARCH_DEP_12
 						if (pte_is_write_combined(*pte)) {
 							pfn |= PFN_WRITE_COMBINED;
 						}
-#else /* POSTK_DEBUG_ARCH_DEP_12 */
-						if ((pte_flags(*pte) & _PAGE_PWT) &&
-						    !(pte_flags(*pte) & _PAGE_PCD)) {
-							pfn |= _PAGE_PWT;
-						}
-#endif /* POSTK_DEBUG_ARCH_DEP_12 */
 					}
 					pte_unmap(pte);
 				}

kernel/devobj.c

@@ -232,14 +232,7 @@ static int devobj_get_page(struct memobj *memobj, off_t off, int p2align, uintpt
 			dkprintf("devobj_get_page(%p %lx,%lx,%d):PFN_PRESENT before %#lx\n", memobj, obj->handle, off, p2align, pfn);
 			attr = pfn & ~PFN_PFN;
 
-			/* TODO: do an arch dependent PTE to mapping flag conversion 
-			 * instead of this inline check, also, we rely on having the
-			 * same PAT config as Linux here.. */
-#ifdef POSTK_DEBUG_ARCH_DEP_12
 			if (pfn_is_write_combined(pfn)) {
-#else /* POSTK_DEBUG_ARCH_DEP_12 */
-			if ((pfn & PFL1_PWT) && !(pfn & PFL1_PCD)) {
-#endif /* POSTK_DEBUG_ARCH_DEP_12 */
 				*flag |= VR_WRITE_COMBINED;
 			}
 

test/issues/1355/C1355.sh

@@ -0,0 +1,81 @@
+#/bin/sh
+
+USELTP=0
+USEOSTEST=0
+
+issue=1355
+tid=01
+
+tname=`printf "C${issue}T%02d" ${tid}`
+echo "*** ${tname} start *******************************"
+
+grep "result=0" ./x86_64_intel_bench.txt
+
+if [ $? -eq 0 ]; then
+	echo "*** ${tname} PASSED ******************************"
+else
+	echo "*** ${tname} FAILED ******************************"
+fi
+let tid++
+echo ""
+
+tname=`printf "C${issue}T%02d" ${tid}`
+echo "*** ${tname} start *******************************"
+
+grep "All processes entering MPI_Finalize" ./x86_64_pingpong.txt
+
+if [ $? -eq 0 ]; then
+	echo "*** ${tname} PASSED ******************************"
+else
+	echo "*** ${tname} FAILED ******************************"
+fi
+let tid++
+echo ""
+
+tname=`printf "C${issue}T%02d" ${tid}`
+echo "*** ${tname} start *******************************"
+
+grep "result=0" ./aarch64_intel_bench.txt
+
+if [ $? -eq 0 ]; then
+	echo "*** ${tname} PASSED ******************************"
+else
+	echo "*** ${tname} FAILED ******************************"
+fi
+let tid++
+echo ""
+
+tname=`printf "C${issue}T%02d" ${tid}`
+echo "*** ${tname} start *******************************"
+
+grep "All processes entering MPI_Finalize" ./aarch64_pingpong.txt
+
+if [ $? -eq 0 ]; then
+	echo "*** ${tname} PASSED ******************************"
+else
+	echo "*** ${tname} FAILED ******************************"
+fi
+let tid++
+echo ""
+
+tname=`printf "C${issue}T%02d" ${tid}`
+echo "*** ${tname} start *******************************"
+ng=0
+
+grep -e ".*apollo15.*rank 0.*" ./aarch64_mvapich.txt
+if [ $? -ne 0 ]; then
+	let ng++
+fi
+grep -e ".*apollo16.*rank 1.*" ./aarch64_mvapich.txt
+if [ $? -ne 0 ]; then
+	let ng++
+fi
+
+if [ $ng -eq 0 ]; then
+	echo "*** ${tname} PASSED ******************************"
+else
+	echo "*** ${tname} FAILED ******************************"
+fi
+let tid++
+echo ""
+

test/issues/1355/Makefile

@@ -0,0 +1,11 @@
+CFLAGS=-g
+LDFLAGS=
+
+TARGET=
+
+all: $(TARGET)
+
+test: all
+	./C1355.sh
+clean:
+	rm -f $(TARGET) *.o

test/issues/1355/README

@@ -0,0 +1,61 @@
+【Issue#1355 動作確認】
+□ テスト内容
+1. 複数ノードによるMPIプログラムを実行し、修正が既存機能に影響していないことを確認
+・x86_64 環境でIntel MPIが動作することの確認
+	wallaby7, 13 の2台を用いて、Intel MPI benchmarkのpingpong を実行し、
+	下記を確認する
+C1355T01: benchmarkのpingpong が正常に終了していることを確認
+C1355T02: pingpongのMPIプロセスが、それぞれのノードで正常に終了していることを確認
+
+・aarch64 環境でIntel MPIが動作することの確認
+	apollo15, 16 の2台を用いて、Intel MPI benchmarkのpingpong を実行し、
+	下記を確認する
+C1355T03: benchmarkのpingpong が正常に終了していることを確認
+C1355T04: pingpongのMPIプロセスが、それぞれのノードで正常に終了していることを確認
+
+・aarch64 環境でMVAPICH が動作することの確認
+	apollo15, 16 の2台を用いて、MVAPCHのMPIプログラムを実行し、
+	下記を確認する
+C1355T05: それぞれのノードでrank情報の表示が行えていることを確認
+
+□ 実行手順
+本テストは複数ノードによるMPIプログラム実行を行うため、自動化はしていない
+それぞれのMPIプログラムの実行方法を示す
+
+<wallaby7, 13 での Intel MPI benchmark>
+$ ssh wallaby7
+$ sudo su mcktest
+$ sudo systemctl stop tasksched
+$ cd /work/mcktest/work_development_development
+$ (cwd=$(pwd); cd <mck_src>; rm -rf build; mkdir build; cd build; cmake ../mckernel -DCMAKE_INSTALL_PREFIX=${cwd}/mck && make -j install)
+$ bash -x ../bin/init-config.sh
+AUTOTEST_HOME=/work/mcktest bash -x /work/mcktest/bin/syncdir-all.sh
+AUTOTEST_HOME=/work/mcktest bash -x /work/mcktest/bin/boot-all.sh
+AUTOTEST_HOME=/work/mcktest MCEXEC=mcexec bash -x /work/mcktest/perf/bin/pingpong-1x2 > x86_64_intel_bench.txt
+cat /work/mcktest/work_development_development/perflog/mckernel/pingpong-1x2 > x86_64_pingpong.txt
+sudo systemctl start tasksched
+
+<apollo15, 16 での Intel MPI benchmark>
+$ ssh apollo15
+$ sudo su mcktest
+$ cd /work/mcktest/work_development_development
+$ (cwd=$(pwd); cd <mck_src>; rm -rf build; mkdir build; cd build; cmake ../mckernel -DCMAKE_INSTALL_PREFIX=${cwd}/mck && make -j install)
+$ bash -x ../bin/init-config.sh
+AUTOTEST_HOME=/work/mcktest bash -x /work/mcktest/bin/syncdir-all.sh
+AUTOTEST_HOME=/work/mcktest bash -x /work/mcktest/bin/boot-all.sh
+AUTOTEST_HOME=/work/mcktest MCEXEC=mcexec bash -x /work/mcktest/perf/bin/pingpong-1x2 > aarch64_intel_bench.txt
+cat /work/mcktest/work_development_development/perflog/mckernel/pingpong-1x2 > aarch64_pingpong.txt
+
+<apollo15, 16 での MVAPICH>
+$ export MODULEPATH=/home/bgerofi/modulefiles:/opt/arm/modulefiles/:$MODULEPATH
+$ module load MVAPICH
+$ module load gcc
+$ mpicc -o mpi_hello ./mpi_hello.c
+$ mpirun -hostfile ./hostfile ./mpi_hello > aarch64_mvapich.txt
+
+これらの出力結果ファイルを同ディレクトリを配置し、下記を実行する
+$ make test
+
+□ 実行結果
+result.log 参照。
+すべての項目をPASSしていることを確認。

test/issues/1355/aarch64_intel_bench.txt

@@ -0,0 +1,95 @@
+++ basename /work/mcktest/perf/bin/pingpong-1x2
++ TESTCASE=pingpong-1x2
++ '[' /work/mcktest == '' ']'
++ export AUTOTEST_HOME
+++ pwd -P
++ CWD=/work/mcktest/work_development_development
+++ dirname /work/mcktest/perf/bin/pingpong-1x2
++ cd /work/mcktest/perf/bin
++ app_dir=/work/mcktest/perf/mpi-benchmarks/src
++ . /work/mcktest/bin/config.sh
+++ export NODETYPE=wallaby
+++ NODETYPE=wallaby
+++ export WGETSSHPROXY=ofp05
+++ WGETSSHPROXY=ofp05
+++ export 'WGETPROXYOPTS=-e use_proxy=yes -e http_proxy=wallaby -e https_proxy=wallaby:80'
+++ WGETPROXYOPTS='-e use_proxy=yes -e http_proxy=wallaby -e https_proxy=wallaby:80'
+++ MCKERNELBRANCH=development
+++ IHKBRANCH=development
+++ export WORKDIR=/work/mcktest/work_development_development
+++ WORKDIR=/work/mcktest/work_development_development
+++ SRC=/work/mcktest/work_development_development/src
+++ MCKBUILD=/work/mcktest/work_development_development/build
+++ MCKINSTALL=/work/mcktest/work_development_development/mck
+++ BUILDID_DIR=/work/mcktest/buildids
+++ GITS=/work/mcktest/git
+++ LTPINSTALL=/work/mcktest/ltp
+++ STRESS_TEST_DIR=/work/mcktest/stress_test/install
+++ MCKBUILDSCRIPT=/work/mcktest/bin/config_and_build_smp_x86.sh
+++ MPICH=/usr/lib64/mpich-3.2
+++ MPI_DIR=/home/bgerofi/install/mvapich2-2.3
+++ export LTPDIR=/work/mcktest/ltp
+++ LTPDIR=/work/mcktest/ltp
+++ export LTPROOT=/work/mcktest/ltp
+++ LTPROOT=/work/mcktest/ltp
+++ export 'MAIL_TO=mckernel@www.pccluster.org toshi@metahack.jp'
+++ MAIL_TO='mckernel@www.pccluster.org toshi@metahack.jp'
+++ export 'MCKERNELCONFIGOPTION=--enable-rusage --with-mpi=/usr/lib64/mpich-3.2 --enable-qlmpi'
+++ MCKERNELCONFIGOPTION='--enable-rusage --with-mpi=/usr/lib64/mpich-3.2 --enable-qlmpi'
+++ export IHKCONFIGOPTION=--enable-rusage
+++ IHKCONFIGOPTION=--enable-rusage
+++ MCREBOOTOPTION='-k 0 -f LOG_LOCAL6  -m 10G@0,10G@1 -c 1-13,15-27,29-41,43-55 -r 1-13:0+15-27:14+29-41:28+43-55:42 -o root -O'
+++ export PATH=/work/mcktest/bin:/work/mcktest/work_development_development/mck/bin:/work/mcktest/work_development_development/mck/sbin:/home/bgerofi/install/mvapich2-2.3/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin:/home/mcktest/project/src/socat/usr/bin:/home/mcktest/.local/bin:/home/mcktest/usr/bin:/usr/lib64/ccache:/sbin:/bin:/usr/sbin:/usr/bin:/opt/ibutils/bin:/opt/pbs/PBSPro_18.2.2-arm/bin:/home/mcktest/go/bin
+++ PATH=/work/mcktest/bin:/work/mcktest/work_development_development/mck/bin:/work/mcktest/work_development_development/mck/sbin:/home/bgerofi/install/mvapich2-2.3/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin:/home/mcktest/project/src/socat/usr/bin:/home/mcktest/.local/bin:/home/mcktest/usr/bin:/usr/lib64/ccache:/sbin:/bin:/usr/sbin:/usr/bin:/opt/ibutils/bin:/opt/pbs/PBSPro_18.2.2-arm/bin:/home/mcktest/go/bin
+++ export LD_LIBRARY_PATH=/home/bgerofi/install/mvapich2-2.3/lib:
+++ LD_LIBRARY_PATH=/home/bgerofi/install/mvapich2-2.3/lib:
+++ export DATADIR=/work/mcktest/data
+++ DATADIR=/work/mcktest/data
+++ export BINDIR=/work/mcktest/bin
+++ BINDIR=/work/mcktest/bin
+++ export LOGDIR=/work/mcktest/log
+++ LOGDIR=/work/mcktest/log
+++ STRESS_TEST_TIMEOUT=60
+++ export OMP_NUM_THREADS_1=1
+++ OMP_NUM_THREADS_1=1
+++ export OMP_NUM_THREADS_2=2
+++ OMP_NUM_THREADS_2=2
+++ export OMP_NUM_THREADS_4=4
+++ OMP_NUM_THREADS_4=4
+++ export OMP_NUM_THREADS_8=8
+++ OMP_NUM_THREADS_8=8
+++ export OMP_NUM_THREADS_16=16
+++ OMP_NUM_THREADS_16=16
+++ export OMP_NUM_THREADS_24=24
+++ OMP_NUM_THREADS_24=24
+++ export OMP_NUM_THREADS_32=32
+++ OMP_NUM_THREADS_32=32
+++ export 'MPI_HOSTS=apollo15 apollo16'
+++ MPI_HOSTS='apollo15 apollo16'
+++ [[ -e /work/mcktest/bin/gerrit-creds ]]
+++ . /work/mcktest/bin/gerrit-creds
++++ GERRITUSER=riken-syssoft-gerrit:ogzuCUSBrLM5gpUoECTKZya52oWf7Bu0fvO+oEeFNg
+++ export GERRITSERVER=review.gerrithub.io
+++ GERRITSERVER=review.gerrithub.io
+++ export 'GERRITCURL=curl -s --proxy http://apollo:3128 https://riken-syssoft-gerrit:ogzuCUSBrLM5gpUoECTKZya52oWf7Bu0fvO+oEeFNg@review.gerrithub.io'
+++ GERRITCURL='curl -s --proxy http://apollo:3128 https://riken-syssoft-gerrit:ogzuCUSBrLM5gpUoECTKZya52oWf7Bu0fvO+oEeFNg@review.gerrithub.io'
++ . ./perf-mpi-config.sh
++++ echo apollo15 apollo16
++++ tr ' ' ,
+++ COMMAHOSTS=apollo15,apollo16
+++ MPIHOSTS='-hosts apollo15,apollo16'
+++ MPIRUNOPTS=
+++ '[' xmcexec = xmcexec ']'
+++ MPIRUN='timeout 120 mpirun'
+++ MCEXECPATH=/work/mcktest/work_development_development/mck/bin/mcexec
+++ MCEXECN1='-n 1'
+++ MCEXECN2='-n 2'
+++ MCEXECN4='-n 4'
+++ MCEXECN8='-n 8'
+++ MCEXECN16='-n 16'
+++ OUTDIR=/work/mcktest/work_development_development/perflog/mckernel
++ mkdir -p /work/mcktest/work_development_development/perflog/mckernel
++ timeout 120 mpirun -hosts apollo15,apollo16 -n 2 -ppn 1 /work/mcktest/work_development_development/mck/bin/mcexec -n 1 /work/mcktest/perf/mpi-benchmarks/src/IMB-MPI1 PingPong
++ result=0
++ '[' 0 '!=' 0 ']'
++ ./ex-pingpong.awk /work/mcktest/work_development_development/perflog/mckernel/pingpong-1x2

test/issues/1355/aarch64_mvapich.txt

@@ -0,0 +1,2 @@
+Hello world from processor apollo15.postk.local.r-ccs.riken.jp, rank 0 out of 2 processors
+Hello world from processor apollo16.postk.local.r-ccs.riken.jp, rank 1 out of 2 processors

test/issues/1355/aarch64_pingpong.txt

@@ -0,0 +1,61 @@
+#------------------------------------------------------------
+#    Intel (R) MPI Benchmarks 2018, MPI-1 part
+#------------------------------------------------------------
+# Date                  : Mon Sep 23 15:01:48 2019
+# Machine               : aarch64
+# System                : Linux
+# Release               : 4.14.0-115.2.2.el7a.aarch64
+# Version               : #1 SMP Mon Nov 5 12:30:15 EST 2018
+# MPI Version           : 3.1
+# MPI Thread Environment:
+
+
+# Calling sequence was:
+
+# /work/mcktest/perf/mpi-benchmarks/src/IMB-MPI1 PingPong
+
+# Minimum message length in bytes:   0
+# Maximum message length in bytes:   4194304
+#
+# MPI_Datatype                   :   MPI_BYTE
+# MPI_Datatype for reductions    :   MPI_FLOAT
+# MPI_Op                         :   MPI_SUM
+#
+#
+
+# List of Benchmarks to run:
+
+# PingPong
+
+#---------------------------------------------------
+# Benchmarking PingPong
+# #processes = 2
+#---------------------------------------------------
+       #bytes #repetitions      t[usec]   Mbytes/sec
+            0         1000         1.82         0.00
+            1         1000         1.86         0.54
+            2         1000         1.91         1.05
+            4         1000         1.94         2.06
+            8         1000         1.87         4.27
+           16         1000         1.92         8.33
+           32         1000         1.93        16.59
+           64         1000         1.95        32.88
+          128         1000         2.04        62.72
+          256         1000         2.71        94.54
+          512         1000         2.85       179.50
+         1024         1000         3.15       324.92
+         2048         1000         3.86       531.16
+         4096         1000         5.69       719.95
+         8192         1000         8.29       988.34
+        16384         1000        12.99      1261.04
+        32768         1000        17.12      1914.39
+        65536          640        22.18      2954.41
+       131072          320        34.36      3814.35
+       262144          160        54.84      4779.84
+       524288           80        95.02      5517.82
+      1048576           40       172.05      6094.54
+      2097152           20       334.97      6260.68
+      4194304           10       661.65      6339.18
+
+
+# All processes entering MPI_Finalize

test/issues/1355/result.log

@@ -0,0 +1,21 @@
+*** C1355T01 start *******************************
++ result=0
+*** C1355T01 PASSED ******************************
+
+*** C1355T02 start *******************************
+# All processes entering MPI_Finalize
+*** C1355T02 PASSED ******************************
+
+*** C1355T03 start *******************************
++ result=0
+*** C1355T03 PASSED ******************************
+
+*** C1355T04 start *******************************
+# All processes entering MPI_Finalize
+*** C1355T04 PASSED ******************************
+
+*** C1355T05 start *******************************
+Hello world from processor apollo15.postk.local.r-ccs.riken.jp, rank 0 out of 2 processors
+Hello world from processor apollo16.postk.local.r-ccs.riken.jp, rank 1 out of 2 processors
+*** C1355T05 PASSED ******************************
+

test/issues/1355/x86_64_intel_bench.txt

@@ -0,0 +1,98 @@
+++ basename /work/mcktest/perf/bin/pingpong-1x2
++ TESTCASE=pingpong-1x2
++ '[' /work/mcktest == '' ']'
++ export AUTOTEST_HOME
+++ pwd -P
++ CWD=/work/mcktest/work_development_development
+++ dirname /work/mcktest/perf/bin/pingpong-1x2
++ cd /work/mcktest/perf/bin
++ app_dir=/work/mcktest/perf/mpi-benchmarks/src
++ . ./config.sh
+++++ dirname /work/mcktest/perf/bin/pingpong-1x2
++++ cd /work/mcktest/perf/bin/../..
++++ pwd -P
+++ export AUTOTEST_HOME=/work/mcktest
+++ AUTOTEST_HOME=/work/mcktest
+++ . /work/mcktest/bin/config.sh
++++ export NODETYPE=wallaby
++++ NODETYPE=wallaby
++++ export WGETSSHPROXY=ofp05
++++ WGETSSHPROXY=ofp05
++++ export 'WGETPROXYOPTS=-e use_proxy=yes -e http_proxy=wallaby -e https_proxy=wallaby:80'
++++ WGETPROXYOPTS='-e use_proxy=yes -e http_proxy=wallaby -e https_proxy=wallaby:80'
++++ MCKERNELBRANCH=development
++++ IHKBRANCH=development
++++ export WORKDIR=/work/mcktest/work_development_development
++++ WORKDIR=/work/mcktest/work_development_development
++++ SRC=/work/mcktest/work_development_development/src
++++ MCKBUILD=/work/mcktest/work_development_development/build
++++ MCKINSTALL=/work/mcktest/work_development_development/mck
++++ BUILDID_DIR=/work/mcktest/buildids
++++ GITS=/work/mcktest/git
++++ LTPINSTALL=/work/mcktest/ltp/install
++++ STRESS_TEST_DIR=/work/mcktest/stress_test/install
++++ MCKBUILDSCRIPT=/work/mcktest/bin/config_and_build_smp_x86.sh
++++ MPICH=/usr/lib64/mpich-3.2
++++ export LTPDIR=/work/mcktest/ltp/install
++++ LTPDIR=/work/mcktest/ltp/install
++++ export LTPROOT=/work/mcktest/ltp/install
++++ LTPROOT=/work/mcktest/ltp/install
++++ export 'MAIL_TO=mckernel@www.pccluster.org toshi@metahack.jp'
++++ MAIL_TO='mckernel@www.pccluster.org toshi@metahack.jp'
++++ export 'MCKERNELCONFIGOPTION=--enable-rusage --with-mpi=/usr/lib64/mpich-3.2 --enable-qlmpi'
++++ MCKERNELCONFIGOPTION='--enable-rusage --with-mpi=/usr/lib64/mpich-3.2 --enable-qlmpi'
++++ export IHKCONFIGOPTION=--enable-rusage
++++ IHKCONFIGOPTION=--enable-rusage
++++ MCREBOOTOPTION='-k 0 -f LOG_LOCAL6  -c 1-7,9-15,17-23,25-31 -m 10G@0,10G@1 -r 1-7:0+9-15:8+17-23:16+25-31:24 -o root -O'
++++ export PATH=/work/mcktest/bin:/work/mcktest/work_development_development/mck/bin:/work/mcktest/work_development_development/mck/sbin:/usr/lib64/mpich-3.2/bin:/usr/bin:/bin:/usr/sbin:/sbin
++++ PATH=/work/mcktest/bin:/work/mcktest/work_development_development/mck/bin:/work/mcktest/work_development_development/mck/sbin:/usr/lib64/mpich-3.2/bin:/usr/bin:/bin:/usr/sbin:/sbin
++++ export DATADIR=/work/mcktest/data
++++ DATADIR=/work/mcktest/data
++++ export BINDIR=/work/mcktest/bin
++++ BINDIR=/work/mcktest/bin
++++ export LOGDIR=/work/mcktest/log
++++ LOGDIR=/work/mcktest/log
++++ STRESS_TEST_TIMEOUT=60
++++ export OMP_NUM_THREADS_1=1
++++ OMP_NUM_THREADS_1=1
++++ export OMP_NUM_THREADS_2=2
++++ OMP_NUM_THREADS_2=2
++++ export OMP_NUM_THREADS_4=4
++++ OMP_NUM_THREADS_4=4
++++ export OMP_NUM_THREADS_8=8
++++ OMP_NUM_THREADS_8=8
++++ export OMP_NUM_THREADS_16=16
++++ OMP_NUM_THREADS_16=16
++++ export OMP_NUM_THREADS_24=24
++++ OMP_NUM_THREADS_24=24
++++ export OMP_NUM_THREADS_32=32
++++ OMP_NUM_THREADS_32=32
++++ export 'MPI_HOSTS=wallaby7 wallaby13'
++++ MPI_HOSTS='wallaby7 wallaby13'
++++ [[ -e /work/mcktest/bin/gerrit-creds ]]
++++ . /work/mcktest/bin/gerrit-creds
+++++ GERRITUSER=riken-syssoft-gerrit:ogzuCUSBrLM5gpUoECTKZya52oWf7Bu0fvO+oEeFNg
++++ export GERRITSERVER=review.gerrithub.io
++++ GERRITSERVER=review.gerrithub.io
++++ export 'GERRITCURL=curl -s --proxy http://wallaby:80 https://riken-syssoft-gerrit:ogzuCUSBrLM5gpUoECTKZya52oWf7Bu0fvO+oEeFNg@review.gerrithub.io'
++++ GERRITCURL='curl -s --proxy http://wallaby:80 https://riken-syssoft-gerrit:ogzuCUSBrLM5gpUoECTKZya52oWf7Bu0fvO+oEeFNg@review.gerrithub.io'
++ . ./perf-mpi-config.sh
++++ echo wallaby7 wallaby13
++++ tr ' ' ,
+++ COMMAHOSTS=wallaby7,wallaby13
+++ MPIHOSTS='-hosts wallaby7,wallaby13'
+++ MPIRUNOPTS='-genv I_MPI_DAPL_PROVIDER=ofa-v2-ib0 -genv I_MPI_DEBUG=2 -genv I_MPI_PIN=0 -genv I_MPI_COLL_INTRANODE_SHM_THRESHOLD=4194304'
+++ '[' xmcexec = xmcexec ']'
+++ MPIRUN='timeout 300 mpirun'
+++ MCEXECPATH=/work/mcktest/work_development_development/mck/bin/mcexec
+++ MCEXECN1='-n 1'
+++ MCEXECN2='-n 2'
+++ MCEXECN4='-n 4'
+++ MCEXECN8='-n 8'
+++ MCEXECN16='-n 16'
+++ OUTDIR=/work/mcktest/work_development_development/perflog/mckernel
++ mkdir -p /work/mcktest/work_development_development/perflog/mckernel
++ timeout 300 mpirun -hosts wallaby7,wallaby13 -genv I_MPI_DAPL_PROVIDER=ofa-v2-ib0 -genv I_MPI_DEBUG=2 -genv I_MPI_PIN=0 -genv I_MPI_COLL_INTRANODE_SHM_THRESHOLD=4194304 -n 2 -ppn 1 /work/mcktest/work_development_development/mck/bin/mcexec -n 1 /work/mcktest/perf/mpi-benchmarks/src/IMB-MPI1 PingPong
++ result=0
++ '[' 0 '!=' 0 ']'
++ ./ex-pingpong.awk /work/mcktest/work_development_development/perflog/mckernel/pingpong-1x2

test/issues/1355/x86_64_pingpong.txt

@@ -0,0 +1,65 @@
+MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
+MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
+[0] MPI startup(): libfabric version: 1.7.2a-impi
+[0] MPI startup(): libfabric provider: verbs;ofi_rxm
+#------------------------------------------------------------
+#    Intel (R) MPI Benchmarks 2018, MPI-1 part
+#------------------------------------------------------------
+# Date                  : Sun Sep 22 15:14:41 2019
+# Machine               : x86_64
+# System                : Linux
+# Release               : 3.10.0-693.2.2.el7.x86_64
+# Version               : #1 SMP Tue Sep 12 22:26:13 UTC 2017
+# MPI Version           : 3.1
+# MPI Thread Environment:
+
+
+# Calling sequence was:
+
+# /work/mcktest/perf/mpi-benchmarks/src/IMB-MPI1 PingPong
+
+# Minimum message length in bytes:   0
+# Maximum message length in bytes:   4194304
+#
+# MPI_Datatype                   :   MPI_BYTE
+# MPI_Datatype for reductions    :   MPI_FLOAT
+# MPI_Op                         :   MPI_SUM
+#
+#
+
+# List of Benchmarks to run:
+
+# PingPong
+
+#---------------------------------------------------
+# Benchmarking PingPong
+# #processes = 2
+#---------------------------------------------------
+       #bytes #repetitions      t[usec]   Mbytes/sec
+            0         1000         2.17         0.00
+            1         1000         2.14         0.47
+            2         1000         2.19         0.91
+            4         1000         2.17         1.84
+            8         1000         2.13         3.75
+           16         1000         2.34         6.83
+           32         1000         2.50        12.79
+           64         1000         2.86        22.40
+          128         1000         3.60        35.54
+          256         1000         2.21       116.09
+          512         1000         2.36       216.96
+         1024         1000         2.65       385.94
+         2048         1000         3.22       636.06
+         4096         1000         4.49       912.60
+         8192         1000         5.80      1412.38
+        16384         1000         8.61      1902.95
+        32768         1000        11.70      2800.47
+        65536          640        17.92      3656.26
+       131072          320        30.26      4331.35
+       262144          160       181.29      1446.01
+       524288           80       248.00      2114.06
+      1048576           40       380.09      2758.73
+      2097152           20       643.60      3258.48
+      4194304           10      1207.54      3473.41
+
+
+# All processes entering MPI_Finalize