gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

get_one_cpu_topology: Renumber core_id (physical core id)

Browse Source 
Change-Id: I4e4857e9a063d16d19d73adfabfc18a4b461bbfb
Refs: #1439
This commit is contained in:
Toshiyuki Takahashi
2020-02-14 11:27:49 +09:00
committed by Masamichi Takagi
parent bb7e140655
commit 0a4e6b49b4
10 changed files with 536 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
executer/kernel/mcctrl/mcctrl.h
View File

@@ -298,6 +298,7 @@ struct mcctrl_cpu_topology {
//struct mcctrl_usrdata *udp;
struct ihk_cpu_topology *saved;
int mckernel_cpu_id;
int mckernel_core_id;
cpumask_t core_siblings;
cpumask_t thread_siblings;

40
executer/kernel/mcctrl/sysfs_files.c
View File

@@ -12,6 +12,7 @@
*/
#include <linux/kernel.h>
#include <linux/hashtable.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/version.h>
@@ -23,6 +24,14 @@
#define wprintk(...) do { if (1) printk(KERN_WARNING __VA_ARGS__); } while (0)
#define eprintk(...) do { if (1) printk(KERN_ERR __VA_ARGS__); } while (0)
struct physical_core_id {
int linux_core_id;
int mckernel_core_id;
struct hlist_node next;
};
DEFINE_HASHTABLE(physical_core_id_map, 10);
static ssize_t
show_int(struct sysfsm_ops *ops, void *instance, void *buf, size_t size)
{
@@ -188,6 +197,9 @@ static void free_cpu_topology(struct mcctrl_usrdata *udp)
void free_topology_info(ihk_os_t os)
{
struct mcctrl_usrdata *udp = ihk_host_os_get_usrdata(os);
int bkt;
struct hlist_node *tmp;
struct physical_core_id *cur;
if (!udp) {
pr_warn("%s: warning: mcctrl_usrdata not found\n", __func__);
@@ -197,6 +209,11 @@ void free_topology_info(ihk_os_t os)
free_node_topology(udp);
free_cpu_topology(udp);
hash_for_each_safe(physical_core_id_map, bkt, tmp, cur, next) {
hash_del(&cur->next);
kfree(cur);
}
return;
} /* free_topology_info() */
@@ -348,6 +365,11 @@ static struct mcctrl_cpu_topology *get_one_cpu_topology(struct mcctrl_usrdata *u
struct mcctrl_cpu_topology *topology = NULL;
struct cache_topology *cache;
struct ihk_cache_topology *saved_cache;
int linux_core_id;
int mckernel_core_id;
struct physical_core_id *entry;
struct physical_core_id *cur;
static int nr_mckernel_core;
dprintk("get_one_cpu_topology(%p,%d)\n", udp, index);
topology = kmalloc(sizeof(*topology), GFP_KERNEL);
@@ -391,6 +413,22 @@ static struct mcctrl_cpu_topology *get_one_cpu_topology(struct mcctrl_usrdata *u
goto out;
}
linux_core_id = topology->saved->core_id;
mckernel_core_id = -1;
hash_for_each_possible(physical_core_id_map, cur, next, linux_core_id) {
mckernel_core_id = cur->mckernel_core_id;
break;
}
if (mckernel_core_id < 0) {
mckernel_core_id = nr_mckernel_core++;
entry = kmalloc(sizeof(struct physical_core_id), GFP_KERNEL);
entry->linux_core_id = linux_core_id;
entry->mckernel_core_id = mckernel_core_id;
hash_add(physical_core_id_map,
&entry->next, entry->linux_core_id);
}
topology->mckernel_core_id = mckernel_core_id;
list_for_each_entry(saved_cache,
&topology->saved->cache_topology_list, chain) {
cache = get_cache_topology(udp, topology, saved_cache);
@@ -512,7 +550,7 @@ static void setup_cpu_sysfs_files(struct mcctrl_usrdata *udp,
"%s/cpu%d/topology/physical_package_id",
prefix, cpu_number);
sysfsm_createf(udp->os, SYSFS_SNOOPING_OPS_d32,
&cpu->saved->core_id, 0444,
&cpu->mckernel_core_id, 0444,
"%s/cpu%d/topology/core_id",
prefix, cpu_number);

92
test/issues/1439/C1439.py Normal file
View File

@@ -0,0 +1,92 @@
#
# Test script for issue #1439
#
import os
import sys
import subprocess
mckdir = os.getenv('MCK_DIR')
mckbin = mckdir + '/bin'
mcksbin = mckdir + '/sbin'
mcreboot = mcksbin + '/mcreboot.sh'
mcstop = mcksbin + '/mcstop+release.sh'
mcexec = mckbin + '/mcexec'
def get_command_result(cmd):
results = subprocess.Popen(
cmd, stdout=subprocess.PIPE,
shell=True).stdout.readlines()
return [str(x).rstrip("\n") for x in results]
def enumerate_cpu(cpu_list):
allcpus = []
for ranged_cpu in cpu_list.split(','):
try:
cpu_begin, cpu_end = ranged_cpu.split('-')
except ValueError:
cpu_begin = cpu_end = ranged_cpu
for i in range(int(cpu_begin), int(cpu_end) + 1):
allcpus.append(i)
allcpus.sort()
return allcpus
def bind_cpu_core(catcmd, allcpus):
cpucores = {}
for cpu in allcpus:
sysfile = '/sys/devices/system/cpu/cpu%d/topology/core_id' % cpu
core_id = get_command_result(catcmd + ' ' + sysfile)[0]
cpucores[cpu] = int(core_id)
return cpucores
def compare_cores(linuxcpucores, mckernelcpucores):
linuxcpus = linuxcpucores.keys()
linuxcpus.sort()
linuxcores = []
for linuxcpu in linuxcpus:
linuxcores.append(linuxcpucores[linuxcpu])
mckernelcpus = mckernelcpucores.keys()
mckernelcpus.sort()
mckernelcores = []
for mckernelcpu in mckernelcpus:
mckernelcores.append(mckernelcpucores[mckernelcpu])
coremap = {}
seq = 0
for i in range(len(linuxcores)):
linuxcore = linuxcores[i]
mckernelcore = mckernelcores[i]
if linuxcore in coremap:
if mckernelcore != coremap[linuxcore]:
print 'FAIL'
quit()
else:
if seq != mckernelcore:
print 'FAIL'
quit()
seq = seq + 1
coremap[linuxcore] = mckernelcore
def main():
argvs = sys.argv
argc = len(argvs)
if (argc != 2):
print 'Usage: python %s <cpu_list>' % argvs[0]
quit()
print 'cpu_list = %s' % argvs[1]
cpulist = argvs[1]
linuxcpus = enumerate_cpu(cpulist)
linuxcpucores = bind_cpu_core('cat', linuxcpus)
print 'linux: '
print linuxcpucores
get_command_result('sudo ' + mcreboot + ' -c ' + cpulist)
mckernelcpus = [i for i in range(0, len(linuxcpus))]
mckernelcpucores = bind_cpu_core(mcexec + ' cat', mckernelcpus)
print 'mckernel: '
print mckernelcpucores
get_command_result('sudo ' + mcstop)
compare_cores(linuxcpucores, mckernelcpucores)
print 'SUCCESS'
if __name__ == '__main__':
main()

16
test/issues/1439/Makefile Normal file
View File

@@ -0,0 +1,16 @@
all:: show-omp-cpu-affinity
show-omp-cpu-affinity:: show-omp-cpu-affinity.c
fcc -Nclang -fopenmp -Nlibomp -o $@ $<
test:: test-core, test-omp
test-core::
. ${HOME}/.mck_test_config && export MCK_DIR && python C1439.py 1-4,30-33,56-61
test-omp:: show-omp-cpu-affinity
./test-affinity.sh
clean::
rm show-omp-cpu-affinity

50
test/issues/1439/README Normal file
View File

@@ -0,0 +1,50 @@
□ テスト内容
/sys/devices/system/cpu/cpuX/topology/core_idについて、以下を確かめる。
(A) 物理コア番号の振り方について以下を確かめる
1) 物理コアの第1の論理コアのcore_idはCPU番号(=cpu_id)と一致する
2) 論理コアのcore_idは、それが属する物理コアの第1の論理コアのcore_idと一致する
(B) McKernelがcpu_idをrenumberすることによる悪影響がないか確認する
1) 富士通OpenMPで、OMP_PROC_BINDの
close（物理コアをラウンドロビンで選ぶ）と
spread（CPUトポロジ上なるべく離れるように配置する）
について、期待通りのバインディングになることを確認する
なお、apolloでのテストでは、OMP_NUM_THREADSは2から物理コア数*2の間で、
OMP_NUM_THREADSと物理コア数のうち、大きいほうが小さい方で
割り切れる関係にあるものに設定した。
□ 実行手順
(1) $HOME/.mck_test_configを、MCK_DIRがMcKernelのインストール先を指すように編集する
(2) apolloログインノードにおいて一般ユーザで以下のコマンドを実行し、
富士通コンパイラでテストに使うOpenMPアプリケーションバイナリを作成する。
　　ソースコードはmckernelユーザでcheckoutしていると思われるが、その場合、
本ディレクトリを、その一般ユーザが書き込めるようにwrite permissionを
つけておく必要がある。
$ make
(3) (2)で作られたバイナリを(mckernelユーザの)apolloの計算ノードにて
動かすために、富士通コンパイラのDLLを当該ユーザが読める場所に
コピーして、その場所にLD_LIBRARY_PATHを通す。
コピー元のディレクトリは富士通コンパイラ0.36では以下のディレクトリである。
/usr/local/FJSVxtclang/fujitsu_compilers_sve_own_20191226/sve_own/lib64
(4) apolloの計算ノードにおいてmckernelユーザで以下のコマンドを実行し、
(A)(B)のテストを実行する。
$ make test-core # .... (A)
$ make test-omp # .... (B)
□ 確認方法
(A) (B) いずれにおいてもSUCCESSが出力され、FAILが出力されていないこと。
サンプル出力は以下のとおり
(A) aarch64_result_core.log
(B) aarch64_result_omp.log

8
test/issues/1439/aarch64_result_core.log Normal file
View File

@@ -0,0 +1,8 @@
$ make test
python C1439.py 1-4,30-33,56-61
cpu_list = 1-4,30-33,56-61
linux:
{32: 4, 1: 1, 2: 2, 3: 3, 4: 4, 33: 5, 56: 256, 57: 257, 58: 258, 59: 259, 60: 260, 61: 261, 30: 2, 31: 3}
mckernel:
{0: 0, 1: 1, 2: 2, 3: 3, 4: 1, 5: 2, 6: 3, 7: 4, 8: 5, 9: 6, 10: 7, 11: 8, 12: 9, 13: 10}
SUCCESS

108
test/issues/1439/aarch64_result_omp.log Normal file
View File

@@ -0,0 +1,108 @@
./test-affinity.sh
sudo /home/toshi/install/dev2-mckernel/sbin/mcreboot.sh -m 1G@0,1G@1 -c 4-27,32-55 -O
/home/toshi/install/dev2-mckernel/bin/mcexec python -u test-affinity.py
CPUS: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]
PLACES: [[0, 24], [1, 25], [2, 26], [3, 27], [4, 28], [5, 29], [6, 30], [7, 31], [8, 32], [9, 33], [10, 34], [11, 35], [12, 36], [13, 37], [14, 38], [15, 39], [16, 40], [17, 41], [18, 42], [19, 43], [20, 44], [21, 45], [22, 46], [23, 47]]
MAP_CPU_TO_PLACE {0: [0, 24], 1: [1, 25], 2: [2, 26], 3: [3, 27], 4: [4, 28], 5: [5, 29], 6: [6, 30], 7: [7, 31], 8: [8, 32], 9: [9, 33], 10: [10, 34], 11: [11, 35], 12: [12, 36], 13: [13, 37], 14: [14, 38], 15: [15, 39], 16: [16, 40], 17: [17, 41], 18: [18, 42], 19: [19, 43], 20: [20, 44], 21: [21, 45], 22: [22, 46], 23: [23, 47], 24: [0, 24], 25: [1, 25], 26: [2, 26], 27: [3, 27], 28: [4, 28], 29: [5, 29], 30: [6, 30], 31: [7, 31], 32: [8, 32], 33: [9, 33], 34: [10, 34], 35: [11, 35], 36: [12, 36], 37: [13, 37], 38: [14, 38], 39: [15, 39], 40: [16, 40], 41: [17, 41], 42: [18, 42], 43: [19, 43], 44: [20, 44], 45: [21, 45], 46: [22, 46], 47: [23, 47]}
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(close): {0: [0, 24], 1: [1, 25]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 1}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: close, OMP_NUM_THREAD: 2, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(spread): {0: [0, 24], 1: [12, 36]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 12}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: spread, OMP_NUM_THREAD: 2, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(close): {0: [0, 24], 1: [1, 25], 2: [2, 26]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 1, 2: 2}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: close, OMP_NUM_THREAD: 3, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(spread): {0: [0, 24], 1: [8, 32], 2: [16, 40]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 8, 2: 16}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: spread, OMP_NUM_THREAD: 3, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(close): {0: [0, 24], 1: [1, 25], 2: [2, 26], 3: [3, 27]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 1, 2: 2, 3: 3}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: close, OMP_NUM_THREAD: 4, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(spread): {0: [0, 24], 1: [6, 30], 2: [12, 36], 3: [18, 42]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 6, 2: 12, 3: 18}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: spread, OMP_NUM_THREAD: 4, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(close): {0: [0, 24], 1: [1, 25], 2: [2, 26], 3: [3, 27], 4: [4, 28], 5: [5, 29]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: close, OMP_NUM_THREAD: 6, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(spread): {0: [0, 24], 1: [4, 28], 2: [8, 32], 3: [12, 36], 4: [16, 40], 5: [20, 44]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 4, 2: 8, 3: 12, 4: 16, 5: 20}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: spread, OMP_NUM_THREAD: 6, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(close): {0: [0, 24], 1: [1, 25], 2: [2, 26], 3: [3, 27], 4: [4, 28], 5: [5, 29], 6: [6, 30], 7: [7, 31]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: close, OMP_NUM_THREAD: 8, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(spread): {0: [0, 24], 1: [3, 27], 2: [6, 30], 3: [9, 33], 4: [12, 36], 5: [15, 39], 6: [18, 42], 7: [21, 45]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 3, 2: 6, 3: 9, 4: 12, 5: 15, 6: 18, 7: 21}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: spread, OMP_NUM_THREAD: 8, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(close): {0: [0, 24], 1: [1, 25], 2: [2, 26], 3: [3, 27], 4: [4, 28], 5: [5, 29], 6: [6, 30], 7: [7, 31], 8: [8, 32], 9: [9, 33], 10: [10, 34], 11: [11, 35]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9, 10: 10, 11: 11}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: close, OMP_NUM_THREAD: 12, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(spread): {0: [0, 24], 1: [2, 26], 2: [4, 28], 3: [6, 30], 4: [8, 32], 5: [10, 34], 6: [12, 36], 7: [14, 38], 8: [16, 40], 9: [18, 42], 10: [20, 44], 11: [22, 46]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 2, 2: 4, 3: 6, 4: 8, 5: 10, 6: 12, 7: 14, 8: 16, 9: 18, 10: 20, 11: 22}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: spread, OMP_NUM_THREAD: 12, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(close): {0: [0, 24], 1: [1, 25], 2: [2, 26], 3: [3, 27], 4: [4, 28], 5: [5, 29], 6: [6, 30], 7: [7, 31], 8: [8, 32], 9: [9, 33], 10: [10, 34], 11: [11, 35], 12: [12, 36], 13: [13, 37], 14: [14, 38], 15: [15, 39], 16: [16, 40], 17: [17, 41], 18: [18, 42], 19: [19, 43], 20: [20, 44], 21: [21, 45], 22: [22, 46], 23: [23, 47]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9, 10: 10, 11: 11, 12: 12, 13: 13, 14: 14, 15: 15, 16: 16, 17: 17, 18: 18, 19: 19, 20: 20, 21: 21, 22: 22, 23: 23}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: close, OMP_NUM_THREAD: 24, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(spread): {0: [0, 24], 1: [1, 25], 2: [2, 26], 3: [3, 27], 4: [4, 28], 5: [5, 29], 6: [6, 30], 7: [7, 31], 8: [8, 32], 9: [9, 33], 10: [10, 34], 11: [11, 35], 12: [12, 36], 13: [13, 37], 14: [14, 38], 15: [15, 39], 16: [16, 40], 17: [17, 41], 18: [18, 42], 19: [19, 43], 20: [20, 44], 21: [21, 45], 22: [22, 46], 23: [23, 47]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9, 10: 10, 11: 11, 12: 12, 13: 13, 14: 14, 15: 15, 16: 16, 17: 17, 18: 18, 19: 19, 20: 20, 21: 21, 22: 22, 23: 23}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: spread, OMP_NUM_THREAD: 24, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(close): {0: [0, 24], 1: [0, 24], 2: [1, 25], 3: [1, 25], 4: [2, 26], 5: [2, 26], 6: [3, 27], 7: [3, 27], 8: [4, 28], 9: [4, 28], 10: [5, 29], 11: [5, 29], 12: [6, 30], 13: [6, 30], 14: [7, 31], 15: [7, 31], 16: [8, 32], 17: [8, 32], 18: [9, 33], 19: [9, 33], 20: [10, 34], 21: [10, 34], 22: [11, 35], 23: [11, 35], 24: [12, 36], 25: [12, 36], 26: [13, 37], 27: [13, 37], 28: [14, 38], 29: [14, 38], 30: [15, 39], 31: [15, 39], 32: [16, 40], 33: [16, 40], 34: [17, 41], 35: [17, 41], 36: [18, 42], 37: [18, 42], 38: [19, 43], 39: [19, 43], 40: [20, 44], 41: [20, 44], 42: [21, 45], 43: [21, 45], 44: [22, 46], 45: [22, 46], 46: [23, 47], 47: [23, 47]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 0, 2: 1, 3: 1, 4: 2, 5: 2, 6: 3, 7: 3, 8: 4, 9: 4, 10: 5, 11: 5, 12: 6, 13: 6, 14: 7, 15: 7, 16: 8, 17: 8, 18: 9, 19: 9, 20: 10, 21: 10, 22: 11, 23: 11, 24: 12, 25: 12, 26: 13, 27: 13, 28: 14, 29: 14, 30: 15, 31: 15, 32: 16, 33: 16, 34: 17, 35: 17, 36: 18, 37: 18, 38: 19, 39: 19, 40: 20, 41: 20, 42: 21, 43: 21, 44: 22, 45: 22, 46: 23, 47: 47}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: close, OMP_NUM_THREAD: 48, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(spread): {0: [0, 24], 1: [0, 24], 2: [1, 25], 3: [1, 25], 4: [2, 26], 5: [2, 26], 6: [3, 27], 7: [3, 27], 8: [4, 28], 9: [4, 28], 10: [5, 29], 11: [5, 29], 12: [6, 30], 13: [6, 30], 14: [7, 31], 15: [7, 31], 16: [8, 32], 17: [8, 32], 18: [9, 33], 19: [9, 33], 20: [10, 34], 21: [10, 34], 22: [11, 35], 23: [11, 35], 24: [12, 36], 25: [12, 36], 26: [13, 37], 27: [13, 37], 28: [14, 38], 29: [14, 38], 30: [15, 39], 31: [15, 39], 32: [16, 40], 33: [16, 40], 34: [17, 41], 35: [17, 41], 36: [18, 42], 37: [18, 42], 38: [19, 43], 39: [19, 43], 40: [20, 44], 41: [20, 44], 42: [21, 45], 43: [21, 45], 44: [22, 46], 45: [22, 46], 46: [23, 47], 47: [23, 47]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 0, 2: 1, 3: 1, 4: 2, 5: 2, 6: 3, 7: 3, 8: 4, 9: 4, 10: 5, 11: 5, 12: 6, 13: 6, 14: 7, 15: 7, 16: 8, 17: 8, 18: 9, 19: 9, 20: 10, 21: 10, 22: 11, 23: 11, 24: 12, 25: 12, 26: 13, 27: 13, 28: 14, 29: 14, 30: 15, 31: 15, 32: 16, 33: 16, 34: 17, 35: 17, 36: 18, 37: 18, 38: 19, 39: 19, 40: 20, 41: 20, 42: 21, 43: 21, 44: 22, 45: 22, 46: 23, 47: 47}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: spread, OMP_NUM_THREAD: 48, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(close): {0: [0, 24], 1: [0, 24], 2: [0, 24], 3: [1, 25], 4: [1, 25], 5: [1, 25], 6: [2, 26], 7: [2, 26], 8: [2, 26], 9: [3, 27], 10: [3, 27], 11: [3, 27], 12: [4, 28], 13: [4, 28], 14: [4, 28], 15: [5, 29], 16: [5, 29], 17: [5, 29], 18: [6, 30], 19: [6, 30], 20: [6, 30], 21: [7, 31], 22: [7, 31], 23: [7, 31], 24: [8, 32], 25: [8, 32], 26: [8, 32], 27: [9, 33], 28: [9, 33], 29: [9, 33], 30: [10, 34], 31: [10, 34], 32: [10, 34], 33: [11, 35], 34: [11, 35], 35: [11, 35], 36: [12, 36], 37: [12, 36], 38: [12, 36], 39: [13, 37], 40: [13, 37], 41: [13, 37], 42: [14, 38], 43: [14, 38], 44: [14, 38], 45: [15, 39], 46: [15, 39], 47: [15, 39], 48: [16, 40], 49: [16, 40], 50: [16, 40], 51: [17, 41], 52: [17, 41], 53: [17, 41], 54: [18, 42], 55: [18, 42], 56: [18, 42], 57: [19, 43], 58: [19, 43], 59: [19, 43], 60: [20, 44], 61: [20, 44], 62: [20, 44], 63: [21, 45], 64: [21, 45], 65: [21, 45], 66: [22, 46], 67: [22, 46], 68: [22, 46], 69: [23, 47], 70: [23, 47], 71: [23, 47]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 0, 2: 0, 3: 1, 4: 1, 5: 1, 6: 2, 7: 2, 8: 2, 9: 3, 10: 3, 11: 3, 12: 4, 13: 4, 14: 4, 15: 5, 16: 5, 17: 5, 18: 6, 19: 6, 20: 6, 21: 7, 22: 7, 23: 7, 24: 8, 25: 8, 26: 8, 27: 9, 28: 9, 29: 9, 30: 10, 31: 10, 32: 10, 33: 11, 34: 11, 35: 35, 36: 36, 37: 12, 38: 12, 39: 13, 40: 13, 41: 13, 42: 14, 43: 14, 44: 14, 45: 15, 46: 15, 47: 15, 48: 16, 49: 16, 50: 16, 51: 17, 52: 17, 53: 17, 54: 18, 55: 18, 56: 18, 57: 19, 58: 19, 59: 19, 60: 20, 61: 20, 62: 20, 63: 21, 64: 21, 65: 21, 66: 22, 67: 22, 68: 22, 69: 23, 70: 23, 71: 23}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: close, OMP_NUM_THREAD: 72, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(spread): {0: [0, 24], 1: [0, 24], 2: [0, 24], 3: [1, 25], 4: [1, 25], 5: [1, 25], 6: [2, 26], 7: [2, 26], 8: [2, 26], 9: [3, 27], 10: [3, 27], 11: [3, 27], 12: [4, 28], 13: [4, 28], 14: [4, 28], 15: [5, 29], 16: [5, 29], 17: [5, 29], 18: [6, 30], 19: [6, 30], 20: [6, 30], 21: [7, 31], 22: [7, 31], 23: [7, 31], 24: [8, 32], 25: [8, 32], 26: [8, 32], 27: [9, 33], 28: [9, 33], 29: [9, 33], 30: [10, 34], 31: [10, 34], 32: [10, 34], 33: [11, 35], 34: [11, 35], 35: [11, 35], 36: [12, 36], 37: [12, 36], 38: [12, 36], 39: [13, 37], 40: [13, 37], 41: [13, 37], 42: [14, 38], 43: [14, 38], 44: [14, 38], 45: [15, 39], 46: [15, 39], 47: [15, 39], 48: [16, 40], 49: [16, 40], 50: [16, 40], 51: [17, 41], 52: [17, 41], 53: [17, 41], 54: [18, 42], 55: [18, 42], 56: [18, 42], 57: [19, 43], 58: [19, 43], 59: [19, 43], 60: [20, 44], 61: [20, 44], 62: [20, 44], 63: [21, 45], 64: [21, 45], 65: [21, 45], 66: [22, 46], 67: [22, 46], 68: [22, 46], 69: [23, 47], 70: [23, 47], 71: [23, 47]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 0, 2: 0, 3: 1, 4: 1, 5: 1, 6: 2, 7: 2, 8: 2, 9: 3, 10: 3, 11: 3, 12: 4, 13: 4, 14: 4, 15: 5, 16: 5, 17: 5, 18: 6, 19: 6, 20: 6, 21: 7, 22: 7, 23: 7, 24: 8, 25: 8, 26: 8, 27: 9, 28: 9, 29: 9, 30: 10, 31: 10, 32: 10, 33: 11, 34: 11, 35: 35, 36: 36, 37: 12, 38: 12, 39: 13, 40: 13, 41: 13, 42: 14, 43: 14, 44: 14, 45: 15, 46: 15, 47: 15, 48: 16, 49: 16, 50: 16, 51: 17, 52: 17, 53: 17, 54: 18, 55: 18, 56: 18, 57: 19, 58: 19, 59: 19, 60: 20, 61: 20, 62: 20, 63: 21, 64: 21, 65: 21, 66: 22, 67: 22, 68: 22, 69: 23, 70: 23, 71: 23}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: spread, OMP_NUM_THREAD: 72, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(close): {0: [0, 24], 1: [0, 24], 2: [0, 24], 3: [0, 24], 4: [1, 25], 5: [1, 25], 6: [1, 25], 7: [1, 25], 8: [2, 26], 9: [2, 26], 10: [2, 26], 11: [2, 26], 12: [3, 27], 13: [3, 27], 14: [3, 27], 15: [3, 27], 16: [4, 28], 17: [4, 28], 18: [4, 28], 19: [4, 28], 20: [5, 29], 21: [5, 29], 22: [5, 29], 23: [5, 29], 24: [6, 30], 25: [6, 30], 26: [6, 30], 27: [6, 30], 28: [7, 31], 29: [7, 31], 30: [7, 31], 31: [7, 31], 32: [8, 32], 33: [8, 32], 34: [8, 32], 35: [8, 32], 36: [9, 33], 37: [9, 33], 38: [9, 33], 39: [9, 33], 40: [10, 34], 41: [10, 34], 42: [10, 34], 43: [10, 34], 44: [11, 35], 45: [11, 35], 46: [11, 35], 47: [11, 35], 48: [12, 36], 49: [12, 36], 50: [12, 36], 51: [12, 36], 52: [13, 37], 53: [13, 37], 54: [13, 37], 55: [13, 37], 56: [14, 38], 57: [14, 38], 58: [14, 38], 59: [14, 38], 60: [15, 39], 61: [15, 39], 62: [15, 39], 63: [15, 39], 64: [16, 40], 65: [16, 40], 66: [16, 40], 67: [16, 40], 68: [17, 41], 69: [17, 41], 70: [17, 41], 71: [17, 41], 72: [18, 42], 73: [18, 42], 74: [18, 42], 75: [18, 42], 76: [19, 43], 77: [19, 43], 78: [19, 43], 79: [19, 43], 80: [20, 44], 81: [20, 44], 82: [20, 44], 83: [20, 44], 84: [21, 45], 85: [21, 45], 86: [21, 45], 87: [21, 45], 88: [22, 46], 89: [22, 46], 90: [22, 46], 91: [22, 46], 92: [23, 47], 93: [23, 47], 94: [23, 47], 95: [23, 47]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1, 8: 2, 9: 2, 10: 2, 11: 2, 12: 3, 13: 3, 14: 3, 15: 3, 16: 4, 17: 4, 18: 4, 19: 4, 20: 5, 21: 5, 22: 5, 23: 5, 24: 6, 25: 6, 26: 6, 27: 6, 28: 7, 29: 7, 30: 7, 31: 31, 32: 32, 33: 8, 34: 8, 35: 8, 36: 9, 37: 9, 38: 9, 39: 9, 40: 10, 41: 10, 42: 10, 43: 10, 44: 11, 45: 11, 46: 11, 47: 11, 48: 12, 49: 12, 50: 12, 51: 12, 52: 13, 53: 13, 54: 13, 55: 13, 56: 14, 57: 14, 58: 14, 59: 14, 60: 15, 61: 15, 62: 15, 63: 15, 64: 16, 65: 16, 66: 16, 67: 16, 68: 17, 69: 17, 70: 17, 71: 17, 72: 18, 73: 18, 74: 18, 75: 18, 76: 19, 77: 19, 78: 19, 79: 19, 80: 20, 81: 20, 82: 20, 83: 20, 84: 21, 85: 21, 86: 21, 87: 21, 88: 22, 89: 22, 90: 22, 91: 22, 92: 23, 93: 23, 94: 47, 95: 23}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: close, OMP_NUM_THREAD: 96, RESULT: SUCCESS
./show-omp-cpu-affinity
MAP_THREAD_TO_PLACE(spread): {0: [0, 24], 1: [0, 24], 2: [0, 24], 3: [0, 24], 4: [1, 25], 5: [1, 25], 6: [1, 25], 7: [1, 25], 8: [2, 26], 9: [2, 26], 10: [2, 26], 11: [2, 26], 12: [3, 27], 13: [3, 27], 14: [3, 27], 15: [3, 27], 16: [4, 28], 17: [4, 28], 18: [4, 28], 19: [4, 28], 20: [5, 29], 21: [5, 29], 22: [5, 29], 23: [5, 29], 24: [6, 30], 25: [6, 30], 26: [6, 30], 27: [6, 30], 28: [7, 31], 29: [7, 31], 30: [7, 31], 31: [7, 31], 32: [8, 32], 33: [8, 32], 34: [8, 32], 35: [8, 32], 36: [9, 33], 37: [9, 33], 38: [9, 33], 39: [9, 33], 40: [10, 34], 41: [10, 34], 42: [10, 34], 43: [10, 34], 44: [11, 35], 45: [11, 35], 46: [11, 35], 47: [11, 35], 48: [12, 36], 49: [12, 36], 50: [12, 36], 51: [12, 36], 52: [13, 37], 53: [13, 37], 54: [13, 37], 55: [13, 37], 56: [14, 38], 57: [14, 38], 58: [14, 38], 59: [14, 38], 60: [15, 39], 61: [15, 39], 62: [15, 39], 63: [15, 39], 64: [16, 40], 65: [16, 40], 66: [16, 40], 67: [16, 40], 68: [17, 41], 69: [17, 41], 70: [17, 41], 71: [17, 41], 72: [18, 42], 73: [18, 42], 74: [18, 42], 75: [18, 42], 76: [19, 43], 77: [19, 43], 78: [19, 43], 79: [19, 43], 80: [20, 44], 81: [20, 44], 82: [20, 44], 83: [20, 44], 84: [21, 45], 85: [21, 45], 86: [21, 45], 87: [21, 45], 88: [22, 46], 89: [22, 46], 90: [22, 46], 91: [22, 46], 92: [23, 47], 93: [23, 47], 94: [23, 47], 95: [23, 47]}
RESULT_MAP_THREAD_TO_CPU: {0: 0, 1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1, 8: 2, 9: 2, 10: 2, 11: 2, 12: 3, 13: 3, 14: 3, 15: 3, 16: 4, 17: 4, 18: 4, 19: 4, 20: 5, 21: 5, 22: 5, 23: 5, 24: 6, 25: 6, 26: 6, 27: 6, 28: 7, 29: 7, 30: 7, 31: 31, 32: 32, 33: 8, 34: 8, 35: 8, 36: 9, 37: 9, 38: 9, 39: 9, 40: 10, 41: 10, 42: 10, 43: 10, 44: 11, 45: 11, 46: 11, 47: 11, 48: 12, 49: 12, 50: 12, 51: 12, 52: 13, 53: 13, 54: 13, 55: 13, 56: 14, 57: 14, 58: 14, 59: 14, 60: 15, 61: 15, 62: 15, 63: 15, 64: 16, 65: 16, 66: 16, 67: 16, 68: 17, 69: 17, 70: 17, 71: 17, 72: 18, 73: 18, 74: 18, 75: 18, 76: 19, 77: 19, 78: 19, 79: 19, 80: 20, 81: 20, 82: 20, 83: 20, 84: 21, 85: 21, 86: 21, 87: 21, 88: 22, 89: 22, 90: 22, 91: 22, 92: 23, 93: 23, 94: 47, 95: 23}
#CPU: 48, #PLACE: 24, OMP_PLACES: cores, OMP_PROC_BIND: spread, OMP_NUM_THREAD: 96, RESULT: SUCCESS
sudo /home/toshi/install/dev2-mckernel/sbin/mcstop+release.sh

19
test/issues/1439/show-omp-cpu-affinity.c Normal file
View File

@@ -0,0 +1,19 @@
#define _GNU_SOURCE
#include <stdio.h>
#include <sched.h>
#include <omp.h>
int main(int argc, char **argv)
{
#pragma omp parallel
{
int thread_num = omp_get_thread_num();
int cpu_num = sched_getcpu();
printf("Thread %d CPU %d\n", thread_num, cpu_num);
}
return 0;
}

190
test/issues/1439/test-affinity.py Normal file
View File

@@ -0,0 +1,190 @@
import os
import sys
import subprocess
import math
def get_command_result(cmd):
print cmd
results = subprocess.Popen(
cmd, stdout=subprocess.PIPE,
shell=True).stdout.readlines()
return [str(x).rstrip("\n") for x in results]
def get_cpus():
cpus = []
online_file = open('/sys/devices/system/cpu/online', 'r')
for cpurange in online_file.readlines()[0].strip().split(','):
try:
cpurange_start, cpurange_end = cpurange.split('-')
except ValueError:
cpurange_start = cpurange_end = cpurange
for cpu in range(int(cpurange_start), int(cpurange_end) + 1):
cpus.append(cpu)
return cpus
def get_omp_places_cores(cpus):
places = []
map_cpu_to_place = {}
for cpu in cpus:
if cpu not in map_cpu_to_place:
siblings_file = open('/sys/devices/system/cpu/cpu{0}/topology/thread_siblings_list'.format(cpu))
place = []
siblings = siblings_file.readlines()[0].strip().split(',')
for sibling in siblings:
place.append(int(sibling))
places.append(place)
for sibling in siblings:
map_cpu_to_place[int(sibling)] = place
return places, map_cpu_to_place
def index_of_place(places, cpu):
i = 0
for place in places:
if cpu in place:
return i
i = i + 1
return -1
def index_of_subpartition(subpartition, place):
i = 0
for placelist in subpartition:
if place in placelist:
return i
i = i + 1
return -1
def get_estimated_bind(omp_proc_bind, nthreads, places):
if omp_proc_bind == 'close':
return get_estimated_bind_close(0, 0, nthreads, places)
elif omp_proc_bind == 'spread':
return get_estimated_bind_spread(0, 0, nthreads, places)
return None
def get_estimated_bind_close(master_thread, master_cpu, nthreads, places):
map_thread_to_place = {}
nplaces = len(places)
# print 'nthreads =', nthreads
# print 'nplaces =', nplaces
if nthreads <= nplaces:
place_idx = index_of_place(places, master_cpu)
# print 'place_idx =', place_idx
for i in range(nthreads):
thread = (master_thread + i) % nthreads
map_thread_to_place[thread] = places[(place_idx + i) % nplaces]
else:
s = [0] * nplaces
for p in range(nplaces):
if nplaces - p <= nthreads % nplaces: # implementation defined
s[p] = nthreads / nplaces + 1 # ceil
else:
s[p] = nthreads / nplaces # floor
# print 's[', p, '] =', s[p]
i_begin = 0
place_idx = index_of_place(places, master_cpu)
for p in range(nplaces):
for i in range(i_begin, i_begin + s[p]):
thread = (master_thread + i) % nthreads
map_thread_to_place[thread] = places[(place_idx + p) % nplaces]
i_begin = i_begin + s[p]
return map_thread_to_place
def get_estimated_bind_spread(master_thread, master_cpu, nthreads, places):
map_thread_to_place = {}
nplaces = len(places)
if nthreads <= nplaces:
places_subpartition = []
p_begin = 0
for i in range(nthreads):
if nthreads - i <= nplaces % nthreads: # implementation defined
size_places_subpartition = nplaces / nthreads + 1 # ceil
else:
size_places_subpartition = nplaces / nthreads # floor
places_subpartition.append(places[p_begin:p_begin + size_places_subpartition])
p_begin = p_begin + size_places_subpartition
place_idx = index_of_place(places, master_cpu)
places_subpartition_idx = index_of_subpartition(places_subpartition, places[place_idx])
for i in range(nthreads):
thread = (master_thread + i) % nthreads
if thread == master_thread:
map_thread_to_place[thread] = places[place_idx]
else:
map_thread_to_place[thread] = places_subpartition[(places_subpartition_idx + i) % nthreads][0]
else:
threads = []
for i in range(nthreads):
threads.append(i)
threads_subpartition = []
i_begin = 0
for p in range(nplaces):
if nplaces - p <= nthreads % nplaces: # implementation defined
size_threads_subpartition = nthreads / nplaces + 1 # ceil
else:
size_threads_subpartition = nthreads / nplaces # floor
threads_subpartition.append(threads[i_begin:i_begin + size_threads_subpartition])
i_begin = i_begin + size_threads_subpartition
place_idx = index_of_place(places, master_cpu)
for p in range(nplaces):
for i in threads_subpartition[p]:
thread = (master_thread + i) % nthreads
map_thread_to_place[thread] = places[p]
return map_thread_to_place
def run_and_get_omp_cpu_affinity(omp_proc_bind, nthreads):
os.environ['NODES'] = '1'
os.environ['PPN'] = '1'
os.environ['HWLOC_HIDE_ERRORS'] = '1'
os.environ['OMP_PLACES'] = 'cores'
os.environ['OMP_PROC_BIND'] = omp_proc_bind
os.environ['OMP_NUM_THREADS'] = str(nthreads)
command = './show-omp-cpu-affinity'
result_map_thread_to_cpu = {}
for line in get_command_result(command):
outputs = line.split(' ')
thread = outputs[1]
cpu = outputs[3]
result_map_thread_to_cpu[int(thread)] = int(cpu)
return result_map_thread_to_cpu
def compare_result(nthreads, map_thread_to_place, result_map_thread_to_cpu):
try:
for thread in range(nthreads):
place = map_thread_to_place[thread]
if result_map_thread_to_cpu[thread] not in place:
return False
return True
except KeyError:
return False
def test_cpu_affinity(cpus, omp_proc_bind, nthreads, places):
map_thread_to_place = get_estimated_bind(omp_proc_bind, nthreads, places)
result_map_thread_to_cpu = run_and_get_omp_cpu_affinity(omp_proc_bind, nthreads)
if compare_result(nthreads, map_thread_to_place, result_map_thread_to_cpu):
result = 'SUCCESS'
else:
result = 'FAIL'
print "MAP_THREAD_TO_PLACE({0}): {1}".format(omp_proc_bind, map_thread_to_place)
print "RESULT_MAP_THREAD_TO_CPU: {0}".format(result_map_thread_to_cpu)
print "#CPU: {}, #PLACE: {}, OMP_PLACES: cores, OMP_PROC_BIND: {}, OMP_NUM_THREAD: {}, RESULT: {}".format(len(cpus), len(places), omp_proc_bind, nthreads, result)
def main():
cpus = get_cpus()
print 'CPUS:', cpus
places, map_cpu_to_place = get_omp_places_cores(cpus)
print 'PLACES:', places
print 'MAP_CPU_TO_PLACE', map_cpu_to_place
print
nplaces = len(places)
for nthreads in range(2, nplaces * 4 + 1):
if nthreads < nplaces and nplaces % nthreads > 0:
continue
if nthreads >= nplaces and nthreads % nplaces > 0:
continue
test_cpu_affinity(cpus, 'close', nthreads, places)
print
test_cpu_affinity(cpus, 'spread', nthreads, places)
print
if __name__ == '__main__':
main()

13
test/issues/1439/test-affinity.sh Executable file
View File

@@ -0,0 +1,13 @@
#!/bin/bash
. ${HOME}/.mck_test_config
export MCK_DIR
echo sudo ${MCK_DIR}/sbin/mcreboot.sh -m 1G@0,1G@1 -c 4-27,32-55 -O
sudo ${MCK_DIR}/sbin/mcreboot.sh -m 1G@0,1G@1 -c 4-27,32-55 -O
#sudo ${MCK_DIR}/sbin/mcreboot.sh -m 1G@0,1G@1 -c 4-27,32-55,60-83,88-111 -r 4-7,32-35:0+8-11,36-39:1+12-15,40-43:2+16-19,44-47:3+20-23,48-51:28+24-27,52-55:29+60-63,88-91:56+64-67,92-95:57+68-71,96-99:58+72-75,100-103:59+76-79,104-107:84+80-83,108-111:85 -O
echo ${MCK_DIR}/bin/mcexec python -u test-affinity.py
${MCK_DIR}/bin/mcexec python -u test-affinity.py
echo sudo ${MCK_DIR}/sbin/mcstop+release.sh
sudo ${MCK_DIR}/sbin/mcstop+release.sh