gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mcexec: memory policy control by environmental variable

Browse Source 
Refs: #1470
Change-Id: I3d556cae90d31d81572b1c4e5c680e826577d428
This commit is contained in:
Masamichi Takagi
2020-12-15 14:17:32 +09:00
parent ed670c03af
commit 33dd2e60b1
21 changed files with 729 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
executer/user/mcexec.c
View File

@@ -84,7 +84,11 @@
#include "../include/pmi.h"
#include "../include/qlmpi.h"
#include <sys/xattr.h>
#include "../include/defs.h"
#include "../../lib/include/list.h"
#include "../../lib/include/bitops-set_bit.h"
#include "../../lib/include/bitops-clear_bit.h"
#include "../../lib/include/bitops-test_bit.h"
//#define DEBUG
#define ADD_ENVS_OPTION
@@ -1056,6 +1060,64 @@ static inline cpu_set_t *numa_node_set(int n)
return (cpu_set_t *)(numa_nodes + n * cpu_set_size);
}
static inline void _numa_local(__cpu_set_unit *localset,
unsigned long *nodemask, int nonlocal)
{
int i;
memset(nodemask, 0, PLD_PROCESS_NUMA_MASK_BITS / 8);
for (i = 0; i < nnodes; i++) {
cpu_set_t *nodeset = numa_node_set(i);
int j;
if (nonlocal) {
set_bit(i, nodemask);
}
for (j = 0; j < ncpu; j++) {
if (test_bit(j, localset)) {
__dprintf("%d belongs to local set\n", j);
}
if (CPU_ISSET_S(j, cpu_set_size, nodeset)) {
__dprintf("%d belongs to node %d\n", j, i);
}
if (test_bit(j, localset) &&
CPU_ISSET_S(j, cpu_set_size, nodeset)) {
if (nonlocal) {
clear_bit(i, nodemask);
} else {
set_bit(i, nodemask);
}
}
}
}
}
static inline void numa_local(__cpu_set_unit *localset, unsigned long *nodemask)
{
_numa_local(localset, nodemask, 0);
}
static inline void numa_nonlocal(__cpu_set_unit *localset,
unsigned long *nodemask)
{
_numa_local(localset, nodemask, 1);
}
static inline void numa_all(unsigned long *nodemask)
{
int i;
memset(nodemask, 0, PLD_PROCESS_NUMA_MASK_BITS / 8);
for (i = 0; i < nnodes; i++) {
set_bit(i, nodemask);
}
}
pid_t master_tid;
pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
@@ -2685,6 +2747,7 @@ int main(int argc, char **argv)
desc->heap_extension = heap_extension;
desc->mpol_bind_mask = 0;
desc->mpol_mode = PLD_MPOL_MAX; /* not specified */
if (mpol_bind_nodes) {
struct bitmask *bind_mask;
bind_mask = numa_parse_nodestring_all(mpol_bind_nodes);
@@ -2698,6 +2761,54 @@ int main(int argc, char **argv)
}
}
}
/* Fujitsu TCS specific: mempolicy */
else if (getenv("OMPI_MCA_plm_ple_memory_allocation_policy")) {
char *mpol =
getenv("OMPI_MCA_plm_ple_memory_allocation_policy");
__dprintf("OMPI_MCA_plm_ple_memory_allocation_policy: %s\n",
mpol);
if (!strncmp(mpol, "localalloc", 10)) {
/* MPOL_DEFAULT has the same effect as MPOL_LOCAL */
desc->mpol_mode = MPOL_DEFAULT;
}
else if (!strncmp(mpol, "interleave_local", 16)) {
desc->mpol_mode = MPOL_INTERLEAVE;
numa_local(desc->cpu_set, desc->mpol_nodemask);
}
else if (!strncmp(mpol, "interleave_nonlocal", 19)) {
desc->mpol_mode = MPOL_INTERLEAVE;
numa_nonlocal(desc->cpu_set, desc->mpol_nodemask);
}
else if (!strncmp(mpol, "interleave_all", 14)) {
desc->mpol_mode = MPOL_INTERLEAVE;
numa_all(desc->mpol_nodemask);
}
else if (!strncmp(mpol, "bind_local", 10)) {
desc->mpol_mode = MPOL_BIND;
numa_local(desc->cpu_set, desc->mpol_nodemask);
}
else if (!strncmp(mpol, "bind_nonlocal", 13)) {
desc->mpol_mode = MPOL_BIND;
numa_nonlocal(desc->cpu_set, desc->mpol_nodemask);
}
else if (!strncmp(mpol, "bind_all", 8)) {
desc->mpol_mode = MPOL_BIND;
numa_all(desc->mpol_nodemask);
}
else if (!strncmp(mpol, "prefer_local", 12)) {
desc->mpol_mode = MPOL_PREFERRED;
numa_local(desc->cpu_set, desc->mpol_nodemask);
}
else if (!strncmp(mpol, "prefer_nonlocal", 15)) {
desc->mpol_mode = MPOL_PREFERRED;
numa_nonlocal(desc->cpu_set, desc->mpol_nodemask);
}
__dprintf("mpol_mode: %d, mpol_nodemask: %ld\n",
desc->mpol_mode, desc->mpol_nodemask[0]);
}
desc->uti_thread_rank = uti_thread_rank;
desc->uti_use_last_cpu = uti_use_last_cpu;