From 1bc3218fc1368550ebc771e4517c3518d19c7e3c Mon Sep 17 00:00:00 2001
From: Balazs Gerofi <bgerofi@riken.jp>
Date: Sun, 22 Jan 2017 13:44:07 +0900
Subject: [PATCH] partitioned execution: bind mcexec to corresponding NUMA node

---
 executer/include/uprotocol.h     |  1 +
 executer/kernel/mcctrl/control.c | 11 ++++++++++
 executer/user/Makefile.in        |  2 +-
 executer/user/mcexec.c           | 37 +++++++++++++++++++++++++++++---
 4 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/executer/include/uprotocol.h b/executer/include/uprotocol.h
index 9a4d5ba7..9cf3eb45 100644
--- a/executer/include/uprotocol.h
+++ b/executer/include/uprotocol.h
@@ -85,6 +85,7 @@ struct get_cpu_set_arg {
 	void *cpu_set;
 	size_t cpu_set_size;	// Size in bytes
 	int *target_core;
+	int *mcexec_linux_numa; // NUMA domain to bind mcexec to
 };
 
 #define PLD_CPU_SET_MAX_CPUS 1024
diff --git a/executer/kernel/mcctrl/control.c b/executer/kernel/mcctrl/control.c
index 3261b26e..edfb8768 100644
--- a/executer/kernel/mcctrl/control.c
+++ b/executer/kernel/mcctrl/control.c
@@ -493,6 +493,7 @@ static long mcexec_get_cpuset(ihk_os_t os, unsigned long arg)
 	struct cache_topology *cache_top;
 	int cpu, cpus_assigned, cpus_to_assign, cpu_prev;
 	int ret = 0;
+	int mcexec_linux_numa;
 	cpumask_t cpus_used;
 	cpumask_t cpus_to_use;
 	struct mcctrl_per_proc_data *ppd;
@@ -643,6 +644,7 @@ next_cpu:
 		goto put_and_unlock_out;
 	}
 
+	/* Copy IKC target core and mcexec Linux NUMA id */
 	cpu = cpumask_next(-1, &cpus_to_use);
 	if (copy_to_user(req.target_core, &cpu, sizeof(cpu))) {
 		printk("%s: error copying target core to user\n",
@@ -651,6 +653,15 @@ next_cpu:
 		goto put_and_unlock_out;
 	}
 
+	mcexec_linux_numa = cpu_to_node(mckernel_cpu_2_linux_cpu(udp, cpu));
+	if (copy_to_user(req.mcexec_linux_numa, &mcexec_linux_numa,
+				sizeof(mcexec_linux_numa))) {
+		printk("%s: error copying mcexec Linux NUMA id\n",
+				__FUNCTION__);
+		ret = -EINVAL;
+		goto put_and_unlock_out;
+	}
+
 	/* Save in per-process structure */
 	memcpy(&ppd->cpu_set, &cpus_to_use, sizeof(cpumask_t));
 	ppd->ikc_target_cpu = cpu;
diff --git a/executer/user/Makefile.in b/executer/user/Makefile.in
index acd948a5..6418c746 100644
--- a/executer/user/Makefile.in
+++ b/executer/user/Makefile.in
@@ -11,7 +11,7 @@ IHKDIR ?= $(VPATH)/../../../ihk/linux/include/
 all: $(TARGET)
 
 mcexec: mcexec.c
-	$(CC) -I${KDIR} $(CFLAGS) $(EXTRA_CFLAGS) -fPIE -pie -lrt -pthread -o $@ $^ $(EXTRA_OBJS)
+	$(CC) -I${KDIR} $(CFLAGS) $(EXTRA_CFLAGS) -fPIE -pie -lrt -lnuma -pthread -o $@ $^ $(EXTRA_OBJS)
 
 eclair: eclair.c
 	$(CC) $(CFLAGS) -I${IHKDIR} -o $@ $^ $(LIBS)
diff --git a/executer/user/mcexec.c b/executer/user/mcexec.c
index 90c1e11d..257cad64 100644
--- a/executer/user/mcexec.c
+++ b/executer/user/mcexec.c
@@ -66,6 +66,8 @@
 #include "../include/uprotocol.h"
 #include <getopt.h>
 #include "../config.h"
+#include <numa.h>
+#include <numaif.h>
 
 //#define DEBUG
 
@@ -1623,21 +1625,50 @@ int main(int argc, char **argv)
 	/* Partitioned execution, obtain CPU set */
 	if (nr_processes > 0) {
 		struct get_cpu_set_arg cpu_set_arg;
+		int mcexec_linux_numa = 0;
 
 		cpu_set_arg.cpu_set = (void *)&desc->cpu_set;
 		cpu_set_arg.cpu_set_size = sizeof(desc->cpu_set);
 		cpu_set_arg.nr_processes = nr_processes;
 		cpu_set_arg.target_core = &target_core;
-		
+		cpu_set_arg.mcexec_linux_numa = &mcexec_linux_numa;
+
 		if (ioctl(fd, MCEXEC_UP_GET_CPUSET, (void *)&cpu_set_arg) != 0) {
 			perror("getting CPU set for partitioned execution");
 			close(fd);
 			return 1;
 		}
-		
+
 		desc->cpu = target_core;
+
+		/* This call may not succeed, but that is fine */
+		if (numa_run_on_node(mcexec_linux_numa) < 0) {
+			__dprint("%s: WARNING: couldn't bind to NUMA %d\n",
+				__FUNCTION__, mcexec_linux_numa);
+		}
+#ifdef DEBUG
+		else {
+			cpu_set_t cpuset;
+			char affinity[BUFSIZ];
+
+			CPU_ZERO(&cpuset);
+			if ((sched_getaffinity(0, sizeof(cpu_set_t), &cpuset)) != 0) {
+				perror("Error sched_getaffinity");
+				exit(1);
+			}
+
+			affinity[0] = '\0';
+			for (i = 0; i < 512; i++) {
+				if (CPU_ISSET(i, &cpuset) == 1) {
+					sprintf(affinity, "%s %d", affinity, i);
+				}
+			}
+			__dprint("%s: PID: %d affinity: %s\n",
+					__FUNCTION__, getpid(), affinity);
+		}
+#endif
 	}
-	
+
 	if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) {
 		perror("prepare");
 		close(fd);