From 0a0e2c04a017c904dc8cef43911e0227b57bc049 Mon Sep 17 00:00:00 2001
From: Balazs Gerofi <bgerofi@riken.jp>
Date: Fri, 7 Aug 2015 08:51:50 +0900
Subject: [PATCH] support for dynamically toggling time sharing when CPU is
 oversubscribed

---
 arch/x86/kernel/cpu.c | 26 ++++++++++++++++++++++++++
 kernel/include/cls.h  |  1 +
 kernel/process.c      | 19 +++++++++++++++++++
 3 files changed, 46 insertions(+)

diff --git a/arch/x86/kernel/cpu.c b/arch/x86/kernel/cpu.c
index 9fffcc38..47784c29 100644
--- a/arch/x86/kernel/cpu.c
+++ b/arch/x86/kernel/cpu.c
@@ -39,6 +39,7 @@
 #define LAPIC_ICR0          0x300
 #define LAPIC_ICR2          0x310
 #define LAPIC_ESR           0x280
+#define LOCAL_TIMER_VECTOR  0xef
 
 #define APIC_INT_LEVELTRIG      0x08000
 #define APIC_INT_ASSERT         0x04000
@@ -48,6 +49,8 @@
 #define APIC_DM_NMI             0x00400
 #define APIC_DM_INIT            0x00500
 #define APIC_DM_STARTUP         0x00600
+#define APIC_DIVISOR            16
+#define APIC_LVT_TIMER_PERIODIC (1 << 17)
 
 
 //#define DEBUG_PRINT_CPU
@@ -252,6 +255,23 @@ void lapic_icr_write(unsigned int h, unsigned int l)
 }
 
 
+void lapic_timer_enable(unsigned int clocks)
+{
+	unsigned int lvtt_value;
+
+	lapic_write(LAPIC_TIMER_INITIAL, clocks / APIC_DIVISOR);
+	lapic_write(LAPIC_TIMER_DIVIDE, 3);
+
+	/* initialize periodic timer */
+	lvtt_value = LOCAL_TIMER_VECTOR | APIC_LVT_TIMER_PERIODIC;
+	lapic_write(LAPIC_TIMER, lvtt_value);
+}
+
+void lapic_timer_disable()
+{
+	lapic_write(LAPIC_TIMER_INITIAL, 0);
+}
+
 void print_msr(int idx)
 {
 	int bit;
@@ -673,6 +693,12 @@ void handle_interrupt(int vector, struct x86_user_context *regs)
 			panic("Unhandled exception");
 		}
 	}
+	else if (vector == LOCAL_TIMER_VECTOR) {
+		/* Timer interrupt, enabled only on oversubscribed CPU cores,
+		 * request reschedule */
+		v->flags |= CPU_FLAG_NEED_RESCHED;
+		dkprintf("timer[%lu]: CPU_FLAG_NEED_RESCHED \n", rdtsc());
+	}
 	else if (vector >= IHK_TLB_FLUSH_IRQ_VECTOR_START && 
 	         vector < IHK_TLB_FLUSH_IRQ_VECTOR_END) {
 
diff --git a/kernel/include/cls.h b/kernel/include/cls.h
index 687a216c..774443fe 100644
--- a/kernel/include/cls.h
+++ b/kernel/include/cls.h
@@ -69,6 +69,7 @@ struct cpu_local_var {
 	struct list_head migq;
 	int in_interrupt;
 	int no_preempt;
+	int timer_enabled;
 } __attribute__((aligned(64)));
 
 
diff --git a/kernel/process.c b/kernel/process.c
index 572e065b..e7e476c5 100644
--- a/kernel/process.c
+++ b/kernel/process.c
@@ -57,6 +57,8 @@ extern void restore_fp_regs(struct process *proc);
 void settid(struct process *proc, int mode, int newcpuid, int oldcpuid);
 extern void __runq_add_proc(struct process *proc, int cpu_id);
 extern void terminate_host(int pid);
+extern void lapic_timer_enable(unsigned int clocks);
+extern void lapic_timer_disable();
 
 int refcount_fork_tree_node(struct fork_tree_node *ftn)
 {
@@ -2264,6 +2266,23 @@ redo:
 			list_add_tail(&prev->sched_list, &(v->runq));
 			++v->runq_len;
 		}
+
+		/* Toggle timesharing if CPU core is oversubscribed 
+		 * (on last CPU core only for now) */
+		if (ihk_mc_get_processor_id() == num_processors - 1) {
+			if (v->runq_len > 1) {
+				if (!cpu_local_var(timer_enabled)) {
+					lapic_timer_enable(10000000);
+					cpu_local_var(timer_enabled) = 1;
+				}
+			}
+			else {
+				if (cpu_local_var(timer_enabled)) {
+					lapic_timer_disable();
+					cpu_local_var(timer_enabled) = 0;
+				}
+			}
+		}
 	}
 
 	if (v->flags & CPU_FLAG_NEED_MIGRATE) {