From f0bc1a6b07efbaf70d0a12d9939299b06d061644 Mon Sep 17 00:00:00 2001
From: Masamichi Takagi <masamichi.takagi@riken.jp>
Date: Wed, 24 Jul 2019 11:08:32 +0900
Subject: [PATCH] cmake: Add option for "mem: per-CPU allocator cache
 (ThunderX2 workaround)"

Change-Id: I7156cf433b2081246d1d9b8e4fde489609676ef1
---
 CMakeLists.txt       | 2 ++
 config.h.in          | 3 +++
 kernel/cls.c         | 2 ++
 kernel/include/cls.h | 3 +++
 lib/page_alloc.c     | 4 ++++
 5 files changed, 14 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f9fc4ad8..0ace2ed1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -87,6 +87,7 @@ option(ENABLE_RUSAGE "Enable rusage support" ON)
 option(ENABLE_QLMPI "Enable qlmpi programs" OFF)
 option(ENABLE_UTI "Enable uti support" OFF)
 option(ENABLE_UBSAN "Enable undefined behaviour sanitizer on mckernel size" OFF)
+option(ENABLE_PER_CPU_ALLOC_CACHE "Enable per-CPU allocator cache (ThunderX2 workaround)" OFF)
 
 find_package(PkgConfig REQUIRED)
 set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON)
@@ -197,4 +198,5 @@ message("ENABLE_UTI: ${ENABLE_UTI}")
 message("ENABLE_WERROR: ${ENABLE_WERROR}")
 message("ENABLE_UBSAN: ${ENABLE_UBSAN}")
 message("ENABLE_LINUX_WORK_IRQ_FOR_IKC: ${ENABLE_LINUX_WORK_IRQ_FOR_IKC}")
+message("ENABLE_PER_CPU_ALLOC_CACHE: ${ENABLE_PER_CPU_ALLOC_CACHE}")
 message("-------------------------------")
diff --git a/config.h.in b/config.h.in
index 1df2263b..f0323e1a 100644
--- a/config.h.in
+++ b/config.h.in
@@ -28,6 +28,9 @@
 /* whether undefined behaviour sanitizer is enabled */
 #cmakedefine ENABLE_UBSAN 1
 
+/* whether per-CPU allocator cache (ThunderX2 workaround) is enabled */
+#cmakedefine ENABLE_PER_CPU_ALLOC_CACHE 1
+
 /* Path of bind-mount source directory */
 #cmakedefine ROOTFSDIR "${ROOTFSDIR}"
 
diff --git a/kernel/cls.c b/kernel/cls.c
index 0d09d076..6d83d93f 100644
--- a/kernel/cls.c
+++ b/kernel/cls.c
@@ -42,7 +42,9 @@ void cpu_local_var_init(void)
 		clv[i].monitor = monitor->cpu + i;
 		clv[i].rusage = rusage.cpu + i;
 		INIT_LIST_HEAD(&clv[i].smp_func_req_list);
+#ifdef ENABLE_PER_CPU_ALLOC_CACHE
 		clv[i].free_chunks.rb_node = NULL;
+#endif
 	}
 
 	cpu_local_var_initialized = 1;
diff --git a/kernel/include/cls.h b/kernel/include/cls.h
index d1bbc252..554e33df 100644
--- a/kernel/include/cls.h
+++ b/kernel/include/cls.h
@@ -15,6 +15,7 @@
 
 #include <process.h>
 #include <syscall.h>
+#include <config.h>
 /*
  * CPU Local Storage (cls)
  */
@@ -103,8 +104,10 @@ struct cpu_local_var {
 
 	/* UTI */
 	void *uti_futex_resp;
+#ifdef ENABLE_PER_CPU_ALLOC_CACHE
 	/* Per-CPU memory allocator cache */
 	struct rb_root free_chunks;
+#endif
 } __attribute__((aligned(64)));
 
 extern int cpu_local_var_initialized;
diff --git a/lib/page_alloc.c b/lib/page_alloc.c
index f0fc5340..79f2360f 100644
--- a/lib/page_alloc.c
+++ b/lib/page_alloc.c
@@ -609,6 +609,7 @@ unsigned long ihk_numa_alloc_pages(struct ihk_mc_numa_node *node,
 	unsigned long addr = 0;
 	mcs_lock_node_t mcs_node;
 
+#ifdef ENABLE_PER_CPU_ALLOC_CACHE
 	/* Check CPU local cache first */
 	if (cpu_local_var_initialized) {
 		unsigned long irqflags;
@@ -624,6 +625,7 @@ unsigned long ihk_numa_alloc_pages(struct ihk_mc_numa_node *node,
 			return addr;
 		}
 	}
+#endif
 
 	mcs_lock_lock(&node->lock, &mcs_node);
 
@@ -652,6 +654,7 @@ void ihk_numa_free_pages(struct ihk_mc_numa_node *node,
 {
 	mcs_lock_node_t mcs_node;
 
+#ifdef ENABLE_PER_CPU_ALLOC_CACHE
 	/* CPU local cache */
 	if (cpu_local_var_initialized) {
 		unsigned long irqflags;
@@ -670,6 +673,7 @@ void ihk_numa_free_pages(struct ihk_mc_numa_node *node,
 			return;
 		}
 	}
+#endif
 
 	if (addr < node->min_addr ||
 			(addr + (npages << PAGE_SHIFT)) > node->max_addr) {