From e056cb799f24b3f935ccd56edfce0d55fcb667a6 Mon Sep 17 00:00:00 2001
From: Balazs Gerofi <bgerofi@riken.jp>
Date: Wed, 27 May 2020 16:15:55 +0900
Subject: [PATCH] memclear: non-temporal memory clean (arm64)

Change-Id: I8f80ff20e98bc01088450282e1790c27c67c16eb
---
 arch/arm64/kernel/include/arch-string.h |  9 ++++++
 arch/arm64/kernel/memset.S              | 38 +++++++++++++++++++++++++
 kernel/process.c                        | 15 ++++------
 3 files changed, 53 insertions(+), 9 deletions(-)

diff --git a/arch/arm64/kernel/include/arch-string.h b/arch/arm64/kernel/include/arch-string.h
index b484bcc5..007a4d8f 100644
--- a/arch/arm64/kernel/include/arch-string.h
+++ b/arch/arm64/kernel/include/arch-string.h
@@ -10,4 +10,13 @@ extern void *__inline_memcpy(void *to, const void *from, size_t t);
 
 extern void *__inline_memset(void *s, unsigned long c, size_t count);
 
+#define ARCH_MEMCLEAR
+
+extern void __memclear(void *addr, unsigned long len, void *tmp);
+inline static void memclear(void *addr, unsigned long len)
+{
+	uint64_t q0q1[4];
+	__memclear(addr, len, (void *)&q0q1);
+}
+
 #endif	/* __HEADER_ARM64_COMMON_ARCH_TIMER_H */
diff --git a/arch/arm64/kernel/memset.S b/arch/arm64/kernel/memset.S
index 5785ab14..32255e19 100644
--- a/arch/arm64/kernel/memset.S
+++ b/arch/arm64/kernel/memset.S
@@ -218,3 +218,41 @@ ENTRY(__inline_memset)
 	ret
 ENDPIPROC(__inline_memset)
 ENDPROC(____inline_memset)
+
+
+/*
+ * Non-temporal vector memory clear
+ *
+ * Parameters:
+ *     x0 - buf (assumed to be aligned to page size)
+ *     x1 - n (assumed to be at least page size)
+ */
+ENTRY(__memclear)
+       stp     q0, q1, [x2] /* Preserve two 128 bit vector regs */
+       eor v0.16B, v0.16B, v0.16B
+       eor v1.16B, v1.16B, v1.16B
+1:
+       stnp q0, q1, [x0, #32 * 0]
+       stnp q0, q1, [x0, #32 * 1]
+       stnp q0, q1, [x0, #32 * 2]
+       stnp q0, q1, [x0, #32 * 3]
+       stnp q0, q1, [x0, #32 * 4]
+       stnp q0, q1, [x0, #32 * 5]
+       stnp q0, q1, [x0, #32 * 6]
+       stnp q0, q1, [x0, #32 * 7]
+       stnp q0, q1, [x0, #32 * 8]
+       stnp q0, q1, [x0, #32 * 9]
+       stnp q0, q1, [x0, #32 * 10]
+       stnp q0, q1, [x0, #32 * 11]
+       stnp q0, q1, [x0, #32 * 12]
+       stnp q0, q1, [x0, #32 * 13]
+       stnp q0, q1, [x0, #32 * 14]
+       stnp q0, q1, [x0, #32 * 15]
+       add x0, x0, #512
+       subs x1, x1, #512
+       cmp x1, #0
+       b.ne    1b
+
+       ldp     q0, q1, [x2] /* Restore vector regs */
+       ret
+ENDPROC(__memclear)
diff --git a/kernel/process.c b/kernel/process.c
index 02458678..78b36c9b 100644
--- a/kernel/process.c
+++ b/kernel/process.c
@@ -1466,17 +1466,14 @@ int add_process_memory_range(struct process_vm *vm,
 	/* Clear content! */
 	if (phys != NOPHYS && !(flag & (VR_REMOTE | VR_DEMAND_PAGING))
 			&& ((flag & VR_PROT_MASK) != VR_PROT_NONE)) {
-#if 1
-		memset((void *)phys_to_virt(phys), 0, end - start);
+
+		if (!zero_at_free) {
+#ifdef ARCH_MEMCLEAR
+			memclear((void *)phys_to_virt(phys), end - start);
 #else
-		if (end - start < (1024*1024)) {
-			memset((void*)phys_to_virt(phys), 0, end - start);
-		}
-		else {
-			memset_smp(&cpu_local_var(current)->cpu_set,
-					(void *)phys_to_virt(phys), 0, end - start);
-		}
+			memset((void *)phys_to_virt(phys), 0, end - start);
 #endif
+		}
 	}
 
 	/* Return range object if requested */