122 lines
3.7 KiB
Diff
122 lines
3.7 KiB
Diff
From bc9d1fa1bd32dca78f38bd2a8557e7fc638308bd Mon Sep 17 00:00:00 2001
|
|
From: Rik van Riel <riel@surriel.com>
|
|
Date: Mon, 30 Dec 2024 12:53:06 -0500
|
|
Subject: x86/mm: add INVLPGB support code
|
|
|
|
Add invlpgb.h with the helper functions and definitions needed to use
|
|
broadcast TLB invalidation on AMD EPYC 3 and newer CPUs.
|
|
|
|
Signed-off-by: Rik van Riel <riel@surriel.com>
|
|
---
|
|
arch/x86/include/asm/invlpgb.h | 93 +++++++++++++++++++++++++++++++++
|
|
arch/x86/include/asm/tlbflush.h | 1 +
|
|
2 files changed, 94 insertions(+)
|
|
create mode 100644 arch/x86/include/asm/invlpgb.h
|
|
|
|
--- /dev/null
|
|
+++ b/arch/x86/include/asm/invlpgb.h
|
|
@@ -0,0 +1,93 @@
|
|
+/* SPDX-License-Identifier: GPL-2.0 */
|
|
+#ifndef _ASM_X86_INVLPGB
|
|
+#define _ASM_X86_INVLPGB
|
|
+
|
|
+#include <vdso/bits.h>
|
|
+
|
|
+/*
|
|
+ * INVLPGB does broadcast TLB invalidation across all the CPUs in the system.
|
|
+ *
|
|
+ * The INVLPGB instruction is weakly ordered, and a batch of invalidations can
|
|
+ * be done in a parallel fashion.
|
|
+ *
|
|
+ * TLBSYNC is used to ensure that pending INVLPGB invalidations initiated from
|
|
+ * this CPU have completed.
|
|
+ */
|
|
+static inline void __invlpgb(unsigned long asid, unsigned long pcid, unsigned long addr,
|
|
+ int extra_count, bool pmd_stride, unsigned long flags)
|
|
+{
|
|
+ u64 rax = addr | flags;
|
|
+ u32 ecx = (pmd_stride << 31) | extra_count;
|
|
+ u32 edx = (pcid << 16) | asid;
|
|
+
|
|
+ asm volatile("invlpgb" : : "a" (rax), "c" (ecx), "d" (edx));
|
|
+}
|
|
+
|
|
+/*
|
|
+ * INVLPGB can be targeted by virtual address, PCID, ASID, or any combination
|
|
+ * of the three. For example:
|
|
+ * - INVLPGB_VA | INVLPGB_INCLUDE_GLOBAL: invalidate all TLB entries at the address
|
|
+ * - INVLPGB_PCID: invalidate all TLB entries matching the PCID
|
|
+ *
|
|
+ * The first can be used to invalidate (kernel) mappings at a particular
|
|
+ * address across all processes.
|
|
+ *
|
|
+ * The latter invalidates all TLB entries matching a PCID.
|
|
+ */
|
|
+#define INVLPGB_VA BIT(0)
|
|
+#define INVLPGB_PCID BIT(1)
|
|
+#define INVLPGB_ASID BIT(2)
|
|
+#define INVLPGB_INCLUDE_GLOBAL BIT(3)
|
|
+#define INVLPGB_FINAL_ONLY BIT(4)
|
|
+#define INVLPGB_INCLUDE_NESTED BIT(5)
|
|
+
|
|
+/* Flush all mappings for a given pcid and addr, not including globals. */
|
|
+static inline void invlpgb_flush_user(unsigned long pcid,
|
|
+ unsigned long addr)
|
|
+{
|
|
+ __invlpgb(0, pcid, addr, 0, 0, INVLPGB_PCID | INVLPGB_VA);
|
|
+}
|
|
+
|
|
+static inline void invlpgb_flush_user_nr(unsigned long pcid, unsigned long addr,
|
|
+ int nr, bool pmd_stride)
|
|
+{
|
|
+ __invlpgb(0, pcid, addr, nr - 1, pmd_stride, INVLPGB_PCID | INVLPGB_VA);
|
|
+}
|
|
+
|
|
+/* Flush all mappings for a given ASID, not including globals. */
|
|
+static inline void invlpgb_flush_single_asid(unsigned long asid)
|
|
+{
|
|
+ __invlpgb(asid, 0, 0, 0, 0, INVLPGB_ASID);
|
|
+}
|
|
+
|
|
+/* Flush all mappings for a given PCID, not including globals. */
|
|
+static inline void invlpgb_flush_single_pcid(unsigned long pcid)
|
|
+{
|
|
+ __invlpgb(0, pcid, 0, 0, 0, INVLPGB_PCID);
|
|
+}
|
|
+
|
|
+/* Flush all mappings, including globals, for all PCIDs. */
|
|
+static inline void invlpgb_flush_all(void)
|
|
+{
|
|
+ __invlpgb(0, 0, 0, 0, 0, INVLPGB_INCLUDE_GLOBAL);
|
|
+}
|
|
+
|
|
+/* Flush addr, including globals, for all PCIDs. */
|
|
+static inline void invlpgb_flush_addr(unsigned long addr, int nr)
|
|
+{
|
|
+ __invlpgb(0, 0, addr, nr - 1, 0, INVLPGB_INCLUDE_GLOBAL);
|
|
+}
|
|
+
|
|
+/* Flush all mappings for all PCIDs except globals. */
|
|
+static inline void invlpgb_flush_all_nonglobals(void)
|
|
+{
|
|
+ __invlpgb(0, 0, 0, 0, 0, 0);
|
|
+}
|
|
+
|
|
+/* Wait for INVLPGB originated by this CPU to complete. */
|
|
+static inline void tlbsync(void)
|
|
+{
|
|
+ asm volatile("tlbsync");
|
|
+}
|
|
+
|
|
+#endif /* _ASM_X86_INVLPGB */
|
|
--- a/arch/x86/include/asm/tlbflush.h
|
|
+++ b/arch/x86/include/asm/tlbflush.h
|
|
@@ -10,6 +10,7 @@
|
|
#include <asm/cpufeature.h>
|
|
#include <asm/special_insns.h>
|
|
#include <asm/smp.h>
|
|
+#include <asm/invlpgb.h>
|
|
#include <asm/invpcid.h>
|
|
#include <asm/pti.h>
|
|
#include <asm/processor-flags.h>
|