104 lines
3.6 KiB
Diff
104 lines
3.6 KiB
Diff
From e5d151337c384934c9b669967d72f9b29781b308 Mon Sep 17 00:00:00 2001
|
|
From: Rik van Riel <riel@surriel.com>
|
|
Date: Tue, 25 Feb 2025 22:00:37 -0500
|
|
Subject: x86/mm: Add INVLPGB feature and Kconfig entry
|
|
|
|
In addition, the CPU advertises the maximum number of pages that can be
|
|
shot down with one INVLPGB instruction in CPUID. Save that information
|
|
for later use.
|
|
|
|
[ bp: use cpu_has(), typos, massage. ]
|
|
|
|
Signed-off-by: Rik van Riel <riel@surriel.com>
|
|
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
|
|
Link: https://lore.kernel.org/r/20250226030129.530345-3-riel@surriel.com
|
|
---
|
|
arch/x86/Kconfig.cpu | 4 ++++
|
|
arch/x86/include/asm/cpufeatures.h | 1 +
|
|
arch/x86/include/asm/disabled-features.h | 8 +++++++-
|
|
arch/x86/include/asm/tlbflush.h | 3 +++
|
|
arch/x86/kernel/cpu/amd.c | 6 ++++++
|
|
5 files changed, 21 insertions(+), 1 deletion(-)
|
|
|
|
--- a/arch/x86/Kconfig.cpu
|
|
+++ b/arch/x86/Kconfig.cpu
|
|
@@ -740,6 +740,10 @@ menuconfig PROCESSOR_SELECT
|
|
This lets you choose what x86 vendor support code your kernel
|
|
will include.
|
|
|
|
+config BROADCAST_TLB_FLUSH
|
|
+ def_bool y
|
|
+ depends on CPU_SUP_AMD && 64BIT
|
|
+
|
|
config CPU_SUP_INTEL
|
|
default y
|
|
bool "Support Intel processors" if PROCESSOR_SELECT
|
|
--- a/arch/x86/include/asm/cpufeatures.h
|
|
+++ b/arch/x86/include/asm/cpufeatures.h
|
|
@@ -338,6 +338,7 @@
|
|
#define X86_FEATURE_CLZERO (13*32+ 0) /* "clzero" CLZERO instruction */
|
|
#define X86_FEATURE_IRPERF (13*32+ 1) /* "irperf" Instructions Retired Count */
|
|
#define X86_FEATURE_XSAVEERPTR (13*32+ 2) /* "xsaveerptr" Always save/restore FP error pointers */
|
|
+#define X86_FEATURE_INVLPGB (13*32+ 3) /* INVLPGB and TLBSYNC instructions supported */
|
|
#define X86_FEATURE_RDPRU (13*32+ 4) /* "rdpru" Read processor register at user level */
|
|
#define X86_FEATURE_WBNOINVD (13*32+ 9) /* "wbnoinvd" WBNOINVD instruction */
|
|
#define X86_FEATURE_AMD_IBPB (13*32+12) /* Indirect Branch Prediction Barrier */
|
|
--- a/arch/x86/include/asm/disabled-features.h
|
|
+++ b/arch/x86/include/asm/disabled-features.h
|
|
@@ -129,6 +129,12 @@
|
|
#define DISABLE_SEV_SNP (1 << (X86_FEATURE_SEV_SNP & 31))
|
|
#endif
|
|
|
|
+#ifdef CONFIG_BROADCAST_TLB_FLUSH
|
|
+#define DISABLE_INVLPGB 0
|
|
+#else
|
|
+#define DISABLE_INVLPGB (1 << (X86_FEATURE_INVLPGB & 31))
|
|
+#endif
|
|
+
|
|
/*
|
|
* Make sure to add features to the correct mask
|
|
*/
|
|
@@ -146,7 +152,7 @@
|
|
#define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \
|
|
DISABLE_CALL_DEPTH_TRACKING|DISABLE_USER_SHSTK)
|
|
#define DISABLED_MASK12 (DISABLE_FRED|DISABLE_LAM)
|
|
-#define DISABLED_MASK13 0
|
|
+#define DISABLED_MASK13 (DISABLE_INVLPGB)
|
|
#define DISABLED_MASK14 0
|
|
#define DISABLED_MASK15 0
|
|
#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \
|
|
--- a/arch/x86/include/asm/tlbflush.h
|
|
+++ b/arch/x86/include/asm/tlbflush.h
|
|
@@ -183,6 +183,9 @@ static inline void cr4_init_shadow(void)
|
|
extern unsigned long mmu_cr4_features;
|
|
extern u32 *trampoline_cr4_features;
|
|
|
|
+/* How many pages can be invalidated with one INVLPGB. */
|
|
+extern u16 invlpgb_count_max;
|
|
+
|
|
extern void initialize_tlbstate_and_flush(void);
|
|
|
|
/*
|
|
--- a/arch/x86/kernel/cpu/amd.c
|
|
+++ b/arch/x86/kernel/cpu/amd.c
|
|
@@ -29,6 +29,8 @@
|
|
|
|
#include "cpu.h"
|
|
|
|
+u16 invlpgb_count_max __ro_after_init;
|
|
+
|
|
static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
|
|
{
|
|
u32 gprs[8] = { 0 };
|
|
@@ -1145,6 +1147,10 @@ static void cpu_detect_tlb_amd(struct cp
|
|
tlb_lli_2m[ENTRIES] = eax & mask;
|
|
|
|
tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
|
|
+
|
|
+ /* Max number of pages INVLPGB can invalidate in one shot */
|
|
+ if (cpu_has(c, X86_FEATURE_INVLPGB))
|
|
+ invlpgb_count_max = (cpuid_edx(0x80000008) & 0xffff) + 1;
|
|
}
|
|
|
|
static const struct cpu_dev amd_cpu_dev = {
|