[PATCH 2/3] x86/speculation: Support Automatic IBRS
From: Kim Phillips
Date: Fri Nov 04 2022 - 17:37:50 EST
The AMD Zen4 core supports a new feature called Automatic IBRS.
It is a "set-and-forget" feature that means that, unlike e.g.,
s/w-toggled SPEC_CTRL.IBRS, h/w manages its IBRS mitigation
resources automatically across CPL transitions.
The feature is advertised by CPUID_Fn80000021_EAX bit 8 and is
enabled by setting MSR C000_0080 (EFER) bit 21.
Enable Automatic IBRS by default if the CPU feature is present.
It typically provides greater performance over the incumbent
generic retpolines mitigation. In addition:
- Don't clear the RSB on VMEXIT when AutoIBRS is enabled:
The internal return address stack used for return address
predictions is automatically cleared on VMEXIT.
- Automatic IBRS removes the need for toggling IBRS during
firmware switches, so don't enable IBRS_FW when Automatic
IBRS is enabled.
- Allow for spectre_v2=autoibrs in the kernel command line,
reverting to auto-selection if the feature isn't available.
Signed-off-by: Kim Phillips <kim.phillips@xxxxxxx>
---
.../admin-guide/kernel-parameters.txt | 1 +
arch/x86/include/asm/msr-index.h | 2 ++
arch/x86/include/asm/nospec-branch.h | 1 +
arch/x86/kernel/cpu/bugs.c | 34 +++++++++++++++++--
4 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index a465d5242774..5ac4422e16a6 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -5702,6 +5702,7 @@
eibrs,retpoline - enhanced IBRS + Retpolines
eibrs,lfence - enhanced IBRS + LFENCE
ibrs - use IBRS to protect kernel
+ autoibrs - AMD Automatic IBRS
Not specifying this option is equivalent to
spectre_v2=auto.
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 10ac52705892..bd73e509cfa6 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -30,6 +30,7 @@
#define _EFER_SVME 12 /* Enable virtualization */
#define _EFER_LMSLE 13 /* Long Mode Segment Limit Enable */
#define _EFER_FFXSR 14 /* Enable Fast FXSAVE/FXRSTOR */
+#define _EFER_AUTOIBRS 21 /* Automatic IBRS Enable */
#define EFER_SCE (1<<_EFER_SCE)
#define EFER_LME (1<<_EFER_LME)
@@ -38,6 +39,7 @@
#define EFER_SVME (1<<_EFER_SVME)
#define EFER_LMSLE (1<<_EFER_LMSLE)
#define EFER_FFXSR (1<<_EFER_FFXSR)
+#define EFER_AUTOIBRS (1<<_EFER_AUTOIBRS)
/* Intel MSRs. Some also available on other CPUs */
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 82580adbca4b..12b2b070caab 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -442,6 +442,7 @@ enum spectre_v2_mitigation {
SPECTRE_V2_EIBRS_RETPOLINE,
SPECTRE_V2_EIBRS_LFENCE,
SPECTRE_V2_IBRS,
+ SPECTRE_V2_AUTO_IBRS,
};
/* The indirect branch speculation control variants */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 66d7addf1784..31e5af78baa0 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -1050,6 +1050,7 @@ enum spectre_v2_mitigation_cmd {
SPECTRE_V2_CMD_EIBRS_RETPOLINE,
SPECTRE_V2_CMD_EIBRS_LFENCE,
SPECTRE_V2_CMD_IBRS,
+ SPECTRE_V2_CMD_AUTOIBRS,
};
enum spectre_v2_user_cmd {
@@ -1124,6 +1125,7 @@ spectre_v2_parse_user_cmdline(void)
return SPECTRE_V2_USER_CMD_AUTO;
}
+/* Checks for Intel IBRS versions */
static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode)
{
return mode == SPECTRE_V2_IBRS ||
@@ -1233,6 +1235,7 @@ static const char * const spectre_v2_strings[] = {
[SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE",
[SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines",
[SPECTRE_V2_IBRS] = "Mitigation: IBRS",
+ [SPECTRE_V2_AUTO_IBRS] = "Mitigation: Automatic IBRS",
};
static const struct {
@@ -1249,6 +1252,7 @@ static const struct {
{ "eibrs", SPECTRE_V2_CMD_EIBRS, false },
{ "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false },
{ "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false },
+ { "autoibrs", SPECTRE_V2_CMD_AUTOIBRS, false },
{ "auto", SPECTRE_V2_CMD_AUTO, false },
{ "ibrs", SPECTRE_V2_CMD_IBRS, false },
};
@@ -1305,6 +1309,13 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
return SPECTRE_V2_CMD_AUTO;
}
+ if (cmd == SPECTRE_V2_CMD_AUTOIBRS &&
+ !boot_cpu_has(X86_FEATURE_AUTOIBRS)) {
+ pr_err("%s selected but CPU doesn't have AMD Automatic IBRS. Switching to AUTO select\n",
+ mitigation_options[i].option);
+ return SPECTRE_V2_CMD_AUTO;
+ }
+
if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
@@ -1392,6 +1403,7 @@ static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_
*/
switch (mode) {
case SPECTRE_V2_NONE:
+ case SPECTRE_V2_AUTO_IBRS:
return;
case SPECTRE_V2_EIBRS_LFENCE:
@@ -1419,6 +1431,7 @@ static void __init spectre_v2_select_mitigation(void)
{
enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
+ uint64_t efer;
/*
* If the CPU is not affected and the command line mode is NONE or AUTO
@@ -1434,6 +1447,11 @@ static void __init spectre_v2_select_mitigation(void)
case SPECTRE_V2_CMD_FORCE:
case SPECTRE_V2_CMD_AUTO:
+ if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) {
+ mode = SPECTRE_V2_AUTO_IBRS;
+ break;
+ }
+
if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
mode = SPECTRE_V2_EIBRS;
break;
@@ -1480,6 +1498,10 @@ static void __init spectre_v2_select_mitigation(void)
case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
mode = SPECTRE_V2_EIBRS_RETPOLINE;
break;
+
+ case SPECTRE_V2_CMD_AUTOIBRS:
+ mode = SPECTRE_V2_AUTO_IBRS;
+ break;
}
if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
@@ -1495,6 +1517,11 @@ static void __init spectre_v2_select_mitigation(void)
case SPECTRE_V2_EIBRS:
break;
+ case SPECTRE_V2_AUTO_IBRS:
+ rdmsrl(MSR_EFER, efer);
+ wrmsrl(MSR_EFER, efer | EFER_AUTOIBRS);
+ break;
+
case SPECTRE_V2_IBRS:
setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS);
if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED))
@@ -1571,8 +1598,8 @@ static void __init spectre_v2_select_mitigation(void)
/*
* Retpoline protects the kernel, but doesn't protect firmware. IBRS
* and Enhanced IBRS protect firmware too, so enable IBRS around
- * firmware calls only when IBRS / Enhanced IBRS aren't otherwise
- * enabled.
+ * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't
+ * otherwise enabled.
*
* Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
* the user might select retpoline on the kernel command line and if
@@ -1589,7 +1616,8 @@ static void __init spectre_v2_select_mitigation(void)
pr_info("Enabling Speculation Barrier for firmware calls\n");
}
- } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) {
+ } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode) &&
+ mode != SPECTRE_V2_AUTO_IBRS) {
setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
pr_info("Enabling Restricted Speculation for firmware calls\n");
}
--
2.34.1