[PATCH v3 4/6] x86/gsseg: move local_irq_save/restore() into asm_load_gs_index()

From: Xin Li
Date: Thu Oct 13 2022 - 16:24:02 EST


From: "H. Peter Anvin (Intel)" <hpa@xxxxxxxxx>

The need to disable/enable interrupts around asm_load_gs_index() is a
consequence of the implementation. Prepare for using the LKGS
instruction, which is locally atomic and therefore doesn't need
interrupts disabled.

Signed-off-by: H. Peter Anvin (Intel) <hpa@xxxxxxxxx>
Signed-off-by: Xin Li <xin3.li@xxxxxxxxx>
---
arch/x86/entry/entry_64.S | 26 +++++++++++++++++++++-----
arch/x86/include/asm/special_insns.h | 4 ----
2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index e0c48998d2fb..cc6ba6672418 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -778,19 +778,35 @@ SYM_CODE_END(common_interrupt_return)
_ASM_NOKPROBE(common_interrupt_return)

/*
- * Reload gs selector with exception handling
+ * Reload gs selector with exception handling. This is used only on
+ * native, so using swapgs, pushf, popf, cli, sti, ... directly is fine.
+ *
* di: new selector
+ * rax: scratch register
*
* Is in entry.text as it shouldn't be instrumented.
+ *
+ * Note: popf is slow, so use pushf to read IF and then execute cli/sti
+ * if necessary.
*/
SYM_FUNC_START(asm_load_gs_index)
FRAME_BEGIN
+ pushf
+ pop %rax
+ andl $X86_EFLAGS_IF, %eax /* Interrupts enabled? */
+ jz 1f
+ cli
+1:
swapgs
.Lgs_change:
ANNOTATE_NOENDBR // error_entry
movl %edi, %gs
2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
swapgs
+ testl %eax, %eax
+ jz 3f
+ sti
+3:
FRAME_END
RET

@@ -799,12 +815,12 @@ SYM_FUNC_START(asm_load_gs_index)
swapgs /* switch back to user gs */
.macro ZAP_GS
/* This can't be a string because the preprocessor needs to see it. */
- movl $__USER_DS, %eax
- movl %eax, %gs
+ movl $__USER_DS, %edi
+ movl %edi, %gs
.endm
ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
- xorl %eax, %eax
- movl %eax, %gs
+ xorl %edi, %edi
+ movl %edi, %gs
jmp 2b

_ASM_EXTABLE(.Lgs_change, .Lbad_gs)
diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h
index a71d0e8d4684..6de00dec6564 100644
--- a/arch/x86/include/asm/special_insns.h
+++ b/arch/x86/include/asm/special_insns.h
@@ -124,11 +124,7 @@ extern asmlinkage void asm_load_gs_index(u16 selector);

static inline void native_load_gs_index(unsigned int selector)
{
- unsigned long flags;
-
- local_irq_save(flags);
asm_load_gs_index(selector);
- local_irq_restore(flags);
}

static inline unsigned long __read_cr4(void)
--
2.34.1