[PATCH v2 06/23] x86,entry_32: Remove .fixup usage

From: Peter Zijlstra
Date: Wed Nov 10 2021 - 05:16:39 EST


Where possible, push the .fixup into code, at the tail of functions.

This is hard for macros since they're used in multiple functions,
therefore introduce a new extable handler to pop zeros.

Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
---
arch/x86/entry/entry_32.S | 28 ++++++++--------------------
arch/x86/include/asm/extable_fixup_types.h | 2 ++
arch/x86/mm/extable.c | 14 ++++++++++++++
3 files changed, 24 insertions(+), 20 deletions(-)

--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -270,17 +270,9 @@
3: popl %fs
addl $(4 + \pop), %esp /* pop the unused "gs" slot */
IRET_FRAME
-.pushsection .fixup, "ax"
-4: movl $0, (%esp)
- jmp 1b
-5: movl $0, (%esp)
- jmp 2b
-6: movl $0, (%esp)
- jmp 3b
-.popsection
- _ASM_EXTABLE(1b, 4b)
- _ASM_EXTABLE(2b, 5b)
- _ASM_EXTABLE(3b, 6b)
+ _ASM_EXTABLE_TYPE(1b, 1b, EX_TYPE_POP_ZERO)
+ _ASM_EXTABLE_TYPE(2b, 2b, EX_TYPE_POP_ZERO)
+ _ASM_EXTABLE_TYPE(3b, 3b, EX_TYPE_POP_ZERO)
.endm

.macro RESTORE_ALL_NMI cr3_reg:req pop=0
@@ -925,10 +917,8 @@ SYM_FUNC_START(entry_SYSENTER_32)
sti
sysexit

-.pushsection .fixup, "ax"
-2: movl $0, PT_FS(%esp)
- jmp 1b
-.popsection
+2: movl $0, PT_FS(%esp)
+ jmp 1b
_ASM_EXTABLE(1b, 2b)

.Lsysenter_fix_flags:
@@ -996,8 +986,7 @@ SYM_FUNC_START(entry_INT80_32)
*/
iret

-.section .fixup, "ax"
-SYM_CODE_START(asm_iret_error)
+.Lasm_iret_error:
pushl $0 # no error code
pushl $iret_error

@@ -1014,9 +1003,8 @@ SYM_CODE_START(asm_iret_error)
#endif

jmp handle_exception
-SYM_CODE_END(asm_iret_error)
-.previous
- _ASM_EXTABLE(.Lirq_return, asm_iret_error)
+
+ _ASM_EXTABLE(.Lirq_return, .Lasm_iret_error)
SYM_FUNC_END(entry_INT80_32)

.macro FIXUP_ESPFIX_STACK
--- a/arch/x86/include/asm/extable_fixup_types.h
+++ b/arch/x86/include/asm/extable_fixup_types.h
@@ -19,4 +19,6 @@
#define EX_TYPE_DEFAULT_MCE_SAFE 12
#define EX_TYPE_FAULT_MCE_SAFE 13

+#define EX_TYPE_POP_ZERO 14
+
#endif
--- a/arch/x86/mm/extable.c
+++ b/arch/x86/mm/extable.c
@@ -99,6 +99,18 @@ static bool ex_handler_clear_fs(const st
return ex_handler_default(fixup, regs);
}

+static bool ex_handler_pop_zero(const struct exception_table_entry *fixup,
+ struct pt_regs *regs)
+{
+ /*
+ * Typically used for when "pop %seg" traps, in which case we'll clear
+ * the stack slot and re-try the instruction, which will then succeed
+ * to pop zero.
+ */
+ *((unsigned long *)regs->sp) = 0;
+ return ex_handler_default(fixup, regs);
+}
+
int ex_get_fixup_type(unsigned long ip)
{
const struct exception_table_entry *e = search_exception_tables(ip);
@@ -156,6 +168,8 @@ int fixup_exception(struct pt_regs *regs
case EX_TYPE_WRMSR_IN_MCE:
ex_handler_msr_mce(regs, true);
break;
+ case EX_TYPE_POP_ZERO:
+ return ex_handler_pop_zero(e, regs);
}
BUG();
}