[PATCH] x86 interrupt stacks

From: Dave Hansen (haveblue@us.ibm.com)
Date: Tue Feb 04 2003 - 15:29:56 EST

This applies cleanly againt 2.5.59. It gets rid of the ugly multi-line
C macros in entry.S and cleans up the declarations of the interrupt
handler functions.

Something in my previous patches was causing faulures with gcc-3.2. I
suspect it was one of the functions using regparm(1) and asmlinkage
(which implies regparm(0)) at the same time.

Tested:
UP gcc-3.2
SMP gcc-3.2 and 2.95.4 

-- 
Dave Hansen
haveblue@us.ibm.com

diff -ur linux-2.5.59-thread_info-infra/arch/i386/Kconfig linux-2.5.59-irqstack/arch/i386/Kconfig --- linux-2.5.59-thread_info-infra/arch/i386/Kconfig Tue Feb 4 10:32:54 2003 +++ linux-2.5.59-irqstack/arch/i386/Kconfig Tue Feb 4 10:44:02 2003 @@ -368,6 +368,11 @@ depends on MK8 || MPENTIUM4 default y +config X86_CMOV + bool + depends on M686 || MPENTIUMII || MPENTIUMIII || MPENTIUM4 || MK8 || MCRUSOE + default y + config HUGETLB_PAGE bool "Huge TLB Page Support" help diff -ur linux-2.5.59-thread_info-infra/arch/i386/kernel/apic.c linux-2.5.59-irqstack/arch/i386/kernel/apic.c --- linux-2.5.59-thread_info-infra/arch/i386/kernel/apic.c Tue Feb 4 10:32:54 2003 +++ linux-2.5.59-irqstack/arch/i386/kernel/apic.c Tue Feb 4 11:37:21 2003 @@ -1038,7 +1038,8 @@ * interrupt as well. Thus we cannot inline the local irq ... ] */ -void smp_apic_timer_interrupt(struct pt_regs regs) +struct pt_regs * IRQHANDLER(smp_apic_timer_interrupt(struct pt_regs* regs)); +struct pt_regs * smp_apic_timer_interrupt(struct pt_regs* regs) { int cpu = smp_processor_id(); @@ -1058,14 +1059,16 @@ * interrupt lock, which is the WrongThing (tm) to do. */ irq_enter(); - smp_local_timer_interrupt(&regs); + smp_local_timer_interrupt(regs); irq_exit(); + return regs; } /* * This interrupt should _never_ happen with our APIC/SMP architecture */ -asmlinkage void smp_spurious_interrupt(void) +struct pt_regs * IRQHANDLER(smp_spurious_interrupt(struct pt_regs* regs)); +struct pt_regs * smp_spurious_interrupt(struct pt_regs* regs) { unsigned long v; @@ -1083,13 +1086,15 @@ printk(KERN_INFO "spurious APIC interrupt on CPU#%d, should never happen.\n", smp_processor_id()); irq_exit(); + return regs; } /* * This interrupt should never happen with our APIC/SMP architecture */ -asmlinkage void smp_error_interrupt(void) +struct pt_regs * IRQHANDLER(smp_error_interrupt(struct pt_regs* regs)); +struct pt_regs * smp_error_interrupt(struct pt_regs* regs) { unsigned long v, v1; @@ -1114,6 +1119,7 @@ printk (KERN_INFO "APIC error on CPU%d: %02lx(%02lx)\n", smp_processor_id(), v , v1); irq_exit(); + return regs; } /* diff -ur linux-2.5.59-thread_info-infra/arch/i386/kernel/cpu/mcheck/p4.c linux-2.5.59-irqstack/arch/i386/kernel/cpu/mcheck/p4.c --- linux-2.5.59-thread_info-infra/arch/i386/kernel/cpu/mcheck/p4.c Tue Feb 4 10:32:54 2003 +++ linux-2.5.59-irqstack/arch/i386/kernel/cpu/mcheck/p4.c Tue Feb 4 11:37:26 2003 @@ -61,11 +61,13 @@ /* Thermal interrupt handler for this CPU setup */ static void (*vendor_thermal_interrupt)(struct pt_regs *regs) = unexpected_thermal_interrupt; -asmlinkage void smp_thermal_interrupt(struct pt_regs regs) +struct pt_regs * IRQHANDLER(smp_thermal_interrupt(struct pt_regs* regs)); +struct pt_regs * smp_thermal_interrupt(struct pt_regs* regs) { irq_enter(); vendor_thermal_interrupt(&regs); irq_exit(); + return regs; } /* P4/Xeon Thermal regulation detect and init */ diff -ur linux-2.5.59-thread_info-infra/arch/i386/kernel/entry.S linux-2.5.59-irqstack/arch/i386/kernel/entry.S --- linux-2.5.59-thread_info-infra/arch/i386/kernel/entry.S Tue Feb 4 12:09:05 2003 +++ linux-2.5.59-irqstack/arch/i386/kernel/entry.S Tue Feb 4 12:13:20 2003 @@ -388,17 +388,78 @@ vector=vector+1 .endr + +# lets play optimizing compiler... +#ifdef CONFIG_X86_CMOV +#define COND_MOVE cmovnz %esi,%esp; +#else +#define COND_MOVE \ + jz 1f; \ + mov %esi,%esp; \ +1: +#endif + +# These macros will switch you to, and from a per-cpu interrupt stack +# They take the pt_regs arg and move it from the normal place on the +# stack to %eax. Any handler function can retrieve it using regparm(1). +# The handlers are expected to return the stack to switch back to in +# the same register. +# +# This means that the irq handlers need to return their arg +# +# SWITCH_TO_IRQSTACK clobbers %ebx, %ecx, %edx, %esi +# old stack gets put in %eax + +.macro SWITCH_TO_IRQSTACK + GET_THREAD_INFO(%ebx); + movl TI_IRQ_STACK(%ebx),%ecx; + movl TI_TASK(%ebx),%edx; + movl %esp,%eax; + + # %ecx+THREAD_SIZE is next stack -4 keeps us in the right one + leal (THREAD_SIZE-4)(%ecx),%esi; + + # is there a valid irq_stack? + testl %ecx,%ecx; + COND_MOVE; + + # update the task pointer in the irq stack + GET_THREAD_INFO(%esi); + movl %edx,TI_TASK(%esi); + + # update the preempt count in the irq stack + movl TI_PRE_COUNT(%ebx),%ecx; + movl %ecx,TI_PRE_COUNT(%esi); +.endm + +# copy flags from the irq stack back into the task's thread_info +# %esi is saved over the irq handler call and contains the irq stack's +# thread_info pointer +# %eax was returned from the handler, as described above +# %ebx contains the original thread_info pointer + +.macro RESTORE_FROM_IRQSTACK + movl %eax,%esp; + movl TI_FLAGS(%esi),%eax; + movl $0,TI_FLAGS(%esi); + LOCK orl %eax,TI_FLAGS(%ebx); +.endm + ALIGN common_interrupt: SAVE_ALL + SWITCH_TO_IRQSTACK call do_IRQ + RESTORE_FROM_IRQSTACK jmp ret_from_intr #define BUILD_INTERRUPT(name, nr) \ ENTRY(name) \ pushl $nr-256; \ SAVE_ALL \ - call smp_/**/name; \ + SWITCH_TO_IRQSTACK; \ + call smp_/**/name; \ + RESTORE_FROM_IRQSTACK; \ jmp ret_from_intr; /* The include is where all of the SMP etc. interrupts come from */ diff -ur linux-2.5.59-thread_info-infra/arch/i386/kernel/init_task.c linux-2.5.59-irqstack/arch/i386/kernel/init_task.c --- linux-2.5.59-thread_info-infra/arch/i386/kernel/init_task.c Tue Feb 4 10:32:54 2003 +++ linux-2.5.59-irqstack/arch/i386/kernel/init_task.c Tue Feb 4 10:44:02 2003 @@ -13,6 +13,10 @@ static struct signal_struct init_signals = INIT_SIGNALS(init_signals); struct mm_struct init_mm = INIT_MM(init_mm); +union thread_union init_irq_union + __attribute__((__section__(".data.init_task"))); + + /* * Initial thread structure. * diff -ur linux-2.5.59-thread_info-infra/arch/i386/kernel/irq.c linux-2.5.59-irqstack/arch/i386/kernel/irq.c --- linux-2.5.59-thread_info-infra/arch/i386/kernel/irq.c Tue Feb 4 10:32:54 2003 +++ linux-2.5.59-irqstack/arch/i386/kernel/irq.c Tue Feb 4 11:37:30 2003 @@ -311,7 +311,8 @@ * SMP cross-CPU interrupts have their own specific * handlers). */ -asmlinkage unsigned int do_IRQ(struct pt_regs regs) +struct pt_regs * IRQHANDLER(do_IRQ(struct pt_regs *regs)); +struct pt_regs * do_IRQ(struct pt_regs *regs) { /* * We ack quickly, we don't want the irq controller @@ -323,7 +324,7 @@ * 0 return value means that this irq is already being * handled by some other CPU. (or is disabled) */ - int irq = regs.orig_eax & 0xff; /* high bits used in ret_from_ code */ + int irq = regs->orig_eax & 0xff; /* high bits used in ret_from_ code */ int cpu = smp_processor_id(); irq_desc_t *desc = irq_desc + irq; struct irqaction * action; @@ -388,7 +389,7 @@ */ for (;;) { spin_unlock(&desc->lock); - handle_IRQ_event(irq, &regs, action); + handle_IRQ_event(irq, regs, action); spin_lock(&desc->lock); if (likely(!(desc->status & IRQ_PENDING))) @@ -407,7 +408,7 @@ irq_exit(); - return 1; + return regs; } /** diff -ur linux-2.5.59-thread_info-infra/arch/i386/kernel/process.c linux-2.5.59-irqstack/arch/i386/kernel/process.c --- linux-2.5.59-thread_info-infra/arch/i386/kernel/process.c Tue Feb 4 10:32:54 2003 +++ linux-2.5.59-irqstack/arch/i386/kernel/process.c Tue Feb 4 10:44:02 2003 @@ -432,6 +432,7 @@ /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */ + next_p->thread_info->irq_stack = prev_p->thread_info->irq_stack; unlazy_fpu(prev_p); /* diff -ur linux-2.5.59-thread_info-infra/arch/i386/kernel/smp.c linux-2.5.59-irqstack/arch/i386/kernel/smp.c --- linux-2.5.59-thread_info-infra/arch/i386/kernel/smp.c Tue Feb 4 10:32:54 2003 +++ linux-2.5.59-irqstack/arch/i386/kernel/smp.c Tue Feb 4 11:37:40 2003 @@ -305,7 +305,8 @@ * 2) Leave the mm if we are in the lazy tlb mode. */ -asmlinkage void smp_invalidate_interrupt (void) +struct pt_regs * IRQHANDLER(smp_invalidate_interrupt(struct pt_regs *regs)); +struct pt_regs * smp_invalidate_interrupt(struct pt_regs *regs) { unsigned long cpu; @@ -336,6 +337,7 @@ out: put_cpu_no_resched(); + return regs; } static void flush_tlb_others (unsigned long cpumask, struct mm_struct *mm, @@ -576,12 +578,15 @@ * all the work is done automatically when * we return from the interrupt. */ -asmlinkage void smp_reschedule_interrupt(void) +struct pt_regs * IRQHANDLER(smp_reschedule_interrupt(struct pt_regs *regs)); +struct pt_regs * smp_reschedule_interrupt(struct pt_regs *regs) { ack_APIC_irq(); + return regs; } -asmlinkage void smp_call_function_interrupt(void) +struct pt_regs * IRQHANDLER(smp_call_function_interrupt(struct pt_regs *regs)); +struct pt_regs * smp_call_function_interrupt(struct pt_regs *regs) { void (*func) (void *info) = call_data->func; void *info = call_data->info; @@ -605,5 +610,6 @@ mb(); atomic_inc(&call_data->finished); } + return regs; } diff -ur linux-2.5.59-thread_info-infra/arch/i386/kernel/smpboot.c linux-2.5.59-irqstack/arch/i386/kernel/smpboot.c --- linux-2.5.59-thread_info-infra/arch/i386/kernel/smpboot.c Tue Feb 4 10:32:54 2003 +++ linux-2.5.59-irqstack/arch/i386/kernel/smpboot.c Tue Feb 4 10:44:02 2003 @@ -71,6 +71,11 @@ /* Per CPU bogomips and other parameters */ struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned; +/* Per CPU interrupt stacks */ +extern union thread_union init_irq_union; +union thread_union *irq_stacks[NR_CPUS] __cacheline_aligned = + { &init_irq_union, }; + /* Set when the idlers are all forked */ int smp_threads_ready; @@ -772,6 +777,28 @@ } #endif /* WAKE_SECONDARY_VIA_INIT */ +static void __init setup_irq_stack(struct task_struct *p, int cpu) +{ + unsigned long stk; + + stk = __get_free_pages(GFP_KERNEL, THREAD_ORDER); + if (!stk) + panic("I can't seem to allocate my irq stack. Oh well, giving up."); + + irq_stacks[cpu] = (void *)stk; + memset(irq_stacks[cpu], 0, THREAD_SIZE); + irq_stacks[cpu]->thread_info.cpu = cpu; + irq_stacks[cpu]->thread_info.preempt_count = 1; + /* interrupts are not preemptable */ + p->thread_info->irq_stack = &irq_stacks[cpu]->thread_info; + + /* If we want to make the irq stack more than one unit + * deep, we can chain then off of the irq_stack pointer + * here. + */ +} + + extern unsigned long cpu_initialized; static int __init do_boot_cpu(int apicid) @@ -795,6 +822,8 @@ idle = fork_by_hand(); if (IS_ERR(idle)) panic("failed fork for CPU %d", cpu); + + setup_irq_stack(idle, cpu); /* * We remove it from the pidhash and the runqueue diff -ur linux-2.5.59-thread_info-infra/include/asm-i386/linkage.h linux-2.5.59-irqstack/include/asm-i386/linkage.h --- linux-2.5.59-thread_info-infra/include/asm-i386/linkage.h Tue Feb 4 10:32:50 2003 +++ linux-2.5.59-irqstack/include/asm-i386/linkage.h Tue Feb 4 11:24:24 2003 @@ -3,6 +3,7 @@ #define asmlinkage CPP_ASMLINKAGE __attribute__((regparm(0))) #define FASTCALL(x) x __attribute__((regparm(3))) +#define IRQHANDLER(x) x __attribute__((regparm(1))) #ifdef CONFIG_X86_ALIGNMENT_16 #define __ALIGN .align 16,0x90 diff -ur linux-2.5.59-thread_info-infra/include/asm-i386/thread_info.h linux-2.5.59-irqstack/include/asm-i386/thread_info.h --- linux-2.5.59-thread_info-infra/include/asm-i386/thread_info.h Tue Feb 4 12:09:05 2003 +++ linux-2.5.59-irqstack/include/asm-i386/thread_info.h Tue Feb 4 10:44:02 2003 @@ -30,9 +30,11 @@ __s32 preempt_count; /* 0 => preemptable, <0 => BUG */ mm_segment_t addr_limit; /* thread address space: + 0 for interrupts: illegal 0-0xBFFFFFFF for user-thead 0-0xFFFFFFFF for kernel-thread */ + struct thread_info *irq_stack; /* pointer to cpu irq stack */ struct restart_block restart_block; __u8 supervisor_stack[0]; @@ -47,7 +49,8 @@ #define TI_CPU 0x0000000C #define TI_PRE_COUNT 0x00000010 #define TI_ADDR_LIMIT 0x00000014 -#define TI_RESTART_BLOCK 0x0000018 +#define TI_IRQ_STACK 0x00000018 +#define TI_RESTART_BLOCK 0x0000022 #endif @@ -63,17 +66,18 @@ #ifndef __ASSEMBLY__ -#define INIT_THREAD_INFO(tsk) \ -{ \ - .task = &tsk, \ - .exec_domain = &default_exec_domain, \ - .flags = 0, \ - .cpu = 0, \ - .preempt_count = 1, \ - .addr_limit = KERNEL_DS, \ - .restart_block = { \ - .fn = do_no_restart_syscall, \ - }, \ +#define INIT_THREAD_INFO(tsk) \ +{ \ + .task = &tsk, \ + .exec_domain = &default_exec_domain, \ + .flags = 0, \ + .cpu = 0, \ + .preempt_count = 1, \ + .addr_limit = KERNEL_DS, \ + .irq_stack = &init_irq_union.thread_info, \ + .restart_block = { \ + .fn = do_no_restart_syscall, \ + } \ } #define init_thread_info (init_thread_union.thread_info)

- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/

This archive was generated by hypermail 2b29 : Fri Feb 07 2003 - 22:00:15 EST