Skip to content

Commit 1006fae

Browse files
torvaldstorvalds
committed
Merge branch 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull IRQ changes from Ingo Molnar: "The biggest change this cycle are the softirq/hardirq stack interaction and nesting fixes, cleanups and reorganizations from Frederic. This is the longer followup story to the softirq nesting fix that is already upstream (commit ded7975: "irq: Force hardirq exit's softirq processing on its own stack")" * 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: irqchip: bcm2835: Convert to use IRQCHIP_DECLARE macro powerpc: Tell about irq stack coverage x86: Tell about irq stack coverage irq: Optimize softirq stack selection in irq exit irq: Justify the various softirq stack choices irq: Improve a bit softirq debugging irq: Optimize call to softirq on hardirq exit irq: Consolidate do_softirq() arch overriden implementations x86/irq: Correct comment about i8259 initialization
2 parents 70fdcb8 + 5702941 commit 1006fae

18 files changed

Lines changed: 145 additions & 255 deletions

File tree

‎arch/Kconfig‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,16 @@ config HAVE_UNDERSCORE_SYMBOL_PREFIX
390390
Some architectures generate an _ in front of C symbols; things like
391391
module loading and assembly files need to know about this.
392392

393+
config HAVE_IRQ_EXIT_ON_IRQ_STACK
394+
bool
395+
help
396+
Architecture doesn't only execute the irq handler on the irq stack
397+
but also irq_exit(). This way we can process softirqs on this irq
398+
stack instead of switching to a new one when we call __do_softirq()
399+
in the end of an hardirq.
400+
This spares a stack switch and improves cache usage on softirq
401+
processing.
402+
393403
#
394404
# ABI hall of shame
395405
#

‎arch/arm/mach-bcm2835/bcm2835.c‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
#include <linux/delay.h>
1616
#include <linux/init.h>
17-
#include <linux/irqchip/bcm2835.h>
17+
#include <linux/irqchip.h>
1818
#include <linux/of_address.h>
1919
#include <linux/of_platform.h>
2020
#include <linux/clk/bcm2835.h>
@@ -130,8 +130,7 @@ static const char * const bcm2835_compat[] = {
130130

131131
DT_MACHINE_START(BCM2835, "BCM2835")
132132
.map_io = bcm2835_map_io,
133-
.init_irq = bcm2835_init_irq,
134-
.handle_irq = bcm2835_handle_irq,
133+
.init_irq = irqchip_init,
135134
.init_machine = bcm2835_init,
136135
.restart = bcm2835_restart,
137136
.dt_compat = bcm2835_compat

‎arch/metag/kernel/irq.c‎

Lines changed: 19 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -159,44 +159,30 @@ void irq_ctx_exit(int cpu)
159159

160160
extern asmlinkage void __do_softirq(void);
161161

162-
asmlinkage void do_softirq(void)
162+
void do_softirq_own_stack(void)
163163
{
164-
unsigned long flags;
165164
struct thread_info *curctx;
166165
union irq_ctx *irqctx;
167166
u32 *isp;
168167

169-
if (in_interrupt())
170-
return;
171-
172-
local_irq_save(flags);
173-
174-
if (local_softirq_pending()) {
175-
curctx = current_thread_info();
176-
irqctx = softirq_ctx[smp_processor_id()];
177-
irqctx->tinfo.task = curctx->task;
178-
179-
/* build the stack frame on the softirq stack */
180-
isp = (u32 *) ((char *)irqctx + sizeof(struct thread_info));
181-
182-
asm volatile (
183-
"MOV D0.5,%0\n"
184-
"SWAP A0StP,D0.5\n"
185-
"CALLR D1RtP,___do_softirq\n"
186-
"MOV A0StP,D0.5\n"
187-
:
188-
: "r" (isp)
189-
: "memory", "cc", "D1Ar1", "D0Ar2", "D1Ar3", "D0Ar4",
190-
"D1Ar5", "D0Ar6", "D0Re0", "D1Re0", "D0.4", "D1RtP",
191-
"D0.5"
192-
);
193-
/*
194-
* Shouldn't happen, we returned above if in_interrupt():
195-
*/
196-
WARN_ON_ONCE(softirq_count());
197-
}
198-
199-
local_irq_restore(flags);
168+
curctx = current_thread_info();
169+
irqctx = softirq_ctx[smp_processor_id()];
170+
irqctx->tinfo.task = curctx->task;
171+
172+
/* build the stack frame on the softirq stack */
173+
isp = (u32 *) ((char *)irqctx + sizeof(struct thread_info));
174+
175+
asm volatile (
176+
"MOV D0.5,%0\n"
177+
"SWAP A0StP,D0.5\n"
178+
"CALLR D1RtP,___do_softirq\n"
179+
"MOV A0StP,D0.5\n"
180+
:
181+
: "r" (isp)
182+
: "memory", "cc", "D1Ar1", "D0Ar2", "D1Ar3", "D0Ar4",
183+
"D1Ar5", "D0Ar6", "D0Re0", "D1Re0", "D0.4", "D1RtP",
184+
"D0.5"
185+
);
200186
}
201187
#endif
202188

‎arch/parisc/kernel/irq.c‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -495,22 +495,9 @@ static void execute_on_irq_stack(void *func, unsigned long param1)
495495
*irq_stack_in_use = 1;
496496
}
497497

498-
asmlinkage void do_softirq(void)
498+
void do_softirq_own_stack(void)
499499
{
500-
__u32 pending;
501-
unsigned long flags;
502-
503-
if (in_interrupt())
504-
return;
505-
506-
local_irq_save(flags);
507-
508-
pending = local_softirq_pending();
509-
510-
if (pending)
511-
execute_on_irq_stack(__do_softirq, 0);
512-
513-
local_irq_restore(flags);
500+
execute_on_irq_stack(__do_softirq, 0);
514501
}
515502
#endif /* CONFIG_IRQSTACKS */
516503

‎arch/powerpc/Kconfig‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ config PPC
138138
select OLD_SIGSUSPEND
139139
select OLD_SIGACTION if PPC32
140140
select HAVE_DEBUG_STACKOVERFLOW
141+
select HAVE_IRQ_EXIT_ON_IRQ_STACK
141142

142143
config EARLY_PRINTK
143144
bool

‎arch/powerpc/kernel/irq.c‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -594,7 +594,7 @@ void irq_ctx_init(void)
594594
}
595595
}
596596

597-
static inline void do_softirq_onstack(void)
597+
void do_softirq_own_stack(void)
598598
{
599599
struct thread_info *curtp, *irqtp;
600600

@@ -612,21 +612,6 @@ static inline void do_softirq_onstack(void)
612612
set_bits(irqtp->flags, &curtp->flags);
613613
}
614614

615-
void do_softirq(void)
616-
{
617-
unsigned long flags;
618-
619-
if (in_interrupt())
620-
return;
621-
622-
local_irq_save(flags);
623-
624-
if (local_softirq_pending())
625-
do_softirq_onstack();
626-
627-
local_irq_restore(flags);
628-
}
629-
630615
irq_hw_number_t virq_to_hw(unsigned int virq)
631616
{
632617
struct irq_data *irq_data = irq_get_irq_data(virq);

‎arch/s390/kernel/irq.c‎

Lines changed: 21 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -157,39 +157,29 @@ int arch_show_interrupts(struct seq_file *p, int prec)
157157
/*
158158
* Switch to the asynchronous interrupt stack for softirq execution.
159159
*/
160-
asmlinkage void do_softirq(void)
160+
void do_softirq_own_stack(void)
161161
{
162-
unsigned long flags, old, new;
163-
164-
if (in_interrupt())
165-
return;
166-
167-
local_irq_save(flags);
168-
169-
if (local_softirq_pending()) {
170-
/* Get current stack pointer. */
171-
asm volatile("la %0,0(15)" : "=a" (old));
172-
/* Check against async. stack address range. */
173-
new = S390_lowcore.async_stack;
174-
if (((new - old) >> (PAGE_SHIFT + THREAD_ORDER)) != 0) {
175-
/* Need to switch to the async. stack. */
176-
new -= STACK_FRAME_OVERHEAD;
177-
((struct stack_frame *) new)->back_chain = old;
178-
179-
asm volatile(" la 15,0(%0)\n"
180-
" basr 14,%2\n"
181-
" la 15,0(%1)\n"
182-
: : "a" (new), "a" (old),
183-
"a" (__do_softirq)
184-
: "0", "1", "2", "3", "4", "5", "14",
185-
"cc", "memory" );
186-
} else {
187-
/* We are already on the async stack. */
188-
__do_softirq();
189-
}
162+
unsigned long old, new;
163+
164+
/* Get current stack pointer. */
165+
asm volatile("la %0,0(15)" : "=a" (old));
166+
/* Check against async. stack address range. */
167+
new = S390_lowcore.async_stack;
168+
if (((new - old) >> (PAGE_SHIFT + THREAD_ORDER)) != 0) {
169+
/* Need to switch to the async. stack. */
170+
new -= STACK_FRAME_OVERHEAD;
171+
((struct stack_frame *) new)->back_chain = old;
172+
asm volatile(" la 15,0(%0)\n"
173+
" basr 14,%2\n"
174+
" la 15,0(%1)\n"
175+
: : "a" (new), "a" (old),
176+
"a" (__do_softirq)
177+
: "0", "1", "2", "3", "4", "5", "14",
178+
"cc", "memory" );
179+
} else {
180+
/* We are already on the async stack. */
181+
__do_softirq();
190182
}
191-
192-
local_irq_restore(flags);
193183
}
194184

195185
/*

‎arch/sh/kernel/irq.c‎

Lines changed: 21 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -149,47 +149,32 @@ void irq_ctx_exit(int cpu)
149149
hardirq_ctx[cpu] = NULL;
150150
}
151151

152-
asmlinkage void do_softirq(void)
152+
void do_softirq_own_stack(void)
153153
{
154-
unsigned long flags;
155154
struct thread_info *curctx;
156155
union irq_ctx *irqctx;
157156
u32 *isp;
158157

159-
if (in_interrupt())
160-
return;
161-
162-
local_irq_save(flags);
163-
164-
if (local_softirq_pending()) {
165-
curctx = current_thread_info();
166-
irqctx = softirq_ctx[smp_processor_id()];
167-
irqctx->tinfo.task = curctx->task;
168-
irqctx->tinfo.previous_sp = current_stack_pointer;
169-
170-
/* build the stack frame on the softirq stack */
171-
isp = (u32 *)((char *)irqctx + sizeof(*irqctx));
172-
173-
__asm__ __volatile__ (
174-
"mov r15, r9 \n"
175-
"jsr @%0 \n"
176-
/* switch to the softirq stack */
177-
" mov %1, r15 \n"
178-
/* restore the thread stack */
179-
"mov r9, r15 \n"
180-
: /* no outputs */
181-
: "r" (__do_softirq), "r" (isp)
182-
: "memory", "r0", "r1", "r2", "r3", "r4",
183-
"r5", "r6", "r7", "r8", "r9", "r15", "t", "pr"
184-
);
185-
186-
/*
187-
* Shouldn't happen, we returned above if in_interrupt():
188-
*/
189-
WARN_ON_ONCE(softirq_count());
190-
}
191-
192-
local_irq_restore(flags);
158+
curctx = current_thread_info();
159+
irqctx = softirq_ctx[smp_processor_id()];
160+
irqctx->tinfo.task = curctx->task;
161+
irqctx->tinfo.previous_sp = current_stack_pointer;
162+
163+
/* build the stack frame on the softirq stack */
164+
isp = (u32 *)((char *)irqctx + sizeof(*irqctx));
165+
166+
__asm__ __volatile__ (
167+
"mov r15, r9 \n"
168+
"jsr @%0 \n"
169+
/* switch to the softirq stack */
170+
" mov %1, r15 \n"
171+
/* restore the thread stack */
172+
"mov r9, r15 \n"
173+
: /* no outputs */
174+
: "r" (__do_softirq), "r" (isp)
175+
: "memory", "r0", "r1", "r2", "r3", "r4",
176+
"r5", "r6", "r7", "r8", "r9", "r15", "t", "pr"
177+
);
193178
}
194179
#else
195180
static inline void handle_one_irq(unsigned int irq)

‎arch/sparc/kernel/irq_64.c‎

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -698,30 +698,19 @@ void __irq_entry handler_irq(int pil, struct pt_regs *regs)
698698
set_irq_regs(old_regs);
699699
}
700700

701-
void do_softirq(void)
701+
void do_softirq_own_stack(void)
702702
{
703-
unsigned long flags;
704-
705-
if (in_interrupt())
706-
return;
707-
708-
local_irq_save(flags);
703+
void *orig_sp, *sp = softirq_stack[smp_processor_id()];
709704

710-
if (local_softirq_pending()) {
711-
void *orig_sp, *sp = softirq_stack[smp_processor_id()];
712-
713-
sp += THREAD_SIZE - 192 - STACK_BIAS;
714-
715-
__asm__ __volatile__("mov %%sp, %0\n\t"
716-
"mov %1, %%sp"
717-
: "=&r" (orig_sp)
718-
: "r" (sp));
719-
__do_softirq();
720-
__asm__ __volatile__("mov %0, %%sp"
721-
: : "r" (orig_sp));
722-
}
705+
sp += THREAD_SIZE - 192 - STACK_BIAS;
723706

724-
local_irq_restore(flags);
707+
__asm__ __volatile__("mov %%sp, %0\n\t"
708+
"mov %1, %%sp"
709+
: "=&r" (orig_sp)
710+
: "r" (sp));
711+
__do_softirq();
712+
__asm__ __volatile__("mov %0, %%sp"
713+
: : "r" (orig_sp));
725714
}
726715

727716
#ifdef CONFIG_HOTPLUG_CPU

‎arch/x86/Kconfig‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ config X86
123123
select COMPAT_OLD_SIGACTION if IA32_EMULATION
124124
select RTC_LIB
125125
select HAVE_DEBUG_STACKOVERFLOW
126+
select HAVE_IRQ_EXIT_ON_IRQ_STACK if X86_64
126127

127128
config INSTRUCTION_DECODER
128129
def_bool y

0 commit comments

Comments
 (0)