diff options
author | Ralf Baechle <ralf@linux-mips.org> | 2000-02-23 00:40:54 +0000 |
---|---|---|
committer | Ralf Baechle <ralf@linux-mips.org> | 2000-02-23 00:40:54 +0000 |
commit | 529c593ece216e4aaffd36bd940cb94f1fa63129 (patch) | |
tree | 78f1c0b805f5656aa7b0417a043c5346f700a2cf /kernel/softirq.c | |
parent | 0bd079751d25808d1972baee5c4eaa1db2227257 (diff) |
Merge with 2.3.43. I did ignore all modifications to the qlogicisp.c
driver due to the Origin A64 hacks.
Diffstat (limited to 'kernel/softirq.c')
-rw-r--r-- | kernel/softirq.c | 291 |
1 files changed, 247 insertions, 44 deletions
diff --git a/kernel/softirq.c b/kernel/softirq.c index d184c944e..0f3c23ee9 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -3,68 +3,271 @@ * * Copyright (C) 1992 Linus Torvalds * - * do_bottom_half() runs at normal kernel priority: all interrupts - * enabled. do_bottom_half() is atomic with respect to itself: a - * bottom_half handler need not be re-entrant. - * * Fixed a disable_bh()/enable_bh() race (was causing a console lockup) * due bh_mask_count not atomic handling. Copyright (C) 1998 Andrea Arcangeli + * + * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) */ #include <linux/mm.h> #include <linux/kernel_stat.h> #include <linux/interrupt.h> #include <linux/smp_lock.h> +#include <linux/init.h> -#include <asm/io.h> +/* + - No shared variables, all the data are CPU local. + - If a softirq needs serialization, let it serialize itself + by its own spinlocks. + - Even if softirq is serialized, only local cpu is marked for + execution. Hence, we get something sort of weak cpu binding. + Though it is still not clear, will it result in better locality + or will not. + - These softirqs are not masked by global cli() and start_bh_atomic() + (by clear reasons). Hence, old parts of code still using global locks + MUST NOT use softirqs, but insert interfacing routines acquiring + global locks. F.e. look at BHs implementation. -/* intr_count died a painless death... -DaveM */ + Examples: + - NET RX softirq. It is multithreaded and does not require + any global serialization. + - NET TX softirq. It kicks software netdevice queues, hence + it is logically serialized per device, but this serialization + is invisible to common code. + - Tasklets: serialized wrt itself. + - Bottom halves: globally serialized, grr... + */ -atomic_t bh_mask_count[32]; -unsigned long bh_active = 0; -unsigned long bh_mask = 0; -void (*bh_base[32])(void); -/* - * This needs to make sure that only one bottom half handler - * is ever active at a time. We do this without locking by - * doing an atomic increment on the intr_count, and checking - * (nonatomically) against 1. Only if it's 1 do we schedule - * the bottom half. - * - * Note that the non-atomicity of the test (as opposed to the - * actual update) means that the test may fail, and _nobody_ - * runs the handlers if there is a race that makes multiple - * CPU's get here at the same time. That's ok, we'll run them - * next time around. - */ -static inline void run_bottom_halves(void) +struct softirq_state softirq_state[NR_CPUS]; +static struct softirq_action softirq_vec[32]; + +asmlinkage void do_softirq() { - unsigned long active; - void (**bh)(void); - - active = get_active_bhs(); - clear_active_bhs(active); - bh = bh_base; - do { - if (active & 1) - (*bh)(); - bh++; - active >>= 1; - } while (active); + int cpu = smp_processor_id(); + __u32 active, mask; + + if (in_interrupt()) + return; + + local_bh_disable(); + + local_irq_disable(); + mask = softirq_state[cpu].mask; + active = softirq_state[cpu].active & mask; + + if (active) { + struct softirq_action *h; + +restart: + /* Reset active bitmask before enabling irqs */ + softirq_state[cpu].active &= ~active; + + local_irq_enable(); + + h = softirq_vec; + mask &= ~active; + + do { + if (active & 1) + h->action(h); + h++; + active >>= 1; + } while (active); + + local_irq_disable(); + + active = softirq_state[cpu].active; + if ((active &= mask) != 0) + goto retry; + } + + local_bh_enable(); + + /* Leave with locally disabled hard irqs. It is critical to close + * window for infinite recursion, while we help local bh count, + * it protected us. Now we are defenceless. + */ + return; + +retry: + goto restart; } -asmlinkage void do_bottom_half(void) + +static spinlock_t softirq_mask_lock = SPIN_LOCK_UNLOCKED; + +void open_softirq(int nr, void (*action)(struct softirq_action*), void *data) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&softirq_mask_lock, flags); + softirq_vec[nr].data = data; + softirq_vec[nr].action = action; + + for (i=0; i<NR_CPUS; i++) + softirq_state[i].mask |= (1<<nr); + spin_unlock_irqrestore(&softirq_mask_lock, flags); +} + + +/* Tasklets */ + +struct tasklet_head tasklet_vec[NR_CPUS] __cacheline_aligned; + +static void tasklet_action(struct softirq_action *a) { int cpu = smp_processor_id(); + struct tasklet_struct *list; + + local_irq_disable(); + list = tasklet_vec[cpu].list; + tasklet_vec[cpu].list = NULL; + local_irq_enable(); - if (softirq_trylock(cpu)) { - if (hardirq_trylock(cpu)) { - __sti(); - run_bottom_halves(); - __cli(); - hardirq_endlock(cpu); + while (list != NULL) { + struct tasklet_struct *t = list; + + list = list->next; + + if (tasklet_trylock(t)) { + if (atomic_read(&t->count) == 0) { + clear_bit(TASKLET_STATE_SCHED, &t->state); + + t->func(t->data); + tasklet_unlock(t); + continue; + } + tasklet_unlock(t); } - softirq_endlock(cpu); + local_irq_disable(); + t->next = tasklet_vec[cpu].list; + tasklet_vec[cpu].list = t; + __cpu_raise_softirq(cpu, TASKLET_SOFTIRQ); + local_irq_enable(); } } + + + +struct tasklet_head tasklet_hi_vec[NR_CPUS] __cacheline_aligned; + +static void tasklet_hi_action(struct softirq_action *a) +{ + int cpu = smp_processor_id(); + struct tasklet_struct *list; + + local_irq_disable(); + list = tasklet_hi_vec[cpu].list; + tasklet_hi_vec[cpu].list = NULL; + local_irq_enable(); + + while (list != NULL) { + struct tasklet_struct *t = list; + + list = list->next; + + if (tasklet_trylock(t)) { + if (atomic_read(&t->count) == 0) { + clear_bit(TASKLET_STATE_SCHED, &t->state); + + t->func(t->data); + tasklet_unlock(t); + continue; + } + tasklet_unlock(t); + } + local_irq_disable(); + t->next = tasklet_hi_vec[cpu].list; + tasklet_hi_vec[cpu].list = t; + __cpu_raise_softirq(cpu, HI_SOFTIRQ); + local_irq_enable(); + } +} + + +void tasklet_init(struct tasklet_struct *t, + void (*func)(unsigned long), unsigned long data) +{ + t->func = func; + t->data = data; + t->state = 0; + atomic_set(&t->count, 0); +} + +void tasklet_kill(struct tasklet_struct *t) +{ + while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { + if (in_interrupt()) + panic("Attempt to kill tasklet from interrupt\n"); + schedule(); + } + tasklet_unlock_wait(t); +} + + + +/* Old style BHs */ + +static void (*bh_base[32])(void); +struct tasklet_struct bh_task_vec[32]; + +/* BHs are serialized by spinlock global_bh_lock. + + It is still possible to make synchronize_bh() as + spin_unlock_wait(&global_bh_lock). This operation is not used + by kernel now, so that this lock is not made private only + due to wait_on_irq(). + + It can be removed only after auditing all the BHs. + */ +spinlock_t global_bh_lock = SPIN_LOCK_UNLOCKED; + +static void bh_action(unsigned long nr) +{ + int cpu = smp_processor_id(); + + if (!spin_trylock(&global_bh_lock)) + goto resched; + + if (!hardirq_trylock(cpu)) + goto resched_unlock; + + if (bh_base[nr]) + bh_base[nr](); + + hardirq_endlock(cpu); + spin_unlock(&global_bh_lock); + return; + +resched_unlock: + spin_unlock(&global_bh_lock); +resched: + mark_bh(nr); +} + +void init_bh(int nr, void (*routine)(void)) +{ + bh_base[nr] = routine; + mb(); +} + +void remove_bh(int nr) +{ + tasklet_kill(bh_task_vec+nr); + bh_base[nr] = NULL; +} + +void __init softirq_init() +{ + int i; + + for (i=0; i<32; i++) + tasklet_init(bh_task_vec+i, bh_action, i); + + open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL); + open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL); +} + + |