1 files changed, 22 insertions, 24 deletions

diff --git a/mm/slab.c b/mm/slab.c
index 815430698..ed5d018f1 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -843,21 +843,17 @@ static kmem_cache_t *cache_to_drain = NULL;
 static DECLARE_WAIT_QUEUE_HEAD(cache_drain_wait);
 unsigned long slab_cache_drain_mask;
 
-static void drain_cpu_caches(kmem_cache_t *cachep)
+/*
+ * Waits for all CPUs to execute slab_drain_local_cache().
+ * Caller must be holding cache_drain_sem.
+ */
+static void slab_drain_all_sync(void)
 {
 	DECLARE_WAITQUEUE(wait, current);
-	unsigned long cpu_mask = 0;
-	int i;
-
-	for (i = 0; i < smp_num_cpus; i++)
-		cpu_mask |= (1UL << cpu_logical_map(i));
-
-	down(&cache_drain_sem);
-
-	cache_to_drain = cachep;
-	slab_cache_drain_mask = cpu_mask;
 
+	local_irq_disable();
 	slab_drain_local_cache();
+	local_irq_enable();
 
 	add_wait_queue(&cache_drain_wait, &wait);
 	current->state = TASK_UNINTERRUPTIBLE;
@@ -865,7 +861,21 @@ static void drain_cpu_caches(kmem_cache_t *cachep)
 		schedule();
 	current->state = TASK_RUNNING;
 	remove_wait_queue(&cache_drain_wait, &wait);
+}
+
+static void drain_cpu_caches(kmem_cache_t *cachep)
+{
+	unsigned long cpu_mask = 0;
+	int i;
 
+	for (i = 0; i < smp_num_cpus; i++)
+		cpu_mask |= (1UL << cpu_logical_map(i));
+
+	down(&cache_drain_sem);
+
+	cache_to_drain = cachep;
+	slab_cache_drain_mask = cpu_mask;
+	slab_drain_all_sync();
 	cache_to_drain = NULL;
 
 	up(&cache_drain_sem);
@@ -1594,7 +1604,6 @@ static ccupdate_struct_t *ccupdate_state = NULL;
 /* Called from per-cpu timer interrupt. */
 void slab_drain_local_cache(void)
 {
-	local_irq_disable();
 	if (ccupdate_state != NULL) {
 		ccupdate_struct_t *new = ccupdate_state;
 		cpucache_t *old = cc_data(new->cachep);
@@ -1610,7 +1619,6 @@ void slab_drain_local_cache(void)
 			cc->avail = 0;
 		}
 	}
-	local_irq_enable();
 
 	clear_bit(smp_processor_id(), &slab_cache_drain_mask);
 	if (slab_cache_drain_mask == 0)
@@ -1619,7 +1627,6 @@ void slab_drain_local_cache(void)
 
 static void do_ccupdate(ccupdate_struct_t *data)
 {
-	DECLARE_WAITQUEUE(wait, current);
 	unsigned long cpu_mask = 0;
 	int i;
 
@@ -1630,16 +1637,7 @@ static void do_ccupdate(ccupdate_struct_t *data)
 
 	ccupdate_state = data;
 	slab_cache_drain_mask = cpu_mask;
-
-	slab_drain_local_cache();
-
-	add_wait_queue(&cache_drain_wait, &wait);
-	current->state = TASK_UNINTERRUPTIBLE;
-	while (slab_cache_drain_mask != 0UL)
-		schedule();
-	current->state = TASK_RUNNING;
-	remove_wait_queue(&cache_drain_wait, &wait);
-
+	slab_drain_all_sync();
 	ccupdate_state = NULL;
 
 	up(&cache_drain_sem);