Merge with 2.3.99-pre9.

8 files changed, 97 insertions, 68 deletions

diff --git a/mm/filemap.c b/mm/filemap.c
index 81f7d7ab9..b1e2b8547 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -244,14 +244,19 @@ repeat:
 	spin_unlock(&pagecache_lock);
 }
 
+/*
+ * nr_dirty represents the number of dirty pages that we will write async
+ * before doing sync writes.  We can only do sync writes if we can
+ * wait for IO (__GFP_IO set).
+ */
 int shrink_mmap(int priority, int gfp_mask)
 {
-	int ret = 0, count;
-	LIST_HEAD(old);
-	struct list_head * page_lru, * dispose;
+	int ret = 0, count, nr_dirty;
+	struct list_head * page_lru;
 	struct page * page = NULL;
 	
 	count = nr_lru_pages / (priority + 1);
+	nr_dirty = priority;
 
 	/* we need pagemap_lru_lock for list_del() ... subtle code below */
 	spin_lock(&pagemap_lru_lock);
@@ -259,25 +264,10 @@ int shrink_mmap(int priority, int gfp_mask)
 		page = list_entry(page_lru, struct page, lru);
 		list_del(page_lru);
 
-		dispose = &lru_cache;
 		if (PageTestandClearReferenced(page))
 			goto dispose_continue;
 
 		count--;
-
-		/*
-		 * I'm ambivalent on this one.. Should we try to
-		 * maintain LRU on the LRU list, and put pages that
-		 * are old at the end of the queue, even if that
-		 * means that we'll re-scan then again soon and
-		 * often waste CPU time? Or should be just let any
-		 * pages we do not want to touch now for one reason
-		 * or another percolate to be "young"?
-		 *
-		dispose = &old;
-		 *
-		 */
-
 		/*
 		 * Avoid unscalable SMP locking for pages we can
 		 * immediate tell are untouchable..
@@ -303,7 +293,8 @@ int shrink_mmap(int priority, int gfp_mask)
 		 * of zone - it's old.
 		 */
 		if (page->buffers) {
-			if (!try_to_free_buffers(page))
+			int wait = ((gfp_mask & __GFP_IO) && (nr_dirty-- < 0));
+			if (!try_to_free_buffers(page, wait))
 				goto unlock_continue;
 			/* page was locked, inode can't go away under us */
 			if (!page->mapping) {
@@ -362,7 +353,7 @@ unlock_continue:
 		UnlockPage(page);
 		page_cache_release(page);
 dispose_continue:
-		list_add(page_lru, dispose);
+		list_add(page_lru, &lru_cache);
 	}
 	goto out;
 
@@ -377,8 +368,6 @@ made_buffer_progress:
 	nr_lru_pages--;
 
 out:
-	list_splice(&old, lru_cache.prev);
-
 	spin_unlock(&pagemap_lru_lock);
 
 	return ret;
@@ -2319,7 +2308,8 @@ out:
 	return error;
 }
 
-struct page *read_cache_page(struct address_space *mapping,
+static inline
+struct page *__read_cache_page(struct address_space *mapping,
 				unsigned long index,
 				int (*filler)(void *,struct page*),
 				void *data)
@@ -2350,6 +2340,35 @@ repeat:
 	return page;
 }
 
+/*
+ * Read into the page cache. If a page already exists,
+ * and Page_Uptodate() is not set, try to fill the page.
+ */
+struct page *read_cache_page(struct address_space *mapping,
+				unsigned long index,
+				int (*filler)(void *,struct page*),
+				void *data)
+{
+	struct page *page = __read_cache_page(mapping, index, filler, data);
+	int err;
+
+	if (IS_ERR(page) || Page_Uptodate(page))
+		goto out;
+
+	lock_page(page);
+	if (Page_Uptodate(page)) {
+		UnlockPage(page);
+		goto out;
+	}
+	err = filler(data, page);
+	if (err < 0) {
+		page_cache_release(page);
+		page = ERR_PTR(err);
+	}
+ out:
+	return page;
+}
+
 static inline struct page * __grab_cache_page(struct address_space *mapping,
 				unsigned long index, struct page **cached_page)
 {
diff --git a/mm/highmem.c b/mm/highmem.c
index 11e03521e..7c9dbc695 100644
--- a/mm/highmem.c
+++ b/mm/highmem.c
@@ -60,7 +60,7 @@ struct page * prepare_highmem_swapout(struct page * page)
 	 * ok, we can just forget about our highmem page since 
 	 * we stored its data into the new regular_page.
 	 */
-	__free_page(page);
+	page_cache_release(page);
 	new_page = mem_map + MAP_NR(regular_page);
 	LockPage(new_page);
 	return new_page;
@@ -78,7 +78,7 @@ struct page * replace_with_highmem(struct page * page)
 	if (!highpage)
 		return page;
 	if (!PageHighMem(highpage)) {
-		__free_page(highpage);
+		page_cache_release(highpage);
 		return page;
 	}
 
@@ -94,7 +94,7 @@ struct page * replace_with_highmem(struct page * page)
 	 * We can just forget the old page since 
 	 * we stored its data into the new highmem-page.
 	 */
-	__free_page(page);
+	page_cache_release(page);
 
 	return highpage;
 }
diff --git a/mm/memory.c b/mm/memory.c
index e5a548925..de7dc07f8 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -156,7 +156,7 @@ int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
 	unsigned long address = vma->vm_start;
 	unsigned long end = vma->vm_end;
 	unsigned long cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
-	
+
 	src_pgd = pgd_offset(src, address)-1;
 	dst_pgd = pgd_offset(dst, address)-1;
 	
@@ -878,7 +878,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma,
 		new_page = old_page;
 	}
 	spin_unlock(&mm->page_table_lock);
-	__free_page(new_page);
+	page_cache_release(new_page);
 	return 1;	/* Minor fault */
 
 bad_wp_page:
@@ -1022,7 +1022,7 @@ void swapin_readahead(swp_entry_t entry)
 		/* Ok, do the async read-ahead now */
 		new_page = read_swap_cache_async(SWP_ENTRY(SWP_TYPE(entry), offset), 0);
 		if (new_page != NULL)
-			__free_page(new_page);
+			page_cache_release(new_page);
 		swap_free(SWP_ENTRY(SWP_TYPE(entry), offset));
 	}
 	return;
diff --git a/mm/slab.c b/mm/slab.c
index 7dbc443fb..64f33cb33 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -1751,7 +1751,7 @@ bad_ptr:
 
 #if 1
 /* FORCE A KERNEL DUMP WHEN THIS HAPPENS. SPEAK IN ALL CAPS. GET THE CALL CHAIN. */
-*(int *) 0 = 0;
+BUG();
 #endif
 
 null_ptr:
diff --git a/mm/swap_state.c b/mm/swap_state.c
index 347f87372..2405aba2f 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -136,7 +136,7 @@ void free_page_and_swap_cache(struct page *page)
 		}
 		UnlockPage(page);
 	}
-        page_cache_release(page);
+	page_cache_release(page);
 }
 
 
diff --git a/mm/swapfile.c b/mm/swapfile.c
index c4b4733b7..55ef476a3 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -377,7 +377,7 @@ static int try_to_unuse(unsigned int type)
                    page we've been using. */
 		if (PageSwapCache(page))
 			delete_from_swap_cache(page);
-		__free_page(page);
+		page_cache_release(page);
 		/*
 		 * Check for and clear any overflowed swap map counts.
 		 */
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 99510c53b..57f3ca56c 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -3,14 +3,17 @@
  *
  *  Copyright (C) 1993  Linus Torvalds
  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
+ *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
  */
 
 #include <linux/malloc.h>
 #include <linux/vmalloc.h>
+#include <linux/spinlock.h>
 
 #include <asm/uaccess.h>
 #include <asm/pgalloc.h>
 
+rwlock_t vmlist_lock = RW_LOCK_UNLOCKED;
 struct vm_struct * vmlist = NULL;
 
 static inline void free_area_pte(pmd_t * pmd, unsigned long address, unsigned long size)
@@ -87,7 +90,8 @@ void vmfree_area_pages(unsigned long address, unsigned long size)
 	flush_tlb_all();
 }
 
-static inline int alloc_area_pte(pte_t * pte, unsigned long address, unsigned long size, pgprot_t prot)
+static inline int alloc_area_pte (pte_t * pte, unsigned long address,
+			unsigned long size, int gfp_mask, pgprot_t prot)
 {
 	unsigned long end;
 
@@ -99,7 +103,7 @@ static inline int alloc_area_pte(pte_t * pte, unsigned long address, unsigned lo
 		struct page * page;
 		if (!pte_none(*pte))
 			printk(KERN_ERR "alloc_area_pte: page already exists\n");
-		page = alloc_page(GFP_KERNEL|__GFP_HIGHMEM);
+		page = alloc_page(gfp_mask);
 		if (!page)
 			return -ENOMEM;
 		set_pte(pte, mk_pte(page, prot));
@@ -109,7 +113,7 @@ static inline int alloc_area_pte(pte_t * pte, unsigned long address, unsigned lo
 	return 0;
 }
 
-static inline int alloc_area_pmd(pmd_t * pmd, unsigned long address, unsigned long size, pgprot_t prot)
+static inline int alloc_area_pmd(pmd_t * pmd, unsigned long address, unsigned long size, int gfp_mask, pgprot_t prot)
 {
 	unsigned long end;
 
@@ -121,7 +125,7 @@ static inline int alloc_area_pmd(pmd_t * pmd, unsigned long address, unsigned lo
 		pte_t * pte = pte_alloc_kernel(pmd, address);
 		if (!pte)
 			return -ENOMEM;
-		if (alloc_area_pte(pte, address, end - address, prot))
+		if (alloc_area_pte(pte, address, end - address, gfp_mask, prot))
 			return -ENOMEM;
 		address = (address + PMD_SIZE) & PMD_MASK;
 		pmd++;
@@ -129,7 +133,8 @@ static inline int alloc_area_pmd(pmd_t * pmd, unsigned long address, unsigned lo
 	return 0;
 }
 
-int vmalloc_area_pages(unsigned long address, unsigned long size, pgprot_t prot)
+inline int vmalloc_area_pages (unsigned long address, unsigned long size,
+                               int gfp_mask, pgprot_t prot)
 {
 	pgd_t * dir;
 	unsigned long end = address + size;
@@ -139,11 +144,11 @@ int vmalloc_area_pages(unsigned long address, unsigned long size, pgprot_t prot)
 	do {
 		pmd_t *pmd;
 		pgd_t olddir = *dir;
-
+		
 		pmd = pmd_alloc_kernel(dir, address);
 		if (!pmd)
 			return -ENOMEM;
-		if (alloc_area_pmd(pmd, address, end - address, prot))
+		if (alloc_area_pmd(pmd, address, end - address, gfp_mask, prot))
 			return -ENOMEM;
 		if (pgd_val(olddir) != pgd_val(*dir))
 			set_pgdir(address, *dir);
@@ -163,11 +168,13 @@ struct vm_struct * get_vm_area(unsigned long size, unsigned long flags)
 	if (!area)
 		return NULL;
 	addr = VMALLOC_START;
+	write_lock(&vmlist_lock);
 	for (p = &vmlist; (tmp = *p) ; p = &tmp->next) {
 		if (size + addr < (unsigned long) tmp->addr)
 			break;
 		addr = tmp->size + (unsigned long) tmp->addr;
 		if (addr > VMALLOC_END-size) {
+			write_unlock(&vmlist_lock);
 			kfree(area);
 			return NULL;
 		}
@@ -177,6 +184,7 @@ struct vm_struct * get_vm_area(unsigned long size, unsigned long flags)
 	area->size = size + PAGE_SIZE;
 	area->next = *p;
 	*p = area;
+	write_unlock(&vmlist_lock);
 	return area;
 }
 
@@ -190,18 +198,21 @@ void vfree(void * addr)
 		printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
 		return;
 	}
+	write_lock(&vmlist_lock);
 	for (p = &vmlist ; (tmp = *p) ; p = &tmp->next) {
 		if (tmp->addr == addr) {
 			*p = tmp->next;
 			vmfree_area_pages(VMALLOC_VMADDR(tmp->addr), tmp->size);
 			kfree(tmp);
+			write_unlock(&vmlist_lock);
 			return;
 		}
 	}
+	write_unlock(&vmlist_lock);
 	printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", addr);
 }
 
-void * vmalloc_prot(unsigned long size, pgprot_t prot)
+void * __vmalloc (unsigned long size, int gfp_mask, pgprot_t prot)
 {
 	void * addr;
 	struct vm_struct *area;
@@ -217,7 +228,7 @@ void * vmalloc_prot(unsigned long size, pgprot_t prot)
 		return NULL;
 	}
 	addr = area->addr;
-	if (vmalloc_area_pages(VMALLOC_VMADDR(addr), size, prot)) {
+	if (vmalloc_area_pages(VMALLOC_VMADDR(addr), size, gfp_mask, prot)) {
 		vfree(addr);
 		BUG();
 		return NULL;
@@ -225,11 +236,6 @@ void * vmalloc_prot(unsigned long size, pgprot_t prot)
 	return addr;
 }
 
-void * vmalloc(unsigned long size)
-{
-	return vmalloc_prot (size, PAGE_KERNEL);
-}
-
 long vread(char *buf, char *addr, unsigned long count)
 {
 	struct vm_struct *tmp;
@@ -240,6 +246,7 @@ long vread(char *buf, char *addr, unsigned long count)
 	if ((unsigned long) addr + count < count)
 		count = -(unsigned long) addr;
 
+	read_lock(&vmlist_lock);
 	for (tmp = vmlist; tmp; tmp = tmp->next) {
 		vaddr = (char *) tmp->addr;
 		if (addr >= vaddr + tmp->size - PAGE_SIZE)
@@ -247,7 +254,7 @@ long vread(char *buf, char *addr, unsigned long count)
 		while (addr < vaddr) {
 			if (count == 0)
 				goto finished;
-			put_user('\0', buf);
+			*buf = '\0';
 			buf++;
 			addr++;
 			count--;
@@ -256,12 +263,13 @@ long vread(char *buf, char *addr, unsigned long count)
 		do {
 			if (count == 0)
 				goto finished;
-			put_user(*addr, buf);
+			*buf = *addr;
 			buf++;
 			addr++;
 			count--;
 		} while (--n > 0);
 	}
 finished:
+	read_unlock(&vmlist_lock);
 	return buf - buf_start;
 }
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 8734cc459..1919c0961 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -48,6 +48,9 @@ static int try_to_swap_out(struct mm_struct * mm, struct vm_area_struct* vma, un
 	if ((page-mem_map >= max_mapnr) || PageReserved(page))
 		goto out_failed;
 
+	if (mm->swap_cnt)
+		mm->swap_cnt--;
+
 	/* Don't look at this pte if it's been accessed recently. */
 	if (pte_young(pte)) {
 		/*
@@ -76,10 +79,9 @@ static int try_to_swap_out(struct mm_struct * mm, struct vm_area_struct* vma, un
 		set_pte(page_table, swp_entry_to_pte(entry));
 drop_pte:
 		UnlockPage(page);
-		mm->swap_cnt--;
 		vma->vm_mm->rss--;
 		flush_tlb_page(vma, address);
-		__free_page(page);
+		page_cache_release(page);
 		goto out_failed;
 	}
 
@@ -142,7 +144,6 @@ drop_pte:
 		struct file *file = vma->vm_file;
 		if (file) get_file(file);
 		pte_clear(page_table);
-		mm->swap_cnt--;
 		vma->vm_mm->rss--;
 		flush_tlb_page(vma, address);
 		vmlist_access_unlock(vma->vm_mm);
@@ -151,7 +152,7 @@ drop_pte:
 		if (file) fput(file);
 		if (!error)
 			goto out_free_success;
-		__free_page(page);
+		page_cache_release(page);
 		return error;
 	}
 
@@ -174,7 +175,6 @@ drop_pte:
 	add_to_swap_cache(page, entry);
 
 	/* Put the swap entry into the pte after the page is in swapcache */
-	mm->swap_cnt--;
 	vma->vm_mm->rss--;
 	set_pte(page_table, swp_entry_to_pte(entry));
 	flush_tlb_page(vma, address);
@@ -184,7 +184,7 @@ drop_pte:
 	rw_swap_page(WRITE, page, 0);
 
 out_free_success:
-	__free_page(page);
+	page_cache_release(page);
 	return 1;
 out_swap_free:
 	swap_free(entry);
@@ -363,7 +363,7 @@ static int swap_out(unsigned int priority, int gfp_mask)
 	 * Think of swap_cnt as a "shadow rss" - it tells us which process
 	 * we want to page out (always try largest first).
 	 */
-	counter = (nr_threads << 1) >> (priority >> 1);
+	counter = (nr_threads << 2) >> (priority >> 2);
 	if (counter < 1)
 		counter = 1;
 
@@ -430,16 +430,17 @@ out:
  * latency.
  */
 #define FREE_COUNT	8
-#define SWAP_COUNT	8
+#define SWAP_COUNT	16
 static int do_try_to_free_pages(unsigned int gfp_mask)
 {
 	int priority;
 	int count = FREE_COUNT;
+	int swap_count;
 
 	/* Always trim SLAB caches when memory gets low. */
 	kmem_cache_reap(gfp_mask);
 
-	priority = 6;
+	priority = 64;
 	do {
 		while (shrink_mmap(priority, gfp_mask)) {
 			if (!--count)
@@ -471,12 +472,11 @@ static int do_try_to_free_pages(unsigned int gfp_mask)
 		 * put in the swap cache), so we must not count this
 		 * as a "count" success.
 		 */
-		{
-			int swap_count = SWAP_COUNT;
-			while (swap_out(priority, gfp_mask))
-				if (--swap_count < 0)
-					break;
-		}
+		swap_count = SWAP_COUNT;
+		while (swap_out(priority, gfp_mask))
+			if (--swap_count < 0)
+				break;
+
 	} while (--priority >= 0);
 
 	/* Always end on a shrink_mmap.. */
@@ -484,8 +484,8 @@ static int do_try_to_free_pages(unsigned int gfp_mask)
 		if (!--count)
 			goto done;
 	}
-
-	return 0;
+	/* We return 1 if we are freed some page */
+	return (count != FREE_COUNT);
 
 done:
 	return 1;
@@ -538,16 +538,18 @@ int kswapd(void *unused)
 			int i;
 			for(i = 0; i < MAX_NR_ZONES; i++) {
 				zone_t *zone = pgdat->node_zones+ i;
+				if (tsk->need_resched)
+					schedule();
 				if (!zone->size || !zone->zone_wake_kswapd)
 					continue;
-				something_to_do = 1;
+				if (zone->free_pages < zone->pages_low)
+					something_to_do = 1;
 				do_try_to_free_pages(GFP_KSWAPD);
 			}
-			run_task_queue(&tq_disk);
 			pgdat = pgdat->node_next;
 		} while (pgdat);
 
-		if (tsk->need_resched || !something_to_do) {
+		if (!something_to_do) {
 			tsk->state = TASK_INTERRUPTIBLE;
 			interruptible_sleep_on(&kswapd_wait);
 		}