o Make it work on CPUs with 128 byte cachelines.

 o Implement handlers for VCEI / VCED exceptions.  Install them only on
   SC / MC versions of the R4000 and R4400 which saves us four
   instructions per exception on other CPUs.
 o Print the number of VCED / VCEI exceptions in /proc/cpuinfo, iff
   available.
 o Many performance fixes for the SC / MC versions of R4000 and R4400.

1 files changed, 355 insertions, 286 deletions

diff --git a/arch/mips/mm/r4xx0.c b/arch/mips/mm/r4xx0.c
index 43c13e263..31b636296 100644
--- a/arch/mips/mm/r4xx0.c
+++ b/arch/mips/mm/r4xx0.c
@@ -1,4 +1,4 @@
-/* $Id: r4xx0.c,v 1.28 1998/08/18 20:45:08 ralf Exp $
+/* $Id: r4xx0.c,v 1.19 1998/08/25 09:14:46 ralf Exp $
  *
  * This file is subject to the terms and conditions of the GNU General Public
  * License.  See the file "COPYING" in the main directory of this archive
@@ -41,7 +41,7 @@ static int icache_size, dcache_size; /* Size in bytes */
 static int ic_lsize, dc_lsize;       /* LineSize in bytes */
 
 /* Secondary cache (if present) parameters. */
-static unsigned long scache_size, sc_lsize;	/* Again, in bytes */
+static unsigned int scache_size, sc_lsize;	/* Again, in bytes */
 
 #include <asm/cacheops.h>
 #include <asm/r4kcache.h>
@@ -78,10 +78,7 @@ struct bcache_ops *bcops = &no_sc_ops;
  * - a version which handles the buggy R4600 v1.x
  * - a version which handles the buggy R4600 v2.0
  * - Finally a last version without fancy cache games for the SC and MC
- *   versions of R4000 and R4400.  Cache instructions are quite expensive
- *   and I guess using them for both the primary and the second level cache
- *   wouldn't be worth the effort.
- *   This needs to be verified by benchmarking.
+ *   versions of R4000 and R4400.
  */
 
 static void r4k_clear_page_d16(unsigned long page)
@@ -245,14 +242,84 @@ static void r4k_clear_page_r4600_v2(unsigned long page)
 	restore_flags(flags);
 }
 
-static void r4k_clear_page(unsigned long page)
+/*
+ * The next 4 versions are optimized for all possible scache configurations
+ * of the SC / MC versions of R4000 and R4400 ...
+ *
+ * Todo: For even better performance we should have a routine optimized for
+ * every legal combination of dcache / scache linesize.  When I (Ralf) tried
+ * this the kernel crashed shortly after mounting the root filesystem.  CPU
+ * bug?  Weirdo cache instruction semantics?
+ */
+static void r4k_clear_page_s16(unsigned long page)
+{
+	__asm__ __volatile__(
+		".set\tnoreorder\n\t"
+		".set\tnoat\n\t"
+		".set\tmips3\n\t"
+		"daddiu\t$1,%0,%2\n"
+		"1:\tcache\t%3,(%0)\n\t"
+		"sd\t$0,(%0)\n\t"
+		"sd\t$0,8(%0)\n\t"
+		"cache\t%3,16(%0)\n\t"
+		"sd\t$0,16(%0)\n\t"
+		"sd\t$0,24(%0)\n\t"
+		"daddiu\t%0,64\n\t"
+		"cache\t%3,-32(%0)\n\t"
+		"sd\t$0,-32(%0)\n\t"
+		"sd\t$0,-24(%0)\n\t"
+		"cache\t%3,-16(%0)\n\t"
+		"sd\t$0,-16(%0)\n\t"
+		"bne\t$1,%0,1b\n\t"
+		"sd\t$0,-8(%0)\n\t"
+		".set\tmips0\n\t"
+		".set\tat\n\t"
+		".set\treorder"
+		:"=r" (page)
+		:"0" (page),
+		 "I" (PAGE_SIZE),
+		 "i" (Create_Dirty_Excl_SD)
+		:"$1","memory");
+}
+
+static void r4k_clear_page_s32(unsigned long page)
+{
+	__asm__ __volatile__(
+		".set\tnoreorder\n\t"
+		".set\tnoat\n\t"
+		".set\tmips3\n\t"
+		"daddiu\t$1,%0,%2\n"
+		"1:\tcache\t%3,(%0)\n\t"
+		"sd\t$0,(%0)\n\t"
+		"sd\t$0,8(%0)\n\t"
+		"sd\t$0,16(%0)\n\t"
+		"sd\t$0,24(%0)\n\t"
+		"daddiu\t%0,64\n\t"
+		"cache\t%3,-32(%0)\n\t"
+		"sd\t$0,-32(%0)\n\t"
+		"sd\t$0,-24(%0)\n\t"
+		"sd\t$0,-16(%0)\n\t"
+		"bne\t$1,%0,1b\n\t"
+		"sd\t$0,-8(%0)\n\t"
+		".set\tmips0\n\t"
+		".set\tat\n\t"
+		".set\treorder"
+		:"=r" (page)
+		:"0" (page),
+		 "I" (PAGE_SIZE),
+		 "i" (Create_Dirty_Excl_SD)
+		:"$1","memory");
+}
+
+static void r4k_clear_page_s64(unsigned long page)
 {
 	__asm__ __volatile__(
 		".set\tnoreorder\n\t"
 		".set\tnoat\n\t"
 		".set\tmips3\n\t"
 		"daddiu\t$1,%0,%2\n"
-		"1:\tsd\t$0,(%0)\n\t"
+		"1:\tcache\t%3,(%0)\n\t"
+		"sd\t$0,(%0)\n\t"
 		"sd\t$0,8(%0)\n\t"
 		"sd\t$0,16(%0)\n\t"
 		"sd\t$0,24(%0)\n\t"
@@ -267,7 +334,44 @@ static void r4k_clear_page(unsigned long page)
 		".set\treorder"
 		:"=r" (page)
 		:"0" (page),
-		 "I" (PAGE_SIZE)
+		 "I" (PAGE_SIZE),
+		 "i" (Create_Dirty_Excl_SD)
+		:"$1","memory");
+}
+
+static void r4k_clear_page_s128(unsigned long page)
+{
+	__asm__ __volatile__(
+		".set\tnoreorder\n\t"
+		".set\tnoat\n\t"
+		".set\tmips3\n\t"
+		"daddiu\t$1,%0,%2\n"
+		"1:\tcache\t%3,(%0)\n\t"
+		"sd\t$0,(%0)\n\t"
+		"sd\t$0,8(%0)\n\t"
+		"sd\t$0,16(%0)\n\t"
+		"sd\t$0,24(%0)\n\t"
+		"sd\t$0,32(%0)\n\t"
+		"sd\t$0,40(%0)\n\t"
+		"sd\t$0,48(%0)\n\t"
+		"sd\t$0,56(%0)\n\t"
+		"daddiu\t%0,128\n\t"
+		"sd\t$0,-64(%0)\n\t"
+		"sd\t$0,-56(%0)\n\t"
+		"sd\t$0,-48(%0)\n\t"
+		"sd\t$0,-40(%0)\n\t"
+		"sd\t$0,-32(%0)\n\t"
+		"sd\t$0,-24(%0)\n\t"
+		"sd\t$0,-16(%0)\n\t"
+		"bne\t$1,%0,1b\n\t"
+		"sd\t$0,-8(%0)\n\t"
+		".set\tmips0\n\t"
+		".set\tat\n\t"
+		".set\treorder"
+		:"=r" (page)
+		:"0" (page),
+		 "I" (PAGE_SIZE),
+		 "i" (Create_Dirty_Excl_SD)
 		:"$1","memory");
 }
 
@@ -529,7 +633,10 @@ static void r4k_copy_page_r4600_v2(unsigned long to, unsigned long from)
 	restore_flags(flags);
 }
 
-static void r4k_copy_page(unsigned long to, unsigned long from)
+/*
+ * These are for R4000SC / R4400MC
+ */
+static void r4k_copy_page_s16(unsigned long to, unsigned long from)
 {
 	unsigned long dummy1, dummy2;
 	unsigned long reg1, reg2, reg3, reg4;
@@ -539,7 +646,8 @@ static void r4k_copy_page(unsigned long to, unsigned long from)
 		".set\tnoat\n\t"
 		".set\tmips3\n\t"
 		"daddiu\t$1,%0,%8\n"
-		"1:\tlw\t%2,(%1)\n\t"
+		"1:\tcache\t%9,(%0)\n\t"
+		"lw\t%2,(%1)\n\t"
 		"lw\t%3,4(%1)\n\t"
 		"lw\t%4,8(%1)\n\t"
 		"lw\t%5,12(%1)\n\t"
@@ -547,6 +655,7 @@ static void r4k_copy_page(unsigned long to, unsigned long from)
 		"sw\t%3,4(%0)\n\t"
 		"sw\t%4,8(%0)\n\t"
 		"sw\t%5,12(%0)\n\t"
+		"cache\t%9,16(%0)\n\t"
 		"lw\t%2,16(%1)\n\t"
 		"lw\t%3,20(%1)\n\t"
 		"lw\t%4,24(%1)\n\t"
@@ -555,6 +664,7 @@ static void r4k_copy_page(unsigned long to, unsigned long from)
 		"sw\t%3,20(%0)\n\t"
 		"sw\t%4,24(%0)\n\t"
 		"sw\t%5,28(%0)\n\t"
+		"cache\t%9,32(%0)\n\t"
 		"daddiu\t%0,64\n\t"
 		"daddiu\t%1,64\n\t"
 		"lw\t%2,-32(%1)\n\t"
@@ -565,6 +675,208 @@ static void r4k_copy_page(unsigned long to, unsigned long from)
 		"sw\t%3,-28(%0)\n\t"
 		"sw\t%4,-24(%0)\n\t"
 		"sw\t%5,-20(%0)\n\t"
+		"cache\t%9,-16(%0)\n\t"
+		"lw\t%2,-16(%1)\n\t"
+		"lw\t%3,-12(%1)\n\t"
+		"lw\t%4,-8(%1)\n\t"
+		"lw\t%5,-4(%1)\n\t"
+		"sw\t%2,-16(%0)\n\t"
+		"sw\t%3,-12(%0)\n\t"
+		"sw\t%4,-8(%0)\n\t"
+		"bne\t$1,%0,1b\n\t"
+		"sw\t%5,-4(%0)\n\t"
+		".set\tmips0\n\t"
+		".set\tat\n\t"
+		".set\treorder"
+		:"=r" (dummy1), "=r" (dummy2),
+		 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
+		:"0" (to), "1" (from),
+		 "I" (PAGE_SIZE),
+		 "i" (Create_Dirty_Excl_SD));
+}
+
+static void r4k_copy_page_s32(unsigned long to, unsigned long from)
+{
+	unsigned long dummy1, dummy2;
+	unsigned long reg1, reg2, reg3, reg4;
+
+	__asm__ __volatile__(
+		".set\tnoreorder\n\t"
+		".set\tnoat\n\t"
+		".set\tmips3\n\t"
+		"daddiu\t$1,%0,%8\n"
+		"1:\tcache\t%9,(%0)\n\t"
+		"lw\t%2,(%1)\n\t"
+		"lw\t%3,4(%1)\n\t"
+		"lw\t%4,8(%1)\n\t"
+		"lw\t%5,12(%1)\n\t"
+		"sw\t%2,(%0)\n\t"
+		"sw\t%3,4(%0)\n\t"
+		"sw\t%4,8(%0)\n\t"
+		"sw\t%5,12(%0)\n\t"
+		"lw\t%2,16(%1)\n\t"
+		"lw\t%3,20(%1)\n\t"
+		"lw\t%4,24(%1)\n\t"
+		"lw\t%5,28(%1)\n\t"
+		"sw\t%2,16(%0)\n\t"
+		"sw\t%3,20(%0)\n\t"
+		"sw\t%4,24(%0)\n\t"
+		"sw\t%5,28(%0)\n\t"
+		"cache\t%9,32(%0)\n\t"
+		"daddiu\t%0,64\n\t"
+		"daddiu\t%1,64\n\t"
+		"lw\t%2,-32(%1)\n\t"
+		"lw\t%3,-28(%1)\n\t"
+		"lw\t%4,-24(%1)\n\t"
+		"lw\t%5,-20(%1)\n\t"
+		"sw\t%2,-32(%0)\n\t"
+		"sw\t%3,-28(%0)\n\t"
+		"sw\t%4,-24(%0)\n\t"
+		"sw\t%5,-20(%0)\n\t"
+		"lw\t%2,-16(%1)\n\t"
+		"lw\t%3,-12(%1)\n\t"
+		"lw\t%4,-8(%1)\n\t"
+		"lw\t%5,-4(%1)\n\t"
+		"sw\t%2,-16(%0)\n\t"
+		"sw\t%3,-12(%0)\n\t"
+		"sw\t%4,-8(%0)\n\t"
+		"bne\t$1,%0,1b\n\t"
+		"sw\t%5,-4(%0)\n\t"
+		".set\tmips0\n\t"
+		".set\tat\n\t"
+		".set\treorder"
+		:"=r" (dummy1), "=r" (dummy2),
+		 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
+		:"0" (to), "1" (from),
+		 "I" (PAGE_SIZE),
+		 "i" (Create_Dirty_Excl_SD));
+}
+
+static void r4k_copy_page_s64(unsigned long to, unsigned long from)
+{
+	unsigned long dummy1, dummy2;
+	unsigned long reg1, reg2, reg3, reg4;
+
+	__asm__ __volatile__(
+		".set\tnoreorder\n\t"
+		".set\tnoat\n\t"
+		".set\tmips3\n\t"
+		"daddiu\t$1,%0,%8\n"
+		"1:\tcache\t%9,(%0)\n\t"
+		"lw\t%2,(%1)\n\t"
+		"lw\t%3,4(%1)\n\t"
+		"lw\t%4,8(%1)\n\t"
+		"lw\t%5,12(%1)\n\t"
+		"sw\t%2,(%0)\n\t"
+		"sw\t%3,4(%0)\n\t"
+		"sw\t%4,8(%0)\n\t"
+		"sw\t%5,12(%0)\n\t"
+		"lw\t%2,16(%1)\n\t"
+		"lw\t%3,20(%1)\n\t"
+		"lw\t%4,24(%1)\n\t"
+		"lw\t%5,28(%1)\n\t"
+		"sw\t%2,16(%0)\n\t"
+		"sw\t%3,20(%0)\n\t"
+		"sw\t%4,24(%0)\n\t"
+		"sw\t%5,28(%0)\n\t"
+		"daddiu\t%0,64\n\t"
+		"daddiu\t%1,64\n\t"
+		"lw\t%2,-32(%1)\n\t"
+		"lw\t%3,-28(%1)\n\t"
+		"lw\t%4,-24(%1)\n\t"
+		"lw\t%5,-20(%1)\n\t"
+		"sw\t%2,-32(%0)\n\t"
+		"sw\t%3,-28(%0)\n\t"
+		"sw\t%4,-24(%0)\n\t"
+		"sw\t%5,-20(%0)\n\t"
+		"lw\t%2,-16(%1)\n\t"
+		"lw\t%3,-12(%1)\n\t"
+		"lw\t%4,-8(%1)\n\t"
+		"lw\t%5,-4(%1)\n\t"
+		"sw\t%2,-16(%0)\n\t"
+		"sw\t%3,-12(%0)\n\t"
+		"sw\t%4,-8(%0)\n\t"
+		"bne\t$1,%0,1b\n\t"
+		"sw\t%5,-4(%0)\n\t"
+		".set\tmips0\n\t"
+		".set\tat\n\t"
+		".set\treorder"
+		:"=r" (dummy1), "=r" (dummy2),
+		 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
+		:"0" (to), "1" (from),
+		 "I" (PAGE_SIZE),
+		 "i" (Create_Dirty_Excl_SD));
+}
+
+static void r4k_copy_page_s128(unsigned long to, unsigned long from)
+{
+	unsigned long dummy1, dummy2;
+	unsigned long reg1, reg2, reg3, reg4;
+
+	__asm__ __volatile__(
+		".set\tnoreorder\n\t"
+		".set\tnoat\n\t"
+		".set\tmips3\n\t"
+		"daddiu\t$1,%0,%8\n"
+		"1:\tcache\t%9,(%0)\n\t"
+		"lw\t%2,(%1)\n\t"
+		"lw\t%3,4(%1)\n\t"
+		"lw\t%4,8(%1)\n\t"
+		"lw\t%5,12(%1)\n\t"
+		"sw\t%2,(%0)\n\t"
+		"sw\t%3,4(%0)\n\t"
+		"sw\t%4,8(%0)\n\t"
+		"sw\t%5,12(%0)\n\t"
+		"lw\t%2,16(%1)\n\t"
+		"lw\t%3,20(%1)\n\t"
+		"lw\t%4,24(%1)\n\t"
+		"lw\t%5,28(%1)\n\t"
+		"sw\t%2,16(%0)\n\t"
+		"sw\t%3,20(%0)\n\t"
+		"sw\t%4,24(%0)\n\t"
+		"sw\t%5,28(%0)\n\t"
+		"lw\t%2,32(%1)\n\t"
+		"lw\t%3,36(%1)\n\t"
+		"lw\t%4,40(%1)\n\t"
+		"lw\t%5,44(%1)\n\t"
+		"sw\t%2,32(%0)\n\t"
+		"sw\t%3,36(%0)\n\t"
+		"sw\t%4,40(%0)\n\t"
+		"sw\t%5,44(%0)\n\t"
+		"lw\t%2,48(%1)\n\t"
+		"lw\t%3,52(%1)\n\t"
+		"lw\t%4,56(%1)\n\t"
+		"lw\t%5,60(%1)\n\t"
+		"sw\t%2,48(%0)\n\t"
+		"sw\t%3,52(%0)\n\t"
+		"sw\t%4,56(%0)\n\t"
+		"sw\t%5,60(%0)\n\t"
+		"daddiu\t%0,128\n\t"
+		"daddiu\t%1,128\n\t"
+		"lw\t%2,-64(%1)\n\t"
+		"lw\t%3,-60(%1)\n\t"
+		"lw\t%4,-56(%1)\n\t"
+		"lw\t%5,-52(%1)\n\t"
+		"sw\t%2,-64(%0)\n\t"
+		"sw\t%3,-60(%0)\n\t"
+		"sw\t%4,-56(%0)\n\t"
+		"sw\t%5,-52(%0)\n\t"
+		"lw\t%2,-48(%1)\n\t"
+		"lw\t%3,-44(%1)\n\t"
+		"lw\t%4,-40(%1)\n\t"
+		"lw\t%5,-36(%1)\n\t"
+		"sw\t%2,-48(%0)\n\t"
+		"sw\t%3,-44(%0)\n\t"
+		"sw\t%4,-40(%0)\n\t"
+		"sw\t%5,-36(%0)\n\t"
+		"lw\t%2,-32(%1)\n\t"
+		"lw\t%3,-28(%1)\n\t"
+		"lw\t%4,-24(%1)\n\t"
+		"lw\t%5,-20(%1)\n\t"
+		"sw\t%2,-32(%0)\n\t"
+		"sw\t%3,-28(%0)\n\t"
+		"sw\t%4,-24(%0)\n\t"
+		"sw\t%5,-20(%0)\n\t"
 		"lw\t%2,-16(%1)\n\t"
 		"lw\t%3,-12(%1)\n\t"
 		"lw\t%4,-8(%1)\n\t"
@@ -580,9 +892,11 @@ static void r4k_copy_page(unsigned long to, unsigned long from)
 		:"=r" (dummy1), "=r" (dummy2),
 		 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
 		:"0" (to), "1" (from),
-		 "I" (PAGE_SIZE));
+		 "I" (PAGE_SIZE),
+		 "i" (Create_Dirty_Excl_SD));
 }
 
+
 /*
  * If you think for one second that this stuff coming up is a lot
  * of bulky code eating too many kernel cache lines.  Think _again_.
@@ -632,15 +946,6 @@ static inline void r4k_flush_cache_all_s128d16i16(void)
 	restore_flags(flags);
 }
 
-static inline void r4k_flush_cache_all_s16d32i32(void)
-{
-	unsigned long flags;
-
-	save_and_cli(flags);
-	blast_dcache32(); blast_icache32(); blast_scache16();
-	restore_flags(flags);
-}
-
 static inline void r4k_flush_cache_all_s32d32i32(void)
 {
 	unsigned long flags;
@@ -718,12 +1023,8 @@ r4k_flush_cache_range_s16d16i16(struct mm_struct *mm,
 				pmd = pmd_offset(pgd, start);
 				pte = pte_offset(pmd, start);
 
-				if(pte_val(*pte) & _PAGE_VALID) {
-					blast_dcache16_page(start);
-					if(text)
-						blast_icache16_page(start);
+				if(pte_val(*pte) & _PAGE_VALID)
 					blast_scache16_page(start);
-				}
 				start += PAGE_SIZE;
 			}
 			restore_flags(flags);
@@ -763,12 +1064,8 @@ r4k_flush_cache_range_s32d16i16(struct mm_struct *mm,
 				pmd = pmd_offset(pgd, start);
 				pte = pte_offset(pmd, start);
 
-				if(pte_val(*pte) & _PAGE_VALID) {
-					blast_dcache16_page(start);
-					if(text)
-						blast_icache16_page(start);
+				if(pte_val(*pte) & _PAGE_VALID)
 					blast_scache32_page(start);
-				}
 				start += PAGE_SIZE;
 			}
 			restore_flags(flags);
@@ -807,12 +1104,8 @@ static void r4k_flush_cache_range_s64d16i16(struct mm_struct *mm,
 				pmd = pmd_offset(pgd, start);
 				pte = pte_offset(pmd, start);
 
-				if(pte_val(*pte) & _PAGE_VALID) {
-					blast_dcache16_page(start);
-					if(text)
-						blast_icache16_page(start);
+				if(pte_val(*pte) & _PAGE_VALID)
 					blast_scache64_page(start);
-				}
 				start += PAGE_SIZE;
 			}
 			restore_flags(flags);
@@ -851,56 +1144,8 @@ static void r4k_flush_cache_range_s128d16i16(struct mm_struct *mm,
 				pmd = pmd_offset(pgd, start);
 				pte = pte_offset(pmd, start);
 
-				if(pte_val(*pte) & _PAGE_VALID) {
-					blast_dcache16_page(start);
-					if(text)
-						blast_icache16_page(start);
+				if(pte_val(*pte) & _PAGE_VALID)
 					blast_scache128_page(start);
-				}
-				start += PAGE_SIZE;
-			}
-			restore_flags(flags);
-		}
-	}
-}
-
-static void r4k_flush_cache_range_s16d32i32(struct mm_struct *mm,
-					    unsigned long start,
-					    unsigned long end)
-{
-	struct vm_area_struct *vma;
-	unsigned long flags;
-
-	if(mm->context == 0)
-		return;
-
-	start &= PAGE_MASK;
-#ifdef DEBUG_CACHE
-	printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
-#endif
-	vma = find_vma(mm, start);
-	if(vma) {
-		if(mm->context != current->mm->context) {
-			r4k_flush_cache_all_s16d32i32();
-		} else {
-			pgd_t *pgd;
-			pmd_t *pmd;
-			pte_t *pte;
-			int text;
-
-			save_and_cli(flags);
-			text = vma->vm_flags & VM_EXEC;
-			while(start < end) {
-				pgd = pgd_offset(mm, start);
-				pmd = pmd_offset(pgd, start);
-				pte = pte_offset(pmd, start);
-
-				if(pte_val(*pte) & _PAGE_VALID) {
-					blast_dcache32_page(start);
-					if(text)
-						blast_icache32_page(start);
-					blast_scache16_page(start);
-				}
 				start += PAGE_SIZE;
 			}
 			restore_flags(flags);
@@ -939,12 +1184,8 @@ static void r4k_flush_cache_range_s32d32i32(struct mm_struct *mm,
 				pmd = pmd_offset(pgd, start);
 				pte = pte_offset(pmd, start);
 
-				if(pte_val(*pte) & _PAGE_VALID) {
-					blast_dcache32_page(start);
-					if(text)
-						blast_icache32_page(start);
+				if(pte_val(*pte) & _PAGE_VALID)
 					blast_scache32_page(start);
-				}
 				start += PAGE_SIZE;
 			}
 			restore_flags(flags);
@@ -983,12 +1224,8 @@ static void r4k_flush_cache_range_s64d32i32(struct mm_struct *mm,
 				pmd = pmd_offset(pgd, start);
 				pte = pte_offset(pmd, start);
 
-				if(pte_val(*pte) & _PAGE_VALID) {
-					blast_dcache32_page(start);
-					if(text)
-						blast_icache32_page(start);
+				if(pte_val(*pte) & _PAGE_VALID)
 					blast_scache64_page(start);
-				}
 				start += PAGE_SIZE;
 			}
 			restore_flags(flags);
@@ -1027,12 +1264,8 @@ static void r4k_flush_cache_range_s128d32i32(struct mm_struct *mm,
 				pmd = pmd_offset(pgd, start);
 				pte = pte_offset(pmd, start);
 
-				if(pte_val(*pte) & _PAGE_VALID) {
-					blast_dcache32_page(start);
-					if(text)
-						blast_icache32_page(start);
+				if(pte_val(*pte) & _PAGE_VALID)
 					blast_scache128_page(start);
-				}
 				start += PAGE_SIZE;
 			}
 			restore_flags(flags);
@@ -1117,16 +1350,6 @@ static void r4k_flush_cache_mm_s128d16i16(struct mm_struct *mm)
 	}
 }
 
-static void r4k_flush_cache_mm_s16d32i32(struct mm_struct *mm)
-{
-	if(mm->context != 0) {
-#ifdef DEBUG_CACHE
-		printk("cmm[%d]", (int)mm->context);
-#endif
-		r4k_flush_cache_all_s16d32i32();
-	}
-}
-
 static void r4k_flush_cache_mm_s32d32i32(struct mm_struct *mm)
 {
 	if(mm->context != 0) {
@@ -1225,12 +1448,8 @@ static void r4k_flush_cache_page_s16d16i16(struct vm_area_struct *vma,
 		if(text)
 			blast_icache16_page_indexed(page);
 		blast_scache16_page_indexed(page);
-	} else {
-		blast_dcache16_page(page);
-		if(text)
-			blast_icache16_page(page);
+	} else
 		blast_scache16_page(page);
-	}
 out:
 	restore_flags(flags);
 }
@@ -1282,12 +1501,8 @@ static void r4k_flush_cache_page_s32d16i16(struct vm_area_struct *vma,
 		if(text)
 			blast_icache16_page_indexed(page);
 		blast_scache32_page_indexed(page);
-	} else {
-		blast_dcache16_page(page);
-		if(text)
-			blast_icache16_page(page);
+	} else
 		blast_scache32_page(page);
-	}
 out:
 	restore_flags(flags);
 }
@@ -1340,12 +1555,8 @@ static void r4k_flush_cache_page_s64d16i16(struct vm_area_struct *vma,
 		if(text)
 			blast_icache16_page_indexed(page);
 		blast_scache64_page_indexed(page);
-	} else {
-		blast_dcache16_page(page);
-		if(text)
-			blast_icache16_page(page);
+	} else
 		blast_scache64_page(page);
-	}
 out:
 	restore_flags(flags);
 }
@@ -1399,70 +1610,8 @@ static void r4k_flush_cache_page_s128d16i16(struct vm_area_struct *vma,
 		if(text)
 			blast_icache16_page_indexed(page);
 		blast_scache128_page_indexed(page);
-	} else {
-		blast_dcache16_page(page);
-		if(text)
-			blast_icache16_page(page);
+	} else
 		blast_scache128_page(page);
-	}
-out:
-	restore_flags(flags);
-}
-
-static void r4k_flush_cache_page_s16d32i32(struct vm_area_struct *vma,
-					   unsigned long page)
-{
-	struct mm_struct *mm = vma->vm_mm;
-	unsigned long flags;
-	pgd_t *pgdp;
-	pmd_t *pmdp;
-	pte_t *ptep;
-	int text;
-
-	/*
-	 * If ownes no valid ASID yet, cannot possibly have gotten
-	 * this page into the cache.
-	 */
-	if(mm->context == 0)
-		return;
-
-#ifdef DEBUG_CACHE
-	printk("cpage[%d,%08lx]", (int)mm->context, page);
-#endif
-	save_and_cli(flags);
-	page &= PAGE_MASK;
-	pgdp = pgd_offset(mm, page);
-	pmdp = pmd_offset(pgdp, page);
-	ptep = pte_offset(pmdp, page);
-
-	/* If the page isn't marked valid, the page cannot possibly be
-	 * in the cache.
-	 */
-	if(!(pte_val(*ptep) & _PAGE_VALID))
-		goto out;
-
-	text = (vma->vm_flags & VM_EXEC);
-	/*
-	 * Doing flushes for another ASID than the current one is
-	 * too difficult since stupid R4k caches do a TLB translation
-	 * for every cache flush operation.  So we do indexed flushes
-	 * in that case, which doesn't overly flush the cache too much.
-	 */
-	if(mm->context != current->mm->context) {
-		/* Do indexed flush, too much work to get the (possible)
-		 * tlb refills to work correctly.
-		 */
-		page = (KSEG0 + (page & (scache_size - 1)));
-		blast_dcache32_page_indexed(page);
-		if(text)
-			blast_icache32_page_indexed(page);
-		blast_scache16_page_indexed(page);
-	} else {
-		blast_dcache32_page(page);
-		if(text)
-			blast_icache32_page(page);
-		blast_scache16_page(page);
-	}
 out:
 	restore_flags(flags);
 }
@@ -1517,12 +1666,8 @@ static void r4k_flush_cache_page_s32d32i32(struct vm_area_struct *vma,
 		if(text)
 			blast_icache32_page_indexed(page);
 		blast_scache32_page_indexed(page);
-	} else {
-		blast_dcache32_page(page);
-		if(text)
-			blast_icache32_page(page);
+	} else
 		blast_scache32_page(page);
-	}
 out:
 	restore_flags(flags);
 }
@@ -1577,12 +1722,8 @@ static void r4k_flush_cache_page_s64d32i32(struct vm_area_struct *vma,
 		if(text)
 			blast_icache32_page_indexed(page);
 		blast_scache64_page_indexed(page);
-	} else {
-		blast_dcache32_page(page);
-		if(text)
-			blast_icache32_page(page);
+	} else
 		blast_scache64_page(page);
-	}
 out:
 	restore_flags(flags);
 }
@@ -1635,12 +1776,8 @@ static void r4k_flush_cache_page_s128d32i32(struct vm_area_struct *vma,
 		if(text)
 			blast_icache32_page_indexed(page);
 		blast_scache128_page_indexed(page);
-	} else {
-		blast_dcache32_page(page);
-		if(text)
-			blast_icache32_page(page);
+	} else
 		blast_scache128_page(page);
-	}
 out:
 	restore_flags(flags);
 }
@@ -1832,15 +1969,10 @@ static void r4k_flush_page_to_ram_s16d16i16(unsigned long page)
 {
 	page &= PAGE_MASK;
 	if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) {
-		unsigned long flags;
-
 #ifdef DEBUG_CACHE
 		printk("cram[%08lx]", page);
 #endif
-		save_and_cli(flags);
-		blast_dcache16_page(page);
 		blast_scache16_page(page);
-		restore_flags(flags);
 	}
 }
 
@@ -1848,15 +1980,10 @@ static void r4k_flush_page_to_ram_s32d16i16(unsigned long page)
 {
 	page &= PAGE_MASK;
 	if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) {
-		unsigned long flags;
-
 #ifdef DEBUG_CACHE
 		printk("cram[%08lx]", page);
 #endif
-		save_and_cli(flags);
-		blast_dcache16_page(page);
 		blast_scache32_page(page);
-		restore_flags(flags);
 	}
 }
 
@@ -1864,15 +1991,10 @@ static void r4k_flush_page_to_ram_s64d16i16(unsigned long page)
 {
 	page &= PAGE_MASK;
 	if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) {
-		unsigned long flags;
-
 #ifdef DEBUG_CACHE
 		printk("cram[%08lx]", page);
 #endif
-		save_and_cli(flags);
-		blast_dcache16_page(page);
 		blast_scache64_page(page);
-		restore_flags(flags);
 	}
 }
 
@@ -1880,31 +2002,10 @@ static void r4k_flush_page_to_ram_s128d16i16(unsigned long page)
 {
 	page &= PAGE_MASK;
 	if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) {
-		unsigned long flags;
-
 #ifdef DEBUG_CACHE
 		printk("cram[%08lx]", page);
 #endif
-		save_and_cli(flags);
-		blast_dcache16_page(page);
 		blast_scache128_page(page);
-		restore_flags(flags);
-	}
-}
-
-static void r4k_flush_page_to_ram_s16d32i32(unsigned long page)
-{
-	page &= PAGE_MASK;
-	if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) {
-		unsigned long flags;
-
-#ifdef DEBUG_CACHE
-		printk("cram[%08lx]", page);
-#endif
-		save_and_cli(flags);
-		blast_dcache32_page(page);
-		blast_scache16_page(page);
-		restore_flags(flags);
 	}
 }
 
@@ -1912,15 +2013,10 @@ static void r4k_flush_page_to_ram_s32d32i32(unsigned long page)
 {
 	page &= PAGE_MASK;
 	if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) {
-		unsigned long flags;
-
 #ifdef DEBUG_CACHE
 		printk("cram[%08lx]", page);
 #endif
-		save_and_cli(flags);
-		blast_dcache32_page(page);
 		blast_scache32_page(page);
-		restore_flags(flags);
 	}
 }
 
@@ -1928,15 +2024,10 @@ static void r4k_flush_page_to_ram_s64d32i32(unsigned long page)
 {
 	page &= PAGE_MASK;
 	if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) {
-		unsigned long flags;
-
 #ifdef DEBUG_CACHE
 		printk("cram[%08lx]", page);
 #endif
-		save_and_cli(flags);
-		blast_dcache32_page(page);
 		blast_scache64_page(page);
-		restore_flags(flags);
 	}
 }
 
@@ -1944,15 +2035,10 @@ static void r4k_flush_page_to_ram_s128d32i32(unsigned long page)
 {
 	page &= PAGE_MASK;
 	if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) {
-		unsigned long flags;
-
 #ifdef DEBUG_CACHE
 		printk("cram[%08lx]", page);
 #endif
-		save_and_cli(flags);
-		blast_dcache32_page(page);
 		blast_scache128_page(page);
-		restore_flags(flags);
 	}
 }
 
@@ -2026,20 +2112,10 @@ static void
 r4k_dma_cache_wback_inv_sc(unsigned long addr, unsigned long size)
 {
 	unsigned long end, a;
-	unsigned int flags;
 
 	if (size >= scache_size) {
 		flush_cache_all();
-	} else {
-		save_and_cli(flags);
-		a = addr & ~(dc_lsize - 1);
-		end = (addr + size) & ~(dc_lsize - 1);
-		while (1) {
-			flush_dcache_line(a); /* Hit_Writeback_Inv_D */
-			if (a == end) break;
-			a += dc_lsize;
-		}
-		restore_flags(flags);
+		return;
 	}
 
 	a = addr & ~(sc_lsize - 1);
@@ -2081,20 +2157,10 @@ static void
 r4k_dma_cache_inv_sc(unsigned long addr, unsigned long size)
 {
 	unsigned long end, a;
-	unsigned int flags;
 
 	if (size >= scache_size) {
 		flush_cache_all();
-	} else {
-		save_and_cli(flags);
-		a = addr & ~(dc_lsize - 1);
-		end = (addr + size) & ~(dc_lsize - 1);
-		while (1) {
-			flush_dcache_line(a); /* Hit_Writeback_Inv_D */
-			if (a == end) break;
-			a += dc_lsize;
-		}
-		restore_flags(flags);
+		return;
 	}
 
 	a = addr & ~(sc_lsize - 1);
@@ -2546,8 +2612,8 @@ __initfunc(static int probe_scache(unsigned long config))
 	}
 	restore_flags(flags);
 	addr -= begin;
-	printk("Secondary cache sized at %dK linesize %d\n", (int) (addr >> 10),
-	       sc_lsize);
+	printk("Secondary cache sized at %dK linesize %d\n",
+	       (int) (addr >> 10), sc_lsize);
 	scache_size = addr;
 	return 1;
 }
@@ -2602,13 +2668,10 @@ __initfunc(static void setup_scache_funcs(void))
 			flush_page_to_ram = r4k_flush_page_to_ram_s16d16i16;
 			break;
 		case 32:
-			flush_cache_all = r4k_flush_cache_all_s16d32i32;
-			flush_cache_mm = r4k_flush_cache_mm_s16d32i32;
-			flush_cache_range = r4k_flush_cache_range_s16d32i32;
-			flush_cache_page = r4k_flush_cache_page_s16d32i32;
-			flush_page_to_ram = r4k_flush_page_to_ram_s16d32i32;
-			break;
+			panic("Invalid cache configuration detected");
 		};
+		clear_page = r4k_clear_page_s16;
+		copy_page = r4k_copy_page_s16;
 		break;
 	case 32:
 		switch(dc_lsize) {
@@ -2627,6 +2690,9 @@ __initfunc(static void setup_scache_funcs(void))
 			flush_page_to_ram = r4k_flush_page_to_ram_s32d32i32;
 			break;
 		};
+		clear_page = r4k_clear_page_s32;
+		copy_page = r4k_copy_page_s32;
+		break;
 	case 64:
 		switch(dc_lsize) {
 		case 16:
@@ -2644,6 +2710,9 @@ __initfunc(static void setup_scache_funcs(void))
 			flush_page_to_ram = r4k_flush_page_to_ram_s64d32i32;
 			break;
 		};
+		clear_page = r4k_clear_page_s64;
+		copy_page = r4k_copy_page_s64;
+		break;
 	case 128:
 		switch(dc_lsize) {
 		case 16:
@@ -2661,10 +2730,10 @@ __initfunc(static void setup_scache_funcs(void))
 			flush_page_to_ram = r4k_flush_page_to_ram_s128d32i32;
 			break;
 		};
+		clear_page = r4k_clear_page_s128;
+		copy_page = r4k_copy_page_s128;
 		break;
 	}
-	clear_page = r4k_clear_page;
-	copy_page = r4k_copy_page;
 	dma_cache_wback_inv = r4k_dma_cache_wback_inv_sc;
 	dma_cache_inv = r4k_dma_cache_inv_sc;
 }