diff options
author | Ralf Baechle <ralf@linux-mips.org> | 2000-07-31 03:39:50 +0000 |
---|---|---|
committer | Ralf Baechle <ralf@linux-mips.org> | 2000-07-31 03:39:50 +0000 |
commit | 17d285e537c498cf18ce79a52ca553ea964f389b (patch) | |
tree | 3905f77168a7a6d92c68bbd47b5ddae2664fe203 | |
parent | ce36512304b7cb5e1a911829b249e403261263ac (diff) |
Shave of 50% of lat_mmap. Our cache routines were plain stupid.
-rw-r--r-- | arch/mips/mm/andes.c | 7 | ||||
-rw-r--r-- | arch/mips/mm/loadmmu.c | 2 | ||||
-rw-r--r-- | arch/mips/mm/r2300.c | 21 | ||||
-rw-r--r-- | arch/mips/mm/r4xx0.c | 116 | ||||
-rw-r--r-- | arch/mips64/mm/andes.c | 2 | ||||
-rw-r--r-- | include/asm-mips/pgtable.h | 10 | ||||
-rw-r--r-- | include/asm-mips64/pgtable.h | 8 | ||||
-rw-r--r-- | kernel/ptrace.c | 2 | ||||
-rw-r--r-- | mm/memory.c | 4 |
9 files changed, 95 insertions, 77 deletions
diff --git a/arch/mips/mm/andes.c b/arch/mips/mm/andes.c index 631d6f002..5e5cb4768 100644 --- a/arch/mips/mm/andes.c +++ b/arch/mips/mm/andes.c @@ -119,6 +119,12 @@ static void andes_flush_page_to_ram(struct page * page) /* XXX */ } +static void andes_flush_icache_page(struct vm_area_struct *vma, + struct page *page, unsigned long addr) +{ + /* XXX */ +} + static void andes_flush_cache_sigtramp(unsigned long page) { /* XXX */ @@ -171,6 +177,7 @@ void __init ld_mmu_andes(void) _flush_cache_page = andes_flush_cache_page; _flush_cache_sigtramp = andes_flush_cache_sigtramp; _flush_page_to_ram = andes_flush_page_to_ram; + _flush_icache_page = andes_flush_icache_page; flush_cache_all(); flush_tlb_all(); diff --git a/arch/mips/mm/loadmmu.c b/arch/mips/mm/loadmmu.c index 6a1ad94ae..69a5e2de5 100644 --- a/arch/mips/mm/loadmmu.c +++ b/arch/mips/mm/loadmmu.c @@ -28,6 +28,8 @@ void (*_flush_cache_range)(struct mm_struct *mm, unsigned long start, void (*_flush_cache_page)(struct vm_area_struct *vma, unsigned long page); void (*_flush_cache_sigtramp)(unsigned long addr); void (*_flush_page_to_ram)(struct page * page); +void (*_flush_icache_page)(struct vm_area_struct *vma, struct page *page, + unsigned long addr); /* DMA cache operations. */ void (*_dma_cache_wback_inv)(unsigned long start, unsigned long size); diff --git a/arch/mips/mm/r2300.c b/arch/mips/mm/r2300.c index 690e3698f..51bf703cd 100644 --- a/arch/mips/mm/r2300.c +++ b/arch/mips/mm/r2300.c @@ -360,6 +360,26 @@ static void r3k_flush_page_to_ram(struct page * page) */ } +static void r3k_flush_icache_page(struct vm_area_struct *vma, + unsigned long page, unsigned long address) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long physpage; + + if (mm->context == 0) + return; + + if (!(vma->vm_flags & VM_EXEC)) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + + if ((physpage = get_phys_page(page, vma->vm_mm))) + r3k_flush_icache_range(physpage, PAGE_SIZE); +} + static void r3k_flush_cache_sigtramp(unsigned long addr) { unsigned long flags; @@ -651,6 +671,7 @@ void __init ld_mmu_r2300(void) _flush_cache_page = r3k_flush_cache_page; _flush_cache_sigtramp = r3k_flush_cache_sigtramp; _flush_page_to_ram = r3k_flush_page_to_ram; + _flush_icache_page = r3k_flush_icache_page; _dma_cache_wback_inv = r3k_dma_cache_wback_inv; diff --git a/arch/mips/mm/r4xx0.c b/arch/mips/mm/r4xx0.c index 0e851688a..0446ec7b8 100644 --- a/arch/mips/mm/r4xx0.c +++ b/arch/mips/mm/r4xx0.c @@ -1390,7 +1390,6 @@ static void r4k_flush_cache_page_s16d16i16(struct vm_area_struct *vma, pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int text; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -1415,20 +1414,19 @@ static void r4k_flush_cache_page_s16d16i16(struct vm_area_struct *vma, if (!(pte_val(*ptep) & _PAGE_VALID)) goto out; - text = (vma->vm_flags & VM_EXEC); - /* Doing flushes for another ASID than the current one is + /* + * Doing flushes for another ASID than the current one is * too difficult since stupid R4k caches do a TLB translation * for every cache flush operation. So we do indexed flushes * in that case, which doesn't overly flush the cache too much. */ if (mm->context != current->active_mm->context) { - /* Do indexed flush, too much work to get the (possible) + /* + * Do indexed flush, too much work to get the (possible) * tlb refills to work correctly. */ page = (KSEG0 + (page & (scache_size - 1))); blast_dcache16_page_indexed(page); - if(text) - blast_icache16_page_indexed(page); blast_scache16_page_indexed(page); } else blast_scache16_page(page); @@ -1444,7 +1442,6 @@ static void r4k_flush_cache_page_s32d16i16(struct vm_area_struct *vma, pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int text; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -1468,20 +1465,19 @@ static void r4k_flush_cache_page_s32d16i16(struct vm_area_struct *vma, if (!(pte_val(*ptep) & _PAGE_VALID)) goto out; - text = (vma->vm_flags & VM_EXEC); - /* Doing flushes for another ASID than the current one is + /* + * Doing flushes for another ASID than the current one is * too difficult since stupid R4k caches do a TLB translation * for every cache flush operation. So we do indexed flushes * in that case, which doesn't overly flush the cache too much. */ if (mm->context != current->active_mm->context) { - /* Do indexed flush, too much work to get the (possible) + /* + * Do indexed flush, too much work to get the (possible) * tlb refills to work correctly. */ page = (KSEG0 + (page & (scache_size - 1))); blast_dcache16_page_indexed(page); - if(text) - blast_icache16_page_indexed(page); blast_scache32_page_indexed(page); } else blast_scache32_page(page); @@ -1497,7 +1493,6 @@ static void r4k_flush_cache_page_s64d16i16(struct vm_area_struct *vma, pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int text; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -1521,7 +1516,6 @@ static void r4k_flush_cache_page_s64d16i16(struct vm_area_struct *vma, if (!(pte_val(*ptep) & _PAGE_VALID)) goto out; - text = (vma->vm_flags & VM_EXEC); /* * Doing flushes for another ASID than the current one is * too difficult since stupid R4k caches do a TLB translation @@ -1529,13 +1523,12 @@ static void r4k_flush_cache_page_s64d16i16(struct vm_area_struct *vma, * in that case, which doesn't overly flush the cache too much. */ if (mm->context != current->active_mm->context) { - /* Do indexed flush, too much work to get the (possible) + /* + * Do indexed flush, too much work to get the (possible) * tlb refills to work correctly. */ page = (KSEG0 + (page & (scache_size - 1))); blast_dcache16_page_indexed(page); - if(text) - blast_icache16_page_indexed(page); blast_scache64_page_indexed(page); } else blast_scache64_page(page); @@ -1551,7 +1544,6 @@ static void r4k_flush_cache_page_s128d16i16(struct vm_area_struct *vma, pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int text; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -1576,8 +1568,8 @@ static void r4k_flush_cache_page_s128d16i16(struct vm_area_struct *vma, if (!(pte_val(*ptep) & _PAGE_VALID)) goto out; - text = (vma->vm_flags & VM_EXEC); - /* Doing flushes for another ASID than the current one is + /* + * Doing flushes for another ASID than the current one is * too difficult since stupid R4k caches do a TLB translation * for every cache flush operation. So we do indexed flushes * in that case, which doesn't overly flush the cache too much. @@ -1589,8 +1581,6 @@ static void r4k_flush_cache_page_s128d16i16(struct vm_area_struct *vma, */ page = (KSEG0 + (page & (scache_size - 1))); blast_dcache16_page_indexed(page); - if(text) - blast_icache16_page_indexed(page); blast_scache128_page_indexed(page); } else blast_scache128_page(page); @@ -1606,7 +1596,6 @@ static void r4k_flush_cache_page_s32d32i32(struct vm_area_struct *vma, pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int text; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -1631,7 +1620,6 @@ static void r4k_flush_cache_page_s32d32i32(struct vm_area_struct *vma, if (!(pte_val(*ptep) & _PAGE_VALID)) goto out; - text = (vma->vm_flags & VM_EXEC); /* * Doing flushes for another ASID than the current one is * too difficult since stupid R4k caches do a TLB translation @@ -1645,8 +1633,6 @@ static void r4k_flush_cache_page_s32d32i32(struct vm_area_struct *vma, */ page = (KSEG0 + (page & (scache_size - 1))); blast_dcache32_page_indexed(page); - if(text) - blast_icache32_page_indexed(page); blast_scache32_page_indexed(page); } else blast_scache32_page(page); @@ -1662,7 +1648,6 @@ static void r4k_flush_cache_page_s64d32i32(struct vm_area_struct *vma, pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int text; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -1687,7 +1672,6 @@ static void r4k_flush_cache_page_s64d32i32(struct vm_area_struct *vma, if (!(pte_val(*ptep) & _PAGE_VALID)) goto out; - text = (vma->vm_flags & VM_EXEC); /* * Doing flushes for another ASID than the current one is * too difficult since stupid R4k caches do a TLB translation @@ -1701,8 +1685,6 @@ static void r4k_flush_cache_page_s64d32i32(struct vm_area_struct *vma, */ page = (KSEG0 + (page & (scache_size - 1))); blast_dcache32_page_indexed(page); - if(text) - blast_icache32_page_indexed(page); blast_scache64_page_indexed(page); } else blast_scache64_page(page); @@ -1718,7 +1700,6 @@ static void r4k_flush_cache_page_s128d32i32(struct vm_area_struct *vma, pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int text; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -1736,13 +1717,13 @@ static void r4k_flush_cache_page_s128d32i32(struct vm_area_struct *vma, pmdp = pmd_offset(pgdp, page); ptep = pte_offset(pmdp, page); - /* If the page isn't marked valid, the page cannot possibly be + /* + * If the page isn't marked valid, the page cannot possibly be * in the cache. */ if (!(pte_val(*ptep) & _PAGE_VALID)) goto out; - text = (vma->vm_flags & VM_EXEC); /* * Doing flushes for another ASID than the current one is * too difficult since stupid R4k caches do a TLB translation @@ -1755,8 +1736,6 @@ static void r4k_flush_cache_page_s128d32i32(struct vm_area_struct *vma, */ page = (KSEG0 + (page & (scache_size - 1))); blast_dcache32_page_indexed(page); - if(text) - blast_icache32_page_indexed(page); blast_scache128_page_indexed(page); } else blast_scache128_page(page); @@ -1772,7 +1751,6 @@ static void r4k_flush_cache_page_d16i16(struct vm_area_struct *vma, pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int text; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -1790,13 +1768,13 @@ static void r4k_flush_cache_page_d16i16(struct vm_area_struct *vma, pmdp = pmd_offset(pgdp, page); ptep = pte_offset(pmdp, page); - /* If the page isn't marked valid, the page cannot possibly be + /* + * If the page isn't marked valid, the page cannot possibly be * in the cache. */ if (!(pte_val(*ptep) & _PAGE_VALID)) goto out; - text = (vma->vm_flags & VM_EXEC); /* * Doing flushes for another ASID than the current one is * too difficult since stupid R4k caches do a TLB translation @@ -1805,16 +1783,12 @@ static void r4k_flush_cache_page_d16i16(struct vm_area_struct *vma, */ if (mm == current->active_mm) { blast_dcache16_page(page); - if(text) - blast_icache16_page(page); } else { /* Do indexed flush, too much work to get the (possible) * tlb refills to work correctly. */ page = (KSEG0 + (page & (dcache_size - 1))); blast_dcache16_page_indexed(page); - if(text) - blast_icache16_page_indexed(page); } out: restore_flags(flags); @@ -1828,7 +1802,6 @@ static void r4k_flush_cache_page_d32i32(struct vm_area_struct *vma, pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int text; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -1853,7 +1826,6 @@ static void r4k_flush_cache_page_d32i32(struct vm_area_struct *vma, if (!(pte_val(*ptep) & _PAGE_PRESENT)) goto out; - text = (vma->vm_flags & VM_EXEC); /* * Doing flushes for another ASID than the current one is * too difficult since stupid R4k caches do a TLB translation @@ -1862,8 +1834,6 @@ static void r4k_flush_cache_page_d32i32(struct vm_area_struct *vma, */ if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) { blast_dcache32_page(page); - if(text) - blast_icache32_page(page); } else { /* * Do indexed flush, too much work to get the (possible) @@ -1871,8 +1841,6 @@ static void r4k_flush_cache_page_d32i32(struct vm_area_struct *vma, */ page = (KSEG0 + (page & (dcache_size - 1))); blast_dcache32_page_indexed(page); - if(text) - blast_icache32_page_indexed(page); } out: restore_flags(flags); @@ -1886,7 +1854,6 @@ static void r4k_flush_cache_page_d32i32_r4600(struct vm_area_struct *vma, pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int text; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -1911,7 +1878,6 @@ static void r4k_flush_cache_page_d32i32_r4600(struct vm_area_struct *vma, if (!(pte_val(*ptep) & _PAGE_PRESENT)) goto out; - text = (vma->vm_flags & VM_EXEC); /* * Doing flushes for another ASID than the current one is * too difficult since stupid R4k caches do a TLB translation @@ -1920,8 +1886,6 @@ static void r4k_flush_cache_page_d32i32_r4600(struct vm_area_struct *vma, */ if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) { blast_dcache32_page(page); - if(text) - blast_icache32_page(page); } else { /* Do indexed flush, too much work to get the (possible) * tlb refills to work correctly. @@ -1929,10 +1893,6 @@ static void r4k_flush_cache_page_d32i32_r4600(struct vm_area_struct *vma, page = (KSEG0 + (page & (dcache_size - 1))); blast_dcache32_page_indexed(page); blast_dcache32_page_indexed(page ^ dcache_waybit); - if(text) { - blast_icache32_page_indexed(page); - blast_icache32_page_indexed(page ^ icache_waybit); - } } out: restore_flags(flags); @@ -1986,6 +1946,35 @@ static void r4k_flush_page_to_ram_d32_r4600(struct page *page) __restore_flags(flags); } +static void +r4k_flush_icache_page_s(struct vm_area_struct *vma, struct page *page, + unsigned long address) +{ + /* + * We did an scache flush therefore PI is already clean. + */ +} + +static void +r4k_flush_icache_page_i16(struct vm_area_struct *vma, struct page *page, + unsigned long address) +{ + if (!(vma->vm_flags & VM_EXEC)) + return; + + blast_icache16_page(address); +} + +static void +r4k_flush_icache_page_i32(struct vm_area_struct *vma, struct page *page, + unsigned long address) +{ + if (!(vma->vm_flags & VM_EXEC)) + return; + + blast_icache32_page(address); +} + /* * Writeback and invalidate the primary cache dcache before DMA. * @@ -2289,12 +2278,6 @@ void pgd_init(unsigned long page) } } -#ifdef DEBUG_TLBUPDATE -static unsigned long ehi_debug[NTLB_ENTRIES]; -static unsigned long el0_debug[NTLB_ENTRIES]; -static unsigned long el1_debug[NTLB_ENTRIES]; -#endif - /* We will need multiple versions of update_mmu_cache(), one that just * updates the TLB with the new pte(s), and another which also checks * for the R4k "end of page" hardware bug and does the needy. @@ -2579,6 +2562,14 @@ static void __init setup_noscache_funcs(void) _flush_cache_page = r4k_flush_cache_page_d32i32; break; } + + switch(ic_lsize) { + case 16: + _flush_icache_page = r4k_flush_icache_page_i16; + case 32: + _flush_icache_page = r4k_flush_icache_page_i32; + } + _dma_cache_wback_inv = r4k_dma_cache_wback_inv_pc; _dma_cache_wback = r4k_dma_cache_wback; _dma_cache_inv = r4k_dma_cache_inv_pc; @@ -2660,6 +2651,7 @@ static void __init setup_scache_funcs(void) _copy_page = r4k_copy_page_s128; break; } + _flush_icache_page = r4k_flush_icache_page_s; _dma_cache_wback_inv = r4k_dma_cache_wback_inv_sc; _dma_cache_wback = r4k_dma_cache_wback; _dma_cache_inv = r4k_dma_cache_inv_sc; diff --git a/arch/mips64/mm/andes.c b/arch/mips64/mm/andes.c index bcbb07760..43edfe728 100644 --- a/arch/mips64/mm/andes.c +++ b/arch/mips64/mm/andes.c @@ -119,7 +119,7 @@ andes_flush_cache_l2(void) } void -andes_flush_icache_page(unsigned long page) +andes_flush_icache_page(unsigned long page, unsigned long addr) { if (scache_lsz64) blast_scache64_page(page); diff --git a/include/asm-mips/pgtable.h b/include/asm-mips/pgtable.h index f72b7ddd3..71defadb8 100644 --- a/include/asm-mips/pgtable.h +++ b/include/asm-mips/pgtable.h @@ -33,6 +33,8 @@ extern void (*_flush_cache_range)(struct mm_struct *mm, unsigned long start, extern void (*_flush_cache_page)(struct vm_area_struct *vma, unsigned long page); extern void (*_flush_cache_sigtramp)(unsigned long addr); extern void (*_flush_page_to_ram)(struct page * page); +extern void (*_flush_icache_page)(struct vm_area_struct *vma, + struct page *page, unsigned long addr); #define flush_cache_all() _flush_cache_all() #define flush_cache_mm(mm) _flush_cache_mm(mm) @@ -42,13 +44,7 @@ extern void (*_flush_page_to_ram)(struct page * page); #define flush_page_to_ram(page) _flush_page_to_ram(page) #define flush_icache_range(start, end) flush_cache_all() - -#define flush_icache_page(vma, page) \ -do { \ - unsigned long addr; \ - addr = page_address(page); \ - _flush_cache_page(vma, addr); \ -} while (0) +#define flush_icache_page(vma, page, addr) _flush_icache_page(vma, page, addr) /* diff --git a/include/asm-mips64/pgtable.h b/include/asm-mips64/pgtable.h index c40380569..94ef5eee8 100644 --- a/include/asm-mips64/pgtable.h +++ b/include/asm-mips64/pgtable.h @@ -43,7 +43,7 @@ extern void (*_flush_page_to_ram)(struct page * page); #define flush_icache_range(start, end) _flush_cache_l1() -#define flush_icache_page(vma, page) \ +#define flush_icache_page(vma, page, address) \ do { \ unsigned long addr; \ addr = page_address(page); \ @@ -59,16 +59,16 @@ do { \ * are io coherent. The only place where we might be overoptimizing * out icache flushes are from mprotect (when PROT_EXEC is added). */ -extern void andes_flush_icache_page(unsigned long); +extern void andes_flush_icache_page(unsigned long, unsigned long); #define flush_cache_mm(mm) do { } while(0) #define flush_cache_range(mm,start,end) do { } while(0) #define flush_cache_page(vma,page) do { } while(0) #define flush_page_to_ram(page) do { } while(0) #define flush_icache_range(start, end) _flush_cache_l1() -#define flush_icache_page(vma, page) \ +#define flush_icache_page(vma, page, address) \ do { \ if ((vma)->vm_flags & VM_EXEC) \ - andes_flush_icache_page(page_address(page)); \ + andes_flush_icache_page(page_address(page), address); \ } while (0) #endif /* !CONFIG_CPU_R10000 */ diff --git a/kernel/ptrace.c b/kernel/ptrace.c index e009bca35..6c5d1f143 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -54,7 +54,7 @@ repeat: maddr = kmap(page); memcpy((char *)maddr + (addr & ~PAGE_MASK), buf, len); flush_page_to_ram(page); - flush_icache_page(vma, page); + flush_icache_page(vma, page, addr); kunmap(page); } else { maddr = kmap(page); diff --git a/mm/memory.c b/mm/memory.c index 7b54fc585..0d93216b5 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1063,7 +1063,7 @@ static int do_swap_page(struct mm_struct * mm, return -1; flush_page_to_ram(page); - flush_icache_page(vma, page); + flush_icache_page(vma, page, address); } mm->rss++; @@ -1159,7 +1159,7 @@ static int do_no_page(struct mm_struct * mm, struct vm_area_struct * vma, * handle that later. */ flush_page_to_ram(new_page); - flush_icache_page(vma, new_page); + flush_icache_page(vma, new_page, address); entry = mk_pte(new_page, vma->vm_page_prot); if (write_access) { entry = pte_mkwrite(pte_mkdirty(entry)); |