From 794ff4dc959a8dfef4f704ef6a5bb7334e421623 Mon Sep 17 00:00:00 2001 From: Ralf Baechle Date: Wed, 18 Aug 1999 23:37:42 +0000 Subject: - Drop support for 32/32 fp register model - Fix siginfo structure definition - Ptrace fixes, one of them was an uninitialized pointer. - Header file fixes. - Fixes to linker scripts required for new binutils. - MIPS64. Chainsaw edition. Btw, Harald, why does the kernel no longer compile ... --- arch/mips64/mm/.cvsignore | 2 + arch/mips64/mm/Makefile | 17 + arch/mips64/mm/andes.c | 134 +++ arch/mips64/mm/extable.c | 60 + arch/mips64/mm/fault.c | 165 +++ arch/mips64/mm/init.c | 421 +++++++ arch/mips64/mm/loadmmu.c | 109 ++ arch/mips64/mm/r4xx0.c | 2821 +++++++++++++++++++++++++++++++++++++++++++++ arch/mips64/mm/tfp.c | 120 ++ arch/mips64/mm/umap.c | 211 ++++ 10 files changed, 4060 insertions(+) create mode 100644 arch/mips64/mm/.cvsignore create mode 100644 arch/mips64/mm/Makefile create mode 100644 arch/mips64/mm/andes.c create mode 100644 arch/mips64/mm/extable.c create mode 100644 arch/mips64/mm/fault.c create mode 100644 arch/mips64/mm/init.c create mode 100644 arch/mips64/mm/loadmmu.c create mode 100644 arch/mips64/mm/r4xx0.c create mode 100644 arch/mips64/mm/tfp.c create mode 100644 arch/mips64/mm/umap.c (limited to 'arch/mips64/mm') diff --git a/arch/mips64/mm/.cvsignore b/arch/mips64/mm/.cvsignore new file mode 100644 index 000000000..857dd22e9 --- /dev/null +++ b/arch/mips64/mm/.cvsignore @@ -0,0 +1,2 @@ +.depend +.*.flags diff --git a/arch/mips64/mm/Makefile b/arch/mips64/mm/Makefile new file mode 100644 index 000000000..a5da59936 --- /dev/null +++ b/arch/mips64/mm/Makefile @@ -0,0 +1,17 @@ +# $Id$ +# +# Makefile for the Linux/MIPS-specific parts of the memory manager. +# + +O_TARGET := mm.o +O_OBJS := extable.o init.o fault.o r4xx0.o tfp.o andes.o loadmmu.o + +ifdef CONFIG_SGI +O_OBJS += umap.o +endif + +ifdef CONFIG_BAGET_MIPS +O_OBJS += umap.o +endif + +include $(TOPDIR)/Rules.make diff --git a/arch/mips64/mm/andes.c b/arch/mips64/mm/andes.c new file mode 100644 index 000000000..13be21622 --- /dev/null +++ b/arch/mips64/mm/andes.c @@ -0,0 +1,134 @@ +/* $Id$ + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) + * Copyright (C) 1997, 1998, 1999 Ralf Baechle (ralf@gnu.org) + * Copyright (C) 1999 Silicon Graphics, Inc. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern unsigned long mips_tlb_entries; + +/* Cache operations. XXX Write these dave... */ +static void +andes_flush_cache_all(void) +{ + /* XXX */ +} + +static void +andes_flush_cache_mm(struct mm_struct *mm) +{ + /* XXX */ +} + +static void +andes_flush_cache_range(struct mm_struct *mm, unsigned long start, + unsigned long end) +{ + /* XXX */ +} + +static void +andes_flush_cache_page(struct vm_area_struct *vma, unsigned long page) +{ + /* XXX */ +} + +static void +andes_flush_page_to_ram(unsigned long page) +{ + /* XXX */ +} + +static void +andes_flush_cache_sigtramp(unsigned long page) +{ + /* XXX */ +} + +/* TLB operations. XXX Write these dave... */ +static void +andes_flush_tlb_all(void) +{ + /* XXX */ +} + +static void +andes_flush_tlb_mm(struct mm_struct *mm) +{ + /* XXX */ +} + +static void +andes_flush_tlb_range(struct mm_struct *mm, unsigned long start, + unsigned long end) +{ + /* XXX */ +} + +static void +andes_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + /* XXX */ +} + +static void +andes_load_pgd(unsigned long pg_dir) +{ +} + +static void +andes_pgd_init(unsigned long page) +{ +} + +static void +andes_add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, + unsigned long entryhi, unsigned long pagemask) +{ + /* XXX */ +} + +static int +andes_user_mode(struct pt_regs *regs) +{ + return (regs->cp0_status & ST0_KSU) == KSU_USER; +} + +void __init ld_mmu_andes(void) +{ + flush_cache_all = andes_flush_cache_all; + flush_cache_mm = andes_flush_cache_mm; + flush_cache_range = andes_flush_cache_range; + flush_cache_page = andes_flush_cache_page; + flush_cache_sigtramp = andes_flush_cache_sigtramp; + flush_page_to_ram = andes_flush_page_to_ram; + + flush_tlb_all = andes_flush_tlb_all; + flush_tlb_mm = andes_flush_tlb_mm; + flush_tlb_range = andes_flush_tlb_range; + flush_tlb_page = andes_flush_tlb_page; + andes_asid_setup(); + + add_wired_entry = andes_add_wired_entry; + + user_mode = andes_user_mode; + + load_pgd = andes_load_pgd; + pgd_init = andes_pgd_init; + + flush_cache_all(); + flush_tlb_all(); +} diff --git a/arch/mips64/mm/extable.c b/arch/mips64/mm/extable.c new file mode 100644 index 000000000..7a7ab27ea --- /dev/null +++ b/arch/mips64/mm/extable.c @@ -0,0 +1,60 @@ +/* $Id$ + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1994, 1995, 1996, 1999 by Ralf Baechle + * Copyright (C) 1999 by Silicon Graphics + */ +#include +#include +#include + +extern const struct exception_table_entry __start___ex_table[]; +extern const struct exception_table_entry __stop___ex_table[]; + +static inline unsigned +search_one_table(const struct exception_table_entry *first, + const struct exception_table_entry *last, + unsigned long value) +{ + while (first <= last) { + const struct exception_table_entry *mid; + long diff; + + mid = (last - first) / 2 + first; + diff = mid->insn - value; + if (diff == 0) + return mid->nextinsn; + else if (diff < 0) + first = mid+1; + else + last = mid-1; + } + return 0; +} + +unsigned long +search_exception_table(unsigned long addr) +{ + unsigned long ret; + +#ifndef CONFIG_MODULES + /* There is only the kernel to search. */ + ret = search_one_table(__start___ex_table, __stop___ex_table-1, addr); + if (ret) return ret; +#else + /* The kernel is the last "module" -- no need to treat it special. */ + struct module *mp; + for (mp = module_list; mp != NULL; mp = mp->next) { + if (mp->ex_table_start == NULL) + continue; + ret = search_one_table(mp->ex_table_start, + mp->ex_table_end - 1, addr); + if (ret) return ret; + } +#endif + + return 0; +} diff --git a/arch/mips64/mm/fault.c b/arch/mips64/mm/fault.c new file mode 100644 index 000000000..1e34de13c --- /dev/null +++ b/arch/mips64/mm/fault.c @@ -0,0 +1,165 @@ +/* $Id$ + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1995, 1996, 1997, 1998, 1999 by Ralf Baechle + * Copyright (C) 1999 by Silicon Graphics + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#define development_version (LINUX_VERSION_CODE & 0x100) + +extern void die(char *, struct pt_regs *, unsigned long write); + +unsigned long asid_cache; + +/* + * Macro for exception fixup code to access integer registers. + */ +#define dpf_reg(r) (regs->regs[r]) + +/* + * This routine handles page faults. It determines the address, + * and the problem, and then passes it off to one of the appropriate + * routines. + */ +asmlinkage void +do_page_fault(struct pt_regs *regs, unsigned long writeaccess, + unsigned long address) +{ + struct vm_area_struct * vma; + struct task_struct *tsk = current; + struct mm_struct *mm = tsk->mm; + unsigned long fixup; + + /* + * If we're in an interrupt or have no user + * context, we must not take the fault.. + */ + if (in_interrupt() || mm == &init_mm) + goto no_context; +#if 0 + printk("[%s:%d:%08lx:%ld:%08lx]\n", current->comm, current->pid, + address, writeaccess, regs->cp0_epc); +#endif + down(&mm->mmap_sem); + vma = find_vma(mm, address); + if (!vma) + goto bad_area; + if (vma->vm_start <= address) + goto good_area; + if (!(vma->vm_flags & VM_GROWSDOWN)) + goto bad_area; + if (expand_stack(vma, address)) + goto bad_area; +/* + * Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ +good_area: + if (writeaccess) { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + } else { + if (!(vma->vm_flags & (VM_READ | VM_EXEC))) + goto bad_area; + } + + /* + * If for any reason at all we couldn't handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ + if (!handle_mm_fault(tsk, vma, address, writeaccess)) + goto do_sigbus; + + up(&mm->mmap_sem); + return; + +/* + * Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ +bad_area: + up(&mm->mmap_sem); + + if (user_mode(regs)) { + tsk->tss.cp0_badvaddr = address; + tsk->tss.error_code = writeaccess; +#if 0 + printk("do_page_fault() #2: sending SIGSEGV to %s for illegal %s\n" + "%08lx (epc == %08lx, ra == %08lx)\n", + tsk->comm, + writeaccess ? "writeaccess to" : "readaccess from", + address, + (unsigned long) regs->cp0_epc, + (unsigned long) regs->regs[31]); +#endif + force_sig(SIGSEGV, tsk); + return; + } + +no_context: + /* Are we prepared to handle this kernel fault? */ + fixup = search_exception_table(regs->cp0_epc); + if (fixup) { + long new_epc; + + tsk->tss.cp0_baduaddr = address; + new_epc = fixup_exception(dpf_reg, fixup, regs->cp0_epc); + if (development_version) + printk(KERN_DEBUG "%s: Exception at [<%lx>] (%lx)\n", + tsk->comm, regs->cp0_epc, new_epc); + regs->cp0_epc = new_epc; + return; + } + + /* + * Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. + */ + printk(KERN_ALERT "Unable to handle kernel paging request at virtual " + "address %08lx, epc == %08lx, ra == %08lx\n", + address, regs->cp0_epc, regs->regs[31]); + die("Oops", regs, writeaccess); + do_exit(SIGKILL); + +/* + * We ran out of memory, or some other thing happened to us that made + * us unable to handle the page fault gracefully. + */ +do_sigbus: + up(&mm->mmap_sem); + + /* + * Send a sigbus, regardless of whether we were in kernel + * or user mode. + */ + tsk->tss.cp0_badvaddr = address; + force_sig(SIGBUS, tsk); + + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + goto no_context; +} diff --git a/arch/mips64/mm/init.c b/arch/mips64/mm/init.c new file mode 100644 index 000000000..d6e489693 --- /dev/null +++ b/arch/mips64/mm/init.c @@ -0,0 +1,421 @@ +/* $Id$ + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1994 - 1999 by Ralf Baechle + * Copyright (C) 1999 by Silicon Graphics + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_BLK_DEV_INITRD +#include +#endif + +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_SGI +#include +#endif +#include + +extern void show_net_buffers(void); + +void __bad_pte_kernel(pmd_t *pmd) +{ + printk("Bad pmd in pte_alloc_kernel: %08lx\n", pmd_val(*pmd)); + pmd_val(*pmd) = BAD_PAGETABLE; +} + +void __bad_pte(pmd_t *pmd) +{ + printk("Bad pmd in pte_alloc: %08lx\n", pmd_val(*pmd)); + pmd_val(*pmd) = BAD_PAGETABLE; +} + +pte_t *get_pte_kernel_slow(pmd_t *pmd, unsigned long offset) +{ + pte_t *page; + + page = (pte_t *) __get_free_page(GFP_USER); + if (pmd_none(*pmd)) { + if (page) { + clear_page((unsigned long)page); + pmd_val(*pmd) = (unsigned long)page; + return page + offset; + } + pmd_val(*pmd) = BAD_PAGETABLE; + return NULL; + } + free_page((unsigned long)page); + if (pmd_bad(*pmd)) { + __bad_pte_kernel(pmd); + return NULL; + } + return (pte_t *) pmd_page(*pmd) + offset; +} + +pte_t *get_pte_slow(pmd_t *pmd, unsigned long offset) +{ + pte_t *page; + + page = (pte_t *) __get_free_page(GFP_KERNEL); + if (pmd_none(*pmd)) { + if (page) { + clear_page((unsigned long)page); + pmd_val(*pmd) = (unsigned long)page; + return page + offset; + } + pmd_val(*pmd) = BAD_PAGETABLE; + return NULL; + } + free_page((unsigned long)page); + if (pmd_bad(*pmd)) { + __bad_pte(pmd); + return NULL; + } + return (pte_t *) pmd_page(*pmd) + offset; +} + + +asmlinkage int sys_cacheflush(void *addr, int bytes, int cache) +{ + /* XXX Just get it working for now... */ + flush_cache_all(); + return 0; +} + +/* + * We have upto 8 empty zeroed pages so we can map one of the right colour + * when needed. This is necessary only on R4000 / R4400 SC and MC versions + * where we have to avoid VCED / VECI exceptions for good performance at + * any price. Since page is never written to after the initialization we + * don't have to care about aliases on other CPUs. + */ +unsigned long empty_zero_page, zero_page_mask; + +static inline unsigned long setup_zero_pages(void) +{ + unsigned long order, size, pg; + + switch (mips_cputype) { + case CPU_R4000SC: + case CPU_R4000MC: + case CPU_R4400SC: + case CPU_R4400MC: + order = 3; + break; + default: + order = 0; + } + + empty_zero_page = __get_free_pages(GFP_KERNEL, order); + if (!empty_zero_page) + panic("Oh boy, that early out of memory?"); + + pg = MAP_NR(empty_zero_page); + while(pg < MAP_NR(empty_zero_page) + (1 << order)) { + set_bit(PG_reserved, &mem_map[pg].flags); + set_page_count(mem_map + pg, 0); + pg++; + } + + size = PAGE_SIZE << order; + zero_page_mask = (size - 1) & PAGE_MASK; + memset((void *)empty_zero_page, 0, size); + + return size; +} + +int do_check_pgt_cache(int low, int high) +{ + int freed = 0; + + if(pgtable_cache_size > high) { + do { + if(pgd_quicklist) + free_pgd_slow(get_pgd_fast()), freed++; + if(pmd_quicklist) + free_pmd_slow(get_pmd_fast()), freed++; + if(pte_quicklist) + free_pte_slow(get_pte_fast()), freed++; + } while(pgtable_cache_size > low); + } + return freed; +} + +/* + * BAD_PAGE is the page that is used for page faults when linux + * is out-of-memory. Older versions of linux just did a + * do_exit(), but using this instead means there is less risk + * for a process dying in kernel mode, possibly leaving a inode + * unused etc.. + * + * BAD_PAGETABLE is the accompanying page-table: it is initialized + * to point to BAD_PAGE entries. + * + * ZERO_PAGE is a special page that is used for zero-initialized + * data and COW. + */ +pte_t * __bad_pagetable(void) +{ + extern char empty_bad_page_table[PAGE_SIZE]; + unsigned long page; + unsigned long dummy1, dummy2; +#if (_MIPS_ISA == _MIPS_ISA_MIPS3) || (_MIPS_ISA == _MIPS_ISA_MIPS4) + unsigned long dummy3; +#endif + + page = (unsigned long) empty_bad_page_table; + /* + * As long as we only save the low 32 bit of the 64 bit wide + * R4000 registers on interrupt we cannot use 64 bit memory accesses + * to the main memory. + */ +#if (_MIPS_ISA == _MIPS_ISA_MIPS3) || (_MIPS_ISA == _MIPS_ISA_MIPS4) + /* + * Use 64bit code even for Linux/MIPS 32bit on R4000 + */ + __asm__ __volatile__( + ".set\tnoreorder\n" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "dsll32\t$1,%2,0\n\t" + "dsrl32\t%2,$1,0\n\t" + "or\t%2,$1\n" + "1:\tsd\t%2,(%0)\n\t" + "subu\t%1,1\n\t" + "bnez\t%1,1b\n\t" + "addiu\t%0,8\n\t" + ".set\tmips0\n\t" + ".set\tat\n" + ".set\treorder" + :"=r" (dummy1), + "=r" (dummy2), + "=r" (dummy3) + :"0" (page), + "1" (PAGE_SIZE/8), + "2" (pte_val(BAD_PAGE))); +#else /* (_MIPS_ISA == _MIPS_ISA_MIPS1) || (_MIPS_ISA == _MIPS_ISA_MIPS2) */ + __asm__ __volatile__( + ".set\tnoreorder\n" + "1:\tsw\t%2,(%0)\n\t" + "subu\t%1,1\n\t" + "bnez\t%1,1b\n\t" + "addiu\t%0,4\n\t" + ".set\treorder" + :"=r" (dummy1), + "=r" (dummy2) + :"r" (pte_val(BAD_PAGE)), + "0" (page), + "1" (PAGE_SIZE/4)); +#endif + + return (pte_t *)page; +} + +pte_t __bad_page(void) +{ + extern char empty_bad_page[PAGE_SIZE]; + unsigned long page = (unsigned long)empty_bad_page; + + clear_page(page); + return pte_mkdirty(mk_pte(page, PAGE_SHARED)); +} + +void show_mem(void) +{ + int i, free = 0, total = 0, reserved = 0; + int shared = 0, cached = 0; + + printk("Mem-info:\n"); + show_free_areas(); + printk("Free swap: %6dkB\n", nr_swap_pages<<(PAGE_SHIFT-10)); + i = max_mapnr; + while (i-- > 0) { + total++; + if (PageReserved(mem_map+i)) + reserved++; + else if (PageSwapCache(mem_map+i)) + cached++; + else if (!page_count(mem_map + i)) + free++; + else + shared += page_count(mem_map + i) - 1; + } + printk("%d pages of RAM\n", total); + printk("%d reserved pages\n", reserved); + printk("%d pages shared\n", shared); + printk("%d pages swap cached\n",cached); + printk("%ld pages in page table cache\n",pgtable_cache_size); + printk("%d free pages\n", free); +#ifdef CONFIG_NET + show_net_buffers(); +#endif +} + +extern unsigned long free_area_init(unsigned long, unsigned long); + +unsigned long __init +paging_init(unsigned long start_mem, unsigned long end_mem) +{ + /* Initialize the entire pgd. */ + pgd_init((unsigned long)swapper_pg_dir); + pgd_init((unsigned long)swapper_pg_dir + PAGE_SIZE / 2); + return free_area_init(start_mem, end_mem); +} + +void __init +mem_init(unsigned long start_mem, unsigned long end_mem) +{ + int codepages = 0; + int datapages = 0; + unsigned long tmp; + extern int _etext, _ftext; + +#ifdef CONFIG_MIPS_JAZZ + if (mips_machgroup == MACH_GROUP_JAZZ) + start_mem = vdma_init(start_mem, end_mem); +#endif + + end_mem &= PAGE_MASK; + max_mapnr = MAP_NR(end_mem); + high_memory = (void *)end_mem; + num_physpages = 0; + + /* mark usable pages in the mem_map[] */ + start_mem = PAGE_ALIGN(start_mem); + + for(tmp = MAP_NR(start_mem);tmp < max_mapnr;tmp++) + clear_bit(PG_reserved, &mem_map[tmp].flags); + + prom_fixup_mem_map(start_mem, (unsigned long)high_memory); + + for (tmp = PAGE_OFFSET; tmp < end_mem; tmp += PAGE_SIZE) { + /* + * This is only for PC-style DMA. The onboard DMA + * of Jazz and Tyne machines is completely different and + * not handled via a flag in mem_map_t. + */ + if (tmp >= MAX_DMA_ADDRESS) + clear_bit(PG_DMA, &mem_map[MAP_NR(tmp)].flags); + if (PageReserved(mem_map+MAP_NR(tmp))) { + if ((tmp < (unsigned long) &_etext) && + (tmp >= (unsigned long) &_ftext)) + codepages++; + else if ((tmp < start_mem) && + (tmp > (unsigned long) &_etext)) + datapages++; + continue; + } + num_physpages++; + set_page_count(mem_map + MAP_NR(tmp), 1); +#ifdef CONFIG_BLK_DEV_INITRD + if (!initrd_start || (tmp < initrd_start || tmp >= + initrd_end)) +#endif + free_page(tmp); + } + tmp = nr_free_pages << PAGE_SHIFT; + + /* Setup zeroed pages. */ + tmp -= setup_zero_pages(); + + printk("Memory: %luk/%luk available (%dk kernel code, %dk data)\n", + tmp >> 10, + max_mapnr << (PAGE_SHIFT-10), + codepages << (PAGE_SHIFT-10), + datapages << (PAGE_SHIFT-10)); +} + +extern char __init_begin, __init_end; + +void +free_initmem(void) +{ + unsigned long addr; + + prom_free_prom_memory (); + + addr = (unsigned long)(&__init_begin); + for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) { + mem_map[MAP_NR(addr)].flags &= ~(1 << PG_reserved); + set_page_count(mem_map + MAP_NR(addr), 1); + free_page(addr); + } + printk("Freeing unused kernel memory: %dk freed\n", + (&__init_end - &__init_begin) >> 10); +} + +void +si_meminfo(struct sysinfo *val) +{ + int i; + + i = MAP_NR(high_memory); + val->totalram = 0; + val->sharedram = 0; + val->freeram = nr_free_pages << PAGE_SHIFT; + val->bufferram = atomic_read(&buffermem); + while (i-- > 0) { + if (PageReserved(mem_map+i)) + continue; + val->totalram++; + if (!page_count(mem_map + i)) + continue; + val->sharedram += page_count(mem_map + i) - 1; + } + val->totalram <<= PAGE_SHIFT; + val->sharedram <<= PAGE_SHIFT; + return; +} + +/* Fixup an immediate instruction */ +static void __init +__i_insn_fixup(unsigned int **start, unsigned int **stop, + unsigned int i_const) +{ + unsigned int **p, *ip; + + for (p = start;p < stop; p++) { + ip = *p; + *ip = (*ip & 0xffff0000) | i_const; + } +} + +#define i_insn_fixup(section, const) \ +do { \ + extern unsigned int *__start_ ## section; \ + extern unsigned int *__stop_ ## section; \ + __i_insn_fixup(&__start_ ## section, &__stop_ ## section, const); \ +} while(0) + +/* Caller is assumed to flush the caches before the first context switch. */ +void __init +__asid_setup(unsigned int inc, unsigned int mask, unsigned int version_mask, + unsigned int first_version) +{ + i_insn_fixup(__asid_inc, inc); + i_insn_fixup(__asid_mask, mask); + i_insn_fixup(__asid_version_mask, version_mask); + i_insn_fixup(__asid_first_version, first_version); + + asid_cache = first_version; +} diff --git a/arch/mips64/mm/loadmmu.c b/arch/mips64/mm/loadmmu.c new file mode 100644 index 000000000..a96ab901c --- /dev/null +++ b/arch/mips64/mm/loadmmu.c @@ -0,0 +1,109 @@ +/* $Id$ + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) + * Copyright (C) 1997, 1999 Ralf Baechle (ralf@gnu.org) + * Copyright (C) 1999 Silicon Graphics, Inc. + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/* memory functions */ +void (*clear_page)(unsigned long page); +void (*copy_page)(unsigned long to, unsigned long from); + +/* Cache operations. */ +void (*flush_cache_all)(void); +void (*flush_cache_mm)(struct mm_struct *mm); +void (*flush_cache_range)(struct mm_struct *mm, unsigned long start, + unsigned long end); +void (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page); +void (*flush_cache_sigtramp)(unsigned long addr); +void (*flush_page_to_ram)(unsigned long page); + +/* DMA cache operations. */ +void (*dma_cache_wback_inv)(unsigned long start, unsigned long size); +void (*dma_cache_wback)(unsigned long start, unsigned long size); +void (*dma_cache_inv)(unsigned long start, unsigned long size); + +/* TLB operations. */ +void (*flush_tlb_all)(void); +void (*flush_tlb_mm)(struct mm_struct *mm); +void (*flush_tlb_range)(struct mm_struct *mm, unsigned long start, + unsigned long end); +void (*flush_tlb_page)(struct vm_area_struct *vma, unsigned long page); + +/* Miscellaneous. */ +void (*load_pgd)(unsigned long pg_dir); +void (*pgd_init)(unsigned long page); +void (*update_mmu_cache)(struct vm_area_struct * vma, + unsigned long address, pte_t pte); + +void (*show_regs)(struct pt_regs *); + +void (*add_wired_entry)(unsigned long entrylo0, unsigned long entrylo1, + unsigned long entryhi, unsigned long pagemask); + +int (*user_mode)(struct pt_regs *); + +asmlinkage void *(*resume)(void *last, void *next); + +extern void ld_mmu_r2300(void); +extern void ld_mmu_r4xx0(void); +extern void ld_mmu_r6000(void); +extern void ld_mmu_tfp(void); +extern void ld_mmu_andes(void); + +void __init loadmmu(void) +{ + switch(mips_cputype) { + case CPU_R4000PC: + case CPU_R4000SC: + case CPU_R4000MC: + case CPU_R4200: + case CPU_R4300: + case CPU_R4400PC: + case CPU_R4400SC: + case CPU_R4400MC: + case CPU_R4600: + case CPU_R4640: + case CPU_R4650: + case CPU_R4700: + case CPU_R5000: + case CPU_R5000A: + case CPU_NEVADA: + printk("Loading R4000 MMU routines.\n"); + ld_mmu_r4xx0(); + break; + + case CPU_R8000: + printk("Loading TFP MMU routines.\n"); + ld_mmu_tfp(); + break; + + case CPU_R10000: + printk("Loading R10000 MMU routines.\n"); + ld_mmu_andes(); + break; + + default: + /* XXX We need an generic routine in the MIPS port + * XXX to jabber stuff onto the screen on all machines + * XXX before the console is setup. The ARCS prom + * XXX routines look good for this, but only the SGI + * XXX code has a full library for that at this time. + */ + panic("Yeee, unsupported mmu/cache architecture."); + } +} diff --git a/arch/mips64/mm/r4xx0.c b/arch/mips64/mm/r4xx0.c new file mode 100644 index 000000000..f6b31f4ba --- /dev/null +++ b/arch/mips64/mm/r4xx0.c @@ -0,0 +1,2821 @@ +/* $Id$ + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * r4xx0.c: R4000 processor variant specific MMU/Cache routines. + * + * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) + * Copyright (C) 1997, 1998, 1999 Ralf Baechle (ralf@gnu.org) + * Copyright (C) 1999 Silicon Graphics, Inc. + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* CP0 hazard avoidance. */ +#define BARRIER __asm__ __volatile__(".set noreorder\n\t" \ + "nop; nop; nop; nop; nop; nop;\n\t" \ + ".set reorder\n\t") + +/* Primary cache parameters. */ +static int icache_size, dcache_size; /* Size in bytes */ +static int ic_lsize, dc_lsize; /* LineSize in bytes */ + +/* Secondary cache (if present) parameters. */ +static unsigned int scache_size, sc_lsize; /* Again, in bytes */ + +#include +#include + +#undef DEBUG_CACHE + +/* + * Dummy cache handling routines for machines without boardcaches + */ +static void no_sc_noop(void) {} + +static struct bcache_ops no_sc_ops = { + (void *)no_sc_noop, (void *)no_sc_noop, + (void *)no_sc_noop, (void *)no_sc_noop +}; + +struct bcache_ops *bcops = &no_sc_ops; + +/* + * On processors with QED R4600 style two set assosicative cache + * this is the bit which selects the way in the cache for the + * indexed cachops. + */ +#define icache_waybit (icache_size >> 1) +#define dcache_waybit (dcache_size >> 1) + +/* + * Zero an entire page. Basically a simple unrolled loop should do the + * job but we want more performance by saving memory bus bandwidth. We + * have five flavours of the routine available for: + * + * - 16byte cachelines and no second level cache + * - 32byte cachelines second level cache + * - a version which handles the buggy R4600 v1.x + * - a version which handles the buggy R4600 v2.0 + * - Finally a last version without fancy cache games for the SC and MC + * versions of R4000 and R4400. + */ + +static void r4k_clear_page_d16(unsigned long page) +{ + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%2\n" + "1:\tcache\t%3,(%0)\n\t" + "sd\t$0,(%0)\n\t" + "sd\t$0,8(%0)\n\t" + "cache\t%3,16(%0)\n\t" + "sd\t$0,16(%0)\n\t" + "sd\t$0,24(%0)\n\t" + "daddiu\t%0,64\n\t" + "cache\t%3,-32(%0)\n\t" + "sd\t$0,-32(%0)\n\t" + "sd\t$0,-24(%0)\n\t" + "cache\t%3,-16(%0)\n\t" + "sd\t$0,-16(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sd\t$0,-8(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (page) + :"0" (page), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_D) + :"$1","memory"); +} + +static void r4k_clear_page_d32(unsigned long page) +{ + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%2\n" + "1:\tcache\t%3,(%0)\n\t" + "sd\t$0,(%0)\n\t" + "sd\t$0,8(%0)\n\t" + "sd\t$0,16(%0)\n\t" + "sd\t$0,24(%0)\n\t" + "daddiu\t%0,64\n\t" + "cache\t%3,-32(%0)\n\t" + "sd\t$0,-32(%0)\n\t" + "sd\t$0,-24(%0)\n\t" + "sd\t$0,-16(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sd\t$0,-8(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (page) + :"0" (page), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_D) + :"$1","memory"); +} + + +/* + * This flavour of r4k_clear_page is for the R4600 V1.x. Cite from the + * IDT R4600 V1.7 errata: + * + * 18. The CACHE instructions Hit_Writeback_Invalidate_D, Hit_Writeback_D, + * Hit_Invalidate_D and Create_Dirty_Excl_D should only be + * executed if there is no other dcache activity. If the dcache is + * accessed for another instruction immeidately preceding when these + * cache instructions are executing, it is possible that the dcache + * tag match outputs used by these cache instructions will be + * incorrect. These cache instructions should be preceded by at least + * four instructions that are not any kind of load or store + * instruction. + * + * This is not allowed: lw + * nop + * nop + * nop + * cache Hit_Writeback_Invalidate_D + * + * This is allowed: lw + * nop + * nop + * nop + * nop + * cache Hit_Writeback_Invalidate_D + */ +static void r4k_clear_page_r4600_v1(unsigned long page) +{ + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%2\n" + "1:\tnop\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + "cache\t%3,(%0)\n\t" + "sd\t$0,(%0)\n\t" + "sd\t$0,8(%0)\n\t" + "sd\t$0,16(%0)\n\t" + "sd\t$0,24(%0)\n\t" + "daddiu\t%0,64\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + "cache\t%3,-32(%0)\n\t" + "sd\t$0,-32(%0)\n\t" + "sd\t$0,-24(%0)\n\t" + "sd\t$0,-16(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sd\t$0,-8(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (page) + :"0" (page), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_D) + :"$1","memory"); +} + +/* + * And this one is for the R4600 V2.0 + */ +static void r4k_clear_page_r4600_v2(unsigned long page) +{ + unsigned int flags; + + save_and_cli(flags); + *(volatile unsigned int *)KSEG1; + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%2\n" + "1:\tcache\t%3,(%0)\n\t" + "sd\t$0,(%0)\n\t" + "sd\t$0,8(%0)\n\t" + "sd\t$0,16(%0)\n\t" + "sd\t$0,24(%0)\n\t" + "daddiu\t%0,64\n\t" + "cache\t%3,-32(%0)\n\t" + "sd\t$0,-32(%0)\n\t" + "sd\t$0,-24(%0)\n\t" + "sd\t$0,-16(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sd\t$0,-8(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (page) + :"0" (page), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_D) + :"$1","memory"); + restore_flags(flags); +} + +/* + * The next 4 versions are optimized for all possible scache configurations + * of the SC / MC versions of R4000 and R4400 ... + * + * Todo: For even better performance we should have a routine optimized for + * every legal combination of dcache / scache linesize. When I (Ralf) tried + * this the kernel crashed shortly after mounting the root filesystem. CPU + * bug? Weirdo cache instruction semantics? + */ +static void r4k_clear_page_s16(unsigned long page) +{ + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%2\n" + "1:\tcache\t%3,(%0)\n\t" + "sd\t$0,(%0)\n\t" + "sd\t$0,8(%0)\n\t" + "cache\t%3,16(%0)\n\t" + "sd\t$0,16(%0)\n\t" + "sd\t$0,24(%0)\n\t" + "daddiu\t%0,64\n\t" + "cache\t%3,-32(%0)\n\t" + "sd\t$0,-32(%0)\n\t" + "sd\t$0,-24(%0)\n\t" + "cache\t%3,-16(%0)\n\t" + "sd\t$0,-16(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sd\t$0,-8(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (page) + :"0" (page), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_SD) + :"$1","memory"); +} + +static void r4k_clear_page_s32(unsigned long page) +{ + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%2\n" + "1:\tcache\t%3,(%0)\n\t" + "sd\t$0,(%0)\n\t" + "sd\t$0,8(%0)\n\t" + "sd\t$0,16(%0)\n\t" + "sd\t$0,24(%0)\n\t" + "daddiu\t%0,64\n\t" + "cache\t%3,-32(%0)\n\t" + "sd\t$0,-32(%0)\n\t" + "sd\t$0,-24(%0)\n\t" + "sd\t$0,-16(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sd\t$0,-8(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (page) + :"0" (page), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_SD) + :"$1","memory"); +} + +static void r4k_clear_page_s64(unsigned long page) +{ + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%2\n" + "1:\tcache\t%3,(%0)\n\t" + "sd\t$0,(%0)\n\t" + "sd\t$0,8(%0)\n\t" + "sd\t$0,16(%0)\n\t" + "sd\t$0,24(%0)\n\t" + "daddiu\t%0,64\n\t" + "sd\t$0,-32(%0)\n\t" + "sd\t$0,-24(%0)\n\t" + "sd\t$0,-16(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sd\t$0,-8(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (page) + :"0" (page), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_SD) + :"$1","memory"); +} + +static void r4k_clear_page_s128(unsigned long page) +{ + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%2\n" + "1:\tcache\t%3,(%0)\n\t" + "sd\t$0,(%0)\n\t" + "sd\t$0,8(%0)\n\t" + "sd\t$0,16(%0)\n\t" + "sd\t$0,24(%0)\n\t" + "sd\t$0,32(%0)\n\t" + "sd\t$0,40(%0)\n\t" + "sd\t$0,48(%0)\n\t" + "sd\t$0,56(%0)\n\t" + "daddiu\t%0,128\n\t" + "sd\t$0,-64(%0)\n\t" + "sd\t$0,-56(%0)\n\t" + "sd\t$0,-48(%0)\n\t" + "sd\t$0,-40(%0)\n\t" + "sd\t$0,-32(%0)\n\t" + "sd\t$0,-24(%0)\n\t" + "sd\t$0,-16(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sd\t$0,-8(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (page) + :"0" (page), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_SD) + :"$1","memory"); +} + + +/* + * This is still inefficient. We only can do better if we know the + * virtual address where the copy will be accessed. + */ + +static void r4k_copy_page_d16(unsigned long to, unsigned long from) +{ + unsigned long dummy1, dummy2; + unsigned long reg1, reg2, reg3, reg4; + + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%8\n" + "1:\tcache\t%9,(%0)\n\t" + "lw\t%2,(%1)\n\t" + "lw\t%3,4(%1)\n\t" + "lw\t%4,8(%1)\n\t" + "lw\t%5,12(%1)\n\t" + "sw\t%2,(%0)\n\t" + "sw\t%3,4(%0)\n\t" + "sw\t%4,8(%0)\n\t" + "sw\t%5,12(%0)\n\t" + "cache\t%9,16(%0)\n\t" + "lw\t%2,16(%1)\n\t" + "lw\t%3,20(%1)\n\t" + "lw\t%4,24(%1)\n\t" + "lw\t%5,28(%1)\n\t" + "sw\t%2,16(%0)\n\t" + "sw\t%3,20(%0)\n\t" + "sw\t%4,24(%0)\n\t" + "sw\t%5,28(%0)\n\t" + "cache\t%9,32(%0)\n\t" + "daddiu\t%0,64\n\t" + "daddiu\t%1,64\n\t" + "lw\t%2,-32(%1)\n\t" + "lw\t%3,-28(%1)\n\t" + "lw\t%4,-24(%1)\n\t" + "lw\t%5,-20(%1)\n\t" + "sw\t%2,-32(%0)\n\t" + "sw\t%3,-28(%0)\n\t" + "sw\t%4,-24(%0)\n\t" + "sw\t%5,-20(%0)\n\t" + "cache\t%9,-16(%0)\n\t" + "lw\t%2,-16(%1)\n\t" + "lw\t%3,-12(%1)\n\t" + "lw\t%4,-8(%1)\n\t" + "lw\t%5,-4(%1)\n\t" + "sw\t%2,-16(%0)\n\t" + "sw\t%3,-12(%0)\n\t" + "sw\t%4,-8(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sw\t%5,-4(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (dummy1), "=r" (dummy2), + "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4) + :"0" (to), "1" (from), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_D)); +} + +static void r4k_copy_page_d32(unsigned long to, unsigned long from) +{ + unsigned long dummy1, dummy2; + unsigned long reg1, reg2, reg3, reg4; + + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%8\n" + "1:\tcache\t%9,(%0)\n\t" + "lw\t%2,(%1)\n\t" + "lw\t%3,4(%1)\n\t" + "lw\t%4,8(%1)\n\t" + "lw\t%5,12(%1)\n\t" + "sw\t%2,(%0)\n\t" + "sw\t%3,4(%0)\n\t" + "sw\t%4,8(%0)\n\t" + "sw\t%5,12(%0)\n\t" + "lw\t%2,16(%1)\n\t" + "lw\t%3,20(%1)\n\t" + "lw\t%4,24(%1)\n\t" + "lw\t%5,28(%1)\n\t" + "sw\t%2,16(%0)\n\t" + "sw\t%3,20(%0)\n\t" + "sw\t%4,24(%0)\n\t" + "sw\t%5,28(%0)\n\t" + "cache\t%9,32(%0)\n\t" + "daddiu\t%0,64\n\t" + "daddiu\t%1,64\n\t" + "lw\t%2,-32(%1)\n\t" + "lw\t%3,-28(%1)\n\t" + "lw\t%4,-24(%1)\n\t" + "lw\t%5,-20(%1)\n\t" + "sw\t%2,-32(%0)\n\t" + "sw\t%3,-28(%0)\n\t" + "sw\t%4,-24(%0)\n\t" + "sw\t%5,-20(%0)\n\t" + "lw\t%2,-16(%1)\n\t" + "lw\t%3,-12(%1)\n\t" + "lw\t%4,-8(%1)\n\t" + "lw\t%5,-4(%1)\n\t" + "sw\t%2,-16(%0)\n\t" + "sw\t%3,-12(%0)\n\t" + "sw\t%4,-8(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sw\t%5,-4(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (dummy1), "=r" (dummy2), + "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4) + :"0" (to), "1" (from), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_D)); +} + +/* + * Again a special version for the R4600 V1.x + */ +static void r4k_copy_page_r4600_v1(unsigned long to, unsigned long from) +{ + unsigned long dummy1, dummy2; + unsigned long reg1, reg2, reg3, reg4; + + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%8\n" + "1:\tnop\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + "\tcache\t%9,(%0)\n\t" + "lw\t%2,(%1)\n\t" + "lw\t%3,4(%1)\n\t" + "lw\t%4,8(%1)\n\t" + "lw\t%5,12(%1)\n\t" + "sw\t%2,(%0)\n\t" + "sw\t%3,4(%0)\n\t" + "sw\t%4,8(%0)\n\t" + "sw\t%5,12(%0)\n\t" + "lw\t%2,16(%1)\n\t" + "lw\t%3,20(%1)\n\t" + "lw\t%4,24(%1)\n\t" + "lw\t%5,28(%1)\n\t" + "sw\t%2,16(%0)\n\t" + "sw\t%3,20(%0)\n\t" + "sw\t%4,24(%0)\n\t" + "sw\t%5,28(%0)\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + "cache\t%9,32(%0)\n\t" + "daddiu\t%0,64\n\t" + "daddiu\t%1,64\n\t" + "lw\t%2,-32(%1)\n\t" + "lw\t%3,-28(%1)\n\t" + "lw\t%4,-24(%1)\n\t" + "lw\t%5,-20(%1)\n\t" + "sw\t%2,-32(%0)\n\t" + "sw\t%3,-28(%0)\n\t" + "sw\t%4,-24(%0)\n\t" + "sw\t%5,-20(%0)\n\t" + "lw\t%2,-16(%1)\n\t" + "lw\t%3,-12(%1)\n\t" + "lw\t%4,-8(%1)\n\t" + "lw\t%5,-4(%1)\n\t" + "sw\t%2,-16(%0)\n\t" + "sw\t%3,-12(%0)\n\t" + "sw\t%4,-8(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sw\t%5,-4(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (dummy1), "=r" (dummy2), + "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4) + :"0" (to), "1" (from), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_D)); +} + +static void r4k_copy_page_r4600_v2(unsigned long to, unsigned long from) +{ + unsigned long dummy1, dummy2; + unsigned long reg1, reg2, reg3, reg4; + unsigned int flags; + + __save_and_cli(flags); + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%8\n" + "1:\tnop\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + "\tcache\t%9,(%0)\n\t" + "lw\t%2,(%1)\n\t" + "lw\t%3,4(%1)\n\t" + "lw\t%4,8(%1)\n\t" + "lw\t%5,12(%1)\n\t" + "sw\t%2,(%0)\n\t" + "sw\t%3,4(%0)\n\t" + "sw\t%4,8(%0)\n\t" + "sw\t%5,12(%0)\n\t" + "lw\t%2,16(%1)\n\t" + "lw\t%3,20(%1)\n\t" + "lw\t%4,24(%1)\n\t" + "lw\t%5,28(%1)\n\t" + "sw\t%2,16(%0)\n\t" + "sw\t%3,20(%0)\n\t" + "sw\t%4,24(%0)\n\t" + "sw\t%5,28(%0)\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + "cache\t%9,32(%0)\n\t" + "daddiu\t%0,64\n\t" + "daddiu\t%1,64\n\t" + "lw\t%2,-32(%1)\n\t" + "lw\t%3,-28(%1)\n\t" + "lw\t%4,-24(%1)\n\t" + "lw\t%5,-20(%1)\n\t" + "sw\t%2,-32(%0)\n\t" + "sw\t%3,-28(%0)\n\t" + "sw\t%4,-24(%0)\n\t" + "sw\t%5,-20(%0)\n\t" + "lw\t%2,-16(%1)\n\t" + "lw\t%3,-12(%1)\n\t" + "lw\t%4,-8(%1)\n\t" + "lw\t%5,-4(%1)\n\t" + "sw\t%2,-16(%0)\n\t" + "sw\t%3,-12(%0)\n\t" + "sw\t%4,-8(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sw\t%5,-4(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (dummy1), "=r" (dummy2), + "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4) + :"0" (to), "1" (from), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_D)); + restore_flags(flags); +} + +/* + * These are for R4000SC / R4400MC + */ +static void r4k_copy_page_s16(unsigned long to, unsigned long from) +{ + unsigned long dummy1, dummy2; + unsigned long reg1, reg2, reg3, reg4; + + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%8\n" + "1:\tcache\t%9,(%0)\n\t" + "lw\t%2,(%1)\n\t" + "lw\t%3,4(%1)\n\t" + "lw\t%4,8(%1)\n\t" + "lw\t%5,12(%1)\n\t" + "sw\t%2,(%0)\n\t" + "sw\t%3,4(%0)\n\t" + "sw\t%4,8(%0)\n\t" + "sw\t%5,12(%0)\n\t" + "cache\t%9,16(%0)\n\t" + "lw\t%2,16(%1)\n\t" + "lw\t%3,20(%1)\n\t" + "lw\t%4,24(%1)\n\t" + "lw\t%5,28(%1)\n\t" + "sw\t%2,16(%0)\n\t" + "sw\t%3,20(%0)\n\t" + "sw\t%4,24(%0)\n\t" + "sw\t%5,28(%0)\n\t" + "cache\t%9,32(%0)\n\t" + "daddiu\t%0,64\n\t" + "daddiu\t%1,64\n\t" + "lw\t%2,-32(%1)\n\t" + "lw\t%3,-28(%1)\n\t" + "lw\t%4,-24(%1)\n\t" + "lw\t%5,-20(%1)\n\t" + "sw\t%2,-32(%0)\n\t" + "sw\t%3,-28(%0)\n\t" + "sw\t%4,-24(%0)\n\t" + "sw\t%5,-20(%0)\n\t" + "cache\t%9,-16(%0)\n\t" + "lw\t%2,-16(%1)\n\t" + "lw\t%3,-12(%1)\n\t" + "lw\t%4,-8(%1)\n\t" + "lw\t%5,-4(%1)\n\t" + "sw\t%2,-16(%0)\n\t" + "sw\t%3,-12(%0)\n\t" + "sw\t%4,-8(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sw\t%5,-4(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (dummy1), "=r" (dummy2), + "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4) + :"0" (to), "1" (from), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_SD)); +} + +static void r4k_copy_page_s32(unsigned long to, unsigned long from) +{ + unsigned long dummy1, dummy2; + unsigned long reg1, reg2, reg3, reg4; + + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%8\n" + "1:\tcache\t%9,(%0)\n\t" + "lw\t%2,(%1)\n\t" + "lw\t%3,4(%1)\n\t" + "lw\t%4,8(%1)\n\t" + "lw\t%5,12(%1)\n\t" + "sw\t%2,(%0)\n\t" + "sw\t%3,4(%0)\n\t" + "sw\t%4,8(%0)\n\t" + "sw\t%5,12(%0)\n\t" + "lw\t%2,16(%1)\n\t" + "lw\t%3,20(%1)\n\t" + "lw\t%4,24(%1)\n\t" + "lw\t%5,28(%1)\n\t" + "sw\t%2,16(%0)\n\t" + "sw\t%3,20(%0)\n\t" + "sw\t%4,24(%0)\n\t" + "sw\t%5,28(%0)\n\t" + "cache\t%9,32(%0)\n\t" + "daddiu\t%0,64\n\t" + "daddiu\t%1,64\n\t" + "lw\t%2,-32(%1)\n\t" + "lw\t%3,-28(%1)\n\t" + "lw\t%4,-24(%1)\n\t" + "lw\t%5,-20(%1)\n\t" + "sw\t%2,-32(%0)\n\t" + "sw\t%3,-28(%0)\n\t" + "sw\t%4,-24(%0)\n\t" + "sw\t%5,-20(%0)\n\t" + "lw\t%2,-16(%1)\n\t" + "lw\t%3,-12(%1)\n\t" + "lw\t%4,-8(%1)\n\t" + "lw\t%5,-4(%1)\n\t" + "sw\t%2,-16(%0)\n\t" + "sw\t%3,-12(%0)\n\t" + "sw\t%4,-8(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sw\t%5,-4(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (dummy1), "=r" (dummy2), + "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4) + :"0" (to), "1" (from), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_SD)); +} + +static void r4k_copy_page_s64(unsigned long to, unsigned long from) +{ + unsigned long dummy1, dummy2; + unsigned long reg1, reg2, reg3, reg4; + + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%8\n" + "1:\tcache\t%9,(%0)\n\t" + "lw\t%2,(%1)\n\t" + "lw\t%3,4(%1)\n\t" + "lw\t%4,8(%1)\n\t" + "lw\t%5,12(%1)\n\t" + "sw\t%2,(%0)\n\t" + "sw\t%3,4(%0)\n\t" + "sw\t%4,8(%0)\n\t" + "sw\t%5,12(%0)\n\t" + "lw\t%2,16(%1)\n\t" + "lw\t%3,20(%1)\n\t" + "lw\t%4,24(%1)\n\t" + "lw\t%5,28(%1)\n\t" + "sw\t%2,16(%0)\n\t" + "sw\t%3,20(%0)\n\t" + "sw\t%4,24(%0)\n\t" + "sw\t%5,28(%0)\n\t" + "daddiu\t%0,64\n\t" + "daddiu\t%1,64\n\t" + "lw\t%2,-32(%1)\n\t" + "lw\t%3,-28(%1)\n\t" + "lw\t%4,-24(%1)\n\t" + "lw\t%5,-20(%1)\n\t" + "sw\t%2,-32(%0)\n\t" + "sw\t%3,-28(%0)\n\t" + "sw\t%4,-24(%0)\n\t" + "sw\t%5,-20(%0)\n\t" + "lw\t%2,-16(%1)\n\t" + "lw\t%3,-12(%1)\n\t" + "lw\t%4,-8(%1)\n\t" + "lw\t%5,-4(%1)\n\t" + "sw\t%2,-16(%0)\n\t" + "sw\t%3,-12(%0)\n\t" + "sw\t%4,-8(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sw\t%5,-4(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (dummy1), "=r" (dummy2), + "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4) + :"0" (to), "1" (from), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_SD)); +} + +static void r4k_copy_page_s128(unsigned long to, unsigned long from) +{ + unsigned long dummy1, dummy2; + unsigned long reg1, reg2, reg3, reg4; + + __asm__ __volatile__( + ".set\tnoreorder\n\t" + ".set\tnoat\n\t" + ".set\tmips3\n\t" + "daddiu\t$1,%0,%8\n" + "1:\tcache\t%9,(%0)\n\t" + "lw\t%2,(%1)\n\t" + "lw\t%3,4(%1)\n\t" + "lw\t%4,8(%1)\n\t" + "lw\t%5,12(%1)\n\t" + "sw\t%2,(%0)\n\t" + "sw\t%3,4(%0)\n\t" + "sw\t%4,8(%0)\n\t" + "sw\t%5,12(%0)\n\t" + "lw\t%2,16(%1)\n\t" + "lw\t%3,20(%1)\n\t" + "lw\t%4,24(%1)\n\t" + "lw\t%5,28(%1)\n\t" + "sw\t%2,16(%0)\n\t" + "sw\t%3,20(%0)\n\t" + "sw\t%4,24(%0)\n\t" + "sw\t%5,28(%0)\n\t" + "lw\t%2,32(%1)\n\t" + "lw\t%3,36(%1)\n\t" + "lw\t%4,40(%1)\n\t" + "lw\t%5,44(%1)\n\t" + "sw\t%2,32(%0)\n\t" + "sw\t%3,36(%0)\n\t" + "sw\t%4,40(%0)\n\t" + "sw\t%5,44(%0)\n\t" + "lw\t%2,48(%1)\n\t" + "lw\t%3,52(%1)\n\t" + "lw\t%4,56(%1)\n\t" + "lw\t%5,60(%1)\n\t" + "sw\t%2,48(%0)\n\t" + "sw\t%3,52(%0)\n\t" + "sw\t%4,56(%0)\n\t" + "sw\t%5,60(%0)\n\t" + "daddiu\t%0,128\n\t" + "daddiu\t%1,128\n\t" + "lw\t%2,-64(%1)\n\t" + "lw\t%3,-60(%1)\n\t" + "lw\t%4,-56(%1)\n\t" + "lw\t%5,-52(%1)\n\t" + "sw\t%2,-64(%0)\n\t" + "sw\t%3,-60(%0)\n\t" + "sw\t%4,-56(%0)\n\t" + "sw\t%5,-52(%0)\n\t" + "lw\t%2,-48(%1)\n\t" + "lw\t%3,-44(%1)\n\t" + "lw\t%4,-40(%1)\n\t" + "lw\t%5,-36(%1)\n\t" + "sw\t%2,-48(%0)\n\t" + "sw\t%3,-44(%0)\n\t" + "sw\t%4,-40(%0)\n\t" + "sw\t%5,-36(%0)\n\t" + "lw\t%2,-32(%1)\n\t" + "lw\t%3,-28(%1)\n\t" + "lw\t%4,-24(%1)\n\t" + "lw\t%5,-20(%1)\n\t" + "sw\t%2,-32(%0)\n\t" + "sw\t%3,-28(%0)\n\t" + "sw\t%4,-24(%0)\n\t" + "sw\t%5,-20(%0)\n\t" + "lw\t%2,-16(%1)\n\t" + "lw\t%3,-12(%1)\n\t" + "lw\t%4,-8(%1)\n\t" + "lw\t%5,-4(%1)\n\t" + "sw\t%2,-16(%0)\n\t" + "sw\t%3,-12(%0)\n\t" + "sw\t%4,-8(%0)\n\t" + "bne\t$1,%0,1b\n\t" + "sw\t%5,-4(%0)\n\t" + ".set\tmips0\n\t" + ".set\tat\n\t" + ".set\treorder" + :"=r" (dummy1), "=r" (dummy2), + "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4) + :"0" (to), "1" (from), + "I" (PAGE_SIZE), + "i" (Create_Dirty_Excl_SD)); +} + + +/* + * If you think for one second that this stuff coming up is a lot + * of bulky code eating too many kernel cache lines. Think _again_. + * + * Consider: + * 1) Taken branches have a 3 cycle penalty on R4k + * 2) The branch itself is a real dead cycle on even R4600/R5000. + * 3) Only one of the following variants of each type is even used by + * the kernel based upon the cache parameters we detect at boot time. + * + * QED. + */ + +static inline void r4k_flush_cache_all_s16d16i16(void) +{ + unsigned long flags; + + save_and_cli(flags); + blast_dcache16(); blast_icache16(); blast_scache16(); + restore_flags(flags); +} + +static inline void r4k_flush_cache_all_s32d16i16(void) +{ + unsigned long flags; + + save_and_cli(flags); + blast_dcache16(); blast_icache16(); blast_scache32(); + restore_flags(flags); +} + +static inline void r4k_flush_cache_all_s64d16i16(void) +{ + unsigned long flags; + + save_and_cli(flags); + blast_dcache16(); blast_icache16(); blast_scache64(); + restore_flags(flags); +} + +static inline void r4k_flush_cache_all_s128d16i16(void) +{ + unsigned long flags; + + save_and_cli(flags); + blast_dcache16(); blast_icache16(); blast_scache128(); + restore_flags(flags); +} + +static inline void r4k_flush_cache_all_s32d32i32(void) +{ + unsigned long flags; + + save_and_cli(flags); + blast_dcache32(); blast_icache32(); blast_scache32(); + restore_flags(flags); +} + +static inline void r4k_flush_cache_all_s64d32i32(void) +{ + unsigned long flags; + + save_and_cli(flags); + blast_dcache32(); blast_icache32(); blast_scache64(); + restore_flags(flags); +} + +static inline void r4k_flush_cache_all_s128d32i32(void) +{ + unsigned long flags; + + save_and_cli(flags); + blast_dcache32(); blast_icache32(); blast_scache128(); + restore_flags(flags); +} + +static inline void r4k_flush_cache_all_d16i16(void) +{ + unsigned long flags; + + save_and_cli(flags); + blast_dcache16(); blast_icache16(); + restore_flags(flags); +} + +static inline void r4k_flush_cache_all_d32i32(void) +{ + unsigned long flags; + + save_and_cli(flags); + blast_dcache32(); blast_icache32(); + restore_flags(flags); +} + +static void +r4k_flush_cache_range_s16d16i16(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + struct vm_area_struct *vma; + unsigned long flags; + + if(mm->context == 0) + return; + + start &= PAGE_MASK; +#ifdef DEBUG_CACHE + printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end); +#endif + vma = find_vma(mm, start); + if(vma) { + if(mm->context != current->mm->context) { + r4k_flush_cache_all_s16d16i16(); + } else { + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int text; + + save_and_cli(flags); + text = vma->vm_flags & VM_EXEC; + while(start < end) { + pgd = pgd_offset(mm, start); + pmd = pmd_offset(pgd, start); + pte = pte_offset(pmd, start); + + if(pte_val(*pte) & _PAGE_VALID) + blast_scache16_page(start); + start += PAGE_SIZE; + } + restore_flags(flags); + } + } +} + +static void +r4k_flush_cache_range_s32d16i16(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + struct vm_area_struct *vma; + unsigned long flags; + + if(mm->context == 0) + return; + + start &= PAGE_MASK; +#ifdef DEBUG_CACHE + printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end); +#endif + vma = find_vma(mm, start); + if(vma) { + if(mm->context != current->mm->context) { + r4k_flush_cache_all_s32d16i16(); + } else { + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int text; + + save_and_cli(flags); + text = vma->vm_flags & VM_EXEC; + while(start < end) { + pgd = pgd_offset(mm, start); + pmd = pmd_offset(pgd, start); + pte = pte_offset(pmd, start); + + if(pte_val(*pte) & _PAGE_VALID) + blast_scache32_page(start); + start += PAGE_SIZE; + } + restore_flags(flags); + } + } +} + +static void r4k_flush_cache_range_s64d16i16(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + struct vm_area_struct *vma; + unsigned long flags; + + if(mm->context == 0) + return; + + start &= PAGE_MASK; +#ifdef DEBUG_CACHE + printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end); +#endif + vma = find_vma(mm, start); + if(vma) { + if(mm->context != current->mm->context) { + r4k_flush_cache_all_s64d16i16(); + } else { + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int text; + + save_and_cli(flags); + text = vma->vm_flags & VM_EXEC; + while(start < end) { + pgd = pgd_offset(mm, start); + pmd = pmd_offset(pgd, start); + pte = pte_offset(pmd, start); + + if(pte_val(*pte) & _PAGE_VALID) + blast_scache64_page(start); + start += PAGE_SIZE; + } + restore_flags(flags); + } + } +} + +static void r4k_flush_cache_range_s128d16i16(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + struct vm_area_struct *vma; + unsigned long flags; + + if(mm->context == 0) + return; + + start &= PAGE_MASK; +#ifdef DEBUG_CACHE + printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end); +#endif + vma = find_vma(mm, start); + if(vma) { + if(mm->context != current->mm->context) { + r4k_flush_cache_all_s128d16i16(); + } else { + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int text; + + save_and_cli(flags); + text = vma->vm_flags & VM_EXEC; + while(start < end) { + pgd = pgd_offset(mm, start); + pmd = pmd_offset(pgd, start); + pte = pte_offset(pmd, start); + + if(pte_val(*pte) & _PAGE_VALID) + blast_scache128_page(start); + start += PAGE_SIZE; + } + restore_flags(flags); + } + } +} + +static void r4k_flush_cache_range_s32d32i32(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + struct vm_area_struct *vma; + unsigned long flags; + + if(mm->context == 0) + return; + + start &= PAGE_MASK; +#ifdef DEBUG_CACHE + printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end); +#endif + vma = find_vma(mm, start); + if(vma) { + if(mm->context != current->mm->context) { + r4k_flush_cache_all_s32d32i32(); + } else { + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int text; + + save_and_cli(flags); + text = vma->vm_flags & VM_EXEC; + while(start < end) { + pgd = pgd_offset(mm, start); + pmd = pmd_offset(pgd, start); + pte = pte_offset(pmd, start); + + if(pte_val(*pte) & _PAGE_VALID) + blast_scache32_page(start); + start += PAGE_SIZE; + } + restore_flags(flags); + } + } +} + +static void r4k_flush_cache_range_s64d32i32(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + struct vm_area_struct *vma; + unsigned long flags; + + if(mm->context == 0) + return; + + start &= PAGE_MASK; +#ifdef DEBUG_CACHE + printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end); +#endif + vma = find_vma(mm, start); + if(vma) { + if(mm->context != current->mm->context) { + r4k_flush_cache_all_s64d32i32(); + } else { + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int text; + + save_and_cli(flags); + text = vma->vm_flags & VM_EXEC; + while(start < end) { + pgd = pgd_offset(mm, start); + pmd = pmd_offset(pgd, start); + pte = pte_offset(pmd, start); + + if(pte_val(*pte) & _PAGE_VALID) + blast_scache64_page(start); + start += PAGE_SIZE; + } + restore_flags(flags); + } + } +} + +static void r4k_flush_cache_range_s128d32i32(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + struct vm_area_struct *vma; + unsigned long flags; + + if(mm->context == 0) + return; + + start &= PAGE_MASK; +#ifdef DEBUG_CACHE + printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end); +#endif + vma = find_vma(mm, start); + if(vma) { + if(mm->context != current->mm->context) { + r4k_flush_cache_all_s128d32i32(); + } else { + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int text; + + save_and_cli(flags); + text = vma->vm_flags & VM_EXEC; + while(start < end) { + pgd = pgd_offset(mm, start); + pmd = pmd_offset(pgd, start); + pte = pte_offset(pmd, start); + + if(pte_val(*pte) & _PAGE_VALID) + blast_scache128_page(start); + start += PAGE_SIZE; + } + restore_flags(flags); + } + } +} + +static void r4k_flush_cache_range_d16i16(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + if(mm->context != 0) { + unsigned long flags; + +#ifdef DEBUG_CACHE + printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end); +#endif + save_and_cli(flags); + blast_dcache16(); blast_icache16(); + restore_flags(flags); + } +} + +static void r4k_flush_cache_range_d32i32(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + if(mm->context != 0) { + unsigned long flags; + +#ifdef DEBUG_CACHE + printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end); +#endif + save_and_cli(flags); + blast_dcache32(); blast_icache32(); + restore_flags(flags); + } +} + +/* + * On architectures like the Sparc, we could get rid of lines in + * the cache created only by a certain context, but on the MIPS + * (and actually certain Sparc's) we cannot. + */ +static void r4k_flush_cache_mm_s16d16i16(struct mm_struct *mm) +{ + if(mm->context != 0) { +#ifdef DEBUG_CACHE + printk("cmm[%d]", (int)mm->context); +#endif + r4k_flush_cache_all_s16d16i16(); + } +} + +static void r4k_flush_cache_mm_s32d16i16(struct mm_struct *mm) +{ + if(mm->context != 0) { +#ifdef DEBUG_CACHE + printk("cmm[%d]", (int)mm->context); +#endif + r4k_flush_cache_all_s32d16i16(); + } +} + +static void r4k_flush_cache_mm_s64d16i16(struct mm_struct *mm) +{ + if(mm->context != 0) { +#ifdef DEBUG_CACHE + printk("cmm[%d]", (int)mm->context); +#endif + r4k_flush_cache_all_s64d16i16(); + } +} + +static void r4k_flush_cache_mm_s128d16i16(struct mm_struct *mm) +{ + if(mm->context != 0) { +#ifdef DEBUG_CACHE + printk("cmm[%d]", (int)mm->context); +#endif + r4k_flush_cache_all_s128d16i16(); + } +} + +static void r4k_flush_cache_mm_s32d32i32(struct mm_struct *mm) +{ + if(mm->context != 0) { +#ifdef DEBUG_CACHE + printk("cmm[%d]", (int)mm->context); +#endif + r4k_flush_cache_all_s32d32i32(); + } +} + +static void r4k_flush_cache_mm_s64d32i32(struct mm_struct *mm) +{ + if(mm->context != 0) { +#ifdef DEBUG_CACHE + printk("cmm[%d]", (int)mm->context); +#endif + r4k_flush_cache_all_s64d32i32(); + } +} + +static void r4k_flush_cache_mm_s128d32i32(struct mm_struct *mm) +{ + if(mm->context != 0) { +#ifdef DEBUG_CACHE + printk("cmm[%d]", (int)mm->context); +#endif + r4k_flush_cache_all_s128d32i32(); + } +} + +static void r4k_flush_cache_mm_d16i16(struct mm_struct *mm) +{ + if(mm->context != 0) { +#ifdef DEBUG_CACHE + printk("cmm[%d]", (int)mm->context); +#endif + r4k_flush_cache_all_d16i16(); + } +} + +static void r4k_flush_cache_mm_d32i32(struct mm_struct *mm) +{ + if(mm->context != 0) { +#ifdef DEBUG_CACHE + printk("cmm[%d]", (int)mm->context); +#endif + r4k_flush_cache_all_d32i32(); + } +} + +static void r4k_flush_cache_page_s16d16i16(struct vm_area_struct *vma, + unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int text; + + /* + * If ownes no valid ASID yet, cannot possibly have gotten + * this page into the cache. + */ + if(mm->context == 0) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + save_and_cli(flags); + page &= PAGE_MASK; + pgdp = pgd_offset(mm, page); + pmdp = pmd_offset(pgdp, page); + ptep = pte_offset(pmdp, page); + + /* + * If the page isn't marked valid, the page cannot possibly be + * in the cache. + */ + if(!(pte_val(*ptep) & _PAGE_VALID)) + goto out; + + text = (vma->vm_flags & VM_EXEC); + /* Doing flushes for another ASID than the current one is + * too difficult since stupid R4k caches do a TLB translation + * for every cache flush operation. So we do indexed flushes + * in that case, which doesn't overly flush the cache too much. + */ + if(mm->context != current->mm->context) { + /* Do indexed flush, too much work to get the (possible) + * tlb refills to work correctly. + */ + page = (KSEG0 + (page & (scache_size - 1))); + blast_dcache16_page_indexed(page); + if(text) + blast_icache16_page_indexed(page); + blast_scache16_page_indexed(page); + } else + blast_scache16_page(page); +out: + restore_flags(flags); +} + +static void r4k_flush_cache_page_s32d16i16(struct vm_area_struct *vma, + unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int text; + + /* + * If ownes no valid ASID yet, cannot possibly have gotten + * this page into the cache. + */ + if(mm->context == 0) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + save_and_cli(flags); + page &= PAGE_MASK; + pgdp = pgd_offset(mm, page); + pmdp = pmd_offset(pgdp, page); + ptep = pte_offset(pmdp, page); + + /* If the page isn't marked valid, the page cannot possibly be + * in the cache. + */ + if(!(pte_val(*ptep) & _PAGE_VALID)) + goto out; + + text = (vma->vm_flags & VM_EXEC); + /* Doing flushes for another ASID than the current one is + * too difficult since stupid R4k caches do a TLB translation + * for every cache flush operation. So we do indexed flushes + * in that case, which doesn't overly flush the cache too much. + */ + if(mm->context != current->mm->context) { + /* Do indexed flush, too much work to get the (possible) + * tlb refills to work correctly. + */ + page = (KSEG0 + (page & (scache_size - 1))); + blast_dcache16_page_indexed(page); + if(text) + blast_icache16_page_indexed(page); + blast_scache32_page_indexed(page); + } else + blast_scache32_page(page); +out: + restore_flags(flags); +} + +static void r4k_flush_cache_page_s64d16i16(struct vm_area_struct *vma, + unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int text; + + /* + * If ownes no valid ASID yet, cannot possibly have gotten + * this page into the cache. + */ + if(mm->context == 0) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + save_and_cli(flags); + page &= PAGE_MASK; + pgdp = pgd_offset(mm, page); + pmdp = pmd_offset(pgdp, page); + ptep = pte_offset(pmdp, page); + + /* If the page isn't marked valid, the page cannot possibly be + * in the cache. + */ + if(!(pte_val(*ptep) & _PAGE_VALID)) + goto out; + + text = (vma->vm_flags & VM_EXEC); + /* + * Doing flushes for another ASID than the current one is + * too difficult since stupid R4k caches do a TLB translation + * for every cache flush operation. So we do indexed flushes + * in that case, which doesn't overly flush the cache too much. + */ + if(mm->context != current->mm->context) { + /* Do indexed flush, too much work to get the (possible) + * tlb refills to work correctly. + */ + page = (KSEG0 + (page & (scache_size - 1))); + blast_dcache16_page_indexed(page); + if(text) + blast_icache16_page_indexed(page); + blast_scache64_page_indexed(page); + } else + blast_scache64_page(page); +out: + restore_flags(flags); +} + +static void r4k_flush_cache_page_s128d16i16(struct vm_area_struct *vma, + unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int text; + + /* + * If ownes no valid ASID yet, cannot possibly have gotten + * this page into the cache. + */ + if(mm->context == 0) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + save_and_cli(flags); + page &= PAGE_MASK; + pgdp = pgd_offset(mm, page); + pmdp = pmd_offset(pgdp, page); + ptep = pte_offset(pmdp, page); + + /* + * If the page isn't marked valid, the page cannot possibly be + * in the cache. + */ + if(!(pte_val(*ptep) & _PAGE_VALID)) + goto out; + + text = (vma->vm_flags & VM_EXEC); + /* Doing flushes for another ASID than the current one is + * too difficult since stupid R4k caches do a TLB translation + * for every cache flush operation. So we do indexed flushes + * in that case, which doesn't overly flush the cache too much. + */ + if(mm->context != current->mm->context) { + /* + * Do indexed flush, too much work to get the (possible) + * tlb refills to work correctly. + */ + page = (KSEG0 + (page & (scache_size - 1))); + blast_dcache16_page_indexed(page); + if(text) + blast_icache16_page_indexed(page); + blast_scache128_page_indexed(page); + } else + blast_scache128_page(page); +out: + restore_flags(flags); +} + +static void r4k_flush_cache_page_s32d32i32(struct vm_area_struct *vma, + unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int text; + + /* + * If ownes no valid ASID yet, cannot possibly have gotten + * this page into the cache. + */ + if(mm->context == 0) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + save_and_cli(flags); + page &= PAGE_MASK; + pgdp = pgd_offset(mm, page); + pmdp = pmd_offset(pgdp, page); + ptep = pte_offset(pmdp, page); + + /* + * If the page isn't marked valid, the page cannot possibly be + * in the cache. + */ + if(!(pte_val(*ptep) & _PAGE_VALID)) + goto out; + + text = (vma->vm_flags & VM_EXEC); + /* + * Doing flushes for another ASID than the current one is + * too difficult since stupid R4k caches do a TLB translation + * for every cache flush operation. So we do indexed flushes + * in that case, which doesn't overly flush the cache too much. + */ + if(mm->context != current->mm->context) { + /* + * Do indexed flush, too much work to get the (possible) + * tlb refills to work correctly. + */ + page = (KSEG0 + (page & (scache_size - 1))); + blast_dcache32_page_indexed(page); + if(text) + blast_icache32_page_indexed(page); + blast_scache32_page_indexed(page); + } else + blast_scache32_page(page); +out: + restore_flags(flags); +} + +static void r4k_flush_cache_page_s64d32i32(struct vm_area_struct *vma, + unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int text; + + /* + * If ownes no valid ASID yet, cannot possibly have gotten + * this page into the cache. + */ + if(mm->context == 0) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + save_and_cli(flags); + page &= PAGE_MASK; + pgdp = pgd_offset(mm, page); + pmdp = pmd_offset(pgdp, page); + ptep = pte_offset(pmdp, page); + + /* + * If the page isn't marked valid, the page cannot possibly be + * in the cache. + */ + if(!(pte_val(*ptep) & _PAGE_VALID)) + goto out; + + text = (vma->vm_flags & VM_EXEC); + /* + * Doing flushes for another ASID than the current one is + * too difficult since stupid R4k caches do a TLB translation + * for every cache flush operation. So we do indexed flushes + * in that case, which doesn't overly flush the cache too much. + */ + if(mm->context != current->mm->context) { + /* + * Do indexed flush, too much work to get the (possible) + * tlb refills to work correctly. + */ + page = (KSEG0 + (page & (scache_size - 1))); + blast_dcache32_page_indexed(page); + if(text) + blast_icache32_page_indexed(page); + blast_scache64_page_indexed(page); + } else + blast_scache64_page(page); +out: + restore_flags(flags); +} + +static void r4k_flush_cache_page_s128d32i32(struct vm_area_struct *vma, + unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int text; + + /* + * If ownes no valid ASID yet, cannot possibly have gotten + * this page into the cache. + */ + if(mm->context == 0) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + save_and_cli(flags); + page &= PAGE_MASK; + pgdp = pgd_offset(mm, page); + pmdp = pmd_offset(pgdp, page); + ptep = pte_offset(pmdp, page); + + /* If the page isn't marked valid, the page cannot possibly be + * in the cache. + */ + if(!(pte_val(*ptep) & _PAGE_VALID)) + goto out; + + text = (vma->vm_flags & VM_EXEC); + /* + * Doing flushes for another ASID than the current one is + * too difficult since stupid R4k caches do a TLB translation + * for every cache flush operation. So we do indexed flushes + * in that case, which doesn't overly flush the cache too much. + */ + if(mm->context != current->mm->context) { + /* Do indexed flush, too much work to get the (possible) + * tlb refills to work correctly. + */ + page = (KSEG0 + (page & (scache_size - 1))); + blast_dcache32_page_indexed(page); + if(text) + blast_icache32_page_indexed(page); + blast_scache128_page_indexed(page); + } else + blast_scache128_page(page); +out: + restore_flags(flags); +} + +static void r4k_flush_cache_page_d16i16(struct vm_area_struct *vma, + unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int text; + + /* + * If ownes no valid ASID yet, cannot possibly have gotten + * this page into the cache. + */ + if(mm->context == 0) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + save_and_cli(flags); + page &= PAGE_MASK; + pgdp = pgd_offset(mm, page); + pmdp = pmd_offset(pgdp, page); + ptep = pte_offset(pmdp, page); + + /* If the page isn't marked valid, the page cannot possibly be + * in the cache. + */ + if(!(pte_val(*ptep) & _PAGE_VALID)) + goto out; + + text = (vma->vm_flags & VM_EXEC); + /* + * Doing flushes for another ASID than the current one is + * too difficult since stupid R4k caches do a TLB translation + * for every cache flush operation. So we do indexed flushes + * in that case, which doesn't overly flush the cache too much. + */ + if(mm == current->mm) { + blast_dcache16_page(page); + if(text) + blast_icache16_page(page); + } else { + /* Do indexed flush, too much work to get the (possible) + * tlb refills to work correctly. + */ + page = (KSEG0 + (page & (dcache_size - 1))); + blast_dcache16_page_indexed(page); + if(text) + blast_icache16_page_indexed(page); + } +out: + restore_flags(flags); +} + +static void r4k_flush_cache_page_d32i32(struct vm_area_struct *vma, + unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int text; + + /* + * If ownes no valid ASID yet, cannot possibly have gotten + * this page into the cache. + */ + if(mm->context == 0) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + save_and_cli(flags); + page &= PAGE_MASK; + pgdp = pgd_offset(mm, page); + pmdp = pmd_offset(pgdp, page); + ptep = pte_offset(pmdp, page); + + /* + * If the page isn't marked valid, the page cannot possibly be + * in the cache. + */ + if(!(pte_val(*ptep) & _PAGE_PRESENT)) + goto out; + + text = (vma->vm_flags & VM_EXEC); + /* + * Doing flushes for another ASID than the current one is + * too difficult since stupid R4k caches do a TLB translation + * for every cache flush operation. So we do indexed flushes + * in that case, which doesn't overly flush the cache too much. + */ + if((mm == current->mm) && (pte_val(*ptep) & _PAGE_VALID)) { + blast_dcache32_page(page); + if(text) + blast_icache32_page(page); + } else { + /* + * Do indexed flush, too much work to get the (possible) + * tlb refills to work correctly. + */ + page = (KSEG0 + (page & (dcache_size - 1))); + blast_dcache32_page_indexed(page); + if(text) + blast_icache32_page_indexed(page); + } +out: + restore_flags(flags); +} + +static void r4k_flush_cache_page_d32i32_r4600(struct vm_area_struct *vma, + unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int text; + + /* + * If ownes no valid ASID yet, cannot possibly have gotten + * this page into the cache. + */ + if(mm->context == 0) + return; + +#ifdef DEBUG_CACHE + printk("cpage[%d,%08lx]", (int)mm->context, page); +#endif + save_and_cli(flags); + page &= PAGE_MASK; + pgdp = pgd_offset(mm, page); + pmdp = pmd_offset(pgdp, page); + ptep = pte_offset(pmdp, page); + + /* + * If the page isn't marked valid, the page cannot possibly be + * in the cache. + */ + if(!(pte_val(*ptep) & _PAGE_PRESENT)) + goto out; + + text = (vma->vm_flags & VM_EXEC); + /* + * Doing flushes for another ASID than the current one is + * too difficult since stupid R4k caches do a TLB translation + * for every cache flush operation. So we do indexed flushes + * in that case, which doesn't overly flush the cache too much. + */ + if((mm == current->mm) && (pte_val(*ptep) & _PAGE_VALID)) { + blast_dcache32_page(page); + if(text) + blast_icache32_page(page); + } else { + /* Do indexed flush, too much work to get the (possible) + * tlb refills to work correctly. + */ + page = (KSEG0 + (page & (dcache_size - 1))); + blast_dcache32_page_indexed(page); + blast_dcache32_page_indexed(page ^ dcache_waybit); + if(text) { + blast_icache32_page_indexed(page); + blast_icache32_page_indexed(page ^ icache_waybit); + } + } +out: + restore_flags(flags); +} + +/* If the addresses passed to these routines are valid, they are + * either: + * + * 1) In KSEG0, so we can do a direct flush of the page. + * 2) In KSEG2, and since every process can translate those + * addresses all the time in kernel mode we can do a direct + * flush. + * 3) In KSEG1, no flush necessary. + */ +static void r4k_flush_page_to_ram_s16d16i16(unsigned long page) +{ + page &= PAGE_MASK; + if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) { +#ifdef DEBUG_CACHE + printk("cram[%08lx]", page); +#endif + blast_scache16_page(page); + } +} + +static void r4k_flush_page_to_ram_s32d16i16(unsigned long page) +{ + page &= PAGE_MASK; + if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) { +#ifdef DEBUG_CACHE + printk("cram[%08lx]", page); +#endif + blast_scache32_page(page); + } +} + +static void r4k_flush_page_to_ram_s64d16i16(unsigned long page) +{ + page &= PAGE_MASK; + if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) { +#ifdef DEBUG_CACHE + printk("cram[%08lx]", page); +#endif + blast_scache64_page(page); + } +} + +static void r4k_flush_page_to_ram_s128d16i16(unsigned long page) +{ + page &= PAGE_MASK; + if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) { +#ifdef DEBUG_CACHE + printk("cram[%08lx]", page); +#endif + blast_scache128_page(page); + } +} + +static void r4k_flush_page_to_ram_s32d32i32(unsigned long page) +{ + page &= PAGE_MASK; + if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) { +#ifdef DEBUG_CACHE + printk("cram[%08lx]", page); +#endif + blast_scache32_page(page); + } +} + +static void r4k_flush_page_to_ram_s64d32i32(unsigned long page) +{ + page &= PAGE_MASK; + if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) { +#ifdef DEBUG_CACHE + printk("cram[%08lx]", page); +#endif + blast_scache64_page(page); + } +} + +static void r4k_flush_page_to_ram_s128d32i32(unsigned long page) +{ + page &= PAGE_MASK; + if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) { +#ifdef DEBUG_CACHE + printk("cram[%08lx]", page); +#endif + blast_scache128_page(page); + } +} + +static void r4k_flush_page_to_ram_d16i16(unsigned long page) +{ + page &= PAGE_MASK; + if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) { + unsigned long flags; + +#ifdef DEBUG_CACHE + printk("cram[%08lx]", page); +#endif + save_and_cli(flags); + blast_dcache16_page(page); + restore_flags(flags); + } +} + +static void r4k_flush_page_to_ram_d32i32(unsigned long page) +{ + page &= PAGE_MASK; + if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) { + unsigned long flags; + +#ifdef DEBUG_CACHE + printk("cram[%08lx]", page); +#endif + save_and_cli(flags); + blast_dcache32_page(page); + restore_flags(flags); + } +} + +/* + * Writeback and invalidate the primary cache dcache before DMA. + * + * R4600 v2.0 bug: "The CACHE instructions Hit_Writeback_Inv_D, + * Hit_Writeback_D, Hit_Invalidate_D and Create_Dirty_Exclusive_D will only + * operate correctly if the internal data cache refill buffer is empty. These + * CACHE instructions should be separated from any potential data cache miss + * by a load instruction to an uncached address to empty the response buffer." + * (Revision 2.0 device errata from IDT available on http://www.idt.com/ + * in .pdf format.) + */ +static void +r4k_dma_cache_wback_inv_pc(unsigned long addr, unsigned long size) +{ + unsigned long end, a; + unsigned int flags; + + if (size >= dcache_size) { + flush_cache_all(); + } else { + /* Workaround for R4600 bug. See comment above. */ + save_and_cli(flags); + *(volatile unsigned long *)KSEG1; + + a = addr & ~(dc_lsize - 1); + end = (addr + size) & ~(dc_lsize - 1); + while (1) { + flush_dcache_line(a); /* Hit_Writeback_Inv_D */ + if (a == end) break; + a += dc_lsize; + } + restore_flags(flags); + } + bcops->bc_wback_inv(addr, size); +} + +static void +r4k_dma_cache_wback_inv_sc(unsigned long addr, unsigned long size) +{ + unsigned long end, a; + + if (size >= scache_size) { + flush_cache_all(); + return; + } + + a = addr & ~(sc_lsize - 1); + end = (addr + size) & ~(sc_lsize - 1); + while (1) { + flush_scache_line(a); /* Hit_Writeback_Inv_SD */ + if (a == end) break; + a += sc_lsize; + } +} + +static void +r4k_dma_cache_inv_pc(unsigned long addr, unsigned long size) +{ + unsigned long end, a; + unsigned int flags; + + if (size >= dcache_size) { + flush_cache_all(); + } else { + /* Workaround for R4600 bug. See comment above. */ + save_and_cli(flags); + *(volatile unsigned long *)KSEG1; + + a = addr & ~(dc_lsize - 1); + end = (addr + size) & ~(dc_lsize - 1); + while (1) { + flush_dcache_line(a); /* Hit_Writeback_Inv_D */ + if (a == end) break; + a += dc_lsize; + } + restore_flags(flags); + } + + bcops->bc_inv(addr, size); +} + +static void +r4k_dma_cache_inv_sc(unsigned long addr, unsigned long size) +{ + unsigned long end, a; + + if (size >= scache_size) { + flush_cache_all(); + return; + } + + a = addr & ~(sc_lsize - 1); + end = (addr + size) & ~(sc_lsize - 1); + while (1) { + flush_scache_line(a); /* Hit_Writeback_Inv_SD */ + if (a == end) break; + a += sc_lsize; + } +} + +static void +r4k_dma_cache_wback(unsigned long addr, unsigned long size) +{ + panic("r4k_dma_cache called - should not happen.\n"); +} + +/* + * While we're protected against bad userland addresses we don't care + * very much about what happens in that case. Usually a segmentation + * fault will dump the process later on anyway ... + */ +static void r4k_flush_cache_sigtramp(unsigned long addr) +{ + unsigned long daddr, iaddr; + + daddr = addr & ~(dc_lsize - 1); + __asm__ __volatile__("nop;nop;nop;nop"); /* R4600 V1.7 */ + protected_writeback_dcache_line(daddr); + protected_writeback_dcache_line(daddr + dc_lsize); + iaddr = addr & ~(ic_lsize - 1); + protected_flush_icache_line(iaddr); + protected_flush_icache_line(iaddr + ic_lsize); +} + +static void r4600v20k_flush_cache_sigtramp(unsigned long addr) +{ + unsigned long daddr, iaddr; + unsigned int flags; + + daddr = addr & ~(dc_lsize - 1); + save_and_cli(flags); + + /* Clear internal cache refill buffer */ + *(volatile unsigned int *)KSEG1; + + protected_writeback_dcache_line(daddr); + protected_writeback_dcache_line(daddr + dc_lsize); + iaddr = addr & ~(ic_lsize - 1); + protected_flush_icache_line(iaddr); + protected_flush_icache_line(iaddr + ic_lsize); + restore_flags(flags); +} + +#undef DEBUG_TLB +#undef DEBUG_TLBUPDATE + +#define NTLB_ENTRIES 48 /* Fixed on all R4XX0 variants... */ + +#define NTLB_ENTRIES_HALF 24 /* Fixed on all R4XX0 variants... */ + +static inline void r4k_flush_tlb_all(void) +{ + unsigned long flags; + unsigned long old_ctx; + int entry; + +#ifdef DEBUG_TLB + printk("[tlball]"); +#endif + + save_and_cli(flags); + /* Save old context and create impossible VPN2 value */ + old_ctx = (get_entryhi() & 0xff); + set_entryhi(KSEG0); + set_entrylo0(0); + set_entrylo1(0); + BARRIER; + + entry = get_wired(); + + /* Blast 'em all away. */ + while(entry < NTLB_ENTRIES) { + set_index(entry); + BARRIER; + tlb_write_indexed(); + BARRIER; + entry++; + } + BARRIER; + set_entryhi(old_ctx); + restore_flags(flags); +} + +static void r4k_flush_tlb_mm(struct mm_struct *mm) +{ + if(mm->context != 0) { + unsigned long flags; + +#ifdef DEBUG_TLB + printk("[tlbmm<%d>]", mm->context); +#endif + save_and_cli(flags); + get_new_mmu_context(mm, asid_cache); + if(mm == current->mm) + set_entryhi(mm->context & 0xff); + restore_flags(flags); + } +} + +static void r4k_flush_tlb_range(struct mm_struct *mm, unsigned long start, + unsigned long end) +{ + if(mm->context != 0) { + unsigned long flags; + int size; + +#ifdef DEBUG_TLB + printk("[tlbrange<%02x,%08lx,%08lx>]", (mm->context & 0xff), + start, end); +#endif + save_and_cli(flags); + size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + size = (size + 1) >> 1; + if(size <= NTLB_ENTRIES_HALF) { + int oldpid = (get_entryhi() & 0xff); + int newpid = (mm->context & 0xff); + + start &= (PAGE_MASK << 1); + end += ((PAGE_SIZE << 1) - 1); + end &= (PAGE_MASK << 1); + while(start < end) { + int idx; + + set_entryhi(start | newpid); + start += (PAGE_SIZE << 1); + BARRIER; + tlb_probe(); + BARRIER; + idx = get_index(); + set_entrylo0(0); + set_entrylo1(0); + set_entryhi(KSEG0); + BARRIER; + if(idx < 0) + continue; + tlb_write_indexed(); + BARRIER; + } + set_entryhi(oldpid); + } else { + get_new_mmu_context(mm, asid_cache); + if(mm == current->mm) + set_entryhi(mm->context & 0xff); + } + restore_flags(flags); + } +} + +static void r4k_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + if(vma->vm_mm->context != 0) { + unsigned long flags; + int oldpid, newpid, idx; + +#ifdef DEBUG_TLB + printk("[tlbpage<%d,%08lx>]", vma->vm_mm->context, page); +#endif + newpid = (vma->vm_mm->context & 0xff); + page &= (PAGE_MASK << 1); + save_and_cli(flags); + oldpid = (get_entryhi() & 0xff); + set_entryhi(page | newpid); + BARRIER; + tlb_probe(); + BARRIER; + idx = get_index(); + set_entrylo0(0); + set_entrylo1(0); + set_entryhi(KSEG0); + if(idx < 0) + goto finish; + BARRIER; + tlb_write_indexed(); + + finish: + BARRIER; + set_entryhi(oldpid); + restore_flags(flags); + } +} + +/* Load a new root pointer into the TLB. */ +static void r4k_load_pgd(unsigned long pg_dir) +{ +} + +static void r4k_pgd_init(unsigned long page) +{ + unsigned long *p = (unsigned long *) page; + int i; + + for(i = 0; i < USER_PTRS_PER_PGD; i+=8) { + p[i + 0] = (unsigned long) invalid_pte_table; + p[i + 1] = (unsigned long) invalid_pte_table; + p[i + 2] = (unsigned long) invalid_pte_table; + p[i + 3] = (unsigned long) invalid_pte_table; + p[i + 4] = (unsigned long) invalid_pte_table; + p[i + 5] = (unsigned long) invalid_pte_table; + p[i + 6] = (unsigned long) invalid_pte_table; + p[i + 7] = (unsigned long) invalid_pte_table; + } +} + +#ifdef DEBUG_TLBUPDATE +static unsigned long ehi_debug[NTLB_ENTRIES]; +static unsigned long el0_debug[NTLB_ENTRIES]; +static unsigned long el1_debug[NTLB_ENTRIES]; +#endif + +/* We will need multiple versions of update_mmu_cache(), one that just + * updates the TLB with the new pte(s), and another which also checks + * for the R4k "end of page" hardware bug and does the needy. + */ +static void r4k_update_mmu_cache(struct vm_area_struct * vma, + unsigned long address, pte_t pte) +{ + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int idx, pid; + + pid = (get_entryhi() & 0xff); + +#ifdef DEBUG_TLB + if((pid != (vma->vm_mm->context & 0xff)) || (vma->vm_mm->context == 0)) { + printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%d tlbpid=%d\n", + (int) (vma->vm_mm->context & 0xff), pid); + } +#endif + + save_and_cli(flags); + address &= (PAGE_MASK << 1); + set_entryhi(address | (pid)); + pgdp = pgd_offset(vma->vm_mm, address); + BARRIER; + tlb_probe(); + BARRIER; + pmdp = pmd_offset(pgdp, address); + idx = get_index(); + ptep = pte_offset(pmdp, address); + BARRIER; + set_entrylo0(pte_val(*ptep++) >> 6); + set_entrylo1(pte_val(*ptep) >> 6); + set_entryhi(address | (pid)); + BARRIER; + if(idx < 0) { + tlb_write_random(); + } else { + tlb_write_indexed(); + } + BARRIER; + set_entryhi(pid); + BARRIER; + restore_flags(flags); +} + +#if 0 +static void r4k_update_mmu_cache_hwbug(struct vm_area_struct * vma, + unsigned long address, pte_t pte) +{ + unsigned long flags; + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + int idx; + + save_and_cli(flags); + address &= (PAGE_MASK << 1); + set_entryhi(address | (get_entryhi() & 0xff)); + pgdp = pgd_offset(vma->vm_mm, address); + tlb_probe(); + pmdp = pmd_offset(pgdp, address); + idx = get_index(); + ptep = pte_offset(pmdp, address); + set_entrylo0(pte_val(*ptep++) >> 6); + set_entrylo1(pte_val(*ptep) >> 6); + BARRIER; + if(idx < 0) + tlb_write_random(); + else + tlb_write_indexed(); + BARRIER; + restore_flags(flags); +} +#endif + +static void r4k_show_regs(struct pt_regs * regs) +{ + /* Saved main processor registers. */ + printk("$0 : %08lx %08lx %08lx %08lx\n", + 0UL, regs->regs[1], regs->regs[2], regs->regs[3]); + printk("$4 : %08lx %08lx %08lx %08lx\n", + regs->regs[4], regs->regs[5], regs->regs[6], regs->regs[7]); + printk("$8 : %08lx %08lx %08lx %08lx\n", + regs->regs[8], regs->regs[9], regs->regs[10], regs->regs[11]); + printk("$12: %08lx %08lx %08lx %08lx\n", + regs->regs[12], regs->regs[13], regs->regs[14], regs->regs[15]); + printk("$16: %08lx %08lx %08lx %08lx\n", + regs->regs[16], regs->regs[17], regs->regs[18], regs->regs[19]); + printk("$20: %08lx %08lx %08lx %08lx\n", + regs->regs[20], regs->regs[21], regs->regs[22], regs->regs[23]); + printk("$24: %08lx %08lx\n", + regs->regs[24], regs->regs[25]); + printk("$28: %08lx %08lx %08lx %08lx\n", + regs->regs[28], regs->regs[29], regs->regs[30], regs->regs[31]); + + /* Saved cp0 registers. */ + printk("epc : %08lx\nStatus: %08lx\nCause : %08lx\n", + regs->cp0_epc, regs->cp0_status, regs->cp0_cause); +} + +static void r4k_add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, + unsigned long entryhi, unsigned long pagemask) +{ + unsigned long flags; + unsigned long wired; + unsigned long old_pagemask; + unsigned long old_ctx; + + save_and_cli(flags); + /* Save old context and create impossible VPN2 value */ + old_ctx = (get_entryhi() & 0xff); + old_pagemask = get_pagemask(); + wired = get_wired(); + set_wired (wired + 1); + set_index (wired); + BARRIER; + set_pagemask (pagemask); + set_entryhi(entryhi); + set_entrylo0(entrylo0); + set_entrylo1(entrylo1); + BARRIER; + tlb_write_indexed(); + BARRIER; + + set_entryhi(old_ctx); + BARRIER; + set_pagemask (old_pagemask); + flush_tlb_all(); + restore_flags(flags); +} + +/* Detect and size the various r4k caches. */ +static void __init probe_icache(unsigned long config) +{ + icache_size = 1 << (12 + ((config >> 9) & 7)); + ic_lsize = 16 << ((config >> 5) & 1); + + printk("Primary instruction cache %dkb, linesize %d bytes)\n", + icache_size >> 10, ic_lsize); +} + +static void __init probe_dcache(unsigned long config) +{ + dcache_size = 1 << (12 + ((config >> 6) & 7)); + dc_lsize = 16 << ((config >> 4) & 1); + + printk("Primary data cache %dkb, linesize %d bytes)\n", + dcache_size >> 10, dc_lsize); +} + + +/* If you even _breathe_ on this function, look at the gcc output + * and make sure it does not pop things on and off the stack for + * the cache sizing loop that executes in KSEG1 space or else + * you will crash and burn badly. You have been warned. + */ +static int __init probe_scache(unsigned long config) +{ + extern unsigned long stext; + unsigned long flags, addr, begin, end, pow2; + int tmp; + + tmp = ((config >> 17) & 1); + if(tmp) + return 0; + tmp = ((config >> 22) & 3); + switch(tmp) { + case 0: + sc_lsize = 16; + break; + case 1: + sc_lsize = 32; + break; + case 2: + sc_lsize = 64; + break; + case 3: + sc_lsize = 128; + break; + } + + begin = (unsigned long) &stext; + begin &= ~((4 * 1024 * 1024) - 1); + end = begin + (4 * 1024 * 1024); + + /* This is such a bitch, you'd think they would make it + * easy to do this. Away you daemons of stupidity! + */ + save_and_cli(flags); + + /* Fill each size-multiple cache line with a valid tag. */ + pow2 = (64 * 1024); + for(addr = begin; addr < end; addr = (begin + pow2)) { + unsigned long *p = (unsigned long *) addr; + __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ + pow2 <<= 1; + } + + /* Load first line with zero (therefore invalid) tag. */ + set_taglo(0); + set_taghi(0); + __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ + __asm__ __volatile__("\n\t.set noreorder\n\t" + ".set mips3\n\t" + "cache 8, (%0)\n\t" + ".set mips0\n\t" + ".set reorder\n\t" : : "r" (begin)); + __asm__ __volatile__("\n\t.set noreorder\n\t" + ".set mips3\n\t" + "cache 9, (%0)\n\t" + ".set mips0\n\t" + ".set reorder\n\t" : : "r" (begin)); + __asm__ __volatile__("\n\t.set noreorder\n\t" + ".set mips3\n\t" + "cache 11, (%0)\n\t" + ".set mips0\n\t" + ".set reorder\n\t" : : "r" (begin)); + + /* Now search for the wrap around point. */ + pow2 = (128 * 1024); + tmp = 0; + for(addr = (begin + (128 * 1024)); addr < (end); addr = (begin + pow2)) { + __asm__ __volatile__("\n\t.set noreorder\n\t" + ".set mips3\n\t" + "cache 7, (%0)\n\t" + ".set mips0\n\t" + ".set reorder\n\t" : : "r" (addr)); + __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ + if(!get_taglo()) + break; + pow2 <<= 1; + } + restore_flags(flags); + addr -= begin; + printk("Secondary cache sized at %dK linesize %d\n", + (int) (addr >> 10), sc_lsize); + scache_size = addr; + return 1; +} + +static void __init setup_noscache_funcs(void) +{ + unsigned int prid; + + switch(dc_lsize) { + case 16: + clear_page = r4k_clear_page_d16; + copy_page = r4k_copy_page_d16; + flush_cache_all = r4k_flush_cache_all_d16i16; + flush_cache_mm = r4k_flush_cache_mm_d16i16; + flush_cache_range = r4k_flush_cache_range_d16i16; + flush_cache_page = r4k_flush_cache_page_d16i16; + flush_page_to_ram = r4k_flush_page_to_ram_d16i16; + break; + case 32: + prid = read_32bit_cp0_register(CP0_PRID) & 0xfff0; + if (prid == 0x2010) { /* R4600 V1.7 */ + clear_page = r4k_clear_page_r4600_v1; + copy_page = r4k_copy_page_r4600_v1; + } else if (prid == 0x2020) { /* R4600 V2.0 */ + clear_page = r4k_clear_page_r4600_v2; + copy_page = r4k_copy_page_r4600_v2; + } else { + clear_page = r4k_clear_page_d32; + copy_page = r4k_copy_page_d32; + } + flush_cache_all = r4k_flush_cache_all_d32i32; + flush_cache_mm = r4k_flush_cache_mm_d32i32; + flush_cache_range = r4k_flush_cache_range_d32i32; + flush_cache_page = r4k_flush_cache_page_d32i32; + flush_page_to_ram = r4k_flush_page_to_ram_d32i32; + break; + } + dma_cache_wback_inv = r4k_dma_cache_wback_inv_pc; + dma_cache_wback = r4k_dma_cache_wback; + dma_cache_inv = r4k_dma_cache_inv_pc; +} + +static void __init setup_scache_funcs(void) +{ + switch(sc_lsize) { + case 16: + switch(dc_lsize) { + case 16: + flush_cache_all = r4k_flush_cache_all_s16d16i16; + flush_cache_mm = r4k_flush_cache_mm_s16d16i16; + flush_cache_range = r4k_flush_cache_range_s16d16i16; + flush_cache_page = r4k_flush_cache_page_s16d16i16; + flush_page_to_ram = r4k_flush_page_to_ram_s16d16i16; + break; + case 32: + panic("Invalid cache configuration detected"); + }; + clear_page = r4k_clear_page_s16; + copy_page = r4k_copy_page_s16; + break; + case 32: + switch(dc_lsize) { + case 16: + flush_cache_all = r4k_flush_cache_all_s32d16i16; + flush_cache_mm = r4k_flush_cache_mm_s32d16i16; + flush_cache_range = r4k_flush_cache_range_s32d16i16; + flush_cache_page = r4k_flush_cache_page_s32d16i16; + flush_page_to_ram = r4k_flush_page_to_ram_s32d16i16; + break; + case 32: + flush_cache_all = r4k_flush_cache_all_s32d32i32; + flush_cache_mm = r4k_flush_cache_mm_s32d32i32; + flush_cache_range = r4k_flush_cache_range_s32d32i32; + flush_cache_page = r4k_flush_cache_page_s32d32i32; + flush_page_to_ram = r4k_flush_page_to_ram_s32d32i32; + break; + }; + clear_page = r4k_clear_page_s32; + copy_page = r4k_copy_page_s32; + break; + case 64: + switch(dc_lsize) { + case 16: + flush_cache_all = r4k_flush_cache_all_s64d16i16; + flush_cache_mm = r4k_flush_cache_mm_s64d16i16; + flush_cache_range = r4k_flush_cache_range_s64d16i16; + flush_cache_page = r4k_flush_cache_page_s64d16i16; + flush_page_to_ram = r4k_flush_page_to_ram_s64d16i16; + break; + case 32: + flush_cache_all = r4k_flush_cache_all_s64d32i32; + flush_cache_mm = r4k_flush_cache_mm_s64d32i32; + flush_cache_range = r4k_flush_cache_range_s64d32i32; + flush_cache_page = r4k_flush_cache_page_s64d32i32; + flush_page_to_ram = r4k_flush_page_to_ram_s64d32i32; + break; + }; + clear_page = r4k_clear_page_s64; + copy_page = r4k_copy_page_s64; + break; + case 128: + switch(dc_lsize) { + case 16: + flush_cache_all = r4k_flush_cache_all_s128d16i16; + flush_cache_mm = r4k_flush_cache_mm_s128d16i16; + flush_cache_range = r4k_flush_cache_range_s128d16i16; + flush_cache_page = r4k_flush_cache_page_s128d16i16; + flush_page_to_ram = r4k_flush_page_to_ram_s128d16i16; + break; + case 32: + flush_cache_all = r4k_flush_cache_all_s128d32i32; + flush_cache_mm = r4k_flush_cache_mm_s128d32i32; + flush_cache_range = r4k_flush_cache_range_s128d32i32; + flush_cache_page = r4k_flush_cache_page_s128d32i32; + flush_page_to_ram = r4k_flush_page_to_ram_s128d32i32; + break; + }; + clear_page = r4k_clear_page_s128; + copy_page = r4k_copy_page_s128; + break; + } + dma_cache_wback_inv = r4k_dma_cache_wback_inv_sc; + dma_cache_wback = r4k_dma_cache_wback; + dma_cache_inv = r4k_dma_cache_inv_sc; +} + +typedef int (*probe_func_t)(unsigned long); + +static inline void __init setup_scache(unsigned int config) +{ + probe_func_t probe_scache_kseg1; + int sc_present = 0; + + /* Maybe the cpu knows about a l2 cache? */ + probe_scache_kseg1 = (probe_func_t) (KSEG1ADDR(&probe_scache)); + sc_present = probe_scache_kseg1(config); + + if (sc_present) { + setup_scache_funcs(); + return; + } + + setup_noscache_funcs(); +} + +static int r4k_user_mode(struct pt_regs *regs) +{ + return (regs->cp0_status & ST0_KSU) == KSU_USER; +} + +void __init ld_mmu_r4xx0(void) +{ + unsigned long config = read_32bit_cp0_register(CP0_CONFIG); + + printk("CPU revision is: %08x\n", read_32bit_cp0_register(CP0_PRID)); + + set_cp0_config(CONF_CM_CMASK, CONF_CM_CACHABLE_NONCOHERENT); + + probe_icache(config); + probe_dcache(config); + setup_scache(config); + + switch(mips_cputype) { + case CPU_R4600: /* QED style two way caches? */ + case CPU_R4700: + case CPU_R5000: + case CPU_NEVADA: + flush_cache_page = r4k_flush_cache_page_d32i32_r4600; + } + + flush_cache_sigtramp = r4k_flush_cache_sigtramp; + if ((read_32bit_cp0_register(CP0_PRID) & 0xfff0) == 0x2020) { + flush_cache_sigtramp = r4600v20k_flush_cache_sigtramp; + } + + flush_tlb_all = r4k_flush_tlb_all; + flush_tlb_mm = r4k_flush_tlb_mm; + flush_tlb_range = r4k_flush_tlb_range; + flush_tlb_page = r4k_flush_tlb_page; + r4xx0_asid_setup(); + + load_pgd = r4k_load_pgd; + pgd_init = r4k_pgd_init; + update_mmu_cache = r4k_update_mmu_cache; + + show_regs = r4k_show_regs; + + add_wired_entry = r4k_add_wired_entry; + + user_mode = r4k_user_mode; + + flush_cache_all(); + write_32bit_cp0_register(CP0_WIRED, 0); + + /* + * You should never change this register: + * - On R4600 1.7 the tlbp never hits for pages smaller than + * the value in the c0_pagemask register. + * - The entire mm handling assumes the c0_pagemask register to + * be set for 4kb pages. + */ + write_32bit_cp0_register(CP0_PAGEMASK, PM_4K); + flush_tlb_all(); +} diff --git a/arch/mips64/mm/tfp.c b/arch/mips64/mm/tfp.c new file mode 100644 index 000000000..30e87bde1 --- /dev/null +++ b/arch/mips64/mm/tfp.c @@ -0,0 +1,120 @@ +/* $Id$ + * + * tfp.c: MMU and cache routines specific to the r8000 (TFP). + * + * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) + * + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +extern unsigned long mips_tlb_entries; + +/* Cache operations. XXX Write these dave... */ +static inline void tfp_flush_cache_all(void) +{ + /* XXX */ +} + +static void tfp_flush_cache_mm(struct mm_struct *mm) +{ + /* XXX */ +} + +static void tfp_flush_cache_range(struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + /* XXX */ +} + +static void tfp_flush_cache_page(struct vm_area_struct *vma, + unsigned long page) +{ + /* XXX */ +} + +static void tfp_flush_page_to_ram(unsigned long page) +{ + /* XXX */ +} + +static void tfp_flush_cache_sigtramp(unsigned long page) +{ + /* XXX */ +} + +/* TLB operations. XXX Write these dave... */ +static inline void tfp_flush_tlb_all(void) +{ + /* XXX */ +} + +static void tfp_flush_tlb_mm(struct mm_struct *mm) +{ + /* XXX */ +} + +static void tfp_flush_tlb_range(struct mm_struct *mm, unsigned long start, + unsigned long end) +{ + /* XXX */ +} + +static void tfp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + /* XXX */ +} + +static void tfp_load_pgd(unsigned long pg_dir) +{ +} + +static void tfp_pgd_init(unsigned long page) +{ +} + +static void tfp_add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, + unsigned long entryhi, unsigned long pagemask) +{ + /* XXX */ +} + +static int tfp_user_mode(struct pt_regs *regs) +{ + return (regs->cp0_status & ST0_KSU) == KSU_USER; +} + +__initfunc(void ld_mmu_tfp(void)) +{ + flush_cache_all = tfp_flush_cache_all; + flush_cache_mm = tfp_flush_cache_mm; + flush_cache_range = tfp_flush_cache_range; + flush_cache_page = tfp_flush_cache_page; + flush_cache_sigtramp = tfp_flush_cache_sigtramp; + flush_page_to_ram = tfp_flush_page_to_ram; + + flush_tlb_all = tfp_flush_tlb_all; + flush_tlb_mm = tfp_flush_tlb_mm; + flush_tlb_range = tfp_flush_tlb_range; + flush_tlb_page = tfp_flush_tlb_page; + tfp_asid_setup(); + + add_wired_entry = tfp_add_wired_entry; + + user_mode = tfp_user_mode; + + load_pgd = tfp_load_pgd; + pgd_init = tfp_pgd_init; + + flush_cache_all(); + flush_tlb_all(); +} diff --git a/arch/mips64/mm/umap.c b/arch/mips64/mm/umap.c new file mode 100644 index 000000000..90d6f19fd --- /dev/null +++ b/arch/mips64/mm/umap.c @@ -0,0 +1,211 @@ +/* $Id$ + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1994 Linus Torvalds + * Copyright (C) 1997 Miguel de Icaza + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +static inline void +remove_mapping_pte_range (pmd_t *pmd, unsigned long address, unsigned long size) +{ + pte_t *pte; + unsigned long end; + + if (pmd_none (*pmd)) + return; + if (pmd_bad (*pmd)){ + printk ("remove_graphics_pte_range: bad pmd (%08lx)\n", pmd_val (*pmd)); + pmd_clear (pmd); + return; + } + pte = pte_offset (pmd, address); + address &= ~PMD_MASK; + end = address + size; + if (end > PMD_SIZE) + end = PMD_SIZE; + do { + pte_t entry = *pte; + if (pte_present (entry)) + set_pte (pte, pte_modify (entry, PAGE_NONE)); + address += PAGE_SIZE; + pte++; + } while (address < end); + +} + +static inline void +remove_mapping_pmd_range (pgd_t *pgd, unsigned long address, unsigned long size) +{ + pmd_t *pmd; + unsigned long end; + + if (pgd_none (*pgd)) + return; + + if (pgd_bad (*pgd)){ + printk ("remove_graphics_pmd_range: bad pgd (%08lx)\n", pgd_val (*pgd)); + pgd_clear (pgd); + return; + } + pmd = pmd_offset (pgd, address); + address &= ~PGDIR_MASK; + end = address + size; + if (end > PGDIR_SIZE) + end = PGDIR_SIZE; + do { + remove_mapping_pte_range (pmd, address, end - address); + address = (address + PMD_SIZE) & PMD_MASK; + pmd++; + } while (address < end); + +} + +/* + * This routine is called from the page fault handler to remove a + * range of active mappings at this point + */ +void +remove_mapping (struct task_struct *task, unsigned long start, unsigned long end) +{ + unsigned long beg = start; + pgd_t *dir; + + down (&task->mm->mmap_sem); + dir = pgd_offset (task->mm, start); + flush_cache_range (task->mm, beg, end); + while (start < end){ + remove_mapping_pmd_range (dir, start, end - start); + start = (start + PGDIR_SIZE) & PGDIR_MASK; + dir++; + } + flush_tlb_range (task->mm, beg, end); + up (&task->mm->mmap_sem); +} + +void *vmalloc_uncached (unsigned long size) +{ + return vmalloc_prot (size, PAGE_KERNEL_UNCACHED); +} + +static inline void free_pte(pte_t page) +{ + if (pte_present(page)) { + unsigned long addr = pte_page(page); + if (MAP_NR(addr) >= max_mapnr || PageReserved(mem_map+MAP_NR(addr))) + return; + free_page(addr); + if (current->mm->rss <= 0) + return; + current->mm->rss--; + return; + } + swap_free(pte_val(page)); +} + +static inline void forget_pte(pte_t page) +{ + if (!pte_none(page)) { + printk("forget_pte: old mapping existed!\n"); + free_pte(page); + } +} + +/* + * maps a range of vmalloc()ed memory into the requested pages. the old + * mappings are removed. + */ +static inline void +vmap_pte_range (pte_t *pte, unsigned long address, unsigned long size, unsigned long vaddr) +{ + unsigned long end; + pgd_t *vdir; + pmd_t *vpmd; + pte_t *vpte; + + address &= ~PMD_MASK; + end = address + size; + if (end > PMD_SIZE) + end = PMD_SIZE; + do { + pte_t oldpage = *pte; + unsigned long page; + pte_clear(pte); + + vdir = pgd_offset_k (vaddr); + vpmd = pmd_offset (vdir, vaddr); + vpte = pte_offset (vpmd, vaddr); + page = pte_page (*vpte); + + set_pte(pte, mk_pte_phys(page, PAGE_USERIO)); + forget_pte(oldpage); + address += PAGE_SIZE; + vaddr += PAGE_SIZE; + pte++; + } while (address < end); +} + +static inline int +vmap_pmd_range (pmd_t *pmd, unsigned long address, unsigned long size, unsigned long vaddr) +{ + unsigned long end; + + address &= ~PGDIR_MASK; + end = address + size; + if (end > PGDIR_SIZE) + end = PGDIR_SIZE; + vaddr -= address; + do { + pte_t * pte = pte_alloc(pmd, address); + if (!pte) + return -ENOMEM; + vmap_pte_range(pte, address, end - address, address + vaddr); + address = (address + PMD_SIZE) & PMD_MASK; + pmd++; + } while (address < end); + return 0; +} + +int +vmap_page_range (unsigned long from, unsigned long size, unsigned long vaddr) +{ + int error = 0; + pgd_t * dir; + unsigned long beg = from; + unsigned long end = from + size; + + vaddr -= from; + dir = pgd_offset(current->mm, from); + flush_cache_range(current->mm, beg, end); + while (from < end) { + pmd_t *pmd = pmd_alloc(dir, from); + error = -ENOMEM; + if (!pmd) + break; + error = vmap_pmd_range(pmd, from, end - from, vaddr + from); + if (error) + break; + from = (from + PGDIR_SIZE) & PGDIR_MASK; + dir++; + } + flush_tlb_range(current->mm, beg, end); + return error; +} -- cgit v1.2.3