summaryrefslogtreecommitdiffstats
path: root/include/asm-s390/pgalloc.h
blob: 72c67617ebc8728d88d9b620795aa2f1aa576745 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
/*
 *  include/asm-s390/bugs.h
 *
 *  S390 version
 *    Copyright (C) 1999,2000 IBM Deutschland Entwicklung GmbH, IBM Corporation
 *    Author(s): Hartmut Penner (hp@de.ibm.com)
 *               Martin Schwidefsky (schwidefsky@de.ibm.com)
 *
 *  Derived from "include/asm-i386/pgalloc.h"
 *    Copyright (C) 1994  Linus Torvalds
 */

#ifndef _S390_PGALLOC_H
#define _S390_PGALLOC_H

#include <linux/config.h>
#include <asm/processor.h>
#include <linux/threads.h>

#define pgd_quicklist (S390_lowcore.cpu_data.pgd_quick)
#define pmd_quicklist ((unsigned long *)0)
#define pte_quicklist (S390_lowcore.cpu_data.pte_quick)
#define pgtable_cache_size (S390_lowcore.cpu_data.pgtable_cache_sz)

/*
 * Allocate and free page tables. The xxx_kernel() versions are
 * used to allocate a kernel page table - this turns on ASN bits
 * if any.
 */

extern __inline__ pgd_t* get_pgd_slow(void)
{
        int i;
        pgd_t *pgd,*ret = (pgd_t *)__get_free_pages(GFP_KERNEL,1);
	if (ret)
		for (i=0,pgd=ret;i<USER_PTRS_PER_PGD;i++,pgd++)
			pmd_clear(pmd_offset(pgd,i*PGDIR_SIZE));
        return ret;
}

extern __inline__ pgd_t* get_pgd_fast(void)
{
        unsigned long *ret = pgd_quicklist;
	
        if (ret != NULL) {
                pgd_quicklist = (unsigned long *)(*ret);
                ret[0] = ret[1];
                pgtable_cache_size -= 2;
        }
        return (pgd_t *)ret;
}

extern __inline__ pgd_t *pgd_alloc(void)
{
	pgd_t *pgd;

	pgd = get_pgd_fast();
	if (!pgd)
		pgd = get_pgd_slow();
	return pgd;
}

extern __inline__ void free_pgd_fast(pgd_t *pgd)
{
        *(unsigned long *)pgd = (unsigned long) pgd_quicklist;
        pgd_quicklist = (unsigned long *) pgd;
        pgtable_cache_size += 2;
}

extern __inline__ void free_pgd_slow(pgd_t *pgd)
{
        free_pages((unsigned long) pgd, 1);
}

#define pgd_free(pgd)           free_pgd_fast(pgd)

/*
 * page middle directory allocation/free routines.
 * We don't use pmd cache, so these are dummy routines.
 */
extern __inline__ pmd_t *get_pmd_fast(void)
{
        return (pmd_t *)0;
}

extern __inline__ void free_pmd_fast(pmd_t *pmd)
{
}

extern inline pmd_t * pmd_alloc(pgd_t * pgd, unsigned long address)
{
        return (pmd_t *) pgd;
}

extern __inline__ void free_pmd_slow(pmd_t *pmd)
{
}

extern inline void pmd_free(pmd_t * pmd)
{
}

#define pmd_free_kernel         pmd_free
#define pmd_alloc_kernel        pmd_alloc

/*
 * page table entry allocation/free routines.
 */
extern pte_t empty_bad_pte_table[];
extern pte_t *get_pte_slow(pmd_t *pmd, unsigned long address_preadjusted);

extern __inline__ pte_t* get_pte_fast(void)
{
        unsigned long *ret = (unsigned long *) pte_quicklist;

        if (ret != NULL) {
                pte_quicklist = (unsigned long *)(*ret);
                ret[0] = ret[1];
                pgtable_cache_size--;
        }
        return (pte_t *)ret;
}

extern __inline__ void free_pte_fast(pte_t *pte)
{
	if (pte == empty_bad_pte_table)
		return;
        *(unsigned long *)pte = (unsigned long) pte_quicklist;
        pte_quicklist = (unsigned long *) pte;
        pgtable_cache_size++;
}

extern __inline__ void free_pte_slow(pte_t *pte)
{
        free_page((unsigned long) pte);
}

extern inline pte_t * pte_alloc(pmd_t * pmd, unsigned long vmaddr)
{
	unsigned long offset;

	offset = (vmaddr >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
        if (pmd_none(*pmd)) {
		unsigned long page = (unsigned long) get_pte_fast();
		
		if (!page)
			return get_pte_slow(pmd, offset);
		pmd_val(pmd[0]) = _PAGE_TABLE + __pa(page);
		pmd_val(pmd[1]) = _PAGE_TABLE + __pa(page+1024);
		pmd_val(pmd[2]) = _PAGE_TABLE + __pa(page+2048);
		pmd_val(pmd[3]) = _PAGE_TABLE + __pa(page+3072);
		return (pte_t *) page + offset;
	}
        if (pmd_bad(*pmd))
		BUG();
        return (pte_t *) pmd_page(*pmd) + offset;
}

#define pte_alloc_kernel(pmd, addr) pte_alloc(pmd, addr)
#define pte_free_kernel(pte)    free_pte_fast(pte)
#define pte_free(pte)           free_pte_fast(pte)

extern int do_check_pgt_cache(int, int);

/*
 * This establishes kernel virtual mappings (e.g., as a result of a
 * vmalloc call).  Since s390-esame uses a separate kernel page table,
 * there is nothing to do here... :)
 */
#define set_pgdir(addr,entry) do { } while(0)

/*
 * TLB flushing:
 *
 *  - flush_tlb() flushes the current mm struct TLBs
 *  - flush_tlb_all() flushes all processes TLBs 
 *    called only from vmalloc/vfree
 *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
 *  - flush_tlb_page(vma, vmaddr) flushes one page
 *  - flush_tlb_range(mm, start, end) flushes a range of pages
 *  - flush_tlb_pgtables(mm, start, end) flushes a range of page tables
 */

/*
 * S/390 has three ways of flushing TLBs
 * 'ptlb' does a flush of the local processor
 * 'csp' flushes the TLBs on all PUs of a SMP
 * 'ipte' invalidates a pte in a page table and flushes that out of
 * the TLBs of all PUs of a SMP
 */

#define local_flush_tlb() \
do {  __asm__ __volatile__("ptlb": : :"memory"); } while (0)


#ifndef CONFIG_SMP

/*
 * We always need to flush, since s390 does not flush tlb
 * on each context switch
 */

#define flush_tlb()			local_flush_tlb()
#define flush_tlb_all()			local_flush_tlb()
#define flush_tlb_mm(mm)		local_flush_tlb()
#define flush_tlb_page(vma, va)		local_flush_tlb()
#define flush_tlb_range(mm, start, end)	local_flush_tlb()

#else

#include <asm/smp.h>

extern void smp_ptlb_all(void);
static inline void global_flush_tlb_csp(void)
{
	int cs1=0,dum=0;
	int *adr;
	long long dummy=0;
	adr = (int*) (((int)(((int*) &dummy)+1) & 0xfffffffc)|1);
	__asm__ __volatile__("lr    2,%0\n\t"
                             "lr    3,%1\n\t"
                             "lr    4,%2\n\t"
                             "csp   2,4" :
                             : "d" (cs1), "d" (dum), "d" (adr)
                             : "2", "3", "4");
}
static inline void global_flush_tlb(void)
{
	if (MACHINE_HAS_CSP)
		global_flush_tlb_csp();
	else
		smp_ptlb_all();
}

/*
 * We only have to do global flush of tlb if process run since last
 * flush on any other pu than current. 
 * If we have threads (mm->count > 1) we always do a global flush, 
 * since the process runs on more than one processor at the same time.
 */

static inline void __flush_tlb_mm(struct mm_struct * mm)
{
	if ((smp_num_cpus > 1) &&
	    ((atomic_read(&mm->mm_count) != 1) ||
	     (mm->cpu_vm_mask != (1UL << smp_processor_id())))) {
		mm->cpu_vm_mask = (1UL << smp_processor_id());
		global_flush_tlb();
	} else {                 
		local_flush_tlb();
	}
}

#define flush_tlb()			__flush_tlb_mm(current->mm)
#define flush_tlb_all()			global_flush_tlb()
#define flush_tlb_mm(mm)		__flush_tlb_mm(mm)
#define flush_tlb_page(vma, va)		__flush_tlb_mm((vma)->vm_mm)
#define flush_tlb_range(mm, start, end)	__flush_tlb_mm(mm)

#endif

extern inline void flush_tlb_pgtables(struct mm_struct *mm,
                                      unsigned long start, unsigned long end)
{
        /* S/390 does not keep any page table caches in TLB */
}


static inline int ptep_test_and_clear_and_flush_young(struct vm_area_struct *vma, 
                                                      unsigned long address, pte_t *ptep)
{
	/* No need to flush TLB; bits are in storage key */
	return ptep_test_and_clear_young(ptep);
}

static inline int ptep_test_and_clear_and_flush_dirty(struct vm_area_struct *vma, 
                                                      unsigned long address, pte_t *ptep)
{
	/* No need to flush TLB; bits are in storage key */
	return ptep_test_and_clear_dirty(ptep);
}

static inline pte_t ptep_invalidate(struct vm_area_struct *vma, 
                                    unsigned long address, pte_t *ptep)
{
	pte_t pte = *ptep;
	if (!(pte_val(pte) & _PAGE_INVALID)) {
		/* S390 has 1mb segments, we are emulating 4MB segments */
		pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
		__asm__ __volatile__ ("ipte %0,%1" : : "a" (pto), "a" (address));
	}
	pte_clear(ptep);
	return pte;
}

static inline void ptep_establish(struct vm_area_struct *vma, 
                                  unsigned long address, pte_t *ptep, pte_t entry)
{
	ptep_invalidate(vma, address, ptep);
	set_pte(ptep, entry);
}

#endif /* _S390_PGALLOC_H */