1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
/* $Id: blockops.S,v 1.19 1999/11/19 05:52:45 davem Exp $
* blockops.S: UltraSparc block zero optimized routines.
*
* Copyright (C) 1996,1998 David S. Miller (davem@caip.rutgers.edu)
* Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
*/
#include "VIS.h"
#include <asm/visasm.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/asm_offsets.h>
#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \
fmovd %reg0, %f48; fmovd %reg1, %f50; \
fmovd %reg2, %f52; fmovd %reg3, %f54; \
fmovd %reg4, %f56; fmovd %reg5, %f58; \
fmovd %reg6, %f60; fmovd %reg7, %f62;
#define TLBTEMP_BASE (8 * 1024 * 1024)
#define DCACHE_SIZE (PAGE_SIZE * 2)
#define TLBTEMP_ENT1 (61 << 3)
#define TLBTEMP_ENT2 (62 << 3)
#define TLBTEMP_ENTSZ (1 << 3)
.text
.align 32
.globl copy_page
.type copy_page,@function
copy_page: /* %o0=dest, %o1=src */
VISEntry
ldx [%g6 + AOFF_task_active_mm], %o2
sub %o0, %g4, %g1
sethi %uhi(_PAGE_VALID), %g3
sub %o1, %g4, %g2
sllx %g3, 32, %g3
ldx [%o2 + AOFF_mm_segments], %o0
or %g3, (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), %g3
or %g1, %g3, %g1
or %g2, %g3, %g2
mov TLB_TAG_ACCESS, %o2
sethi %hi(TLBTEMP_BASE), %o3
sethi %hi(DCACHE_SIZE), %o1
add %o0, %o3, %o0
add %o0, %o1, %o1
sethi %hi(TLBTEMP_ENT1), %o3
rdpr %pstate, %g3
wrpr %g3, PSTATE_IE, %pstate
ldxa [%o3] ASI_DTLB_TAG_READ, %o4
ldxa [%o3] ASI_DTLB_DATA_ACCESS, %o5
stxa %o0, [%o2] ASI_DMMU
stxa %g1, [%o3] ASI_DTLB_DATA_ACCESS
membar #Sync
add %o3, (TLBTEMP_ENTSZ), %o3
ldxa [%o3] ASI_DTLB_TAG_READ, %g5
ldxa [%o3] ASI_DTLB_DATA_ACCESS, %g7
stxa %o1, [%o2] ASI_DMMU
stxa %g2, [%o3] ASI_DTLB_DATA_ACCESS
membar #Sync
membar #LoadStore | #StoreStore | #StoreLoad
ldda [%o1] ASI_BLK_P, %f0
add %o1, 0x40, %o1
ldda [%o1] ASI_BLK_P, %f16
add %o1, 0x40, %o1
sethi %hi(8192), %o2
1: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
ldda [%o1] ASI_BLK_P, %f32
add %o1, 0x40, %o1
sub %o2, 0x40, %o2
stda %f48, [%o0] ASI_BLK_P
add %o0, 0x40, %o0
TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
ldda [%o1] ASI_BLK_P, %f0
add %o1, 0x40, %o1
sub %o2, 0x40, %o2
stda %f48, [%o0] ASI_BLK_P
add %o0, 0x40, %o0
TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
ldda [%o1] ASI_BLK_P, %f16
add %o1, 0x40, %o1
sub %o2, 0x40, %o2
stda %f48, [%o0] ASI_BLK_P
cmp %o2, 0x80
bne,pt %xcc, 1b
add %o0, 0x40, %o0
membar #Sync
stda %f0, [%o0] ASI_BLK_P
add %o0, 0x40, %o0
stda %f16, [%o0] ASI_BLK_P
membar #Sync
VISExit
mov TLB_TAG_ACCESS, %o2
stxa %g5, [%o2] ASI_DMMU
stxa %g7, [%o3] ASI_DTLB_DATA_ACCESS
membar #Sync
sub %o3, (TLBTEMP_ENTSZ), %o3
stxa %o4, [%o2] ASI_DMMU
stxa %o5, [%o3] ASI_DTLB_DATA_ACCESS
membar #Sync
jmpl %o7 + 0x8, %g0
wrpr %g3, 0x0, %pstate
.align 32
.globl clear_page
.type clear_page,@function
clear_page: /* %o0=dest */
VISEntryHalf
ldx [%g6 + AOFF_task_active_mm], %o2
sub %o0, %g4, %g1
sethi %uhi(_PAGE_VALID), %g3
sllx %g3, 32, %g3
or %g3, (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), %g3
ldx [%o2 + AOFF_mm_segments], %o0
or %g1, %g3, %g1
mov TLB_TAG_ACCESS, %o2
sethi %hi(TLBTEMP_BASE), %o3
add %o0, %o3, %o0
sethi %hi(TLBTEMP_ENT2), %o3
rdpr %pstate, %g3
wrpr %g3, PSTATE_IE, %pstate
ldxa [%o3] ASI_DTLB_TAG_READ, %g5
ldxa [%o3] ASI_DTLB_DATA_ACCESS, %g7
stxa %o0, [%o2] ASI_DMMU
stxa %g1, [%o3] ASI_DTLB_DATA_ACCESS
membar #Sync
fzero %f0 ! FPA Group
mov 32, %o1 ! IEU0
fzero %f2 ! FPA Group
faddd %f0, %f2, %f4 ! FPA Group
fmuld %f0, %f2, %f6 ! FPM
faddd %f0, %f2, %f8 ! FPA Group
fmuld %f0, %f2, %f10 ! FPM
faddd %f0, %f2, %f12 ! FPA Group
fmuld %f0, %f2, %f14 ! FPM
membar #StoreLoad | #StoreStore | #LoadStore ! LSU Group
1: stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
add %o0, 0x40, %o0 ! IEU0
stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
add %o0, 0x40, %o0 ! IEU0
stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
add %o0, 0x40, %o0 ! IEU0 Group
stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
subcc %o1, 1, %o1 ! IEU1
bne,pt %icc, 1b ! CTI
add %o0, 0x40, %o0 ! IEU0 Group
membar #Sync ! LSU Group
VISExitHalf
stxa %g5, [%o2] ASI_DMMU
stxa %g7, [%o3] ASI_DTLB_DATA_ACCESS
membar #Sync
jmpl %o7 + 0x8, %g0
wrpr %g3, 0x0, %pstate
|
