Gitiles
Code Review Sign In
review.evervolv.com / android_kernel_oneplus_msm8996 / df7d6aec96ab98cb182dd5138a85bdc363a9bf0d / . / arch / sparc64 / kernel / ktlb.S
blob: f6bb2e08964a2f8e1a9b22050ec4e370707281ed [file] [log] [blame]
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]1/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
2 *
3 * Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net>
4 * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
5 * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
6 * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]7 */
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]8
9#include <linux/config.h>
10#include <asm/head.h>
11#include <asm/asi.h>
12#include <asm/page.h>
13#include <asm/pgtable.h>
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]14#include <asm/tsb.h>
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]15
16 .text
17 .align 32
18
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]19kvmap_itlb:
20 /* g6: TAG TARGET */
21 mov TLB_TAG_ACCESS, %g4
22 ldxa [%g4] ASI_IMMU, %g4
23
David S. Millerd257d5d2006-02-06 23:44:37 -0800[diff] [blame]24 /* sun4v_itlb_miss branches here with the missing virtual
25 * address already loaded into %g4
26 */
27kvmap_itlb_4v:
28
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]29kvmap_itlb_nonlinear:
30 /* Catch kernel NULL pointer calls. */
31 sethi %hi(PAGE_SIZE), %g5
32 cmp %g4, %g5
33 bleu,pn %xcc, kvmap_dtlb_longpath
34 nop
35
36 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
37
38kvmap_itlb_tsb_miss:
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]39 sethi %hi(LOW_OBP_ADDRESS), %g5
40 cmp %g4, %g5
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]41 blu,pn %xcc, kvmap_itlb_vmalloc_addr
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]42 mov 0x1, %g5
43 sllx %g5, 32, %g5
44 cmp %g4, %g5
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]45 blu,pn %xcc, kvmap_itlb_obp
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]46 nop
47
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]48kvmap_itlb_vmalloc_addr:
49 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]50
David S. Miller517af332006-02-01 15:55:21 -0800[diff] [blame]51 KTSB_LOCK_TAG(%g1, %g2, %g4)
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]52
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]53 /* Load and check PTE. */
54 ldxa [%g5] ASI_PHYS_USE_EC, %g5
55 brgez,a,pn %g5, kvmap_itlb_longpath
David S. Miller517af332006-02-01 15:55:21 -0800[diff] [blame]56 KTSB_STORE(%g1, %g0)
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]57
David S. Miller517af332006-02-01 15:55:21 -0800[diff] [blame]58 KTSB_WRITE(%g1, %g5, %g6)
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]59
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]60 /* fallthrough to TLB load */
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]61
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]62kvmap_itlb_load:
63 stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Reload TLB
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]64 retry
65
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]66kvmap_itlb_longpath:
David S. Miller45fec052006-02-05 22:27:28 -0800[diff] [blame]67
68661: rdpr %pstate, %g5
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]69 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
David S. Millerdf7d6ae2006-02-07 00:00:16 -0800[diff] [blame^]70 .section .sun4v_2insn_patch, "ax"
David S. Miller45fec052006-02-05 22:27:28 -0800[diff] [blame]71 .word 661b
72 nop
73 nop
74 .previous
75
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]76 rdpr %tpc, %g5
77 ba,pt %xcc, sparc64_realfault_common
78 mov FAULT_CODE_ITLB, %g4
David S. Millerc9c10832005-10-12 12:22:46 -0700[diff] [blame]79
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]80kvmap_itlb_obp:
81 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]82
David S. Miller517af332006-02-01 15:55:21 -0800[diff] [blame]83 KTSB_LOCK_TAG(%g1, %g2, %g4)
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]84
David S. Miller517af332006-02-01 15:55:21 -0800[diff] [blame]85 KTSB_WRITE(%g1, %g5, %g6)
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]86
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]87 ba,pt %xcc, kvmap_itlb_load
88 nop
David S. Millerc9c10832005-10-12 12:22:46 -0700[diff] [blame]89
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]90kvmap_dtlb_obp:
91 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
92
David S. Miller517af332006-02-01 15:55:21 -0800[diff] [blame]93 KTSB_LOCK_TAG(%g1, %g2, %g4)
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]94
David S. Miller517af332006-02-01 15:55:21 -0800[diff] [blame]95 KTSB_WRITE(%g1, %g5, %g6)
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]96
97 ba,pt %xcc, kvmap_dtlb_load
98 nop
99
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]100 .align 32
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]101kvmap_dtlb:
102 /* %g6: TAG TARGET */
103 mov TLB_TAG_ACCESS, %g4
104 ldxa [%g4] ASI_DMMU, %g4
David S. Millerd257d5d2006-02-06 23:44:37 -0800[diff] [blame]105
106 /* sun4v_dtlb_miss branches here with the missing virtual
107 * address already loaded into %g4
108 */
109kvmap_dtlb_4v:
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]110 brgez,pn %g4, kvmap_dtlb_nonlinear
David S. Miller56425302005-09-25 16:46:57 -0700[diff] [blame]111 nop
112
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]113#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
114#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
115
116 sethi %uhi(KERN_HIGHBITS), %g2
117 or %g2, %ulo(KERN_HIGHBITS), %g2
118 sllx %g2, 32, %g2
119 or %g2, KERN_LOWBITS, %g2
120
121#undef KERN_HIGHBITS
122#undef KERN_LOWBITS
123
David S. Miller56425302005-09-25 16:46:57 -0700[diff] [blame]124 .globl kvmap_linear_patch
125kvmap_linear_patch:
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]126 ba,pt %xcc, kvmap_dtlb_load
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]127 xor %g2, %g4, %g5
128
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]129kvmap_dtlb_vmalloc_addr:
130 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
David S. Miller56425302005-09-25 16:46:57 -0700[diff] [blame]131
David S. Miller517af332006-02-01 15:55:21 -0800[diff] [blame]132 KTSB_LOCK_TAG(%g1, %g2, %g4)
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]133
134 /* Load and check PTE. */
135 ldxa [%g5] ASI_PHYS_USE_EC, %g5
136 brgez,a,pn %g5, kvmap_dtlb_longpath
David S. Miller517af332006-02-01 15:55:21 -0800[diff] [blame]137 KTSB_STORE(%g1, %g0)
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]138
David S. Miller517af332006-02-01 15:55:21 -0800[diff] [blame]139 KTSB_WRITE(%g1, %g5, %g6)
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]140
141 /* fallthrough to TLB load */
142
143kvmap_dtlb_load:
144 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
145 retry
146
147kvmap_dtlb_nonlinear:
148 /* Catch kernel NULL pointer derefs. */
149 sethi %hi(PAGE_SIZE), %g5
150 cmp %g4, %g5
151 bleu,pn %xcc, kvmap_dtlb_longpath
152 nop
153
154 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
155
156kvmap_dtlb_tsbmiss:
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]157 sethi %hi(MODULES_VADDR), %g5
158 cmp %g4, %g5
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]159 blu,pn %xcc, kvmap_dtlb_longpath
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]160 mov (VMALLOC_END >> 24), %g5
161 sllx %g5, 24, %g5
162 cmp %g4, %g5
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]163 bgeu,pn %xcc, kvmap_dtlb_longpath
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]164 nop
165
166kvmap_check_obp:
167 sethi %hi(LOW_OBP_ADDRESS), %g5
168 cmp %g4, %g5
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]169 blu,pn %xcc, kvmap_dtlb_vmalloc_addr
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]170 mov 0x1, %g5
171 sllx %g5, 32, %g5
172 cmp %g4, %g5
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]173 blu,pn %xcc, kvmap_dtlb_obp
174 nop
175 ba,pt %xcc, kvmap_dtlb_vmalloc_addr
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]176 nop
177
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]178kvmap_dtlb_longpath:
David S. Miller45fec052006-02-05 22:27:28 -0800[diff] [blame]179
180661: rdpr %pstate, %g5
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]181 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
David S. Millerdf7d6ae2006-02-07 00:00:16 -0800[diff] [blame^]182 .section .sun4v_2insn_patch, "ax"
David S. Miller45fec052006-02-05 22:27:28 -0800[diff] [blame]183 .word 661b
184 nop
185 nop
186 .previous
187
David S. Miller74bf4312006-01-31 18:29:18 -0800[diff] [blame]188 rdpr %tl, %g4
189 cmp %g4, 1
190 mov TLB_TAG_ACCESS, %g4
191 ldxa [%g4] ASI_DMMU, %g5
192 be,pt %xcc, sparc64_realfault_common
193 mov FAULT_CODE_DTLB, %g4
194 ba,pt %xcc, winfix_trampoline
David S. Miller2a7e2992005-09-21 18:50:51 -0700[diff] [blame]195 nop