David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 1 | /* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling. |
| 2 | * |
| 3 | * Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net> |
| 4 | * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) |
| 5 | * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) |
| 6 | * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 7 | */ |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 8 | |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 9 | #include <asm/head.h> |
| 10 | #include <asm/asi.h> |
| 11 | #include <asm/page.h> |
| 12 | #include <asm/pgtable.h> |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 13 | #include <asm/tsb.h> |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 14 | |
| 15 | .text |
| 16 | .align 32 |
| 17 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 18 | kvmap_itlb: |
| 19 | /* g6: TAG TARGET */ |
| 20 | mov TLB_TAG_ACCESS, %g4 |
| 21 | ldxa [%g4] ASI_IMMU, %g4 |
| 22 | |
David S. Miller | d257d5d | 2006-02-06 23:44:37 -0800 | [diff] [blame] | 23 | /* sun4v_itlb_miss branches here with the missing virtual |
| 24 | * address already loaded into %g4 |
| 25 | */ |
| 26 | kvmap_itlb_4v: |
| 27 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 28 | kvmap_itlb_nonlinear: |
| 29 | /* Catch kernel NULL pointer calls. */ |
| 30 | sethi %hi(PAGE_SIZE), %g5 |
| 31 | cmp %g4, %g5 |
| 32 | bleu,pn %xcc, kvmap_dtlb_longpath |
| 33 | nop |
| 34 | |
| 35 | KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load) |
| 36 | |
| 37 | kvmap_itlb_tsb_miss: |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 38 | sethi %hi(LOW_OBP_ADDRESS), %g5 |
| 39 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 40 | blu,pn %xcc, kvmap_itlb_vmalloc_addr |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 41 | mov 0x1, %g5 |
| 42 | sllx %g5, 32, %g5 |
| 43 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 44 | blu,pn %xcc, kvmap_itlb_obp |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 45 | nop |
| 46 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 47 | kvmap_itlb_vmalloc_addr: |
| 48 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 49 | |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 50 | KTSB_LOCK_TAG(%g1, %g2, %g7) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 51 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 52 | /* Load and check PTE. */ |
| 53 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 54 | mov 1, %g7 |
| 55 | sllx %g7, TSB_TAG_INVALID_BIT, %g7 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 56 | brgez,a,pn %g5, kvmap_itlb_longpath |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 57 | KTSB_STORE(%g1, %g7) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 58 | |
David S. Miller | 517af33 | 2006-02-01 15:55:21 -0800 | [diff] [blame] | 59 | KTSB_WRITE(%g1, %g5, %g6) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 60 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 61 | /* fallthrough to TLB load */ |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 62 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 63 | kvmap_itlb_load: |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 64 | |
| 65 | 661: stxa %g5, [%g0] ASI_ITLB_DATA_IN |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 66 | retry |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 67 | .section .sun4v_2insn_patch, "ax" |
| 68 | .word 661b |
| 69 | nop |
| 70 | nop |
| 71 | .previous |
| 72 | |
| 73 | /* For sun4v the ASI_ITLB_DATA_IN store and the retry |
| 74 | * instruction get nop'd out and we get here to branch |
| 75 | * to the sun4v tlb load code. The registers are setup |
| 76 | * as follows: |
| 77 | * |
| 78 | * %g4: vaddr |
| 79 | * %g5: PTE |
| 80 | * %g6: TAG |
| 81 | * |
| 82 | * The sun4v TLB load wants the PTE in %g3 so we fix that |
| 83 | * up here. |
| 84 | */ |
| 85 | ba,pt %xcc, sun4v_itlb_load |
| 86 | mov %g5, %g3 |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 87 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 88 | kvmap_itlb_longpath: |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 89 | |
| 90 | 661: rdpr %pstate, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 91 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate |
David S. Miller | df7d6ae | 2006-02-07 00:00:16 -0800 | [diff] [blame] | 92 | .section .sun4v_2insn_patch, "ax" |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 93 | .word 661b |
David S. Miller | 6cc200d | 2006-02-18 16:36:39 -0800 | [diff] [blame] | 94 | SET_GL(1) |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 95 | nop |
| 96 | .previous |
| 97 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 98 | rdpr %tpc, %g5 |
| 99 | ba,pt %xcc, sparc64_realfault_common |
| 100 | mov FAULT_CODE_ITLB, %g4 |
David S. Miller | c9c1083 | 2005-10-12 12:22:46 -0700 | [diff] [blame] | 101 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 102 | kvmap_itlb_obp: |
| 103 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 104 | |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 105 | KTSB_LOCK_TAG(%g1, %g2, %g7) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 106 | |
David S. Miller | 517af33 | 2006-02-01 15:55:21 -0800 | [diff] [blame] | 107 | KTSB_WRITE(%g1, %g5, %g6) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 108 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 109 | ba,pt %xcc, kvmap_itlb_load |
| 110 | nop |
David S. Miller | c9c1083 | 2005-10-12 12:22:46 -0700 | [diff] [blame] | 111 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 112 | kvmap_dtlb_obp: |
| 113 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath) |
| 114 | |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 115 | KTSB_LOCK_TAG(%g1, %g2, %g7) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 116 | |
David S. Miller | 517af33 | 2006-02-01 15:55:21 -0800 | [diff] [blame] | 117 | KTSB_WRITE(%g1, %g5, %g6) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 118 | |
| 119 | ba,pt %xcc, kvmap_dtlb_load |
| 120 | nop |
| 121 | |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 122 | .align 32 |
David S. Miller | d7744a0 | 2006-02-21 22:31:11 -0800 | [diff] [blame] | 123 | kvmap_dtlb_tsb4m_load: |
| 124 | KTSB_LOCK_TAG(%g1, %g2, %g7) |
| 125 | KTSB_WRITE(%g1, %g5, %g6) |
| 126 | ba,pt %xcc, kvmap_dtlb_load |
| 127 | nop |
| 128 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 129 | kvmap_dtlb: |
| 130 | /* %g6: TAG TARGET */ |
| 131 | mov TLB_TAG_ACCESS, %g4 |
| 132 | ldxa [%g4] ASI_DMMU, %g4 |
David S. Miller | d257d5d | 2006-02-06 23:44:37 -0800 | [diff] [blame] | 133 | |
| 134 | /* sun4v_dtlb_miss branches here with the missing virtual |
| 135 | * address already loaded into %g4 |
| 136 | */ |
| 137 | kvmap_dtlb_4v: |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 138 | brgez,pn %g4, kvmap_dtlb_nonlinear |
David S. Miller | 5642530 | 2005-09-25 16:46:57 -0700 | [diff] [blame] | 139 | nop |
| 140 | |
David S. Miller | d7744a0 | 2006-02-21 22:31:11 -0800 | [diff] [blame] | 141 | /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */ |
| 142 | KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) |
| 143 | |
| 144 | /* TSB entry address left in %g1, lookup linear PTE. |
| 145 | * Must preserve %g1 and %g6 (TAG). |
| 146 | */ |
| 147 | kvmap_dtlb_tsb4m_miss: |
David S. Miller | 9cc3a1a | 2006-02-21 20:51:13 -0800 | [diff] [blame] | 148 | sethi %hi(kpte_linear_bitmap), %g2 |
| 149 | or %g2, %lo(kpte_linear_bitmap), %g2 |
| 150 | |
| 151 | /* Clear the PAGE_OFFSET top virtual bits, then shift |
| 152 | * down to get a 256MB physical address index. |
| 153 | */ |
| 154 | sllx %g4, 21, %g5 |
| 155 | mov 1, %g7 |
| 156 | srlx %g5, 21 + 28, %g5 |
| 157 | |
| 158 | /* Don't try this at home kids... this depends upon srlx |
| 159 | * only taking the low 6 bits of the shift count in %g5. |
| 160 | */ |
| 161 | sllx %g7, %g5, %g7 |
| 162 | |
| 163 | /* Divide by 64 to get the offset into the bitmask. */ |
| 164 | srlx %g5, 6, %g5 |
David S. Miller | 6889331 | 2006-02-26 23:09:37 -0800 | [diff] [blame] | 165 | sllx %g5, 3, %g5 |
David S. Miller | 9cc3a1a | 2006-02-21 20:51:13 -0800 | [diff] [blame] | 166 | |
| 167 | /* kern_linear_pte_xor[((mask & bit) ? 1 : 0)] */ |
| 168 | ldx [%g2 + %g5], %g2 |
| 169 | andcc %g2, %g7, %g0 |
| 170 | sethi %hi(kern_linear_pte_xor), %g5 |
| 171 | or %g5, %lo(kern_linear_pte_xor), %g5 |
| 172 | bne,a,pt %xcc, 1f |
| 173 | add %g5, 8, %g5 |
| 174 | |
| 175 | 1: ldx [%g5], %g2 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 176 | |
David S. Miller | 5642530 | 2005-09-25 16:46:57 -0700 | [diff] [blame] | 177 | .globl kvmap_linear_patch |
| 178 | kvmap_linear_patch: |
David S. Miller | d7744a0 | 2006-02-21 22:31:11 -0800 | [diff] [blame] | 179 | ba,pt %xcc, kvmap_dtlb_tsb4m_load |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 180 | xor %g2, %g4, %g5 |
| 181 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 182 | kvmap_dtlb_vmalloc_addr: |
| 183 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) |
David S. Miller | 5642530 | 2005-09-25 16:46:57 -0700 | [diff] [blame] | 184 | |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 185 | KTSB_LOCK_TAG(%g1, %g2, %g7) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 186 | |
| 187 | /* Load and check PTE. */ |
| 188 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 189 | mov 1, %g7 |
| 190 | sllx %g7, TSB_TAG_INVALID_BIT, %g7 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 191 | brgez,a,pn %g5, kvmap_dtlb_longpath |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 192 | KTSB_STORE(%g1, %g7) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 193 | |
David S. Miller | 517af33 | 2006-02-01 15:55:21 -0800 | [diff] [blame] | 194 | KTSB_WRITE(%g1, %g5, %g6) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 195 | |
| 196 | /* fallthrough to TLB load */ |
| 197 | |
| 198 | kvmap_dtlb_load: |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 199 | |
| 200 | 661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 201 | retry |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 202 | .section .sun4v_2insn_patch, "ax" |
| 203 | .word 661b |
| 204 | nop |
| 205 | nop |
| 206 | .previous |
| 207 | |
| 208 | /* For sun4v the ASI_DTLB_DATA_IN store and the retry |
| 209 | * instruction get nop'd out and we get here to branch |
| 210 | * to the sun4v tlb load code. The registers are setup |
| 211 | * as follows: |
| 212 | * |
| 213 | * %g4: vaddr |
| 214 | * %g5: PTE |
| 215 | * %g6: TAG |
| 216 | * |
| 217 | * The sun4v TLB load wants the PTE in %g3 so we fix that |
| 218 | * up here. |
| 219 | */ |
| 220 | ba,pt %xcc, sun4v_dtlb_load |
| 221 | mov %g5, %g3 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 222 | |
| 223 | kvmap_dtlb_nonlinear: |
| 224 | /* Catch kernel NULL pointer derefs. */ |
| 225 | sethi %hi(PAGE_SIZE), %g5 |
| 226 | cmp %g4, %g5 |
| 227 | bleu,pn %xcc, kvmap_dtlb_longpath |
| 228 | nop |
| 229 | |
| 230 | KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) |
| 231 | |
| 232 | kvmap_dtlb_tsbmiss: |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 233 | sethi %hi(MODULES_VADDR), %g5 |
| 234 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 235 | blu,pn %xcc, kvmap_dtlb_longpath |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 236 | mov (VMALLOC_END >> 24), %g5 |
| 237 | sllx %g5, 24, %g5 |
| 238 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 239 | bgeu,pn %xcc, kvmap_dtlb_longpath |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 240 | nop |
| 241 | |
| 242 | kvmap_check_obp: |
| 243 | sethi %hi(LOW_OBP_ADDRESS), %g5 |
| 244 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 245 | blu,pn %xcc, kvmap_dtlb_vmalloc_addr |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 246 | mov 0x1, %g5 |
| 247 | sllx %g5, 32, %g5 |
| 248 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 249 | blu,pn %xcc, kvmap_dtlb_obp |
| 250 | nop |
| 251 | ba,pt %xcc, kvmap_dtlb_vmalloc_addr |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 252 | nop |
| 253 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 254 | kvmap_dtlb_longpath: |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 255 | |
| 256 | 661: rdpr %pstate, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 257 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate |
David S. Miller | df7d6ae | 2006-02-07 00:00:16 -0800 | [diff] [blame] | 258 | .section .sun4v_2insn_patch, "ax" |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 259 | .word 661b |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 260 | SET_GL(1) |
| 261 | ldxa [%g0] ASI_SCRATCHPAD, %g5 |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 262 | .previous |
| 263 | |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 264 | rdpr %tl, %g3 |
| 265 | cmp %g3, 1 |
| 266 | |
| 267 | 661: mov TLB_TAG_ACCESS, %g4 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 268 | ldxa [%g4] ASI_DMMU, %g5 |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 269 | .section .sun4v_2insn_patch, "ax" |
| 270 | .word 661b |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 271 | ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 272 | nop |
| 273 | .previous |
| 274 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 275 | be,pt %xcc, sparc64_realfault_common |
| 276 | mov FAULT_CODE_DTLB, %g4 |
| 277 | ba,pt %xcc, winfix_trampoline |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 278 | nop |