David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 1 | /* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling. |
| 2 | * |
| 3 | * Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net> |
| 4 | * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) |
| 5 | * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) |
| 6 | * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 7 | */ |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 8 | |
| 9 | #include <linux/config.h> |
| 10 | #include <asm/head.h> |
| 11 | #include <asm/asi.h> |
| 12 | #include <asm/page.h> |
| 13 | #include <asm/pgtable.h> |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 14 | #include <asm/tsb.h> |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 15 | |
| 16 | .text |
| 17 | .align 32 |
| 18 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 19 | kvmap_itlb: |
| 20 | /* g6: TAG TARGET */ |
| 21 | mov TLB_TAG_ACCESS, %g4 |
| 22 | ldxa [%g4] ASI_IMMU, %g4 |
| 23 | |
David S. Miller | d257d5d | 2006-02-06 23:44:37 -0800 | [diff] [blame] | 24 | /* sun4v_itlb_miss branches here with the missing virtual |
| 25 | * address already loaded into %g4 |
| 26 | */ |
| 27 | kvmap_itlb_4v: |
| 28 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 29 | kvmap_itlb_nonlinear: |
| 30 | /* Catch kernel NULL pointer calls. */ |
| 31 | sethi %hi(PAGE_SIZE), %g5 |
| 32 | cmp %g4, %g5 |
| 33 | bleu,pn %xcc, kvmap_dtlb_longpath |
| 34 | nop |
| 35 | |
| 36 | KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load) |
| 37 | |
| 38 | kvmap_itlb_tsb_miss: |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 39 | sethi %hi(LOW_OBP_ADDRESS), %g5 |
| 40 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 41 | blu,pn %xcc, kvmap_itlb_vmalloc_addr |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 42 | mov 0x1, %g5 |
| 43 | sllx %g5, 32, %g5 |
| 44 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 45 | blu,pn %xcc, kvmap_itlb_obp |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 46 | nop |
| 47 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 48 | kvmap_itlb_vmalloc_addr: |
| 49 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 50 | |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 51 | KTSB_LOCK_TAG(%g1, %g2, %g7) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 52 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 53 | /* Load and check PTE. */ |
| 54 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 55 | mov 1, %g7 |
| 56 | sllx %g7, TSB_TAG_INVALID_BIT, %g7 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 57 | brgez,a,pn %g5, kvmap_itlb_longpath |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 58 | KTSB_STORE(%g1, %g7) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 59 | |
David S. Miller | 517af33 | 2006-02-01 15:55:21 -0800 | [diff] [blame] | 60 | KTSB_WRITE(%g1, %g5, %g6) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 61 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 62 | /* fallthrough to TLB load */ |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 63 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 64 | kvmap_itlb_load: |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 65 | |
| 66 | 661: stxa %g5, [%g0] ASI_ITLB_DATA_IN |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 67 | retry |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 68 | .section .sun4v_2insn_patch, "ax" |
| 69 | .word 661b |
| 70 | nop |
| 71 | nop |
| 72 | .previous |
| 73 | |
| 74 | /* For sun4v the ASI_ITLB_DATA_IN store and the retry |
| 75 | * instruction get nop'd out and we get here to branch |
| 76 | * to the sun4v tlb load code. The registers are setup |
| 77 | * as follows: |
| 78 | * |
| 79 | * %g4: vaddr |
| 80 | * %g5: PTE |
| 81 | * %g6: TAG |
| 82 | * |
| 83 | * The sun4v TLB load wants the PTE in %g3 so we fix that |
| 84 | * up here. |
| 85 | */ |
| 86 | ba,pt %xcc, sun4v_itlb_load |
| 87 | mov %g5, %g3 |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 88 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 89 | kvmap_itlb_longpath: |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 90 | |
| 91 | 661: rdpr %pstate, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 92 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate |
David S. Miller | df7d6ae | 2006-02-07 00:00:16 -0800 | [diff] [blame] | 93 | .section .sun4v_2insn_patch, "ax" |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 94 | .word 661b |
David S. Miller | 6cc200d | 2006-02-18 16:36:39 -0800 | [diff] [blame] | 95 | SET_GL(1) |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 96 | nop |
| 97 | .previous |
| 98 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 99 | rdpr %tpc, %g5 |
| 100 | ba,pt %xcc, sparc64_realfault_common |
| 101 | mov FAULT_CODE_ITLB, %g4 |
David S. Miller | c9c1083 | 2005-10-12 12:22:46 -0700 | [diff] [blame] | 102 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 103 | kvmap_itlb_obp: |
| 104 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 105 | |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 106 | KTSB_LOCK_TAG(%g1, %g2, %g7) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 107 | |
David S. Miller | 517af33 | 2006-02-01 15:55:21 -0800 | [diff] [blame] | 108 | KTSB_WRITE(%g1, %g5, %g6) |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 109 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 110 | ba,pt %xcc, kvmap_itlb_load |
| 111 | nop |
David S. Miller | c9c1083 | 2005-10-12 12:22:46 -0700 | [diff] [blame] | 112 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 113 | kvmap_dtlb_obp: |
| 114 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath) |
| 115 | |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 116 | KTSB_LOCK_TAG(%g1, %g2, %g7) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 117 | |
David S. Miller | 517af33 | 2006-02-01 15:55:21 -0800 | [diff] [blame] | 118 | KTSB_WRITE(%g1, %g5, %g6) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 119 | |
| 120 | ba,pt %xcc, kvmap_dtlb_load |
| 121 | nop |
| 122 | |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 123 | .align 32 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 124 | kvmap_dtlb: |
| 125 | /* %g6: TAG TARGET */ |
| 126 | mov TLB_TAG_ACCESS, %g4 |
| 127 | ldxa [%g4] ASI_DMMU, %g4 |
David S. Miller | d257d5d | 2006-02-06 23:44:37 -0800 | [diff] [blame] | 128 | |
| 129 | /* sun4v_dtlb_miss branches here with the missing virtual |
| 130 | * address already loaded into %g4 |
| 131 | */ |
| 132 | kvmap_dtlb_4v: |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 133 | brgez,pn %g4, kvmap_dtlb_nonlinear |
David S. Miller | 5642530 | 2005-09-25 16:46:57 -0700 | [diff] [blame] | 134 | nop |
| 135 | |
David S. Miller | 9cc3a1a | 2006-02-21 20:51:13 -0800 | [diff] [blame^] | 136 | sethi %hi(kpte_linear_bitmap), %g2 |
| 137 | or %g2, %lo(kpte_linear_bitmap), %g2 |
| 138 | |
| 139 | /* Clear the PAGE_OFFSET top virtual bits, then shift |
| 140 | * down to get a 256MB physical address index. |
| 141 | */ |
| 142 | sllx %g4, 21, %g5 |
| 143 | mov 1, %g7 |
| 144 | srlx %g5, 21 + 28, %g5 |
| 145 | |
| 146 | /* Don't try this at home kids... this depends upon srlx |
| 147 | * only taking the low 6 bits of the shift count in %g5. |
| 148 | */ |
| 149 | sllx %g7, %g5, %g7 |
| 150 | |
| 151 | /* Divide by 64 to get the offset into the bitmask. */ |
| 152 | srlx %g5, 6, %g5 |
| 153 | |
| 154 | /* kern_linear_pte_xor[((mask & bit) ? 1 : 0)] */ |
| 155 | ldx [%g2 + %g5], %g2 |
| 156 | andcc %g2, %g7, %g0 |
| 157 | sethi %hi(kern_linear_pte_xor), %g5 |
| 158 | or %g5, %lo(kern_linear_pte_xor), %g5 |
| 159 | bne,a,pt %xcc, 1f |
| 160 | add %g5, 8, %g5 |
| 161 | |
| 162 | 1: ldx [%g5], %g2 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 163 | |
David S. Miller | 5642530 | 2005-09-25 16:46:57 -0700 | [diff] [blame] | 164 | .globl kvmap_linear_patch |
| 165 | kvmap_linear_patch: |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 166 | ba,pt %xcc, kvmap_dtlb_load |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 167 | xor %g2, %g4, %g5 |
| 168 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 169 | kvmap_dtlb_vmalloc_addr: |
| 170 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) |
David S. Miller | 5642530 | 2005-09-25 16:46:57 -0700 | [diff] [blame] | 171 | |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 172 | KTSB_LOCK_TAG(%g1, %g2, %g7) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 173 | |
| 174 | /* Load and check PTE. */ |
| 175 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 176 | mov 1, %g7 |
| 177 | sllx %g7, TSB_TAG_INVALID_BIT, %g7 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 178 | brgez,a,pn %g5, kvmap_dtlb_longpath |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 179 | KTSB_STORE(%g1, %g7) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 180 | |
David S. Miller | 517af33 | 2006-02-01 15:55:21 -0800 | [diff] [blame] | 181 | KTSB_WRITE(%g1, %g5, %g6) |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 182 | |
| 183 | /* fallthrough to TLB load */ |
| 184 | |
| 185 | kvmap_dtlb_load: |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 186 | |
| 187 | 661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 188 | retry |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 189 | .section .sun4v_2insn_patch, "ax" |
| 190 | .word 661b |
| 191 | nop |
| 192 | nop |
| 193 | .previous |
| 194 | |
| 195 | /* For sun4v the ASI_DTLB_DATA_IN store and the retry |
| 196 | * instruction get nop'd out and we get here to branch |
| 197 | * to the sun4v tlb load code. The registers are setup |
| 198 | * as follows: |
| 199 | * |
| 200 | * %g4: vaddr |
| 201 | * %g5: PTE |
| 202 | * %g6: TAG |
| 203 | * |
| 204 | * The sun4v TLB load wants the PTE in %g3 so we fix that |
| 205 | * up here. |
| 206 | */ |
| 207 | ba,pt %xcc, sun4v_dtlb_load |
| 208 | mov %g5, %g3 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 209 | |
| 210 | kvmap_dtlb_nonlinear: |
| 211 | /* Catch kernel NULL pointer derefs. */ |
| 212 | sethi %hi(PAGE_SIZE), %g5 |
| 213 | cmp %g4, %g5 |
| 214 | bleu,pn %xcc, kvmap_dtlb_longpath |
| 215 | nop |
| 216 | |
| 217 | KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) |
| 218 | |
| 219 | kvmap_dtlb_tsbmiss: |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 220 | sethi %hi(MODULES_VADDR), %g5 |
| 221 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 222 | blu,pn %xcc, kvmap_dtlb_longpath |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 223 | mov (VMALLOC_END >> 24), %g5 |
| 224 | sllx %g5, 24, %g5 |
| 225 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 226 | bgeu,pn %xcc, kvmap_dtlb_longpath |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 227 | nop |
| 228 | |
| 229 | kvmap_check_obp: |
| 230 | sethi %hi(LOW_OBP_ADDRESS), %g5 |
| 231 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 232 | blu,pn %xcc, kvmap_dtlb_vmalloc_addr |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 233 | mov 0x1, %g5 |
| 234 | sllx %g5, 32, %g5 |
| 235 | cmp %g4, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 236 | blu,pn %xcc, kvmap_dtlb_obp |
| 237 | nop |
| 238 | ba,pt %xcc, kvmap_dtlb_vmalloc_addr |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 239 | nop |
| 240 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 241 | kvmap_dtlb_longpath: |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 242 | |
| 243 | 661: rdpr %pstate, %g5 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 244 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate |
David S. Miller | df7d6ae | 2006-02-07 00:00:16 -0800 | [diff] [blame] | 245 | .section .sun4v_2insn_patch, "ax" |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 246 | .word 661b |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 247 | SET_GL(1) |
| 248 | ldxa [%g0] ASI_SCRATCHPAD, %g5 |
David S. Miller | 45fec05 | 2006-02-05 22:27:28 -0800 | [diff] [blame] | 249 | .previous |
| 250 | |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 251 | rdpr %tl, %g3 |
| 252 | cmp %g3, 1 |
| 253 | |
| 254 | 661: mov TLB_TAG_ACCESS, %g4 |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 255 | ldxa [%g4] ASI_DMMU, %g5 |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 256 | .section .sun4v_2insn_patch, "ax" |
| 257 | .word 661b |
David S. Miller | 8b23427 | 2006-02-17 18:01:02 -0800 | [diff] [blame] | 258 | ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 |
David S. Miller | 459b6e6 | 2006-02-11 12:21:20 -0800 | [diff] [blame] | 259 | nop |
| 260 | .previous |
| 261 | |
David S. Miller | 74bf431 | 2006-01-31 18:29:18 -0800 | [diff] [blame] | 262 | be,pt %xcc, sparc64_realfault_common |
| 263 | mov FAULT_CODE_DTLB, %g4 |
| 264 | ba,pt %xcc, winfix_trampoline |
David S. Miller | 2a7e299 | 2005-09-21 18:50:51 -0700 | [diff] [blame] | 265 | nop |