| /* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling. |
| * |
| * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net> |
| * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) |
| * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) |
| * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) |
| */ |
| |
| #include <asm/head.h> |
| #include <asm/asi.h> |
| #include <asm/page.h> |
| #include <asm/pgtable.h> |
| #include <asm/tsb.h> |
| |
| .text |
| .align 32 |
| |
| kvmap_itlb: |
| /* g6: TAG TARGET */ |
| mov TLB_TAG_ACCESS, %g4 |
| ldxa [%g4] ASI_IMMU, %g4 |
| |
| /* sun4v_itlb_miss branches here with the missing virtual |
| * address already loaded into %g4 |
| */ |
| kvmap_itlb_4v: |
| |
| kvmap_itlb_nonlinear: |
| /* Catch kernel NULL pointer calls. */ |
| sethi %hi(PAGE_SIZE), %g5 |
| cmp %g4, %g5 |
| bleu,pn %xcc, kvmap_dtlb_longpath |
| nop |
| |
| KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load) |
| |
| kvmap_itlb_tsb_miss: |
| sethi %hi(LOW_OBP_ADDRESS), %g5 |
| cmp %g4, %g5 |
| blu,pn %xcc, kvmap_itlb_vmalloc_addr |
| mov 0x1, %g5 |
| sllx %g5, 32, %g5 |
| cmp %g4, %g5 |
| blu,pn %xcc, kvmap_itlb_obp |
| nop |
| |
| kvmap_itlb_vmalloc_addr: |
| KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) |
| |
| KTSB_LOCK_TAG(%g1, %g2, %g7) |
| |
| /* Load and check PTE. */ |
| ldxa [%g5] ASI_PHYS_USE_EC, %g5 |
| mov 1, %g7 |
| sllx %g7, TSB_TAG_INVALID_BIT, %g7 |
| brgez,a,pn %g5, kvmap_itlb_longpath |
| KTSB_STORE(%g1, %g7) |
| |
| KTSB_WRITE(%g1, %g5, %g6) |
| |
| /* fallthrough to TLB load */ |
| |
| kvmap_itlb_load: |
| |
| 661: stxa %g5, [%g0] ASI_ITLB_DATA_IN |
| retry |
| .section .sun4v_2insn_patch, "ax" |
| .word 661b |
| nop |
| nop |
| .previous |
| |
| /* For sun4v the ASI_ITLB_DATA_IN store and the retry |
| * instruction get nop'd out and we get here to branch |
| * to the sun4v tlb load code. The registers are setup |
| * as follows: |
| * |
| * %g4: vaddr |
| * %g5: PTE |
| * %g6: TAG |
| * |
| * The sun4v TLB load wants the PTE in %g3 so we fix that |
| * up here. |
| */ |
| ba,pt %xcc, sun4v_itlb_load |
| mov %g5, %g3 |
| |
| kvmap_itlb_longpath: |
| |
| 661: rdpr %pstate, %g5 |
| wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate |
| .section .sun4v_2insn_patch, "ax" |
| .word 661b |
| SET_GL(1) |
| nop |
| .previous |
| |
| rdpr %tpc, %g5 |
| ba,pt %xcc, sparc64_realfault_common |
| mov FAULT_CODE_ITLB, %g4 |
| |
| kvmap_itlb_obp: |
| OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath) |
| |
| KTSB_LOCK_TAG(%g1, %g2, %g7) |
| |
| KTSB_WRITE(%g1, %g5, %g6) |
| |
| ba,pt %xcc, kvmap_itlb_load |
| nop |
| |
| kvmap_dtlb_obp: |
| OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath) |
| |
| KTSB_LOCK_TAG(%g1, %g2, %g7) |
| |
| KTSB_WRITE(%g1, %g5, %g6) |
| |
| ba,pt %xcc, kvmap_dtlb_load |
| nop |
| |
| .align 32 |
| kvmap_dtlb_tsb4m_load: |
| KTSB_LOCK_TAG(%g1, %g2, %g7) |
| KTSB_WRITE(%g1, %g5, %g6) |
| ba,pt %xcc, kvmap_dtlb_load |
| nop |
| |
| kvmap_dtlb: |
| /* %g6: TAG TARGET */ |
| mov TLB_TAG_ACCESS, %g4 |
| ldxa [%g4] ASI_DMMU, %g4 |
| |
| /* sun4v_dtlb_miss branches here with the missing virtual |
| * address already loaded into %g4 |
| */ |
| kvmap_dtlb_4v: |
| brgez,pn %g4, kvmap_dtlb_nonlinear |
| nop |
| |
| #ifdef CONFIG_DEBUG_PAGEALLOC |
| /* Index through the base page size TSB even for linear |
| * mappings when using page allocation debugging. |
| */ |
| KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) |
| #else |
| /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */ |
| KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) |
| #endif |
| /* TSB entry address left in %g1, lookup linear PTE. |
| * Must preserve %g1 and %g6 (TAG). |
| */ |
| kvmap_dtlb_tsb4m_miss: |
| sethi %hi(kpte_linear_bitmap), %g2 |
| or %g2, %lo(kpte_linear_bitmap), %g2 |
| |
| /* Clear the PAGE_OFFSET top virtual bits, then shift |
| * down to get a 256MB physical address index. |
| */ |
| sllx %g4, 21, %g5 |
| mov 1, %g7 |
| srlx %g5, 21 + 28, %g5 |
| |
| /* Don't try this at home kids... this depends upon srlx |
| * only taking the low 6 bits of the shift count in %g5. |
| */ |
| sllx %g7, %g5, %g7 |
| |
| /* Divide by 64 to get the offset into the bitmask. */ |
| srlx %g5, 6, %g5 |
| sllx %g5, 3, %g5 |
| |
| /* kern_linear_pte_xor[((mask & bit) ? 1 : 0)] */ |
| ldx [%g2 + %g5], %g2 |
| andcc %g2, %g7, %g0 |
| sethi %hi(kern_linear_pte_xor), %g5 |
| or %g5, %lo(kern_linear_pte_xor), %g5 |
| bne,a,pt %xcc, 1f |
| add %g5, 8, %g5 |
| |
| 1: ldx [%g5], %g2 |
| |
| .globl kvmap_linear_patch |
| kvmap_linear_patch: |
| ba,pt %xcc, kvmap_dtlb_tsb4m_load |
| xor %g2, %g4, %g5 |
| |
| kvmap_dtlb_vmalloc_addr: |
| KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) |
| |
| KTSB_LOCK_TAG(%g1, %g2, %g7) |
| |
| /* Load and check PTE. */ |
| ldxa [%g5] ASI_PHYS_USE_EC, %g5 |
| mov 1, %g7 |
| sllx %g7, TSB_TAG_INVALID_BIT, %g7 |
| brgez,a,pn %g5, kvmap_dtlb_longpath |
| KTSB_STORE(%g1, %g7) |
| |
| KTSB_WRITE(%g1, %g5, %g6) |
| |
| /* fallthrough to TLB load */ |
| |
| kvmap_dtlb_load: |
| |
| 661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB |
| retry |
| .section .sun4v_2insn_patch, "ax" |
| .word 661b |
| nop |
| nop |
| .previous |
| |
| /* For sun4v the ASI_DTLB_DATA_IN store and the retry |
| * instruction get nop'd out and we get here to branch |
| * to the sun4v tlb load code. The registers are setup |
| * as follows: |
| * |
| * %g4: vaddr |
| * %g5: PTE |
| * %g6: TAG |
| * |
| * The sun4v TLB load wants the PTE in %g3 so we fix that |
| * up here. |
| */ |
| ba,pt %xcc, sun4v_dtlb_load |
| mov %g5, %g3 |
| |
| #ifdef CONFIG_SPARSEMEM_VMEMMAP |
| kvmap_vmemmap: |
| sub %g4, %g5, %g5 |
| srlx %g5, 22, %g5 |
| sethi %hi(vmemmap_table), %g1 |
| sllx %g5, 3, %g5 |
| or %g1, %lo(vmemmap_table), %g1 |
| ba,pt %xcc, kvmap_dtlb_load |
| ldx [%g1 + %g5], %g5 |
| #endif |
| |
| kvmap_dtlb_nonlinear: |
| /* Catch kernel NULL pointer derefs. */ |
| sethi %hi(PAGE_SIZE), %g5 |
| cmp %g4, %g5 |
| bleu,pn %xcc, kvmap_dtlb_longpath |
| nop |
| |
| #ifdef CONFIG_SPARSEMEM_VMEMMAP |
| /* Do not use the TSB for vmemmap. */ |
| mov (VMEMMAP_BASE >> 24), %g5 |
| sllx %g5, 24, %g5 |
| cmp %g4,%g5 |
| bgeu,pn %xcc, kvmap_vmemmap |
| nop |
| #endif |
| |
| KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) |
| |
| kvmap_dtlb_tsbmiss: |
| sethi %hi(MODULES_VADDR), %g5 |
| cmp %g4, %g5 |
| blu,pn %xcc, kvmap_dtlb_longpath |
| mov (VMALLOC_END >> 24), %g5 |
| sllx %g5, 24, %g5 |
| cmp %g4, %g5 |
| bgeu,pn %xcc, kvmap_dtlb_longpath |
| nop |
| |
| kvmap_check_obp: |
| sethi %hi(LOW_OBP_ADDRESS), %g5 |
| cmp %g4, %g5 |
| blu,pn %xcc, kvmap_dtlb_vmalloc_addr |
| mov 0x1, %g5 |
| sllx %g5, 32, %g5 |
| cmp %g4, %g5 |
| blu,pn %xcc, kvmap_dtlb_obp |
| nop |
| ba,pt %xcc, kvmap_dtlb_vmalloc_addr |
| nop |
| |
| kvmap_dtlb_longpath: |
| |
| 661: rdpr %pstate, %g5 |
| wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate |
| .section .sun4v_2insn_patch, "ax" |
| .word 661b |
| SET_GL(1) |
| ldxa [%g0] ASI_SCRATCHPAD, %g5 |
| .previous |
| |
| rdpr %tl, %g3 |
| cmp %g3, 1 |
| |
| 661: mov TLB_TAG_ACCESS, %g4 |
| ldxa [%g4] ASI_DMMU, %g5 |
| .section .sun4v_2insn_patch, "ax" |
| .word 661b |
| ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 |
| nop |
| .previous |
| |
| be,pt %xcc, sparc64_realfault_common |
| mov FAULT_CODE_DTLB, %g4 |
| ba,pt %xcc, winfix_trampoline |
| nop |