/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2020-2022 Loongson Technology Corporation Limited */ #include #include #include #include #include #include #define INVTLB_ADDR_GFALSE_AND_ASID 5 #define PTRS_PER_PGD_BITS (PAGE_SHIFT - PTRLOG) #define PTRS_PER_PUD_BITS (PAGE_SHIFT - PTRLOG) #define PTRS_PER_PMD_BITS (PAGE_SHIFT - PTRLOG) #define PTRS_PER_PTE_BITS (PAGE_SHIFT - PTRLOG) #ifdef CONFIG_32BIT #define PTE_LL ll.w #define PTE_SC sc.w #else #define PTE_LL ll.d #define PTE_SC sc.d #endif .macro tlb_do_page_fault, write SYM_CODE_START(tlb_do_page_fault_\write) UNWIND_HINT_UNDEFINED SAVE_ALL csrrd a2, LOONGARCH_CSR_BADV move a0, sp REG_S a2, sp, PT_BVADDR li.w a1, \write bl do_page_fault RESTORE_ALL_AND_RET SYM_CODE_END(tlb_do_page_fault_\write) .endm tlb_do_page_fault 0 tlb_do_page_fault 1 SYM_CODE_START(handle_tlb_protect) UNWIND_HINT_UNDEFINED BACKUP_T0T1 SAVE_ALL move a0, sp move a1, zero csrrd a2, LOONGARCH_CSR_BADV REG_S a2, sp, PT_BVADDR la_abs t0, do_page_fault jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_CODE_END(handle_tlb_protect) SYM_CODE_START(handle_tlb_load) UNWIND_HINT_UNDEFINED csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ csrrd t0, LOONGARCH_CSR_BADV bltz t0, vmalloc_load csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_load: /* Get PGD offset in bytes */ #ifdef CONFIG_32BIT PTR_BSTRPICK ra, t0, 31, PGDIR_SHIFT #else PTR_BSTRPICK ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT #endif PTR_ALSL t1, ra, t1, _PGD_T_LOG2 #if CONFIG_PGTABLE_LEVELS > 3 PTR_L t1, t1, 0 PTR_BSTRPICK ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT PTR_ALSL t1, ra, t1, _PMD_T_LOG2 #endif #if CONFIG_PGTABLE_LEVELS > 2 PTR_L t1, t1, 0 PTR_BSTRPICK ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT PTR_ALSL t1, ra, t1, _PMD_T_LOG2 #endif PTR_L ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ PTR_ROTRI ra, ra, _PAGE_HUGE_SHIFT + 1 bltz ra, tlb_huge_update_load PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) PTR_BSTRPICK t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT PTR_ALSL t1, t0, ra, _PTE_T_LOG2 #ifdef CONFIG_SMP smp_pgtable_change_load: PTE_LL t0, t1, 0 #else PTR_L t0, t1, 0 #endif andi ra, t0, _PAGE_PRESENT beqz ra, nopage_tlb_load ori t0, t0, _PAGE_VALID #ifdef CONFIG_SMP PTE_SC t0, t1, 0 beqz t0, smp_pgtable_change_load #else PTR_S t0, t1, 0 #endif tlbsrch PTR_BSTRINS t1, zero, _PTE_T_LOG2, _PTE_T_LOG2 PTR_L t0, t1, 0 PTR_L t1, t1, _PTE_T_SIZE csrwr t0, LOONGARCH_CSR_TLBELO0 csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn vmalloc_load: la_abs t1, swapper_pg_dir b vmalloc_done_load /* This is the entry point of a huge page. */ tlb_huge_update_load: #ifdef CONFIG_SMP PTE_LL ra, t1, 0 #else PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) #endif andi t0, ra, _PAGE_PRESENT beqz t0, nopage_tlb_load #ifdef CONFIG_SMP ori t0, ra, _PAGE_VALID PTE_SC t0, t1, 0 beqz t0, tlb_huge_update_load ori t0, ra, _PAGE_VALID #else ori t0, ra, _PAGE_VALID PTR_S t0, t1, 0 #endif csrrd ra, LOONGARCH_CSR_ASID csrrd t1, LOONGARCH_CSR_BADV andi ra, ra, CSR_ASID_ASID invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the * of the large TLB entry size we intend to use. * A TLB entry half the size of the configured * huge page size is configured into entrylo0 * and entrylo1 to cover the contiguous huge PTE * address space. */ /* Huge page: Move Global bit */ xori t0, t0, _PAGE_HUGE lu12i.w t1, _PAGE_HGLOBAL >> 12 and t1, t0, t1 PTR_SRLI t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) or t0, t0, t1 move ra, t0 csrwr ra, LOONGARCH_CSR_TLBELO0 /* Convert to entrylo1 */ PTR_ADDI t1, zero, 1 PTR_SLLI t1, t1, (HPAGE_SHIFT - 1) PTR_ADD t0, t0, t1 csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 PTR_LI t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX tlbfill PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 PTR_LI t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn nopage_tlb_load: dbar 0x700 csrrd ra, EXCEPTION_KS2 la_abs t0, tlb_do_page_fault_0 jr t0 SYM_CODE_END(handle_tlb_load) SYM_CODE_START(handle_tlb_load_ptw) UNWIND_HINT_UNDEFINED csrwr t0, LOONGARCH_CSR_KS0 csrwr t1, LOONGARCH_CSR_KS1 la_abs t0, tlb_do_page_fault_0 jr t0 SYM_CODE_END(handle_tlb_load_ptw) SYM_CODE_START(handle_tlb_store) UNWIND_HINT_UNDEFINED csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ csrrd t0, LOONGARCH_CSR_BADV bltz t0, vmalloc_store csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_store: /* Get PGD offset in bytes */ #ifdef CONFIG_32BIT PTR_BSTRPICK ra, t0, 31, PGDIR_SHIFT #else PTR_BSTRPICK ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT #endif PTR_ALSL t1, ra, t1, _PGD_T_LOG2 #if CONFIG_PGTABLE_LEVELS > 3 PTR_L t1, t1, 0 PTR_BSTRPICK ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT PTR_ALSL t1, ra, t1, _PMD_T_LOG2 #endif #if CONFIG_PGTABLE_LEVELS > 2 PTR_L t1, t1, 0 PTR_BSTRPICK ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT PTR_ALSL t1, ra, t1, _PMD_T_LOG2 #endif PTR_L ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ PTR_ROTRI ra, ra, _PAGE_HUGE_SHIFT + 1 bltz ra, tlb_huge_update_store PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) PTR_BSTRPICK t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT PTR_ALSL t1, t0, ra, _PTE_T_LOG2 #ifdef CONFIG_SMP smp_pgtable_change_store: PTE_LL t0, t1, 0 #else PTR_L t0, t1, 0 #endif #ifdef CONFIG_64BIT andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE #else PTR_LI ra, _PAGE_PRESENT | _PAGE_WRITE and ra, ra, t0 nor ra, ra, zero #endif bnez ra, nopage_tlb_store #ifdef CONFIG_64BIT ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #else PTR_LI ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) or t0, ra, t0 #endif #ifdef CONFIG_SMP PTE_SC t0, t1, 0 beqz t0, smp_pgtable_change_store #else PTR_S t0, t1, 0 #endif tlbsrch PTR_BSTRINS t1, zero, _PTE_T_LOG2, _PTE_T_LOG2 PTR_L t0, t1, 0 PTR_L t1, t1, _PTE_T_SIZE csrwr t0, LOONGARCH_CSR_TLBELO0 csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn vmalloc_store: la_abs t1, swapper_pg_dir b vmalloc_done_store /* This is the entry point of a huge page. */ tlb_huge_update_store: #ifdef CONFIG_SMP PTE_LL ra, t1, 0 #else PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) #endif #ifdef CONFIG_64BIT andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE #else PTR_LI t0, _PAGE_PRESENT | _PAGE_WRITE and t0, t0, ra nor t0, t0, zero #endif bnez t0, nopage_tlb_store #ifdef CONFIG_SMP ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) PTE_SC t0, t1, 0 beqz t0, tlb_huge_update_store ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #else #ifdef CONFIG_64BIT ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #else PTR_LI t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) or t0, ra, t0 #endif PTR_S t0, t1, 0 #endif csrrd ra, LOONGARCH_CSR_ASID csrrd t1, LOONGARCH_CSR_BADV andi ra, ra, CSR_ASID_ASID invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the * of the large TLB entry size we intend to use. * A TLB entry half the size of the configured * huge page size is configured into entrylo0 * and entrylo1 to cover the contiguous huge PTE * address space. */ /* Huge page: Move Global bit */ xori t0, t0, _PAGE_HUGE lu12i.w t1, _PAGE_HGLOBAL >> 12 and t1, t0, t1 PTR_SRLI t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) or t0, t0, t1 move ra, t0 csrwr ra, LOONGARCH_CSR_TLBELO0 /* Convert to entrylo1 */ PTR_ADDI t1, zero, 1 PTR_SLLI t1, t1, (HPAGE_SHIFT - 1) PTR_ADD t0, t0, t1 csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 PTR_LI t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX tlbfill /* Reset default page size */ PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 PTR_LI t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn nopage_tlb_store: dbar 0x700 csrrd ra, EXCEPTION_KS2 la_abs t0, tlb_do_page_fault_1 jr t0 SYM_CODE_END(handle_tlb_store) SYM_CODE_START(handle_tlb_store_ptw) UNWIND_HINT_UNDEFINED csrwr t0, LOONGARCH_CSR_KS0 csrwr t1, LOONGARCH_CSR_KS1 la_abs t0, tlb_do_page_fault_1 jr t0 SYM_CODE_END(handle_tlb_store_ptw) SYM_CODE_START(handle_tlb_modify) UNWIND_HINT_UNDEFINED csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ csrrd t0, LOONGARCH_CSR_BADV bltz t0, vmalloc_modify csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_modify: /* Get PGD offset in bytes */ #ifdef CONFIG_32BIT PTR_BSTRPICK ra, t0, 31, PGDIR_SHIFT #else PTR_BSTRPICK ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT #endif PTR_ALSL t1, ra, t1, _PGD_T_LOG2 #if CONFIG_PGTABLE_LEVELS > 3 PTR_L t1, t1, 0 PTR_BSTRPICK ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT PTR_ALSL t1, ra, t1, _PMD_T_LOG2 #endif #if CONFIG_PGTABLE_LEVELS > 2 PTR_L t1, t1, 0 PTR_BSTRPICK ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT PTR_ALSL t1, ra, t1, _PMD_T_LOG2 #endif PTR_L ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ PTR_ROTRI ra, ra, _PAGE_HUGE_SHIFT + 1 bltz ra, tlb_huge_update_modify PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) PTR_BSTRPICK t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT PTR_ALSL t1, t0, ra, _PTE_T_LOG2 #ifdef CONFIG_SMP smp_pgtable_change_modify: PTE_LL t0, t1, 0 #else PTR_L t0, t1, 0 #endif #ifdef CONFIG_64BIT andi ra, t0, _PAGE_WRITE #else PTR_LI ra, _PAGE_WRITE and ra, t0, ra #endif beqz ra, nopage_tlb_modify #ifdef CONFIG_64BIT ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #else PTR_LI ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) or t0, ra, t0 #endif #ifdef CONFIG_SMP PTE_SC t0, t1, 0 beqz t0, smp_pgtable_change_modify #else PTR_S t0, t1, 0 #endif tlbsrch PTR_BSTRINS t1, zero, _PTE_T_LOG2, _PTE_T_LOG2 PTR_L t0, t1, 0 PTR_L t1, t1, _PTE_T_SIZE csrwr t0, LOONGARCH_CSR_TLBELO0 csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn vmalloc_modify: la_abs t1, swapper_pg_dir b vmalloc_done_modify /* This is the entry point of a huge page. */ tlb_huge_update_modify: #ifdef CONFIG_SMP PTE_LL ra, t1, 0 #else PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) #endif #ifdef CONFIG_64BIT andi t0, ra, _PAGE_WRITE #else PTR_LI t0, _PAGE_WRITE and t0, ra, t0 #endif beqz t0, nopage_tlb_modify #ifdef CONFIG_SMP ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) PTE_SC t0, t1, 0 beqz t0, tlb_huge_update_modify ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #else #ifdef CONFIG_64BIT ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #else PTR_LI t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) or t0, ra, t0 #endif PTR_S t0, t1, 0 #endif csrrd ra, LOONGARCH_CSR_ASID csrrd t1, LOONGARCH_CSR_BADV andi ra, ra, CSR_ASID_ASID invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the * of the large TLB entry size we intend to use. * A TLB entry half the size of the configured * huge page size is configured into entrylo0 * and entrylo1 to cover the contiguous huge PTE * address space. */ /* Huge page: Move Global bit */ xori t0, t0, _PAGE_HUGE lu12i.w t1, _PAGE_HGLOBAL >> 12 and t1, t0, t1 PTR_SRLI t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) or t0, t0, t1 move ra, t0 csrwr ra, LOONGARCH_CSR_TLBELO0 /* Convert to entrylo1 */ PTR_ADDI t1, zero, 1 PTR_SLLI t1, t1, (HPAGE_SHIFT - 1) PTR_ADD t0, t0, t1 csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 PTR_LI t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX tlbfill /* Reset default page size */ PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 PTR_LI t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn nopage_tlb_modify: dbar 0x700 csrrd ra, EXCEPTION_KS2 la_abs t0, tlb_do_page_fault_1 jr t0 SYM_CODE_END(handle_tlb_modify) SYM_CODE_START(handle_tlb_modify_ptw) UNWIND_HINT_UNDEFINED csrwr t0, LOONGARCH_CSR_KS0 csrwr t1, LOONGARCH_CSR_KS1 la_abs t0, tlb_do_page_fault_1 jr t0 SYM_CODE_END(handle_tlb_modify_ptw) #ifdef CONFIG_32BIT SYM_CODE_START(handle_tlb_refill) UNWIND_HINT_UNDEFINED csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 li.w ra, 0x1fffffff csrrd t0, LOONGARCH_CSR_PGD csrrd t1, LOONGARCH_CSR_TLBRBADV srli.w t1, t1, PGDIR_SHIFT slli.w t1, t1, 0x2 add.w t0, t0, t1 and t0, t0, ra ld.w t0, t0, 0 csrrd t1, LOONGARCH_CSR_TLBRBADV slli.w t1, t1, (32 - PGDIR_SHIFT) srli.w t1, t1, (32 - PGDIR_SHIFT + PAGE_SHIFT + 1) slli.w t1, t1, (0x2 + 1) add.w t0, t0, t1 and t0, t0, ra ld.w t1, t0, 0x0 csrwr t1, LOONGARCH_CSR_TLBRELO0 ld.w t1, t0, 0x4 csrwr t1, LOONGARCH_CSR_TLBRELO1 tlbfill csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn SYM_CODE_END(handle_tlb_refill) #endif #ifdef CONFIG_64BIT SYM_CODE_START(handle_tlb_refill) UNWIND_HINT_UNDEFINED csrwr t0, LOONGARCH_CSR_TLBRSAVE csrrd t0, LOONGARCH_CSR_PGD lddir t0, t0, 3 #if CONFIG_PGTABLE_LEVELS > 3 lddir t0, t0, 2 #endif #if CONFIG_PGTABLE_LEVELS > 2 lddir t0, t0, 1 #endif ldpte t0, 0 ldpte t0, 1 tlbfill csrrd t0, LOONGARCH_CSR_TLBRSAVE ertn SYM_CODE_END(handle_tlb_refill) #endif