[RFC PATCH V1 07/11] riscv: Adapt satp operations to gap between hw page and sw page
From: Xu Lu
Date: Thu Nov 23 2023 - 01:58:56 EST
The control register CSR_SATP on RISC-V, which points to the root page
table page, is used by MMU to translate va to pa when TLB miss happens.
Thus it should be encoded at a granularity of hardware page, while
existing code usually encodes it via software page frame number.
This commit corrects encoding operations of CSR_SATP register. To get
developers rid of the annoying encoding format of CSR_SATP and the
conversion between sw pfn and hw pfn, we abstract the encoding
operations of CSR_SATP into a specific function.
Signed-off-by: Xu Lu <luxu.kernel@xxxxxxxxxxxxx>
---
arch/riscv/include/asm/pgtable.h | 7 +++++++
arch/riscv/kernel/head.S | 4 ++--
arch/riscv/kernel/hibernate.c | 3 ++-
arch/riscv/mm/context.c | 7 +++----
arch/riscv/mm/fault.c | 1 +
arch/riscv/mm/init.c | 7 +++++--
arch/riscv/mm/kasan_init.c | 7 +++++--
7 files changed, 25 insertions(+), 11 deletions(-)
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 9f81fe046cb8..56366f07985d 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -213,6 +213,13 @@ extern pgd_t swapper_pg_dir[];
extern pgd_t trampoline_pg_dir[];
extern pgd_t early_pg_dir[];
+static inline unsigned long make_satp(unsigned long pfn,
+ unsigned long asid, unsigned long satp_mode)
+{
+ return (pfn_to_hwpfn(pfn) |
+ ((asid & SATP_ASID_MASK) << SATP_ASID_SHIFT) | satp_mode);
+}
+
static __always_inline int __pte_present(unsigned long pteval)
{
return (pteval & (_PAGE_PRESENT | _PAGE_PROT_NONE));
diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
index b77397432403..dace2e4e6164 100644
--- a/arch/riscv/kernel/head.S
+++ b/arch/riscv/kernel/head.S
@@ -87,7 +87,7 @@ relocate_enable_mmu:
csrw CSR_TVEC, a2
/* Compute satp for kernel page tables, but don't load it yet */
- srl a2, a0, PAGE_SHIFT
+ srl a2, a0, HW_PAGE_SHIFT
la a1, satp_mode
REG_L a1, 0(a1)
or a2, a2, a1
@@ -100,7 +100,7 @@ relocate_enable_mmu:
*/
la a0, trampoline_pg_dir
XIP_FIXUP_OFFSET a0
- srl a0, a0, PAGE_SHIFT
+ srl a0, a0, HW_PAGE_SHIFT
or a0, a0, a1
sfence.vma
csrw CSR_SATP, a0
diff --git a/arch/riscv/kernel/hibernate.c b/arch/riscv/kernel/hibernate.c
index 671b686c0158..155be6b1d32c 100644
--- a/arch/riscv/kernel/hibernate.c
+++ b/arch/riscv/kernel/hibernate.c
@@ -395,7 +395,8 @@ int swsusp_arch_resume(void)
if (ret)
return ret;
- hibernate_restore_image(resume_hdr.saved_satp, (PFN_DOWN(__pa(resume_pg_dir)) | satp_mode),
+ hibernate_restore_image(resume_hdr.saved_satp,
+ make_satp(PFN_DOWN(__pa(resume_pg_dir)), 0, satp_mode),
resume_hdr.restore_cpu_addr);
return 0;
diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c
index 217fd4de6134..2ecf87433dfc 100644
--- a/arch/riscv/mm/context.c
+++ b/arch/riscv/mm/context.c
@@ -190,9 +190,8 @@ static void set_mm_asid(struct mm_struct *mm, unsigned int cpu)
raw_spin_unlock_irqrestore(&context_lock, flags);
switch_mm_fast:
- csr_write(CSR_SATP, virt_to_pfn(mm->pgd) |
- ((cntx & asid_mask) << SATP_ASID_SHIFT) |
- satp_mode);
+ csr_write(CSR_SATP, make_satp(virt_to_pfn(mm->pgd), (cntx & asid_mask),
+ satp_mode));
if (need_flush_tlb)
local_flush_tlb_all();
@@ -201,7 +200,7 @@ static void set_mm_asid(struct mm_struct *mm, unsigned int cpu)
static void set_mm_noasid(struct mm_struct *mm)
{
/* Switch the page table and blindly nuke entire local TLB */
- csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | satp_mode);
+ csr_write(CSR_SATP, make_satp(virt_to_pfn(mm->pgd), 0, satp_mode));
local_flush_tlb_all();
}
diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c
index 90d4ba36d1d0..026ac007febf 100644
--- a/arch/riscv/mm/fault.c
+++ b/arch/riscv/mm/fault.c
@@ -133,6 +133,7 @@ static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long a
*/
index = pgd_index(addr);
pfn = csr_read(CSR_SATP) & SATP_PPN;
+ pfn = hwpfn_to_pfn(pfn);
pgd = (pgd_t *)pfn_to_virt(pfn) + index;
pgd_k = init_mm.pgd + index;
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index a768b2b3ff05..c33a90d0c51d 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -805,7 +805,7 @@ static __init void set_satp_mode(uintptr_t dtb_pa)
(uintptr_t)early_p4d : (uintptr_t)early_pud,
PGDIR_SIZE, PAGE_TABLE);
- identity_satp = PFN_DOWN((uintptr_t)&early_pg_dir) | satp_mode;
+ identity_satp = make_satp(PFN_DOWN((uintptr_t)&early_pg_dir), 0, satp_mode);
local_flush_tlb_all();
csr_write(CSR_SATP, identity_satp);
@@ -1285,6 +1285,8 @@ static void __init create_linear_mapping_page_table(void)
static void __init setup_vm_final(void)
{
+ unsigned long satp;
+
/* Setup swapper PGD for fixmap */
#if !defined(CONFIG_64BIT)
/*
@@ -1318,7 +1320,8 @@ static void __init setup_vm_final(void)
clear_fixmap(FIX_P4D);
/* Move to swapper page table */
- csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | satp_mode);
+ satp = make_satp(PFN_DOWN(__pa_symbol(swapper_pg_dir)), 0, satp_mode);
+ csr_write(CSR_SATP, satp);
local_flush_tlb_all();
pt_ops_set_late();
diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c
index 5e39dcf23fdb..72269e9f1964 100644
--- a/arch/riscv/mm/kasan_init.c
+++ b/arch/riscv/mm/kasan_init.c
@@ -471,11 +471,13 @@ static void __init create_tmp_mapping(void)
void __init kasan_init(void)
{
+ unsigned long satp;
phys_addr_t p_start, p_end;
u64 i;
create_tmp_mapping();
- csr_write(CSR_SATP, PFN_DOWN(__pa(tmp_pg_dir)) | satp_mode);
+ satp = make_satp(PFN_DOWN(__pa(tmp_pg_dir)), 0, satp_mode);
+ csr_write(CSR_SATP, satp);
kasan_early_clear_pgd(pgd_offset_k(KASAN_SHADOW_START),
KASAN_SHADOW_START, KASAN_SHADOW_END);
@@ -520,6 +522,7 @@ void __init kasan_init(void)
memset(kasan_early_shadow_page, KASAN_SHADOW_INIT, PAGE_SIZE);
init_task.kasan_depth = 0;
- csr_write(CSR_SATP, PFN_DOWN(__pa(swapper_pg_dir)) | satp_mode);
+ satp = make_satp(PFN_DOWN(__pa(swapper_pg_dir)), 0, satp_mode);
+ csr_write(CSR_SATP, satp);
local_flush_tlb_all();
}
--
2.20.1