@@ -68,6 +68,8 @@ FIELD(TLBENTRY, PLV, 2, 2)
FIELD(TLBENTRY, MAT, 4, 2)
FIELD(TLBENTRY, G, 6, 1)
FIELD(TLBENTRY, HUGE, 6, 1)
+FIELD(TLBENTRY, PRESENT, 7, 1)
+FIELD(TLBENTRY, WRITE, 8, 1)
FIELD(TLBENTRY, HGLOBAL, 12, 1)
FIELD(TLBENTRY, LEVEL, 13, 2)
FIELD(TLBENTRY_32, PPN, 8, 24)
@@ -103,6 +105,7 @@ FIELD(CSR_PWCH, DIR3_BASE, 0, 6)
FIELD(CSR_PWCH, DIR3_WIDTH, 6, 6)
FIELD(CSR_PWCH, DIR4_BASE, 12, 6)
FIELD(CSR_PWCH, DIR4_WIDTH, 18, 6)
+FIELD(CSR_PWCH, HPTW_EN, 24, 1)
#define LOONGARCH_CSR_STLBPS 0x1e /* Stlb page size */
FIELD(CSR_STLBPS, PS, 0, 5)
@@ -476,6 +476,7 @@ static void loongarch_la664_initfn(Object *obj)
env->cpucfg[0] = 0x14d000; /* PRID */
loongarch_common_initfn(env, obj);
+ env->cpucfg[2] = FIELD_DP32(env->cpucfg[2], CPUCFG2, HPTW, 1);
}
static void loongarch_la464_initfn(Object *obj)
@@ -155,6 +155,7 @@ FIELD(CPUCFG2, LBT_ARM, 19, 1)
FIELD(CPUCFG2, LBT_MIPS, 20, 1)
FIELD(CPUCFG2, LSPW, 21, 1)
FIELD(CPUCFG2, LAM, 22, 1)
+FIELD(CPUCFG2, HPTW, 24, 1)
/* cpucfg[3] bits */
FIELD(CPUCFG3, CCDMA, 0, 1)
@@ -178,10 +178,11 @@ static hwaddr dmw_va2pa(CPULoongArchState *env, target_ulong va,
int get_physical_address(CPULoongArchState *env, hwaddr *physical,
int *prot, target_ulong address,
- MMUAccessType access_type, int mmu_idx)
+ MMUAccessType access_type, int mmu_idx, bool is_debug)
{
int user_mode = mmu_idx == MMU_USER_IDX;
int kernel_mode = mmu_idx == MMU_KERNEL_IDX;
+ int ret;
uint32_t plv, base_c, base_v;
int64_t addr_high;
uint8_t da = FIELD_EX64(env->CSR_CRMD, CSR_CRMD, DA);
@@ -221,8 +222,25 @@ int get_physical_address(CPULoongArchState *env, hwaddr *physical,
}
/* Mapped address */
- return loongarch_map_address(env, physical, prot, address,
- access_type, mmu_idx);
+ ret = loongarch_map_address(env, physical, prot, address,
+ access_type, mmu_idx);
+#ifdef CONFIG_TCG
+ if (!FIELD_EX32(env->cpucfg[2], CPUCFG2, HPTW)) {
+ return ret;
+ }
+
+ if (!FIELD_EX32(env->CSR_PWCH, CSR_PWCH, HPTW_EN)) {
+ return ret;
+ }
+
+ if (do_page_walk(env, address, access_type, ret, physical, is_debug)) {
+ if (!is_debug) {
+ ret = loongarch_map_address(env, physical, prot, address,
+ access_type, mmu_idx);
+ }
+ }
+#endif
+ return ret;
}
hwaddr loongarch_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
@@ -232,7 +250,7 @@ hwaddr loongarch_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
int prot;
if (get_physical_address(env, &phys_addr, &prot, addr, MMU_DATA_LOAD,
- cpu_mmu_index(cs, false)) != 0) {
+ cpu_mmu_index(cs, false), true) != 0) {
return -1;
}
return phys_addr;
@@ -56,13 +56,15 @@ bool loongarch_tlb_search(CPULoongArchState *env, target_ulong vaddr,
int *index);
int get_physical_address(CPULoongArchState *env, hwaddr *physical,
int *prot, target_ulong address,
- MMUAccessType access_type, int mmu_idx);
+ MMUAccessType access_type, int mmu_idx, bool is_debug);
hwaddr loongarch_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
#ifdef CONFIG_TCG
bool loongarch_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
MMUAccessType access_type, int mmu_idx,
bool probe, uintptr_t retaddr);
+bool do_page_walk(CPULoongArchState *env, vaddr address,
+ MMUAccessType, int tlb_error, hwaddr *physical, bool is_debug);
#endif
#endif /* !CONFIG_USER_ONLY */
@@ -8,6 +8,7 @@
#include "qemu/osdep.h"
#include "qemu/guest-random.h"
+#include "qemu/atomic.h"
#include "cpu.h"
#include "internals.h"
@@ -504,7 +505,7 @@ bool loongarch_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
/* Data access */
ret = get_physical_address(env, &physical, &prot, address,
- access_type, mmu_idx);
+ access_type, mmu_idx, false);
if (ret == TLBRET_MATCH) {
tlb_set_page(cs, address & TARGET_PAGE_MASK,
@@ -651,3 +652,147 @@ void helper_ldpte(CPULoongArchState *env, target_ulong base, target_ulong odd,
}
env->CSR_TLBREHI = FIELD_DP64(env->CSR_TLBREHI, CSR_TLBREHI, PS, ps);
}
+
+static target_ulong get_pte_base(CPULoongArchState *env, vaddr address)
+{
+ uint64_t dir_base, dir_width;
+ target_ulong base;
+ int i;
+
+ /* Get PGD */
+ base = ((address >> 63) & 0x1) ? env->CSR_PGDH : env->CSR_PGDL;
+
+ for (i = 4; i > 0; i--) {
+ get_dir_base_width(env, &dir_base, &dir_width, i);
+ /*
+ * LDDIR: level = 2 corresponds to Dir1 in PWCL.
+ * PWCL/PWCH: dir >= 1 && dir_width == 0 means no such level.
+ */
+ if (i >= 2 && dir_width == 0) {
+ continue;
+ }
+ base = do_lddir(env, base, address, i);
+ }
+
+ return base;
+}
+
+bool do_page_walk(CPULoongArchState *env, vaddr address,
+ MMUAccessType access_type, int tlb_error,
+ hwaddr *physical, bool is_debug)
+{
+ CPUState *cs = env_cpu(env);
+ target_ulong base, ps, tmp0, tmp1, ptindex, ptoffset, cur_val, new_val, old_val;
+ uint64_t entrylo0, entrylo1, tlbehi, vppn;
+ uint64_t ptbase = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTBASE);
+ uint64_t ptwidth = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTWIDTH);
+ int index, shift;
+ bool ret = false;
+
+ /*
+ * tlb error map :
+ * TLBRET_NOMATCH : tlb fill
+ * TLBRET_INVALID : access_type = 0/2 tlb_load
+ * : access_type = 1 tlb_store
+ * TLBRET_DIRTY : tlb_modify
+ */
+ switch (tlb_error) {
+ case TLBRET_NOMATCH:
+ base = get_pte_base(env, address);
+ if (base == 0) {
+ return ret;
+ }
+ do_ldpte(env, base, address, &tmp0, &tmp1, &ps);
+ entrylo0 = tmp0;
+ entrylo1 = tmp1;
+ tlbehi = address & (TARGET_PAGE_MASK << 1);
+ vppn = FIELD_EX64(tlbehi, CSR_TLBEHI_64, VPPN);
+ if (is_debug) {
+ uint64_t tlb_ppn, tlb_entry;
+ uint8_t n;
+ n = (address >> ps) & 0x1;/* Odd or even */
+
+ tlb_entry = n ? entrylo1: entrylo1;
+ tlb_ppn = FIELD_EX64(tlb_entry, TLBENTRY_64, PPN);
+ /* Remove sw bit between bit12 -- bit PS */
+ tlb_ppn = tlb_ppn & ~(((0x1UL << (ps - 12)) -1));
+ *physical = (tlb_ppn << R_TLBENTRY_64_PPN_SHIFT) |
+ (address & MAKE_64BIT_MASK(0, ps));
+ } else {
+ index = get_random_tlb_index(env, tlbehi, ps);
+ invalidate_tlb(env, index);
+ do_fill_tlb_entry(env, vppn, entrylo0, entrylo1, index, ps);
+ }
+
+ ret = true;
+ break;
+ case TLBRET_DIRTY:
+ case TLBRET_INVALID:
+ base = get_pte_base(env, address);
+
+ /* 0:64bit, 1:128bit, 2:192bit, 3:256bit */
+ shift = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTEWIDTH);
+ shift = (shift + 1) * 3;
+ ptindex = (address >> ptbase) & ((1 << ptwidth) -1);
+ ptoffset = ptindex << shift;
+ tmp0 = base | ptoffset;
+ retry:
+ old_val = ldq_phys(cs->as, tmp0) & TARGET_PHYS_MASK;
+
+ if (old_val == 0) {
+ return ret;
+ }
+
+ new_val = old_val;
+ /* Check pte val, and do tlb modify. */
+ if ((tlb_error == TLBRET_INVALID) &&
+ (access_type == MMU_INST_FETCH )) {
+ if (!(FIELD_EX64(new_val, TLBENTRY, PRESENT))) {
+ break;
+ }
+ new_val = FIELD_DP64(new_val, TLBENTRY, V, 1);
+ } else if ((tlb_error == TLBRET_INVALID) &&
+ access_type == MMU_DATA_STORE) {
+ if (!((FIELD_EX64(new_val, TLBENTRY, PRESENT) &&
+ (FIELD_EX64(new_val, TLBENTRY, WRITE))))){
+ break;
+ }
+ new_val = FIELD_DP64(new_val, TLBENTRY, V, 1);
+ } else if (tlb_error == TLBRET_DIRTY) {
+ if (!(FIELD_EX64(new_val, TLBENTRY, PRESENT) &&
+ (FIELD_EX64(new_val, TLBENTRY, WRITE)))) {
+ break;
+ }
+ new_val = FIELD_DP64(new_val, TLBENTRY, V, 1);
+ }
+
+ if (old_val != new_val) {
+ cur_val = qatomic_cmpxchg((uint64_t *)tmp0, old_val, new_val);
+ if (cur_val == old_val) {
+ goto retry;
+ }
+ }
+
+ tmp0 = tmp0 & (~0x8);
+ entrylo0 = ldq_phys(cs->as, tmp0) & TARGET_PHYS_MASK;
+ entrylo1 = ldq_phys(cs->as, tmp0 | 0x8) & TARGET_PHYS_MASK;
+ tlbehi = address & (TARGET_PAGE_MASK << 1);
+ ps = FIELD_EX64(env->CSR_PWCL, CSR_PWCL, PTBASE);
+ vppn = FIELD_EX64(tlbehi, CSR_TLBEHI_64, VPPN);
+
+ /*
+ * srch tlb index with tlb entryhi
+ * if no match, we use get_random_tlb_index() to get random index.
+ */
+ if (!loongarch_tlb_search(env, tlbehi, &index)) {
+ index = get_random_tlb_index(env, tlbehi, ps);
+ }
+ invalidate_tlb(env, index);
+ do_fill_tlb_entry(env, vppn, entrylo0, entrylo1, index, ps);
+ ret = true;
+ break;
+ default:
+ ;
+ }
+ return ret;
+}
Add hardware page table walker (HPTW) feature for la664. Set CPUCFG2.HPTW = 1 to indicate that HPTW is implemented on this CPU. Set PWCH.HPTW_EN = 1 to enable HPTW. Signed-off-by: Song Gao <gaosong@loongson.cn> --- target/loongarch/cpu-csr.h | 3 + target/loongarch/cpu.c | 1 + target/loongarch/cpu.h | 1 + target/loongarch/cpu_helper.c | 26 +++++- target/loongarch/internals.h | 4 +- target/loongarch/tcg/tlb_helper.c | 147 +++++++++++++++++++++++++++++- 6 files changed, 176 insertions(+), 6 deletions(-)