@@ -1085,7 +1085,7 @@ void tlb_set_page_full(CPUState *cpu, int mmu_idx,
}
} else {
/* I/O or ROMD */
- iotlb = memory_region_section_get_iotlb(cpu, section) + xlat;
+ iotlb = xlat;
/*
* Writes to romd devices must go through MMIO to enable write.
* Reads to romd devices go through the ram_ptr found above,
@@ -1136,10 +1136,9 @@ void tlb_set_page_full(CPUState *cpu, int mmu_idx,
/*
* When memory region is ram, iotlb contains a TARGET_PAGE_BITS
* aligned ram_addr_t of the page base of the target RAM.
- * Otherwise, iotlb contains
- * - a physical section number in the lower TARGET_PAGE_BITS
- * - the offset within section->mr of the page base (I/O, ROMD) with the
- * TARGET_PAGE_BITS masked off.
+ * Otherwise, iotlb contains a TARGET_PAGE_BITS aligned
+ * offset within section->mr of the page base (I/O, ROMD)
+ *
* We subtract addr_page (which is page aligned and thus won't
* disturb the low bits) to give an offset which can be added to the
* (non-page-aligned) vaddr of the eventual memory access to get
@@ -1149,7 +1148,8 @@ void tlb_set_page_full(CPUState *cpu, int mmu_idx,
*/
desc->fulltlb[index] = *full;
full = &desc->fulltlb[index];
- full->xlat_section = iotlb - addr_page;
+ full->xlat = iotlb - addr_page;
+ full->section = section;
full->phys_addr = paddr_page;
/* Now calculate the new entry */
@@ -1265,14 +1265,14 @@ static inline void cpu_unaligned_access(CPUState *cpu, vaddr addr,
}
static MemoryRegionSection *
-io_prepare(hwaddr *out_offset, CPUState *cpu, hwaddr xlat,
+io_prepare(hwaddr *out_offset, CPUState *cpu, CPUTLBEntryFull *full,
MemTxAttrs attrs, vaddr addr, uintptr_t retaddr)
{
MemoryRegionSection *section;
hwaddr mr_offset;
- section = iotlb_to_section(cpu, xlat, attrs);
- mr_offset = (xlat & TARGET_PAGE_MASK) + addr;
+ section = full->section;
+ mr_offset = full->xlat + addr;
cpu->mem_io_pc = retaddr;
if (!cpu->neg.can_do_io) {
cpu_io_recompile(cpu, retaddr);
@@ -1331,7 +1331,7 @@ static bool victim_tlb_hit(CPUState *cpu, size_t mmu_idx, size_t index,
static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size,
CPUTLBEntryFull *full, uintptr_t retaddr)
{
- ram_addr_t ram_addr = mem_vaddr + full->xlat_section;
+ ram_addr_t ram_addr = mem_vaddr + full->xlat;
trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size);
@@ -1588,9 +1588,7 @@ bool tlb_plugin_lookup(CPUState *cpu, vaddr addr, int mmu_idx,
/* We must have an iotlb entry for MMIO */
if (tlb_addr & TLB_MMIO) {
- MemoryRegionSection *section =
- iotlb_to_section(cpu, full->xlat_section & ~TARGET_PAGE_MASK,
- full->attrs);
+ MemoryRegionSection *section = full->section;
data->is_io = true;
data->mr = section->mr;
} else {
@@ -1980,7 +1978,7 @@ static uint64_t do_ld_mmio_beN(CPUState *cpu, CPUTLBEntryFull *full,
tcg_debug_assert(size > 0 && size <= 8);
attrs = full->attrs;
- section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
+ section = io_prepare(&mr_offset, cpu, full, attrs, addr, ra);
mr = section->mr;
BQL_LOCK_GUARD();
@@ -2001,7 +1999,7 @@ static Int128 do_ld16_mmio_beN(CPUState *cpu, CPUTLBEntryFull *full,
tcg_debug_assert(size > 8 && size <= 16);
attrs = full->attrs;
- section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
+ section = io_prepare(&mr_offset, cpu, full, attrs, addr, ra);
mr = section->mr;
BQL_LOCK_GUARD();
@@ -2521,7 +2519,7 @@ static uint64_t do_st_mmio_leN(CPUState *cpu, CPUTLBEntryFull *full,
tcg_debug_assert(size > 0 && size <= 8);
attrs = full->attrs;
- section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
+ section = io_prepare(&mr_offset, cpu, full, attrs, addr, ra);
mr = section->mr;
BQL_LOCK_GUARD();
@@ -2541,7 +2539,7 @@ static uint64_t do_st16_mmio_leN(CPUState *cpu, CPUTLBEntryFull *full,
tcg_debug_assert(size > 8 && size <= 16);
attrs = full->attrs;
- section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
+ section = io_prepare(&mr_offset, cpu, full, attrs, addr, ra);
mr = section->mr;
BQL_LOCK_GUARD();
@@ -43,7 +43,7 @@ void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length);
* @full: the details of the tlb entry
*
* Add an entry to @cpu tlb index @mmu_idx. All of the fields of
- * @full must be filled, except for xlat_section, and constitute
+ * @full must be filled, except for xlat, and constitute
* the complete description of the translated page.
*
* This is generally called by the target tlb_fill function after
@@ -178,21 +178,6 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last);
void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr);
-#if !defined(CONFIG_USER_ONLY)
-
-/**
- * iotlb_to_section:
- * @cpu: CPU performing the access
- * @index: TCG CPU IOTLB entry
- *
- * Given a TCG CPU IOTLB entry, return the MemoryRegionSection that
- * it refers to. @index will have been initially created and returned
- * by memory_region_section_get_iotlb().
- */
-struct MemoryRegionSection *iotlb_to_section(CPUState *cpu,
- hwaddr index, MemTxAttrs attrs);
-#endif
-
/**
* get_page_addr_code_hostp()
* @env: CPUArchState
@@ -208,15 +208,17 @@ struct CPUClass {
*/
struct CPUTLBEntryFull {
/*
- * @xlat_section contains:
- * - in the lower TARGET_PAGE_BITS, a physical section number
- * - with the lower TARGET_PAGE_BITS masked off, an offset which
- * must be added to the virtual address to obtain:
+ * @xlat contains:
+ * - a TARGET_PAGE_BITS aligned offset which must be added to
+ * the virtual address to obtain:
* + the ram_addr_t of the target RAM (if the physical section
* number is PHYS_SECTION_NOTDIRTY or PHYS_SECTION_ROM)
* + the offset within the target MemoryRegion (otherwise)
*/
- hwaddr xlat_section;
+ hwaddr xlat;
+
+ /* @section contains physical section. */
+ MemoryRegionSection *section;
/*
* @phys_addr contains the physical address in the address space
@@ -2668,23 +2668,6 @@ static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
return phys_section_add(map, §ion);
}
-MemoryRegionSection *iotlb_to_section(CPUState *cpu,
- hwaddr index, MemTxAttrs attrs)
-{
- int asidx = cpu_asidx_from_attrs(cpu, attrs);
- CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
- AddressSpaceDispatch *d = cpuas->memory_dispatch;
- int section_index = index & ~TARGET_PAGE_MASK;
- MemoryRegionSection *ret;
-
- assert(section_index < d->map.sections_nb);
- ret = d->map.sections + section_index;
- assert(ret->mr);
- assert(ret->mr->ops);
-
- return ret;
-}
-
static void io_mem_init(void)
{
memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
To fix the bug of iotlb_to_section(), store complete MemoryRegionSection pointer in CPUTLBEntryFull to replace the section_index in xlat_section. Rename 'xlat_section' to 'xlat' as we remove last 12 bits section_index inside. Remove iotlb_to_section() as we directly use section pointer in the CPUTLBEntryFull (full->section). iotlb_to_section() bug description: 'CPUTLBEntryFull.xlat_section' stores section_index in last 12 bits to find the correct section when CPU access the IO region over the IOTLB. However, section_index is only unique inside single AddressSpace. If address space translation is over IOMMUMemoryRegion, it could return section from other AddressSpace. 'iotlb_to_section()' API only finds the sections from CPU's AddressSpace so that it couldn't find section in other AddressSpace. Thus, using 'iotlb_to_section()' API will find the wrong section and QEMU will have wrong load/store access. This bug occurs only when (1) IOMMUMemoryRegion is in the path of CPU access. (2) IOMMUMemoryRegion returns different target_as and the section is in the IO region. Common IOMMU devices don't have this issue since they are only in the path of DMA access. Currently, the bug only occurs when ARM MPC device (hw/misc/tz-mpc.c) returns 'blocked_io_as' to emulate blocked access handling. Upcoming RISC-V wgChecker device is also affected by this bug. Signed-off-by: Jim Shu <jim.shu@sifive.com> --- accel/tcg/cputlb.c | 32 +++++++++++++++----------------- include/exec/cputlb.h | 2 +- include/exec/exec-all.h | 15 --------------- include/hw/core/cpu.h | 12 +++++++----- system/physmem.c | 17 ----------------- 5 files changed, 23 insertions(+), 55 deletions(-)