@@ -214,10 +214,10 @@ static u64 gen8_pte_encode(dma_addr_t addr,
return pte;
}
-static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
- const enum i915_cache_level level)
+static u64 gen8_pde_encode(const dma_addr_t addr,
+ const enum i915_cache_level level)
{
- gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
+ u64 pde = _PAGE_PRESENT | _PAGE_RW;
pde |= addr;
if (level != I915_CACHE_NONE)
pde |= PPAT_CACHED_PDE;
@@ -226,9 +226,6 @@ static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
return pde;
}
-#define gen8_pdpe_encode gen8_pde_encode
-#define gen8_pml4e_encode gen8_pde_encode
-
static u64 snb_pte_encode(dma_addr_t addr,
enum i915_cache_level level,
u32 flags)
@@ -733,24 +730,36 @@ static void free_pd(struct i915_address_space *vm,
kfree(pd);
}
-static void init_pd_with_page(struct i915_address_space *vm,
- struct i915_page_directory * const pd,
- struct i915_page_table *pt)
-{
- fill_px(vm, pd, gen8_pde_encode(px_dma(pt), I915_CACHE_LLC));
- memset_p(pd->entry, pt, 512);
+#define init_pd(vm, pd, to) { \
+ GEM_DEBUG_BUG_ON(!pd_has_phys_page(pd)); \
+ fill_px((vm), (pd), gen8_pde_encode(px_dma(to), I915_CACHE_LLC)); \
+ memset_p((pd)->entry, (to), 512); \
}
-static void init_pd(struct i915_address_space *vm,
- struct i915_page_directory * const pd,
- struct i915_page_directory * const to)
+static void __set_pd_entry(struct i915_page_directory * const pd,
+ const unsigned short pde,
+ const u64 encoded_entry)
{
- GEM_DEBUG_BUG_ON(!pd_has_phys_page(pd));
+ u64 *vaddr;
- fill_px(vm, pd, gen8_pdpe_encode(px_dma(to), I915_CACHE_LLC));
- memset_p(pd->entry, to, 512);
+ vaddr = kmap_atomic(pd->base.page);
+ vaddr[pde] = encoded_entry;
+ kunmap_atomic(vaddr);
}
+#define set_pd_entry(pd, pde, to) ({ \
+ (pd)->entry[(pde)] = (to); \
+ __set_pd_entry((pd), (pde), \
+ gen8_pde_encode(px_dma(to), I915_CACHE_LLC)); \
+})
+
+#define set_pdp_entry(pdp, pdpe, to) ({ \
+ (pdp)->entry[(pdpe)] = (to); \
+ if (pd_has_phys_page(pdp)) \
+ __set_pd_entry((pdp), (pdpe), \
+ gen8_pde_encode(px_dma(to), I915_CACHE_LLC));\
+})
+
/*
* PDE TLBs are a pain to invalidate on GEN8+. When we modify
* the page table structures, we mark them dirty so that
@@ -780,18 +789,6 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
return !atomic_sub_return(num_entries, &pt->used);
}
-static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
- struct i915_page_directory *pd,
- struct i915_page_table *pt,
- unsigned int pde)
-{
- gen8_pde_t *vaddr;
-
- vaddr = kmap_atomic_px(pd);
- vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
- kunmap_atomic(vaddr);
-}
-
static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
struct i915_page_directory *pd,
u64 start, u64 length)
@@ -809,8 +806,7 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
spin_lock(&pd->lock);
if (!atomic_read(&pt->used)) {
- gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
- pd->entry[pde] = vm->scratch_pt;
+ set_pd_entry(pd, pde, vm->scratch_pt);
GEM_BUG_ON(!atomic_read(&pd->used));
atomic_dec(&pd->used);
@@ -824,20 +820,6 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
return !atomic_read(&pd->used);
}
-static void gen8_ppgtt_set_pdpe(struct i915_page_directory *pdp,
- struct i915_page_directory *pd,
- unsigned int pdpe)
-{
- gen8_ppgtt_pdpe_t *vaddr;
-
- if (!pd_has_phys_page(pdp))
- return;
-
- vaddr = kmap_atomic_px(pdp);
- vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
- kunmap_atomic(vaddr);
-}
-
/* Removes entries from a single page dir pointer, releasing it if it's empty.
* Caller can use the return value to update higher-level entries
*/
@@ -858,8 +840,7 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
spin_lock(&pdp->lock);
if (!atomic_read(&pd->used)) {
- gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe);
- pdp->entry[pdpe] = vm->scratch_pd;
+ set_pdp_entry(pdp, pdpe, vm->scratch_pd);
GEM_BUG_ON(!atomic_read(&pdp->used));
atomic_dec(&pdp->used);
@@ -879,17 +860,6 @@ static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
gen8_ppgtt_clear_pdp(vm, i915_vm_to_ppgtt(vm)->pd, start, length);
}
-static void gen8_ppgtt_set_pml4e(struct i915_page_directory *pml4,
- struct i915_page_directory *pdp,
- unsigned int pml4e)
-{
- gen8_ppgtt_pml4e_t *vaddr;
-
- vaddr = kmap_atomic_px(pml4);
- vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
- kunmap_atomic(vaddr);
-}
-
/* Removes entries from a single pml4.
* This is the top-level structure in 4-level page tables used on gen8+.
* Empty entries are always scratch pml4e.
@@ -913,8 +883,7 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
spin_lock(&pml4->lock);
if (!atomic_read(&pdp->used)) {
- gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
- pml4->entry[pml4e] = vm->scratch_pdp;
+ set_pd_entry(pml4, pml4e, vm->scratch_pdp);
free = true;
}
spin_unlock(&pml4->lock);
@@ -1231,7 +1200,7 @@ static int gen8_init_scratch(struct i915_address_space *vm)
}
gen8_initialize_pt(vm, vm->scratch_pt);
- init_pd_with_page(vm, vm->scratch_pd, vm->scratch_pt);
+ init_pd(vm, vm->scratch_pd, vm->scratch_pt);
if (i915_vm_is_4lvl(vm))
init_pd(vm, vm->scratch_pdp, vm->scratch_pd);
@@ -1367,7 +1336,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt);
if (old == vm->scratch_pt) {
- gen8_ppgtt_set_pde(vm, pd, pt, pde);
+ set_pd_entry(pd, pde, pt);
atomic_inc(&pd->used);
} else {
free_pt(vm, pt);
@@ -1408,11 +1377,11 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
if (IS_ERR(pd))
goto unwind;
- init_pd_with_page(vm, pd, vm->scratch_pt);
+ init_pd(vm, pd, vm->scratch_pt);
old = cmpxchg(&pdp->entry[pdpe], vm->scratch_pd, pd);
if (old == vm->scratch_pd) {
- gen8_ppgtt_set_pdpe(pdp, pd, pdpe);
+ set_pdp_entry(pdp, pdpe, pd);
atomic_inc(&pdp->used);
} else {
free_pd(vm, pd);
@@ -1438,7 +1407,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
unwind_pd:
spin_lock(&pdp->lock);
if (atomic_dec_and_test(&pd->used)) {
- gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe);
+ set_pdp_entry(pdp, pdpe, vm->scratch_pd);
GEM_BUG_ON(!atomic_read(&pdp->used));
atomic_dec(&pdp->used);
free_pd(vm, pd);
@@ -1481,7 +1450,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
old = cmpxchg(&pml4->entry[pml4e], vm->scratch_pdp, pdp);
if (old == vm->scratch_pdp) {
- gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
+ set_pd_entry(pml4, pml4e, pdp);
} else {
free_pd(vm, pdp);
pdp = old;
@@ -1506,7 +1475,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
unwind_pdp:
spin_lock(&pml4->lock);
if (atomic_dec_and_test(&pdp->used)) {
- gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
+ set_pd_entry(pml4, pml4e, vm->scratch_pdp);
free_pd(vm, pdp);
}
spin_unlock(&pml4->lock);
@@ -1529,8 +1498,8 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
if (IS_ERR(pd))
goto unwind;
- init_pd_with_page(vm, pd, vm->scratch_pt);
- gen8_ppgtt_set_pdpe(pdp, pd, pdpe);
+ init_pd(vm, pd, vm->scratch_pt);
+ set_pdp_entry(pdp, pdpe, pd);
atomic_inc(&pdp->used);
}
@@ -1542,7 +1511,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
unwind:
start -= from;
gen8_for_each_pdpe(pd, pdp, from, start, pdpe) {
- gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe);
+ set_pdp_entry(pdp, pdpe, vm->scratch_pd);
free_pd(vm, pd);
}
atomic_set(&pdp->used, 0);
@@ -67,9 +67,6 @@ struct i915_vma;
typedef u32 gen6_pte_t;
typedef u64 gen8_pte_t;
-typedef u64 gen8_pde_t;
-typedef u64 gen8_ppgtt_pdpe_t;
-typedef u64 gen8_ppgtt_pml4e_t;
#define ggtt_total_entries(ggtt) ((ggtt)->vm.total >> PAGE_SHIFT)
For all page directory entries, the pde encoding is identical. Don't compilicate call sites with different versions of doing the same thing. Only wart that remains is a 4 entry gen8/bsw pdp, for which we need to check the backing phys page. Signed-off-by: Mika Kuoppala <mika.kuoppala@linux.intel.com> --- drivers/gpu/drm/i915/i915_gem_gtt.c | 111 ++++++++++------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 3 - 2 files changed, 40 insertions(+), 74 deletions(-)