From patchwork Tue Jun 11 17:27:23 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mika Kuoppala X-Patchwork-Id: 10987937 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 0C27C14C0 for ; Tue, 11 Jun 2019 17:27:41 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id E0DD526E1A for ; Tue, 11 Jun 2019 17:27:40 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id D52E5287B1; Tue, 11 Jun 2019 17:27:40 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 9624926E1A for ; Tue, 11 Jun 2019 17:27:40 +0000 (UTC) Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id 046C7891F2; Tue, 11 Jun 2019 17:27:37 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by gabe.freedesktop.org (Postfix) with ESMTPS id C8E8D891D2 for ; Tue, 11 Jun 2019 17:27:35 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga004.jf.intel.com ([10.7.209.38]) by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Jun 2019 10:27:35 -0700 X-ExtLoop1: 1 Received: from rosetta.fi.intel.com ([10.237.72.186]) by orsmga004.jf.intel.com with ESMTP; 11 Jun 2019 10:27:34 -0700 Received: by rosetta.fi.intel.com (Postfix, from userid 1000) id 27B8B84016A; Tue, 11 Jun 2019 20:27:33 +0300 (EEST) From: Mika Kuoppala To: intel-gfx@lists.freedesktop.org Date: Tue, 11 Jun 2019 20:27:23 +0300 Message-Id: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> X-Mailer: git-send-email 2.17.1 Subject: [Intel-gfx] [PATCH 1/9] drm/i915/gtt: No need to zero the table for page dirs X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" X-Virus-Scanned: ClamAV using ClamSMTP We set them to scratch right after allocation so prevent useless zeroing before. v2: atomic_t Cc: Chris Wilson Signed-off-by: Mika Kuoppala Reviewed-by: Chris Wilson --- drivers/gpu/drm/i915/i915_gem_gtt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index e70675bfb51d..07f86d474fa2 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -687,7 +687,7 @@ static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) { struct i915_page_directory *pd; - pd = kzalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL); + pd = kmalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL); if (unlikely(!pd)) return ERR_PTR(-ENOMEM); @@ -747,7 +747,7 @@ alloc_pdp(struct i915_address_space *vm) GEM_BUG_ON(!i915_vm_is_4lvl(vm)); - pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); + pdp = kmalloc(sizeof(*pdp), GFP_KERNEL); if (!pdp) return ERR_PTR(-ENOMEM); From patchwork Tue Jun 11 17:27:24 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mika Kuoppala X-Patchwork-Id: 10987945 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id E384B14B6 for ; Tue, 11 Jun 2019 17:27:45 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C0AB626E1A for ; Tue, 11 Jun 2019 17:27:45 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id B4CAA287B1; Tue, 11 Jun 2019 17:27:45 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id C301A26E1A for ; Tue, 11 Jun 2019 17:27:43 +0000 (UTC) Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id AAB8889218; Tue, 11 Jun 2019 17:27:38 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by gabe.freedesktop.org (Postfix) with ESMTPS id 019C0891D4 for ; Tue, 11 Jun 2019 17:27:36 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga004.jf.intel.com ([10.7.209.38]) by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Jun 2019 10:27:36 -0700 X-ExtLoop1: 1 Received: from rosetta.fi.intel.com ([10.237.72.186]) by orsmga004.jf.intel.com with ESMTP; 11 Jun 2019 10:27:34 -0700 Received: by rosetta.fi.intel.com (Postfix, from userid 1000) id 2A3A08400FD; Tue, 11 Jun 2019 20:27:33 +0300 (EEST) From: Mika Kuoppala To: intel-gfx@lists.freedesktop.org Date: Tue, 11 Jun 2019 20:27:24 +0300 Message-Id: <20190611172731.19174-2-mika.kuoppala@linux.intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> References: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> Subject: [Intel-gfx] [PATCH 2/9] drm/i915/gtt: Use a common type for page directories X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" X-Virus-Scanned: ClamAV using ClamSMTP All page directories are identical in function, only the position in the hierarchy differ. Use same base type for directory functionality. Cc: Chris Wilson Cc: Joonas Lahtinen Cc: Matthew Auld Cc: Abdiel Janulgue Signed-off-by: Mika Kuoppala --- drivers/gpu/drm/i915/gem/i915_gem_context.c | 2 +- drivers/gpu/drm/i915/gt/intel_lrc_reg.h | 2 +- drivers/gpu/drm/i915/gt/intel_ringbuffer.c | 2 +- drivers/gpu/drm/i915/gvt/scheduler.c | 30 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 349 ++++++++++---------- drivers/gpu/drm/i915/i915_gem_gtt.h | 64 ++-- 6 files changed, 234 insertions(+), 215 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index c86ca9f21532..dbab0ab1cef1 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -1038,7 +1038,7 @@ static int emit_ppgtt_update(struct i915_request *rq, void *data) if (i915_vm_is_4lvl(vm)) { struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - const dma_addr_t pd_daddr = px_dma(&ppgtt->pml4); + const dma_addr_t pd_daddr = px_dma(ppgtt->pd); cs = intel_ring_begin(rq, 6); if (IS_ERR(cs)) diff --git a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h index 5ef932d810a7..6bf34738b4e5 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h +++ b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h @@ -55,7 +55,7 @@ #define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \ u32 *reg_state__ = (reg_state); \ - const u64 addr__ = px_dma(&ppgtt->pml4); \ + const u64 addr__ = px_dma(ppgtt->pd); \ (reg_state__)[CTX_PDP0_UDW + 1] = upper_32_bits(addr__); \ (reg_state__)[CTX_PDP0_LDW + 1] = lower_32_bits(addr__); \ } while (0) diff --git a/drivers/gpu/drm/i915/gt/intel_ringbuffer.c b/drivers/gpu/drm/i915/gt/intel_ringbuffer.c index c834d016c965..3b857994943c 100644 --- a/drivers/gpu/drm/i915/gt/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/gt/intel_ringbuffer.c @@ -1523,7 +1523,7 @@ static int load_pd_dir(struct i915_request *rq, const struct i915_ppgtt *ppgtt) *cs++ = MI_LOAD_REGISTER_IMM(1); *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); - *cs++ = ppgtt->pd.base.ggtt_offset << 10; + *cs++ = ppgtt->pd->base.ggtt_offset << 10; intel_ring_advance(rq, cs); diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index e301efb18d45..f1e1261ac3db 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -375,11 +375,13 @@ static int set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, return -EINVAL; if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { - px_dma(&ppgtt->pml4) = mm->ppgtt_mm.shadow_pdps[0]; + px_dma(ppgtt->pd) = mm->ppgtt_mm.shadow_pdps[0]; } else { for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) { - px_dma(ppgtt->pdp.page_directory[i]) = - mm->ppgtt_mm.shadow_pdps[i]; + struct i915_page_directory * const pd = + i915_pd_entry(ppgtt->pd, i); + + px_dma(pd) = mm->ppgtt_mm.shadow_pdps[i]; } } @@ -1128,11 +1130,14 @@ i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s, int i; if (i915_vm_is_4lvl(&ppgtt->vm)) { - px_dma(&ppgtt->pml4) = s->i915_context_pml4; + px_dma(ppgtt->pd) = s->i915_context_pml4; } else { - for (i = 0; i < GEN8_3LVL_PDPES; i++) - px_dma(ppgtt->pdp.page_directory[i]) = - s->i915_context_pdps[i]; + for (i = 0; i < GEN8_3LVL_PDPES; i++) { + struct i915_page_directory * const pd = + i915_pd_entry(ppgtt->pd, i); + + px_dma(pd) = s->i915_context_pdps[i]; + } } } @@ -1186,11 +1191,14 @@ i915_context_ppgtt_root_save(struct intel_vgpu_submission *s, int i; if (i915_vm_is_4lvl(&ppgtt->vm)) { - s->i915_context_pml4 = px_dma(&ppgtt->pml4); + s->i915_context_pml4 = px_dma(ppgtt->pd); } else { - for (i = 0; i < GEN8_3LVL_PDPES; i++) - s->i915_context_pdps[i] = - px_dma(ppgtt->pdp.page_directory[i]); + for (i = 0; i < GEN8_3LVL_PDPES; i++) { + struct i915_page_directory * const pd = + i915_pd_entry(ppgtt->pd, i); + + s->i915_context_pdps[i] = px_dma(pd); + } } } diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 07f86d474fa2..9a1f956a817a 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -661,7 +661,8 @@ static struct i915_page_table *alloc_pt(struct i915_address_space *vm) return ERR_PTR(-ENOMEM); } - atomic_set(&pt->used_ptes, 0); + atomic_set(&pt->used, 0); + return pt; } @@ -683,11 +684,28 @@ static void gen6_initialize_pt(struct i915_address_space *vm, fill32_px(vm, pt, vm->scratch_pte); } +static struct i915_page_directory *__alloc_pd(const unsigned int entries) +{ + struct i915_page_directory *pd; + + pd = kmalloc(sizeof(*pd) + + entries * sizeof(pd->entry[0]), I915_GFP_ALLOW_FAIL); + + if (unlikely(!pd)) + return NULL; + + memset(&pd->base, 0, sizeof(pd->base)); + atomic_set(&pd->used, 0); + spin_lock_init(&pd->lock); + + return pd; +} + static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) { struct i915_page_directory *pd; - pd = kmalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL); + pd = __alloc_pd(512); if (unlikely(!pd)) return ERR_PTR(-ENOMEM); @@ -696,8 +714,6 @@ static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) return ERR_PTR(-ENOMEM); } - atomic_set(&pd->used_pdes, 0); - spin_lock_init(&pd->lock); return pd; } @@ -713,88 +729,56 @@ static void gen8_initialize_pd(struct i915_address_space *vm, { fill_px(vm, pd, gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC)); - memset_p((void **)pd->page_table, vm->scratch_pt, I915_PDES); + memset_p(pd->entry, vm->scratch_pt, I915_PDES); } -static int __pdp_init(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp) +static struct i915_page_directory *alloc_pdp(struct i915_address_space *vm) { - const unsigned int pdpes = i915_pdpes_per_pdp(vm); + struct i915_page_directory *pdp; - pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory), - I915_GFP_ALLOW_FAIL); - if (unlikely(!pdp->page_directory)) - return -ENOMEM; - - memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes); - - atomic_set(&pdp->used_pdpes, 0); - spin_lock_init(&pdp->lock); - return 0; -} - -static void __pdp_fini(struct i915_page_directory_pointer *pdp) -{ - kfree(pdp->page_directory); - pdp->page_directory = NULL; -} - -static struct i915_page_directory_pointer * -alloc_pdp(struct i915_address_space *vm) -{ - struct i915_page_directory_pointer *pdp; - int ret = -ENOMEM; - - GEM_BUG_ON(!i915_vm_is_4lvl(vm)); - - pdp = kmalloc(sizeof(*pdp), GFP_KERNEL); + pdp = __alloc_pd(i915_pdpes_per_pdp(vm)); if (!pdp) return ERR_PTR(-ENOMEM); - ret = __pdp_init(vm, pdp); - if (ret) - goto fail_bitmap; - - ret = setup_px(vm, pdp); - if (ret) - goto fail_page_m; + if (i915_vm_is_4lvl(vm)) { + if (unlikely(setup_px(vm, pdp))) { + kfree(pdp); + return ERR_PTR(-ENOMEM); + } + } return pdp; - -fail_page_m: - __pdp_fini(pdp); -fail_bitmap: - kfree(pdp); - - return ERR_PTR(ret); } static void free_pdp(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp) + struct i915_page_directory *pdp) { - __pdp_fini(pdp); - - if (!i915_vm_is_4lvl(vm)) - return; + if (i915_vm_is_4lvl(vm)) + cleanup_px(vm, pdp); - cleanup_px(vm, pdp); kfree(pdp); } -static void gen8_initialize_pdp(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp) +static void gen8_initialize_4lvl_pdp(struct i915_address_space *vm, + struct i915_page_directory *pdp) { fill_px(vm, pdp, gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC)); + memset_p(pdp->entry, vm->scratch_pd, 512); +} + +static void gen8_initialize_3lvl_pdp(struct i915_address_space *vm, + struct i915_page_directory *pdp) +{ + memset_p(pdp->entry, vm->scratch_pd, GEN8_3LVL_PDPES); } static void gen8_initialize_pml4(struct i915_address_space *vm, - struct i915_pml4 *pml4) + struct i915_page_directory *pml4) { fill_px(vm, pml4, gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC)); - memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4); - spin_lock_init(&pml4->lock); + memset_p(pml4->entry, vm->scratch_pdp, GEN8_PML4ES_PER_PML4); } /* @@ -822,8 +806,8 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm, memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries); kunmap_atomic(vaddr); - GEM_BUG_ON(num_entries > atomic_read(&pt->used_ptes)); - return !atomic_sub_return(num_entries, &pt->used_ptes); + GEM_BUG_ON(num_entries > atomic_read(&pt->used)); + return !atomic_sub_return(num_entries, &pt->used); } static void gen8_ppgtt_set_pde(struct i915_address_space *vm, @@ -854,12 +838,12 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, continue; spin_lock(&pd->lock); - if (!atomic_read(&pt->used_ptes)) { + if (!atomic_read(&pt->used)) { gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde); - pd->page_table[pde] = vm->scratch_pt; + pd->entry[pde] = vm->scratch_pt; - GEM_BUG_ON(!atomic_read(&pd->used_pdes)); - atomic_dec(&pd->used_pdes); + GEM_BUG_ON(!atomic_read(&pd->used)); + atomic_dec(&pd->used); free = true; } spin_unlock(&pd->lock); @@ -867,11 +851,11 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, free_pt(vm, pt); } - return !atomic_read(&pd->used_pdes); + return !atomic_read(&pd->used); } static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp, + struct i915_page_directory *pdp, struct i915_page_directory *pd, unsigned int pdpe) { @@ -889,7 +873,7 @@ static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm, * Caller can use the return value to update higher-level entries */ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp, + struct i915_page_directory * const pdp, u64 start, u64 length) { struct i915_page_directory *pd; @@ -904,12 +888,12 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, continue; spin_lock(&pdp->lock); - if (!atomic_read(&pd->used_pdes)) { + if (!atomic_read(&pd->used)) { gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); - pdp->page_directory[pdpe] = vm->scratch_pd; + pdp->entry[pdpe] = vm->scratch_pd; - GEM_BUG_ON(!atomic_read(&pdp->used_pdpes)); - atomic_dec(&pdp->used_pdpes); + GEM_BUG_ON(!atomic_read(&pdp->used)); + atomic_dec(&pdp->used); free = true; } spin_unlock(&pdp->lock); @@ -917,17 +901,17 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, free_pd(vm, pd); } - return !atomic_read(&pdp->used_pdpes); + return !atomic_read(&pdp->used); } static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm, u64 start, u64 length) { - gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length); + gen8_ppgtt_clear_pdp(vm, i915_vm_to_ppgtt(vm)->pd, start, length); } -static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4, - struct i915_page_directory_pointer *pdp, +static void gen8_ppgtt_set_pml4e(struct i915_page_directory *pml4, + struct i915_page_directory *pdp, unsigned int pml4e) { gen8_ppgtt_pml4e_t *vaddr; @@ -945,8 +929,8 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm, u64 start, u64 length) { struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - struct i915_pml4 *pml4 = &ppgtt->pml4; - struct i915_page_directory_pointer *pdp; + struct i915_page_directory * const pml4 = ppgtt->pd; + struct i915_page_directory *pdp; unsigned int pml4e; GEM_BUG_ON(!i915_vm_is_4lvl(vm)); @@ -959,9 +943,9 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm, continue; spin_lock(&pml4->lock); - if (!atomic_read(&pdp->used_pdpes)) { + if (!atomic_read(&pdp->used)) { gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); - pml4->pdps[pml4e] = vm->scratch_pdp; + pml4->entry[pml4e] = vm->scratch_pdp; free = true; } spin_unlock(&pml4->lock); @@ -998,7 +982,7 @@ static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start) static __always_inline bool gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt, - struct i915_page_directory_pointer *pdp, + struct i915_page_directory *pdp, struct sgt_dma *iter, struct gen8_insert_pte *idx, enum i915_cache_level cache_level, @@ -1010,8 +994,8 @@ gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt, bool ret; GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm)); - pd = pdp->page_directory[idx->pdpe]; - vaddr = kmap_atomic_px(pd->page_table[idx->pde]); + pd = i915_pd_entry(pdp, idx->pdpe); + vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde)); do { vaddr[idx->pte] = pte_encode | iter->dma; @@ -1041,11 +1025,11 @@ gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt, } GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm)); - pd = pdp->page_directory[idx->pdpe]; + pd = pdp->entry[idx->pdpe]; } kunmap_atomic(vaddr); - vaddr = kmap_atomic_px(pd->page_table[idx->pde]); + vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde)); } } while (1); kunmap_atomic(vaddr); @@ -1062,14 +1046,14 @@ static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm, struct sgt_dma iter = sgt_dma(vma); struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start); - gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx, + gen8_ppgtt_insert_pte_entries(ppgtt, ppgtt->pd, &iter, &idx, cache_level, flags); vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; } static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, - struct i915_page_directory_pointer **pdps, + struct i915_page_directory *pml4, struct sgt_dma *iter, enum i915_cache_level cache_level, u32 flags) @@ -1080,8 +1064,9 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, do { struct gen8_insert_pte idx = gen8_insert_pte(start); - struct i915_page_directory_pointer *pdp = pdps[idx.pml4e]; - struct i915_page_directory *pd = pdp->page_directory[idx.pdpe]; + struct i915_page_directory *pdp = + i915_pdp_entry(pml4, idx.pml4e); + struct i915_page_directory *pd = i915_pd_entry(pdp, idx.pdpe); unsigned int page_size; bool maybe_64K = false; gen8_pte_t encode = pte_encode; @@ -1099,7 +1084,7 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, vaddr = kmap_atomic_px(pd); } else { - struct i915_page_table *pt = pd->page_table[idx.pde]; + struct i915_page_table *pt = i915_pt_entry(pd, idx.pde); index = idx.pte; max = GEN8_PTES; @@ -1174,7 +1159,8 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, u16 i; encode = vma->vm->scratch_pte; - vaddr = kmap_atomic_px(pd->page_table[idx.pde]); + vaddr = kmap_atomic_px(i915_pt_entry(pd, + idx.pde)); for (i = 1; i < index; i += 16) memset64(vaddr + i, encode, 15); @@ -1194,15 +1180,16 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm, { struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); struct sgt_dma iter = sgt_dma(vma); - struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps; + struct i915_page_directory * const pml4 = ppgtt->pd; if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) { - gen8_ppgtt_insert_huge_entries(vma, pdps, &iter, cache_level, + gen8_ppgtt_insert_huge_entries(vma, pml4, &iter, cache_level, flags); } else { struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start); - while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++], + while (gen8_ppgtt_insert_pte_entries(ppgtt, + i915_pdp_entry(pml4, idx.pml4e++), &iter, &idx, cache_level, flags)) GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4); @@ -1217,8 +1204,8 @@ static void gen8_free_page_tables(struct i915_address_space *vm, int i; for (i = 0; i < I915_PDES; i++) { - if (pd->page_table[i] != vm->scratch_pt) - free_pt(vm, pd->page_table[i]); + if (pd->entry[i] != vm->scratch_pt) + free_pt(vm, pd->entry[i]); } } @@ -1277,7 +1264,7 @@ static int gen8_init_scratch(struct i915_address_space *vm) gen8_initialize_pt(vm, vm->scratch_pt); gen8_initialize_pd(vm, vm->scratch_pd); if (i915_vm_is_4lvl(vm)) - gen8_initialize_pdp(vm, vm->scratch_pdp); + gen8_initialize_4lvl_pdp(vm, vm->scratch_pdp); return 0; @@ -1299,7 +1286,7 @@ static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create) int i; if (i915_vm_is_4lvl(vm)) { - const u64 daddr = px_dma(&ppgtt->pml4); + const u64 daddr = px_dma(ppgtt->pd); I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr)); I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr)); @@ -1336,17 +1323,17 @@ static void gen8_free_scratch(struct i915_address_space *vm) } static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp) + struct i915_page_directory *pdp) { const unsigned int pdpes = i915_pdpes_per_pdp(vm); int i; for (i = 0; i < pdpes; i++) { - if (pdp->page_directory[i] == vm->scratch_pd) + if (pdp->entry[i] == vm->scratch_pd) continue; - gen8_free_page_tables(vm, pdp->page_directory[i]); - free_pd(vm, pdp->page_directory[i]); + gen8_free_page_tables(vm, pdp->entry[i]); + free_pd(vm, pdp->entry[i]); } free_pdp(vm, pdp); @@ -1354,16 +1341,19 @@ static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm, static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt) { + struct i915_page_directory * const pml4 = ppgtt->pd; int i; for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) { - if (ppgtt->pml4.pdps[i] == ppgtt->vm.scratch_pdp) + struct i915_page_directory *pdp = i915_pdp_entry(pml4, i); + + if (pdp == ppgtt->vm.scratch_pdp) continue; - gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pml4.pdps[i]); + gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp); } - cleanup_px(&ppgtt->vm, &ppgtt->pml4); + cleanup_px(&ppgtt->vm, pml4); } static void gen8_ppgtt_cleanup(struct i915_address_space *vm) @@ -1377,7 +1367,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm) if (i915_vm_is_4lvl(vm)) gen8_ppgtt_cleanup_4lvl(ppgtt); else - gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, &ppgtt->pdp); + gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pd); gen8_free_scratch(vm); } @@ -1406,10 +1396,10 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, if (count < GEN8_PTES || intel_vgpu_active(vm->i915)) gen8_initialize_pt(vm, pt); - old = cmpxchg(&pd->page_table[pde], vm->scratch_pt, pt); + old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt); if (old == vm->scratch_pt) { gen8_ppgtt_set_pde(vm, pd, pt, pde); - atomic_inc(&pd->used_pdes); + atomic_inc(&pd->used); } else { free_pt(vm, pt); pt = old; @@ -1418,7 +1408,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, spin_lock(&pd->lock); } - atomic_add(count, &pt->used_ptes); + atomic_add(count, &pt->used); } spin_unlock(&pd->lock); @@ -1430,7 +1420,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, } static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, - struct i915_page_directory_pointer *pdp, + struct i915_page_directory *pdp, u64 start, u64 length) { struct i915_page_directory *pd; @@ -1451,11 +1441,10 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, gen8_initialize_pd(vm, pd); - old = cmpxchg(&pdp->page_directory[pdpe], - vm->scratch_pd, pd); + old = cmpxchg(&pdp->entry[pdpe], vm->scratch_pd, pd); if (old == vm->scratch_pd) { gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); - atomic_inc(&pdp->used_pdpes); + atomic_inc(&pdp->used); } else { free_pd(vm, pd); pd = old; @@ -1463,7 +1452,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, spin_lock(&pdp->lock); } - atomic_inc(&pd->used_pdes); + atomic_inc(&pd->used); spin_unlock(&pdp->lock); ret = gen8_ppgtt_alloc_pd(vm, pd, start, length); @@ -1471,7 +1460,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, goto unwind_pd; spin_lock(&pdp->lock); - atomic_dec(&pd->used_pdes); + atomic_dec(&pd->used); } spin_unlock(&pdp->lock); @@ -1479,10 +1468,10 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, unwind_pd: spin_lock(&pdp->lock); - if (atomic_dec_and_test(&pd->used_pdes)) { + if (atomic_dec_and_test(&pd->used)) { gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); - GEM_BUG_ON(!atomic_read(&pdp->used_pdpes)); - atomic_dec(&pdp->used_pdpes); + GEM_BUG_ON(!atomic_read(&pdp->used)); + atomic_dec(&pdp->used); free_pd(vm, pd); } spin_unlock(&pdp->lock); @@ -1495,23 +1484,24 @@ static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm, u64 start, u64 length) { return gen8_ppgtt_alloc_pdp(vm, - &i915_vm_to_ppgtt(vm)->pdp, start, length); + i915_vm_to_ppgtt(vm)->pd, start, length); } static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, u64 start, u64 length) { struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - struct i915_pml4 *pml4 = &ppgtt->pml4; - struct i915_page_directory_pointer *pdp; + struct i915_page_directory * const pml4 = ppgtt->pd; + struct i915_page_directory *pdp; u64 from = start; u32 pml4e; int ret; spin_lock(&pml4->lock); gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { + if (pdp == vm->scratch_pdp) { - struct i915_page_directory_pointer *old; + struct i915_page_directory *old; spin_unlock(&pml4->lock); @@ -1519,9 +1509,9 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, if (IS_ERR(pdp)) goto unwind; - gen8_initialize_pdp(vm, pdp); + gen8_initialize_4lvl_pdp(vm, pdp); - old = cmpxchg(&pml4->pdps[pml4e], vm->scratch_pdp, pdp); + old = cmpxchg(&pml4->entry[pml4e], vm->scratch_pdp, pdp); if (old == vm->scratch_pdp) { gen8_ppgtt_set_pml4e(pml4, pdp, pml4e); } else { @@ -1531,7 +1521,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, spin_lock(&pml4->lock); } - atomic_inc(&pdp->used_pdpes); + atomic_inc(&pdp->used); spin_unlock(&pml4->lock); ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length); @@ -1539,7 +1529,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, goto unwind_pdp; spin_lock(&pml4->lock); - atomic_dec(&pdp->used_pdpes); + atomic_dec(&pdp->used); } spin_unlock(&pml4->lock); @@ -1547,7 +1537,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, unwind_pdp: spin_lock(&pml4->lock); - if (atomic_dec_and_test(&pdp->used_pdpes)) { + if (atomic_dec_and_test(&pdp->used)) { gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); free_pdp(vm, pdp); } @@ -1560,7 +1550,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) { struct i915_address_space *vm = &ppgtt->vm; - struct i915_page_directory_pointer *pdp = &ppgtt->pdp; + struct i915_page_directory *pdp = ppgtt->pd; struct i915_page_directory *pd; u64 start = 0, length = ppgtt->vm.total; u64 from = start; @@ -1573,10 +1563,12 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) gen8_initialize_pd(vm, pd); gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); - atomic_inc(&pdp->used_pdpes); + + atomic_inc(&pdp->used); } - atomic_inc(&pdp->used_pdpes); /* never remove */ + atomic_inc(&pdp->used); /* never remove */ + return 0; unwind: @@ -1585,7 +1577,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); free_pd(vm, pd); } - atomic_set(&pdp->used_pdpes, 0); + atomic_set(&pdp->used, 0); return -ENOMEM; } @@ -1640,27 +1632,25 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) if (err) goto err_free; - if (i915_vm_is_4lvl(&ppgtt->vm)) { - err = setup_px(&ppgtt->vm, &ppgtt->pml4); - if (err) - goto err_scratch; + ppgtt->pd = alloc_pdp(&ppgtt->vm); + if (IS_ERR(ppgtt->pd)) { + err = PTR_ERR(ppgtt->pd); + goto err_scratch; + } - gen8_initialize_pml4(&ppgtt->vm, &ppgtt->pml4); + if (i915_vm_is_4lvl(&ppgtt->vm)) { + gen8_initialize_pml4(&ppgtt->vm, ppgtt->pd); ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl; ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl; ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl; } else { - err = __pdp_init(&ppgtt->vm, &ppgtt->pdp); - if (err) - goto err_scratch; + gen8_initialize_3lvl_pdp(&ppgtt->vm, ppgtt->pd); if (intel_vgpu_active(i915)) { err = gen8_preallocate_top_level_pdp(ppgtt); - if (err) { - __pdp_fini(&ppgtt->pdp); - goto err_scratch; - } + if (err) + goto err_pdp; } ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl; @@ -1675,6 +1665,8 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) return ppgtt; +err_pdp: + free_pdp(&ppgtt->vm, ppgtt->pd); err_scratch: gen8_free_scratch(&ppgtt->vm); err_free: @@ -1740,15 +1732,16 @@ static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv) static void gen6_ppgtt_clear_range(struct i915_address_space *vm, u64 start, u64 length) { - struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); - unsigned int first_entry = start / I915_GTT_PAGE_SIZE; + struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); + const unsigned int first_entry = start / I915_GTT_PAGE_SIZE; + const gen6_pte_t scratch_pte = vm->scratch_pte; unsigned int pde = first_entry / GEN6_PTES; unsigned int pte = first_entry % GEN6_PTES; unsigned int num_entries = length / I915_GTT_PAGE_SIZE; - const gen6_pte_t scratch_pte = vm->scratch_pte; while (num_entries) { - struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++]; + struct i915_page_table * const pt = + i915_pt_entry(ppgtt->base.pd, pde++); const unsigned int count = min(num_entries, GEN6_PTES - pte); gen6_pte_t *vaddr; @@ -1756,8 +1749,8 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, num_entries -= count; - GEM_BUG_ON(count > atomic_read(&pt->used_ptes)); - if (!atomic_sub_return(count, &pt->used_ptes)) + GEM_BUG_ON(count > atomic_read(&pt->used)); + if (!atomic_sub_return(count, &pt->used)) ppgtt->scan_for_unused_pt = true; /* @@ -1781,6 +1774,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, u32 flags) { struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); + struct i915_page_directory * const pd = ppgtt->pd; unsigned first_entry = vma->node.start / I915_GTT_PAGE_SIZE; unsigned act_pt = first_entry / GEN6_PTES; unsigned act_pte = first_entry % GEN6_PTES; @@ -1788,9 +1782,9 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, struct sgt_dma iter = sgt_dma(vma); gen6_pte_t *vaddr; - GEM_BUG_ON(ppgtt->pd.page_table[act_pt] == vm->scratch_pt); + GEM_BUG_ON(i915_pt_entry(pd, act_pt) == vm->scratch_pt); - vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]); + vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt)); do { vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma); @@ -1806,7 +1800,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, if (++act_pte == GEN6_PTES) { kunmap_atomic(vaddr); - vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]); + vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt)); act_pte = 0; } } while (1); @@ -1819,6 +1813,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, u64 start, u64 length) { struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); + struct i915_page_directory * const pd = ppgtt->base.pd; struct i915_page_table *pt; intel_wakeref_t wakeref; u64 from = start; @@ -1827,14 +1822,14 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, wakeref = intel_runtime_pm_get(vm->i915); - spin_lock(&ppgtt->base.pd.lock); - gen6_for_each_pde(pt, &ppgtt->base.pd, start, length, pde) { + spin_lock(&pd->lock); + gen6_for_each_pde(pt, pd, start, length, pde) { const unsigned int count = gen6_pte_count(start, length); if (pt == vm->scratch_pt) { struct i915_page_table *old; - spin_unlock(&ppgtt->base.pd.lock); + spin_unlock(&pd->lock); pt = alloc_pt(vm); if (IS_ERR(pt)) @@ -1842,10 +1837,8 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, gen6_initialize_pt(vm, pt); - old = cmpxchg(&ppgtt->base.pd.page_table[pde], - vm->scratch_pt, pt); + old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt); if (old == vm->scratch_pt) { - ppgtt->base.pd.page_table[pde] = pt; if (i915_vma_is_bound(ppgtt->vma, I915_VMA_GLOBAL_BIND)) { gen6_write_pde(ppgtt, pde, pt); @@ -1856,12 +1849,12 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, pt = old; } - spin_lock(&ppgtt->base.pd.lock); + spin_lock(&pd->lock); } - atomic_add(count, &pt->used_ptes); + atomic_add(count, &pt->used); } - spin_unlock(&ppgtt->base.pd.lock); + spin_unlock(&pd->lock); if (flush) { mark_tlbs_dirty(&ppgtt->base); @@ -1881,6 +1874,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) { struct i915_address_space * const vm = &ppgtt->base.vm; + struct i915_page_directory * const pd = ppgtt->base.pd; struct i915_page_table *unused; u32 pde; int ret; @@ -1900,9 +1894,9 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) } gen6_initialize_pt(vm, vm->scratch_pt); - gen6_for_all_pdes(unused, &ppgtt->base.pd, pde) - ppgtt->base.pd.page_table[pde] = vm->scratch_pt; - spin_lock_init(&ppgtt->base.pd.lock); + + gen6_for_all_pdes(unused, pd, pde) + pd->entry[pde] = vm->scratch_pt; return 0; } @@ -1915,10 +1909,11 @@ static void gen6_ppgtt_free_scratch(struct i915_address_space *vm) static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt) { + struct i915_page_directory * const pd = ppgtt->base.pd; struct i915_page_table *pt; u32 pde; - gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) + gen6_for_all_pdes(pt, pd, pde) if (pt != ppgtt->base.vm.scratch_pt) free_pt(&ppgtt->base.vm, pt); } @@ -1982,6 +1977,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm) gen6_ppgtt_free_pd(ppgtt); gen6_ppgtt_free_scratch(vm); + kfree(ppgtt->base.pd); } static int pd_vma_set_pages(struct i915_vma *vma) @@ -2007,10 +2003,10 @@ static int pd_vma_bind(struct i915_vma *vma, struct i915_page_table *pt; unsigned int pde; - ppgtt->base.pd.base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t); + ppgtt->base.pd->base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t); ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset; - gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) + gen6_for_all_pdes(pt, ppgtt->base.pd, pde) gen6_write_pde(ppgtt, pde, pt); mark_tlbs_dirty(&ppgtt->base); @@ -2022,6 +2018,7 @@ static int pd_vma_bind(struct i915_vma *vma, static void pd_vma_unbind(struct i915_vma *vma) { struct gen6_ppgtt *ppgtt = vma->private; + struct i915_page_directory * const pd = ppgtt->base.pd; struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt; struct i915_page_table *pt; unsigned int pde; @@ -2030,12 +2027,12 @@ static void pd_vma_unbind(struct i915_vma *vma) return; /* Free all no longer used page tables */ - gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) { - if (atomic_read(&pt->used_ptes) || pt == scratch_pt) + gen6_for_all_pdes(pt, ppgtt->base.pd, pde) { + if (atomic_read(&pt->used) || pt == scratch_pt) continue; free_pt(&ppgtt->base.vm, pt); - ppgtt->base.pd.page_table[pde] = scratch_pt; + pd->entry[pde] = scratch_pt; } ppgtt->scan_for_unused_pt = false; @@ -2164,9 +2161,15 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915) goto err_free; } + ppgtt->base.pd = __alloc_pd(512); + if (!ppgtt->base.pd) { + err = -ENOMEM; + goto err_work; + } + err = gen6_ppgtt_init_scratch(ppgtt); if (err) - goto err_work; + goto err_pd; ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE); if (IS_ERR(ppgtt->vma)) { @@ -2178,6 +2181,8 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915) err_scratch: gen6_ppgtt_free_scratch(&ppgtt->base.vm); +err_pd: + kfree(ppgtt->base.pd); err_work: kfree(ppgtt->work); err_free: diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index 89437d0a721c..49f44071def4 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -248,28 +248,14 @@ struct i915_page_dma { struct i915_page_table { struct i915_page_dma base; - atomic_t used_ptes; + atomic_t used; }; struct i915_page_directory { struct i915_page_dma base; - - struct i915_page_table *page_table[I915_PDES]; /* PDEs */ - atomic_t used_pdes; - spinlock_t lock; -}; - -struct i915_page_directory_pointer { - struct i915_page_dma base; - struct i915_page_directory **page_directory; - atomic_t used_pdpes; - spinlock_t lock; -}; - -struct i915_pml4 { - struct i915_page_dma base; - struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4]; + atomic_t used; spinlock_t lock; + void *entry[0]; }; struct i915_vma_ops { @@ -321,7 +307,7 @@ struct i915_address_space { struct i915_page_dma scratch_page; struct i915_page_table *scratch_pt; struct i915_page_directory *scratch_pd; - struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */ + struct i915_page_directory *scratch_pdp; /* GEN8+ & 48b PPGTT */ /** * List of vma currently bound. @@ -416,11 +402,7 @@ struct i915_ppgtt { struct i915_address_space vm; intel_engine_mask_t pd_dirty_engines; - union { - struct i915_pml4 pml4; /* GEN8+ & 48b PPGTT */ - struct i915_page_directory_pointer pdp; /* GEN8+ */ - struct i915_page_directory pd; /* GEN6-7 */ - }; + struct i915_page_directory *pd; }; struct gen6_ppgtt { @@ -454,7 +436,7 @@ static inline struct gen6_ppgtt *to_gen6_ppgtt(struct i915_ppgtt *base) #define gen6_for_each_pde(pt, pd, start, length, iter) \ for (iter = gen6_pde_index(start); \ length > 0 && iter < I915_PDES && \ - (pt = (pd)->page_table[iter], true); \ + (pt = i915_pt_entry(pd, iter), true); \ ({ u32 temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT); \ temp = min(temp - start, length); \ start += temp, length -= temp; }), ++iter) @@ -462,7 +444,7 @@ static inline struct gen6_ppgtt *to_gen6_ppgtt(struct i915_ppgtt *base) #define gen6_for_all_pdes(pt, pd, iter) \ for (iter = 0; \ iter < I915_PDES && \ - (pt = (pd)->page_table[iter], true); \ + (pt = i915_pt_entry(pd, iter), true); \ ++iter) static inline u32 i915_pte_index(u64 address, unsigned int pde_shift) @@ -521,6 +503,27 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm) return GEN8_3LVL_PDPES; } +static inline struct i915_page_table * +i915_pt_entry(const struct i915_page_directory * const pd, + const unsigned short n) +{ + return pd->entry[n]; +} + +static inline struct i915_page_directory * +i915_pd_entry(const struct i915_page_directory * const pdp, + const unsigned short n) +{ + return pdp->entry[n]; +} + +static inline struct i915_page_directory * +i915_pdp_entry(const struct i915_page_directory * const pml4, + const unsigned short n) +{ + return pml4->entry[n]; +} + /* Equivalent to the gen6 version, For each pde iterates over every pde * between from start until start + length. On gen8+ it simply iterates * over every page directory entry in a page directory. @@ -528,7 +531,7 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm) #define gen8_for_each_pde(pt, pd, start, length, iter) \ for (iter = gen8_pde_index(start); \ length > 0 && iter < I915_PDES && \ - (pt = (pd)->page_table[iter], true); \ + (pt = i915_pt_entry(pd, iter), true); \ ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT); \ temp = min(temp - start, length); \ start += temp, length -= temp; }), ++iter) @@ -536,7 +539,7 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm) #define gen8_for_each_pdpe(pd, pdp, start, length, iter) \ for (iter = gen8_pdpe_index(start); \ length > 0 && iter < i915_pdpes_per_pdp(vm) && \ - (pd = (pdp)->page_directory[iter], true); \ + (pd = i915_pd_entry(pdp, iter), true); \ ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT); \ temp = min(temp - start, length); \ start += temp, length -= temp; }), ++iter) @@ -544,7 +547,7 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm) #define gen8_for_each_pml4e(pdp, pml4, start, length, iter) \ for (iter = gen8_pml4e_index(start); \ length > 0 && iter < GEN8_PML4ES_PER_PML4 && \ - (pdp = (pml4)->pdps[iter], true); \ + (pdp = i915_pdp_entry(pml4, iter), true); \ ({ u64 temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT); \ temp = min(temp - start, length); \ start += temp, length -= temp; }), ++iter) @@ -577,7 +580,10 @@ static inline u64 gen8_pte_count(u64 address, u64 length) static inline dma_addr_t i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n) { - return px_dma(ppgtt->pdp.page_directory[n]); + struct i915_page_directory *pd; + + pd = i915_pdp_entry(ppgtt->pd, n); + return px_dma(pd); } static inline struct i915_ggtt * From patchwork Tue Jun 11 17:27:25 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mika Kuoppala X-Patchwork-Id: 10987933 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 7AFB214B6 for ; Tue, 11 Jun 2019 17:27:38 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 5AD8826E1A for ; Tue, 11 Jun 2019 17:27:38 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 4F332287B1; Tue, 11 Jun 2019 17:27:38 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 081C826E1A for ; Tue, 11 Jun 2019 17:27:38 +0000 (UTC) Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id 6265B891D2; Tue, 11 Jun 2019 17:27:36 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) by gabe.freedesktop.org (Postfix) with ESMTPS id C75D2891D5 for ; Tue, 11 Jun 2019 17:27:35 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga103.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Jun 2019 10:27:35 -0700 X-ExtLoop1: 1 Received: from rosetta.fi.intel.com ([10.237.72.186]) by fmsmga005.fm.intel.com with ESMTP; 11 Jun 2019 10:27:33 -0700 Received: by rosetta.fi.intel.com (Postfix, from userid 1000) id 2EC578402D0; Tue, 11 Jun 2019 20:27:33 +0300 (EEST) From: Mika Kuoppala To: intel-gfx@lists.freedesktop.org Date: Tue, 11 Jun 2019 20:27:25 +0300 Message-Id: <20190611172731.19174-3-mika.kuoppala@linux.intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> References: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> Subject: [Intel-gfx] [PATCH 3/9] drm/i915/gtt: Introduce init_pd_with_page X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" X-Virus-Scanned: ClamAV using ClamSMTP We set the page directory entries to point into a page table. There is no gen specifics in here so make it simple and obvious. Signed-off-by: Mika Kuoppala --- drivers/gpu/drm/i915/i915_gem_gtt.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 9a1f956a817a..9d87f0fb5b16 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -724,12 +724,12 @@ static void free_pd(struct i915_address_space *vm, kfree(pd); } -static void gen8_initialize_pd(struct i915_address_space *vm, - struct i915_page_directory *pd) +static void init_pd_with_page(struct i915_address_space *vm, + struct i915_page_directory * const pd, + struct i915_page_table *pt) { - fill_px(vm, pd, - gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC)); - memset_p(pd->entry, vm->scratch_pt, I915_PDES); + fill_px(vm, pd, gen8_pde_encode(px_dma(pt), I915_CACHE_LLC)); + memset_p(pd->entry, pt, 512); } static struct i915_page_directory *alloc_pdp(struct i915_address_space *vm) @@ -1262,7 +1262,7 @@ static int gen8_init_scratch(struct i915_address_space *vm) } gen8_initialize_pt(vm, vm->scratch_pt); - gen8_initialize_pd(vm, vm->scratch_pd); + init_pd_with_page(vm, vm->scratch_pd, vm->scratch_pt); if (i915_vm_is_4lvl(vm)) gen8_initialize_4lvl_pdp(vm, vm->scratch_pdp); @@ -1439,7 +1439,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, if (IS_ERR(pd)) goto unwind; - gen8_initialize_pd(vm, pd); + init_pd_with_page(vm, pd, vm->scratch_pt); old = cmpxchg(&pdp->entry[pdpe], vm->scratch_pd, pd); if (old == vm->scratch_pd) { @@ -1561,7 +1561,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) if (IS_ERR(pd)) goto unwind; - gen8_initialize_pd(vm, pd); + init_pd_with_page(vm, pd, vm->scratch_pt); gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); atomic_inc(&pdp->used); From patchwork Tue Jun 11 17:27:26 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mika Kuoppala X-Patchwork-Id: 10987939 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 004ED14B6 for ; Tue, 11 Jun 2019 17:27:42 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id D46FF26E1A for ; Tue, 11 Jun 2019 17:27:41 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id C8C55287B1; Tue, 11 Jun 2019 17:27:41 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 79D3D26E1A for ; Tue, 11 Jun 2019 17:27:41 +0000 (UTC) Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id 318B4891E2; Tue, 11 Jun 2019 17:27:37 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by gabe.freedesktop.org (Postfix) with ESMTPS id B9A52891D4 for ; Tue, 11 Jun 2019 17:27:35 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga006.jf.intel.com ([10.7.209.51]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Jun 2019 10:27:35 -0700 X-ExtLoop1: 1 Received: from rosetta.fi.intel.com ([10.237.72.186]) by orsmga006.jf.intel.com with ESMTP; 11 Jun 2019 10:27:34 -0700 Received: by rosetta.fi.intel.com (Postfix, from userid 1000) id 2FE798402D2; Tue, 11 Jun 2019 20:27:33 +0300 (EEST) From: Mika Kuoppala To: intel-gfx@lists.freedesktop.org Date: Tue, 11 Jun 2019 20:27:26 +0300 Message-Id: <20190611172731.19174-4-mika.kuoppala@linux.intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> References: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> Subject: [Intel-gfx] [PATCH 4/9] drm/i915/gtt: Introduce init_pd X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" X-Virus-Scanned: ClamAV using ClamSMTP All page directories, excluding last level, are initialized with pointer to next level page directories. Make common function for it. Signed-off-by: Mika Kuoppala --- drivers/gpu/drm/i915/i915_gem_gtt.c | 37 +++++++++++------------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 9d87f0fb5b16..73aaf9481dab 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -759,26 +759,12 @@ static void free_pdp(struct i915_address_space *vm, kfree(pdp); } -static void gen8_initialize_4lvl_pdp(struct i915_address_space *vm, - struct i915_page_directory *pdp) +static void init_pd(struct i915_address_space *vm, + struct i915_page_directory * const pd, + struct i915_page_directory * const to) { - fill_px(vm, pdp, - gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC)); - memset_p(pdp->entry, vm->scratch_pd, 512); -} - -static void gen8_initialize_3lvl_pdp(struct i915_address_space *vm, - struct i915_page_directory *pdp) -{ - memset_p(pdp->entry, vm->scratch_pd, GEN8_3LVL_PDPES); -} - -static void gen8_initialize_pml4(struct i915_address_space *vm, - struct i915_page_directory *pml4) -{ - fill_px(vm, pml4, - gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC)); - memset_p(pml4->entry, vm->scratch_pdp, GEN8_PML4ES_PER_PML4); + fill_px(vm, pd, gen8_pdpe_encode(px_dma(to), I915_CACHE_LLC)); + memset_p(pd->entry, to, 512); } /* @@ -1264,7 +1250,7 @@ static int gen8_init_scratch(struct i915_address_space *vm) gen8_initialize_pt(vm, vm->scratch_pt); init_pd_with_page(vm, vm->scratch_pd, vm->scratch_pt); if (i915_vm_is_4lvl(vm)) - gen8_initialize_4lvl_pdp(vm, vm->scratch_pdp); + init_pd(vm, vm->scratch_pdp, vm->scratch_pd); return 0; @@ -1509,7 +1495,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, if (IS_ERR(pdp)) goto unwind; - gen8_initialize_4lvl_pdp(vm, pdp); + init_pd(vm, pdp, vm->scratch_pd); old = cmpxchg(&pml4->entry[pml4e], vm->scratch_pdp, pdp); if (old == vm->scratch_pdp) { @@ -1639,13 +1625,18 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) } if (i915_vm_is_4lvl(&ppgtt->vm)) { - gen8_initialize_pml4(&ppgtt->vm, ppgtt->pd); + init_pd(&ppgtt->vm, ppgtt->pd, ppgtt->vm.scratch_pdp); ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl; ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl; ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl; } else { - gen8_initialize_3lvl_pdp(&ppgtt->vm, ppgtt->pd); + /* + * We don't need to setup dma for top level pdp, only + * for entries. So point entries to scratch. + */ + memset_p(ppgtt->pd->entry, ppgtt->vm.scratch_pd, + GEN8_3LVL_PDPES); if (intel_vgpu_active(i915)) { err = gen8_preallocate_top_level_pdp(ppgtt); From patchwork Tue Jun 11 17:27:27 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mika Kuoppala X-Patchwork-Id: 10987935 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 7067C14BB for ; Tue, 11 Jun 2019 17:27:40 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 505B526E1A for ; Tue, 11 Jun 2019 17:27:40 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 44B1A287B1; Tue, 11 Jun 2019 17:27:40 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id B247F26E1A for ; Tue, 11 Jun 2019 17:27:39 +0000 (UTC) Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id A8EBC891D5; Tue, 11 Jun 2019 17:27:36 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by gabe.freedesktop.org (Postfix) with ESMTPS id 934C4891D2 for ; Tue, 11 Jun 2019 17:27:35 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by fmsmga105.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Jun 2019 10:27:35 -0700 X-ExtLoop1: 1 Received: from rosetta.fi.intel.com ([10.237.72.186]) by fmsmga001.fm.intel.com with ESMTP; 11 Jun 2019 10:27:33 -0700 Received: by rosetta.fi.intel.com (Postfix, from userid 1000) id 3187984055B; Tue, 11 Jun 2019 20:27:33 +0300 (EEST) From: Mika Kuoppala To: intel-gfx@lists.freedesktop.org Date: Tue, 11 Jun 2019 20:27:27 +0300 Message-Id: <20190611172731.19174-5-mika.kuoppala@linux.intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> References: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> Subject: [Intel-gfx] [PATCH 5/9] drm/i915/gtt: Generalize alloc_pd X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" X-Virus-Scanned: ClamAV using ClamSMTP Allocate all page directory variants with alloc_pd. As the lvl3 and lvl4 variants differ in manipulation, we need to check for existence of backing phys page before accessing it. Signed-off-by: Mika Kuoppala --- drivers/gpu/drm/i915/i915_gem_gtt.c | 100 ++++++++++++---------------- 1 file changed, 42 insertions(+), 58 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 73aaf9481dab..bbcf3be28e19 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -717,10 +717,17 @@ static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) return pd; } +static inline bool pd_has_phys_page(const struct i915_page_directory * const pd) +{ + return pd->base.page; +} + static void free_pd(struct i915_address_space *vm, struct i915_page_directory *pd) { - cleanup_px(vm, pd); + if (likely(pd_has_phys_page(pd))) + cleanup_px(vm, pd); + kfree(pd); } @@ -732,37 +739,12 @@ static void init_pd_with_page(struct i915_address_space *vm, memset_p(pd->entry, pt, 512); } -static struct i915_page_directory *alloc_pdp(struct i915_address_space *vm) -{ - struct i915_page_directory *pdp; - - pdp = __alloc_pd(i915_pdpes_per_pdp(vm)); - if (!pdp) - return ERR_PTR(-ENOMEM); - - if (i915_vm_is_4lvl(vm)) { - if (unlikely(setup_px(vm, pdp))) { - kfree(pdp); - return ERR_PTR(-ENOMEM); - } - } - - return pdp; -} - -static void free_pdp(struct i915_address_space *vm, - struct i915_page_directory *pdp) -{ - if (i915_vm_is_4lvl(vm)) - cleanup_px(vm, pdp); - - kfree(pdp); -} - static void init_pd(struct i915_address_space *vm, struct i915_page_directory * const pd, struct i915_page_directory * const to) { + GEM_DEBUG_BUG_ON(!pd_has_phys_page(pd)); + fill_px(vm, pd, gen8_pdpe_encode(px_dma(to), I915_CACHE_LLC)); memset_p(pd->entry, to, 512); } @@ -840,14 +822,13 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, return !atomic_read(&pd->used); } -static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm, - struct i915_page_directory *pdp, +static void gen8_ppgtt_set_pdpe(struct i915_page_directory *pdp, struct i915_page_directory *pd, unsigned int pdpe) { gen8_ppgtt_pdpe_t *vaddr; - if (!i915_vm_is_4lvl(vm)) + if (!pd_has_phys_page(pdp)) return; vaddr = kmap_atomic_px(pdp); @@ -875,7 +856,7 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, spin_lock(&pdp->lock); if (!atomic_read(&pd->used)) { - gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); + gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe); pdp->entry[pdpe] = vm->scratch_pd; GEM_BUG_ON(!atomic_read(&pdp->used)); @@ -936,7 +917,7 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm, } spin_unlock(&pml4->lock); if (free) - free_pdp(vm, pdp); + free_pd(vm, pdp); } } @@ -1240,7 +1221,7 @@ static int gen8_init_scratch(struct i915_address_space *vm) } if (i915_vm_is_4lvl(vm)) { - vm->scratch_pdp = alloc_pdp(vm); + vm->scratch_pdp = alloc_pd(vm); if (IS_ERR(vm->scratch_pdp)) { ret = PTR_ERR(vm->scratch_pdp); goto free_pd; @@ -1302,7 +1283,7 @@ static void gen8_free_scratch(struct i915_address_space *vm) return; if (i915_vm_is_4lvl(vm)) - free_pdp(vm, vm->scratch_pdp); + free_pd(vm, vm->scratch_pdp); free_pd(vm, vm->scratch_pd); free_pt(vm, vm->scratch_pt); cleanup_scratch_page(vm); @@ -1322,7 +1303,7 @@ static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm, free_pd(vm, pdp->entry[i]); } - free_pdp(vm, pdp); + free_pd(vm, pdp); } static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt) @@ -1429,7 +1410,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, old = cmpxchg(&pdp->entry[pdpe], vm->scratch_pd, pd); if (old == vm->scratch_pd) { - gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); + gen8_ppgtt_set_pdpe(pdp, pd, pdpe); atomic_inc(&pdp->used); } else { free_pd(vm, pd); @@ -1455,7 +1436,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, unwind_pd: spin_lock(&pdp->lock); if (atomic_dec_and_test(&pd->used)) { - gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); + gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe); GEM_BUG_ON(!atomic_read(&pdp->used)); atomic_dec(&pdp->used); free_pd(vm, pd); @@ -1485,13 +1466,12 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, spin_lock(&pml4->lock); gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { - if (pdp == vm->scratch_pdp) { struct i915_page_directory *old; spin_unlock(&pml4->lock); - pdp = alloc_pdp(vm); + pdp = alloc_pd(vm); if (IS_ERR(pdp)) goto unwind; @@ -1501,7 +1481,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, if (old == vm->scratch_pdp) { gen8_ppgtt_set_pml4e(pml4, pdp, pml4e); } else { - free_pdp(vm, pdp); + free_pd(vm, pdp); pdp = old; } @@ -1525,7 +1505,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, spin_lock(&pml4->lock); if (atomic_dec_and_test(&pdp->used)) { gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); - free_pdp(vm, pdp); + free_pd(vm, pdp); } spin_unlock(&pml4->lock); unwind: @@ -1548,7 +1528,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) goto unwind; init_pd_with_page(vm, pd, vm->scratch_pt); - gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); + gen8_ppgtt_set_pdpe(pdp, pd, pdpe); atomic_inc(&pdp->used); } @@ -1560,7 +1540,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) unwind: start -= from; gen8_for_each_pdpe(pd, pdp, from, start, pdpe) { - gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); + gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe); free_pd(vm, pd); } atomic_set(&pdp->used, 0); @@ -1592,7 +1572,7 @@ static void ppgtt_init(struct drm_i915_private *i915, static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) { struct i915_ppgtt *ppgtt; - int err; + int ret; ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); if (!ppgtt) @@ -1614,17 +1594,21 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) if (IS_CHERRYVIEW(i915) || IS_BROXTON(i915)) ppgtt->vm.pt_kmap_wc = true; - err = gen8_init_scratch(&ppgtt->vm); - if (err) + ret = gen8_init_scratch(&ppgtt->vm); + if (ret) goto err_free; - ppgtt->pd = alloc_pdp(&ppgtt->vm); - if (IS_ERR(ppgtt->pd)) { - err = PTR_ERR(ppgtt->pd); - goto err_scratch; + ppgtt->pd = __alloc_pd(i915_pdpes_per_pdp(&ppgtt->vm)); + if (!ppgtt->pd) { + ret = -ENOMEM; + goto err_free_scratch; } if (i915_vm_is_4lvl(&ppgtt->vm)) { + ret = setup_px(&ppgtt->vm, ppgtt->pd); + if (ret) + goto err_free_pdp; + init_pd(&ppgtt->vm, ppgtt->pd, ppgtt->vm.scratch_pdp); ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl; @@ -1639,9 +1623,9 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) GEN8_3LVL_PDPES); if (intel_vgpu_active(i915)) { - err = gen8_preallocate_top_level_pdp(ppgtt); - if (err) - goto err_pdp; + ret = gen8_preallocate_top_level_pdp(ppgtt); + if (ret) + goto err_free_pdp; } ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl; @@ -1656,13 +1640,13 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) return ppgtt; -err_pdp: - free_pdp(&ppgtt->vm, ppgtt->pd); -err_scratch: +err_free_pdp: + free_pd(&ppgtt->vm, ppgtt->pd); +err_free_scratch: gen8_free_scratch(&ppgtt->vm); err_free: kfree(ppgtt); - return ERR_PTR(err); + return ERR_PTR(ret); } /* Write pde (index) from the page directory @pd to the page table @pt */ From patchwork Tue Jun 11 17:27:28 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mika Kuoppala X-Patchwork-Id: 10987949 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 1A53514B6 for ; Tue, 11 Jun 2019 17:27:47 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id ECDDE26E1A for ; Tue, 11 Jun 2019 17:27:46 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id E1141287B1; Tue, 11 Jun 2019 17:27:46 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 47BA526E1A for ; Tue, 11 Jun 2019 17:27:46 +0000 (UTC) Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id 57D8E89228; Tue, 11 Jun 2019 17:27:45 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by gabe.freedesktop.org (Postfix) with ESMTPS id F3FEE89221 for ; Tue, 11 Jun 2019 17:27:38 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by fmsmga105.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Jun 2019 10:27:37 -0700 X-ExtLoop1: 1 Received: from rosetta.fi.intel.com ([10.237.72.186]) by fmsmga001.fm.intel.com with ESMTP; 11 Jun 2019 10:27:35 -0700 Received: by rosetta.fi.intel.com (Postfix, from userid 1000) id 3495584055F; Tue, 11 Jun 2019 20:27:33 +0300 (EEST) From: Mika Kuoppala To: intel-gfx@lists.freedesktop.org Date: Tue, 11 Jun 2019 20:27:28 +0300 Message-Id: <20190611172731.19174-6-mika.kuoppala@linux.intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> References: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> Subject: [Intel-gfx] [PATCH 6/9] drm/i915/gtt: pde entry encoding is identical X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" X-Virus-Scanned: ClamAV using ClamSMTP For all page directory entries, the pde encoding is identical. Don't compilicate call sites with different versions of doing the same thing. Only wart that remains is a 4 entry gen8/bsw pdp, for which we need to check the backing phys page. Signed-off-by: Mika Kuoppala --- drivers/gpu/drm/i915/i915_gem_gtt.c | 111 ++++++++++------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 3 - 2 files changed, 40 insertions(+), 74 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index bbcf3be28e19..815950658b12 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -214,10 +214,10 @@ static u64 gen8_pte_encode(dma_addr_t addr, return pte; } -static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, - const enum i915_cache_level level) +static u64 gen8_pde_encode(const dma_addr_t addr, + const enum i915_cache_level level) { - gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; + u64 pde = _PAGE_PRESENT | _PAGE_RW; pde |= addr; if (level != I915_CACHE_NONE) pde |= PPAT_CACHED_PDE; @@ -226,9 +226,6 @@ static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, return pde; } -#define gen8_pdpe_encode gen8_pde_encode -#define gen8_pml4e_encode gen8_pde_encode - static u64 snb_pte_encode(dma_addr_t addr, enum i915_cache_level level, u32 flags) @@ -731,24 +728,36 @@ static void free_pd(struct i915_address_space *vm, kfree(pd); } -static void init_pd_with_page(struct i915_address_space *vm, - struct i915_page_directory * const pd, - struct i915_page_table *pt) -{ - fill_px(vm, pd, gen8_pde_encode(px_dma(pt), I915_CACHE_LLC)); - memset_p(pd->entry, pt, 512); +#define init_pd(vm, pd, to) { \ + GEM_DEBUG_BUG_ON(!pd_has_phys_page(pd)); \ + fill_px((vm), (pd), gen8_pde_encode(px_dma(to), I915_CACHE_LLC)); \ + memset_p((pd)->entry, (to), 512); \ } -static void init_pd(struct i915_address_space *vm, - struct i915_page_directory * const pd, - struct i915_page_directory * const to) +static void __set_pd_entry(struct i915_page_directory * const pd, + const unsigned short pde, + const u64 encoded_entry) { - GEM_DEBUG_BUG_ON(!pd_has_phys_page(pd)); + u64 *vaddr; - fill_px(vm, pd, gen8_pdpe_encode(px_dma(to), I915_CACHE_LLC)); - memset_p(pd->entry, to, 512); + vaddr = kmap_atomic(pd->base.page); + vaddr[pde] = encoded_entry; + kunmap_atomic(vaddr); } +#define set_pd_entry(pd, pde, to) ({ \ + (pd)->entry[(pde)] = (to); \ + __set_pd_entry((pd), (pde), \ + gen8_pde_encode(px_dma(to), I915_CACHE_LLC)); \ +}) + +#define set_pdp_entry(pdp, pdpe, to) ({ \ + (pdp)->entry[(pdpe)] = (to); \ + if (pd_has_phys_page(pdp)) \ + __set_pd_entry((pdp), (pdpe), \ + gen8_pde_encode(px_dma(to), I915_CACHE_LLC));\ +}) + /* * PDE TLBs are a pain to invalidate on GEN8+. When we modify * the page table structures, we mark them dirty so that @@ -778,18 +787,6 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm, return !atomic_sub_return(num_entries, &pt->used); } -static void gen8_ppgtt_set_pde(struct i915_address_space *vm, - struct i915_page_directory *pd, - struct i915_page_table *pt, - unsigned int pde) -{ - gen8_pde_t *vaddr; - - vaddr = kmap_atomic_px(pd); - vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC); - kunmap_atomic(vaddr); -} - static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, struct i915_page_directory *pd, u64 start, u64 length) @@ -807,8 +804,7 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, spin_lock(&pd->lock); if (!atomic_read(&pt->used)) { - gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde); - pd->entry[pde] = vm->scratch_pt; + set_pd_entry(pd, pde, vm->scratch_pt); GEM_BUG_ON(!atomic_read(&pd->used)); atomic_dec(&pd->used); @@ -822,20 +818,6 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm, return !atomic_read(&pd->used); } -static void gen8_ppgtt_set_pdpe(struct i915_page_directory *pdp, - struct i915_page_directory *pd, - unsigned int pdpe) -{ - gen8_ppgtt_pdpe_t *vaddr; - - if (!pd_has_phys_page(pdp)) - return; - - vaddr = kmap_atomic_px(pdp); - vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC); - kunmap_atomic(vaddr); -} - /* Removes entries from a single page dir pointer, releasing it if it's empty. * Caller can use the return value to update higher-level entries */ @@ -856,8 +838,7 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, spin_lock(&pdp->lock); if (!atomic_read(&pd->used)) { - gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe); - pdp->entry[pdpe] = vm->scratch_pd; + set_pdp_entry(pdp, pdpe, vm->scratch_pd); GEM_BUG_ON(!atomic_read(&pdp->used)); atomic_dec(&pdp->used); @@ -877,17 +858,6 @@ static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm, gen8_ppgtt_clear_pdp(vm, i915_vm_to_ppgtt(vm)->pd, start, length); } -static void gen8_ppgtt_set_pml4e(struct i915_page_directory *pml4, - struct i915_page_directory *pdp, - unsigned int pml4e) -{ - gen8_ppgtt_pml4e_t *vaddr; - - vaddr = kmap_atomic_px(pml4); - vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); - kunmap_atomic(vaddr); -} - /* Removes entries from a single pml4. * This is the top-level structure in 4-level page tables used on gen8+. * Empty entries are always scratch pml4e. @@ -911,8 +881,7 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm, spin_lock(&pml4->lock); if (!atomic_read(&pdp->used)) { - gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); - pml4->entry[pml4e] = vm->scratch_pdp; + set_pd_entry(pml4, pml4e, vm->scratch_pdp); free = true; } spin_unlock(&pml4->lock); @@ -1229,7 +1198,7 @@ static int gen8_init_scratch(struct i915_address_space *vm) } gen8_initialize_pt(vm, vm->scratch_pt); - init_pd_with_page(vm, vm->scratch_pd, vm->scratch_pt); + init_pd(vm, vm->scratch_pd, vm->scratch_pt); if (i915_vm_is_4lvl(vm)) init_pd(vm, vm->scratch_pdp, vm->scratch_pd); @@ -1365,7 +1334,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt); if (old == vm->scratch_pt) { - gen8_ppgtt_set_pde(vm, pd, pt, pde); + set_pd_entry(pd, pde, pt); atomic_inc(&pd->used); } else { free_pt(vm, pt); @@ -1406,11 +1375,11 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, if (IS_ERR(pd)) goto unwind; - init_pd_with_page(vm, pd, vm->scratch_pt); + init_pd(vm, pd, vm->scratch_pt); old = cmpxchg(&pdp->entry[pdpe], vm->scratch_pd, pd); if (old == vm->scratch_pd) { - gen8_ppgtt_set_pdpe(pdp, pd, pdpe); + set_pdp_entry(pdp, pdpe, pd); atomic_inc(&pdp->used); } else { free_pd(vm, pd); @@ -1436,7 +1405,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, unwind_pd: spin_lock(&pdp->lock); if (atomic_dec_and_test(&pd->used)) { - gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe); + set_pdp_entry(pdp, pdpe, vm->scratch_pd); GEM_BUG_ON(!atomic_read(&pdp->used)); atomic_dec(&pdp->used); free_pd(vm, pd); @@ -1479,7 +1448,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, old = cmpxchg(&pml4->entry[pml4e], vm->scratch_pdp, pdp); if (old == vm->scratch_pdp) { - gen8_ppgtt_set_pml4e(pml4, pdp, pml4e); + set_pd_entry(pml4, pml4e, pdp); } else { free_pd(vm, pdp); pdp = old; @@ -1504,7 +1473,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, unwind_pdp: spin_lock(&pml4->lock); if (atomic_dec_and_test(&pdp->used)) { - gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); + set_pd_entry(pml4, pml4e, vm->scratch_pdp); free_pd(vm, pdp); } spin_unlock(&pml4->lock); @@ -1527,8 +1496,8 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) if (IS_ERR(pd)) goto unwind; - init_pd_with_page(vm, pd, vm->scratch_pt); - gen8_ppgtt_set_pdpe(pdp, pd, pdpe); + init_pd(vm, pd, vm->scratch_pt); + set_pdp_entry(pdp, pdpe, pd); atomic_inc(&pdp->used); } @@ -1540,7 +1509,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) unwind: start -= from; gen8_for_each_pdpe(pd, pdp, from, start, pdpe) { - gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe); + set_pdp_entry(pdp, pdpe, vm->scratch_pd); free_pd(vm, pd); } atomic_set(&pdp->used, 0); diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index 49f44071def4..4dfc5783404a 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -67,9 +67,6 @@ struct i915_vma; typedef u32 gen6_pte_t; typedef u64 gen8_pte_t; -typedef u64 gen8_pde_t; -typedef u64 gen8_ppgtt_pdpe_t; -typedef u64 gen8_ppgtt_pml4e_t; #define ggtt_total_entries(ggtt) ((ggtt)->vm.total >> PAGE_SHIFT) From patchwork Tue Jun 11 17:27:29 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mika Kuoppala X-Patchwork-Id: 10987941 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id CF92914BB for ; Tue, 11 Jun 2019 17:27:42 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id B010F26E1A for ; Tue, 11 Jun 2019 17:27:42 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id A4977287B1; Tue, 11 Jun 2019 17:27:42 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 546A126E1A for ; Tue, 11 Jun 2019 17:27:42 +0000 (UTC) Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id 6216F89220; Tue, 11 Jun 2019 17:27:37 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by gabe.freedesktop.org (Postfix) with ESMTPS id 89852891D4 for ; Tue, 11 Jun 2019 17:27:36 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Jun 2019 10:27:36 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.63,362,1557212400"; d="scan'208";a="183872738" Received: from rosetta.fi.intel.com ([10.237.72.186]) by fmsmga002.fm.intel.com with ESMTP; 11 Jun 2019 10:27:35 -0700 Received: by rosetta.fi.intel.com (Postfix, from userid 1000) id 3762684064E; Tue, 11 Jun 2019 20:27:33 +0300 (EEST) From: Mika Kuoppala To: intel-gfx@lists.freedesktop.org Date: Tue, 11 Jun 2019 20:27:29 +0300 Message-Id: <20190611172731.19174-7-mika.kuoppala@linux.intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> References: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> Subject: [Intel-gfx] [PATCH 7/9] drm/i915/gtt: Check for physical page for pd entry always X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" X-Virus-Scanned: ClamAV using ClamSMTP Check the physical page before writing the entry into the physical page. This further generalizes the pd so that manipulation in callsites will be identical, removing the need to handle pdps differently for gen8. Signed-off-by: Mika Kuoppala --- drivers/gpu/drm/i915/i915_gem_gtt.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 815950658b12..f1d7874834e2 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -747,17 +747,11 @@ static void __set_pd_entry(struct i915_page_directory * const pd, #define set_pd_entry(pd, pde, to) ({ \ (pd)->entry[(pde)] = (to); \ - __set_pd_entry((pd), (pde), \ + if (likely(pd_has_phys_page(pd))) \ + __set_pd_entry((pd), (pde), \ gen8_pde_encode(px_dma(to), I915_CACHE_LLC)); \ }) -#define set_pdp_entry(pdp, pdpe, to) ({ \ - (pdp)->entry[(pdpe)] = (to); \ - if (pd_has_phys_page(pdp)) \ - __set_pd_entry((pdp), (pdpe), \ - gen8_pde_encode(px_dma(to), I915_CACHE_LLC));\ -}) - /* * PDE TLBs are a pain to invalidate on GEN8+. When we modify * the page table structures, we mark them dirty so that @@ -838,7 +832,7 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, spin_lock(&pdp->lock); if (!atomic_read(&pd->used)) { - set_pdp_entry(pdp, pdpe, vm->scratch_pd); + set_pd_entry(pdp, pdpe, vm->scratch_pd); GEM_BUG_ON(!atomic_read(&pdp->used)); atomic_dec(&pdp->used); @@ -1379,7 +1373,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, old = cmpxchg(&pdp->entry[pdpe], vm->scratch_pd, pd); if (old == vm->scratch_pd) { - set_pdp_entry(pdp, pdpe, pd); + set_pd_entry(pdp, pdpe, pd); atomic_inc(&pdp->used); } else { free_pd(vm, pd); @@ -1405,7 +1399,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, unwind_pd: spin_lock(&pdp->lock); if (atomic_dec_and_test(&pd->used)) { - set_pdp_entry(pdp, pdpe, vm->scratch_pd); + set_pd_entry(pdp, pdpe, vm->scratch_pd); GEM_BUG_ON(!atomic_read(&pdp->used)); atomic_dec(&pdp->used); free_pd(vm, pd); @@ -1497,7 +1491,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) goto unwind; init_pd(vm, pd, vm->scratch_pt); - set_pdp_entry(pdp, pdpe, pd); + set_pd_entry(pdp, pdpe, pd); atomic_inc(&pdp->used); } @@ -1509,7 +1503,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) unwind: start -= from; gen8_for_each_pdpe(pd, pdp, from, start, pdpe) { - set_pdp_entry(pdp, pdpe, vm->scratch_pd); + set_pd_entry(pdp, pdpe, vm->scratch_pd); free_pd(vm, pd); } atomic_set(&pdp->used, 0); From patchwork Tue Jun 11 17:27:30 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mika Kuoppala X-Patchwork-Id: 10987943 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 8FF2714BB for ; Tue, 11 Jun 2019 17:27:43 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 701B826E1A for ; Tue, 11 Jun 2019 17:27:43 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 64A20287B1; Tue, 11 Jun 2019 17:27:43 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id 11D1D26E1A for ; Tue, 11 Jun 2019 17:27:43 +0000 (UTC) Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id 9460189202; Tue, 11 Jun 2019 17:27:38 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by gabe.freedesktop.org (Postfix) with ESMTPS id 5510689218 for ; Tue, 11 Jun 2019 17:27:37 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga001.jf.intel.com ([10.7.209.18]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Jun 2019 10:27:37 -0700 X-ExtLoop1: 1 Received: from rosetta.fi.intel.com ([10.237.72.186]) by orsmga001.jf.intel.com with ESMTP; 11 Jun 2019 10:27:35 -0700 Received: by rosetta.fi.intel.com (Postfix, from userid 1000) id 3ACDE840684; Tue, 11 Jun 2019 20:27:33 +0300 (EEST) From: Mika Kuoppala To: intel-gfx@lists.freedesktop.org Date: Tue, 11 Jun 2019 20:27:30 +0300 Message-Id: <20190611172731.19174-8-mika.kuoppala@linux.intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> References: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> Subject: [Intel-gfx] [PATCH 8/9] drm/i915/gtt: Make swapping the pd entry generic X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" X-Virus-Scanned: ClamAV using ClamSMTP Swapping a pd entry is same across the page directories, if we succeed we need to increment the count and write the phys page entry. Make a common function for it. Signed-off-by: Mika Kuoppala --- drivers/gpu/drm/i915/i915_gem_gtt.c | 41 +++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index f1d7874834e2..9b0d0a077e31 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -752,6 +752,27 @@ static void __set_pd_entry(struct i915_page_directory * const pd, gen8_pde_encode(px_dma(to), I915_CACHE_LLC)); \ }) +static void *__swap_pd_entry(struct i915_page_directory * const pd, + const unsigned short pde, + void * const old_val, + void * const new_val, + const u64 encoded_entry) +{ + void * const old = cmpxchg(&pd->entry[pde], old_val, new_val); + + if (likely(old == old_val)) { + atomic_inc(&pd->used); + if (likely(pd_has_phys_page(pd))) + __set_pd_entry(pd, pde, encoded_entry); + } + + return old; +} + +#define swap_pd_entry(pd, pde, old, to) \ + __swap_pd_entry((pd), (pde), (old), (to), \ + gen8_pde_encode(px_dma(to), I915_CACHE_LLC)) + /* * PDE TLBs are a pain to invalidate on GEN8+. When we modify * the page table structures, we mark them dirty so that @@ -1326,11 +1347,8 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm, if (count < GEN8_PTES || intel_vgpu_active(vm->i915)) gen8_initialize_pt(vm, pt); - old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt); - if (old == vm->scratch_pt) { - set_pd_entry(pd, pde, pt); - atomic_inc(&pd->used); - } else { + old = swap_pd_entry(pd, pde, vm->scratch_pt, pt); + if (unlikely(old != vm->scratch_pt)) { free_pt(vm, pt); pt = old; } @@ -1371,11 +1389,8 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, init_pd(vm, pd, vm->scratch_pt); - old = cmpxchg(&pdp->entry[pdpe], vm->scratch_pd, pd); - if (old == vm->scratch_pd) { - set_pd_entry(pdp, pdpe, pd); - atomic_inc(&pdp->used); - } else { + old = swap_pd_entry(pdp, pdpe, vm->scratch_pd, pd); + if (unlikely(old != vm->scratch_pd)) { free_pd(vm, pd); pd = old; } @@ -1440,10 +1455,8 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, init_pd(vm, pdp, vm->scratch_pd); - old = cmpxchg(&pml4->entry[pml4e], vm->scratch_pdp, pdp); - if (old == vm->scratch_pdp) { - set_pd_entry(pml4, pml4e, pdp); - } else { + old = swap_pd_entry(pml4, pml4e, vm->scratch_pdp, pdp); + if (unlikely(old != vm->scratch_pdp)) { free_pd(vm, pdp); pdp = old; } From patchwork Tue Jun 11 17:27:31 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mika Kuoppala X-Patchwork-Id: 10987947 Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 6A92214C0 for ; Tue, 11 Jun 2019 17:27:46 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id 4AFD8287AD for ; Tue, 11 Jun 2019 17:27:46 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id 3FC09287B1; Tue, 11 Jun 2019 17:27:46 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-5.2 required=2.0 tests=BAYES_00,MAILING_LIST_MULTI, RCVD_IN_DNSWL_MED autolearn=ham version=3.3.1 Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) (using TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.wl.linuxfoundation.org (Postfix) with ESMTPS id E7A2D26E1A for ; Tue, 11 Jun 2019 17:27:45 +0000 (UTC) Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id 430B689226; Tue, 11 Jun 2019 17:27:45 +0000 (UTC) X-Original-To: intel-gfx@lists.freedesktop.org Delivered-To: intel-gfx@lists.freedesktop.org Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by gabe.freedesktop.org (Postfix) with ESMTPS id 38B1D89204 for ; Tue, 11 Jun 2019 17:27:37 +0000 (UTC) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga002.jf.intel.com ([10.7.209.21]) by orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 11 Jun 2019 10:27:36 -0700 X-ExtLoop1: 1 Received: from rosetta.fi.intel.com ([10.237.72.186]) by orsmga002.jf.intel.com with ESMTP; 11 Jun 2019 10:27:35 -0700 Received: by rosetta.fi.intel.com (Postfix, from userid 1000) id 3E4DF840688; Tue, 11 Jun 2019 20:27:33 +0300 (EEST) From: Mika Kuoppala To: intel-gfx@lists.freedesktop.org Date: Tue, 11 Jun 2019 20:27:31 +0300 Message-Id: <20190611172731.19174-9-mika.kuoppala@linux.intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> References: <20190611172731.19174-1-mika.kuoppala@linux.intel.com> Subject: [Intel-gfx] [PATCH 9/9] drm/i915/gtt: Tear down setup and cleanup macros for page dma X-BeenThere: intel-gfx@lists.freedesktop.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: Intel graphics driver community testing & development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" X-Virus-Scanned: ClamAV using ClamSMTP We don't use common codepaths to setup and cleanup page directories vs page tables. So their setup and cleanup macros are of no use. Signed-off-by: Mika Kuoppala --- drivers/gpu/drm/i915/i915_gem_gtt.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 9b0d0a077e31..a2022f240fb6 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -550,8 +550,6 @@ static void cleanup_page_dma(struct i915_address_space *vm, #define kmap_atomic_px(px) kmap_atomic(px_base(px)->page) -#define setup_px(vm, px) setup_page_dma((vm), px_base(px)) -#define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px)) #define fill_px(vm, px, v) fill_page_dma((vm), px_base(px), (v)) #define fill32_px(vm, px, v) fill_page_dma_32((vm), px_base(px), (v)) @@ -653,7 +651,7 @@ static struct i915_page_table *alloc_pt(struct i915_address_space *vm) if (unlikely(!pt)) return ERR_PTR(-ENOMEM); - if (unlikely(setup_px(vm, pt))) { + if (unlikely(setup_page_dma(vm, &pt->base))) { kfree(pt); return ERR_PTR(-ENOMEM); } @@ -665,7 +663,7 @@ static struct i915_page_table *alloc_pt(struct i915_address_space *vm) static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt) { - cleanup_px(vm, pt); + cleanup_page_dma(vm, &pt->base); kfree(pt); } @@ -706,7 +704,7 @@ static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) if (unlikely(!pd)) return ERR_PTR(-ENOMEM); - if (unlikely(setup_px(vm, pd))) { + if (unlikely(setup_page_dma(vm, &pd->base))) { kfree(pd); return ERR_PTR(-ENOMEM); } @@ -723,7 +721,7 @@ static void free_pd(struct i915_address_space *vm, struct i915_page_directory *pd) { if (likely(pd_has_phys_page(pd))) - cleanup_px(vm, pd); + cleanup_page_dma(vm, &pd->base); kfree(pd); } @@ -1304,7 +1302,7 @@ static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt) gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp); } - cleanup_px(&ppgtt->vm, pml4); + cleanup_page_dma(&ppgtt->vm, &pml4->base); } static void gen8_ppgtt_cleanup(struct i915_address_space *vm) @@ -1581,7 +1579,7 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) } if (i915_vm_is_4lvl(&ppgtt->vm)) { - ret = setup_px(&ppgtt->vm, ppgtt->pd); + ret = setup_page_dma(&ppgtt->vm, &ppgtt->pd->base); if (ret) goto err_free_pdp;