diff mbox

[v3,02/17] drm/i915/gen8: Make pdp allocation more dynamic

Message ID 1435764453-11954-3-git-send-email-michel.thierry@intel.com (mailing list archive)
State New, archived
Headers show

Commit Message

Michel Thierry July 1, 2015, 3:27 p.m. UTC
This transitional patch doesn't do much for the existing code. However,
it should make upcoming patches to use the full 48b address space a bit
easier. The patch also introduces the PML4, ie. the new top level structure
of the page tables.

v2: Renamed  pdp_free to be similar to  pd/pt (unmap_and_free_pdp).
v3: To facilitate testing, 48b mode will be available on Broadwell and
GEN9+, when i915.enable_ppgtt = 3.
v4: Rebase after s/page_tables/page_table/, added extra information
about 4-level page table formats and use IS_ENABLED macro.
v5: Check CONFIG_X86_64 instead of CONFIG_64BIT.
v6: Rebase after Mika's ppgtt cleanup / scratch merge patch series, and follow
his nomenclature in pdp functions (there is no alloc_pdp yet).
v7: Rebase after merged version of Mika's ppgtt cleanup patch series.
v8: Rebase after final merged version of Mika's ppgtt/scratch patches.

Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
Signed-off-by: Michel Thierry <michel.thierry@intel.com> (v2+)
---
 drivers/gpu/drm/i915/i915_drv.h     |   7 ++-
 drivers/gpu/drm/i915/i915_gem_gtt.c | 116 ++++++++++++++++++++++++++++--------
 drivers/gpu/drm/i915/i915_gem_gtt.h |  41 ++++++++++---
 3 files changed, 128 insertions(+), 36 deletions(-)

Comments

akash.goel@intel.com July 7, 2015, 12:36 p.m. UTC | #1
On 7/1/2015 8:57 PM, Michel Thierry wrote:
> This transitional patch doesn't do much for the existing code. However,
> it should make upcoming patches to use the full 48b address space a bit
> easier. The patch also introduces the PML4, ie. the new top level structure
> of the page tables.
>

Would be better to move the introduction of PML4 to a separate patch & 
keep this patch only for the dynamic allocation of pdp changes.

> v2: Renamed  pdp_free to be similar to  pd/pt (unmap_and_free_pdp).
> v3: To facilitate testing, 48b mode will be available on Broadwell and
> GEN9+, when i915.enable_ppgtt = 3.
> v4: Rebase after s/page_tables/page_table/, added extra information
> about 4-level page table formats and use IS_ENABLED macro.
> v5: Check CONFIG_X86_64 instead of CONFIG_64BIT.
> v6: Rebase after Mika's ppgtt cleanup / scratch merge patch series, and follow
> his nomenclature in pdp functions (there is no alloc_pdp yet).
> v7: Rebase after merged version of Mika's ppgtt cleanup patch series.
> v8: Rebase after final merged version of Mika's ppgtt/scratch patches.
>
> Signed-off-by: Ben Widawsky <ben@bwidawsk.net>
> Signed-off-by: Michel Thierry <michel.thierry@intel.com> (v2+)
> ---
>   drivers/gpu/drm/i915/i915_drv.h     |   7 ++-
>   drivers/gpu/drm/i915/i915_gem_gtt.c | 116 ++++++++++++++++++++++++++++--------
>   drivers/gpu/drm/i915/i915_gem_gtt.h |  41 ++++++++++---
>   3 files changed, 128 insertions(+), 36 deletions(-)
>
> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
> index 1dbd957..7bccfd5 100644
> --- a/drivers/gpu/drm/i915/i915_drv.h
> +++ b/drivers/gpu/drm/i915/i915_drv.h
> @@ -2490,7 +2490,12 @@ struct drm_i915_cmd_table {
>   #define HAS_HW_CONTEXTS(dev)	(INTEL_INFO(dev)->gen >= 6)
>   #define HAS_LOGICAL_RING_CONTEXTS(dev)	(INTEL_INFO(dev)->gen >= 8)
>   #define USES_PPGTT(dev)		(i915.enable_ppgtt)
> -#define USES_FULL_PPGTT(dev)	(i915.enable_ppgtt == 2)
> +#define USES_FULL_PPGTT(dev)	(i915.enable_ppgtt >= 2)
> +#ifdef CONFIG_X86_64
> +# define USES_FULL_48BIT_PPGTT(dev)	(i915.enable_ppgtt == 3)
> +#else
> +# define USES_FULL_48BIT_PPGTT(dev)	false
> +#endif
>
>   #define HAS_OVERLAY(dev)		(INTEL_INFO(dev)->has_overlay)
>   #define OVERLAY_NEEDS_PHYSICAL(dev)	(INTEL_INFO(dev)->overlay_needs_physical)
> diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
> index 712ca34..cdcc778 100644
> --- a/drivers/gpu/drm/i915/i915_gem_gtt.c
> +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
> @@ -104,9 +104,13 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
>   {
>   	bool has_aliasing_ppgtt;
>   	bool has_full_ppgtt;
> +	bool has_full_64bit_ppgtt;
>
>   	has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
>   	has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
> +	has_full_64bit_ppgtt = IS_ENABLED(CONFIG_X86_64) &&
> +			       (IS_BROADWELL(dev) ||
> +				INTEL_INFO(dev)->gen >= 9) && false; /* FIXME: 64b */
>
>   	if (intel_vgpu_active(dev))
>   		has_full_ppgtt = false; /* emulation is too hard */
> @@ -125,6 +129,9 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
>   	if (enable_ppgtt == 2 && has_full_ppgtt)
>   		return 2;
>
> +	if (enable_ppgtt == 3 && has_full_64bit_ppgtt)
> +		return 3;
> +
>   #ifdef CONFIG_INTEL_IOMMU
>   	/* Disable ppgtt on SNB if VT-d is on. */
>   	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
> @@ -522,6 +529,45 @@ static void gen8_initialize_pd(struct i915_address_space *vm,
>   	fill_px(vm->dev, pd, scratch_pde);
>   }
>
> +static int __pdp_init(struct drm_device *dev,
> +		      struct i915_page_directory_pointer *pdp)
> +{
> +	size_t pdpes = I915_PDPES_PER_PDP(dev);
> +
> +	pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
> +				  sizeof(unsigned long),
> +				  GFP_KERNEL);
> +	if (!pdp->used_pdpes)
> +		return -ENOMEM;
> +
> +	pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
> +				      GFP_KERNEL);
> +	if (!pdp->page_directory) {
> +		kfree(pdp->used_pdpes);
> +		/* the PDP might be the statically allocated top level. Keep it
> +		 * as clean as possible */
> +		pdp->used_pdpes = NULL;
> +		return -ENOMEM;
> +	}
> +
> +	return 0;
> +}
> +
> +static void __pdp_fini(struct i915_page_directory_pointer *pdp)
> +{
> +	kfree(pdp->used_pdpes);
> +	kfree(pdp->page_directory);
> +	pdp->page_directory = NULL;
> +}
> +
> +static void free_pdp(struct drm_device *dev,
> +		     struct i915_page_directory_pointer *pdp)
> +{
> +	__pdp_fini(pdp);
> +	if (USES_FULL_48BIT_PPGTT(dev))
> +		kfree(pdp);
> +}
> +
>   /* Broadwell Page Directory Pointer Descriptors */
>   static int gen8_write_pdp(struct drm_i915_gem_request *req,
>   			  unsigned entry,
> @@ -634,9 +680,6 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
>   	pt_vaddr = NULL;
>
>   	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
> -		if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
> -			break;
> -
>   		if (pt_vaddr == NULL) {
>   			struct i915_page_directory *pd = ppgtt->pdp.page_directory[pdpe];
>   			struct i915_page_table *pt = pd->page_table[pde];
> @@ -720,7 +763,8 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
>   		container_of(vm, struct i915_hw_ppgtt, base);
>   	int i;
>
> -	for_each_set_bit(i, ppgtt->pdp.used_pdpes, GEN8_LEGACY_PDPES) {
> +	for_each_set_bit(i, ppgtt->pdp.used_pdpes,
> +				I915_PDPES_PER_PDP(ppgtt->base.dev)) {
>   		if (WARN_ON(!ppgtt->pdp.page_directory[i]))
>   			continue;
>
> @@ -729,6 +773,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
>   		free_pd(ppgtt->base.dev, ppgtt->pdp.page_directory[i]);
>   	}
>
> +	free_pdp(ppgtt->base.dev, &ppgtt->pdp);
>   	gen8_free_scratch(vm);
>   }
>
> @@ -820,8 +865,9 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,
>   	struct i915_page_directory *pd;
>   	uint64_t temp;
>   	uint32_t pdpe;
> +	uint32_t pdpes =  I915_PDPES_PER_PDP(ppgtt->base.dev);
>
> -	WARN_ON(!bitmap_empty(new_pds, GEN8_LEGACY_PDPES));
> +	WARN_ON(!bitmap_empty(new_pds, pdpes));
>
>   	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
>   		if (pd)
> @@ -839,18 +885,19 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,
>   	return 0;
>
>   unwind_out:
> -	for_each_set_bit(pdpe, new_pds, GEN8_LEGACY_PDPES)
> +	for_each_set_bit(pdpe, new_pds, pdpes)
>   		free_pd(dev, pdp->page_directory[pdpe]);
>
>   	return -ENOMEM;
>   }
>
>   static void
> -free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts)
> +free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts,
> +		       uint32_t pdpes)
>   {
>   	int i;
>
> -	for (i = 0; i < GEN8_LEGACY_PDPES; i++)
> +	for (i = 0; i < pdpes; i++)
>   		kfree(new_pts[i]);
>   	kfree(new_pts);
>   	kfree(new_pds);
> @@ -861,23 +908,24 @@ free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts)
>    */
>   static
>   int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
> -					 unsigned long ***new_pts)
> +					 unsigned long ***new_pts,
> +					 uint32_t pdpes)
>   {
>   	int i;
>   	unsigned long *pds;
>   	unsigned long **pts;
>
> -	pds = kcalloc(BITS_TO_LONGS(GEN8_LEGACY_PDPES), sizeof(unsigned long), GFP_KERNEL);
> +	pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_KERNEL);
>   	if (!pds)
>   		return -ENOMEM;
>
> -	pts = kcalloc(GEN8_LEGACY_PDPES, sizeof(unsigned long *), GFP_KERNEL);
> +	pts = kcalloc(pdpes, sizeof(unsigned long *), GFP_KERNEL);
>   	if (!pts) {
>   		kfree(pds);
>   		return -ENOMEM;
>   	}
>
> -	for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
> +	for (i = 0; i < pdpes; i++) {
>   		pts[i] = kcalloc(BITS_TO_LONGS(I915_PDES),
>   				 sizeof(unsigned long), GFP_KERNEL);
>   		if (!pts[i])
> @@ -890,7 +938,7 @@ int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
>   	return 0;
>
>   err_out:
> -	free_gen8_temp_bitmaps(pds, pts);
> +	free_gen8_temp_bitmaps(pds, pts, pdpes);
>   	return -ENOMEM;
>   }
>
> @@ -916,6 +964,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
>   	const uint64_t orig_length = length;
>   	uint64_t temp;
>   	uint32_t pdpe;
> +	uint32_t pdpes = I915_PDPES_PER_PDP(dev);
>   	int ret;
>
>   	/* Wrap is never okay since we can only represent 48b, and we don't
> @@ -927,7 +976,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
>   	if (WARN_ON(start + length > ppgtt->base.total))
>   		return -ENODEV;
>
> -	ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables);
> +	ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
>   	if (ret)
>   		return ret;
>
> @@ -935,7 +984,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
>   	ret = gen8_ppgtt_alloc_page_directories(ppgtt, &ppgtt->pdp, start, length,
>   					new_page_dirs);
>   	if (ret) {
> -		free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
> +		free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
>   		return ret;
>   	}
>
> @@ -989,7 +1038,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
>   		__set_bit(pdpe, ppgtt->pdp.used_pdpes);
>   	}
>
> -	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
> +	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
>   	mark_tlbs_dirty(ppgtt);
>   	return 0;
>
> @@ -999,10 +1048,10 @@ err_out:
>   			free_pt(vm->dev, ppgtt->pdp.page_directory[pdpe]->page_table[temp]);
>   	}
>
> -	for_each_set_bit(pdpe, new_page_dirs, GEN8_LEGACY_PDPES)
> +	for_each_set_bit(pdpe, new_page_dirs, pdpes)
>   		free_pd(vm->dev, ppgtt->pdp.page_directory[pdpe]);
>
> -	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
> +	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
>   	mark_tlbs_dirty(ppgtt);
>   	return ret;
>   }
> @@ -1023,14 +1072,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
>   		return ret;
>
>   	ppgtt->base.start = 0;
> -	ppgtt->base.total = 1ULL << 32;
> -	if (IS_ENABLED(CONFIG_X86_32))
> -		/* While we have a proliferation of size_t variables
> -		 * we cannot represent the full ppgtt size on 32bit,
> -		 * so limit it to the same size as the GGTT (currently
> -		 * 2GiB).
> -		 */
> -		ppgtt->base.total = to_i915(ppgtt->base.dev)->gtt.base.total;
>   	ppgtt->base.cleanup = gen8_ppgtt_cleanup;
>   	ppgtt->base.allocate_va_range = gen8_alloc_va_range;
>   	ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
> @@ -1040,7 +1081,30 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
>
>   	ppgtt->switch_mm = gen8_mm_switch;
>
> +	if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
> +		ret = __pdp_init(false, &ppgtt->pdp);
> +
> +		if (ret)
> +			goto free_scratch;
> +
> +		ppgtt->base.total = 1ULL << 32;
> +		if (IS_ENABLED(CONFIG_X86_32))
> +			/* While we have a proliferation of size_t variables
> +			 * we cannot represent the full ppgtt size on 32bit,
> +			 * so limit it to the same size as the GGTT (currently
> +			 * 2GiB).
> +			 */
> +			ppgtt->base.total = to_i915(ppgtt->base.dev)->gtt.base.total;
> +	} else {
> +		ppgtt->base.total = 1ULL << 48;
> +		return -EPERM; /* Not yet implemented */
> +	}
> +
>   	return 0;
> +
> +free_scratch:
> +	gen8_free_scratch(&ppgtt->base);
> +	return ret;
>   }
>
>   static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
> diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
> index d5bf953..e2b684e 100644
> --- a/drivers/gpu/drm/i915/i915_gem_gtt.h
> +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
> @@ -88,9 +88,17 @@ typedef uint64_t gen8_pde_t;
>    * PDPE  |  PDE  |  PTE  | offset
>    * The difference as compared to normal x86 3 level page table is the PDPEs are
>    * programmed via register.
> + *
> + * GEN8 48b legacy style address is defined as a 4 level page table:
> + * 47:39 | 38:30 | 29:21 | 20:12 |  11:0
> + * PML4E | PDPE  |  PDE  |  PTE  | offset
>    */
> +#define GEN8_PML4ES_PER_PML4		512
> +#define GEN8_PML4E_SHIFT		39
>   #define GEN8_PDPE_SHIFT			30
> -#define GEN8_PDPE_MASK			0x3
> +/* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page
> + * tables */
> +#define GEN8_PDPE_MASK			0x1ff
>   #define GEN8_PDE_SHIFT			21
>   #define GEN8_PDE_MASK			0x1ff
>   #define GEN8_PTE_SHIFT			12
> @@ -98,6 +106,9 @@ typedef uint64_t gen8_pde_t;
>   #define GEN8_LEGACY_PDPES		4
>   #define GEN8_PTES			I915_PTES(sizeof(gen8_pte_t))
>
> +#define I915_PDPES_PER_PDP(dev) (USES_FULL_48BIT_PPGTT(dev) ?\
> +				GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES)
> +
>   #define PPAT_UNCACHED_INDEX		(_PAGE_PWT | _PAGE_PCD)
>   #define PPAT_CACHED_PDE_INDEX		0 /* WB LLC */
>   #define PPAT_CACHED_INDEX		_PAGE_PAT /* WB LLCeLLC */
> @@ -241,9 +252,17 @@ struct i915_page_directory {
>   };
>
>   struct i915_page_directory_pointer {
> -	/* struct page *page; */
> -	DECLARE_BITMAP(used_pdpes, GEN8_LEGACY_PDPES);
> -	struct i915_page_directory *page_directory[GEN8_LEGACY_PDPES];
> +	struct i915_page_dma base;
> +
> +	unsigned long *used_pdpes;
> +	struct i915_page_directory **page_directory;
> +};
> +
> +struct i915_pml4 {
> +	struct i915_page_dma base;
> +
> +	DECLARE_BITMAP(used_pml4es, GEN8_PML4ES_PER_PML4);
> +	struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4];
>   };
>
>   struct i915_address_space {
> @@ -341,8 +360,9 @@ struct i915_hw_ppgtt {
>   	struct drm_mm_node node;
>   	unsigned long pd_dirty_rings;
>   	union {
> -		struct i915_page_directory_pointer pdp;
> -		struct i915_page_directory pd;
> +		struct i915_pml4 pml4;		/* GEN8+ & 48b PPGTT */
> +		struct i915_page_directory_pointer pdp;	/* GEN8+ */
> +		struct i915_page_directory pd;		/* GEN6-7 */
>   	};
>
>   	struct drm_i915_file_private *file_priv;
> @@ -436,14 +456,17 @@ static inline uint32_t gen6_pde_index(uint32_t addr)
>   	     temp = min(temp, length),					\
>   	     start += temp, length -= temp)
>
> -#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter)		\
> -	for (iter = gen8_pdpe_index(start);	\
> -	     pd = (pdp)->page_directory[iter], length > 0 && iter < GEN8_LEGACY_PDPES;	\
> +#define gen8_for_each_pdpe_e(pd, pdp, start, length, temp, iter, b)	\
> +	for (iter = gen8_pdpe_index(start); \
> +	     pd = (pdp)->page_directory[iter], length > 0 && (iter < b);	\
>   	     iter++,				\
>   	     temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT) - start,	\
>   	     temp = min(temp, length),					\
>   	     start += temp, length -= temp)
>
> +#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter)		\
> +	gen8_for_each_pdpe_e(pd, pdp, start, length, temp, iter, I915_PDPES_PER_PDP(dev))
> +
>   static inline uint32_t gen8_pte_index(uint64_t address)
>   {
>   	return i915_pte_index(address, GEN8_PDE_SHIFT);
>
Michel Thierry July 7, 2015, 12:56 p.m. UTC | #2
On 7/7/2015 1:36 PM, Goel, Akash wrote:
>
>
> On 7/1/2015 8:57 PM, Michel Thierry wrote:
>> This transitional patch doesn't do much for the existing code. However,
>> it should make upcoming patches to use the full 48b address space a bit
>> easier. The patch also introduces the PML4, ie. the new top level
>> structure
>> of the page tables.
>>
>
> Would be better to move the introduction of PML4 to a separate patch &
> keep this patch only for the dynamic allocation of pdp changes.
>
I'll move the PML4 declaration to a subsequent patch.

Thanks
diff mbox

Patch

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 1dbd957..7bccfd5 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2490,7 +2490,12 @@  struct drm_i915_cmd_table {
 #define HAS_HW_CONTEXTS(dev)	(INTEL_INFO(dev)->gen >= 6)
 #define HAS_LOGICAL_RING_CONTEXTS(dev)	(INTEL_INFO(dev)->gen >= 8)
 #define USES_PPGTT(dev)		(i915.enable_ppgtt)
-#define USES_FULL_PPGTT(dev)	(i915.enable_ppgtt == 2)
+#define USES_FULL_PPGTT(dev)	(i915.enable_ppgtt >= 2)
+#ifdef CONFIG_X86_64
+# define USES_FULL_48BIT_PPGTT(dev)	(i915.enable_ppgtt == 3)
+#else
+# define USES_FULL_48BIT_PPGTT(dev)	false
+#endif
 
 #define HAS_OVERLAY(dev)		(INTEL_INFO(dev)->has_overlay)
 #define OVERLAY_NEEDS_PHYSICAL(dev)	(INTEL_INFO(dev)->overlay_needs_physical)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 712ca34..cdcc778 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -104,9 +104,13 @@  static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 {
 	bool has_aliasing_ppgtt;
 	bool has_full_ppgtt;
+	bool has_full_64bit_ppgtt;
 
 	has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
 	has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
+	has_full_64bit_ppgtt = IS_ENABLED(CONFIG_X86_64) &&
+			       (IS_BROADWELL(dev) ||
+				INTEL_INFO(dev)->gen >= 9) && false; /* FIXME: 64b */
 
 	if (intel_vgpu_active(dev))
 		has_full_ppgtt = false; /* emulation is too hard */
@@ -125,6 +129,9 @@  static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 	if (enable_ppgtt == 2 && has_full_ppgtt)
 		return 2;
 
+	if (enable_ppgtt == 3 && has_full_64bit_ppgtt)
+		return 3;
+
 #ifdef CONFIG_INTEL_IOMMU
 	/* Disable ppgtt on SNB if VT-d is on. */
 	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
@@ -522,6 +529,45 @@  static void gen8_initialize_pd(struct i915_address_space *vm,
 	fill_px(vm->dev, pd, scratch_pde);
 }
 
+static int __pdp_init(struct drm_device *dev,
+		      struct i915_page_directory_pointer *pdp)
+{
+	size_t pdpes = I915_PDPES_PER_PDP(dev);
+
+	pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
+				  sizeof(unsigned long),
+				  GFP_KERNEL);
+	if (!pdp->used_pdpes)
+		return -ENOMEM;
+
+	pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
+				      GFP_KERNEL);
+	if (!pdp->page_directory) {
+		kfree(pdp->used_pdpes);
+		/* the PDP might be the statically allocated top level. Keep it
+		 * as clean as possible */
+		pdp->used_pdpes = NULL;
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void __pdp_fini(struct i915_page_directory_pointer *pdp)
+{
+	kfree(pdp->used_pdpes);
+	kfree(pdp->page_directory);
+	pdp->page_directory = NULL;
+}
+
+static void free_pdp(struct drm_device *dev,
+		     struct i915_page_directory_pointer *pdp)
+{
+	__pdp_fini(pdp);
+	if (USES_FULL_48BIT_PPGTT(dev))
+		kfree(pdp);
+}
+
 /* Broadwell Page Directory Pointer Descriptors */
 static int gen8_write_pdp(struct drm_i915_gem_request *req,
 			  unsigned entry,
@@ -634,9 +680,6 @@  static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 	pt_vaddr = NULL;
 
 	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
-		if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
-			break;
-
 		if (pt_vaddr == NULL) {
 			struct i915_page_directory *pd = ppgtt->pdp.page_directory[pdpe];
 			struct i915_page_table *pt = pd->page_table[pde];
@@ -720,7 +763,8 @@  static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 		container_of(vm, struct i915_hw_ppgtt, base);
 	int i;
 
-	for_each_set_bit(i, ppgtt->pdp.used_pdpes, GEN8_LEGACY_PDPES) {
+	for_each_set_bit(i, ppgtt->pdp.used_pdpes,
+				I915_PDPES_PER_PDP(ppgtt->base.dev)) {
 		if (WARN_ON(!ppgtt->pdp.page_directory[i]))
 			continue;
 
@@ -729,6 +773,7 @@  static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 		free_pd(ppgtt->base.dev, ppgtt->pdp.page_directory[i]);
 	}
 
+	free_pdp(ppgtt->base.dev, &ppgtt->pdp);
 	gen8_free_scratch(vm);
 }
 
@@ -820,8 +865,9 @@  static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,
 	struct i915_page_directory *pd;
 	uint64_t temp;
 	uint32_t pdpe;
+	uint32_t pdpes =  I915_PDPES_PER_PDP(ppgtt->base.dev);
 
-	WARN_ON(!bitmap_empty(new_pds, GEN8_LEGACY_PDPES));
+	WARN_ON(!bitmap_empty(new_pds, pdpes));
 
 	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
 		if (pd)
@@ -839,18 +885,19 @@  static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,
 	return 0;
 
 unwind_out:
-	for_each_set_bit(pdpe, new_pds, GEN8_LEGACY_PDPES)
+	for_each_set_bit(pdpe, new_pds, pdpes)
 		free_pd(dev, pdp->page_directory[pdpe]);
 
 	return -ENOMEM;
 }
 
 static void
-free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts)
+free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts,
+		       uint32_t pdpes)
 {
 	int i;
 
-	for (i = 0; i < GEN8_LEGACY_PDPES; i++)
+	for (i = 0; i < pdpes; i++)
 		kfree(new_pts[i]);
 	kfree(new_pts);
 	kfree(new_pds);
@@ -861,23 +908,24 @@  free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts)
  */
 static
 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
-					 unsigned long ***new_pts)
+					 unsigned long ***new_pts,
+					 uint32_t pdpes)
 {
 	int i;
 	unsigned long *pds;
 	unsigned long **pts;
 
-	pds = kcalloc(BITS_TO_LONGS(GEN8_LEGACY_PDPES), sizeof(unsigned long), GFP_KERNEL);
+	pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_KERNEL);
 	if (!pds)
 		return -ENOMEM;
 
-	pts = kcalloc(GEN8_LEGACY_PDPES, sizeof(unsigned long *), GFP_KERNEL);
+	pts = kcalloc(pdpes, sizeof(unsigned long *), GFP_KERNEL);
 	if (!pts) {
 		kfree(pds);
 		return -ENOMEM;
 	}
 
-	for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
+	for (i = 0; i < pdpes; i++) {
 		pts[i] = kcalloc(BITS_TO_LONGS(I915_PDES),
 				 sizeof(unsigned long), GFP_KERNEL);
 		if (!pts[i])
@@ -890,7 +938,7 @@  int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
 	return 0;
 
 err_out:
-	free_gen8_temp_bitmaps(pds, pts);
+	free_gen8_temp_bitmaps(pds, pts, pdpes);
 	return -ENOMEM;
 }
 
@@ -916,6 +964,7 @@  static int gen8_alloc_va_range(struct i915_address_space *vm,
 	const uint64_t orig_length = length;
 	uint64_t temp;
 	uint32_t pdpe;
+	uint32_t pdpes = I915_PDPES_PER_PDP(dev);
 	int ret;
 
 	/* Wrap is never okay since we can only represent 48b, and we don't
@@ -927,7 +976,7 @@  static int gen8_alloc_va_range(struct i915_address_space *vm,
 	if (WARN_ON(start + length > ppgtt->base.total))
 		return -ENODEV;
 
-	ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables);
+	ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
 	if (ret)
 		return ret;
 
@@ -935,7 +984,7 @@  static int gen8_alloc_va_range(struct i915_address_space *vm,
 	ret = gen8_ppgtt_alloc_page_directories(ppgtt, &ppgtt->pdp, start, length,
 					new_page_dirs);
 	if (ret) {
-		free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
+		free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
 		return ret;
 	}
 
@@ -989,7 +1038,7 @@  static int gen8_alloc_va_range(struct i915_address_space *vm,
 		__set_bit(pdpe, ppgtt->pdp.used_pdpes);
 	}
 
-	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
+	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
 	mark_tlbs_dirty(ppgtt);
 	return 0;
 
@@ -999,10 +1048,10 @@  err_out:
 			free_pt(vm->dev, ppgtt->pdp.page_directory[pdpe]->page_table[temp]);
 	}
 
-	for_each_set_bit(pdpe, new_page_dirs, GEN8_LEGACY_PDPES)
+	for_each_set_bit(pdpe, new_page_dirs, pdpes)
 		free_pd(vm->dev, ppgtt->pdp.page_directory[pdpe]);
 
-	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
+	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
 	mark_tlbs_dirty(ppgtt);
 	return ret;
 }
@@ -1023,14 +1072,6 @@  static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 		return ret;
 
 	ppgtt->base.start = 0;
-	ppgtt->base.total = 1ULL << 32;
-	if (IS_ENABLED(CONFIG_X86_32))
-		/* While we have a proliferation of size_t variables
-		 * we cannot represent the full ppgtt size on 32bit,
-		 * so limit it to the same size as the GGTT (currently
-		 * 2GiB).
-		 */
-		ppgtt->base.total = to_i915(ppgtt->base.dev)->gtt.base.total;
 	ppgtt->base.cleanup = gen8_ppgtt_cleanup;
 	ppgtt->base.allocate_va_range = gen8_alloc_va_range;
 	ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
@@ -1040,7 +1081,30 @@  static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 
 	ppgtt->switch_mm = gen8_mm_switch;
 
+	if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
+		ret = __pdp_init(false, &ppgtt->pdp);
+
+		if (ret)
+			goto free_scratch;
+
+		ppgtt->base.total = 1ULL << 32;
+		if (IS_ENABLED(CONFIG_X86_32))
+			/* While we have a proliferation of size_t variables
+			 * we cannot represent the full ppgtt size on 32bit,
+			 * so limit it to the same size as the GGTT (currently
+			 * 2GiB).
+			 */
+			ppgtt->base.total = to_i915(ppgtt->base.dev)->gtt.base.total;
+	} else {
+		ppgtt->base.total = 1ULL << 48;
+		return -EPERM; /* Not yet implemented */
+	}
+
 	return 0;
+
+free_scratch:
+	gen8_free_scratch(&ppgtt->base);
+	return ret;
 }
 
 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index d5bf953..e2b684e 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -88,9 +88,17 @@  typedef uint64_t gen8_pde_t;
  * PDPE  |  PDE  |  PTE  | offset
  * The difference as compared to normal x86 3 level page table is the PDPEs are
  * programmed via register.
+ *
+ * GEN8 48b legacy style address is defined as a 4 level page table:
+ * 47:39 | 38:30 | 29:21 | 20:12 |  11:0
+ * PML4E | PDPE  |  PDE  |  PTE  | offset
  */
+#define GEN8_PML4ES_PER_PML4		512
+#define GEN8_PML4E_SHIFT		39
 #define GEN8_PDPE_SHIFT			30
-#define GEN8_PDPE_MASK			0x3
+/* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page
+ * tables */
+#define GEN8_PDPE_MASK			0x1ff
 #define GEN8_PDE_SHIFT			21
 #define GEN8_PDE_MASK			0x1ff
 #define GEN8_PTE_SHIFT			12
@@ -98,6 +106,9 @@  typedef uint64_t gen8_pde_t;
 #define GEN8_LEGACY_PDPES		4
 #define GEN8_PTES			I915_PTES(sizeof(gen8_pte_t))
 
+#define I915_PDPES_PER_PDP(dev) (USES_FULL_48BIT_PPGTT(dev) ?\
+				GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES)
+
 #define PPAT_UNCACHED_INDEX		(_PAGE_PWT | _PAGE_PCD)
 #define PPAT_CACHED_PDE_INDEX		0 /* WB LLC */
 #define PPAT_CACHED_INDEX		_PAGE_PAT /* WB LLCeLLC */
@@ -241,9 +252,17 @@  struct i915_page_directory {
 };
 
 struct i915_page_directory_pointer {
-	/* struct page *page; */
-	DECLARE_BITMAP(used_pdpes, GEN8_LEGACY_PDPES);
-	struct i915_page_directory *page_directory[GEN8_LEGACY_PDPES];
+	struct i915_page_dma base;
+
+	unsigned long *used_pdpes;
+	struct i915_page_directory **page_directory;
+};
+
+struct i915_pml4 {
+	struct i915_page_dma base;
+
+	DECLARE_BITMAP(used_pml4es, GEN8_PML4ES_PER_PML4);
+	struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4];
 };
 
 struct i915_address_space {
@@ -341,8 +360,9 @@  struct i915_hw_ppgtt {
 	struct drm_mm_node node;
 	unsigned long pd_dirty_rings;
 	union {
-		struct i915_page_directory_pointer pdp;
-		struct i915_page_directory pd;
+		struct i915_pml4 pml4;		/* GEN8+ & 48b PPGTT */
+		struct i915_page_directory_pointer pdp;	/* GEN8+ */
+		struct i915_page_directory pd;		/* GEN6-7 */
 	};
 
 	struct drm_i915_file_private *file_priv;
@@ -436,14 +456,17 @@  static inline uint32_t gen6_pde_index(uint32_t addr)
 	     temp = min(temp, length),					\
 	     start += temp, length -= temp)
 
-#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter)		\
-	for (iter = gen8_pdpe_index(start);	\
-	     pd = (pdp)->page_directory[iter], length > 0 && iter < GEN8_LEGACY_PDPES;	\
+#define gen8_for_each_pdpe_e(pd, pdp, start, length, temp, iter, b)	\
+	for (iter = gen8_pdpe_index(start); \
+	     pd = (pdp)->page_directory[iter], length > 0 && (iter < b);	\
 	     iter++,				\
 	     temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT) - start,	\
 	     temp = min(temp, length),					\
 	     start += temp, length -= temp)
 
+#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter)		\
+	gen8_for_each_pdpe_e(pd, pdp, start, length, temp, iter, I915_PDPES_PER_PDP(dev))
+
 static inline uint32_t gen8_pte_index(uint64_t address)
 {
 	return i915_pte_index(address, GEN8_PDE_SHIFT);