[v2,08/15] drm/panthor: Add the MMU/VM logical block

Commit Message

Boris Brezillon Aug. 9, 2023, 4:53 p.m. UTC

MMU and VM management is related and placed in the same source file.

Page table updates are delegated to the io-pgtable-arm driver that's in
the iommu subsystem.

The VM management logic is based on drm_gpuva_mgr, and is assuming the
VA space is mostly managed by the usermode driver, except for a reserved
portion of this VA-space that's used for kernel objects (like the heap
contexts/chunks).

Both asynchronous and synchronous VM operations are supported, and
internal helpers are exposed to allow other logical blocks to map their
buffers in the GPU VA space.

There's one VM_BIND queue per-VM (meaning the Vulkan driver can only
expose one sparse-binding queue), and this bind queue is managed with
a 1:1 drm_sched_entity:drm_gpu_scheduler, such that each VM gets its own
independent execution queue, avoiding VM operation serialization at the
device level (things are still serialized at the VM level).

The rest is just implementation details that are hopefully well explained
in the documentation.

v2:
- Rename the driver (pancsf -> panthor)
- Change the license (GPL2 -> MIT + GPL2)
- Split the driver addition commit
- Document the code
- Use drm_gpuva_mgr
- Replace VM_MAP/UNMAP by VM_BIND
- Add support for asynchronous VM_BIND (VM_BIND queue implemented with
  drm_sched)
- Use drm_dev_{unplug,enter,exit}() to provide safe device removal
- Use the panthor_irq layer to manage/process IRQs

Signed-off-by: Boris Brezillon <[e-mail]>
---
 drivers/gpu/drm/panthor/panthor_mmu.c | 2611 +++++++++++++++++++++++++
 drivers/gpu/drm/panthor/panthor_mmu.h |   81 +
 2 files changed, 2692 insertions(+)
 create mode 100644 drivers/gpu/drm/panthor/panthor_mmu.c
 create mode 100644 drivers/gpu/drm/panthor/panthor_mmu.h

Comments

Steven Price Aug. 14, 2023, 3:53 p.m. UTC | #1

On 09/08/2023 17:53, Boris Brezillon wrote:
> MMU and VM management is related and placed in the same source file.
> 
> Page table updates are delegated to the io-pgtable-arm driver that's in
> the iommu subsystem.
> 
> The VM management logic is based on drm_gpuva_mgr, and is assuming the
> VA space is mostly managed by the usermode driver, except for a reserved
> portion of this VA-space that's used for kernel objects (like the heap
> contexts/chunks).
> 
> Both asynchronous and synchronous VM operations are supported, and
> internal helpers are exposed to allow other logical blocks to map their
> buffers in the GPU VA space.
> 
> There's one VM_BIND queue per-VM (meaning the Vulkan driver can only
> expose one sparse-binding queue), and this bind queue is managed with
> a 1:1 drm_sched_entity:drm_gpu_scheduler, such that each VM gets its own
> independent execution queue, avoiding VM operation serialization at the
> device level (things are still serialized at the VM level).
> 
> The rest is just implementation details that are hopefully well explained
> in the documentation.
> 
> v2:
> - Rename the driver (pancsf -> panthor)
> - Change the license (GPL2 -> MIT + GPL2)
> - Split the driver addition commit
> - Document the code
> - Use drm_gpuva_mgr
> - Replace VM_MAP/UNMAP by VM_BIND
> - Add support for asynchronous VM_BIND (VM_BIND queue implemented with
>   drm_sched)
> - Use drm_dev_{unplug,enter,exit}() to provide safe device removal
> - Use the panthor_irq layer to manage/process IRQs
> 
> Signed-off-by: Boris Brezillon <[e-mail]>
> ---
>  drivers/gpu/drm/panthor/panthor_mmu.c | 2611 +++++++++++++++++++++++++
>  drivers/gpu/drm/panthor/panthor_mmu.h |   81 +
>  2 files changed, 2692 insertions(+)
>  create mode 100644 drivers/gpu/drm/panthor/panthor_mmu.c
>  create mode 100644 drivers/gpu/drm/panthor/panthor_mmu.h
> 
> diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c
> new file mode 100644
> index 000000000000..3ba784473023
> --- /dev/null
> +++ b/drivers/gpu/drm/panthor/panthor_mmu.c
> @@ -0,0 +1,2611 @@
> +// SPDX-License-Identifier: GPL-2.0 or MIT
> +/* Copyright 2019 Linaro, Ltd, Rob Herring <[e-mail]> */
> +/* Copyright 2023 Collabora ltd. */
> +
> +#include <drm/drm_debugfs.h>
> +#include <drm/drm_drv.h>
> +#include <drm/drm_exec.h>
> +#include <drm/drm_gpuva_mgr.h>
> +#include <drm/drm_managed.h>
> +#include <drm/gpu_scheduler.h>
> +#include <drm/panthor_drm.h>
> +
> +#include <linux/atomic.h>
> +#include <linux/bitfield.h>
> +#include <linux/delay.h>
> +#include <linux/dma-mapping.h>
> +#include <linux/interrupt.h>
> +#include <linux/io.h>
> +#include <linux/iopoll.h>
> +#include <linux/io-pgtable.h>
> +#include <linux/iommu.h>
> +#include <linux/kmemleak.h>
> +#include <linux/platform_device.h>
> +#include <linux/pm_runtime.h>
> +#include <linux/rwsem.h>
> +#include <linux/shmem_fs.h>
> +#include <linux/sizes.h>
> +
> +#include "panthor_device.h"
> +#include "panthor_heap.h"
> +#include "panthor_mmu.h"
> +#include "panthor_sched.h"
> +#include "panthor_gem.h"
> +#include "panthor_regs.h"
> +
> +#define MAX_AS_SLOTS			32
> +
> +struct panthor_vm;
> +
> +/**
> + * struct panthor_as_slot - Address space slot
> + */
> +struct panthor_as_slot {
> +	/** @vm: VM bound to this slot. NULL is no VM is bound. */
> +	struct panthor_vm *vm;
> +
> +	/** @lock: Lock used to serialize access to the AS registers. */
> +	spinlock_t lock;
> +};
> +
> +/**
> + * struct panthor_mmu - MMU related data
> + */
> +struct panthor_mmu {
> +	/** @irq: The MMU irq. */
> +	struct panthor_irq irq;
> +
> +	/** @as: Address space related fields.
> +	 *
> +	 * The GPU has a limited number of address spaces (AS) slots, forcing
> +	 * us to re-assign them to re-assign slots on-demand.
> +	 */
> +	struct {
> +		/** @slots_lock: Lock protecting access to all other AS fields. */
> +		struct mutex slots_lock;
> +
> +		/** @alloc_mask: Bitmask encoding the allocated slots. */
> +		unsigned long alloc_mask;
> +
> +		/** @faulty_mask: Bitmask encoding the faulty slots. */
> +		unsigned long faulty_mask;
> +
> +		/** @slots: VMs currently bound to the AS slots. */
> +		struct panthor_as_slot slots[MAX_AS_SLOTS];
> +
> +		/**
> +		 * @lru_list: List of least recently used VMs.
> +		 *
> +		 * We use this list to pick a VM to evict when all slots are
> +		 * used.
> +		 *
> +		 * There should be no more active VMs than there are AS slots,
> +		 * so this LRU is just here to keep VMs bound until there's
> +		 * a need to release a slot, thus avoid unnecessary TLB/cache
> +		 * flushes.
> +		 */
> +		struct list_head lru_list;
> +	} as;
> +
> +	/** @vm: VMs management fields */
> +	struct {
> +		/** @lock: Lock protecting access to list. */
> +		struct mutex lock;
> +
> +		/** @list: List containing all VMs. */
> +		struct list_head list;
> +
> +		/** @reset_in_progress: True if a reset is in progress. */
> +		bool reset_in_progress;
> +
> +		/** @wq: Workqueue used for the VM_BIND queues. */
> +		struct workqueue_struct *wq;
> +	} vm;
> +};
> +
> +/**
> + * struct panthor_vm_pool - VM pool object
> + */
> +struct panthor_vm_pool {
> +	/** @xa: Array used for VM handle tracking. */
> +	struct xarray xa;
> +};
> +
> +/**
> + * struct panthor_vma - GPU mapping object
> + *
> + * This is used to track GEM mappings in GPU space.
> + */
> +struct panthor_vma {
> +	/** @base: Inherits from drm_gpuva. */
> +	struct drm_gpuva base;
> +
> +	/** @node: Used to insert the mapping in the panthor_vm::shared_bos list. */
> +	struct list_head node;
> +
> +	/**
> +	 * @flags: Combination of drm_panthor_vm_bind_op_flags.
> +	 *
> +	 * Only map related flags are accepted.
> +	 */
> +	u32 flags;
> +};
> +
> +/**
> + * struct panthor_vm_op_ctx - VM operation context
> + *
> + * With VM operations potentially taking place in a dma-signaling path, we
> + * need to make sure everything that might require resource allocation is
> + * pre-allocated upfront. This is what this operation context is far.
> + *
> + * We also collect resources that have been freed, so we can release them
> + * asynchronously, and let the VM_BIND scheduler process the next VM_BIND
> + * request.
> + */
> +struct panthor_vm_op_ctx {
> +	/** @rsvd_page_tables: Pages reserved for the MMU page table update. */
> +	struct {
> +		/** @count: Number of pages reserved. */
> +		u32 count;
> +
> +		/** @ptr: Point to the first unused page in the @pages table. */
> +		u32 ptr;
> +
> +		/**
> +		 * @page: Array of pages that can be used for an MMU page table update.
> +		 *
> +		 * After an VM operation, there might be free pages left in this array.
> +		 * They should be returned to the pt_cache as part of the op_ctx cleanup.
> +		 */
> +		void **pages;
> +	} rsvd_page_tables;

Two questions:

1) Would a mempool simplify the implementation? It looks like a
reasonable match.

2) Does it really make sense to have a separate pool of memory for every
operation? Instead of having a separate pool for each operation, it
would be possible to just keep track of the total number needed for all
outstanding operations. Then a single (per device or maybe per-VM if
necessary) mempool could be resized to ensure it has the right amount of
space.

I'm also a little wary that the VM_BIND infrastructure could potentially
be abused to trigger a large amount of kernel allocation as it allocates
up-front for the worst case but those pages are not charged to the
process (AFAICT). But I haven't fully got my head round that yet.

> +
> +	/** @flags: Combination of drm_panthor_vm_bind_op_flags. */
> +	u32 flags;
> +
> +	/** @va: Virtual range targeted by the VM operation. */
> +	struct {
> +		/** @addr: Start address. */
> +		u64 addr;
> +
> +		/** @range: Range size. */
> +		u64 range;
> +	} va;
> +
> +	/**
> +	 * @returned_vmas: List of panthor_vma objects returned after a VM operation.
> +	 *
> +	 * For unmap operations, this will contain all VMAs that were covered by the
> +	 * specified VA range.
> +	 *
> +	 * For map operations, this will contain all VMAs that previously mapped to
> +	 * the specified VA range.
> +	 *
> +	 * Those VMAs, and the resources they point to will be released as part of
> +	 * the op_ctx cleanup operation.
> +	 */
> +	struct list_head returned_vmas;
> +
> +	/** @map: Fields specific to a map operation. */
> +	struct {
> +		/** @gem: GEM object information. */
> +		struct {
> +			/** @obj: GEM object to map. */
> +			struct drm_gem_object *obj;
> +
> +			/** @offset: Offset in the GEM object. */
> +			u64 offset;
> +		} gem;
> +
> +		/**
> +		 * @sgt: sg-table pointing to pages backing the GEM object.
> +		 *
> +		 * This is gathered at job creation time, such that we don't have
> +		 * to allocate in ::run_job().
> +		 */
> +		struct sg_table *sgt;
> +
> +		/**
> +		 * @prev_vma: Pre-allocated VMA object to deal with a remap situation.
> +		 *
> +		 * If the map request covers a region that's inside another VMA, the
> +		 * previous VMA will be split, requiring instantiation of a maximum of
> +		 * two new VMA objects.
> +		 */
> +		struct panthor_vma *prev_vma;
> +
> +		/**
> +		 * @new_vma: The new VMA object that will be inserted to the VA tree.
> +		 */
> +		struct panthor_vma *new_vma;
> +
> +		/**
> +		 * @next_vma: Pre-allocated VMA object to deal with a remap situation.
> +		 *
> +		 * See @prev_vma.
> +		 */
> +		struct panthor_vma *next_vma;

It's probably premature optimization, but it feels like having a cache
of these VMA structures might be an idea. I'm also struggling to
understand how both a new prev and new next VMA are needed - but I
haven't dug into the GPU VA manager.

> +	} map;
> +};
> +
> +/**
> + * struct panthor_vm - VM object
> + *
> + * A VM is an object representing a GPU (or MCU) virtual address space.
> + * It embeds the MMU page table for this address space, a tree containing
> + * all the virtual mappings of GEM objects, and other things needed to manage
> + * the VM.
> + *
> + * Except for the MCU VM, which is managed by the kernel, all other VMs are
> + * created by userspace and mostly managed by userspace, using the
> + * %DRM_IOCTL_PANTHOR_VM_BIND ioctl.
> + *
> + * A portion of the virtual address space is reserved for kernel objects,
> + * like heap chunks, and userspace gets to decide how much of the virtual
> + * address space is left to the kernel (half of the virtual address space
> + * by default).
> + */
> +struct panthor_vm {
> +	/**
> +	 * @va_mgr: GPU VA manager.
> +	 *
> +	 * We delegate all the VA management to the common drm_gpuva_mgr framework
> +	 * and only implement hooks to update the MMU page table.
> +	 */
> +	struct drm_gpuva_manager va_mgr;
> +
> +	/**
> +	 * @sched: Scheduler used for asynchronous VM_BIND request.
> +	 *
> +	 * We use a 1:1 scheduler here.
> +	 */
> +	struct drm_gpu_scheduler sched;
> +
> +	/**
> +	 * @entity: Scheduling entity representing the VM_BIND queue.
> +	 *
> +	 * There's currently one bind queue per VM. It doesn't make sense to
> +	 * allow more given the VM operations are serialized anyway.
> +	 */
> +	struct drm_sched_entity entity;
> +
> +	/** @ptdev: Device. */
> +	struct panthor_device *ptdev;
> +
> +	/** @refcount: Reference count. */
> +	struct kref refcount;
> +
> +	/** @memattr: Value to program to the AS_MEMATTR register. */
> +	u64 memattr;
> +
> +	/** @pgtbl_ops: Page table operations. */
> +	struct io_pgtable_ops *pgtbl_ops;
> +
> +	/**
> +	 * @dummy_gem: Used as a VM reservation object.
> +	 *
> +	 * We declare a drm_gem_object and no a dma_resv, so we can use drm_exec()
> +	 * for the VM reservation.
> +	 *
> +	 * All private BOs use the resv of this dummy GEM object instead of
> +	 * drm_gem_object::_resv, such that private GEM preparation is O(1)
> +	 * instead of O(N).
> +	 */
> +	struct drm_gem_object dummy_gem;
> +
> +	/**
> +	 * @op_lock: Lock used to serialize operations on a VM.
> +	 *
> +	 * The serialization of jobs queued to the VM_BIND queue is already
> +	 * taken care of by drm_sched, but we need to serialize synchronous
> +	 * and asynchronous VM_BIND request. This is what this lock is for.
> +	 */
> +	struct mutex op_lock;
> +
> +	/**
> +	 * @op_ctx: The context attached to the currently executing VM operation.
> +	 *
> +	 * NULL when no operation is in progress.
> +	 */
> +	struct panthor_vm_op_ctx *op_ctx;
> +
> +	/**
> +	 * @shared_bos: List of shared BOs.
> +	 *
> +	 * Shared BOs don't use the VM resv, and need to be prepared
> +	 * independently. This list keeps track of all VMAs that target
> +	 * non-private BOs.
> +	 *
> +	 * There might be duplicates, but drm_exec and dma_resv should
> +	 * handle that for us.
> +	 *
> +	 * TODO: This is not optimal. We should probably switch to the
> +	 * drm_gpuva_mgr solution for handling shared BOs once it's
> +	 * ready.
> +	 */
> +	struct list_head shared_bos;
> +
> +	/**
> +	 * @mm: Memory management object representing the auto-VA/kernel-VA.
> +	 *
> +	 * Used to auto-allocate VA space for kernel-managed objects (tiler
> +	 * heaps, ...).
> +	 *
> +	 * For the MCU VM, this is managing the VA range that's used to map
> +	 * all shared interfaces.
> +	 *
> +	 * For user VMs, the range is specified by userspace, and must not
> +	 * exceed half of the VA space addressable.
> +	 */
> +	struct drm_mm mm;
> +
> +	/** @mm_lock: Lock protecting the @mm field. */
> +	struct mutex mm_lock;
> +
> +	/** @as: Address space related fields. */
> +	struct {
> +		/**
> +		 * @id: ID of the address space this VM is bound to.
> +		 *
> +		 * A value of -1 means the VM is inactive/not bound.
> +		 */
> +		int id;
> +
> +		/**
> +		 * @lru_node: Used to instead the VM in the panthor_mmu::as::lru_list.
> +		 *
> +		 * Active VMs should not be inserted in the LRU list.
> +		 */
> +		struct list_head lru_node;
> +	} as;
> +
> +	/**
> +	 * @heaps: Tiler heap related fields.
> +	 */
> +	struct {
> +		/**
> +		 * @pool: The heap pool attached to this VM.
> +		 *
> +		 * Will stay NULL until someone creates a heap context on this VM.
> +		 */
> +		struct panthor_heap_pool *pool;
> +
> +		/** @lock: Lock used to protect access to @pool. */
> +		struct mutex lock;
> +	} heaps;
> +
> +	/** @node: Used to insert the VM in the panthor_mmu::vm::list. */
> +	struct list_head node;
> +
> +	/** @for_mcu: True if this is the MCU VM. */
> +	bool for_mcu;
> +
> +	/**
> +	 * @destroyed: True if the VM was destroyed.
> +	 *
> +	 * No further bind requests should be queued to a destroyed VM.
> +	 */
> +	bool destroyed;
> +
> +	/**
> +	 * @unusable: True if the VM has turned unusable because something
> +	 * bad happened during an asynchronous request.
> +	 *
> +	 * We don't try to recover from such failures, because this implies
> +	 * informing userspace about the specific operation that failed, and
> +	 * hoping the userspace driver can replay things from there. This all
> +	 * sounds very complicated for little gain.
> +	 *
> +	 * Instead, we should just flag the VM as unusable, and fail any
> +	 * further request targeting this VM.
> +	 *
> +	 * We also provide a way to query a VM state, so userspace can destroy
> +	 * it and create a new one.
> +	 *
> +	 * As an analogy, this would be mapped to a VK_ERROR_DEVICE_LOST
> +	 * situation, where the logical device needs to be re-created.
> +	 */
> +	bool unusable;
> +};
> +
> +/**
> + * struct panthor_vm_bind_job - VM bind job
> + */
> +struct panthor_vm_bind_job {
> +	/** @base: Inherit from drm_sched_job. */
> +	struct drm_sched_job base;
> +
> +	/** @refcount: Reference count. */
> +	struct kref refcount;
> +
> +	/** @cleanup_op_ctx_work: Work used to cleanup the VM operation context. */
> +	struct work_struct cleanup_op_ctx_work;
> +
> +	/** @vm: VM targeted by the VM operation. */
> +	struct panthor_vm *vm;
> +
> +	/** @ctx: Operation context. */
> +	struct panthor_vm_op_ctx ctx;
> +};
> +
> +/**
> + * @pt_cache: Cache used to allocate MMU page tables.
> + *
> + * The pre-allocation pattern forces us to over-allocate to plan for
> + * the worst case scenario, and return the pages we didn't use.
> + *
> + * Having a kmem_cache allows us to speed allocations.
> + */
> +static struct kmem_cache *pt_cache;
> +
> +/**
> + * alloc_pt() - Custom page table allocator
> + * @cookie: Cookie passed at page table allocation time.
> + * @size: Size of the page table. This size should be fixed,
> + * and determined at creation time based on the granule size.
> + * @gfp: GFP flags.
> + *
> + * We want a custom allocator so we can use a cache for page table
> + * allocations and amortize the cost of the over-reservation that's
> + * done to allow asynchronous VM operations.
> + *
> + * Return: non-NULL on success, NULL if the allocation failed for any
> + * reason.
> + */
> +static void *alloc_pt(void *cookie, size_t size, gfp_t gfp)
> +{
> +	struct panthor_vm *vm = cookie;
> +	void *page;
> +
> +	/* We're not supposed to have anything bigger than 4k here, because we picked a
> +	 * 4k granule size at init time.
> +	 */
> +	if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
> +		return NULL;
> +
> +	/* Allocation of the root page table happening during init. */
> +	if (!vm->pgtbl_ops) {
> +		drm_WARN_ON(&vm->ptdev->base, vm->op_ctx);
> +		page = kmem_cache_alloc(pt_cache, gfp);
> +		goto out;
> +	}
> +
> +	/* We must have some op_ctx attached to the VM and it must have at least one
> +	 * free page.
> +	 */
> +	if (drm_WARN_ON(&vm->ptdev->base, !vm->op_ctx) ||
> +	    drm_WARN_ON(&vm->ptdev->base,
> +			vm->op_ctx->rsvd_page_tables.ptr >= vm->op_ctx->rsvd_page_tables.count))
> +		return NULL;
> +
> +	page = vm->op_ctx->rsvd_page_tables.pages[vm->op_ctx->rsvd_page_tables.ptr++];
> +	memset(page, 0, SZ_4K);
> +
> +out:
> +	/* Page table entries don't use virtual addresses, which trips out
> +	 * kmemleak. kmemleak_alloc_phys() might work, but physical addresses
> +	 * are mixed with other fields, and I fear kmemleak won't detect that
> +	 * either.
> +	 *
> +	 * Let's just ignore memory passed to the page-table driver for now.
> +	 */
> +	kmemleak_ignore(page);
> +	return page;
> +}
> +
> +/**
> + * @free_pt() - Custom page table free function
> + * @cookie: Cookie passed at page table allocation time.
> + * @data: Page table to free.
> + * @size: Size of the page table. This size should be fixed,
> + * and determined at creation time based on the granule size.
> + */
> +static void free_pt(void *cookie, void *data, size_t size)
> +{
> +	struct panthor_vm *vm = cookie;
> +
> +	if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
> +		return;
> +
> +	/* Return the page to the pt_cache. */
> +	kmem_cache_free(pt_cache, data);
> +}
> +
> +static int wait_ready(struct panthor_device *ptdev, u32 as_nr)
> +{
> +	int ret;
> +	u32 val;
> +
> +	/* Wait for the MMU status to indicate there is no active command, in
> +	 * case one is pending.
> +	 */
> +	ret = readl_relaxed_poll_timeout_atomic(ptdev->iomem + AS_STATUS(as_nr),
> +						val, !(val & AS_STATUS_AS_ACTIVE),
> +						10, 100000);
> +
> +	if (ret) {
> +		panthor_device_schedule_reset(ptdev);
> +		drm_err(&ptdev->base, "AS_ACTIVE bit stuck\n");
> +	}
> +
> +	return ret;
> +}
> +
> +static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd)
> +{
> +	int status;
> +
> +	/* write AS_COMMAND when MMU is ready to accept another command */
> +	status = wait_ready(ptdev, as_nr);
> +	if (!status)
> +		gpu_write(ptdev, AS_COMMAND(as_nr), cmd);
> +
> +	return status;
> +}
> +
> +static void lock_region(struct panthor_device *ptdev, u32 as_nr,
> +			u64 region_start, u64 size)
> +{
> +	u8 region_width;
> +	u64 region;
> +	u64 region_end = region_start + size;
> +
> +	if (!size)
> +		return;
> +
> +	/*
> +	 * The locked region is a naturally aligned power of 2 block encoded as
> +	 * log2 minus(1).
> +	 * Calculate the desired start/end and look for the highest bit which
> +	 * differs. The smallest naturally aligned block must include this bit
> +	 * change, the desired region starts with this bit (and subsequent bits)
> +	 * zeroed and ends with the bit (and subsequent bits) set to one.
> +	 */
> +	region_width = max(fls64(region_start ^ (region_end - 1)),
> +			   const_ilog2(AS_LOCK_REGION_MIN_SIZE)) - 1;
> +
> +	/*
> +	 * Mask off the low bits of region_start (which would be ignored by
> +	 * the hardware anyway)
> +	 */
> +	region_start &= GENMASK_ULL(63, region_width);
> +
> +	region = region_width | region_start;
> +
> +	/* Lock the region that needs to be updated */
> +	gpu_write(ptdev, AS_LOCKADDR_LO(as_nr), lower_32_bits(region));
> +	gpu_write(ptdev, AS_LOCKADDR_HI(as_nr), upper_32_bits(region));
> +	write_cmd(ptdev, as_nr, AS_COMMAND_LOCK);
> +}
> +
> +static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr,
> +				      u64 iova, u64 size, u32 op)
> +{
> +	if (as_nr < 0)
> +		return 0;
> +
> +	if (op != AS_COMMAND_UNLOCK)
> +		lock_region(ptdev, as_nr, iova, size);
> +
> +	/* Run the MMU operation */
> +	write_cmd(ptdev, as_nr, op);
> +
> +	/* Wait for the flush to complete */
> +	return wait_ready(ptdev, as_nr);
> +}
> +
> +static int mmu_hw_do_operation(struct panthor_vm *vm,
> +			       u64 iova, u64 size, u32 op)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +	int ret;
> +
> +	spin_lock(&ptdev->mmu->as.slots[vm->as.id].lock);
> +	ret = mmu_hw_do_operation_locked(ptdev, vm->as.id, iova, size, op);
> +	spin_unlock(&ptdev->mmu->as.slots[vm->as.id].lock);
> +	return ret;
> +}
> +
> +static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
> +				 u64 transtab, u64 transcfg, u64 memattr)
> +{
> +	int ret;
> +
> +	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
> +	if (ret)
> +		return ret;
> +
> +	gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), lower_32_bits(transtab));
> +	gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), upper_32_bits(transtab));
> +
> +	gpu_write(ptdev, AS_MEMATTR_LO(as_nr), lower_32_bits(memattr));
> +	gpu_write(ptdev, AS_MEMATTR_HI(as_nr), upper_32_bits(memattr));
> +
> +	gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), lower_32_bits(transcfg));
> +	gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), upper_32_bits(transcfg));
> +
> +	return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
> +}
> +
> +static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr)
> +{
> +	int ret;
> +
> +	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
> +	if (ret)
> +		return ret;
> +
> +	gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), 0);
> +	gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), 0);
> +
> +	gpu_write(ptdev, AS_MEMATTR_LO(as_nr), 0);
> +	gpu_write(ptdev, AS_MEMATTR_HI(as_nr), 0);
> +
> +	gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED);
> +	gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), 0);
> +
> +	return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
> +}
> +
> +static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value)
> +{
> +	/* Bits 16 to 31 mean REQ_COMPLETE. */
> +	return value & GENMASK(15, 0);
> +}
> +
> +static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)
> +{
> +	return BIT(as);
> +}
> +
> +/**
> + * panthor_vm_active() - Flag a VM as active
> + * @VM: VM to flag as active.
> + *
> + * Assigns an address space to a VM so it can be used by the GPU/MCU.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_active(struct panthor_vm *vm)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +	struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
> +	int ret = 0, as, cookie;
> +	u64 transtab, transcfg;
> +
> +	if (!drm_dev_enter(&ptdev->base, &cookie))
> +		return -ENODEV;
> +
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +
> +	as = vm->as.id;
> +	if (as >= 0) {
> +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
> +
> +		if (ptdev->mmu->as.faulty_mask & mask) {
> +			/* Unhandled pagefault on this AS, the MMU was
> +			 * disabled. We need to re-enable the MMU after
> +			 * clearing+unmasking the AS interrupts.
> +			 */
> +			gpu_write(ptdev, MMU_INT_CLEAR, mask);
> +			ptdev->mmu->as.faulty_mask &= ~mask;
> +			gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
> +			goto out_enable_as;
> +		}
> +
> +		goto out_unlock;
> +	}
> +
> +	/* Check for a free AS */
> +	if (vm->for_mcu) {
> +		drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));
> +		as = 0;
> +	} else {
> +		as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));
> +	}
> +
> +	if (!(BIT(as) & ptdev->gpu_info.as_present)) {
> +		struct panthor_vm *lru_vm;
> +
> +		lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,
> +						  struct panthor_vm,
> +						  as.lru_node);
> +		if (drm_WARN_ON(&ptdev->base, !lru_vm)) {
> +			ret = -EBUSY;
> +			goto out_unlock;
> +		}
> +
> +		list_del_init(&lru_vm->as.lru_node);
> +		as = lru_vm->as.id;

Should this not set lru_vm->as.id = -1, so that the code knows the VM no
longer has an address space?

> +	} else {
> +		set_bit(as, &ptdev->mmu->as.alloc_mask);
> +	}
> +
> +	/* Assign the free or reclaimed AS to the FD */
> +	vm->as.id = as;
> +	ptdev->mmu->as.slots[as].vm = vm;
> +
> +out_enable_as:
> +	transtab = cfg->arm_lpae_s1_cfg.ttbr;
> +	transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |
> +		   AS_TRANSCFG_PTW_RA |
> +		   AS_TRANSCFG_ADRMODE_AARCH64_4K;
> +	if (ptdev->coherent)
> +		transcfg |= AS_TRANSCFG_PTW_SH_OS;
> +
> +	ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
> +
> +out_unlock:
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +	drm_dev_exit(cookie);
> +	return ret;
> +}
> +
> +/**
> + * panthor_vm_idle() - Flag a VM idle
> + * @VM: VM to flag as idle.
> + *
> + * When we know the GPU is done with the VM (no more jobs to process),
> + * we can relinquish the AS slot attached to this VM, if any.
> + *
> + * We don't release the slot immediately, but instead place the VM in
> + * the LRU list, so it can be evicted if another VM needs an AS slot.
> + * This way, VMs keep attached to the AS they were given until we run
> + * out of free slot, limiting the number of MMU operations (TLB flush
> + * and other AS updates).
> + */
> +void panthor_vm_idle(struct panthor_vm *vm)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +	if (vm->as.id >= 0 && list_empty(&vm->as.lru_node))
> +		list_add_tail(&vm->as.lru_node, &ptdev->mmu->as.lru_list);
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +}
> +
> +static void panthor_vm_stop(struct panthor_vm *vm)
> +{
> +	drm_sched_stop(&vm->sched, NULL);
> +}
> +
> +static void panthor_vm_start(struct panthor_vm *vm)
> +{
> +	drm_sched_start(&vm->sched, true);
> +}
> +
> +/**
> + * panthor_vm_as() - Get the AS slot attached to a VM
> + * @vm: VM to get the AS slot of.
> + *
> + * Return: -1 if the VM is not assigned an AS slot yet, >= 0 otherwise.
> + */
> +int panthor_vm_as(struct panthor_vm *vm)
> +{
> +	return vm->as.id;
> +}
> +
> +static size_t get_pgsize(u64 addr, size_t size, size_t *count)
> +{
> +	/*
> +	 * io-pgtable only operates on multiple pages within a single table
> +	 * entry, so we need to split at boundaries of the table size, i.e.
> +	 * the next block size up. The distance from address A to the next
> +	 * boundary of block size B is logically B - A % B, but in unsigned
> +	 * two's complement where B is a power of two we get the equivalence
> +	 * B - A % B == (B - A) % B == (n * B - A) % B, and choose n = 0 :)
> +	 */
> +	size_t blk_offset = -addr % SZ_2M;
> +
> +	if (blk_offset || size < SZ_2M) {
> +		*count = min_not_zero(blk_offset, size) / SZ_4K;
> +		return SZ_4K;
> +	}
> +	blk_offset = -addr % SZ_1G ?: SZ_1G;
> +	*count = min(blk_offset, size) / SZ_2M;
> +	return SZ_2M;
> +}
> +
> +static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +	int ret = 0, cookie;
> +
> +	if (vm->as.id < 0)
> +		return 0;
> +
> +	/* If the device is unplugged, we just silently skip the flush. */
> +	if (!drm_dev_enter(&ptdev->base, &cookie))
> +		return 0;
> +
> +	/* Flush the PTs only if we're already awake */
> +	if (pm_runtime_active(ptdev->base.dev))
> +		ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT);
> +
> +	drm_dev_exit(cookie);
> +	return ret;
> +}
> +
> +static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, size_t size)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +	struct io_pgtable_ops *ops = vm->pgtbl_ops;
> +	size_t offset = 0;
> +
> +	drm_dbg(&ptdev->base, "unmap: as=%d, iova=%llx, len=%zx", vm->as.id, iova, size);
> +
> +	while (offset < size) {
> +		size_t unmapped_sz = 0, pgcount;
> +		size_t pgsize = get_pgsize(iova + offset, size - offset, &pgcount);
> +
> +		unmapped_sz = ops->unmap_pages(ops, iova + offset, pgsize, pgcount, NULL);
> +
> +		if (drm_WARN_ON(&ptdev->base, unmapped_sz != pgsize * pgcount)) {
> +			drm_err(&ptdev->base, "failed to unmap range %llx-%llx (requested range %llx-%llx)\n",
> +				iova + offset + unmapped_sz,
> +				iova + offset + pgsize * pgcount,
> +				iova, iova + size);
> +			panthor_vm_flush_range(vm, iova, offset + unmapped_sz);
> +			return  -EINVAL;
> +		}
> +		offset += unmapped_sz;
> +	}
> +
> +	return panthor_vm_flush_range(vm, iova, size);
> +}
> +
> +static int
> +panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
> +		     struct sg_table *sgt, u64 offset, ssize_t size)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +	unsigned int count;
> +	struct scatterlist *sgl;
> +	struct io_pgtable_ops *ops = vm->pgtbl_ops;
> +	u64 start_iova = iova;
> +	int ret;
> +
> +	if (!size)
> +		return 0;
> +
> +	for_each_sgtable_dma_sg(sgt, sgl, count) {
> +		dma_addr_t paddr = sg_dma_address(sgl);
> +		size_t len = sg_dma_len(sgl);
> +
> +		if (len <= offset) {
> +			offset -= len;
> +			continue;
> +		}
> +
> +		paddr -= offset;
> +		len -= offset;
> +
> +		if (size >= 0) {
> +			len = min_t(size_t, len, size);
> +			size -= len;
> +		}
> +
> +		drm_dbg(&ptdev->base, "map: as=%d, iova=%llx, paddr=%llx, len=%zx",
> +			vm->as.id, iova, paddr, len);
> +
> +		while (len) {
> +			size_t pgcount, mapped = 0;
> +			size_t pgsize = get_pgsize(iova | paddr, len, &pgcount);
> +
> +			ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot,
> +					     GFP_KERNEL, &mapped);
> +			iova += mapped;
> +			paddr += mapped;
> +			len -= mapped;
> +
> +			if (drm_WARN_ON(&ptdev->base, !ret && !mapped))
> +				ret = -ENOMEM;
> +
> +			if (ret) {
> +				/* If something failed, unmap what we've already mapped before
> +				 * returning. The unmap call is not supposed to fail.
> +				 */
> +				drm_WARN_ON(&ptdev->base,
> +					    panthor_vm_unmap_pages(vm, start_iova,
> +								   iova - start_iova));
> +				return ret;
> +			}
> +		}
> +
> +		if (!size)
> +			break;
> +	}
> +
> +	return panthor_vm_flush_range(vm, start_iova, iova - start_iova);
> +}
> +
> +static int flags_to_prot(u32 flags)
> +{
> +	int prot = 0;
> +
> +	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC)
> +		prot |= IOMMU_NOEXEC;
> +
> +	if (!(flags & DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED))
> +		prot |= IOMMU_CACHE;
> +
> +	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_READONLY)
> +		prot |= IOMMU_READ;
> +	else
> +		prot |= IOMMU_READ | IOMMU_WRITE;
> +
> +	return prot;
> +}
> +
> +/**
> + * panthor_vm_alloc_va() - Allocate a region in the auto-va space
> + * @VM: VM to allocate a region on.
> + * @size: Size of the region.
> + *
> + * Some GPU objects, like heap chunks, are fully managed by the kernel and
> + * need to be mapped to the userspace VM, in the region reserved for kernel
> + * objects.
> + *
> + * This function takes care of allocating a region in this reserved space.
> + *
> + * Return: A valid pointer on success, and ERR_PTR() otherwise.
> + */
> +struct drm_mm_node *
> +panthor_vm_alloc_va(struct panthor_vm *vm, size_t size)
> +{
> +	struct drm_mm_node *mm_node;
> +	int ret;
> +
> +	if (!size || (size & ~PAGE_MASK))
> +		return ERR_PTR(-EINVAL);
> +
> +	mm_node = kzalloc(sizeof(*mm_node), GFP_KERNEL);
> +	if (!mm_node)
> +		return ERR_PTR(-ENOMEM);
> +
> +	mutex_lock(&vm->mm_lock);
> +	ret = drm_mm_insert_node(&vm->mm, mm_node, size);
> +	mutex_unlock(&vm->mm_lock);
> +
> +	if (ret) {
> +		kfree(mm_node);
> +		return ERR_PTR(ret);
> +	}
> +
> +	return mm_node;
> +}
> +
> +/**
> + * panthor_vm_free_va() - Free a region allocated with panthor_vm_alloc_va()
> + * @VM: VM to free the region on.
> + * @mm_node: Memory node representing the region to free.
> + */
> +void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *mm_node)
> +{
> +	if (!mm_node)
> +		return;
> +
> +	mutex_lock(&vm->mm_lock);
> +	drm_mm_remove_node(mm_node);
> +	mutex_unlock(&vm->mm_lock);
> +
> +	kfree(mm_node);
> +}
> +
> +static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,
> +				      struct panthor_vm *vm)
> +{
> +	struct panthor_vma *vma, *tmp_vma;
> +
> +	u32 remaining_pt_count = op_ctx->rsvd_page_tables.count -
> +				 op_ctx->rsvd_page_tables.ptr;
> +
> +	if (remaining_pt_count) {
> +		kmem_cache_free_bulk(pt_cache, remaining_pt_count,
> +				     op_ctx->rsvd_page_tables.pages +
> +				     op_ctx->rsvd_page_tables.ptr);
> +	}
> +
> +	kfree(op_ctx->rsvd_page_tables.pages);
> +	memset(&op_ctx->rsvd_page_tables, 0, sizeof(op_ctx->rsvd_page_tables));
> +
> +	if (op_ctx->map.gem.obj) {
> +		struct panthor_gem_object *bo = to_panthor_bo(op_ctx->map.gem.obj);
> +
> +		if (!bo->base.base.import_attach)
> +			drm_gem_shmem_unpin(&bo->base);
> +
> +		drm_gem_object_put(&bo->base.base);
> +	}
> +
> +	kfree(op_ctx->map.new_vma);
> +	kfree(op_ctx->map.next_vma);
> +	kfree(op_ctx->map.prev_vma);
> +	memset(&op_ctx->map, 0, sizeof(op_ctx->map));
> +
> +	list_for_each_entry_safe(vma, tmp_vma, &op_ctx->returned_vmas, node) {
> +		struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
> +
> +		if (!bo->base.base.import_attach)
> +			drm_gem_shmem_unpin(&bo->base);
> +
> +		drm_gem_object_put(&bo->base.base);
> +		list_del(&vma->node);
> +		kfree(vma);
> +	}
> +}
> +
> +#define PANTHOR_VM_BIND_OP_MAP_FLAGS \
> +	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
> +	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
> +	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \
> +	 DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
> +
> +static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
> +					 struct panthor_vm *vm,
> +					 struct panthor_gem_object *bo,
> +					 u64 offset,
> +					 size_t size, u64 va,
> +					 u32 flags)
> +{
> +	struct sg_table *sgt = NULL;
> +	u64 pt_count;
> +	int ret;
> +
> +	if (!bo)
> +		return -EINVAL;
> +
> +	if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) ||
> +	    (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP)
> +		return -EINVAL;
> +
> +	/* Make sure the VA and size are aligned and in-bounds. */
> +	if (size > bo->base.base.size || offset > bo->base.base.size - size)
> +		return -EINVAL;
> +
> +	/* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */
> +	if (bo->exclusive_vm && bo->exclusive_vm != vm)
> +		return -EINVAL;
> +
> +	memset(op_ctx, 0, sizeof(*op_ctx));
> +	INIT_LIST_HEAD(&op_ctx->returned_vmas);
> +	op_ctx->flags = flags;
> +	op_ctx->va.range = size;
> +	op_ctx->va.addr = va;
> +
> +	op_ctx->map.new_vma = kzalloc(sizeof(*op_ctx->map.new_vma), GFP_KERNEL);
> +	op_ctx->map.next_vma = kzalloc(sizeof(*op_ctx->map.next_vma), GFP_KERNEL);
> +	op_ctx->map.prev_vma = kzalloc(sizeof(*op_ctx->map.prev_vma), GFP_KERNEL);
> +	if (!op_ctx->map.new_vma || !op_ctx->map.next_vma || !op_ctx->map.prev_vma) {
> +		ret = -ENOMEM;
> +		goto err_cleanup;
> +	}
> +
> +	if (!bo->base.base.import_attach) {
> +		/* Pre-reserve the BO pages, so the map operation doesn't have to
> +		 * allocate.
> +		 */
> +		ret = drm_gem_shmem_pin(&bo->base);
> +		if (ret)
> +			goto err_cleanup;
> +	}
> +
> +	sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
> +	if (IS_ERR(sgt)) {
> +		if (!bo->base.base.import_attach)
> +			drm_gem_shmem_unpin(&bo->base);
> +
> +		ret = PTR_ERR(sgt);
> +		goto err_cleanup;
> +	}
> +
> +	op_ctx->map.sgt = sgt;
> +	op_ctx->map.gem.obj = &bo->base.base;
> +	op_ctx->map.gem.offset = offset;
> +	drm_gem_object_get(op_ctx->map.gem.obj);
> +
> +	/* L1, L2 and L3 page tables.
> +	 * We could optimize L3 allocation by iterating over the sgt and merging
> +	 * 2M contiguous blocks, but it's simpler to over-provision and return
> +	 * the pages if they're not used.
> +	 */
> +	pt_count = ((ALIGN(va + size, 1ull << 39) - ALIGN_DOWN(va, 1ull << 39)) >> 39) +
> +		   ((ALIGN(va + size, 1ull << 30) - ALIGN_DOWN(va, 1ull << 30)) >> 30) +
> +		   ((ALIGN(va + size, 1ull << 21) - ALIGN_DOWN(va, 1ull << 21)) >> 21);
> +
> +	op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
> +						 sizeof(*op_ctx->rsvd_page_tables.pages),
> +						 GFP_KERNEL);
> +	if (!op_ctx->rsvd_page_tables.pages)
> +		goto err_cleanup;
> +
> +	ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
> +				    op_ctx->rsvd_page_tables.pages);
> +	op_ctx->rsvd_page_tables.count = ret;
> +	if (ret != pt_count) {
> +		ret = -ENOMEM;
> +		goto err_cleanup;
> +	}
> +
> +	return 0;
> +
> +err_cleanup:
> +	panthor_vm_cleanup_op_ctx(op_ctx, vm);
> +	return ret;
> +}
> +
> +static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx,
> +					   struct panthor_vm *vm,
> +					   u64 va, size_t size)
> +{
> +	u32 pt_count = 0;
> +	int ret;
> +
> +	memset(op_ctx, 0, sizeof(*op_ctx));
> +	INIT_LIST_HEAD(&op_ctx->returned_vmas);
> +	op_ctx->va.range = size;
> +	op_ctx->va.addr = va;
> +	op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP;
> +
> +	/* Pre-allocate L3 page tables to account for the split-2M-block
> +	 * situation on unmap.
> +	 */
> +	if (va != ALIGN(va, SZ_2M))
> +		pt_count++;
> +
> +	if (va + size != ALIGN(va + size, SZ_2M) &&
> +	    ALIGN(va + size, SZ_2M) != ALIGN(va, SZ_2M))
> +		pt_count++;
> +
> +	if (pt_count) {
> +		op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
> +							 sizeof(*op_ctx->rsvd_page_tables.pages),
> +							 GFP_KERNEL);
> +		if (!op_ctx->rsvd_page_tables.pages)
> +			goto err_cleanup;
> +
> +		ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
> +					    op_ctx->rsvd_page_tables.pages);
> +		if (ret != pt_count) {
> +			ret = -ENOMEM;
> +			goto err_cleanup;
> +		}
> +		op_ctx->rsvd_page_tables.count = pt_count;
> +	}
> +
> +	return 0;
> +
> +err_cleanup:
> +	panthor_vm_cleanup_op_ctx(op_ctx, vm);
> +	return ret;
> +}
> +
> +/**
> + * panthor_vm_get_bo_for_va() - Get the GEM object mapped at a virtual address
> + * @vm: VM to look into.
> + * @va: Virtual address to search for.
> + * @bo_offset: Offset of the GEM object mapped at this virtual address.
> + * Only valid on success.
> + *
> + * The object returned by this function might no longer be mapped when the
> + * function returns. It's the caller responsibility to ensure there's no
> + * concurrent map/unmap operations making the returned value invalid, or
> + * make sure it doesn't matter if the object is no longer mapped.
> + *
> + * Return: A valid pointer on success, an ERR_PTR() otherwise.
> + */
> +struct panthor_gem_object *
> +panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset)
> +{
> +	struct panthor_gem_object *bo = ERR_PTR(-ENOENT);
> +	struct drm_gpuva *gpuva;
> +	struct panthor_vma *vma;
> +	int ret;
> +
> +	/* Take the VM lock to prevent concurrent map/unmap operation. */
> +	ret = dma_resv_lock(vm->dummy_gem.resv, NULL);
> +	if (drm_WARN_ON(&vm->ptdev->base, ret))
> +		return NULL;
> +
> +	gpuva = drm_gpuva_find_first(&vm->va_mgr, va, 1);
> +	vma = gpuva ? container_of(gpuva, struct panthor_vma, base) : NULL;
> +	if (vma && vma->base.gem.obj) {
> +		drm_gem_object_get(vma->base.gem.obj);
> +		bo = to_panthor_bo(vma->base.gem.obj);
> +		*bo_offset = vma->base.gem.offset;
> +	}
> +	dma_resv_unlock(vm->dummy_gem.resv);
> +
> +	return bo;
> +}
> +
> +/*
> + * Only 32 VMs per open file. If that becomes a limiting factor, we can
> + * increase this number.
> + */
> +#define PANTHOR_MAX_VMS_PER_FILE	 32
> +
> +/**
> + * panthor_vm_pool_create_vm() - Create a VM
> + * @pool: The VM to create this VM on.
> + * @kernel_va_start: Start of the region reserved for kernel objects.
> + * @kernel_va_range: Size of the region reserved for kernel objects.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_pool_create_vm(struct panthor_device *ptdev, struct panthor_vm_pool *pool,
> +			      u64 kernel_va_start, u64 kernel_va_range)
> +{
> +	struct panthor_vm *vm;
> +	int ret;
> +	u32 id;
> +
> +	vm = panthor_vm_create(ptdev, false, kernel_va_start, kernel_va_range);
> +	if (IS_ERR(vm))
> +		return PTR_ERR(vm);
> +
> +	ret = xa_alloc(&pool->xa, &id, vm,
> +		       XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL);
> +
> +	if (ret) {
> +		panthor_vm_put(vm);
> +		return ret;
> +	}
> +
> +	return id;
> +}
> +
> +static void panthor_vm_destroy(struct panthor_vm *vm)
> +{
> +	if (!vm)
> +		return;
> +
> +	vm->destroyed = true;
> +
> +	mutex_lock(&vm->heaps.lock);
> +	panthor_heap_pool_destroy(vm->heaps.pool);
> +	vm->heaps.pool = NULL;
> +	mutex_unlock(&vm->heaps.lock);
> +
> +	drm_WARN_ON(&vm->ptdev->base,
> +		    panthor_vm_unmap_range(vm, vm->va_mgr.mm_start, vm->va_mgr.mm_range));
> +	panthor_vm_put(vm);
> +}
> +
> +/**
> + * panthor_vm_destroy() - Destroy a VM.
> + * @pool: VM pool.
> + * @handle: VM handle.
> + *
> + * This function doesn't free the VM object or its resources, it just kills
> + * all mappings, and makes sure nothing can be mapped after that point.
> + *
> + * If there was any active jobs at the time this function is called, these
> + * jobs should experience page faults and be killed as a result.
> + *
> + * The VM resources are freed when the last reference on the VM object is
> + * dropped.
> + */
> +int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle)
> +{
> +	struct panthor_vm *vm;
> +
> +	vm = xa_erase(&pool->xa, handle);
> +
> +	panthor_vm_destroy(vm);
> +
> +	return vm ? 0 : -EINVAL;
> +}
> +
> +/**
> + * panthor_vm_pool_get_vm() - Retrieve VM object bound to a VM handle
> + * @pool: VM pool to check.
> + * @handle: Handle of the VM to retrieve.
> + *
> + * Return: A valid pointer if the VM exists, NULL otherwise.
> + */
> +struct panthor_vm *
> +panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle)
> +{
> +	struct panthor_vm *vm;
> +
> +	vm = panthor_vm_get(xa_load(&pool->xa, handle));
> +
> +	return vm;
> +}
> +
> +/**
> + * panthor_vm_pool_destroy() - Destroy a VM pool.
> + * @pfile: File.
> + *
> + * Destroy all VMs in the pool, and release the pool resources.
> + *
> + * Note that VMs can outlive the pool they were created from if other
> + * objects hold a reference to there VMs.
> + */
> +void panthor_vm_pool_destroy(struct panthor_file *pfile)
> +{
> +	struct panthor_vm *vm;
> +	unsigned long i;
> +
> +	if (!pfile->vms)
> +		return;
> +
> +	xa_for_each(&pfile->vms->xa, i, vm)
> +		panthor_vm_destroy(vm);
> +
> +	xa_destroy(&pfile->vms->xa);
> +	kfree(pfile->vms);
> +}
> +
> +/**
> + * panthor_vm_pool_create() - Create a VM pool
> + * @pfile: File.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_pool_create(struct panthor_file *pfile)
> +{
> +	pfile->vms = kzalloc(sizeof(*pfile->vms), GFP_KERNEL);
> +	if (!pfile->vms)
> +		return -ENOMEM;
> +
> +	xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1);
> +	return 0;
> +}
> +
> +/* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */
> +static void mmu_tlb_flush_all(void *cookie)
> +{
> +}
> +
> +static void mmu_tlb_flush_walk(unsigned long iova, size_t size, size_t granule, void *cookie)
> +{
> +}
> +
> +static const struct iommu_flush_ops mmu_tlb_ops = {
> +	.tlb_flush_all = mmu_tlb_flush_all,
> +	.tlb_flush_walk = mmu_tlb_flush_walk,
> +};
> +
> +static const char *access_type_name(struct panthor_device *ptdev,
> +				    u32 fault_status)
> +{
> +	switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
> +	case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
> +		return "ATOMIC";
> +	case AS_FAULTSTATUS_ACCESS_TYPE_READ:
> +		return "READ";
> +	case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
> +		return "WRITE";
> +	case AS_FAULTSTATUS_ACCESS_TYPE_EX:
> +		return "EXECUTE";
> +	default:
> +		drm_WARN_ON(&ptdev->base, 1);
> +		return NULL;
> +	}
> +}
> +
> +static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
> +{
> +	status = panthor_mmu_fault_mask(ptdev, status);
> +	while (status) {
> +		u32 as = ffs(status | (status >> 16)) - 1;
> +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
> +		u32 new_int_mask;
> +		u64 addr;
> +		u32 fault_status;
> +		u32 exception_type;
> +		u32 access_type;
> +		u32 source_id;
> +
> +		fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
> +		addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as));
> +		addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32;
> +
> +		/* decode the fault status */
> +		exception_type = fault_status & 0xFF;
> +		access_type = (fault_status >> 8) & 0x3;
> +		source_id = (fault_status >> 16);
> +
> +		/* Page fault only */

This comment makes no sense - it looks like it's copied over from panfrost.

If I understand correctly we don't (currently) support growing on page
fault - and it's not really needed now the MCU can handle the tiler heaps.

> +		mutex_lock(&ptdev->mmu->as.slots_lock);
> +
> +		new_int_mask =
> +			panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);
> +
> +		/* terminal fault, print info about the fault */
> +		drm_err(&ptdev->base,
> +			"Unhandled Page fault in AS%d at VA 0x%016llX\n"
> +			"raw fault status: 0x%X\n"
> +			"decoded fault status: %s\n"
> +			"exception type 0x%X: %s\n"
> +			"access type 0x%X: %s\n"
> +			"source id 0x%X\n",
> +			as, addr,
> +			fault_status,
> +			(fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
> +			exception_type, panthor_exception_name(ptdev, exception_type),
> +			access_type, access_type_name(ptdev, fault_status),
> +			source_id);
> +
> +		/* Ignore MMU interrupts on this AS until it's been
> +		 * re-enabled.
> +		 */
> +		ptdev->mmu->irq.mask = new_int_mask;
> +		gpu_write(ptdev, MMU_INT_MASK, new_int_mask);
> +
> +		/* Disable the MMU to kill jobs on this AS. */
> +		panthor_mmu_as_disable(ptdev, as);
> +		mutex_unlock(&ptdev->mmu->as.slots_lock);
> +
> +		status &= ~mask;
> +	}
> +}
> +PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
> +
> +/**
> + * panthor_mmu_suspend() - Suspend the MMU logic
> + * @ptdev: Device.
> + *
> + * All we do here is de-assign the AS slots on all active VMs, so things
> + * get flushed to the main memory, and no further access to these VMs are
> + * possible.
> + *
> + * We also suspend the MMU IRQ.
> + */
> +void panthor_mmu_suspend(struct panthor_device *ptdev)
> +{
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
> +		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
> +
> +		if (vm) {
> +			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
> +			vm->as.id = -1;
> +			list_del_init(&vm->as.lru_node);
> +			ptdev->mmu->as.slots[i].vm = NULL;
> +		}
> +	}
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +
> +	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
> +}
> +
> +/**
> + * panthor_mmu_resume() - Resume the MMU logic
> + * @ptdev: Device.
> + *
> + * Resume the IRQ.
> + *
> + * We don't re-enable previously active VMs. We assume other parts of the
> + * driver will call panthor_vm_active() on the VMs they intend to use.
> + */
> +void panthor_mmu_resume(struct panthor_device *ptdev)
> +{
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +	ptdev->mmu->as.alloc_mask = 0;
> +	ptdev->mmu->as.faulty_mask = 0;
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +
> +	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
> +}
> +
> +/**
> + * panthor_mmu_pre_reset() - Prepare for a reset
> + * @ptdev: Device.
> + *
> + * Suspend the IRQ, and make sure all VM_BIND queues are stopped, so we
> + * don't get asked to do a VM operation while the GPU is down.
> + *
> + * We don't cleanly shutdown the AS slots here, because the reset might
> + * come from an AS_ACTIVE_BIT stuck situation.
> + */
> +void panthor_mmu_pre_reset(struct panthor_device *ptdev)
> +{
> +	struct panthor_vm *vm;
> +
> +	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
> +
> +	mutex_lock(&ptdev->mmu->vm.lock);
> +	ptdev->mmu->vm.reset_in_progress = true;
> +	list_for_each_entry(vm, &ptdev->mmu->vm.list, node)
> +		panthor_vm_stop(vm);
> +	mutex_unlock(&ptdev->mmu->vm.lock);
> +}
> +
> +/**
> + * panthor_mmu_post_reset() - Restore things after a reset
> + * @ptdev: Device.
> + *
> + * Put the MMU logic back in action after a reset. That implies resuming the
> + * IRQ and re-enabling the VM_BIND queues.
> + */
> +void panthor_mmu_post_reset(struct panthor_device *ptdev)
> +{
> +	struct panthor_vm *vm;
> +
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +
> +	/* Now that the reset is effective, we can assume that none of the
> +	 * AS slots are setup, and clear the faulty flags too.
> +	 */
> +	ptdev->mmu->as.alloc_mask = 0;
> +	ptdev->mmu->as.faulty_mask = 0;
> +
> +	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
> +		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
> +
> +		if (vm) {
> +			vm->as.id = -1;
> +			list_del_init(&vm->as.lru_node);
> +			ptdev->mmu->as.slots[i].vm = NULL;
> +		}
> +	}
> +
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +
> +	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
> +
> +	/* Restart the VM_BIND queues. */
> +	mutex_lock(&ptdev->mmu->vm.lock);
> +	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
> +		panthor_vm_start(vm);
> +	}
> +	ptdev->mmu->vm.reset_in_progress = false;
> +	mutex_unlock(&ptdev->mmu->vm.lock);
> +}
> +
> +static void panthor_vm_release(struct kref *kref)
> +{
> +	struct panthor_vm *vm = container_of(kref, struct panthor_vm, refcount);
> +	struct panthor_device *ptdev = vm->ptdev;
> +
> +	mutex_lock(&vm->heaps.lock);
> +	if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))
> +		panthor_heap_pool_destroy(vm->heaps.pool);
> +	mutex_unlock(&vm->heaps.lock);
> +	mutex_destroy(&vm->heaps.lock);
> +
> +	mutex_lock(&ptdev->mmu->vm.lock);
> +	list_del(&vm->node);
> +	/* Restore the scheduler state so we can call drm_sched_entity_destroy()
> +	 * and drm_sched_fini(). If get there, that means we have no job left
> +	 * and no new jobs can be queued, so we can start the scheduler without
> +	 * risking interfering with the reset.
> +	 */
> +	if (ptdev->mmu->vm.reset_in_progress)
> +		panthor_vm_start(vm);
> +	mutex_unlock(&ptdev->mmu->vm.lock);
> +
> +	drm_sched_entity_destroy(&vm->entity);
> +	drm_sched_fini(&vm->sched);
> +
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +	if (vm->as.id >= 0) {
> +		int cookie;
> +
> +		if (drm_dev_enter(&ptdev->base, &cookie)) {
> +			panthor_mmu_as_disable(ptdev, vm->as.id);
> +			drm_dev_exit(cookie);
> +		}
> +
> +		ptdev->mmu->as.slots[vm->as.id].vm = NULL;
> +		clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
> +		list_del(&vm->as.lru_node);
> +	}
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +
> +	drm_WARN_ON(&ptdev->base,
> +		    panthor_vm_unmap_range(vm, vm->va_mgr.mm_start, vm->va_mgr.mm_range));
> +
> +	free_io_pgtable_ops(vm->pgtbl_ops);
> +
> +	drm_mm_takedown(&vm->mm);
> +	mutex_destroy(&vm->mm_lock);
> +	drm_gpuva_manager_destroy(&vm->va_mgr);
> +	drm_gem_private_object_fini(&vm->dummy_gem);
> +	mutex_destroy(&vm->op_lock);
> +	kfree(vm);
> +}
> +
> +/**
> + * panthor_vm_put() - Release a reference on a VM
> + * @vm: VM to release the reference on. Can be NULL.
> + */
> +void panthor_vm_put(struct panthor_vm *vm)
> +{
> +	if (vm)
> +		kref_put(&vm->refcount, panthor_vm_release);
> +}
> +
> +/**
> + * panthor_vm_get() - Get a VM reference
> + * @vm: VM to get the reference on. Can be NULL.
> + *
> + * Return: @vm value.
> + */
> +struct panthor_vm *panthor_vm_get(struct panthor_vm *vm)
> +{
> +	if (vm)
> +		kref_get(&vm->refcount);
> +
> +	return vm;
> +}
> +
> +/**
> + * panthor_vm_get_heap_pool() - Get the heap pool attached to a VM
> + * @vm: VM to query the heap pool on.
> + * @create: True if the heap pool should be created when it doesn't exist.
> + *
> + * Heap pools are per-VM. This function allows one to retrieve the heap pool
> + * attached to a VM.
> + *
> + * If no heap pool exists yet, and @create is true, we create one.
> + *
> + * The returned panthor_heap_pool should be released with panthor_heap_pool_put().
> + *
> + * Return: A valid pointer on success, an ERR_PTR() otherwise.
> + */
> +struct panthor_heap_pool *panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create)
> +{
> +	struct panthor_heap_pool *pool;
> +
> +	mutex_lock(&vm->heaps.lock);
> +	if (!vm->heaps.pool && create) {
> +		if (vm->destroyed)
> +			pool = ERR_PTR(-EINVAL);
> +		else
> +			pool = panthor_heap_pool_create(vm->ptdev, vm);
> +
> +		if (!IS_ERR(pool))
> +			vm->heaps.pool = panthor_heap_pool_get(pool);
> +	} else {
> +		pool = panthor_heap_pool_get(vm->heaps.pool);
> +	}
> +	mutex_unlock(&vm->heaps.lock);
> +
> +	return pool;
> +}
> +
> +static u64 mair_to_memattr(u64 mair)
> +{
> +	u64 memattr = 0;
> +	u32 i;
> +
> +	for (i = 0; i < 8; i++) {
> +		u8 in_attr = mair >> (8 * i), out_attr;
> +		u8 outer = in_attr >> 4, inner = in_attr & 0xf;
> +
> +		/* For caching to be enabled, inner and outer caching policy
> +		 * have to be both write-back, if one of them is write-through
> +		 * or non-cacheable, we just choose non-cacheable. Device
> +		 * memory is also translated to non-cacheable.
> +		 */
> +		if (!(outer & 3) || !(outer & 4) || !(inner & 4)) {
> +			out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_NC |
> +				   AS_MEMATTR_AARCH64_SH_MIDGARD_INNER |
> +				   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(false, false);
> +		} else {
> +			/* Use SH_CPU_INNER mode so SH_IS, which is used when
> +			 * IOMMU_CACHE is set, actually maps to the standard
> +			 * definition of inner-shareable and not Mali's
> +			 * internal-shareable mode.
> +			 */
> +			out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_WB |
> +				   AS_MEMATTR_AARCH64_SH_CPU_INNER |
> +				   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(inner & 1, inner & 2);
> +		}
> +
> +		memattr |= (u64)out_attr << (8 * i);
> +	}
> +
> +	return memattr;
> +}
> +
> +static void panthor_vma_link(struct panthor_vm *vm, struct panthor_vma *vma)
> +{
> +	struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
> +
> +	mutex_lock(&bo->gpuva_list_lock);
> +	drm_gpuva_link(&vma->base);
> +	mutex_unlock(&bo->gpuva_list_lock);
> +
> +	if (!bo->exclusive_vm)
> +		list_add_tail(&vma->node, &vm->shared_bos);
> +}
> +
> +static void panthor_vma_unlink(struct panthor_vm_op_ctx *op_ctx,
> +			       struct panthor_vma *vma)
> +{
> +	struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
> +
> +	mutex_lock(&bo->gpuva_list_lock);
> +	drm_gpuva_unlink(&vma->base);
> +	mutex_unlock(&bo->gpuva_list_lock);
> +
> +	list_move_tail(&vma->node, &op_ctx->returned_vmas);
> +}
> +
> +static void panthor_vma_init(struct panthor_vma *vma,
> +			     struct drm_gem_object *obj,
> +			     u64 offset,
> +			     u64 va, u64 range, u32 flags)
> +{
> +	INIT_LIST_HEAD(&vma->node);
> +	vma->flags = flags;
> +	vma->base.gem.obj = obj;
> +	vma->base.gem.offset = offset;
> +	vma->base.va.addr = va;
> +	vma->base.va.range = range;
> +}
> +
> +#define PANTHOR_VM_MAP_FLAGS \
> +	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
> +	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
> +	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED)
> +
> +static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
> +{
> +	struct panthor_vm *vm = priv;
> +	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
> +	struct panthor_vma *vma = op_ctx->map.new_vma;
> +	int ret;
> +
> +	panthor_vma_init(vma, op->map.gem.obj, op->map.gem.offset, op->map.va.addr,
> +			 op->map.va.range, op_ctx->flags & PANTHOR_VM_MAP_FLAGS);
> +
> +	ret = panthor_vm_map_pages(vm, vma->base.va.addr, flags_to_prot(vma->flags),
> +				   op_ctx->map.sgt, vma->base.gem.offset,
> +				   vma->base.va.range);
> +	if (ret)
> +		return ret;
> +
> +	/* Ref owned by the mapping now, clear the obj field so we don't release the
> +	 * pinning/obj ref behind GPUVA's back.
> +	 */
> +	drm_gpuva_map(&vm->va_mgr, &vma->base, &op->map);
> +	panthor_vma_link(vm, op_ctx->map.new_vma);
> +	op_ctx->map.gem.obj = NULL;
> +	op_ctx->map.new_vma = NULL;
> +	return 0;
> +}
> +
> +static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
> +				       void *priv)
> +{
> +	struct panthor_vma *unmap_vma = container_of(op->remap.unmap->va, struct panthor_vma, base);
> +	const u64 va_start = op->remap.prev ?
> +			     op->remap.prev->va.addr + op->remap.prev->va.range :
> +			     op->remap.unmap->va->va.addr;
> +	const u64 va_end = op->remap.next ?
> +			   op->remap.next->va.addr :
> +			   op->remap.unmap->va->va.addr + op->remap.unmap->va->va.range;
> +	struct panthor_vm *vm = priv;
> +	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
> +	struct drm_gpuva *prev_va = NULL, *next_va = NULL;
> +	int ret;
> +
> +	ret = panthor_vm_unmap_pages(vm, va_start, va_end - va_start);
> +	if (ret)
> +		return ret;
> +
> +	if (op->remap.prev) {
> +		struct panthor_gem_object *bo = to_panthor_bo(op->remap.prev->gem.obj);
> +
> +		if (!bo->base.base.import_attach) {
> +			ret = drm_gem_shmem_pin(&bo->base);
> +			if (drm_WARN_ON(&vm->ptdev->base, ret))
> +				return ret;
> +		}
> +
> +		panthor_vma_init(op_ctx->map.prev_vma,
> +				 op->remap.prev->gem.obj,
> +				 op->remap.prev->gem.offset,
> +				 op->remap.prev->va.addr,
> +				 op->remap.prev->va.range,
> +				 unmap_vma->flags);
> +		prev_va = &op_ctx->map.prev_vma->base;
> +	}
> +
> +	if (op->remap.next) {
> +		struct panthor_gem_object *bo = to_panthor_bo(op->remap.next->gem.obj);
> +
> +		if (!bo->base.base.import_attach) {
> +			ret = drm_gem_shmem_pin(&bo->base);
> +			if (drm_WARN_ON(&vm->ptdev->base, ret))
> +				return ret;
> +		}
> +
> +		panthor_vma_init(op_ctx->map.next_vma,
> +				 op->remap.next->gem.obj,
> +				 op->remap.next->gem.offset,
> +				 op->remap.next->va.addr,
> +				 op->remap.next->va.range,
> +				 unmap_vma->flags);
> +		next_va = &op_ctx->map.next_vma->base;
> +	}
> +
> +	drm_gpuva_remap(prev_va, next_va, &op->remap);
> +
> +	if (prev_va) {
> +		drm_gem_object_get(prev_va->gem.obj);
> +		panthor_vma_link(vm, op_ctx->map.prev_vma);
> +		op_ctx->map.prev_vma = NULL;
> +	}
> +
> +	if (next_va) {
> +		drm_gem_object_get(next_va->gem.obj);
> +		panthor_vma_link(vm, op_ctx->map.next_vma);
> +		op_ctx->map.next_vma = NULL;
> +	}
> +
> +	panthor_vma_unlink(op_ctx, unmap_vma);
> +	return 0;
> +}
> +
> +static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op,
> +				       void *priv)
> +{
> +	struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base);
> +	struct panthor_vm *vm = priv;
> +	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
> +	int ret;
> +
> +	ret = panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
> +				     unmap_vma->base.va.range);
> +	if (drm_WARN_ON(&vm->ptdev->base, ret))
> +		return ret;
> +
> +	drm_gpuva_unmap(&op->unmap);
> +	panthor_vma_unlink(op_ctx, unmap_vma);
> +	return 0;
> +}
> +
> +static const struct drm_gpuva_fn_ops panthor_gpuva_ops = {
> +	.sm_step_map = panthor_gpuva_sm_step_map,
> +	.sm_step_remap = panthor_gpuva_sm_step_remap,
> +	.sm_step_unmap = panthor_gpuva_sm_step_unmap,
> +};
> +
> +/**
> + * panthor_vm_resv() - Get the dma_resv object attached to a VM.
> + * @vm: VM to get the dma_resv of.
> + *
> + * Return: A dma_resv object.
> + */
> +struct dma_resv *panthor_vm_resv(struct panthor_vm *vm)
> +{
> +	return vm->dummy_gem.resv;
> +}
> +
> +static int
> +panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op,
> +		   bool flag_vm_unusable_on_failure)
> +{
> +	int ret;
> +
> +	mutex_lock(&vm->op_lock);
> +	vm->op_ctx = op;
> +	switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
> +	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
> +		if (vm->unusable) {
> +			ret = -EINVAL;
> +			break;
> +		}
> +
> +		ret = drm_gpuva_sm_map(&vm->va_mgr, vm, op->va.addr, op->va.range,
> +				       op->map.gem.obj, op->map.gem.offset);
> +		break;
> +
> +	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
> +		ret = drm_gpuva_sm_unmap(&vm->va_mgr, vm, op->va.addr, op->va.range);
> +		break;
> +
> +	default:
> +		ret = -EINVAL;
> +		break;
> +	}
> +
> +	if (ret && flag_vm_unusable_on_failure)
> +		vm->unusable = true;
> +
> +	vm->op_ctx = NULL;
> +	mutex_unlock(&vm->op_lock);
> +
> +	return ret;
> +}
> +
> +static struct dma_fence *
> +panthor_vm_bind_run_job(struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
> +	bool cookie;
> +	int ret;
> +
> +	/* Not only we report an error whose result is propagated to the
> +	 * drm_sched finished fence, but we also flag the VM as unusable, because
> +	 * a failure in the async VM_BIND results in an inconsistent state. VM needs
> +	 * to be destroyed and recreated.
> +	 */
> +	cookie = dma_fence_begin_signalling();
> +	ret = panthor_vm_exec_op(job->vm, &job->ctx, true);
> +	dma_fence_end_signalling(cookie);
> +
> +	return ret ? ERR_PTR(ret) : NULL;
> +}
> +
> +static void panthor_vm_bind_job_release(struct kref *kref)
> +{
> +	struct panthor_vm_bind_job *job = container_of(kref, struct panthor_vm_bind_job, refcount);
> +
> +	if (job->base.s_fence)
> +		drm_sched_job_cleanup(&job->base);
> +
> +	panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);
> +	panthor_vm_put(job->vm);
> +	kfree(job);
> +}
> +
> +/**
> + * panthor_vm_bind_job_put() - Release a VM_BIND job reference
> + * @sched_job: Job to release the reference on.
> + */
> +void panthor_vm_bind_job_put(struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job =
> +		container_of(sched_job, struct panthor_vm_bind_job, base);
> +
> +	if (sched_job)
> +		kref_put(&job->refcount, panthor_vm_bind_job_release);
> +}
> +
> +static void
> +panthor_vm_bind_free_job(struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job =
> +		container_of(sched_job, struct panthor_vm_bind_job, base);
> +
> +	drm_sched_job_cleanup(sched_job);
> +
> +	/* Do the heavy cleanups asynchronously, so we're out of the
> +	 * dma-signaling path and can acquire dma-resv locks safely.
> +	 */
> +	queue_work(panthor_cleanup_wq, &job->cleanup_op_ctx_work);
> +}
> +
> +static enum drm_gpu_sched_stat
> +panthor_vm_bind_timedout_job(struct drm_sched_job *sched_job)
> +{
> +	WARN(1, "VM_BIND ops are synchronous for now, there should be no timeout!");
> +	return DRM_GPU_SCHED_STAT_NOMINAL;
> +}
> +
> +static const struct drm_sched_backend_ops panthor_vm_bind_ops = {
> +	.run_job = panthor_vm_bind_run_job,
> +	.free_job = panthor_vm_bind_free_job,
> +	.timedout_job = panthor_vm_bind_timedout_job,
> +};
> +
> +/**
> + * panthor_vm_create() - Create a VM
> + * @ptdev: Device.
> + * @for_mcu: True if this is the FW MCU VM.
> + * @auto_va_start: Start of the auto-VA range.
> + * @auto_va_range: Size of the auto-VA range.
> + *
> + * Return: A valid pointer on success, an ERR_PTR() otherwise.
> + */
> +struct panthor_vm *
> +panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
> +		  u64 auto_va_start, u64 auto_va_range)
> +{
> +	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
> +	u32 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features);
> +	struct drm_gpu_scheduler *sched;
> +	struct io_pgtable_cfg pgtbl_cfg;
> +	u64 mair, min_va, va_range;
> +	struct panthor_vm *vm;
> +	int ret;
> +
> +	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
> +	if (!vm)
> +		return ERR_PTR(-ENOMEM);
> +
> +	mutex_init(&vm->heaps.lock);
> +	kref_init(&vm->refcount);
> +	drm_gem_private_object_init(&ptdev->base, &vm->dummy_gem, 0);
> +	vm->for_mcu = for_mcu;
> +	vm->ptdev = ptdev;
> +	INIT_LIST_HEAD(&vm->shared_bos);
> +	mutex_init(&vm->op_lock);
> +
> +	if (for_mcu) {
> +		/* CSF MCU is a cortex M7, and can only address 4G */
> +		min_va = 0;
> +		va_range = SZ_4G;
> +	} else {
> +		min_va = 0;
> +		va_range = (1ull << va_bits);
> +
> +		/* If the auto_va_range is zero, we reserve half of the VA
> +		 * space for kernel stuff.
> +		 */
> +		if (!auto_va_range) {
> +			auto_va_range = va_range / 2;
> +			auto_va_start = va_range - auto_va_range;
> +		}
> +	}
> +
> +	mutex_init(&vm->mm_lock);
> +	drm_mm_init(&vm->mm, auto_va_start, auto_va_range);
> +
> +	/* We intentionally leave the reserved range to zero, because we want kernel VMAs
> +	 * to be handled the same way user VMAs are.
> +	 */
> +	drm_gpuva_manager_init(&vm->va_mgr,
> +			       for_mcu ? "panthor-MCU-VA-manager" : "panthor-GPU-VA-manager",
> +			       min_va, va_range, 0, 0,
> +			       &panthor_gpuva_ops);
> +	INIT_LIST_HEAD(&vm->node);
> +	INIT_LIST_HEAD(&vm->as.lru_node);
> +	vm->as.id = -1;
> +
> +	pgtbl_cfg = (struct io_pgtable_cfg) {
> +		.pgsize_bitmap	= SZ_4K | SZ_2M,
> +		.ias		= va_bits,
> +		.oas		= pa_bits,
> +		.coherent_walk	= ptdev->coherent,
> +		.tlb		= &mmu_tlb_ops,
> +		.iommu_dev	= ptdev->base.dev,
> +		.alloc		= alloc_pt,
> +		.free		= free_pt,
> +	};
> +
> +	vm->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, &pgtbl_cfg, vm);
> +	if (!vm->pgtbl_ops) {
> +		ret = -EINVAL;
> +		goto err_gpuva_destroy;
> +	}
> +
> +	/* Bind operations are synchronous for now, no timeout needed. */
> +	ret = drm_sched_init(&vm->sched, &panthor_vm_bind_ops, ptdev->mmu->vm.wq, 1, 0,
> +			     MAX_SCHEDULE_TIMEOUT, NULL, NULL,
> +			     "panthor-vm-bind", DRM_SCHED_POLICY_SINGLE_ENTITY,
> +			     ptdev->base.dev);
> +	if (ret)
> +		goto err_free_io_pgtable;
> +
> +	sched = &vm->sched;
> +	ret = drm_sched_entity_init(&vm->entity, DRM_SCHED_PRIORITY_NORMAL,
> +				    &sched, 1, NULL);
> +	if (ret)
> +		goto err_sched_fini;
> +
> +	mair = io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg.arm_lpae_s1_cfg.mair;
> +	vm->memattr = mair_to_memattr(mair);
> +
> +	mutex_lock(&ptdev->mmu->vm.lock);
> +	list_add_tail(&vm->node, &ptdev->mmu->vm.list);
> +
> +	/* If a reset is in progress, stop the scheduler. */
> +	if (ptdev->mmu->vm.reset_in_progress)
> +		panthor_vm_stop(vm);
> +	mutex_unlock(&ptdev->mmu->vm.lock);
> +
> +	return vm;
> +
> +err_sched_fini:
> +	drm_sched_fini(&vm->sched);
> +
> +err_free_io_pgtable:
> +	free_io_pgtable_ops(vm->pgtbl_ops);
> +
> +err_gpuva_destroy:
> +	drm_mm_takedown(&vm->mm);
> +	drm_gpuva_manager_destroy(&vm->va_mgr);
> +	drm_gem_private_object_fini(&vm->dummy_gem);
> +	kfree(vm);
> +
> +	return ERR_PTR(ret);
> +}
> +
> +static int
> +panthor_vm_bind_prepare_op_ctx(struct drm_file *file,
> +			       struct panthor_vm *vm,
> +			       const struct drm_panthor_vm_bind_op *op,
> +			       struct panthor_vm_op_ctx *op_ctx)
> +{
> +	struct drm_gem_object *gem;
> +	int ret;
> +
> +	/* Aligned on page size. */
> +	if ((op->va | op->size) & ~PAGE_MASK)
> +		return -EINVAL;
> +
> +	switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
> +	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
> +		gem = drm_gem_object_lookup(file, op->bo_handle);
> +		ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm,
> +						    gem ? to_panthor_bo(gem) : NULL,
> +						    op->bo_offset,
> +						    op->size,
> +						    op->va,
> +						    op->flags);
> +		drm_gem_object_put(gem);
> +		return ret;
> +
> +	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
> +		return panthor_vm_prepare_unmap_op_ctx(op_ctx, vm, op->va, op->size);
> +
> +	default:
> +		return -EINVAL;
> +	}
> +}
> +
> +static void panthor_vm_bind_job_cleanup_op_ctx_work(struct work_struct *work)
> +{
> +	struct panthor_vm_bind_job *job =
> +		container_of(work, struct panthor_vm_bind_job, cleanup_op_ctx_work);
> +
> +	panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);
> +	panthor_vm_bind_job_put(&job->base);
> +}
> +
> +/**
> + * panthor_vm_bind_job_create() - Create a VM_BIND job
> + * @file: File.
> + * @vm: VM targeted by the VM_BIND job.
> + * @op: VM operation data.
> + *
> + * Return: A valid pointer on success, an ERR_PTR() otherwise.
> + */
> +struct drm_sched_job *
> +panthor_vm_bind_job_create(struct drm_file *file,
> +			   struct panthor_vm *vm,
> +			   const struct drm_panthor_vm_bind_op *op)
> +{
> +	struct panthor_vm_bind_job *job;
> +	int ret;
> +
> +	if (!vm)
> +		return ERR_PTR(-EINVAL);
> +
> +	if (vm->destroyed || vm->unusable)
> +		return ERR_PTR(-EINVAL);
> +
> +	job = kzalloc(sizeof(*job), GFP_KERNEL);
> +	if (!job)
> +		return ERR_PTR(-ENOMEM);
> +
> +	INIT_WORK(&job->cleanup_op_ctx_work, panthor_vm_bind_job_cleanup_op_ctx_work);
> +	kref_init(&job->refcount);
> +	job->vm = panthor_vm_get(vm);
> +
> +	ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &job->ctx);
> +	if (ret)
> +		goto err_put_job;
> +
> +	ret = drm_sched_job_init(&job->base, &vm->entity, vm);
> +	if (ret)
> +		goto err_put_job;
> +
> +	return &job->base;
> +
> +err_put_job:
> +	panthor_vm_bind_job_put(&job->base);
> +	return ERR_PTR(ret);
> +}
> +
> +/**
> + * panthor_vm_bind_job_prepare_resvs() - Prepare VM_BIND job dma_resvs
> + * @exec: The locking/preparation context.
> + * @sched_job: The job to prepare resvs on.
> + *
> + * Locks and prepare the VM resv.
> + *
> + * If this is a map operation, locks and prepares the GEM resv.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,
> +				      struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
> +	int ret;
> +
> +	/* Acquire the VM lock an reserve a slot for this VM bind job. */
> +	ret = drm_exec_prepare_obj(exec, &job->vm->dummy_gem, 1);
> +	if (ret)
> +		return ret;
> +
> +	if (job->ctx.map.gem.obj) {
> +		/* Lock/prepare the GEM being mapped. */
> +		ret = drm_exec_prepare_obj(exec, job->ctx.map.gem.obj, 1);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * panthor_vm_bind_job_add_resvs_deps() - Add implicit deps to the VM_BIND job
> + * @sched_job: Job to add implicit deps on.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_bind_job_add_resvs_deps(struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
> +	int ret;
> +
> +	/* We use explicit fencing, so no need to wait for anything else but
> +	 * DMA_RESV_USAGE_KERNEL on the BO being mapped or VM. If there are extra
> +	 * dependencies, they should be passed to the VM_BIND ioctl.
> +	 */
> +	ret = drm_sched_job_add_resv_dependencies(sched_job,
> +						  job->vm->dummy_gem.resv,
> +						  DMA_RESV_USAGE_KERNEL);
> +	if (ret)
> +		return ret;
> +
> +	if (job->ctx.map.gem.obj) {
> +		ret = drm_sched_job_add_resv_dependencies(sched_job,
> +							  job->ctx.map.gem.obj->resv,
> +							  DMA_RESV_USAGE_KERNEL);
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * panthor_vm_bind_job_update_resvs() - Update the resv objects touched by a job
> + * @sched_job: Job to update the resvs on.
> + */
> +void panthor_vm_bind_job_update_resvs(struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
> +
> +	/* Explicit sync => we just register our job finished fence as bookkeep. */
> +	dma_resv_add_fence(job->vm->dummy_gem.resv,
> +			   &sched_job->s_fence->finished,
> +			   DMA_RESV_USAGE_BOOKKEEP);
> +
> +	if (job->ctx.map.gem.obj) {
> +		dma_resv_add_fence(job->ctx.map.gem.obj->resv,
> +				   &sched_job->s_fence->finished,
> +				   DMA_RESV_USAGE_BOOKKEEP);
> +	}
> +}
> +
> +/**
> + * panthor_vm_bind_exec_sync_op() - Execute a VM_BIND operation synchronously.
> + * @file: File.
> + * @vm: VM targeted by the VM operation.
> + * @op: Data describing the VM operation.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_bind_exec_sync_op(struct drm_file *file,
> +				 struct panthor_vm *vm,
> +				 struct drm_panthor_vm_bind_op *op)
> +{
> +	struct panthor_vm_op_ctx op_ctx;
> +	int ret;
> +
> +	/* No sync objects allowed on synchronous operations. */
> +	if (op->syncs.count)
> +		return -EINVAL;
> +
> +	if (!op->size)
> +		return 0;
> +
> +	ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &op_ctx);
> +	if (ret)
> +		return ret;
> +
> +	ret = panthor_vm_exec_op(vm, &op_ctx, false);
> +	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
> +
> +	return ret;
> +}
> +
> +/**
> + * panthor_vm_map_bo_range() - Map a GEM object range to a VM
> + * @vm: VM to map the GEM to.
> + * @bo: GEM object to map.
> + * @offset: Offset in the GEM object.
> + * @size: Size to map.
> + * @va: Virtual address to map the object to.
> + * @flags: Combination of drm_panthor_vm_bind_op_flags flags.
> + * Only map-related flags are valid.
> + *
> + * Internal use only. For userspace requests, use
> + * panthor_vm_bind_exec_sync_op() instead.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
> +			    u64 offset, size_t size, u64 va, u32 flags)
> +{
> +	struct panthor_vm_op_ctx op_ctx;
> +	int ret;
> +
> +	ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags);
> +	if (ret)
> +		return ret;
> +
> +	ret = panthor_vm_exec_op(vm, &op_ctx, false);
> +	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
> +
> +	return ret;
> +}
> +
> +/**
> + * panthor_vm_unmap_range() - Unmap a portion of the VA space
> + * @vm: VM to unmap the region from.
> + * @va: Virtual address to unmap. Must be 4k aligned.
> + * @size: Size of the region to unmap. Must be 4k aligned.
> + *
> + * Internal use only. For userspace requests, use
> + * panthor_vm_bind_exec_sync_op() instead.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, size_t size)
> +{
> +	struct panthor_vm_op_ctx op_ctx;
> +	int ret;
> +
> +	ret = panthor_vm_prepare_unmap_op_ctx(&op_ctx, vm, va, size);
> +	if (ret)
> +		return ret;
> +
> +	ret = panthor_vm_exec_op(vm, &op_ctx, false);
> +	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
> +
> +	return ret;
> +}
> +
> +/**
> + * panthor_vm_prepare_mapped_bos_resvs() - Prepare resvs on VM BOs.
> + * @exec: Locking/preparation context.
> + * @vm: VM targeted by the GPU job.
> + *
> + * GPU jobs assume all BOs bound to the VM at the time the job is submitted
> + * are available when the job is executed. In order to guarantee that, we
> + * need to reserve a slot on all BOs mapped to a VM and update this slot with
> + * the job fence after its submission.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm *vm)
> +{
> +	struct panthor_vma *vma;
> +	int ret;
> +
> +	/* Acquire the VM lock an reserve a slot for this GPU job. */
> +	ret = drm_exec_prepare_obj(exec, &vm->dummy_gem, 1);
> +	if (ret)
> +		return ret;
> +
> +	list_for_each_entry(vma, &vm->shared_bos, node) {
> +		ret = drm_exec_prepare_obj(exec, vma->base.gem.obj, 1);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * panthor_vm_add_bos_resvs_deps_to_job() - Add implicit VM deps to a GPU job
> + * @vm: VM targeted by the GPU job.
> + * @job: GPU job.
> + *
> + * We just take care of kernel access. Other accesses should be passed as
> + * explicit dependencies to the job.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_add_bos_resvs_deps_to_job(struct panthor_vm *vm,
> +					 struct drm_sched_job *job)
> +{
> +	struct panthor_vma *vma;
> +	int ret;
> +
> +	/* We use explicit fencing, so no need to wait for anything else but
> +	 * DMA_RESV_USAGE_KERNEL on the BO being mapped or VM. If there are extra
> +	 * dependencies, they should be passed to the VM_BIND ioctl.
> +	 */
> +	ret = drm_sched_job_add_resv_dependencies(job,
> +						  vm->dummy_gem.resv,
> +						  DMA_RESV_USAGE_KERNEL);
> +	if (ret)
> +		return ret;
> +
> +	list_for_each_entry(vma, &vm->shared_bos, node) {
> +		ret = drm_sched_job_add_resv_dependencies(job,
> +							  vma->base.gem.obj->resv,
> +							  DMA_RESV_USAGE_KERNEL);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * panthor_vm_add_job_fence_to_bos_resvs() - Add GPU job fence to GEM resvs
> + * @vm: VM targeted by the GPU job.
> + * @job: GPU job.
> + *
> + * Update the GEM resvs after a job has been submitted. All GEMs currently
> + * bound to the VMs get the job fence added to their resv as bookkeep. If
> + * another type of implicit dependency is needed, it should be updated
> + * with %DMA_BUF_IOCTL_IMPORT_SYNC_FILE after the
> + * %DRM_IOCTL_PANTHOR_GROUP_SUBMIT ioctl has returned.
> + */
> +void panthor_vm_add_job_fence_to_bos_resvs(struct panthor_vm *vm,
> +					   struct drm_sched_job *job)
> +{
> +	struct panthor_vma *vma;
> +
> +	/* Explicit sync => we just register our job finished fence as bookkeep. */
> +	dma_resv_add_fence(vm->dummy_gem.resv,
> +			   &job->s_fence->finished,
> +			   DMA_RESV_USAGE_BOOKKEEP);
> +
> +	list_for_each_entry(vma, &vm->shared_bos, node) {
> +		dma_resv_add_fence(vma->base.gem.obj->resv,
> +				   &job->s_fence->finished,
> +				   DMA_RESV_USAGE_BOOKKEEP);
> +	}
> +}
> +
> +/**
> + * panthor_mmu_unplug() - Unplug the MMU logic
> + * @ptdev: Device.
> + *
> + * No access to the MMU regs should be done after this function is called.
> + * We suspend the IRQ and disable all VMs to guarantee that.
> + */
> +void panthor_mmu_unplug(struct panthor_device *ptdev)
> +{
> +	if (ptdev->mmu->irq.irq > 0)

In what situation is this not true? AFAICT the driver probe will fail if
the IRQ can't be obtained.

Steve

> +		panthor_mmu_irq_suspend(&ptdev->mmu->irq);
> +
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
> +		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
> +
> +		if (vm) {
> +			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
> +			vm->as.id = -1;
> +			list_del_init(&vm->as.lru_node);
> +			clear_bit(i, &ptdev->mmu->as.alloc_mask);
> +			ptdev->mmu->as.slots[i].vm = NULL;
> +		}
> +	}
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +}
> +
> +static void panthor_mmu_release_wq(struct drm_device *ddev, void *res)
> +{
> +	destroy_workqueue(res);
> +}
> +
> +/**
> + * panthor_mmu_init() - Initialize the MMU logic.
> + * @ptdev: Device.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_mmu_init(struct panthor_device *ptdev)
> +{
> +	struct panthor_mmu *mmu;
> +	int ret, irq;
> +
> +	mmu = drmm_kzalloc(&ptdev->base, sizeof(*mmu), GFP_KERNEL);
> +	if (!mmu)
> +		return -ENOMEM;
> +
> +	INIT_LIST_HEAD(&mmu->as.lru_list);
> +
> +	for (u32 i = 0; i < ARRAY_SIZE(mmu->as.slots); i++)
> +		spin_lock_init(&mmu->as.slots[i].lock);
> +
> +	drmm_mutex_init(&ptdev->base, &mmu->as.slots_lock);
> +	INIT_LIST_HEAD(&mmu->vm.list);
> +	drmm_mutex_init(&ptdev->base, &mmu->vm.lock);
> +
> +	ptdev->mmu = mmu;
> +
> +	irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "mmu");
> +	if (irq <= 0)
> +		return -ENODEV;
> +
> +	ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq,
> +				      panthor_mmu_fault_mask(ptdev, ~0));
> +	if (ret)
> +		return ret;
> +
> +	mmu->vm.wq = alloc_workqueue("panthor-vm-bind", WQ_UNBOUND, 0);
> +	if (!mmu->vm.wq)
> +		return -ENOMEM;
> +
> +	return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq);
> +}
> +
> +#ifdef CONFIG_DEBUG_FS
> +static int show_vm_gpuvas(struct panthor_vm *vm, struct seq_file *m)
> +{
> +	int ret;
> +
> +	mutex_lock(&vm->op_lock);
> +	ret = drm_debugfs_gpuva_info(m, &vm->va_mgr);
> +	mutex_unlock(&vm->op_lock);
> +
> +	return ret;
> +}
> +
> +static int show_each_vm(struct seq_file *m, void *arg)
> +{
> +	struct drm_info_node *node = (struct drm_info_node *)m->private;
> +	struct drm_device *ddev = node->minor->dev;
> +	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
> +	int (*show)(struct panthor_vm *, struct seq_file *) = node->info_ent->data;
> +	struct panthor_vm *vm;
> +	int ret = 0;
> +
> +	mutex_lock(&ptdev->mmu->vm.lock);
> +	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
> +		ret = show(vm, m);
> +		if (ret < 0)
> +			break;
> +
> +		seq_puts(m, "\n");
> +	}
> +	mutex_unlock(&ptdev->mmu->vm.lock);
> +
> +	return ret;
> +}
> +
> +static struct drm_info_list panthor_mmu_debugfs_list[] = {
> +	DRM_DEBUGFS_GPUVA_INFO(show_each_vm, show_vm_gpuvas),
> +};
> +
> +/**
> + * panthor_mmu_debugfs_init() - Initialize MMU debugfs entries
> + * @minor: Minor.
> + */
> +void panthor_mmu_debugfs_init(struct drm_minor *minor)
> +{
> +	drm_debugfs_create_files(panthor_mmu_debugfs_list,
> +				 ARRAY_SIZE(panthor_mmu_debugfs_list),
> +				 minor->debugfs_root, minor);
> +}
> +#endif /* CONFIG_DEBUG_FS */
> +
> +/**
> + * panthor_mmu_pt_cache_init() - Initialize the page table cache.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_mmu_pt_cache_init(void)
> +{
> +	pt_cache = kmem_cache_create("panthor-mmu-pt", SZ_4K, SZ_4K, 0, NULL);
> +	if (!pt_cache)
> +		return -ENOMEM;
> +
> +	return 0;
> +}
> +
> +/**
> + * panthor_mmu_pt_cache_fini() - Destroy the page table cache.
> + */
> +void panthor_mmu_pt_cache_fini(void)
> +{
> +	kmem_cache_destroy(pt_cache);
> +}
> diff --git a/drivers/gpu/drm/panthor/panthor_mmu.h b/drivers/gpu/drm/panthor/panthor_mmu.h
> new file mode 100644
> index 000000000000..d94925ccdc8c
> --- /dev/null
> +++ b/drivers/gpu/drm/panthor/panthor_mmu.h
> @@ -0,0 +1,81 @@
> +/* SPDX-License-Identifier: GPL-2.0 or MIT */
> +/* Copyright 2019 Linaro, Ltd, Rob Herring <[e-mail]> */
> +/* Copyright 2023 Collabora ltd. */
> +
> +#ifndef __PANTHOR_MMU_H__
> +#define __PANTHOR_MMU_H__
> +
> +struct drm_exec;
> +struct drm_sched_job;
> +struct panthor_gem_object;
> +struct panthor_heap_pool;
> +struct panthor_vm;
> +struct panthor_vma;
> +struct panthor_mmu;
> +
> +int panthor_mmu_init(struct panthor_device *ptdev);
> +void panthor_mmu_unplug(struct panthor_device *ptdev);
> +void panthor_mmu_pre_reset(struct panthor_device *ptdev);
> +void panthor_mmu_post_reset(struct panthor_device *ptdev);
> +void panthor_mmu_suspend(struct panthor_device *ptdev);
> +void panthor_mmu_resume(struct panthor_device *ptdev);
> +
> +int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
> +			    u64 offset, size_t size, u64 va, u32 flags);
> +int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, size_t size);
> +struct panthor_gem_object *
> +panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset);
> +
> +int panthor_vm_active(struct panthor_vm *vm);
> +void panthor_vm_idle(struct panthor_vm *vm);
> +int panthor_vm_as(struct panthor_vm *vm);
> +
> +struct panthor_heap_pool *
> +panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create);
> +
> +struct panthor_vm *panthor_vm_get(struct panthor_vm *vm);
> +void panthor_vm_put(struct panthor_vm *vm);
> +struct panthor_vm *panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
> +				     u64 auto_va_start, u64 auto_va_range);
> +
> +int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec,
> +					struct panthor_vm *vm);
> +int panthor_vm_add_bos_resvs_deps_to_job(struct panthor_vm *vm,
> +					 struct drm_sched_job *job);
> +void panthor_vm_add_job_fence_to_bos_resvs(struct panthor_vm *vm,
> +					   struct drm_sched_job *job);
> +
> +struct dma_resv *panthor_vm_resv(struct panthor_vm *vm);
> +
> +void panthor_vm_pool_destroy(struct panthor_file *pfile);
> +int panthor_vm_pool_create(struct panthor_file *pfile);
> +int panthor_vm_pool_create_vm(struct panthor_device *ptdev, struct panthor_vm_pool *pool,
> +			      u64 kernel_va_start, u64 kernel_va_range);
> +int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle);
> +struct panthor_vm *panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle);
> +
> +struct drm_mm_node *panthor_vm_alloc_va(struct panthor_vm *vm, size_t size);
> +void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *mm_node);
> +
> +int panthor_vm_bind_exec_sync_op(struct drm_file *file,
> +				 struct panthor_vm *vm,
> +				 struct drm_panthor_vm_bind_op *op);
> +
> +struct drm_sched_job *
> +panthor_vm_bind_job_create(struct drm_file *file,
> +			   struct panthor_vm *vm,
> +			   const struct drm_panthor_vm_bind_op *op);
> +void panthor_vm_bind_job_put(struct drm_sched_job *job);
> +int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,
> +				      struct drm_sched_job *job);
> +int panthor_vm_bind_job_add_resvs_deps(struct drm_sched_job *job);
> +void panthor_vm_bind_job_update_resvs(struct drm_sched_job *job);
> +
> +int panthor_mmu_pt_cache_init(void);
> +void panthor_mmu_pt_cache_fini(void);
> +
> +#ifdef CONFIG_DEBUG_FS
> +void panthor_mmu_debugfs_init(struct drm_minor *minor);
> +#endif
> +
> +#endif

Boris Brezillon Aug. 29, 2023, 3:33 p.m. UTC | #2

On Mon, 14 Aug 2023 16:53:09 +0100
Steven Price <[e-mail]> wrote:

> > +
> > +/**
> > + * struct panthor_vm_op_ctx - VM operation context
> > + *
> > + * With VM operations potentially taking place in a dma-signaling path, we
> > + * need to make sure everything that might require resource allocation is
> > + * pre-allocated upfront. This is what this operation context is far.
> > + *
> > + * We also collect resources that have been freed, so we can release them
> > + * asynchronously, and let the VM_BIND scheduler process the next VM_BIND
> > + * request.
> > + */
> > +struct panthor_vm_op_ctx {
> > +	/** @rsvd_page_tables: Pages reserved for the MMU page table update. */
> > +	struct {
> > +		/** @count: Number of pages reserved. */
> > +		u32 count;
> > +
> > +		/** @ptr: Point to the first unused page in the @pages table. */
> > +		u32 ptr;
> > +
> > +		/**
> > +		 * @page: Array of pages that can be used for an MMU page table update.
> > +		 *
> > +		 * After an VM operation, there might be free pages left in this array.
> > +		 * They should be returned to the pt_cache as part of the op_ctx cleanup.
> > +		 */
> > +		void **pages;
> > +	} rsvd_page_tables;  
> 
> Two questions:
> 
> 1) Would a mempool simplify the implementation? It looks like a
> reasonable match.

Not sure what you mean by mempool, but I'm using a kmem_cache here for
all page table allocations. The pages that are passed to
panthor_vm_op_ctx::rsvd_page_tables::pages are allocated from this
pool. It's just that for each VM operation we pre-allocate page-tables,
and release those that were not used when the operation is done (we
over-allocate for the worst case scenario).

> 
> 2) Does it really make sense to have a separate pool of memory for every
> operation? Instead of having a separate pool for each operation, it
> would be possible to just keep track of the total number needed for all
> outstanding operations. Then a single (per device or maybe per-VM if
> necessary) mempool could be resized to ensure it has the right amount of
> space.

The pool is per-driver (see the global pt_cache). rsvd_page_tables just
holds pages needed for a specific VM operation. To be more specific, it
holds pages for the worst case (page table tree is empty, except for the
root page table).

> 
> I'm also a little wary that the VM_BIND infrastructure could potentially
> be abused to trigger a large amount of kernel allocation as it allocates
> up-front for the worst case but those pages are not charged to the
> process (AFAICT). But I haven't fully got my head round that yet.

Yep, that's problematic, indeed. I considered allocating page tables
as GEM objects, but the overhead of a GEM object is quite big
(hundreds of bytes of meta-data) compared to the size of a page table
(4k), and kmem_cache was just super convenient for this page table
cache :-).

> 
> > +
> > +	/** @flags: Combination of drm_panthor_vm_bind_op_flags. */
> > +	u32 flags;
> > +
> > +	/** @va: Virtual range targeted by the VM operation. */
> > +	struct {
> > +		/** @addr: Start address. */
> > +		u64 addr;
> > +
> > +		/** @range: Range size. */
> > +		u64 range;
> > +	} va;
> > +
> > +	/**
> > +	 * @returned_vmas: List of panthor_vma objects returned after a VM operation.
> > +	 *
> > +	 * For unmap operations, this will contain all VMAs that were covered by the
> > +	 * specified VA range.
> > +	 *
> > +	 * For map operations, this will contain all VMAs that previously mapped to
> > +	 * the specified VA range.
> > +	 *
> > +	 * Those VMAs, and the resources they point to will be released as part of
> > +	 * the op_ctx cleanup operation.
> > +	 */
> > +	struct list_head returned_vmas;
> > +
> > +	/** @map: Fields specific to a map operation. */
> > +	struct {
> > +		/** @gem: GEM object information. */
> > +		struct {
> > +			/** @obj: GEM object to map. */
> > +			struct drm_gem_object *obj;
> > +
> > +			/** @offset: Offset in the GEM object. */
> > +			u64 offset;
> > +		} gem;
> > +
> > +		/**
> > +		 * @sgt: sg-table pointing to pages backing the GEM object.
> > +		 *
> > +		 * This is gathered at job creation time, such that we don't have
> > +		 * to allocate in ::run_job().
> > +		 */
> > +		struct sg_table *sgt;
> > +
> > +		/**
> > +		 * @prev_vma: Pre-allocated VMA object to deal with a remap situation.
> > +		 *
> > +		 * If the map request covers a region that's inside another VMA, the
> > +		 * previous VMA will be split, requiring instantiation of a maximum of
> > +		 * two new VMA objects.
> > +		 */
> > +		struct panthor_vma *prev_vma;
> > +
> > +		/**
> > +		 * @new_vma: The new VMA object that will be inserted to the VA tree.
> > +		 */
> > +		struct panthor_vma *new_vma;
> > +
> > +		/**
> > +		 * @next_vma: Pre-allocated VMA object to deal with a remap situation.
> > +		 *
> > +		 * See @prev_vma.
> > +		 */
> > +		struct panthor_vma *next_vma;  
> 
> It's probably premature optimization, but it feels like having a cache
> of these VMA structures might be an idea.

If it's needed, I'll probably go for a kmem_cache, but I need to
check if it's worth it first (if the closest kmalloc cache is
significantly biffer than the struct size).

> I'm also struggling to
> understand how both a new prev and new next VMA are needed - but I
> haven't dug into the GPU VA manager.

prev/next are for mapping splits: an object is already mapped, and a new
object is mapped in the middle of this pre-existing mapping. In that
case, we need 2 vma object for the preceeding and succeeding mappings,
since the old mapping object will be released.

new_vma is for the new mapping.

> 
> > +	} map;
> > +};
> > +

[...]

> > +/**
> > + * panthor_vm_active() - Flag a VM as active
> > + * @VM: VM to flag as active.
> > + *
> > + * Assigns an address space to a VM so it can be used by the GPU/MCU.
> > + *
> > + * Return: 0 on success, a negative error code otherwise.
> > + */
> > +int panthor_vm_active(struct panthor_vm *vm)
> > +{
> > +	struct panthor_device *ptdev = vm->ptdev;
> > +	struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
> > +	int ret = 0, as, cookie;
> > +	u64 transtab, transcfg;
> > +
> > +	if (!drm_dev_enter(&ptdev->base, &cookie))
> > +		return -ENODEV;
> > +
> > +	mutex_lock(&ptdev->mmu->as.slots_lock);
> > +
> > +	as = vm->as.id;
> > +	if (as >= 0) {
> > +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
> > +
> > +		if (ptdev->mmu->as.faulty_mask & mask) {
> > +			/* Unhandled pagefault on this AS, the MMU was
> > +			 * disabled. We need to re-enable the MMU after
> > +			 * clearing+unmasking the AS interrupts.
> > +			 */
> > +			gpu_write(ptdev, MMU_INT_CLEAR, mask);
> > +			ptdev->mmu->as.faulty_mask &= ~mask;
> > +			gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
> > +			goto out_enable_as;
> > +		}
> > +
> > +		goto out_unlock;
> > +	}
> > +
> > +	/* Check for a free AS */
> > +	if (vm->for_mcu) {
> > +		drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));
> > +		as = 0;
> > +	} else {
> > +		as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));
> > +	}
> > +
> > +	if (!(BIT(as) & ptdev->gpu_info.as_present)) {
> > +		struct panthor_vm *lru_vm;
> > +
> > +		lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,
> > +						  struct panthor_vm,
> > +						  as.lru_node);
> > +		if (drm_WARN_ON(&ptdev->base, !lru_vm)) {
> > +			ret = -EBUSY;
> > +			goto out_unlock;
> > +		}
> > +
> > +		list_del_init(&lru_vm->as.lru_node);
> > +		as = lru_vm->as.id;  
> 
> Should this not set lru_vm->as.id = -1, so that the code knows the VM no
> longer has an address space?

Good catch!

> 
> > +	} else {
> > +		set_bit(as, &ptdev->mmu->as.alloc_mask);
> > +	}
> > +
> > +	/* Assign the free or reclaimed AS to the FD */
> > +	vm->as.id = as;
> > +	ptdev->mmu->as.slots[as].vm = vm;
> > +
> > +out_enable_as:
> > +	transtab = cfg->arm_lpae_s1_cfg.ttbr;
> > +	transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |
> > +		   AS_TRANSCFG_PTW_RA |
> > +		   AS_TRANSCFG_ADRMODE_AARCH64_4K;
> > +	if (ptdev->coherent)
> > +		transcfg |= AS_TRANSCFG_PTW_SH_OS;
> > +
> > +	ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
> > +
> > +out_unlock:
> > +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> > +	drm_dev_exit(cookie);
> > +	return ret;
> > +}
> > +

[...]

> > +
> > +static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
> > +{
> > +	status = panthor_mmu_fault_mask(ptdev, status);
> > +	while (status) {
> > +		u32 as = ffs(status | (status >> 16)) - 1;
> > +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
> > +		u32 new_int_mask;
> > +		u64 addr;
> > +		u32 fault_status;
> > +		u32 exception_type;
> > +		u32 access_type;
> > +		u32 source_id;
> > +
> > +		fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
> > +		addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as));
> > +		addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32;
> > +
> > +		/* decode the fault status */
> > +		exception_type = fault_status & 0xFF;
> > +		access_type = (fault_status >> 8) & 0x3;
> > +		source_id = (fault_status >> 16);
> > +
> > +		/* Page fault only */  
> 
> This comment makes no sense - it looks like it's copied over from panfrost.

Uh, it made sense before I dropped map/alloc-on-fault :-).

> 
> If I understand correctly we don't (currently) support growing on page
> fault - and it's not really needed now the MCU can handle the tiler heaps.

Exaclty. Map/alloc on fault is a bit challenging because of the whole
'we have to guarantee that a job is done in finite time, and we must
make sure fence signaling is not blocked on allocation'. Given
drm_gem_get_pages() doesn't do non-blocking allocations, I thought it'd
be preferable to postpone map-on-fault until we actually decide we need
it. Note that i915 seems to have some sort of non-blocking page
allocator in shmem_sg_alloc_table()[1].

> 
> > +		mutex_lock(&ptdev->mmu->as.slots_lock);
> > +
> > +		new_int_mask =
> > +			panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);
> > +
> > +		/* terminal fault, print info about the fault */
> > +		drm_err(&ptdev->base,
> > +			"Unhandled Page fault in AS%d at VA 0x%016llX\n"
> > +			"raw fault status: 0x%X\n"
> > +			"decoded fault status: %s\n"
> > +			"exception type 0x%X: %s\n"
> > +			"access type 0x%X: %s\n"
> > +			"source id 0x%X\n",
> > +			as, addr,
> > +			fault_status,
> > +			(fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
> > +			exception_type, panthor_exception_name(ptdev, exception_type),
> > +			access_type, access_type_name(ptdev, fault_status),
> > +			source_id);
> > +
> > +		/* Ignore MMU interrupts on this AS until it's been
> > +		 * re-enabled.
> > +		 */
> > +		ptdev->mmu->irq.mask = new_int_mask;
> > +		gpu_write(ptdev, MMU_INT_MASK, new_int_mask);
> > +
> > +		/* Disable the MMU to kill jobs on this AS. */
> > +		panthor_mmu_as_disable(ptdev, as);
> > +		mutex_unlock(&ptdev->mmu->as.slots_lock);
> > +
> > +		status &= ~mask;
> > +	}
> > +}
> > +PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
> > +

[...]

> > +
> > +/**
> > + * panthor_mmu_unplug() - Unplug the MMU logic
> > + * @ptdev: Device.
> > + *
> > + * No access to the MMU regs should be done after this function is called.
> > + * We suspend the IRQ and disable all VMs to guarantee that.
> > + */
> > +void panthor_mmu_unplug(struct panthor_device *ptdev)
> > +{
> > +	if (ptdev->mmu->irq.irq > 0)  
> 
> In what situation is this not true? AFAICT the driver probe will fail if
> the IRQ can't be obtained.

Right, I'll drop this test.

[1]https://elixir.bootlin.com/linux/v6.5/source/drivers/gpu/drm/i915/gem/i915_gem_shmem.c#L63

Steven Price Aug. 30, 2023, 2:12 p.m. UTC | #3

On 29/08/2023 16:33, Boris Brezillon wrote:
> On Mon, 14 Aug 2023 16:53:09 +0100
> Steven Price <[e-mail]> wrote:
> 
>>> +
>>> +/**
>>> + * struct panthor_vm_op_ctx - VM operation context
>>> + *
>>> + * With VM operations potentially taking place in a dma-signaling path, we
>>> + * need to make sure everything that might require resource allocation is
>>> + * pre-allocated upfront. This is what this operation context is far.
>>> + *
>>> + * We also collect resources that have been freed, so we can release them
>>> + * asynchronously, and let the VM_BIND scheduler process the next VM_BIND
>>> + * request.
>>> + */
>>> +struct panthor_vm_op_ctx {
>>> +	/** @rsvd_page_tables: Pages reserved for the MMU page table update. */
>>> +	struct {
>>> +		/** @count: Number of pages reserved. */
>>> +		u32 count;
>>> +
>>> +		/** @ptr: Point to the first unused page in the @pages table. */
>>> +		u32 ptr;
>>> +
>>> +		/**
>>> +		 * @page: Array of pages that can be used for an MMU page table update.
>>> +		 *
>>> +		 * After an VM operation, there might be free pages left in this array.
>>> +		 * They should be returned to the pt_cache as part of the op_ctx cleanup.
>>> +		 */
>>> +		void **pages;
>>> +	} rsvd_page_tables;  
>>
>> Two questions:
>>
>> 1) Would a mempool simplify the implementation? It looks like a
>> reasonable match.
> 
> Not sure what you mean by mempool,

See include/linux/mempool.h

> but I'm using a kmem_cache here for
> all page table allocations. The pages that are passed to
> panthor_vm_op_ctx::rsvd_page_tables::pages are allocated from this
> pool. It's just that for each VM operation we pre-allocate page-tables,
> and release those that were not used when the operation is done (we
> over-allocate for the worst case scenario).

The mempool could, potentially, replace the rsvd_page_tables structure.
The kmem_cache you would still want as that's per-driver.

>>
>> 2) Does it really make sense to have a separate pool of memory for every
>> operation? Instead of having a separate pool for each operation, it
>> would be possible to just keep track of the total number needed for all
>> outstanding operations. Then a single (per device or maybe per-VM if
>> necessary) mempool could be resized to ensure it has the right amount of
>> space.
> 
> The pool is per-driver (see the global pt_cache). rsvd_page_tables just
> holds pages needed for a specific VM operation. To be more specific, it
> holds pages for the worst case (page table tree is empty, except for the
> root page table).

What I'm wondering is to we need to keep the pages for each operation in
separate pools. So instead of having a rsvd_page_tables for each
operation, can we have one global one which is sized appropriately for
all operations that are in flight for the device? The operations are
serialized so there's no contention. Or at least a per-VM pool if we can
operate on multiple VMs at once.

>>
>> I'm also a little wary that the VM_BIND infrastructure could potentially
>> be abused to trigger a large amount of kernel allocation as it allocates
>> up-front for the worst case but those pages are not charged to the
>> process (AFAICT). But I haven't fully got my head round that yet.
> 
> Yep, that's problematic, indeed. I considered allocating page tables
> as GEM objects, but the overhead of a GEM object is quite big
> (hundreds of bytes of meta-data) compared to the size of a page table
> (4k), and kmem_cache was just super convenient for this page table
> cache :-).

I think page tables as GEM objects is likely to be overkill, we
obviously also have to be careful not to allow user space to get access
to the contents - whereas GEM objects are usually to provide user space
access ;) I'm not sure quite what the best solution here is, clearly one
'solution' is to just cap the number of outstanding VM_BINDs.

>>
>>> +
>>> +	/** @flags: Combination of drm_panthor_vm_bind_op_flags. */
>>> +	u32 flags;
>>> +
>>> +	/** @va: Virtual range targeted by the VM operation. */
>>> +	struct {
>>> +		/** @addr: Start address. */
>>> +		u64 addr;
>>> +
>>> +		/** @range: Range size. */
>>> +		u64 range;
>>> +	} va;
>>> +
>>> +	/**
>>> +	 * @returned_vmas: List of panthor_vma objects returned after a VM operation.
>>> +	 *
>>> +	 * For unmap operations, this will contain all VMAs that were covered by the
>>> +	 * specified VA range.
>>> +	 *
>>> +	 * For map operations, this will contain all VMAs that previously mapped to
>>> +	 * the specified VA range.
>>> +	 *
>>> +	 * Those VMAs, and the resources they point to will be released as part of
>>> +	 * the op_ctx cleanup operation.
>>> +	 */
>>> +	struct list_head returned_vmas;
>>> +
>>> +	/** @map: Fields specific to a map operation. */
>>> +	struct {
>>> +		/** @gem: GEM object information. */
>>> +		struct {
>>> +			/** @obj: GEM object to map. */
>>> +			struct drm_gem_object *obj;
>>> +
>>> +			/** @offset: Offset in the GEM object. */
>>> +			u64 offset;
>>> +		} gem;
>>> +
>>> +		/**
>>> +		 * @sgt: sg-table pointing to pages backing the GEM object.
>>> +		 *
>>> +		 * This is gathered at job creation time, such that we don't have
>>> +		 * to allocate in ::run_job().
>>> +		 */
>>> +		struct sg_table *sgt;
>>> +
>>> +		/**
>>> +		 * @prev_vma: Pre-allocated VMA object to deal with a remap situation.
>>> +		 *
>>> +		 * If the map request covers a region that's inside another VMA, the
>>> +		 * previous VMA will be split, requiring instantiation of a maximum of
>>> +		 * two new VMA objects.
>>> +		 */
>>> +		struct panthor_vma *prev_vma;
>>> +
>>> +		/**
>>> +		 * @new_vma: The new VMA object that will be inserted to the VA tree.
>>> +		 */
>>> +		struct panthor_vma *new_vma;
>>> +
>>> +		/**
>>> +		 * @next_vma: Pre-allocated VMA object to deal with a remap situation.
>>> +		 *
>>> +		 * See @prev_vma.
>>> +		 */
>>> +		struct panthor_vma *next_vma;  
>>
>> It's probably premature optimization, but it feels like having a cache
>> of these VMA structures might be an idea.
> 
> If it's needed, I'll probably go for a kmem_cache, but I need to
> check if it's worth it first (if the closest kmalloc cache is
> significantly biffer than the struct size).
> 
>> I'm also struggling to
>> understand how both a new prev and new next VMA are needed - but I
>> haven't dug into the GPU VA manager.
> 
> prev/next are for mapping splits: an object is already mapped, and a new
> object is mapped in the middle of this pre-existing mapping. In that
> case, we need 2 vma object for the preceeding and succeeding mappings,
> since the old mapping object will be released.
> 
> new_vma is for the new mapping.

Yeah, looking into the GPU VA manager I see now. My problem was that I
assumed in the case of a split one of the original mappings would simply
be resized, so you'd only need one new VMA (plus the one being added).
But AFAICT that resize doesn't happen and instead new VMA are created.

>>
>>> +	} map;
>>> +};
>>> +
> 
> [...]
> 
>>> +/**
>>> + * panthor_vm_active() - Flag a VM as active
>>> + * @VM: VM to flag as active.
>>> + *
>>> + * Assigns an address space to a VM so it can be used by the GPU/MCU.
>>> + *
>>> + * Return: 0 on success, a negative error code otherwise.
>>> + */
>>> +int panthor_vm_active(struct panthor_vm *vm)
>>> +{
>>> +	struct panthor_device *ptdev = vm->ptdev;
>>> +	struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
>>> +	int ret = 0, as, cookie;
>>> +	u64 transtab, transcfg;
>>> +
>>> +	if (!drm_dev_enter(&ptdev->base, &cookie))
>>> +		return -ENODEV;
>>> +
>>> +	mutex_lock(&ptdev->mmu->as.slots_lock);
>>> +
>>> +	as = vm->as.id;
>>> +	if (as >= 0) {
>>> +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
>>> +
>>> +		if (ptdev->mmu->as.faulty_mask & mask) {
>>> +			/* Unhandled pagefault on this AS, the MMU was
>>> +			 * disabled. We need to re-enable the MMU after
>>> +			 * clearing+unmasking the AS interrupts.
>>> +			 */
>>> +			gpu_write(ptdev, MMU_INT_CLEAR, mask);
>>> +			ptdev->mmu->as.faulty_mask &= ~mask;
>>> +			gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
>>> +			goto out_enable_as;
>>> +		}
>>> +
>>> +		goto out_unlock;
>>> +	}
>>> +
>>> +	/* Check for a free AS */
>>> +	if (vm->for_mcu) {
>>> +		drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));
>>> +		as = 0;
>>> +	} else {
>>> +		as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));
>>> +	}
>>> +
>>> +	if (!(BIT(as) & ptdev->gpu_info.as_present)) {
>>> +		struct panthor_vm *lru_vm;
>>> +
>>> +		lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,
>>> +						  struct panthor_vm,
>>> +						  as.lru_node);
>>> +		if (drm_WARN_ON(&ptdev->base, !lru_vm)) {
>>> +			ret = -EBUSY;
>>> +			goto out_unlock;
>>> +		}
>>> +
>>> +		list_del_init(&lru_vm->as.lru_node);
>>> +		as = lru_vm->as.id;  
>>
>> Should this not set lru_vm->as.id = -1, so that the code knows the VM no
>> longer has an address space?
> 
> Good catch!
> 
>>
>>> +	} else {
>>> +		set_bit(as, &ptdev->mmu->as.alloc_mask);
>>> +	}
>>> +
>>> +	/* Assign the free or reclaimed AS to the FD */
>>> +	vm->as.id = as;
>>> +	ptdev->mmu->as.slots[as].vm = vm;
>>> +
>>> +out_enable_as:
>>> +	transtab = cfg->arm_lpae_s1_cfg.ttbr;
>>> +	transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |
>>> +		   AS_TRANSCFG_PTW_RA |
>>> +		   AS_TRANSCFG_ADRMODE_AARCH64_4K;
>>> +	if (ptdev->coherent)
>>> +		transcfg |= AS_TRANSCFG_PTW_SH_OS;
>>> +
>>> +	ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
>>> +
>>> +out_unlock:
>>> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
>>> +	drm_dev_exit(cookie);
>>> +	return ret;
>>> +}
>>> +
> 
> [...]
> 
>>> +
>>> +static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
>>> +{
>>> +	status = panthor_mmu_fault_mask(ptdev, status);
>>> +	while (status) {
>>> +		u32 as = ffs(status | (status >> 16)) - 1;
>>> +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
>>> +		u32 new_int_mask;
>>> +		u64 addr;
>>> +		u32 fault_status;
>>> +		u32 exception_type;
>>> +		u32 access_type;
>>> +		u32 source_id;
>>> +
>>> +		fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
>>> +		addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as));
>>> +		addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32;
>>> +
>>> +		/* decode the fault status */
>>> +		exception_type = fault_status & 0xFF;
>>> +		access_type = (fault_status >> 8) & 0x3;
>>> +		source_id = (fault_status >> 16);
>>> +
>>> +		/* Page fault only */  
>>
>> This comment makes no sense - it looks like it's copied over from panfrost.
> 
> Uh, it made sense before I dropped map/alloc-on-fault :-).

:)

>>
>> If I understand correctly we don't (currently) support growing on page
>> fault - and it's not really needed now the MCU can handle the tiler heaps.
> 
> Exaclty. Map/alloc on fault is a bit challenging because of the whole
> 'we have to guarantee that a job is done in finite time, and we must
> make sure fence signaling is not blocked on allocation'. Given
> drm_gem_get_pages() doesn't do non-blocking allocations, I thought it'd
> be preferable to postpone map-on-fault until we actually decide we need
> it. Note that i915 seems to have some sort of non-blocking page
> allocator in shmem_sg_alloc_table()[1].

Agreed, the intention is definitely to move away from map/alloc-on-fault
- handling page faults from the GPU on the CPU is expensive even without
the can-of-worms of fence signalling.

Steve

>>
>>> +		mutex_lock(&ptdev->mmu->as.slots_lock);
>>> +
>>> +		new_int_mask =
>>> +			panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);
>>> +
>>> +		/* terminal fault, print info about the fault */
>>> +		drm_err(&ptdev->base,
>>> +			"Unhandled Page fault in AS%d at VA 0x%016llX\n"
>>> +			"raw fault status: 0x%X\n"
>>> +			"decoded fault status: %s\n"
>>> +			"exception type 0x%X: %s\n"
>>> +			"access type 0x%X: %s\n"
>>> +			"source id 0x%X\n",
>>> +			as, addr,
>>> +			fault_status,
>>> +			(fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
>>> +			exception_type, panthor_exception_name(ptdev, exception_type),
>>> +			access_type, access_type_name(ptdev, fault_status),
>>> +			source_id);
>>> +
>>> +		/* Ignore MMU interrupts on this AS until it's been
>>> +		 * re-enabled.
>>> +		 */
>>> +		ptdev->mmu->irq.mask = new_int_mask;
>>> +		gpu_write(ptdev, MMU_INT_MASK, new_int_mask);
>>> +
>>> +		/* Disable the MMU to kill jobs on this AS. */
>>> +		panthor_mmu_as_disable(ptdev, as);
>>> +		mutex_unlock(&ptdev->mmu->as.slots_lock);
>>> +
>>> +		status &= ~mask;
>>> +	}
>>> +}
>>> +PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
>>> +
> 
> [...]
> 
>>> +
>>> +/**
>>> + * panthor_mmu_unplug() - Unplug the MMU logic
>>> + * @ptdev: Device.
>>> + *
>>> + * No access to the MMU regs should be done after this function is called.
>>> + * We suspend the IRQ and disable all VMs to guarantee that.
>>> + */
>>> +void panthor_mmu_unplug(struct panthor_device *ptdev)
>>> +{
>>> +	if (ptdev->mmu->irq.irq > 0)  
>>
>> In what situation is this not true? AFAICT the driver probe will fail if
>> the IRQ can't be obtained.
> 
> Right, I'll drop this test.
> 
> [1]https://elixir.bootlin.com/linux/v6.5/source/drivers/gpu/drm/i915/gem/i915_gem_shmem.c#L63

Boris Brezillon Aug. 30, 2023, 2:53 p.m. UTC | #4

On Wed, 30 Aug 2023 15:12:43 +0100
Steven Price <[e-mail]> wrote:

> On 29/08/2023 16:33, Boris Brezillon wrote:
> > On Mon, 14 Aug 2023 16:53:09 +0100
> > Steven Price <[e-mail]> wrote:
> >   
> >>> +
> >>> +/**
> >>> + * struct panthor_vm_op_ctx - VM operation context
> >>> + *
> >>> + * With VM operations potentially taking place in a dma-signaling path, we
> >>> + * need to make sure everything that might require resource allocation is
> >>> + * pre-allocated upfront. This is what this operation context is far.
> >>> + *
> >>> + * We also collect resources that have been freed, so we can release them
> >>> + * asynchronously, and let the VM_BIND scheduler process the next VM_BIND
> >>> + * request.
> >>> + */
> >>> +struct panthor_vm_op_ctx {
> >>> +	/** @rsvd_page_tables: Pages reserved for the MMU page table update. */
> >>> +	struct {
> >>> +		/** @count: Number of pages reserved. */
> >>> +		u32 count;
> >>> +
> >>> +		/** @ptr: Point to the first unused page in the @pages table. */
> >>> +		u32 ptr;
> >>> +
> >>> +		/**
> >>> +		 * @page: Array of pages that can be used for an MMU page table update.
> >>> +		 *
> >>> +		 * After an VM operation, there might be free pages left in this array.
> >>> +		 * They should be returned to the pt_cache as part of the op_ctx cleanup.
> >>> +		 */
> >>> +		void **pages;
> >>> +	} rsvd_page_tables;    
> >>
> >> Two questions:
> >>
> >> 1) Would a mempool simplify the implementation? It looks like a
> >> reasonable match.  
> > 
> > Not sure what you mean by mempool,  
> 
> See include/linux/mempool.h

Oh, okay.

> 
> > but I'm using a kmem_cache here for
> > all page table allocations. The pages that are passed to
> > panthor_vm_op_ctx::rsvd_page_tables::pages are allocated from this
> > pool. It's just that for each VM operation we pre-allocate page-tables,
> > and release those that were not used when the operation is done (we
> > over-allocate for the worst case scenario).  
> 
> The mempool could, potentially, replace the rsvd_page_tables structure.
> The kmem_cache you would still want as that's per-driver.

Need to have a closer look at the API to make my mind, but at first
glance it seems to be overkill for what I initially had in mind.

> 
> >>
> >> 2) Does it really make sense to have a separate pool of memory for every
> >> operation? Instead of having a separate pool for each operation, it
> >> would be possible to just keep track of the total number needed for all
> >> outstanding operations. Then a single (per device or maybe per-VM if
> >> necessary) mempool could be resized to ensure it has the right amount of
> >> space.  
> > 
> > The pool is per-driver (see the global pt_cache). rsvd_page_tables just
> > holds pages needed for a specific VM operation. To be more specific, it
> > holds pages for the worst case (page table tree is empty, except for the
> > root page table).  
> 
> What I'm wondering is to we need to keep the pages for each operation in
> separate pools.

I was not really considering it a pool, more a set of pages that will
be used by the VM operation, some of them being returned to the
kmem_cache pool if we end up using less (over-provisioning). If we have
a mempool, say, at the VM level, that means we have 2 levels of caching:
the kmem_cache itself, and the mempool attached to the VM. Is there any
benefit here? Do we expect kmem_cache to be too slow for fast/already
allocated pages?

I do see how over-provisioning can cause us to allocate a lot of pages
that end up being unused, but I fail to see how VM/device level caching
would solve that, because we still have to dequeue some operations to
return pages to the intermediate pool, at which point, we've already
lost, because already queued operations reserved the amount of pages
they thought they needed for the worst case scenario.
Operations being queued after that can pick from the returned pages of
course, but that's already the case right now, because we return pages
to the kmem_cache as soon as we're done executing a VM operation.

The only thing that might help is limiting the number of in-flight
VM_BIND jobs per VM (or globally), and then have the submit path return
EBUSY or EGAIN so the userspace driver knows it has to retry at a later
time.

> So instead of having a rsvd_page_tables for each
> operation, can we have one global one which is sized appropriately for
> all operations that are in flight for the device? The operations are
> serialized so there's no contention. Or at least a per-VM pool if we can
> operate on multiple VMs at once.

We can operate on multiple VMs at once (VM is basically your VkDevice,
AKA the logical device), but I'm not too worried about the
synchronization that would be incurred by the caching at the
panthor_device level. I'm just curious to know what value it would add.
I'm also worried that it makes the reservation logic more complex:
we need to track what's still reserved and how many pages can be
re-used because they were never reclaimed by the VM operation that had
reserved it. Also didn't check how mempool plays with memory reclaim,
but if the memory in a mempool is not reclaimable, that might be
another problem.

To sum-up, I'd really like to refrain adding an intermediate cache
until the per-driver kmem cache is proven to be too slow.

> 
> >>
> >> I'm also a little wary that the VM_BIND infrastructure could potentially
> >> be abused to trigger a large amount of kernel allocation as it allocates
> >> up-front for the worst case but those pages are not charged to the
> >> process (AFAICT). But I haven't fully got my head round that yet.  
> > 
> > Yep, that's problematic, indeed. I considered allocating page tables
> > as GEM objects, but the overhead of a GEM object is quite big
> > (hundreds of bytes of meta-data) compared to the size of a page table
> > (4k), and kmem_cache was just super convenient for this page table
> > cache :-).  
> 
> I think page tables as GEM objects is likely to be overkill,

I agree it's overkill/not easy to interface with io_pgtbl, but there
was one aspect I was interested in, more than the memory accounting:
being able to reclaim page tables the same way we reclaim regular GEMs,
which would simplify the shrinker/reclaim logic. After discussing it
with Robin, I realized it was pretty much useless, because reclaiming
the GEM will also teardown all VM mappings, which will return the page
table memory to the kmem_cache, and then the kmem_cache layer can
reclaim it.

> we
> obviously also have to be careful not to allow user space to get access
> to the contents - whereas GEM objects are usually to provide user space
> access ;).

GEMs can be hidden to userspace if we want. We are the ones in control
of the mmap() (I think there's a BO flag for preventing users access
already).

> I'm not sure quite what the best solution here is, clearly one
> 'solution' is to just cap the number of outstanding VM_BINDs.

I think that'd make sense.

> 
> >>  
> >>> +
> >>> +	/** @flags: Combination of drm_panthor_vm_bind_op_flags. */
> >>> +	u32 flags;
> >>> +
> >>> +	/** @va: Virtual range targeted by the VM operation. */
> >>> +	struct {
> >>> +		/** @addr: Start address. */
> >>> +		u64 addr;
> >>> +
> >>> +		/** @range: Range size. */
> >>> +		u64 range;
> >>> +	} va;
> >>> +
> >>> +	/**
> >>> +	 * @returned_vmas: List of panthor_vma objects returned after a VM operation.
> >>> +	 *
> >>> +	 * For unmap operations, this will contain all VMAs that were covered by the
> >>> +	 * specified VA range.
> >>> +	 *
> >>> +	 * For map operations, this will contain all VMAs that previously mapped to
> >>> +	 * the specified VA range.
> >>> +	 *
> >>> +	 * Those VMAs, and the resources they point to will be released as part of
> >>> +	 * the op_ctx cleanup operation.
> >>> +	 */
> >>> +	struct list_head returned_vmas;
> >>> +
> >>> +	/** @map: Fields specific to a map operation. */
> >>> +	struct {
> >>> +		/** @gem: GEM object information. */
> >>> +		struct {
> >>> +			/** @obj: GEM object to map. */
> >>> +			struct drm_gem_object *obj;
> >>> +
> >>> +			/** @offset: Offset in the GEM object. */
> >>> +			u64 offset;
> >>> +		} gem;
> >>> +
> >>> +		/**
> >>> +		 * @sgt: sg-table pointing to pages backing the GEM object.
> >>> +		 *
> >>> +		 * This is gathered at job creation time, such that we don't have
> >>> +		 * to allocate in ::run_job().
> >>> +		 */
> >>> +		struct sg_table *sgt;
> >>> +
> >>> +		/**
> >>> +		 * @prev_vma: Pre-allocated VMA object to deal with a remap situation.
> >>> +		 *
> >>> +		 * If the map request covers a region that's inside another VMA, the
> >>> +		 * previous VMA will be split, requiring instantiation of a maximum of
> >>> +		 * two new VMA objects.
> >>> +		 */
> >>> +		struct panthor_vma *prev_vma;
> >>> +
> >>> +		/**
> >>> +		 * @new_vma: The new VMA object that will be inserted to the VA tree.
> >>> +		 */
> >>> +		struct panthor_vma *new_vma;
> >>> +
> >>> +		/**
> >>> +		 * @next_vma: Pre-allocated VMA object to deal with a remap situation.
> >>> +		 *
> >>> +		 * See @prev_vma.
> >>> +		 */
> >>> +		struct panthor_vma *next_vma;    
> >>
> >> It's probably premature optimization, but it feels like having a cache
> >> of these VMA structures might be an idea.  
> > 
> > If it's needed, I'll probably go for a kmem_cache, but I need to
> > check if it's worth it first (if the closest kmalloc cache is
> > significantly biffer than the struct size).
> >   
> >> I'm also struggling to
> >> understand how both a new prev and new next VMA are needed - but I
> >> haven't dug into the GPU VA manager.  
> > 
> > prev/next are for mapping splits: an object is already mapped, and a new
> > object is mapped in the middle of this pre-existing mapping. In that
> > case, we need 2 vma object for the preceeding and succeeding mappings,
> > since the old mapping object will be released.
> > 
> > new_vma is for the new mapping.  
> 
> Yeah, looking into the GPU VA manager I see now. My problem was that I
> assumed in the case of a split one of the original mappings would simply
> be resized, so you'd only need one new VMA (plus the one being added).
> But AFAICT that resize doesn't happen and instead new VMA are created.

Yes. On the other hand, if we have a kmem_cache for panthor_vma
objects, that shouldn't make a big difference.

> 
> >>  
> >>> +	} map;
> >>> +};
> >>> +  
> > 
> > [...]
> >   
> >>> +/**
> >>> + * panthor_vm_active() - Flag a VM as active
> >>> + * @VM: VM to flag as active.
> >>> + *
> >>> + * Assigns an address space to a VM so it can be used by the GPU/MCU.
> >>> + *
> >>> + * Return: 0 on success, a negative error code otherwise.
> >>> + */
> >>> +int panthor_vm_active(struct panthor_vm *vm)
> >>> +{
> >>> +	struct panthor_device *ptdev = vm->ptdev;
> >>> +	struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
> >>> +	int ret = 0, as, cookie;
> >>> +	u64 transtab, transcfg;
> >>> +
> >>> +	if (!drm_dev_enter(&ptdev->base, &cookie))
> >>> +		return -ENODEV;
> >>> +
> >>> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> >>> +
> >>> +	as = vm->as.id;
> >>> +	if (as >= 0) {
> >>> +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
> >>> +
> >>> +		if (ptdev->mmu->as.faulty_mask & mask) {
> >>> +			/* Unhandled pagefault on this AS, the MMU was
> >>> +			 * disabled. We need to re-enable the MMU after
> >>> +			 * clearing+unmasking the AS interrupts.
> >>> +			 */
> >>> +			gpu_write(ptdev, MMU_INT_CLEAR, mask);
> >>> +			ptdev->mmu->as.faulty_mask &= ~mask;
> >>> +			gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
> >>> +			goto out_enable_as;
> >>> +		}
> >>> +
> >>> +		goto out_unlock;
> >>> +	}
> >>> +
> >>> +	/* Check for a free AS */
> >>> +	if (vm->for_mcu) {
> >>> +		drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));
> >>> +		as = 0;
> >>> +	} else {
> >>> +		as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));
> >>> +	}
> >>> +
> >>> +	if (!(BIT(as) & ptdev->gpu_info.as_present)) {
> >>> +		struct panthor_vm *lru_vm;
> >>> +
> >>> +		lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,
> >>> +						  struct panthor_vm,
> >>> +						  as.lru_node);
> >>> +		if (drm_WARN_ON(&ptdev->base, !lru_vm)) {
> >>> +			ret = -EBUSY;
> >>> +			goto out_unlock;
> >>> +		}
> >>> +
> >>> +		list_del_init(&lru_vm->as.lru_node);
> >>> +		as = lru_vm->as.id;    
> >>
> >> Should this not set lru_vm->as.id = -1, so that the code knows the VM no
> >> longer has an address space?  
> > 
> > Good catch!
> >   
> >>  
> >>> +	} else {
> >>> +		set_bit(as, &ptdev->mmu->as.alloc_mask);
> >>> +	}
> >>> +
> >>> +	/* Assign the free or reclaimed AS to the FD */
> >>> +	vm->as.id = as;
> >>> +	ptdev->mmu->as.slots[as].vm = vm;
> >>> +
> >>> +out_enable_as:
> >>> +	transtab = cfg->arm_lpae_s1_cfg.ttbr;
> >>> +	transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |
> >>> +		   AS_TRANSCFG_PTW_RA |
> >>> +		   AS_TRANSCFG_ADRMODE_AARCH64_4K;
> >>> +	if (ptdev->coherent)
> >>> +		transcfg |= AS_TRANSCFG_PTW_SH_OS;
> >>> +
> >>> +	ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
> >>> +
> >>> +out_unlock:
> >>> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> >>> +	drm_dev_exit(cookie);
> >>> +	return ret;
> >>> +}
> >>> +  
> > 
> > [...]
> >   
> >>> +
> >>> +static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
> >>> +{
> >>> +	status = panthor_mmu_fault_mask(ptdev, status);
> >>> +	while (status) {
> >>> +		u32 as = ffs(status | (status >> 16)) - 1;
> >>> +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
> >>> +		u32 new_int_mask;
> >>> +		u64 addr;
> >>> +		u32 fault_status;
> >>> +		u32 exception_type;
> >>> +		u32 access_type;
> >>> +		u32 source_id;
> >>> +
> >>> +		fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
> >>> +		addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as));
> >>> +		addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32;
> >>> +
> >>> +		/* decode the fault status */
> >>> +		exception_type = fault_status & 0xFF;
> >>> +		access_type = (fault_status >> 8) & 0x3;
> >>> +		source_id = (fault_status >> 16);
> >>> +
> >>> +		/* Page fault only */    
> >>
> >> This comment makes no sense - it looks like it's copied over from panfrost.  
> > 
> > Uh, it made sense before I dropped map/alloc-on-fault :-).  
> 
> :)
> 
> >>
> >> If I understand correctly we don't (currently) support growing on page
> >> fault - and it's not really needed now the MCU can handle the tiler heaps.  
> > 
> > Exaclty. Map/alloc on fault is a bit challenging because of the whole
> > 'we have to guarantee that a job is done in finite time, and we must
> > make sure fence signaling is not blocked on allocation'. Given
> > drm_gem_get_pages() doesn't do non-blocking allocations, I thought it'd
> > be preferable to postpone map-on-fault until we actually decide we need
> > it. Note that i915 seems to have some sort of non-blocking page
> > allocator in shmem_sg_alloc_table()[1].  
> 
> Agreed, the intention is definitely to move away from map/alloc-on-fault
> - handling page faults from the GPU on the CPU is expensive even without
> the can-of-worms of fence signalling.

Yeah, I agree, but I'd bet on Khronos members being inventive enough
to come with a use case for this map/alloc-on-fault feature :-).
Anyway, that's not something we have to worry about just yet.

Steven Price Aug. 30, 2023, 3:55 p.m. UTC | #5

On 30/08/2023 15:53, Boris Brezillon wrote:
> On Wed, 30 Aug 2023 15:12:43 +0100
> Steven Price <[e-mail]> wrote:
> 
>> On 29/08/2023 16:33, Boris Brezillon wrote:
>>> On Mon, 14 Aug 2023 16:53:09 +0100
>>> Steven Price <[e-mail]> wrote:
>>>   
>>>>> +
>>>>> +/**
>>>>> + * struct panthor_vm_op_ctx - VM operation context
>>>>> + *
>>>>> + * With VM operations potentially taking place in a dma-signaling path, we
>>>>> + * need to make sure everything that might require resource allocation is
>>>>> + * pre-allocated upfront. This is what this operation context is far.
>>>>> + *
>>>>> + * We also collect resources that have been freed, so we can release them
>>>>> + * asynchronously, and let the VM_BIND scheduler process the next VM_BIND
>>>>> + * request.
>>>>> + */
>>>>> +struct panthor_vm_op_ctx {
>>>>> +	/** @rsvd_page_tables: Pages reserved for the MMU page table update. */
>>>>> +	struct {
>>>>> +		/** @count: Number of pages reserved. */
>>>>> +		u32 count;
>>>>> +
>>>>> +		/** @ptr: Point to the first unused page in the @pages table. */
>>>>> +		u32 ptr;
>>>>> +
>>>>> +		/**
>>>>> +		 * @page: Array of pages that can be used for an MMU page table update.
>>>>> +		 *
>>>>> +		 * After an VM operation, there might be free pages left in this array.
>>>>> +		 * They should be returned to the pt_cache as part of the op_ctx cleanup.
>>>>> +		 */
>>>>> +		void **pages;
>>>>> +	} rsvd_page_tables;    
>>>>
>>>> Two questions:
>>>>
>>>> 1) Would a mempool simplify the implementation? It looks like a
>>>> reasonable match.  
>>>
>>> Not sure what you mean by mempool,  
>>
>> See include/linux/mempool.h
> 
> Oh, okay.
> 
>>
>>> but I'm using a kmem_cache here for
>>> all page table allocations. The pages that are passed to
>>> panthor_vm_op_ctx::rsvd_page_tables::pages are allocated from this
>>> pool. It's just that for each VM operation we pre-allocate page-tables,
>>> and release those that were not used when the operation is done (we
>>> over-allocate for the worst case scenario).  
>>
>> The mempool could, potentially, replace the rsvd_page_tables structure.
>> The kmem_cache you would still want as that's per-driver.
> 
> Need to have a closer look at the API to make my mind, but at first
> glance it seems to be overkill for what I initially had in mind.

I agree it has more functionality than is needed here - but equally the
code is already there. struct rsvd_page_tables looks to me to be a very
simplified version of it, so rather than reinventing it we could just
use the existing code.

Having said that I haven't tried the conversion, so perhaps there are
issues with the mempool implementation which makes it better to roll our
own.

>>
>>>>
>>>> 2) Does it really make sense to have a separate pool of memory for every
>>>> operation? Instead of having a separate pool for each operation, it
>>>> would be possible to just keep track of the total number needed for all
>>>> outstanding operations. Then a single (per device or maybe per-VM if
>>>> necessary) mempool could be resized to ensure it has the right amount of
>>>> space.  
>>>
>>> The pool is per-driver (see the global pt_cache). rsvd_page_tables just
>>> holds pages needed for a specific VM operation. To be more specific, it
>>> holds pages for the worst case (page table tree is empty, except for the
>>> root page table).  
>>
>> What I'm wondering is to we need to keep the pages for each operation in
>> separate pools.
> 
> I was not really considering it a pool, more a set of pages that will
> be used by the VM operation, some of them being returned to the
> kmem_cache pool if we end up using less (over-provisioning). If we have
> a mempool, say, at the VM level, that means we have 2 levels of caching:
> the kmem_cache itself, and the mempool attached to the VM. Is there any
> benefit here? Do we expect kmem_cache to be too slow for fast/already
> allocated pages?

My understanding is that we can't (easily) ensure that sufficient pages
remain in the kmem_cache. So we need a second level 'cache' to hold the
pages that might be used by the pending VM_BIND operations. That's what
the rsvd_page_tables struct does currently.

> I do see how over-provisioning can cause us to allocate a lot of pages
> that end up being unused, but I fail to see how VM/device level caching
> would solve that, because we still have to dequeue some operations to
> return pages to the intermediate pool, at which point, we've already
> lost, because already queued operations reserved the amount of pages
> they thought they needed for the worst case scenario.
> Operations being queued after that can pick from the returned pages of
> course, but that's already the case right now, because we return pages
> to the kmem_cache as soon as we're done executing a VM operation.

I don't think it would make a difference other than having one shared
data structure rather than many struct rsvd_page_tables instances.

> The only thing that might help is limiting the number of in-flight
> VM_BIND jobs per VM (or globally), and then have the submit path return
> EBUSY or EGAIN so the userspace driver knows it has to retry at a later
> time.

I think we're going to need this in some form to avoid exposing an
effective DoS vector (unless some way can be found to account the pages).

>> So instead of having a rsvd_page_tables for each
>> operation, can we have one global one which is sized appropriately for
>> all operations that are in flight for the device? The operations are
>> serialized so there's no contention. Or at least a per-VM pool if we can
>> operate on multiple VMs at once.
> 
> We can operate on multiple VMs at once (VM is basically your VkDevice,
> AKA the logical device), but I'm not too worried about the
> synchronization that would be incurred by the caching at the
> panthor_device level. I'm just curious to know what value it would add.
> I'm also worried that it makes the reservation logic more complex:
> we need to track what's still reserved and how many pages can be
> re-used because they were never reclaimed by the VM operation that had
> reserved it.

Well every operation would need to keep (an equivalent of) the
rsvd_page_tables.count variable so that the number of pages in the
mempool could be kept correct (returning pages from the pool to kmem_cache).

> Also didn't check how mempool plays with memory reclaim,
> but if the memory in a mempool is not reclaimable, that might be
> another problem.

AFAIK mempool doesn't support reclaiming, and I don't think we'd want to
reclaim the pages which were reserved for pending VM_BIND ops.

> To sum-up, I'd really like to refrain adding an intermediate cache
> until the per-driver kmem cache is proven to be too slow.

To be honest, I'm having second thoughts - the code you've got works,
and I'm not sure it's worth the time/effort to investigate mempools now.
It just struck me during the review that I would have looked at using that.

>>
>>>>
>>>> I'm also a little wary that the VM_BIND infrastructure could potentially
>>>> be abused to trigger a large amount of kernel allocation as it allocates
>>>> up-front for the worst case but those pages are not charged to the
>>>> process (AFAICT). But I haven't fully got my head round that yet.  
>>>
>>> Yep, that's problematic, indeed. I considered allocating page tables
>>> as GEM objects, but the overhead of a GEM object is quite big
>>> (hundreds of bytes of meta-data) compared to the size of a page table
>>> (4k), and kmem_cache was just super convenient for this page table
>>> cache :-).  
>>
>> I think page tables as GEM objects is likely to be overkill,
> 
> I agree it's overkill/not easy to interface with io_pgtbl, but there
> was one aspect I was interested in, more than the memory accounting:
> being able to reclaim page tables the same way we reclaim regular GEMs,
> which would simplify the shrinker/reclaim logic. After discussing it
> with Robin, I realized it was pretty much useless, because reclaiming
> the GEM will also teardown all VM mappings, which will return the page
> table memory to the kmem_cache, and then the kmem_cache layer can
> reclaim it.
> 
>> we
>> obviously also have to be careful not to allow user space to get access
>> to the contents - whereas GEM objects are usually to provide user space
>> access ;).
> 
> GEMs can be hidden to userspace if we want. We are the ones in control
> of the mmap() (I think there's a BO flag for preventing users access
> already).

True, it's just my gut reaction that I expect the contents of GEMs to be
controlled by user space in some way. Usually hidden GEM buffers are for
intermediate data that doesn't actually need to be protected from user
space, it's just user space has no interest in it. I'd have to dig into
the security model to satisfy myself whether it was completely safe to
put page tables in there.

>> I'm not sure quite what the best solution here is, clearly one
>> 'solution' is to just cap the number of outstanding VM_BINDs.
> 
> I think that'd make sense.
> 
>>
>>>>  
>>>>> +
>>>>> +	/** @flags: Combination of drm_panthor_vm_bind_op_flags. */
>>>>> +	u32 flags;
>>>>> +
>>>>> +	/** @va: Virtual range targeted by the VM operation. */
>>>>> +	struct {
>>>>> +		/** @addr: Start address. */
>>>>> +		u64 addr;
>>>>> +
>>>>> +		/** @range: Range size. */
>>>>> +		u64 range;
>>>>> +	} va;
>>>>> +
>>>>> +	/**
>>>>> +	 * @returned_vmas: List of panthor_vma objects returned after a VM operation.
>>>>> +	 *
>>>>> +	 * For unmap operations, this will contain all VMAs that were covered by the
>>>>> +	 * specified VA range.
>>>>> +	 *
>>>>> +	 * For map operations, this will contain all VMAs that previously mapped to
>>>>> +	 * the specified VA range.
>>>>> +	 *
>>>>> +	 * Those VMAs, and the resources they point to will be released as part of
>>>>> +	 * the op_ctx cleanup operation.
>>>>> +	 */
>>>>> +	struct list_head returned_vmas;
>>>>> +
>>>>> +	/** @map: Fields specific to a map operation. */
>>>>> +	struct {
>>>>> +		/** @gem: GEM object information. */
>>>>> +		struct {
>>>>> +			/** @obj: GEM object to map. */
>>>>> +			struct drm_gem_object *obj;
>>>>> +
>>>>> +			/** @offset: Offset in the GEM object. */
>>>>> +			u64 offset;
>>>>> +		} gem;
>>>>> +
>>>>> +		/**
>>>>> +		 * @sgt: sg-table pointing to pages backing the GEM object.
>>>>> +		 *
>>>>> +		 * This is gathered at job creation time, such that we don't have
>>>>> +		 * to allocate in ::run_job().
>>>>> +		 */
>>>>> +		struct sg_table *sgt;
>>>>> +
>>>>> +		/**
>>>>> +		 * @prev_vma: Pre-allocated VMA object to deal with a remap situation.
>>>>> +		 *
>>>>> +		 * If the map request covers a region that's inside another VMA, the
>>>>> +		 * previous VMA will be split, requiring instantiation of a maximum of
>>>>> +		 * two new VMA objects.
>>>>> +		 */
>>>>> +		struct panthor_vma *prev_vma;
>>>>> +
>>>>> +		/**
>>>>> +		 * @new_vma: The new VMA object that will be inserted to the VA tree.
>>>>> +		 */
>>>>> +		struct panthor_vma *new_vma;
>>>>> +
>>>>> +		/**
>>>>> +		 * @next_vma: Pre-allocated VMA object to deal with a remap situation.
>>>>> +		 *
>>>>> +		 * See @prev_vma.
>>>>> +		 */
>>>>> +		struct panthor_vma *next_vma;    
>>>>
>>>> It's probably premature optimization, but it feels like having a cache
>>>> of these VMA structures might be an idea.  
>>>
>>> If it's needed, I'll probably go for a kmem_cache, but I need to
>>> check if it's worth it first (if the closest kmalloc cache is
>>> significantly biffer than the struct size).
>>>   
>>>> I'm also struggling to
>>>> understand how both a new prev and new next VMA are needed - but I
>>>> haven't dug into the GPU VA manager.  
>>>
>>> prev/next are for mapping splits: an object is already mapped, and a new
>>> object is mapped in the middle of this pre-existing mapping. In that
>>> case, we need 2 vma object for the preceeding and succeeding mappings,
>>> since the old mapping object will be released.
>>>
>>> new_vma is for the new mapping.  
>>
>> Yeah, looking into the GPU VA manager I see now. My problem was that I
>> assumed in the case of a split one of the original mappings would simply
>> be resized, so you'd only need one new VMA (plus the one being added).
>> But AFAICT that resize doesn't happen and instead new VMA are created.
> 
> Yes. On the other hand, if we have a kmem_cache for panthor_vma
> objects, that shouldn't make a big difference.

Yeah I can understand the design now - I just hadn't dug deep enough before.

>>
>>>>  
>>>>> +	} map;
>>>>> +};
>>>>> +  
>>>
>>> [...]
>>>   
>>>>> +/**
>>>>> + * panthor_vm_active() - Flag a VM as active
>>>>> + * @VM: VM to flag as active.
>>>>> + *
>>>>> + * Assigns an address space to a VM so it can be used by the GPU/MCU.
>>>>> + *
>>>>> + * Return: 0 on success, a negative error code otherwise.
>>>>> + */
>>>>> +int panthor_vm_active(struct panthor_vm *vm)
>>>>> +{
>>>>> +	struct panthor_device *ptdev = vm->ptdev;
>>>>> +	struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
>>>>> +	int ret = 0, as, cookie;
>>>>> +	u64 transtab, transcfg;
>>>>> +
>>>>> +	if (!drm_dev_enter(&ptdev->base, &cookie))
>>>>> +		return -ENODEV;
>>>>> +
>>>>> +	mutex_lock(&ptdev->mmu->as.slots_lock);
>>>>> +
>>>>> +	as = vm->as.id;
>>>>> +	if (as >= 0) {
>>>>> +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
>>>>> +
>>>>> +		if (ptdev->mmu->as.faulty_mask & mask) {
>>>>> +			/* Unhandled pagefault on this AS, the MMU was
>>>>> +			 * disabled. We need to re-enable the MMU after
>>>>> +			 * clearing+unmasking the AS interrupts.
>>>>> +			 */
>>>>> +			gpu_write(ptdev, MMU_INT_CLEAR, mask);
>>>>> +			ptdev->mmu->as.faulty_mask &= ~mask;
>>>>> +			gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
>>>>> +			goto out_enable_as;
>>>>> +		}
>>>>> +
>>>>> +		goto out_unlock;
>>>>> +	}
>>>>> +
>>>>> +	/* Check for a free AS */
>>>>> +	if (vm->for_mcu) {
>>>>> +		drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));
>>>>> +		as = 0;
>>>>> +	} else {
>>>>> +		as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));
>>>>> +	}
>>>>> +
>>>>> +	if (!(BIT(as) & ptdev->gpu_info.as_present)) {
>>>>> +		struct panthor_vm *lru_vm;
>>>>> +
>>>>> +		lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,
>>>>> +						  struct panthor_vm,
>>>>> +						  as.lru_node);
>>>>> +		if (drm_WARN_ON(&ptdev->base, !lru_vm)) {
>>>>> +			ret = -EBUSY;
>>>>> +			goto out_unlock;
>>>>> +		}
>>>>> +
>>>>> +		list_del_init(&lru_vm->as.lru_node);
>>>>> +		as = lru_vm->as.id;    
>>>>
>>>> Should this not set lru_vm->as.id = -1, so that the code knows the VM no
>>>> longer has an address space?  
>>>
>>> Good catch!
>>>   
>>>>  
>>>>> +	} else {
>>>>> +		set_bit(as, &ptdev->mmu->as.alloc_mask);
>>>>> +	}
>>>>> +
>>>>> +	/* Assign the free or reclaimed AS to the FD */
>>>>> +	vm->as.id = as;
>>>>> +	ptdev->mmu->as.slots[as].vm = vm;
>>>>> +
>>>>> +out_enable_as:
>>>>> +	transtab = cfg->arm_lpae_s1_cfg.ttbr;
>>>>> +	transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |
>>>>> +		   AS_TRANSCFG_PTW_RA |
>>>>> +		   AS_TRANSCFG_ADRMODE_AARCH64_4K;
>>>>> +	if (ptdev->coherent)
>>>>> +		transcfg |= AS_TRANSCFG_PTW_SH_OS;
>>>>> +
>>>>> +	ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
>>>>> +
>>>>> +out_unlock:
>>>>> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
>>>>> +	drm_dev_exit(cookie);
>>>>> +	return ret;
>>>>> +}
>>>>> +  
>>>
>>> [...]
>>>   
>>>>> +
>>>>> +static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
>>>>> +{
>>>>> +	status = panthor_mmu_fault_mask(ptdev, status);
>>>>> +	while (status) {
>>>>> +		u32 as = ffs(status | (status >> 16)) - 1;
>>>>> +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
>>>>> +		u32 new_int_mask;
>>>>> +		u64 addr;
>>>>> +		u32 fault_status;
>>>>> +		u32 exception_type;
>>>>> +		u32 access_type;
>>>>> +		u32 source_id;
>>>>> +
>>>>> +		fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
>>>>> +		addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as));
>>>>> +		addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32;
>>>>> +
>>>>> +		/* decode the fault status */
>>>>> +		exception_type = fault_status & 0xFF;
>>>>> +		access_type = (fault_status >> 8) & 0x3;
>>>>> +		source_id = (fault_status >> 16);
>>>>> +
>>>>> +		/* Page fault only */    
>>>>
>>>> This comment makes no sense - it looks like it's copied over from panfrost.  
>>>
>>> Uh, it made sense before I dropped map/alloc-on-fault :-).  
>>
>> :)
>>
>>>>
>>>> If I understand correctly we don't (currently) support growing on page
>>>> fault - and it's not really needed now the MCU can handle the tiler heaps.  
>>>
>>> Exaclty. Map/alloc on fault is a bit challenging because of the whole
>>> 'we have to guarantee that a job is done in finite time, and we must
>>> make sure fence signaling is not blocked on allocation'. Given
>>> drm_gem_get_pages() doesn't do non-blocking allocations, I thought it'd
>>> be preferable to postpone map-on-fault until we actually decide we need
>>> it. Note that i915 seems to have some sort of non-blocking page
>>> allocator in shmem_sg_alloc_table()[1].  
>>
>> Agreed, the intention is definitely to move away from map/alloc-on-fault
>> - handling page faults from the GPU on the CPU is expensive even without
>> the can-of-worms of fence signalling.
> 
> Yeah, I agree, but I'd bet on Khronos members being inventive enough
> to come with a use case for this map/alloc-on-fault feature :-).
> Anyway, that's not something we have to worry about just yet.

:) Vulkan already has sparse textures ('Sparse Residency'), but AFAIK
that's not (yet) tied into userfaultfd... and now I've possibly put the
idea into someone's head! ;)

Steve

Patch

diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c
new file mode 100644
index 000000000000..3ba784473023
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_mmu.c
@@ -0,0 +1,2611 @@ 
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2019 Linaro, Ltd, Rob Herring <[e-mail]> */
+/* Copyright 2023 Collabora ltd. */
+
+#include <drm/drm_debugfs.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_exec.h>
+#include <drm/drm_gpuva_mgr.h>
+#include <drm/drm_managed.h>
+#include <drm/gpu_scheduler.h>
+#include <drm/panthor_drm.h>
+
+#include <linux/atomic.h>
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/io-pgtable.h>
+#include <linux/iommu.h>
+#include <linux/kmemleak.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/rwsem.h>
+#include <linux/shmem_fs.h>
+#include <linux/sizes.h>
+
+#include "panthor_device.h"
+#include "panthor_heap.h"
+#include "panthor_mmu.h"
+#include "panthor_sched.h"
+#include "panthor_gem.h"
+#include "panthor_regs.h"
+
+#define MAX_AS_SLOTS			32
+
+struct panthor_vm;
+
+/**
+ * struct panthor_as_slot - Address space slot
+ */
+struct panthor_as_slot {
+	/** @vm: VM bound to this slot. NULL is no VM is bound. */
+	struct panthor_vm *vm;
+
+	/** @lock: Lock used to serialize access to the AS registers. */
+	spinlock_t lock;
+};
+
+/**
+ * struct panthor_mmu - MMU related data
+ */
+struct panthor_mmu {
+	/** @irq: The MMU irq. */
+	struct panthor_irq irq;
+
+	/** @as: Address space related fields.
+	 *
+	 * The GPU has a limited number of address spaces (AS) slots, forcing
+	 * us to re-assign them to re-assign slots on-demand.
+	 */
+	struct {
+		/** @slots_lock: Lock protecting access to all other AS fields. */
+		struct mutex slots_lock;
+
+		/** @alloc_mask: Bitmask encoding the allocated slots. */
+		unsigned long alloc_mask;
+
+		/** @faulty_mask: Bitmask encoding the faulty slots. */
+		unsigned long faulty_mask;
+
+		/** @slots: VMs currently bound to the AS slots. */
+		struct panthor_as_slot slots[MAX_AS_SLOTS];
+
+		/**
+		 * @lru_list: List of least recently used VMs.
+		 *
+		 * We use this list to pick a VM to evict when all slots are
+		 * used.
+		 *
+		 * There should be no more active VMs than there are AS slots,
+		 * so this LRU is just here to keep VMs bound until there's
+		 * a need to release a slot, thus avoid unnecessary TLB/cache
+		 * flushes.
+		 */
+		struct list_head lru_list;
+	} as;
+
+	/** @vm: VMs management fields */
+	struct {
+		/** @lock: Lock protecting access to list. */
+		struct mutex lock;
+
+		/** @list: List containing all VMs. */
+		struct list_head list;
+
+		/** @reset_in_progress: True if a reset is in progress. */
+		bool reset_in_progress;
+
+		/** @wq: Workqueue used for the VM_BIND queues. */
+		struct workqueue_struct *wq;
+	} vm;
+};
+
+/**
+ * struct panthor_vm_pool - VM pool object
+ */
+struct panthor_vm_pool {
+	/** @xa: Array used for VM handle tracking. */
+	struct xarray xa;
+};
+
+/**
+ * struct panthor_vma - GPU mapping object
+ *
+ * This is used to track GEM mappings in GPU space.
+ */
+struct panthor_vma {
+	/** @base: Inherits from drm_gpuva. */
+	struct drm_gpuva base;
+
+	/** @node: Used to insert the mapping in the panthor_vm::shared_bos list. */
+	struct list_head node;
+
+	/**
+	 * @flags: Combination of drm_panthor_vm_bind_op_flags.
+	 *
+	 * Only map related flags are accepted.
+	 */
+	u32 flags;
+};
+
+/**
+ * struct panthor_vm_op_ctx - VM operation context
+ *
+ * With VM operations potentially taking place in a dma-signaling path, we
+ * need to make sure everything that might require resource allocation is
+ * pre-allocated upfront. This is what this operation context is far.
+ *
+ * We also collect resources that have been freed, so we can release them
+ * asynchronously, and let the VM_BIND scheduler process the next VM_BIND
+ * request.
+ */
+struct panthor_vm_op_ctx {
+	/** @rsvd_page_tables: Pages reserved for the MMU page table update. */
+	struct {
+		/** @count: Number of pages reserved. */
+		u32 count;
+
+		/** @ptr: Point to the first unused page in the @pages table. */
+		u32 ptr;
+
+		/**
+		 * @page: Array of pages that can be used for an MMU page table update.
+		 *
+		 * After an VM operation, there might be free pages left in this array.
+		 * They should be returned to the pt_cache as part of the op_ctx cleanup.
+		 */
+		void **pages;
+	} rsvd_page_tables;
+
+	/** @flags: Combination of drm_panthor_vm_bind_op_flags. */
+	u32 flags;
+
+	/** @va: Virtual range targeted by the VM operation. */
+	struct {
+		/** @addr: Start address. */
+		u64 addr;
+
+		/** @range: Range size. */
+		u64 range;
+	} va;
+
+	/**
+	 * @returned_vmas: List of panthor_vma objects returned after a VM operation.
+	 *
+	 * For unmap operations, this will contain all VMAs that were covered by the
+	 * specified VA range.
+	 *
+	 * For map operations, this will contain all VMAs that previously mapped to
+	 * the specified VA range.
+	 *
+	 * Those VMAs, and the resources they point to will be released as part of
+	 * the op_ctx cleanup operation.
+	 */
+	struct list_head returned_vmas;
+
+	/** @map: Fields specific to a map operation. */
+	struct {
+		/** @gem: GEM object information. */
+		struct {
+			/** @obj: GEM object to map. */
+			struct drm_gem_object *obj;
+
+			/** @offset: Offset in the GEM object. */
+			u64 offset;
+		} gem;
+
+		/**
+		 * @sgt: sg-table pointing to pages backing the GEM object.
+		 *
+		 * This is gathered at job creation time, such that we don't have
+		 * to allocate in ::run_job().
+		 */
+		struct sg_table *sgt;
+
+		/**
+		 * @prev_vma: Pre-allocated VMA object to deal with a remap situation.
+		 *
+		 * If the map request covers a region that's inside another VMA, the
+		 * previous VMA will be split, requiring instantiation of a maximum of
+		 * two new VMA objects.
+		 */
+		struct panthor_vma *prev_vma;
+
+		/**
+		 * @new_vma: The new VMA object that will be inserted to the VA tree.
+		 */
+		struct panthor_vma *new_vma;
+
+		/**
+		 * @next_vma: Pre-allocated VMA object to deal with a remap situation.
+		 *
+		 * See @prev_vma.
+		 */
+		struct panthor_vma *next_vma;
+	} map;
+};
+
+/**
+ * struct panthor_vm - VM object
+ *
+ * A VM is an object representing a GPU (or MCU) virtual address space.
+ * It embeds the MMU page table for this address space, a tree containing
+ * all the virtual mappings of GEM objects, and other things needed to manage
+ * the VM.
+ *
+ * Except for the MCU VM, which is managed by the kernel, all other VMs are
+ * created by userspace and mostly managed by userspace, using the
+ * %DRM_IOCTL_PANTHOR_VM_BIND ioctl.
+ *
+ * A portion of the virtual address space is reserved for kernel objects,
+ * like heap chunks, and userspace gets to decide how much of the virtual
+ * address space is left to the kernel (half of the virtual address space
+ * by default).
+ */
+struct panthor_vm {
+	/**
+	 * @va_mgr: GPU VA manager.
+	 *
+	 * We delegate all the VA management to the common drm_gpuva_mgr framework
+	 * and only implement hooks to update the MMU page table.
+	 */
+	struct drm_gpuva_manager va_mgr;
+
+	/**
+	 * @sched: Scheduler used for asynchronous VM_BIND request.
+	 *
+	 * We use a 1:1 scheduler here.
+	 */
+	struct drm_gpu_scheduler sched;
+
+	/**
+	 * @entity: Scheduling entity representing the VM_BIND queue.
+	 *
+	 * There's currently one bind queue per VM. It doesn't make sense to
+	 * allow more given the VM operations are serialized anyway.
+	 */
+	struct drm_sched_entity entity;
+
+	/** @ptdev: Device. */
+	struct panthor_device *ptdev;
+
+	/** @refcount: Reference count. */
+	struct kref refcount;
+
+	/** @memattr: Value to program to the AS_MEMATTR register. */
+	u64 memattr;
+
+	/** @pgtbl_ops: Page table operations. */
+	struct io_pgtable_ops *pgtbl_ops;
+
+	/**
+	 * @dummy_gem: Used as a VM reservation object.
+	 *
+	 * We declare a drm_gem_object and no a dma_resv, so we can use drm_exec()
+	 * for the VM reservation.
+	 *
+	 * All private BOs use the resv of this dummy GEM object instead of
+	 * drm_gem_object::_resv, such that private GEM preparation is O(1)
+	 * instead of O(N).
+	 */
+	struct drm_gem_object dummy_gem;
+
+	/**
+	 * @op_lock: Lock used to serialize operations on a VM.
+	 *
+	 * The serialization of jobs queued to the VM_BIND queue is already
+	 * taken care of by drm_sched, but we need to serialize synchronous
+	 * and asynchronous VM_BIND request. This is what this lock is for.
+	 */
+	struct mutex op_lock;
+
+	/**
+	 * @op_ctx: The context attached to the currently executing VM operation.
+	 *
+	 * NULL when no operation is in progress.
+	 */
+	struct panthor_vm_op_ctx *op_ctx;
+
+	/**
+	 * @shared_bos: List of shared BOs.
+	 *
+	 * Shared BOs don't use the VM resv, and need to be prepared
+	 * independently. This list keeps track of all VMAs that target
+	 * non-private BOs.
+	 *
+	 * There might be duplicates, but drm_exec and dma_resv should
+	 * handle that for us.
+	 *
+	 * TODO: This is not optimal. We should probably switch to the
+	 * drm_gpuva_mgr solution for handling shared BOs once it's
+	 * ready.
+	 */
+	struct list_head shared_bos;
+
+	/**
+	 * @mm: Memory management object representing the auto-VA/kernel-VA.
+	 *
+	 * Used to auto-allocate VA space for kernel-managed objects (tiler
+	 * heaps, ...).
+	 *
+	 * For the MCU VM, this is managing the VA range that's used to map
+	 * all shared interfaces.
+	 *
+	 * For user VMs, the range is specified by userspace, and must not
+	 * exceed half of the VA space addressable.
+	 */
+	struct drm_mm mm;
+
+	/** @mm_lock: Lock protecting the @mm field. */
+	struct mutex mm_lock;
+
+	/** @as: Address space related fields. */
+	struct {
+		/**
+		 * @id: ID of the address space this VM is bound to.
+		 *
+		 * A value of -1 means the VM is inactive/not bound.
+		 */
+		int id;
+
+		/**
+		 * @lru_node: Used to instead the VM in the panthor_mmu::as::lru_list.
+		 *
+		 * Active VMs should not be inserted in the LRU list.
+		 */
+		struct list_head lru_node;
+	} as;
+
+	/**
+	 * @heaps: Tiler heap related fields.
+	 */
+	struct {
+		/**
+		 * @pool: The heap pool attached to this VM.
+		 *
+		 * Will stay NULL until someone creates a heap context on this VM.
+		 */
+		struct panthor_heap_pool *pool;
+
+		/** @lock: Lock used to protect access to @pool. */
+		struct mutex lock;
+	} heaps;
+
+	/** @node: Used to insert the VM in the panthor_mmu::vm::list. */
+	struct list_head node;
+
+	/** @for_mcu: True if this is the MCU VM. */
+	bool for_mcu;
+
+	/**
+	 * @destroyed: True if the VM was destroyed.
+	 *
+	 * No further bind requests should be queued to a destroyed VM.
+	 */
+	bool destroyed;
+
+	/**
+	 * @unusable: True if the VM has turned unusable because something
+	 * bad happened during an asynchronous request.
+	 *
+	 * We don't try to recover from such failures, because this implies
+	 * informing userspace about the specific operation that failed, and
+	 * hoping the userspace driver can replay things from there. This all
+	 * sounds very complicated for little gain.
+	 *
+	 * Instead, we should just flag the VM as unusable, and fail any
+	 * further request targeting this VM.
+	 *
+	 * We also provide a way to query a VM state, so userspace can destroy
+	 * it and create a new one.
+	 *
+	 * As an analogy, this would be mapped to a VK_ERROR_DEVICE_LOST
+	 * situation, where the logical device needs to be re-created.
+	 */
+	bool unusable;
+};
+
+/**
+ * struct panthor_vm_bind_job - VM bind job
+ */
+struct panthor_vm_bind_job {
+	/** @base: Inherit from drm_sched_job. */
+	struct drm_sched_job base;
+
+	/** @refcount: Reference count. */
+	struct kref refcount;
+
+	/** @cleanup_op_ctx_work: Work used to cleanup the VM operation context. */
+	struct work_struct cleanup_op_ctx_work;
+
+	/** @vm: VM targeted by the VM operation. */
+	struct panthor_vm *vm;
+
+	/** @ctx: Operation context. */
+	struct panthor_vm_op_ctx ctx;
+};
+
+/**
+ * @pt_cache: Cache used to allocate MMU page tables.
+ *
+ * The pre-allocation pattern forces us to over-allocate to plan for
+ * the worst case scenario, and return the pages we didn't use.
+ *
+ * Having a kmem_cache allows us to speed allocations.
+ */
+static struct kmem_cache *pt_cache;
+
+/**
+ * alloc_pt() - Custom page table allocator
+ * @cookie: Cookie passed at page table allocation time.
+ * @size: Size of the page table. This size should be fixed,
+ * and determined at creation time based on the granule size.
+ * @gfp: GFP flags.
+ *
+ * We want a custom allocator so we can use a cache for page table
+ * allocations and amortize the cost of the over-reservation that's
+ * done to allow asynchronous VM operations.
+ *
+ * Return: non-NULL on success, NULL if the allocation failed for any
+ * reason.
+ */
+static void *alloc_pt(void *cookie, size_t size, gfp_t gfp)
+{
+	struct panthor_vm *vm = cookie;
+	void *page;
+
+	/* We're not supposed to have anything bigger than 4k here, because we picked a
+	 * 4k granule size at init time.
+	 */
+	if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
+		return NULL;
+
+	/* Allocation of the root page table happening during init. */
+	if (!vm->pgtbl_ops) {
+		drm_WARN_ON(&vm->ptdev->base, vm->op_ctx);
+		page = kmem_cache_alloc(pt_cache, gfp);
+		goto out;
+	}
+
+	/* We must have some op_ctx attached to the VM and it must have at least one
+	 * free page.
+	 */
+	if (drm_WARN_ON(&vm->ptdev->base, !vm->op_ctx) ||
+	    drm_WARN_ON(&vm->ptdev->base,
+			vm->op_ctx->rsvd_page_tables.ptr >= vm->op_ctx->rsvd_page_tables.count))
+		return NULL;
+
+	page = vm->op_ctx->rsvd_page_tables.pages[vm->op_ctx->rsvd_page_tables.ptr++];
+	memset(page, 0, SZ_4K);
+
+out:
+	/* Page table entries don't use virtual addresses, which trips out
+	 * kmemleak. kmemleak_alloc_phys() might work, but physical addresses
+	 * are mixed with other fields, and I fear kmemleak won't detect that
+	 * either.
+	 *
+	 * Let's just ignore memory passed to the page-table driver for now.
+	 */
+	kmemleak_ignore(page);
+	return page;
+}
+
+/**
+ * @free_pt() - Custom page table free function
+ * @cookie: Cookie passed at page table allocation time.
+ * @data: Page table to free.
+ * @size: Size of the page table. This size should be fixed,
+ * and determined at creation time based on the granule size.
+ */
+static void free_pt(void *cookie, void *data, size_t size)
+{
+	struct panthor_vm *vm = cookie;
+
+	if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
+		return;
+
+	/* Return the page to the pt_cache. */
+	kmem_cache_free(pt_cache, data);
+}
+
+static int wait_ready(struct panthor_device *ptdev, u32 as_nr)
+{
+	int ret;
+	u32 val;
+
+	/* Wait for the MMU status to indicate there is no active command, in
+	 * case one is pending.
+	 */
+	ret = readl_relaxed_poll_timeout_atomic(ptdev->iomem + AS_STATUS(as_nr),
+						val, !(val & AS_STATUS_AS_ACTIVE),
+						10, 100000);
+
+	if (ret) {
+		panthor_device_schedule_reset(ptdev);
+		drm_err(&ptdev->base, "AS_ACTIVE bit stuck\n");
+	}
+
+	return ret;
+}
+
+static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd)
+{
+	int status;
+
+	/* write AS_COMMAND when MMU is ready to accept another command */
+	status = wait_ready(ptdev, as_nr);
+	if (!status)
+		gpu_write(ptdev, AS_COMMAND(as_nr), cmd);
+
+	return status;
+}
+
+static void lock_region(struct panthor_device *ptdev, u32 as_nr,
+			u64 region_start, u64 size)
+{
+	u8 region_width;
+	u64 region;
+	u64 region_end = region_start + size;
+
+	if (!size)
+		return;
+
+	/*
+	 * The locked region is a naturally aligned power of 2 block encoded as
+	 * log2 minus(1).
+	 * Calculate the desired start/end and look for the highest bit which
+	 * differs. The smallest naturally aligned block must include this bit
+	 * change, the desired region starts with this bit (and subsequent bits)
+	 * zeroed and ends with the bit (and subsequent bits) set to one.
+	 */
+	region_width = max(fls64(region_start ^ (region_end - 1)),
+			   const_ilog2(AS_LOCK_REGION_MIN_SIZE)) - 1;
+
+	/*
+	 * Mask off the low bits of region_start (which would be ignored by
+	 * the hardware anyway)
+	 */
+	region_start &= GENMASK_ULL(63, region_width);
+
+	region = region_width | region_start;
+
+	/* Lock the region that needs to be updated */
+	gpu_write(ptdev, AS_LOCKADDR_LO(as_nr), lower_32_bits(region));
+	gpu_write(ptdev, AS_LOCKADDR_HI(as_nr), upper_32_bits(region));
+	write_cmd(ptdev, as_nr, AS_COMMAND_LOCK);
+}
+
+static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr,
+				      u64 iova, u64 size, u32 op)
+{
+	if (as_nr < 0)
+		return 0;
+
+	if (op != AS_COMMAND_UNLOCK)
+		lock_region(ptdev, as_nr, iova, size);
+
+	/* Run the MMU operation */
+	write_cmd(ptdev, as_nr, op);
+
+	/* Wait for the flush to complete */
+	return wait_ready(ptdev, as_nr);
+}
+
+static int mmu_hw_do_operation(struct panthor_vm *vm,
+			       u64 iova, u64 size, u32 op)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	int ret;
+
+	spin_lock(&ptdev->mmu->as.slots[vm->as.id].lock);
+	ret = mmu_hw_do_operation_locked(ptdev, vm->as.id, iova, size, op);
+	spin_unlock(&ptdev->mmu->as.slots[vm->as.id].lock);
+	return ret;
+}
+
+static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
+				 u64 transtab, u64 transcfg, u64 memattr)
+{
+	int ret;
+
+	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
+	if (ret)
+		return ret;
+
+	gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), lower_32_bits(transtab));
+	gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), upper_32_bits(transtab));
+
+	gpu_write(ptdev, AS_MEMATTR_LO(as_nr), lower_32_bits(memattr));
+	gpu_write(ptdev, AS_MEMATTR_HI(as_nr), upper_32_bits(memattr));
+
+	gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), lower_32_bits(transcfg));
+	gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), upper_32_bits(transcfg));
+
+	return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
+}
+
+static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr)
+{
+	int ret;
+
+	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
+	if (ret)
+		return ret;
+
+	gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), 0);
+	gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), 0);
+
+	gpu_write(ptdev, AS_MEMATTR_LO(as_nr), 0);
+	gpu_write(ptdev, AS_MEMATTR_HI(as_nr), 0);
+
+	gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED);
+	gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), 0);
+
+	return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
+}
+
+static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value)
+{
+	/* Bits 16 to 31 mean REQ_COMPLETE. */
+	return value & GENMASK(15, 0);
+}
+
+static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)
+{
+	return BIT(as);
+}
+
+/**
+ * panthor_vm_active() - Flag a VM as active
+ * @VM: VM to flag as active.
+ *
+ * Assigns an address space to a VM so it can be used by the GPU/MCU.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_active(struct panthor_vm *vm)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
+	int ret = 0, as, cookie;
+	u64 transtab, transcfg;
+
+	if (!drm_dev_enter(&ptdev->base, &cookie))
+		return -ENODEV;
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+
+	as = vm->as.id;
+	if (as >= 0) {
+		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
+
+		if (ptdev->mmu->as.faulty_mask & mask) {
+			/* Unhandled pagefault on this AS, the MMU was
+			 * disabled. We need to re-enable the MMU after
+			 * clearing+unmasking the AS interrupts.
+			 */
+			gpu_write(ptdev, MMU_INT_CLEAR, mask);
+			ptdev->mmu->as.faulty_mask &= ~mask;
+			gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
+			goto out_enable_as;
+		}
+
+		goto out_unlock;
+	}
+
+	/* Check for a free AS */
+	if (vm->for_mcu) {
+		drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));
+		as = 0;
+	} else {
+		as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));
+	}
+
+	if (!(BIT(as) & ptdev->gpu_info.as_present)) {
+		struct panthor_vm *lru_vm;
+
+		lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,
+						  struct panthor_vm,
+						  as.lru_node);
+		if (drm_WARN_ON(&ptdev->base, !lru_vm)) {
+			ret = -EBUSY;
+			goto out_unlock;
+		}
+
+		list_del_init(&lru_vm->as.lru_node);
+		as = lru_vm->as.id;
+	} else {
+		set_bit(as, &ptdev->mmu->as.alloc_mask);
+	}
+
+	/* Assign the free or reclaimed AS to the FD */
+	vm->as.id = as;
+	ptdev->mmu->as.slots[as].vm = vm;
+
+out_enable_as:
+	transtab = cfg->arm_lpae_s1_cfg.ttbr;
+	transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |
+		   AS_TRANSCFG_PTW_RA |
+		   AS_TRANSCFG_ADRMODE_AARCH64_4K;
+	if (ptdev->coherent)
+		transcfg |= AS_TRANSCFG_PTW_SH_OS;
+
+	ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
+
+out_unlock:
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+	drm_dev_exit(cookie);
+	return ret;
+}
+
+/**
+ * panthor_vm_idle() - Flag a VM idle
+ * @VM: VM to flag as idle.
+ *
+ * When we know the GPU is done with the VM (no more jobs to process),
+ * we can relinquish the AS slot attached to this VM, if any.
+ *
+ * We don't release the slot immediately, but instead place the VM in
+ * the LRU list, so it can be evicted if another VM needs an AS slot.
+ * This way, VMs keep attached to the AS they were given until we run
+ * out of free slot, limiting the number of MMU operations (TLB flush
+ * and other AS updates).
+ */
+void panthor_vm_idle(struct panthor_vm *vm)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	if (vm->as.id >= 0 && list_empty(&vm->as.lru_node))
+		list_add_tail(&vm->as.lru_node, &ptdev->mmu->as.lru_list);
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+}
+
+static void panthor_vm_stop(struct panthor_vm *vm)
+{
+	drm_sched_stop(&vm->sched, NULL);
+}
+
+static void panthor_vm_start(struct panthor_vm *vm)
+{
+	drm_sched_start(&vm->sched, true);
+}
+
+/**
+ * panthor_vm_as() - Get the AS slot attached to a VM
+ * @vm: VM to get the AS slot of.
+ *
+ * Return: -1 if the VM is not assigned an AS slot yet, >= 0 otherwise.
+ */
+int panthor_vm_as(struct panthor_vm *vm)
+{
+	return vm->as.id;
+}
+
+static size_t get_pgsize(u64 addr, size_t size, size_t *count)
+{
+	/*
+	 * io-pgtable only operates on multiple pages within a single table
+	 * entry, so we need to split at boundaries of the table size, i.e.
+	 * the next block size up. The distance from address A to the next
+	 * boundary of block size B is logically B - A % B, but in unsigned
+	 * two's complement where B is a power of two we get the equivalence
+	 * B - A % B == (B - A) % B == (n * B - A) % B, and choose n = 0 :)
+	 */
+	size_t blk_offset = -addr % SZ_2M;
+
+	if (blk_offset || size < SZ_2M) {
+		*count = min_not_zero(blk_offset, size) / SZ_4K;
+		return SZ_4K;
+	}
+	blk_offset = -addr % SZ_1G ?: SZ_1G;
+	*count = min(blk_offset, size) / SZ_2M;
+	return SZ_2M;
+}
+
+static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	int ret = 0, cookie;
+
+	if (vm->as.id < 0)
+		return 0;
+
+	/* If the device is unplugged, we just silently skip the flush. */
+	if (!drm_dev_enter(&ptdev->base, &cookie))
+		return 0;
+
+	/* Flush the PTs only if we're already awake */
+	if (pm_runtime_active(ptdev->base.dev))
+		ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT);
+
+	drm_dev_exit(cookie);
+	return ret;
+}
+
+static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, size_t size)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	struct io_pgtable_ops *ops = vm->pgtbl_ops;
+	size_t offset = 0;
+
+	drm_dbg(&ptdev->base, "unmap: as=%d, iova=%llx, len=%zx", vm->as.id, iova, size);
+
+	while (offset < size) {
+		size_t unmapped_sz = 0, pgcount;
+		size_t pgsize = get_pgsize(iova + offset, size - offset, &pgcount);
+
+		unmapped_sz = ops->unmap_pages(ops, iova + offset, pgsize, pgcount, NULL);
+
+		if (drm_WARN_ON(&ptdev->base, unmapped_sz != pgsize * pgcount)) {
+			drm_err(&ptdev->base, "failed to unmap range %llx-%llx (requested range %llx-%llx)\n",
+				iova + offset + unmapped_sz,
+				iova + offset + pgsize * pgcount,
+				iova, iova + size);
+			panthor_vm_flush_range(vm, iova, offset + unmapped_sz);
+			return  -EINVAL;
+		}
+		offset += unmapped_sz;
+	}
+
+	return panthor_vm_flush_range(vm, iova, size);
+}
+
+static int
+panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
+		     struct sg_table *sgt, u64 offset, ssize_t size)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	unsigned int count;
+	struct scatterlist *sgl;
+	struct io_pgtable_ops *ops = vm->pgtbl_ops;
+	u64 start_iova = iova;
+	int ret;
+
+	if (!size)
+		return 0;
+
+	for_each_sgtable_dma_sg(sgt, sgl, count) {
+		dma_addr_t paddr = sg_dma_address(sgl);
+		size_t len = sg_dma_len(sgl);
+
+		if (len <= offset) {
+			offset -= len;
+			continue;
+		}
+
+		paddr -= offset;
+		len -= offset;
+
+		if (size >= 0) {
+			len = min_t(size_t, len, size);
+			size -= len;
+		}
+
+		drm_dbg(&ptdev->base, "map: as=%d, iova=%llx, paddr=%llx, len=%zx",
+			vm->as.id, iova, paddr, len);
+
+		while (len) {
+			size_t pgcount, mapped = 0;
+			size_t pgsize = get_pgsize(iova | paddr, len, &pgcount);
+
+			ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot,
+					     GFP_KERNEL, &mapped);
+			iova += mapped;
+			paddr += mapped;
+			len -= mapped;
+
+			if (drm_WARN_ON(&ptdev->base, !ret && !mapped))
+				ret = -ENOMEM;
+
+			if (ret) {
+				/* If something failed, unmap what we've already mapped before
+				 * returning. The unmap call is not supposed to fail.
+				 */
+				drm_WARN_ON(&ptdev->base,
+					    panthor_vm_unmap_pages(vm, start_iova,
+								   iova - start_iova));
+				return ret;
+			}
+		}
+
+		if (!size)
+			break;
+	}
+
+	return panthor_vm_flush_range(vm, start_iova, iova - start_iova);
+}
+
+static int flags_to_prot(u32 flags)
+{
+	int prot = 0;
+
+	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC)
+		prot |= IOMMU_NOEXEC;
+
+	if (!(flags & DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED))
+		prot |= IOMMU_CACHE;
+
+	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_READONLY)
+		prot |= IOMMU_READ;
+	else
+		prot |= IOMMU_READ | IOMMU_WRITE;
+
+	return prot;
+}
+
+/**
+ * panthor_vm_alloc_va() - Allocate a region in the auto-va space
+ * @VM: VM to allocate a region on.
+ * @size: Size of the region.
+ *
+ * Some GPU objects, like heap chunks, are fully managed by the kernel and
+ * need to be mapped to the userspace VM, in the region reserved for kernel
+ * objects.
+ *
+ * This function takes care of allocating a region in this reserved space.
+ *
+ * Return: A valid pointer on success, and ERR_PTR() otherwise.
+ */
+struct drm_mm_node *
+panthor_vm_alloc_va(struct panthor_vm *vm, size_t size)
+{
+	struct drm_mm_node *mm_node;
+	int ret;
+
+	if (!size || (size & ~PAGE_MASK))
+		return ERR_PTR(-EINVAL);
+
+	mm_node = kzalloc(sizeof(*mm_node), GFP_KERNEL);
+	if (!mm_node)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_lock(&vm->mm_lock);
+	ret = drm_mm_insert_node(&vm->mm, mm_node, size);
+	mutex_unlock(&vm->mm_lock);
+
+	if (ret) {
+		kfree(mm_node);
+		return ERR_PTR(ret);
+	}
+
+	return mm_node;
+}
+
+/**
+ * panthor_vm_free_va() - Free a region allocated with panthor_vm_alloc_va()
+ * @VM: VM to free the region on.
+ * @mm_node: Memory node representing the region to free.
+ */
+void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *mm_node)
+{
+	if (!mm_node)
+		return;
+
+	mutex_lock(&vm->mm_lock);
+	drm_mm_remove_node(mm_node);
+	mutex_unlock(&vm->mm_lock);
+
+	kfree(mm_node);
+}
+
+static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,
+				      struct panthor_vm *vm)
+{
+	struct panthor_vma *vma, *tmp_vma;
+
+	u32 remaining_pt_count = op_ctx->rsvd_page_tables.count -
+				 op_ctx->rsvd_page_tables.ptr;
+
+	if (remaining_pt_count) {
+		kmem_cache_free_bulk(pt_cache, remaining_pt_count,
+				     op_ctx->rsvd_page_tables.pages +
+				     op_ctx->rsvd_page_tables.ptr);
+	}
+
+	kfree(op_ctx->rsvd_page_tables.pages);
+	memset(&op_ctx->rsvd_page_tables, 0, sizeof(op_ctx->rsvd_page_tables));
+
+	if (op_ctx->map.gem.obj) {
+		struct panthor_gem_object *bo = to_panthor_bo(op_ctx->map.gem.obj);
+
+		if (!bo->base.base.import_attach)
+			drm_gem_shmem_unpin(&bo->base);
+
+		drm_gem_object_put(&bo->base.base);
+	}
+
+	kfree(op_ctx->map.new_vma);
+	kfree(op_ctx->map.next_vma);
+	kfree(op_ctx->map.prev_vma);
+	memset(&op_ctx->map, 0, sizeof(op_ctx->map));
+
+	list_for_each_entry_safe(vma, tmp_vma, &op_ctx->returned_vmas, node) {
+		struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
+
+		if (!bo->base.base.import_attach)
+			drm_gem_shmem_unpin(&bo->base);
+
+		drm_gem_object_put(&bo->base.base);
+		list_del(&vma->node);
+		kfree(vma);
+	}
+}
+
+#define PANTHOR_VM_BIND_OP_MAP_FLAGS \
+	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
+	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
+	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \
+	 DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
+
+static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
+					 struct panthor_vm *vm,
+					 struct panthor_gem_object *bo,
+					 u64 offset,
+					 size_t size, u64 va,
+					 u32 flags)
+{
+	struct sg_table *sgt = NULL;
+	u64 pt_count;
+	int ret;
+
+	if (!bo)
+		return -EINVAL;
+
+	if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) ||
+	    (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP)
+		return -EINVAL;
+
+	/* Make sure the VA and size are aligned and in-bounds. */
+	if (size > bo->base.base.size || offset > bo->base.base.size - size)
+		return -EINVAL;
+
+	/* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */
+	if (bo->exclusive_vm && bo->exclusive_vm != vm)
+		return -EINVAL;
+
+	memset(op_ctx, 0, sizeof(*op_ctx));
+	INIT_LIST_HEAD(&op_ctx->returned_vmas);
+	op_ctx->flags = flags;
+	op_ctx->va.range = size;
+	op_ctx->va.addr = va;
+
+	op_ctx->map.new_vma = kzalloc(sizeof(*op_ctx->map.new_vma), GFP_KERNEL);
+	op_ctx->map.next_vma = kzalloc(sizeof(*op_ctx->map.next_vma), GFP_KERNEL);
+	op_ctx->map.prev_vma = kzalloc(sizeof(*op_ctx->map.prev_vma), GFP_KERNEL);
+	if (!op_ctx->map.new_vma || !op_ctx->map.next_vma || !op_ctx->map.prev_vma) {
+		ret = -ENOMEM;
+		goto err_cleanup;
+	}
+
+	if (!bo->base.base.import_attach) {
+		/* Pre-reserve the BO pages, so the map operation doesn't have to
+		 * allocate.
+		 */
+		ret = drm_gem_shmem_pin(&bo->base);
+		if (ret)
+			goto err_cleanup;
+	}
+
+	sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
+	if (IS_ERR(sgt)) {
+		if (!bo->base.base.import_attach)
+			drm_gem_shmem_unpin(&bo->base);
+
+		ret = PTR_ERR(sgt);
+		goto err_cleanup;
+	}
+
+	op_ctx->map.sgt = sgt;
+	op_ctx->map.gem.obj = &bo->base.base;
+	op_ctx->map.gem.offset = offset;
+	drm_gem_object_get(op_ctx->map.gem.obj);
+
+	/* L1, L2 and L3 page tables.
+	 * We could optimize L3 allocation by iterating over the sgt and merging
+	 * 2M contiguous blocks, but it's simpler to over-provision and return
+	 * the pages if they're not used.
+	 */
+	pt_count = ((ALIGN(va + size, 1ull << 39) - ALIGN_DOWN(va, 1ull << 39)) >> 39) +
+		   ((ALIGN(va + size, 1ull << 30) - ALIGN_DOWN(va, 1ull << 30)) >> 30) +
+		   ((ALIGN(va + size, 1ull << 21) - ALIGN_DOWN(va, 1ull << 21)) >> 21);
+
+	op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
+						 sizeof(*op_ctx->rsvd_page_tables.pages),
+						 GFP_KERNEL);
+	if (!op_ctx->rsvd_page_tables.pages)
+		goto err_cleanup;
+
+	ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
+				    op_ctx->rsvd_page_tables.pages);
+	op_ctx->rsvd_page_tables.count = ret;
+	if (ret != pt_count) {
+		ret = -ENOMEM;
+		goto err_cleanup;
+	}
+
+	return 0;
+
+err_cleanup:
+	panthor_vm_cleanup_op_ctx(op_ctx, vm);
+	return ret;
+}
+
+static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx,
+					   struct panthor_vm *vm,
+					   u64 va, size_t size)
+{
+	u32 pt_count = 0;
+	int ret;
+
+	memset(op_ctx, 0, sizeof(*op_ctx));
+	INIT_LIST_HEAD(&op_ctx->returned_vmas);
+	op_ctx->va.range = size;
+	op_ctx->va.addr = va;
+	op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP;
+
+	/* Pre-allocate L3 page tables to account for the split-2M-block
+	 * situation on unmap.
+	 */
+	if (va != ALIGN(va, SZ_2M))
+		pt_count++;
+
+	if (va + size != ALIGN(va + size, SZ_2M) &&
+	    ALIGN(va + size, SZ_2M) != ALIGN(va, SZ_2M))
+		pt_count++;
+
+	if (pt_count) {
+		op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
+							 sizeof(*op_ctx->rsvd_page_tables.pages),
+							 GFP_KERNEL);
+		if (!op_ctx->rsvd_page_tables.pages)
+			goto err_cleanup;
+
+		ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
+					    op_ctx->rsvd_page_tables.pages);
+		if (ret != pt_count) {
+			ret = -ENOMEM;
+			goto err_cleanup;
+		}
+		op_ctx->rsvd_page_tables.count = pt_count;
+	}
+
+	return 0;
+
+err_cleanup:
+	panthor_vm_cleanup_op_ctx(op_ctx, vm);
+	return ret;
+}
+
+/**
+ * panthor_vm_get_bo_for_va() - Get the GEM object mapped at a virtual address
+ * @vm: VM to look into.
+ * @va: Virtual address to search for.
+ * @bo_offset: Offset of the GEM object mapped at this virtual address.
+ * Only valid on success.
+ *
+ * The object returned by this function might no longer be mapped when the
+ * function returns. It's the caller responsibility to ensure there's no
+ * concurrent map/unmap operations making the returned value invalid, or
+ * make sure it doesn't matter if the object is no longer mapped.
+ *
+ * Return: A valid pointer on success, an ERR_PTR() otherwise.
+ */
+struct panthor_gem_object *
+panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset)
+{
+	struct panthor_gem_object *bo = ERR_PTR(-ENOENT);
+	struct drm_gpuva *gpuva;
+	struct panthor_vma *vma;
+	int ret;
+
+	/* Take the VM lock to prevent concurrent map/unmap operation. */
+	ret = dma_resv_lock(vm->dummy_gem.resv, NULL);
+	if (drm_WARN_ON(&vm->ptdev->base, ret))
+		return NULL;
+
+	gpuva = drm_gpuva_find_first(&vm->va_mgr, va, 1);
+	vma = gpuva ? container_of(gpuva, struct panthor_vma, base) : NULL;
+	if (vma && vma->base.gem.obj) {
+		drm_gem_object_get(vma->base.gem.obj);
+		bo = to_panthor_bo(vma->base.gem.obj);
+		*bo_offset = vma->base.gem.offset;
+	}
+	dma_resv_unlock(vm->dummy_gem.resv);
+
+	return bo;
+}
+
+/*
+ * Only 32 VMs per open file. If that becomes a limiting factor, we can
+ * increase this number.
+ */
+#define PANTHOR_MAX_VMS_PER_FILE	 32
+
+/**
+ * panthor_vm_pool_create_vm() - Create a VM
+ * @pool: The VM to create this VM on.
+ * @kernel_va_start: Start of the region reserved for kernel objects.
+ * @kernel_va_range: Size of the region reserved for kernel objects.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_pool_create_vm(struct panthor_device *ptdev, struct panthor_vm_pool *pool,
+			      u64 kernel_va_start, u64 kernel_va_range)
+{
+	struct panthor_vm *vm;
+	int ret;
+	u32 id;
+
+	vm = panthor_vm_create(ptdev, false, kernel_va_start, kernel_va_range);
+	if (IS_ERR(vm))
+		return PTR_ERR(vm);
+
+	ret = xa_alloc(&pool->xa, &id, vm,
+		       XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL);
+
+	if (ret) {
+		panthor_vm_put(vm);
+		return ret;
+	}
+
+	return id;
+}
+
+static void panthor_vm_destroy(struct panthor_vm *vm)
+{
+	if (!vm)
+		return;
+
+	vm->destroyed = true;
+
+	mutex_lock(&vm->heaps.lock);
+	panthor_heap_pool_destroy(vm->heaps.pool);
+	vm->heaps.pool = NULL;
+	mutex_unlock(&vm->heaps.lock);
+
+	drm_WARN_ON(&vm->ptdev->base,
+		    panthor_vm_unmap_range(vm, vm->va_mgr.mm_start, vm->va_mgr.mm_range));
+	panthor_vm_put(vm);
+}
+
+/**
+ * panthor_vm_destroy() - Destroy a VM.
+ * @pool: VM pool.
+ * @handle: VM handle.
+ *
+ * This function doesn't free the VM object or its resources, it just kills
+ * all mappings, and makes sure nothing can be mapped after that point.
+ *
+ * If there was any active jobs at the time this function is called, these
+ * jobs should experience page faults and be killed as a result.
+ *
+ * The VM resources are freed when the last reference on the VM object is
+ * dropped.
+ */
+int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle)
+{
+	struct panthor_vm *vm;
+
+	vm = xa_erase(&pool->xa, handle);
+
+	panthor_vm_destroy(vm);
+
+	return vm ? 0 : -EINVAL;
+}
+
+/**
+ * panthor_vm_pool_get_vm() - Retrieve VM object bound to a VM handle
+ * @pool: VM pool to check.
+ * @handle: Handle of the VM to retrieve.
+ *
+ * Return: A valid pointer if the VM exists, NULL otherwise.
+ */
+struct panthor_vm *
+panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle)
+{
+	struct panthor_vm *vm;
+
+	vm = panthor_vm_get(xa_load(&pool->xa, handle));
+
+	return vm;
+}
+
+/**
+ * panthor_vm_pool_destroy() - Destroy a VM pool.
+ * @pfile: File.
+ *
+ * Destroy all VMs in the pool, and release the pool resources.
+ *
+ * Note that VMs can outlive the pool they were created from if other
+ * objects hold a reference to there VMs.
+ */
+void panthor_vm_pool_destroy(struct panthor_file *pfile)
+{
+	struct panthor_vm *vm;
+	unsigned long i;
+
+	if (!pfile->vms)
+		return;
+
+	xa_for_each(&pfile->vms->xa, i, vm)
+		panthor_vm_destroy(vm);
+
+	xa_destroy(&pfile->vms->xa);
+	kfree(pfile->vms);
+}
+
+/**
+ * panthor_vm_pool_create() - Create a VM pool
+ * @pfile: File.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_pool_create(struct panthor_file *pfile)
+{
+	pfile->vms = kzalloc(sizeof(*pfile->vms), GFP_KERNEL);
+	if (!pfile->vms)
+		return -ENOMEM;
+
+	xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1);
+	return 0;
+}
+
+/* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */
+static void mmu_tlb_flush_all(void *cookie)
+{
+}
+
+static void mmu_tlb_flush_walk(unsigned long iova, size_t size, size_t granule, void *cookie)
+{
+}
+
+static const struct iommu_flush_ops mmu_tlb_ops = {
+	.tlb_flush_all = mmu_tlb_flush_all,
+	.tlb_flush_walk = mmu_tlb_flush_walk,
+};
+
+static const char *access_type_name(struct panthor_device *ptdev,
+				    u32 fault_status)
+{
+	switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
+	case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
+		return "ATOMIC";
+	case AS_FAULTSTATUS_ACCESS_TYPE_READ:
+		return "READ";
+	case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
+		return "WRITE";
+	case AS_FAULTSTATUS_ACCESS_TYPE_EX:
+		return "EXECUTE";
+	default:
+		drm_WARN_ON(&ptdev->base, 1);
+		return NULL;
+	}
+}
+
+static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
+{
+	status = panthor_mmu_fault_mask(ptdev, status);
+	while (status) {
+		u32 as = ffs(status | (status >> 16)) - 1;
+		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
+		u32 new_int_mask;
+		u64 addr;
+		u32 fault_status;
+		u32 exception_type;
+		u32 access_type;
+		u32 source_id;
+
+		fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
+		addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as));
+		addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32;
+
+		/* decode the fault status */
+		exception_type = fault_status & 0xFF;
+		access_type = (fault_status >> 8) & 0x3;
+		source_id = (fault_status >> 16);
+
+		/* Page fault only */
+		mutex_lock(&ptdev->mmu->as.slots_lock);
+
+		new_int_mask =
+			panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);
+
+		/* terminal fault, print info about the fault */
+		drm_err(&ptdev->base,
+			"Unhandled Page fault in AS%d at VA 0x%016llX\n"
+			"raw fault status: 0x%X\n"
+			"decoded fault status: %s\n"
+			"exception type 0x%X: %s\n"
+			"access type 0x%X: %s\n"
+			"source id 0x%X\n",
+			as, addr,
+			fault_status,
+			(fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
+			exception_type, panthor_exception_name(ptdev, exception_type),
+			access_type, access_type_name(ptdev, fault_status),
+			source_id);
+
+		/* Ignore MMU interrupts on this AS until it's been
+		 * re-enabled.
+		 */
+		ptdev->mmu->irq.mask = new_int_mask;
+		gpu_write(ptdev, MMU_INT_MASK, new_int_mask);
+
+		/* Disable the MMU to kill jobs on this AS. */
+		panthor_mmu_as_disable(ptdev, as);
+		mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+		status &= ~mask;
+	}
+}
+PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
+
+/**
+ * panthor_mmu_suspend() - Suspend the MMU logic
+ * @ptdev: Device.
+ *
+ * All we do here is de-assign the AS slots on all active VMs, so things
+ * get flushed to the main memory, and no further access to these VMs are
+ * possible.
+ *
+ * We also suspend the MMU IRQ.
+ */
+void panthor_mmu_suspend(struct panthor_device *ptdev)
+{
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
+		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
+
+		if (vm) {
+			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
+			vm->as.id = -1;
+			list_del_init(&vm->as.lru_node);
+			ptdev->mmu->as.slots[i].vm = NULL;
+		}
+	}
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
+}
+
+/**
+ * panthor_mmu_resume() - Resume the MMU logic
+ * @ptdev: Device.
+ *
+ * Resume the IRQ.
+ *
+ * We don't re-enable previously active VMs. We assume other parts of the
+ * driver will call panthor_vm_active() on the VMs they intend to use.
+ */
+void panthor_mmu_resume(struct panthor_device *ptdev)
+{
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	ptdev->mmu->as.alloc_mask = 0;
+	ptdev->mmu->as.faulty_mask = 0;
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
+}
+
+/**
+ * panthor_mmu_pre_reset() - Prepare for a reset
+ * @ptdev: Device.
+ *
+ * Suspend the IRQ, and make sure all VM_BIND queues are stopped, so we
+ * don't get asked to do a VM operation while the GPU is down.
+ *
+ * We don't cleanly shutdown the AS slots here, because the reset might
+ * come from an AS_ACTIVE_BIT stuck situation.
+ */
+void panthor_mmu_pre_reset(struct panthor_device *ptdev)
+{
+	struct panthor_vm *vm;
+
+	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
+
+	mutex_lock(&ptdev->mmu->vm.lock);
+	ptdev->mmu->vm.reset_in_progress = true;
+	list_for_each_entry(vm, &ptdev->mmu->vm.list, node)
+		panthor_vm_stop(vm);
+	mutex_unlock(&ptdev->mmu->vm.lock);
+}
+
+/**
+ * panthor_mmu_post_reset() - Restore things after a reset
+ * @ptdev: Device.
+ *
+ * Put the MMU logic back in action after a reset. That implies resuming the
+ * IRQ and re-enabling the VM_BIND queues.
+ */
+void panthor_mmu_post_reset(struct panthor_device *ptdev)
+{
+	struct panthor_vm *vm;
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+
+	/* Now that the reset is effective, we can assume that none of the
+	 * AS slots are setup, and clear the faulty flags too.
+	 */
+	ptdev->mmu->as.alloc_mask = 0;
+	ptdev->mmu->as.faulty_mask = 0;
+
+	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
+		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
+
+		if (vm) {
+			vm->as.id = -1;
+			list_del_init(&vm->as.lru_node);
+			ptdev->mmu->as.slots[i].vm = NULL;
+		}
+	}
+
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
+
+	/* Restart the VM_BIND queues. */
+	mutex_lock(&ptdev->mmu->vm.lock);
+	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
+		panthor_vm_start(vm);
+	}
+	ptdev->mmu->vm.reset_in_progress = false;
+	mutex_unlock(&ptdev->mmu->vm.lock);
+}
+
+static void panthor_vm_release(struct kref *kref)
+{
+	struct panthor_vm *vm = container_of(kref, struct panthor_vm, refcount);
+	struct panthor_device *ptdev = vm->ptdev;
+
+	mutex_lock(&vm->heaps.lock);
+	if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))
+		panthor_heap_pool_destroy(vm->heaps.pool);
+	mutex_unlock(&vm->heaps.lock);
+	mutex_destroy(&vm->heaps.lock);
+
+	mutex_lock(&ptdev->mmu->vm.lock);
+	list_del(&vm->node);
+	/* Restore the scheduler state so we can call drm_sched_entity_destroy()
+	 * and drm_sched_fini(). If get there, that means we have no job left
+	 * and no new jobs can be queued, so we can start the scheduler without
+	 * risking interfering with the reset.
+	 */
+	if (ptdev->mmu->vm.reset_in_progress)
+		panthor_vm_start(vm);
+	mutex_unlock(&ptdev->mmu->vm.lock);
+
+	drm_sched_entity_destroy(&vm->entity);
+	drm_sched_fini(&vm->sched);
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	if (vm->as.id >= 0) {
+		int cookie;
+
+		if (drm_dev_enter(&ptdev->base, &cookie)) {
+			panthor_mmu_as_disable(ptdev, vm->as.id);
+			drm_dev_exit(cookie);
+		}
+
+		ptdev->mmu->as.slots[vm->as.id].vm = NULL;
+		clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
+		list_del(&vm->as.lru_node);
+	}
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+	drm_WARN_ON(&ptdev->base,
+		    panthor_vm_unmap_range(vm, vm->va_mgr.mm_start, vm->va_mgr.mm_range));
+
+	free_io_pgtable_ops(vm->pgtbl_ops);
+
+	drm_mm_takedown(&vm->mm);
+	mutex_destroy(&vm->mm_lock);
+	drm_gpuva_manager_destroy(&vm->va_mgr);
+	drm_gem_private_object_fini(&vm->dummy_gem);
+	mutex_destroy(&vm->op_lock);
+	kfree(vm);
+}
+
+/**
+ * panthor_vm_put() - Release a reference on a VM
+ * @vm: VM to release the reference on. Can be NULL.
+ */
+void panthor_vm_put(struct panthor_vm *vm)
+{
+	if (vm)
+		kref_put(&vm->refcount, panthor_vm_release);
+}
+
+/**
+ * panthor_vm_get() - Get a VM reference
+ * @vm: VM to get the reference on. Can be NULL.
+ *
+ * Return: @vm value.
+ */
+struct panthor_vm *panthor_vm_get(struct panthor_vm *vm)
+{
+	if (vm)
+		kref_get(&vm->refcount);
+
+	return vm;
+}
+
+/**
+ * panthor_vm_get_heap_pool() - Get the heap pool attached to a VM
+ * @vm: VM to query the heap pool on.
+ * @create: True if the heap pool should be created when it doesn't exist.
+ *
+ * Heap pools are per-VM. This function allows one to retrieve the heap pool
+ * attached to a VM.
+ *
+ * If no heap pool exists yet, and @create is true, we create one.
+ *
+ * The returned panthor_heap_pool should be released with panthor_heap_pool_put().
+ *
+ * Return: A valid pointer on success, an ERR_PTR() otherwise.
+ */
+struct panthor_heap_pool *panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create)
+{
+	struct panthor_heap_pool *pool;
+
+	mutex_lock(&vm->heaps.lock);
+	if (!vm->heaps.pool && create) {
+		if (vm->destroyed)
+			pool = ERR_PTR(-EINVAL);
+		else
+			pool = panthor_heap_pool_create(vm->ptdev, vm);
+
+		if (!IS_ERR(pool))
+			vm->heaps.pool = panthor_heap_pool_get(pool);
+	} else {
+		pool = panthor_heap_pool_get(vm->heaps.pool);
+	}
+	mutex_unlock(&vm->heaps.lock);
+
+	return pool;
+}
+
+static u64 mair_to_memattr(u64 mair)
+{
+	u64 memattr = 0;
+	u32 i;
+
+	for (i = 0; i < 8; i++) {
+		u8 in_attr = mair >> (8 * i), out_attr;
+		u8 outer = in_attr >> 4, inner = in_attr & 0xf;
+
+		/* For caching to be enabled, inner and outer caching policy
+		 * have to be both write-back, if one of them is write-through
+		 * or non-cacheable, we just choose non-cacheable. Device
+		 * memory is also translated to non-cacheable.
+		 */
+		if (!(outer & 3) || !(outer & 4) || !(inner & 4)) {
+			out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_NC |
+				   AS_MEMATTR_AARCH64_SH_MIDGARD_INNER |
+				   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(false, false);
+		} else {
+			/* Use SH_CPU_INNER mode so SH_IS, which is used when
+			 * IOMMU_CACHE is set, actually maps to the standard
+			 * definition of inner-shareable and not Mali's
+			 * internal-shareable mode.
+			 */
+			out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_WB |
+				   AS_MEMATTR_AARCH64_SH_CPU_INNER |
+				   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(inner & 1, inner & 2);
+		}
+
+		memattr |= (u64)out_attr << (8 * i);
+	}
+
+	return memattr;
+}
+
+static void panthor_vma_link(struct panthor_vm *vm, struct panthor_vma *vma)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
+
+	mutex_lock(&bo->gpuva_list_lock);
+	drm_gpuva_link(&vma->base);
+	mutex_unlock(&bo->gpuva_list_lock);
+
+	if (!bo->exclusive_vm)
+		list_add_tail(&vma->node, &vm->shared_bos);
+}
+
+static void panthor_vma_unlink(struct panthor_vm_op_ctx *op_ctx,
+			       struct panthor_vma *vma)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
+
+	mutex_lock(&bo->gpuva_list_lock);
+	drm_gpuva_unlink(&vma->base);
+	mutex_unlock(&bo->gpuva_list_lock);
+
+	list_move_tail(&vma->node, &op_ctx->returned_vmas);
+}
+
+static void panthor_vma_init(struct panthor_vma *vma,
+			     struct drm_gem_object *obj,
+			     u64 offset,
+			     u64 va, u64 range, u32 flags)
+{
+	INIT_LIST_HEAD(&vma->node);
+	vma->flags = flags;
+	vma->base.gem.obj = obj;
+	vma->base.gem.offset = offset;
+	vma->base.va.addr = va;
+	vma->base.va.range = range;
+}
+
+#define PANTHOR_VM_MAP_FLAGS \
+	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
+	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
+	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED)
+
+static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
+{
+	struct panthor_vm *vm = priv;
+	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
+	struct panthor_vma *vma = op_ctx->map.new_vma;
+	int ret;
+
+	panthor_vma_init(vma, op->map.gem.obj, op->map.gem.offset, op->map.va.addr,
+			 op->map.va.range, op_ctx->flags & PANTHOR_VM_MAP_FLAGS);
+
+	ret = panthor_vm_map_pages(vm, vma->base.va.addr, flags_to_prot(vma->flags),
+				   op_ctx->map.sgt, vma->base.gem.offset,
+				   vma->base.va.range);
+	if (ret)
+		return ret;
+
+	/* Ref owned by the mapping now, clear the obj field so we don't release the
+	 * pinning/obj ref behind GPUVA's back.
+	 */
+	drm_gpuva_map(&vm->va_mgr, &vma->base, &op->map);
+	panthor_vma_link(vm, op_ctx->map.new_vma);
+	op_ctx->map.gem.obj = NULL;
+	op_ctx->map.new_vma = NULL;
+	return 0;
+}
+
+static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
+				       void *priv)
+{
+	struct panthor_vma *unmap_vma = container_of(op->remap.unmap->va, struct panthor_vma, base);
+	const u64 va_start = op->remap.prev ?
+			     op->remap.prev->va.addr + op->remap.prev->va.range :
+			     op->remap.unmap->va->va.addr;
+	const u64 va_end = op->remap.next ?
+			   op->remap.next->va.addr :
+			   op->remap.unmap->va->va.addr + op->remap.unmap->va->va.range;
+	struct panthor_vm *vm = priv;
+	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
+	struct drm_gpuva *prev_va = NULL, *next_va = NULL;
+	int ret;
+
+	ret = panthor_vm_unmap_pages(vm, va_start, va_end - va_start);
+	if (ret)
+		return ret;
+
+	if (op->remap.prev) {
+		struct panthor_gem_object *bo = to_panthor_bo(op->remap.prev->gem.obj);
+
+		if (!bo->base.base.import_attach) {
+			ret = drm_gem_shmem_pin(&bo->base);
+			if (drm_WARN_ON(&vm->ptdev->base, ret))
+				return ret;
+		}
+
+		panthor_vma_init(op_ctx->map.prev_vma,
+				 op->remap.prev->gem.obj,
+				 op->remap.prev->gem.offset,
+				 op->remap.prev->va.addr,
+				 op->remap.prev->va.range,
+				 unmap_vma->flags);
+		prev_va = &op_ctx->map.prev_vma->base;
+	}
+
+	if (op->remap.next) {
+		struct panthor_gem_object *bo = to_panthor_bo(op->remap.next->gem.obj);
+
+		if (!bo->base.base.import_attach) {
+			ret = drm_gem_shmem_pin(&bo->base);
+			if (drm_WARN_ON(&vm->ptdev->base, ret))
+				return ret;
+		}
+
+		panthor_vma_init(op_ctx->map.next_vma,
+				 op->remap.next->gem.obj,
+				 op->remap.next->gem.offset,
+				 op->remap.next->va.addr,
+				 op->remap.next->va.range,
+				 unmap_vma->flags);
+		next_va = &op_ctx->map.next_vma->base;
+	}
+
+	drm_gpuva_remap(prev_va, next_va, &op->remap);
+
+	if (prev_va) {
+		drm_gem_object_get(prev_va->gem.obj);
+		panthor_vma_link(vm, op_ctx->map.prev_vma);
+		op_ctx->map.prev_vma = NULL;
+	}
+
+	if (next_va) {
+		drm_gem_object_get(next_va->gem.obj);
+		panthor_vma_link(vm, op_ctx->map.next_vma);
+		op_ctx->map.next_vma = NULL;
+	}
+
+	panthor_vma_unlink(op_ctx, unmap_vma);
+	return 0;
+}
+
+static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op,
+				       void *priv)
+{
+	struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base);
+	struct panthor_vm *vm = priv;
+	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
+	int ret;
+
+	ret = panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
+				     unmap_vma->base.va.range);
+	if (drm_WARN_ON(&vm->ptdev->base, ret))
+		return ret;
+
+	drm_gpuva_unmap(&op->unmap);
+	panthor_vma_unlink(op_ctx, unmap_vma);
+	return 0;
+}
+
+static const struct drm_gpuva_fn_ops panthor_gpuva_ops = {
+	.sm_step_map = panthor_gpuva_sm_step_map,
+	.sm_step_remap = panthor_gpuva_sm_step_remap,
+	.sm_step_unmap = panthor_gpuva_sm_step_unmap,
+};
+
+/**
+ * panthor_vm_resv() - Get the dma_resv object attached to a VM.
+ * @vm: VM to get the dma_resv of.
+ *
+ * Return: A dma_resv object.
+ */
+struct dma_resv *panthor_vm_resv(struct panthor_vm *vm)
+{
+	return vm->dummy_gem.resv;
+}
+
+static int
+panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op,
+		   bool flag_vm_unusable_on_failure)
+{
+	int ret;
+
+	mutex_lock(&vm->op_lock);
+	vm->op_ctx = op;
+	switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
+		if (vm->unusable) {
+			ret = -EINVAL;
+			break;
+		}
+
+		ret = drm_gpuva_sm_map(&vm->va_mgr, vm, op->va.addr, op->va.range,
+				       op->map.gem.obj, op->map.gem.offset);
+		break;
+
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
+		ret = drm_gpuva_sm_unmap(&vm->va_mgr, vm, op->va.addr, op->va.range);
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	if (ret && flag_vm_unusable_on_failure)
+		vm->unusable = true;
+
+	vm->op_ctx = NULL;
+	mutex_unlock(&vm->op_lock);
+
+	return ret;
+}
+
+static struct dma_fence *
+panthor_vm_bind_run_job(struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
+	bool cookie;
+	int ret;
+
+	/* Not only we report an error whose result is propagated to the
+	 * drm_sched finished fence, but we also flag the VM as unusable, because
+	 * a failure in the async VM_BIND results in an inconsistent state. VM needs
+	 * to be destroyed and recreated.
+	 */
+	cookie = dma_fence_begin_signalling();
+	ret = panthor_vm_exec_op(job->vm, &job->ctx, true);
+	dma_fence_end_signalling(cookie);
+
+	return ret ? ERR_PTR(ret) : NULL;
+}
+
+static void panthor_vm_bind_job_release(struct kref *kref)
+{
+	struct panthor_vm_bind_job *job = container_of(kref, struct panthor_vm_bind_job, refcount);
+
+	if (job->base.s_fence)
+		drm_sched_job_cleanup(&job->base);
+
+	panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);
+	panthor_vm_put(job->vm);
+	kfree(job);
+}
+
+/**
+ * panthor_vm_bind_job_put() - Release a VM_BIND job reference
+ * @sched_job: Job to release the reference on.
+ */
+void panthor_vm_bind_job_put(struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job =
+		container_of(sched_job, struct panthor_vm_bind_job, base);
+
+	if (sched_job)
+		kref_put(&job->refcount, panthor_vm_bind_job_release);
+}
+
+static void
+panthor_vm_bind_free_job(struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job =
+		container_of(sched_job, struct panthor_vm_bind_job, base);
+
+	drm_sched_job_cleanup(sched_job);
+
+	/* Do the heavy cleanups asynchronously, so we're out of the
+	 * dma-signaling path and can acquire dma-resv locks safely.
+	 */
+	queue_work(panthor_cleanup_wq, &job->cleanup_op_ctx_work);
+}
+
+static enum drm_gpu_sched_stat
+panthor_vm_bind_timedout_job(struct drm_sched_job *sched_job)
+{
+	WARN(1, "VM_BIND ops are synchronous for now, there should be no timeout!");
+	return DRM_GPU_SCHED_STAT_NOMINAL;
+}
+
+static const struct drm_sched_backend_ops panthor_vm_bind_ops = {
+	.run_job = panthor_vm_bind_run_job,
+	.free_job = panthor_vm_bind_free_job,
+	.timedout_job = panthor_vm_bind_timedout_job,
+};
+
+/**
+ * panthor_vm_create() - Create a VM
+ * @ptdev: Device.
+ * @for_mcu: True if this is the FW MCU VM.
+ * @auto_va_start: Start of the auto-VA range.
+ * @auto_va_range: Size of the auto-VA range.
+ *
+ * Return: A valid pointer on success, an ERR_PTR() otherwise.
+ */
+struct panthor_vm *
+panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
+		  u64 auto_va_start, u64 auto_va_range)
+{
+	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
+	u32 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features);
+	struct drm_gpu_scheduler *sched;
+	struct io_pgtable_cfg pgtbl_cfg;
+	u64 mair, min_va, va_range;
+	struct panthor_vm *vm;
+	int ret;
+
+	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
+	if (!vm)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&vm->heaps.lock);
+	kref_init(&vm->refcount);
+	drm_gem_private_object_init(&ptdev->base, &vm->dummy_gem, 0);
+	vm->for_mcu = for_mcu;
+	vm->ptdev = ptdev;
+	INIT_LIST_HEAD(&vm->shared_bos);
+	mutex_init(&vm->op_lock);
+
+	if (for_mcu) {
+		/* CSF MCU is a cortex M7, and can only address 4G */
+		min_va = 0;
+		va_range = SZ_4G;
+	} else {
+		min_va = 0;
+		va_range = (1ull << va_bits);
+
+		/* If the auto_va_range is zero, we reserve half of the VA
+		 * space for kernel stuff.
+		 */
+		if (!auto_va_range) {
+			auto_va_range = va_range / 2;
+			auto_va_start = va_range - auto_va_range;
+		}
+	}
+
+	mutex_init(&vm->mm_lock);
+	drm_mm_init(&vm->mm, auto_va_start, auto_va_range);
+
+	/* We intentionally leave the reserved range to zero, because we want kernel VMAs
+	 * to be handled the same way user VMAs are.
+	 */
+	drm_gpuva_manager_init(&vm->va_mgr,
+			       for_mcu ? "panthor-MCU-VA-manager" : "panthor-GPU-VA-manager",
+			       min_va, va_range, 0, 0,
+			       &panthor_gpuva_ops);
+	INIT_LIST_HEAD(&vm->node);
+	INIT_LIST_HEAD(&vm->as.lru_node);
+	vm->as.id = -1;
+
+	pgtbl_cfg = (struct io_pgtable_cfg) {
+		.pgsize_bitmap	= SZ_4K | SZ_2M,
+		.ias		= va_bits,
+		.oas		= pa_bits,
+		.coherent_walk	= ptdev->coherent,
+		.tlb		= &mmu_tlb_ops,
+		.iommu_dev	= ptdev->base.dev,
+		.alloc		= alloc_pt,
+		.free		= free_pt,
+	};
+
+	vm->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, &pgtbl_cfg, vm);
+	if (!vm->pgtbl_ops) {
+		ret = -EINVAL;
+		goto err_gpuva_destroy;
+	}
+
+	/* Bind operations are synchronous for now, no timeout needed. */
+	ret = drm_sched_init(&vm->sched, &panthor_vm_bind_ops, ptdev->mmu->vm.wq, 1, 0,
+			     MAX_SCHEDULE_TIMEOUT, NULL, NULL,
+			     "panthor-vm-bind", DRM_SCHED_POLICY_SINGLE_ENTITY,
+			     ptdev->base.dev);
+	if (ret)
+		goto err_free_io_pgtable;
+
+	sched = &vm->sched;
+	ret = drm_sched_entity_init(&vm->entity, DRM_SCHED_PRIORITY_NORMAL,
+				    &sched, 1, NULL);
+	if (ret)
+		goto err_sched_fini;
+
+	mair = io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg.arm_lpae_s1_cfg.mair;
+	vm->memattr = mair_to_memattr(mair);
+
+	mutex_lock(&ptdev->mmu->vm.lock);
+	list_add_tail(&vm->node, &ptdev->mmu->vm.list);
+
+	/* If a reset is in progress, stop the scheduler. */
+	if (ptdev->mmu->vm.reset_in_progress)
+		panthor_vm_stop(vm);
+	mutex_unlock(&ptdev->mmu->vm.lock);
+
+	return vm;
+
+err_sched_fini:
+	drm_sched_fini(&vm->sched);
+
+err_free_io_pgtable:
+	free_io_pgtable_ops(vm->pgtbl_ops);
+
+err_gpuva_destroy:
+	drm_mm_takedown(&vm->mm);
+	drm_gpuva_manager_destroy(&vm->va_mgr);
+	drm_gem_private_object_fini(&vm->dummy_gem);
+	kfree(vm);
+
+	return ERR_PTR(ret);
+}
+
+static int
+panthor_vm_bind_prepare_op_ctx(struct drm_file *file,
+			       struct panthor_vm *vm,
+			       const struct drm_panthor_vm_bind_op *op,
+			       struct panthor_vm_op_ctx *op_ctx)
+{
+	struct drm_gem_object *gem;
+	int ret;
+
+	/* Aligned on page size. */
+	if ((op->va | op->size) & ~PAGE_MASK)
+		return -EINVAL;
+
+	switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
+		gem = drm_gem_object_lookup(file, op->bo_handle);
+		ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm,
+						    gem ? to_panthor_bo(gem) : NULL,
+						    op->bo_offset,
+						    op->size,
+						    op->va,
+						    op->flags);
+		drm_gem_object_put(gem);
+		return ret;
+
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
+		return panthor_vm_prepare_unmap_op_ctx(op_ctx, vm, op->va, op->size);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static void panthor_vm_bind_job_cleanup_op_ctx_work(struct work_struct *work)
+{
+	struct panthor_vm_bind_job *job =
+		container_of(work, struct panthor_vm_bind_job, cleanup_op_ctx_work);
+
+	panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);
+	panthor_vm_bind_job_put(&job->base);
+}
+
+/**
+ * panthor_vm_bind_job_create() - Create a VM_BIND job
+ * @file: File.
+ * @vm: VM targeted by the VM_BIND job.
+ * @op: VM operation data.
+ *
+ * Return: A valid pointer on success, an ERR_PTR() otherwise.
+ */
+struct drm_sched_job *
+panthor_vm_bind_job_create(struct drm_file *file,
+			   struct panthor_vm *vm,
+			   const struct drm_panthor_vm_bind_op *op)
+{
+	struct panthor_vm_bind_job *job;
+	int ret;
+
+	if (!vm)
+		return ERR_PTR(-EINVAL);
+
+	if (vm->destroyed || vm->unusable)
+		return ERR_PTR(-EINVAL);
+
+	job = kzalloc(sizeof(*job), GFP_KERNEL);
+	if (!job)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_WORK(&job->cleanup_op_ctx_work, panthor_vm_bind_job_cleanup_op_ctx_work);
+	kref_init(&job->refcount);
+	job->vm = panthor_vm_get(vm);
+
+	ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &job->ctx);
+	if (ret)
+		goto err_put_job;
+
+	ret = drm_sched_job_init(&job->base, &vm->entity, vm);
+	if (ret)
+		goto err_put_job;
+
+	return &job->base;
+
+err_put_job:
+	panthor_vm_bind_job_put(&job->base);
+	return ERR_PTR(ret);
+}
+
+/**
+ * panthor_vm_bind_job_prepare_resvs() - Prepare VM_BIND job dma_resvs
+ * @exec: The locking/preparation context.
+ * @sched_job: The job to prepare resvs on.
+ *
+ * Locks and prepare the VM resv.
+ *
+ * If this is a map operation, locks and prepares the GEM resv.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,
+				      struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
+	int ret;
+
+	/* Acquire the VM lock an reserve a slot for this VM bind job. */
+	ret = drm_exec_prepare_obj(exec, &job->vm->dummy_gem, 1);
+	if (ret)
+		return ret;
+
+	if (job->ctx.map.gem.obj) {
+		/* Lock/prepare the GEM being mapped. */
+		ret = drm_exec_prepare_obj(exec, job->ctx.map.gem.obj, 1);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_vm_bind_job_add_resvs_deps() - Add implicit deps to the VM_BIND job
+ * @sched_job: Job to add implicit deps on.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_bind_job_add_resvs_deps(struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
+	int ret;
+
+	/* We use explicit fencing, so no need to wait for anything else but
+	 * DMA_RESV_USAGE_KERNEL on the BO being mapped or VM. If there are extra
+	 * dependencies, they should be passed to the VM_BIND ioctl.
+	 */
+	ret = drm_sched_job_add_resv_dependencies(sched_job,
+						  job->vm->dummy_gem.resv,
+						  DMA_RESV_USAGE_KERNEL);
+	if (ret)
+		return ret;
+
+	if (job->ctx.map.gem.obj) {
+		ret = drm_sched_job_add_resv_dependencies(sched_job,
+							  job->ctx.map.gem.obj->resv,
+							  DMA_RESV_USAGE_KERNEL);
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_vm_bind_job_update_resvs() - Update the resv objects touched by a job
+ * @sched_job: Job to update the resvs on.
+ */
+void panthor_vm_bind_job_update_resvs(struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
+
+	/* Explicit sync => we just register our job finished fence as bookkeep. */
+	dma_resv_add_fence(job->vm->dummy_gem.resv,
+			   &sched_job->s_fence->finished,
+			   DMA_RESV_USAGE_BOOKKEEP);
+
+	if (job->ctx.map.gem.obj) {
+		dma_resv_add_fence(job->ctx.map.gem.obj->resv,
+				   &sched_job->s_fence->finished,
+				   DMA_RESV_USAGE_BOOKKEEP);
+	}
+}
+
+/**
+ * panthor_vm_bind_exec_sync_op() - Execute a VM_BIND operation synchronously.
+ * @file: File.
+ * @vm: VM targeted by the VM operation.
+ * @op: Data describing the VM operation.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_bind_exec_sync_op(struct drm_file *file,
+				 struct panthor_vm *vm,
+				 struct drm_panthor_vm_bind_op *op)
+{
+	struct panthor_vm_op_ctx op_ctx;
+	int ret;
+
+	/* No sync objects allowed on synchronous operations. */
+	if (op->syncs.count)
+		return -EINVAL;
+
+	if (!op->size)
+		return 0;
+
+	ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &op_ctx);
+	if (ret)
+		return ret;
+
+	ret = panthor_vm_exec_op(vm, &op_ctx, false);
+	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
+
+	return ret;
+}
+
+/**
+ * panthor_vm_map_bo_range() - Map a GEM object range to a VM
+ * @vm: VM to map the GEM to.
+ * @bo: GEM object to map.
+ * @offset: Offset in the GEM object.
+ * @size: Size to map.
+ * @va: Virtual address to map the object to.
+ * @flags: Combination of drm_panthor_vm_bind_op_flags flags.
+ * Only map-related flags are valid.
+ *
+ * Internal use only. For userspace requests, use
+ * panthor_vm_bind_exec_sync_op() instead.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
+			    u64 offset, size_t size, u64 va, u32 flags)
+{
+	struct panthor_vm_op_ctx op_ctx;
+	int ret;
+
+	ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags);
+	if (ret)
+		return ret;
+
+	ret = panthor_vm_exec_op(vm, &op_ctx, false);
+	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
+
+	return ret;
+}
+
+/**
+ * panthor_vm_unmap_range() - Unmap a portion of the VA space
+ * @vm: VM to unmap the region from.
+ * @va: Virtual address to unmap. Must be 4k aligned.
+ * @size: Size of the region to unmap. Must be 4k aligned.
+ *
+ * Internal use only. For userspace requests, use
+ * panthor_vm_bind_exec_sync_op() instead.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, size_t size)
+{
+	struct panthor_vm_op_ctx op_ctx;
+	int ret;
+
+	ret = panthor_vm_prepare_unmap_op_ctx(&op_ctx, vm, va, size);
+	if (ret)
+		return ret;
+
+	ret = panthor_vm_exec_op(vm, &op_ctx, false);
+	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
+
+	return ret;
+}
+
+/**
+ * panthor_vm_prepare_mapped_bos_resvs() - Prepare resvs on VM BOs.
+ * @exec: Locking/preparation context.
+ * @vm: VM targeted by the GPU job.
+ *
+ * GPU jobs assume all BOs bound to the VM at the time the job is submitted
+ * are available when the job is executed. In order to guarantee that, we
+ * need to reserve a slot on all BOs mapped to a VM and update this slot with
+ * the job fence after its submission.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm *vm)
+{
+	struct panthor_vma *vma;
+	int ret;
+
+	/* Acquire the VM lock an reserve a slot for this GPU job. */
+	ret = drm_exec_prepare_obj(exec, &vm->dummy_gem, 1);
+	if (ret)
+		return ret;
+
+	list_for_each_entry(vma, &vm->shared_bos, node) {
+		ret = drm_exec_prepare_obj(exec, vma->base.gem.obj, 1);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_vm_add_bos_resvs_deps_to_job() - Add implicit VM deps to a GPU job
+ * @vm: VM targeted by the GPU job.
+ * @job: GPU job.
+ *
+ * We just take care of kernel access. Other accesses should be passed as
+ * explicit dependencies to the job.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_add_bos_resvs_deps_to_job(struct panthor_vm *vm,
+					 struct drm_sched_job *job)
+{
+	struct panthor_vma *vma;
+	int ret;
+
+	/* We use explicit fencing, so no need to wait for anything else but
+	 * DMA_RESV_USAGE_KERNEL on the BO being mapped or VM. If there are extra
+	 * dependencies, they should be passed to the VM_BIND ioctl.
+	 */
+	ret = drm_sched_job_add_resv_dependencies(job,
+						  vm->dummy_gem.resv,
+						  DMA_RESV_USAGE_KERNEL);
+	if (ret)
+		return ret;
+
+	list_for_each_entry(vma, &vm->shared_bos, node) {
+		ret = drm_sched_job_add_resv_dependencies(job,
+							  vma->base.gem.obj->resv,
+							  DMA_RESV_USAGE_KERNEL);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_vm_add_job_fence_to_bos_resvs() - Add GPU job fence to GEM resvs
+ * @vm: VM targeted by the GPU job.
+ * @job: GPU job.
+ *
+ * Update the GEM resvs after a job has been submitted. All GEMs currently
+ * bound to the VMs get the job fence added to their resv as bookkeep. If
+ * another type of implicit dependency is needed, it should be updated
+ * with %DMA_BUF_IOCTL_IMPORT_SYNC_FILE after the
+ * %DRM_IOCTL_PANTHOR_GROUP_SUBMIT ioctl has returned.
+ */
+void panthor_vm_add_job_fence_to_bos_resvs(struct panthor_vm *vm,
+					   struct drm_sched_job *job)
+{
+	struct panthor_vma *vma;
+
+	/* Explicit sync => we just register our job finished fence as bookkeep. */
+	dma_resv_add_fence(vm->dummy_gem.resv,
+			   &job->s_fence->finished,
+			   DMA_RESV_USAGE_BOOKKEEP);
+
+	list_for_each_entry(vma, &vm->shared_bos, node) {
+		dma_resv_add_fence(vma->base.gem.obj->resv,
+				   &job->s_fence->finished,
+				   DMA_RESV_USAGE_BOOKKEEP);
+	}
+}
+
+/**
+ * panthor_mmu_unplug() - Unplug the MMU logic
+ * @ptdev: Device.
+ *
+ * No access to the MMU regs should be done after this function is called.
+ * We suspend the IRQ and disable all VMs to guarantee that.
+ */
+void panthor_mmu_unplug(struct panthor_device *ptdev)
+{
+	if (ptdev->mmu->irq.irq > 0)
+		panthor_mmu_irq_suspend(&ptdev->mmu->irq);
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
+		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
+
+		if (vm) {
+			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
+			vm->as.id = -1;
+			list_del_init(&vm->as.lru_node);
+			clear_bit(i, &ptdev->mmu->as.alloc_mask);
+			ptdev->mmu->as.slots[i].vm = NULL;
+		}
+	}
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+}
+
+static void panthor_mmu_release_wq(struct drm_device *ddev, void *res)
+{
+	destroy_workqueue(res);
+}
+
+/**
+ * panthor_mmu_init() - Initialize the MMU logic.
+ * @ptdev: Device.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_mmu_init(struct panthor_device *ptdev)
+{
+	struct panthor_mmu *mmu;
+	int ret, irq;
+
+	mmu = drmm_kzalloc(&ptdev->base, sizeof(*mmu), GFP_KERNEL);
+	if (!mmu)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&mmu->as.lru_list);
+
+	for (u32 i = 0; i < ARRAY_SIZE(mmu->as.slots); i++)
+		spin_lock_init(&mmu->as.slots[i].lock);
+
+	drmm_mutex_init(&ptdev->base, &mmu->as.slots_lock);
+	INIT_LIST_HEAD(&mmu->vm.list);
+	drmm_mutex_init(&ptdev->base, &mmu->vm.lock);
+
+	ptdev->mmu = mmu;
+
+	irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "mmu");
+	if (irq <= 0)
+		return -ENODEV;
+
+	ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq,
+				      panthor_mmu_fault_mask(ptdev, ~0));
+	if (ret)
+		return ret;
+
+	mmu->vm.wq = alloc_workqueue("panthor-vm-bind", WQ_UNBOUND, 0);
+	if (!mmu->vm.wq)
+		return -ENOMEM;
+
+	return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq);
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int show_vm_gpuvas(struct panthor_vm *vm, struct seq_file *m)
+{
+	int ret;
+
+	mutex_lock(&vm->op_lock);
+	ret = drm_debugfs_gpuva_info(m, &vm->va_mgr);
+	mutex_unlock(&vm->op_lock);
+
+	return ret;
+}
+
+static int show_each_vm(struct seq_file *m, void *arg)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *ddev = node->minor->dev;
+	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
+	int (*show)(struct panthor_vm *, struct seq_file *) = node->info_ent->data;
+	struct panthor_vm *vm;
+	int ret = 0;
+
+	mutex_lock(&ptdev->mmu->vm.lock);
+	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
+		ret = show(vm, m);
+		if (ret < 0)
+			break;
+
+		seq_puts(m, "\n");
+	}
+	mutex_unlock(&ptdev->mmu->vm.lock);
+
+	return ret;
+}
+
+static struct drm_info_list panthor_mmu_debugfs_list[] = {
+	DRM_DEBUGFS_GPUVA_INFO(show_each_vm, show_vm_gpuvas),
+};
+
+/**
+ * panthor_mmu_debugfs_init() - Initialize MMU debugfs entries
+ * @minor: Minor.
+ */
+void panthor_mmu_debugfs_init(struct drm_minor *minor)
+{
+	drm_debugfs_create_files(panthor_mmu_debugfs_list,
+				 ARRAY_SIZE(panthor_mmu_debugfs_list),
+				 minor->debugfs_root, minor);
+}
+#endif /* CONFIG_DEBUG_FS */
+
+/**
+ * panthor_mmu_pt_cache_init() - Initialize the page table cache.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_mmu_pt_cache_init(void)
+{
+	pt_cache = kmem_cache_create("panthor-mmu-pt", SZ_4K, SZ_4K, 0, NULL);
+	if (!pt_cache)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * panthor_mmu_pt_cache_fini() - Destroy the page table cache.
+ */
+void panthor_mmu_pt_cache_fini(void)
+{
+	kmem_cache_destroy(pt_cache);
+}
diff --git a/drivers/gpu/drm/panthor/panthor_mmu.h b/drivers/gpu/drm/panthor/panthor_mmu.h
new file mode 100644
index 000000000000..d94925ccdc8c
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_mmu.h
@@ -0,0 +1,81 @@ 
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <[e-mail]> */
+/* Copyright 2023 Collabora ltd. */
+
+#ifndef __PANTHOR_MMU_H__
+#define __PANTHOR_MMU_H__
+
+struct drm_exec;
+struct drm_sched_job;
+struct panthor_gem_object;
+struct panthor_heap_pool;
+struct panthor_vm;
+struct panthor_vma;
+struct panthor_mmu;
+
+int panthor_mmu_init(struct panthor_device *ptdev);
+void panthor_mmu_unplug(struct panthor_device *ptdev);
+void panthor_mmu_pre_reset(struct panthor_device *ptdev);
+void panthor_mmu_post_reset(struct panthor_device *ptdev);
+void panthor_mmu_suspend(struct panthor_device *ptdev);
+void panthor_mmu_resume(struct panthor_device *ptdev);
+
+int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
+			    u64 offset, size_t size, u64 va, u32 flags);
+int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, size_t size);
+struct panthor_gem_object *
+panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset);
+
+int panthor_vm_active(struct panthor_vm *vm);
+void panthor_vm_idle(struct panthor_vm *vm);
+int panthor_vm_as(struct panthor_vm *vm);
+
+struct panthor_heap_pool *
+panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create);
+
+struct panthor_vm *panthor_vm_get(struct panthor_vm *vm);
+void panthor_vm_put(struct panthor_vm *vm);
+struct panthor_vm *panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
+				     u64 auto_va_start, u64 auto_va_range);
+
+int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec,
+					struct panthor_vm *vm);
+int panthor_vm_add_bos_resvs_deps_to_job(struct panthor_vm *vm,
+					 struct drm_sched_job *job);
+void panthor_vm_add_job_fence_to_bos_resvs(struct panthor_vm *vm,
+					   struct drm_sched_job *job);
+
+struct dma_resv *panthor_vm_resv(struct panthor_vm *vm);
+
+void panthor_vm_pool_destroy(struct panthor_file *pfile);
+int panthor_vm_pool_create(struct panthor_file *pfile);
+int panthor_vm_pool_create_vm(struct panthor_device *ptdev, struct panthor_vm_pool *pool,
+			      u64 kernel_va_start, u64 kernel_va_range);
+int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle);
+struct panthor_vm *panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle);
+
+struct drm_mm_node *panthor_vm_alloc_va(struct panthor_vm *vm, size_t size);
+void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *mm_node);
+
+int panthor_vm_bind_exec_sync_op(struct drm_file *file,
+				 struct panthor_vm *vm,
+				 struct drm_panthor_vm_bind_op *op);
+
+struct drm_sched_job *
+panthor_vm_bind_job_create(struct drm_file *file,
+			   struct panthor_vm *vm,
+			   const struct drm_panthor_vm_bind_op *op);
+void panthor_vm_bind_job_put(struct drm_sched_job *job);
+int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,
+				      struct drm_sched_job *job);
+int panthor_vm_bind_job_add_resvs_deps(struct drm_sched_job *job);
+void panthor_vm_bind_job_update_resvs(struct drm_sched_job *job);
+
+int panthor_mmu_pt_cache_init(void);
+void panthor_mmu_pt_cache_fini(void);
+
+#ifdef CONFIG_DEBUG_FS
+void panthor_mmu_debugfs_init(struct drm_minor *minor);
+#endif
+
+#endif