diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig
index fb7446d6d682..7e8c64f61486 100644
--- a/drivers/nvme/target/Kconfig
+++ b/drivers/nvme/target/Kconfig
@@ -94,6 +94,17 @@ config NVME_TARGET_TCP
 
 	  If unsure, say N.
 
+config NVME_TARGET_MDEV_PCI
+	tristate "NVMe MDEV VFIO PCI target support"
+	select VFIO
+	select VFIO_MDEV
+	depends on NVME_TARGET
+	help
+	  This enables the NVMe MDEV VFIO PCI target support, which allows
+	  exporting NVMe PCI devices using VFIO.
+
+	  If unsure, say N.
+
 config NVME_TARGET_TCP_TLS
 	bool "NVMe over Fabrics TCP target TLS encryption support"
 	depends on NVME_TARGET_TCP
diff --git a/drivers/nvme/target/Makefile b/drivers/nvme/target/Makefile
index ed8522911d1f..369dab7a03f5 100644
--- a/drivers/nvme/target/Makefile
+++ b/drivers/nvme/target/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_NVME_TARGET_FC)		+= nvmet-fc.o
 obj-$(CONFIG_NVME_TARGET_FCLOOP)	+= nvme-fcloop.o
 obj-$(CONFIG_NVME_TARGET_TCP)		+= nvmet-tcp.o
 obj-$(CONFIG_NVME_TARGET_PCI_EPF)	+= nvmet-pci-epf.o
+obj-$(CONFIG_NVME_TARGET_MDEV_PCI)	+= mdev-pci/
 
 nvmet-y		+= core.o configfs.o admin-cmd.o fabrics-cmd.o \
 			discovery.o io-cmd-file.o io-cmd-bdev.o pr.o
diff --git a/drivers/nvme/target/mdev-pci/Makefile b/drivers/nvme/target/mdev-pci/Makefile
new file mode 100644
index 000000000000..03cc34b13e07
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/Makefile
@@ -0,0 +1,5 @@
+
+obj-$(CONFIG_NVME_TARGET_MDEV_PCI) 	+= nvmet-mdev-pci.o
+
+nvmet-mdev-pci-y += instance.o target.o io.o irq.o udata.o viommu.o vsq.o \
+			vcq.o vctrl.o mmio.o pci.o
diff --git a/drivers/nvme/target/mdev-pci/instance.c b/drivers/nvme/target/mdev-pci/instance.c
new file mode 100644
index 000000000000..c14bcf47c409
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/instance.c
@@ -0,0 +1,818 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Mediated NVMe instance VFIO code
+ * Copyright (c) 2019 - Maxim Levitsky
+ * Copyright (C) 2025 Oracle Corporation
+ */
+
+#include <linux/init.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/vfio.h>
+#include <linux/sysfs.h>
+#include <linux/mdev.h>
+#include "../../../vfio/vfio.h"
+#include "../nvmet.h"
+#include "priv.h"
+
+#define OFFSET_TO_REGION(offset) ((offset) >> 20)
+#define REGION_TO_OFFSET(nr) (((u64)nr) << 20)
+
+#define NVMET_MDEV_NAME "nvmet_mdev_pci"
+
+static struct device *nvmet_mdev_root_dev;
+
+struct mdev_nvme_vfio_region_info {
+	struct vfio_region_info base;
+	struct vfio_region_info_cap_sparse_mmap mmap_cap;
+};
+
+/* User memory removed */
+static void nvmet_mdev_dma_unmap(struct vfio_device *vfio_dev, u64 iova,
+				 u64 length)
+{
+	struct nvmet_mdev_vctrl *vctrl = vfio_dev_to_nvmet_mdev_vctrl(vfio_dev);
+
+	mutex_lock(&vctrl->lock);
+	/*
+	 * TODO: This cannot be run while the device is in use.
+	 */
+	nvmet_mdev_vctrl_viommu_unmap(vctrl, iova, length);
+	mutex_unlock(&vctrl->lock);
+}
+
+/* Helper function for bar/pci config read/write access */
+static ssize_t nvmet_mdev_access(struct nvmet_mdev_vctrl *vctrl,
+				 char *buf, size_t count,
+				 loff_t pos, bool is_write)
+{
+	int index = OFFSET_TO_REGION(pos);
+	int ret = -EINVAL;
+	unsigned int offset;
+
+	if (index >= VFIO_PCI_NUM_REGIONS || !vctrl->regions[index].rw)
+		goto out;
+
+	offset = pos - REGION_TO_OFFSET(index);
+	if (offset + count > vctrl->regions[index].size)
+		goto out;
+
+	ret = vctrl->regions[index].rw(vctrl, offset, buf, count, is_write);
+out:
+	return ret;
+}
+
+/* Called when read() is done on the device */
+static ssize_t nvmet_mdev_read(struct vfio_device *vfio_dev, char __user *buf,
+			       size_t count, loff_t *ppos)
+{
+	struct nvmet_mdev_vctrl *vctrl = vfio_dev_to_nvmet_mdev_vctrl(vfio_dev);
+	unsigned int done = 0;
+	int ret;
+
+	while (count) {
+		size_t filled;
+
+		if (count >= 4 && !(*ppos % 4)) {
+			u32 val;
+
+			ret = nvmet_mdev_access(vctrl, (char *)&val,
+						sizeof(val), *ppos, false);
+			if (ret <= 0)
+				goto read_err;
+
+			if (copy_to_user(buf, &val, sizeof(val)))
+				goto read_err;
+			filled = sizeof(val);
+		} else if (count >= 2 && !(*ppos % 2)) {
+			u16 val;
+
+			ret = nvmet_mdev_access(vctrl, (char *)&val,
+						sizeof(val), *ppos, false);
+			if (ret <= 0)
+				goto read_err;
+			if (copy_to_user(buf, &val, sizeof(val)))
+				goto read_err;
+			filled = sizeof(val);
+		} else {
+			u8 val;
+
+			ret = nvmet_mdev_access(vctrl, (char *)&val,
+						sizeof(val), *ppos, false);
+			if (ret <= 0)
+				goto read_err;
+			if (copy_to_user(buf, &val, sizeof(val)))
+				goto read_err;
+			filled = sizeof(val);
+		}
+
+		count -= filled;
+		done += filled;
+		*ppos += filled;
+		buf += filled;
+	}
+	return done;
+read_err:
+	return -EFAULT;
+}
+
+/* Called when write() is done on the device */
+static ssize_t nvmet_mdev_write(struct vfio_device *vfio_dev,
+				const char __user *buf, size_t count,
+				loff_t *ppos)
+{
+	struct nvmet_mdev_vctrl *vctrl = vfio_dev_to_nvmet_mdev_vctrl(vfio_dev);
+	unsigned int done = 0;
+	int ret;
+
+	while (count) {
+		size_t filled;
+
+		if (count >= 4 && !(*ppos % 4)) {
+			u32 val;
+
+			if (copy_from_user(&val, buf, sizeof(val)))
+				goto write_err;
+			ret = nvmet_mdev_access(vctrl, (char *)&val,
+						sizeof(val), *ppos, true);
+			if (ret <= 0)
+				goto write_err;
+			filled = sizeof(val);
+		} else if (count >= 2 && !(*ppos % 2)) {
+			u16 val;
+
+			if (copy_from_user(&val, buf, sizeof(val)))
+				goto write_err;
+
+			ret = nvmet_mdev_access(vctrl, (char *)&val,
+						sizeof(val), *ppos, true);
+			if (ret <= 0)
+				goto write_err;
+			filled = sizeof(val);
+		} else {
+			u8 val;
+
+			if (copy_from_user(&val, buf, sizeof(val)))
+				goto write_err;
+			ret = nvmet_mdev_access(vctrl, (char *)&val,
+						sizeof(val), *ppos, true);
+			if (ret <= 0)
+				goto write_err;
+			filled = sizeof(val);
+		}
+		count -= filled;
+		done += filled;
+		*ppos += filled;
+		buf += filled;
+	}
+	return done;
+write_err:
+	return -EFAULT;
+}
+
+/* Helper for IRQ number VFIO query */
+static int nvmet_mdev_irq_counts(struct nvmet_mdev_vctrl *vctrl,
+				 unsigned int irq_type)
+{
+	switch (irq_type) {
+	case VFIO_PCI_INTX_IRQ_INDEX:
+		return 1;
+	case VFIO_PCI_MSIX_IRQ_INDEX:
+		return MAX_VIRTUAL_IRQS;
+	case VFIO_PCI_REQ_IRQ_INDEX:
+		return 1;
+	default:
+		return 0;
+	}
+}
+
+/* VFIO VFIO_IRQ_SET_ACTION_TRIGGER implementation */
+static int nvmet_mdev_ioctl_set_irqs_trigger(struct nvmet_mdev_vctrl *vctrl,
+					     u32 flags,
+					     unsigned int irq_type,
+					     unsigned int start,
+					     unsigned int count,
+					     void *data)
+{
+	u32 data_type = flags & VFIO_IRQ_SET_DATA_TYPE_MASK;
+	u8 *bools = NULL;
+	unsigned int i;
+	int ret = -EINVAL;
+
+	/* Asked to disable the current interrupt mode */
+	if (data_type == VFIO_IRQ_SET_DATA_NONE && count == 0) {
+		switch (irq_type) {
+		case VFIO_PCI_REQ_IRQ_INDEX:
+			nvmet_mdev_irqs_set_unplug_trigger(vctrl, -1);
+			return 0;
+		case VFIO_PCI_INTX_IRQ_INDEX:
+			nvmet_mdev_irqs_disable(vctrl, NVME_MDEV_IMODE_INTX);
+			return 0;
+		case VFIO_PCI_MSIX_IRQ_INDEX:
+			nvmet_mdev_irqs_disable(vctrl, NVME_MDEV_IMODE_MSIX);
+			return 0;
+		default:
+			return -EINVAL;
+		}
+	}
+
+	if (start + count > nvmet_mdev_irq_counts(vctrl, irq_type))
+		return -EINVAL;
+
+	switch (data_type) {
+	case VFIO_IRQ_SET_DATA_BOOL:
+		bools = (u8 *)data;
+		fallthrough;
+	case VFIO_IRQ_SET_DATA_NONE:
+		if (irq_type == VFIO_PCI_REQ_IRQ_INDEX)
+			return -EINVAL;
+
+		for (i = 0 ; i < count ; i++) {
+			int index = start + i;
+
+			if (!bools || bools[i])
+				nvmet_mdev_irq_trigger(vctrl, index);
+		}
+		return 0;
+
+	case VFIO_IRQ_SET_DATA_EVENTFD:
+		switch (irq_type) {
+		case VFIO_PCI_REQ_IRQ_INDEX:
+			return nvmet_mdev_irqs_set_unplug_trigger(vctrl,
+							*(int32_t *)data);
+		case VFIO_PCI_INTX_IRQ_INDEX:
+			ret = nvmet_mdev_irqs_enable(vctrl,
+						     NVME_MDEV_IMODE_INTX);
+			break;
+		case VFIO_PCI_MSIX_IRQ_INDEX:
+			ret = nvmet_mdev_irqs_enable(vctrl,
+						     NVME_MDEV_IMODE_MSIX);
+			break;
+		default:
+			return -EINVAL;
+		}
+		if (ret)
+			return ret;
+
+		return nvmet_mdev_irqs_set_triggers(vctrl, start, count,
+						    (int32_t *)data);
+	default:
+		return -EINVAL;
+	}
+}
+
+/* VFIO_DEVICE_GET_INFO ioctl implementation */
+static int nvmet_mdev_ioctl_get_info(struct nvmet_mdev_vctrl *vctrl,
+				     void __user *arg)
+{
+	struct vfio_device_info info;
+	unsigned int minsz = offsetofend(struct vfio_device_info, num_irqs);
+
+	if (copy_from_user(&info, (void __user *)arg, minsz))
+		return -EFAULT;
+	if (info.argsz < minsz)
+		return -EINVAL;
+
+	info.flags = VFIO_DEVICE_FLAGS_PCI | VFIO_DEVICE_FLAGS_RESET;
+	info.num_regions = VFIO_PCI_NUM_REGIONS;
+	info.num_irqs = VFIO_PCI_NUM_IRQS;
+
+	if (copy_to_user(arg, &info, minsz))
+		return -EFAULT;
+	return 0;
+}
+
+/* VFIO_DEVICE_GET_REGION_INFO ioctl implementation */
+static int nvmet_mdev_ioctl_get_reg_info(struct nvmet_mdev_vctrl *vctrl,
+					 void __user *arg)
+{
+	struct nvmet_mdev_io_region *region;
+	struct mdev_nvme_vfio_region_info *info;
+	unsigned long minsz, outsz, maxsz;
+	int ret = 0;
+
+	minsz = offsetofend(struct vfio_region_info, offset);
+	maxsz = sizeof(struct mdev_nvme_vfio_region_info) +
+				sizeof(struct vfio_region_sparse_mmap_area);
+
+	info = kzalloc(maxsz, GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	if (copy_from_user(info, arg, minsz)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	outsz = info->base.argsz;
+	if (outsz < minsz || outsz > maxsz) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (info->base.index >= VFIO_PCI_NUM_REGIONS) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	region = &vctrl->regions[info->base.index];
+	info->base.offset = REGION_TO_OFFSET(info->base.index);
+	info->base.argsz = maxsz;
+	info->base.size = region->size;
+
+	info->base.flags = VFIO_REGION_INFO_FLAG_READ |
+				VFIO_REGION_INFO_FLAG_WRITE;
+
+	if (region->mmap_ops) {
+		info->base.flags |= (VFIO_REGION_INFO_FLAG_MMAP |
+						VFIO_REGION_INFO_FLAG_CAPS);
+
+		info->base.cap_offset =
+			offsetof(struct mdev_nvme_vfio_region_info, mmap_cap);
+
+		info->mmap_cap.header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
+		info->mmap_cap.header.version = 1;
+		info->mmap_cap.header.next = 0;
+		info->mmap_cap.nr_areas = 1;
+		info->mmap_cap.areas[0].offset = region->mmap_area_start;
+		info->mmap_cap.areas[0].size = region->mmap_area_size;
+	}
+
+	if (copy_to_user(arg, info, outsz))
+		ret = -EFAULT;
+out:
+	kfree(info);
+	return ret;
+}
+
+/* VFIO_DEVICE_GET_IRQ_INFO ioctl implementation */
+static int nvmet_mdev_ioctl_get_irq_info(struct nvmet_mdev_vctrl *vctrl,
+					 void __user *arg)
+{
+	struct vfio_irq_info info;
+	unsigned int minsz = offsetofend(struct vfio_irq_info, count);
+
+	if (copy_from_user(&info, arg, minsz))
+		return -EFAULT;
+	if (info.argsz < minsz)
+		return -EINVAL;
+
+	info.count = nvmet_mdev_irq_counts(vctrl, info.index);
+	info.flags = VFIO_IRQ_INFO_EVENTFD;
+
+	if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
+		info.flags |= VFIO_IRQ_INFO_MASKABLE | VFIO_IRQ_INFO_AUTOMASKED;
+
+	if (copy_to_user(arg, &info, minsz))
+		return -EFAULT;
+	return 0;
+}
+
+/* VFIO VFIO_DEVICE_SET_IRQS ioctl implementation */
+static int nvmet_mdev_ioctl_set_irqs(struct nvmet_mdev_vctrl *vctrl,
+				     void __user *arg)
+{
+	int ret, irqcount;
+	struct vfio_irq_set hdr;
+	u8 *data = NULL;
+	size_t data_size = 0;
+	unsigned long minsz = offsetofend(struct vfio_irq_set, count);
+
+	if (copy_from_user(&hdr, arg, minsz))
+		return -EFAULT;
+
+	irqcount = nvmet_mdev_irq_counts(vctrl, hdr.index);
+	ret = vfio_set_irqs_validate_and_prepare(&hdr,
+						 irqcount,
+						 VFIO_PCI_NUM_IRQS,
+						 &data_size);
+	if (ret)
+		return ret;
+
+	if (data_size) {
+		data = memdup_user((arg + minsz), data_size);
+		if (IS_ERR(data))
+			return PTR_ERR(data);
+	}
+
+	ret = -ENOTTY;
+	switch (hdr.index) {
+	case VFIO_PCI_INTX_IRQ_INDEX:
+	case VFIO_PCI_MSIX_IRQ_INDEX:
+	case VFIO_PCI_REQ_IRQ_INDEX:
+		switch (hdr.flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			/* pretend to support this (even with eventfd) */
+			ret = hdr.index == VFIO_PCI_INTX_IRQ_INDEX ?
+					0 : -EINVAL;
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			ret = nvmet_mdev_ioctl_set_irqs_trigger(vctrl,
+								hdr.flags,
+								hdr.index,
+								hdr.start,
+								hdr.count,
+								data);
+			break;
+		}
+		break;
+	}
+
+	kfree(data);
+	return ret;
+}
+
+static long nvmet_mdev_ioctl(struct vfio_device *vfio_dev, unsigned int cmd,
+			     unsigned long arg)
+{
+	struct nvmet_mdev_vctrl *vctrl = vfio_dev_to_nvmet_mdev_vctrl(vfio_dev);
+
+	switch (cmd) {
+	case VFIO_DEVICE_GET_INFO:
+		return nvmet_mdev_ioctl_get_info(vctrl, (void __user *)arg);
+	case VFIO_DEVICE_GET_REGION_INFO:
+		return nvmet_mdev_ioctl_get_reg_info(vctrl, (void __user *)arg);
+	case VFIO_DEVICE_GET_IRQ_INFO:
+		return nvmet_mdev_ioctl_get_irq_info(vctrl, (void __user *)arg);
+	case VFIO_DEVICE_SET_IRQS:
+		return nvmet_mdev_ioctl_set_irqs(vctrl, (void __user *)arg);
+	case VFIO_DEVICE_RESET:
+		nvmet_mdev_vctrl_reset(vctrl);
+		return 0;
+	default:
+		return -ENOTTY;
+	}
+}
+
+/* mmap() implementation (doorbell area) */
+static int nvmet_mdev_mmap(struct vfio_device *vfio_dev,
+			   struct vm_area_struct *vma)
+{
+	struct nvmet_mdev_vctrl *vctrl = vfio_dev_to_nvmet_mdev_vctrl(vfio_dev);
+	int index = OFFSET_TO_REGION((u64)vma->vm_pgoff << PAGE_SHIFT);
+	unsigned long size, start;
+
+	if (index >= VFIO_PCI_NUM_REGIONS || !vctrl->regions[index].mmap_ops)
+		return -EINVAL;
+
+	if (vma->vm_end < vma->vm_start)
+		return -EINVAL;
+
+	size = vma->vm_end - vma->vm_start;
+	start = vma->vm_pgoff << PAGE_SHIFT;
+
+	if (start < vctrl->regions[index].mmap_area_start)
+		return -EINVAL;
+	if (size > vctrl->regions[index].mmap_area_size)
+		return -EINVAL;
+
+	if ((vma->vm_flags & VM_SHARED) == 0)
+		return -EINVAL;
+
+	vma->vm_ops = vctrl->regions[index].mmap_ops;
+	vma->vm_private_data = vctrl;
+	return 0;
+}
+
+static const struct vfio_device_ops nvme_vfio_dev_ops = {
+	.open_device		= nvmet_mdev_vctrl_open,
+	.close_device		= nvmet_mdev_vctrl_release,
+	.read			= nvmet_mdev_read,
+	.write			= nvmet_mdev_write,
+	.mmap			= nvmet_mdev_mmap,
+	.ioctl			= nvmet_mdev_ioctl,
+	.dma_unmap		= nvmet_mdev_dma_unmap,
+	.bind_iommufd		= vfio_iommufd_emulated_bind,
+	.unbind_iommufd		= vfio_iommufd_emulated_unbind,
+	.attach_ioas		= vfio_iommufd_emulated_attach_ioas,
+	.detach_ioas		= vfio_iommufd_emulated_detach_ioas,
+};
+
+/* Called when new mediated device is created */
+static int nvmet_mdev_probe(struct mdev_device *mdev)
+{
+	struct nvmet_mdev_port *mport = container_of(mdev->type->parent,
+						     struct nvmet_mdev_port,
+						     parent);
+	struct nvmet_mdev_vctrl *vctrl;
+	struct nvmet_ctrl *ctrl;
+	u16 cntlid;
+	int ret;
+
+	ret = kstrtou16(mdev->type->sysfs_name, 10, &cntlid);
+	if (ret)
+		return -EINVAL;
+
+	mutex_lock(&mport->mutex);
+	/* Release the refcount taken on the ctrl at remove. */
+	ctrl = nvmet_find_get_static_ctrl(mport->nvmet_port,
+					  nvmet_mdev_ops.type, cntlid);
+	if (!ctrl) {
+		ret = -ENODEV;
+		goto unlock;
+	}
+
+	vctrl = vfio_alloc_device(nvmet_mdev_vctrl, vfio_dev, &mdev->dev,
+				  &nvme_vfio_dev_ops);
+	if (IS_ERR(vctrl)) {
+		ret = PTR_ERR(vctrl);
+		goto put_ctrl;
+	}
+
+	vctrl->nvmet_ctrl = ctrl;
+	ctrl->drvdata = vctrl;
+	dev_set_drvdata(&mdev->dev, vctrl);
+
+	ret = vfio_register_emulated_iommu_dev(&vctrl->vfio_dev);
+	if (ret)
+		goto put_vfio_dev;
+
+	ret = nvmet_mdev_vctrl_create(vctrl, mdev);
+	if (ret)
+		goto unreg_vfio_dev;
+
+	mutex_unlock(&mport->mutex);
+	return 0;
+
+unreg_vfio_dev:
+	vfio_unregister_group_dev(&vctrl->vfio_dev);
+put_vfio_dev:
+	vfio_put_device(&vctrl->vfio_dev);
+put_ctrl:
+	nvmet_ctrl_put(ctrl);
+unlock:
+	mutex_unlock(&mport->mutex);
+	return ret;
+}
+
+/*
+ * TODO handle if this is called while opened.
+ *
+ * Depending on if we go with configfs based setup we can get a refcount to
+ * the conifgfs objects to prevent them from being removed while in use.
+ * However, the mdev bus hotplug removal cannot be stopped right now
+ * but we may want to modify it to just not allow it.
+ */
+void nvmet_mdev_remove_ctrl(struct nvmet_mdev_vctrl *vctrl)
+{
+	struct nvmet_ctrl *ctrl;
+
+	if (!vctrl)
+		return;
+
+	ctrl = vctrl->nvmet_ctrl;
+	if (!ctrl)
+		return;
+
+	nvmet_mdev_irq_raise_unplug_event(vctrl);
+	nvmet_mdev_vctrl_destroy(vctrl);
+
+	dev_set_drvdata(vctrl->vfio_dev.dev, vctrl);
+	vctrl->nvmet_ctrl = NULL;
+	ctrl->drvdata = NULL;
+
+	vfio_unregister_group_dev(&vctrl->vfio_dev);
+	vfio_put_device(&vctrl->vfio_dev);
+
+	if (ctrl)
+		nvmet_ctrl_put(ctrl);
+}
+
+/* Called when a mediated device is removed via sysfs or parent removal */
+static void nvmet_mdev_remove(struct mdev_device *mdev)
+{
+	struct nvmet_mdev_port *mport = container_of(mdev->type->parent,
+						     struct nvmet_mdev_port,
+						     parent);
+
+	mutex_lock(&mport->mutex);
+	nvmet_mdev_remove_ctrl(dev_get_drvdata(&mdev->dev));
+	mutex_unlock(&mport->mutex);
+}
+
+static unsigned int nvmet_mdev_get_available(struct mdev_type *mtype)
+{
+	/* No limit since nvmet does not restrict it either */
+	return UINT_MAX;
+}
+
+static ssize_t poll_timeout_ms_store(struct device *dev,
+		struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct nvmet_mdev_vctrl *vctrl = dev_get_drvdata(dev);
+	int ret;
+
+	if (!vctrl)
+		return -ENODEV;
+
+	ret = kstrtoint(buf, 10, &vctrl->poll_timeout_ms);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static ssize_t poll_timeout_ms_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct nvmet_mdev_vctrl *vctrl = dev_get_drvdata(dev);
+
+	if (!vctrl)
+		return -ENODEV;
+
+	return sysfs_emit(buf, "%d\n", vctrl->poll_timeout_ms);
+}
+static DEVICE_ATTR_RW(poll_timeout_ms);
+
+static struct attribute *nvmet_mdev_dev_settings_atttributes[] = {
+	&dev_attr_poll_timeout_ms.attr,
+	NULL
+};
+
+static const struct attribute_group nvmet_mdev_setting_attr_group = {
+	.name = "settings",
+	.attrs = nvmet_mdev_dev_settings_atttributes,
+};
+
+static const struct attribute_group *nvmet_mdev_dev_attr_groups[] = {
+	&nvmet_mdev_setting_attr_group,
+	NULL,
+};
+
+static struct mdev_driver nvmet_mdev_driver = {
+	.device_api		= VFIO_DEVICE_API_PCI_STRING,
+	.driver = {
+		.name		= NVMET_MDEV_NAME,
+		.owner		= THIS_MODULE,
+		.dev_groups	= nvmet_mdev_dev_attr_groups,
+	},
+	.probe			= nvmet_mdev_probe,
+	.remove			= nvmet_mdev_remove,
+	.get_available		= nvmet_mdev_get_available,
+};
+
+static const struct bus_type nvmet_mdev_bus = {
+	.name		= NVMET_MDEV_NAME,
+};
+
+static void nvmet_mdev_destroy_types(struct nvmet_mdev_port *mport)
+{
+	int i;
+
+	for (i = 0; i < mport->type_count; i++)
+		kfree(mport->types[i].name);
+}
+
+static int nvmet_mdev_setup_types(void *priv, struct nvmet_port *port,
+				  struct nvmet_ctrl *ctrl)
+{
+	struct nvmet_mdev_port *mport = priv;
+	struct nvmet_mdev_type *type;
+
+	if (mport->type_count == mport->ctrl_count) {
+		pr_err("Invalid number of controllers found %d\n",
+		       mport->ctrl_count);
+		return -EINVAL;
+	}
+
+	type = &mport->types[mport->type_count];
+	type->name = kasprintf(GFP_KERNEL, "%hu", ctrl->cntlid);
+	if (!type->name)
+		return -ENOMEM;
+
+	mport->mdev_types[mport->type_count] = &type->type;
+	mport->mdev_types[mport->type_count]->sysfs_name = type->name;
+
+	mport->type_count++;
+	return 0;
+}
+
+static void nvmet_mdev_dev_release(struct device *dev)
+{
+	struct nvmet_mdev_port *mport = container_of(dev,
+						     struct nvmet_mdev_port,
+						     device);
+	nvmet_mdev_destroy_types(mport);
+	kfree(mport->types);
+	kfree(mport->mdev_types);
+	kfree(mport);
+}
+
+int nvmet_mdev_register_port(struct nvmet_mdev_port *mport)
+{
+	int ret;
+
+	mport->types = kcalloc(mport->ctrl_count, sizeof(*mport->types),
+			       GFP_KERNEL);
+	if (!mport->types)
+		return -ENOMEM;
+
+	mport->mdev_types = kcalloc(mport->ctrl_count,
+				    sizeof(*mport->mdev_types), GFP_KERNEL);
+	if (!mport->mdev_types) {
+		ret = -ENOMEM;
+		goto free_types;
+	}
+
+	ret = nvmet_for_each_static_ctrl(mport->nvmet_port, nvmet_mdev_ops.type,
+					 nvmet_mdev_setup_types, mport);
+	if (ret)
+		/* we might have partially setup the types arrays */
+		goto destroy_types;
+
+	mport->device.parent = nvmet_mdev_root_dev;
+	mport->device.bus = &nvmet_mdev_bus;
+	mport->device.release = nvmet_mdev_dev_release;
+	dev_set_name(&mport->device, "port-%s",
+		     config_item_name(&mport->nvmet_port->group.cg_item));
+
+	ret = device_register(&mport->device);
+	if (ret)
+		goto free_mdev_types;
+
+	ret = mdev_register_parent(&mport->parent, &mport->device,
+				   &nvmet_mdev_driver, mport->mdev_types,
+				   mport->type_count);
+	if (ret)
+		goto unreg_dev;
+
+	return 0;
+
+unreg_dev:
+	device_unregister(&mport->device);
+destroy_types:
+	nvmet_mdev_destroy_types(mport);
+free_mdev_types:
+	kfree(mport->mdev_types);
+free_types:
+	kfree(mport->types);
+	return ret;
+}
+
+void nvmet_mdev_unregister_port(struct nvmet_mdev_port *mport)
+{
+	mdev_unregister_parent(&mport->parent);
+	device_unregister(&mport->device);
+}
+
+static int nvmet_mdev_register_root_device(void)
+{
+	int ret;
+
+	nvmet_mdev_root_dev = root_device_register(NVMET_MDEV_NAME);
+	if (IS_ERR(nvmet_mdev_root_dev))
+		return PTR_ERR(nvmet_mdev_root_dev);
+
+	ret = bus_register(&nvmet_mdev_bus);
+	if (ret)
+		goto unreg_root;
+
+	nvmet_unregister_transport(&nvmet_mdev_ops);
+	return 0;
+
+unreg_root:
+	root_device_unregister(nvmet_mdev_root_dev);
+	return ret;
+}
+
+static void nvmet_mdev_unregister_root_device(void)
+{
+	root_device_unregister(nvmet_mdev_root_dev);
+	bus_unregister(&nvmet_mdev_bus);
+}
+
+static int __init nvmet_mdev_init(void)
+{
+	int ret;
+
+	ret = nvmet_mdev_register_root_device();
+	if (ret)
+		return ret;
+
+	ret = mdev_register_driver(&nvmet_mdev_driver);
+	if (ret)
+		goto unreg_root;
+
+	ret = nvmet_register_transport(&nvmet_mdev_ops);
+	if (ret)
+		goto unreg_driver;
+
+	return 0;
+
+unreg_driver:
+	mdev_unregister_driver(&nvmet_mdev_driver);
+unreg_root:
+	nvmet_mdev_unregister_root_device();
+	return ret;
+}
+
+static void __exit nvmet_mdev_exit(void)
+{
+	nvmet_unregister_transport(&nvmet_mdev_ops);
+	nvmet_mdev_unregister_root_device();
+	mdev_unregister_driver(&nvmet_mdev_driver);
+}
+
+module_init(nvmet_mdev_init);
+module_exit(nvmet_mdev_exit);
diff --git a/drivers/nvme/target/mdev-pci/io.c b/drivers/nvme/target/mdev-pci/io.c
new file mode 100644
index 000000000000..ece3a01434ac
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/io.c
@@ -0,0 +1,332 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * NVMe IO command translation and polling IO thread
+ * Copyright (c) 2019 - Maxim Levitsky
+ * Copyright (C) 2025 Oracle Corporation
+ */
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/slab.h>
+#include <linux/nvme.h>
+#include <linux/timekeeping.h>
+#include <linux/ktime.h>
+#include <linux/delay.h>
+#include "priv.h"
+
+static int nvmet_mdev_create_sgl(struct nvmet_mdev_req *mreq, u64 data_len,
+				 const struct nvme_command *cmd)
+{
+	struct nvmet_ext_data_iter *iter = &mreq->data_iter;
+	struct nvmet_req *req = &mreq->req;
+	struct scatterlist *sg;
+	int i, ret;
+
+	ret = nvmet_mdev_udata_iter_set_dptr(&mreq->data_iter,
+					     &cmd->common.dptr, data_len);
+	if (ret)
+		return ret;
+
+	ret = sg_alloc_table(&mreq->sgt, iter->count, GFP_KERNEL);
+	if (ret)
+		return ret;
+
+	for_each_sgtable_sg(&mreq->sgt, sg, i) {
+		int seg_len = min(PAGE_SIZE, data_len);
+		struct page *page;
+		int offset;
+
+		page = pfn_to_page(PHYS_PFN(iter->physical));
+		offset = offset_in_page(iter->physical);
+
+		sg_set_page(sg, page, seg_len, offset);
+
+		ret = iter->next(iter);
+		if (WARN_ON(ret))
+			goto release_iter;
+		data_len -= seg_len;
+	}
+
+	req->sg = mreq->sgt.sgl;
+	req->sg_cnt = mreq->sgt.nents;
+	return 0;
+
+release_iter:
+	if (iter->release)
+		iter->release(iter);
+
+	sg_free_table(&mreq->sgt);
+	return ret;
+}
+
+static bool nvmet_mdev_submit_cmd(struct nvmet_mdev_vctrl *vctrl,
+				  struct nvmet_mdev_vsq *vsq)
+{
+	struct nvmet_mdev_vcq *vcq = vsq->vcq;
+	struct nvme_completion cqe = {};
+	struct nvmet_mdev_req *mreq;
+	struct nvme_command *cmd;
+	struct nvmet_req *req;
+	u16 ucid;
+	int ret;
+
+	cmd = nvmet_mdev_vsq_get_cmd(vctrl, vsq, &ucid);
+	if (!cmd)
+		return false;
+
+	if (ucid >= vsq->size) {
+		ret = DNR(NVME_SC_INVALID_FIELD);
+		goto complete;
+	}
+
+	if (cmd->common.flags != 0) {
+		ret = DNR(NVME_SC_INVALID_FIELD);
+		goto complete;
+	}
+
+	mreq = &vsq->reqs[ucid];
+	memset(mreq, 0, sizeof(*mreq));
+
+	INIT_LIST_HEAD(&mreq->mem_map_list);
+	nvmet_mdev_udata_iter_setup(&vctrl->viommu, &mreq->data_iter,
+				    &mreq->mem_map_list);
+	init_llist_node(&mreq->cq_node);
+	mreq->vcq = vcq;
+
+	req = &mreq->req;
+	req->cmd = cmd;
+	req->cqe = &mreq->cqe;
+	req->port = vctrl->nvmet_ctrl->port;
+
+	if (!nvmet_req_init(req, &vcq->nvmet_cq, &vsq->nvmet_sq,
+			    &nvmet_mdev_ops)) {
+		vctrl->expected_responses++;
+		/* nvmet will complete via queue_response */
+		return true;
+	}
+
+	req->transfer_len = nvmet_req_transfer_len(req);
+	if (req->transfer_len) {
+		ret = nvmet_mdev_create_sgl(mreq, req->transfer_len, cmd);
+		if (ret) {
+			ret = nvmet_mdev_translate_error(ret);
+			goto uninit_req;
+		}
+	}
+
+	vctrl->expected_responses++;
+	req->execute(req);
+	return true;
+
+uninit_req:
+	nvmet_req_uninit(req);
+complete:
+	cqe.sq_head = cpu_to_le16(vsq->head);
+	cqe.sq_id = cpu_to_le16(vsq->qid);
+	cqe.command_id = cmd->common.command_id;
+	cqe.status = cpu_to_le16(ret << 1);
+
+	nvmet_mdev_vcq_write_cqe(vctrl, vcq, &cqe);
+	return true;
+}
+
+bool nvmet_mdev_process_responses(struct nvmet_mdev_vctrl *vctrl,
+				  struct nvmet_mdev_vcq *vcq)
+{
+	struct nvmet_mdev_req *mreq, *mreq_next;
+	struct nvmet_ext_data_iter *iter;
+	struct llist_node *node;
+	bool processed = false;
+
+	node = llist_del_all(&vcq->mreq_list);
+	if (!node)
+		return processed;
+
+	llist_for_each_entry_safe(mreq, mreq_next, node, cq_node) {
+		iter = &mreq->data_iter;
+
+		nvmet_mdev_vcq_write_cqe(vctrl, vcq, mreq->req.cqe);
+
+		if (iter->release)
+			iter->release(iter);
+
+		if (mreq->req.sg_cnt)
+			sg_free_table(&mreq->sgt);
+
+		vctrl->expected_responses--;
+		processed = true;
+	}
+
+	return processed;
+}
+
+void nvmet_mdev_io_resume(struct nvmet_mdev_vctrl *vctrl)
+{
+	if (!vctrl->iothread || !vctrl->iothread_parked || vctrl->io_idle)
+		return;
+
+	vctrl->iothread_parked = false;
+	/* has memory barrier */
+	kthread_unpark(vctrl->iothread);
+}
+
+bool nvmet_mdev_io_pause(struct nvmet_mdev_vctrl *vctrl)
+{
+	if (!vctrl->iothread || vctrl->iothread_parked)
+		return false;
+
+	vctrl->iothread_parked = true;
+	kthread_park(vctrl->iothread);
+	return true;
+}
+
+static int nvmet_mdev_get_poll_tmo(struct nvmet_mdev_vctrl *vctrl)
+{
+	/* can't stop polling when shadow db not enabled */
+	return vctrl->mmio.shadow_db_en ? vctrl->poll_timeout_ms : 0;
+}
+
+static void nvmet_mdev_process_io(struct nvmet_mdev_vctrl *vctrl)
+{
+	struct nvmet_mdev_vcq *vcq;
+	struct nvmet_mdev_vsq *vsq;
+	unsigned long last = jiffies;
+	bool idle = false;
+	int timeout;
+	u16 qid;
+	int i;
+
+	vctrl->now = ktime_get();
+
+	/* main loop */
+	while (!kthread_should_park()) {
+		vctrl->now = ktime_get();
+
+		for_each_set_bit(qid, vctrl->vsq_en, NVMET_MDEV_MAX_NR_QUEUES) {
+			vsq = &vctrl->vsqs[qid];
+
+			for (i = 0 ; i < (1 << vctrl->arb_burst_shift) ; i++)
+				if (nvmet_mdev_submit_cmd(vctrl, vsq))
+					last = jiffies;
+		}
+
+		for_each_set_bit(qid, vctrl->vcq_en, NVMET_MDEV_MAX_NR_QUEUES) {
+			vcq = &vctrl->vcqs[qid];
+
+			nvmet_mdev_vcq_process(vctrl, vcq, true, vctrl->now);
+		}
+
+		for_each_set_bit(qid, vctrl->vcq_en, NVMET_MDEV_MAX_NR_QUEUES) {
+			vcq = &vctrl->vcqs[qid];
+
+			if (nvmet_mdev_process_responses(vctrl, vcq))
+				last = jiffies;
+		}
+
+		/* Check if we need to stop polling */
+		timeout = nvmet_mdev_get_poll_tmo(vctrl);
+		if (timeout &&
+		    time_is_before_jiffies(last + msecs_to_jiffies(timeout))) {
+			idle = true;
+			break;
+		}
+		cond_resched();
+	}
+
+	for_each_set_bit(qid, vctrl->vcq_en, NVMET_MDEV_MAX_NR_QUEUES) {
+		vcq = &vctrl->vcqs[qid];
+
+		/* Drain all the pending completion interrupts to the guest */
+		if (nvmet_mdev_vcq_flush(vctrl, vcq, vctrl->now))
+			idle = false;
+	}
+
+	/*
+	 * Park IO thread if IO is truly idle.
+	 * TODO - expected_responses will always be > 1 because of async
+	 * events.
+	 */
+	if (!vctrl->expected_responses && idle) {
+		if (!mutex_trylock(&vctrl->lock))
+			return;
+
+		for_each_set_bit(qid, vctrl->vsq_en, NVMET_MDEV_MAX_NR_QUEUES) {
+			vsq = &vctrl->vsqs[qid];
+
+			if (!nvmet_mdev_vsq_suspend_io(vsq))
+				idle = false;
+		}
+
+		if (idle) {
+			_DBG(vctrl, "IO: self-parking\n");
+			vctrl->io_idle = true;
+			nvmet_mdev_io_pause(vctrl);
+		}
+		mutex_unlock(&vctrl->lock);
+	}
+}
+
+static int nvmet_mdev_poll(void *data)
+{
+	struct nvmet_mdev_vctrl *vctrl = data;
+
+	if (kthread_should_stop())
+		return 0;
+
+	_DBG(vctrl, "IO: iothread started\n");
+
+	for (;;) {
+		if (kthread_should_park()) {
+			_DBG(vctrl, "IO: iothread parked\n");
+			kthread_parkme();
+		}
+
+		if (kthread_should_stop())
+			break;
+
+		nvmet_mdev_process_io(vctrl);
+	}
+
+	_DBG(vctrl, "IO: iothread stopped\n");
+	return 0;
+}
+
+int nvmet_mdev_io_create(struct nvmet_mdev_vctrl *vctrl)
+{
+	char name[TASK_COMM_LEN];
+
+	_DBG(vctrl, "IO: creating the polling iothread\n");
+
+	snprintf(name, sizeof(name), "nvmet_mdev%d", vctrl->nvmet_ctrl->cntlid);
+
+	vctrl->iothread_parked = false;
+	vctrl->io_idle = true;
+
+	vctrl->iothread = kthread_create(nvmet_mdev_poll, vctrl, name);
+	if (IS_ERR(vctrl->iothread)) {
+		vctrl->iothread = NULL;
+		return PTR_ERR(vctrl->iothread);
+	}
+
+	if (vctrl->io_idle) {
+		vctrl->iothread_parked = true;
+		kthread_park(vctrl->iothread);
+		return 0;
+	}
+
+	wake_up_process(vctrl->iothread);
+	return 0;
+}
+
+void nvmet_mdev_io_free(struct nvmet_mdev_vctrl *vctrl)
+{
+	_DBG(vctrl, "IO: destroying the polling iothread\n");
+	nvmet_mdev_io_pause(vctrl);
+	kthread_stop(vctrl->iothread);
+	vctrl->iothread = NULL;
+}
+
+void nvmet_mdev_assert_io_not_running(struct nvmet_mdev_vctrl *vctrl)
+{
+	if (WARN_ON(vctrl->iothread && !vctrl->iothread_parked))
+		nvmet_mdev_io_pause(vctrl);
+}
diff --git a/drivers/nvme/target/mdev-pci/irq.c b/drivers/nvme/target/mdev-pci/irq.c
new file mode 100644
index 000000000000..38a096427dcb
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/irq.c
@@ -0,0 +1,269 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * NVMe virtual controller IRQ implementation (MSIx and INTx)
+ * Copyright (c) 2019 - Maxim Levitsky
+ */
+
+#include <linux/init.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include "priv.h"
+
+/* Setup the interrupt subsystem */
+void nvmet_mdev_irqs_setup(struct nvmet_mdev_vctrl *vctrl)
+{
+	vctrl->irqs.mode = NVME_MDEV_IMODE_NONE;
+	vctrl->irqs.irq_coalesc_max = 1;
+}
+
+/* Enable INTx or MSIx interrupts  */
+static int __nvmet_mdev_irqs_enable(struct nvmet_mdev_vctrl *vctrl,
+				    enum nvmet_mdev_irq_mode mode)
+{
+	bool paused;
+
+	if (vctrl->irqs.mode == mode)
+		return 0;
+	if (vctrl->irqs.mode != NVME_MDEV_IMODE_NONE)
+		return -EBUSY;
+
+	if (mode == NVME_MDEV_IMODE_INTX)
+		_DBG(vctrl, "IRQ: enable INTx interrupts\n");
+	else if (mode == NVME_MDEV_IMODE_MSIX)
+		_DBG(vctrl, "IRQ: enable MSIX interrupts\n");
+	else
+		WARN_ON(1);
+
+	paused = nvmet_mdev_io_pause(vctrl);
+	vctrl->irqs.mode = mode;
+	if (paused)
+		nvmet_mdev_io_resume(vctrl);
+	return 0;
+}
+
+int nvmet_mdev_irqs_enable(struct nvmet_mdev_vctrl *vctrl,
+			   enum nvmet_mdev_irq_mode mode)
+{
+	int retval;
+
+	mutex_lock(&vctrl->lock);
+	retval = __nvmet_mdev_irqs_enable(vctrl, mode);
+	mutex_unlock(&vctrl->lock);
+	return retval;
+}
+
+/* Disable INTx or MSIx interrupts  */
+static void __nvmet_mdev_irqs_disable(struct nvmet_mdev_vctrl *vctrl,
+				      enum nvmet_mdev_irq_mode mode)
+{
+	unsigned int i;
+	bool paused;
+
+	if (vctrl->irqs.mode == NVME_MDEV_IMODE_NONE)
+		return;
+	if (vctrl->irqs.mode != mode)
+		return;
+
+	if (vctrl->irqs.mode == NVME_MDEV_IMODE_INTX)
+		_DBG(vctrl, "IRQ: disable INTx interrupts\n");
+	else if (vctrl->irqs.mode == NVME_MDEV_IMODE_MSIX)
+		_DBG(vctrl, "IRQ: disable MSIX interrupts\n");
+	else
+		WARN_ON(1);
+
+	paused = nvmet_mdev_io_pause(vctrl);
+
+	for (i = 0; i < MAX_VIRTUAL_IRQS; i++) {
+		struct nvmet_mdev_user_irq *vec = &vctrl->irqs.vecs[i];
+
+		if (vec->trigger) {
+			eventfd_ctx_put(vec->trigger);
+			vec->trigger = NULL;
+		}
+		vec->irq_pending_cnt = 0;
+		vec->irq_time = 0;
+	}
+	vctrl->irqs.mode = NVME_MDEV_IMODE_NONE;
+	if (paused)
+		nvmet_mdev_io_resume(vctrl);
+}
+
+void nvmet_mdev_irqs_disable(struct nvmet_mdev_vctrl *vctrl,
+			     enum nvmet_mdev_irq_mode mode)
+{
+	mutex_lock(&vctrl->lock);
+	__nvmet_mdev_irqs_disable(vctrl, mode);
+	mutex_unlock(&vctrl->lock);
+}
+
+/* Set eventfd triggers for INTx or MSIx interrupts */
+int nvmet_mdev_irqs_set_triggers(struct nvmet_mdev_vctrl *vctrl, int start,
+				 int count, int32_t *fds)
+{
+	unsigned int i;
+	bool paused;
+
+	mutex_lock(&vctrl->lock);
+	paused = nvmet_mdev_io_pause(vctrl);
+
+	for (i = 0; i < count; i++) {
+		int irqindex = start + i;
+		struct eventfd_ctx *trigger;
+		struct nvmet_mdev_user_irq *irq = &vctrl->irqs.vecs[irqindex];
+
+		if (irq->trigger) {
+			eventfd_ctx_put(irq->trigger);
+			irq->trigger = NULL;
+		}
+
+		if (fds[i] < 0)
+			continue;
+
+		trigger = eventfd_ctx_fdget(fds[i]);
+		if (IS_ERR(trigger)) {
+			mutex_unlock(&vctrl->lock);
+			return PTR_ERR(trigger);
+		}
+
+		irq->trigger = trigger;
+	}
+	if (paused)
+		nvmet_mdev_io_resume(vctrl);
+	mutex_unlock(&vctrl->lock);
+	return 0;
+}
+
+/* Set eventfd trigger for unplug interrupt */
+static int __nvmet_mdev_irqs_set_unplug_trigger(struct nvmet_mdev_vctrl *vctrl,
+						int32_t fd)
+{
+	struct eventfd_ctx *trigger;
+
+	if (vctrl->irqs.request_trigger) {
+		_DBG(vctrl, "IRQ: clear hotplug trigger\n");
+		eventfd_ctx_put(vctrl->irqs.request_trigger);
+		vctrl->irqs.request_trigger = NULL;
+	}
+
+	if (fd < 0)
+		return 0;
+
+	_DBG(vctrl, "IRQ: set hotplug trigger\n");
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger))
+		return PTR_ERR(trigger);
+
+	vctrl->irqs.request_trigger = trigger;
+	return 0;
+}
+
+int nvmet_mdev_irqs_set_unplug_trigger(struct nvmet_mdev_vctrl *vctrl,
+				       int32_t fd)
+{
+	int retval;
+
+	mutex_lock(&vctrl->lock);
+	retval = __nvmet_mdev_irqs_set_unplug_trigger(vctrl, fd);
+	mutex_unlock(&vctrl->lock);
+	return retval;
+}
+
+/* Reset the interrupts subsystem */
+void nvmet_mdev_irqs_reset(struct nvmet_mdev_vctrl *vctrl)
+{
+	int i;
+
+	lockdep_assert_held(&vctrl->lock);
+
+	if (vctrl->irqs.mode != NVME_MDEV_IMODE_NONE)
+		__nvmet_mdev_irqs_disable(vctrl, vctrl->irqs.mode);
+
+	__nvmet_mdev_irqs_set_unplug_trigger(vctrl, -1);
+
+	for (i = 0; i < MAX_VIRTUAL_IRQS; i++) {
+		struct nvmet_mdev_user_irq *vec = &vctrl->irqs.vecs[i];
+
+		vec->irq_coalesc_en = false;
+		vec->irq_pending_cnt = 0;
+		vec->irq_time = 0;
+	}
+
+	vctrl->irqs.irq_coalesc_time_us = 0;
+}
+
+/* Check if interrupt can be coalesced */
+static bool nvmet_mdev_irq_coalesce(struct nvmet_mdev_vctrl *vctrl,
+				    struct nvmet_mdev_user_irq *irq,
+				    ktime_t now)
+{
+	s64 delta;
+
+	if (!irq->irq_coalesc_en)
+		return false;
+
+	if (irq->irq_pending_cnt >= vctrl->irqs.irq_coalesc_max)
+		return false;
+
+	delta = ktime_us_delta(now, irq->irq_time);
+	return (delta < vctrl->irqs.irq_coalesc_time_us);
+}
+
+void nvmet_mdev_irq_raise_unplug_event(struct nvmet_mdev_vctrl *vctrl)
+{
+	mutex_lock(&vctrl->lock);
+
+	if (vctrl->irqs.request_trigger) {
+		dev_notice_ratelimited(mdev_dev(vctrl->mdev),
+				       "Relaying device request to user\n");
+		eventfd_signal(vctrl->irqs.request_trigger);
+
+	} else {
+		dev_notice(mdev_dev(vctrl->mdev),
+			   "No device request channel registered, blocked until released by user\n");
+	}
+	mutex_unlock(&vctrl->lock);
+}
+
+/* Raise an interrupt */
+void nvmet_mdev_irq_raise(struct nvmet_mdev_vctrl *vctrl, unsigned int index)
+{
+	struct nvmet_mdev_user_irq *irq = &vctrl->irqs.vecs[index];
+
+	irq->irq_pending_cnt++;
+}
+
+/* Unraise an interrupt */
+void nvmet_mdev_irq_clear(struct nvmet_mdev_vctrl *vctrl, unsigned int index,
+			  ktime_t now)
+{
+	struct nvmet_mdev_user_irq *irq = &vctrl->irqs.vecs[index];
+
+	irq->irq_time = now;
+	irq->irq_pending_cnt = 0;
+}
+
+/* Directly trigger an interrupt without affecting irq coalescing settings */
+void nvmet_mdev_irq_trigger(struct nvmet_mdev_vctrl *vctrl, unsigned int index)
+{
+	struct nvmet_mdev_user_irq *irq = &vctrl->irqs.vecs[index];
+
+	if (irq->trigger)
+		eventfd_signal(irq->trigger);
+}
+
+/* Trigger previously raised interrupt */
+void nvmet_mdev_irq_cond_trigger(struct nvmet_mdev_vctrl *vctrl,
+				 unsigned int index, ktime_t now)
+{
+	struct nvmet_mdev_user_irq *irq = &vctrl->irqs.vecs[index];
+
+	if (irq->irq_pending_cnt == 0)
+		return;
+
+	if (!nvmet_mdev_irq_coalesce(vctrl, irq, now)) {
+		nvmet_mdev_irq_trigger(vctrl, index);
+		nvmet_mdev_irq_clear(vctrl, index, now);
+	}
+}
diff --git a/drivers/nvme/target/mdev-pci/mmio.c b/drivers/nvme/target/mdev-pci/mmio.c
new file mode 100644
index 000000000000..f41246b73044
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/mmio.c
@@ -0,0 +1,561 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * NVMe virtual controller MMIO implementation
+ * Copyright (c) 2019 - Maxim Levitsky
+ */
+#include <linux/kernel.h>
+#include <linux/highmem.h>
+#include "priv.h"
+
+#define DB_AREA_SIZE (NVMET_MDEV_MAX_NR_QUEUES * 2 * (4 << DB_STRIDE_SHIFT))
+#define DB_MASK ((4 << DB_STRIDE_SHIFT) - 1)
+#define MMIO_BAR_SIZE __roundup_pow_of_two(NVME_REG_DBS + DB_AREA_SIZE)
+
+/* Put the controller into fatal error state. Only way out is reset */
+static void nvmet_mdev_mmio_fatal_error(struct nvmet_mdev_vctrl *vctrl)
+{
+	if (vctrl->mmio.csts & NVME_CSTS_CFS)
+		return;
+
+	vctrl->mmio.csts |= NVME_CSTS_CFS;
+	nvmet_mdev_io_pause(vctrl);
+
+	if (vctrl->mmio.csts & NVME_CSTS_RDY)
+		nvmet_mdev_vctrl_disable(vctrl);
+}
+
+/* This is memory fault handler for the mmap area of the doorbells */
+static vm_fault_t nvmet_mdev_mmio_dbs_mmap_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct nvmet_mdev_vctrl *vctrl = vma->vm_private_data;
+
+	/* DB area is just one page, starting at offset 4096 of the mmio */
+	if (WARN_ON(vmf->pgoff != 1))
+		return VM_FAULT_SIGBUS;
+
+	get_page(vctrl->mmio.dbs_page);
+	vmf->page = vctrl->mmio.dbs_page;
+	return 0;
+}
+
+static const struct vm_operations_struct nvmet_mdev_mmio_dbs_vm_ops = {
+	.fault = nvmet_mdev_mmio_dbs_mmap_fault,
+};
+
+/* check that user db write is valid and send an error if not */
+bool nvmet_mdev_mmio_db_check(struct nvmet_mdev_vctrl *vctrl, u16 qid, u16 size,
+			      u16 db)
+{
+	if (db < size)
+		return true;
+	if (qid == 0) {
+		_DBG(vctrl, "MMIO: invalid admin DB write - fatal error\n");
+		nvmet_mdev_mmio_fatal_error(vctrl);
+		return false;
+	}
+
+	_DBG(vctrl, "MMIO: invalid DB value write qid=%d, size=%d, value=%d\n",
+	     qid, size, db);
+
+	nvmet_add_async_event(vctrl->nvmet_ctrl, NVME_AER_ERROR,
+			      NVME_AER_ERROR_INVALID_DB_VALUE, NVME_LOG_ERROR);
+	return false;
+}
+
+/* handle submission queue doorbell write */
+static void nvmet_mdev_mmio_db_write_sq(struct nvmet_mdev_vctrl *vctrl, u32 qid,
+					u32 val)
+{
+	_DBG(vctrl, "MMIO: doorbell SQID %d, DB write %d\n", qid, val);
+
+	/* check if the db belongs to a valid queue */
+	if (qid >= NVMET_MDEV_MAX_NR_QUEUES || !test_bit(qid, vctrl->vsq_en))
+		goto err_db;
+
+	vctrl->io_idle = false;
+
+	if (!vctrl->mmio.shadow_db_supported)
+		return;
+
+	nvmet_mdev_io_resume(vctrl);
+	return;
+
+err_db:
+	_DBG(vctrl, "MMIO: inactive/invalid SQ DB write qid=%d, value=%d\n",
+	     qid, val);
+
+	nvmet_add_async_event(vctrl->nvmet_ctrl, NVME_AER_ERROR,
+			      NVME_AER_ERROR_INVALID_DB_REG, NVME_LOG_ERROR);
+}
+
+/* handle doorbell write */
+static void nvmet_mdev_mmio_db_write_cq(struct nvmet_mdev_vctrl *vctrl, u32 qid,
+					u32 val)
+{
+	_DBG(vctrl, "MMIO: doorbell CQID %d, DB write %d\n", qid, val);
+
+	lockdep_assert_held(&vctrl->lock);
+	/* check if the db belongs to a valid queue */
+	if (qid >= NVMET_MDEV_MAX_NR_QUEUES || !test_bit(qid, vctrl->vcq_en))
+		goto err_db;
+
+	if (!vctrl->mmio.shadow_db_supported)
+		return;
+
+	nvmet_mdev_io_resume(vctrl);
+	return;
+
+err_db:
+	_DBG(vctrl,
+	     "MMIO: inactive/invalid CQ DB write qid=%d, value=%d\n",
+	     qid, val);
+
+	nvmet_add_async_event(vctrl->nvmet_ctrl, NVME_AER_ERROR,
+			      NVME_AER_ERROR_INVALID_DB_REG, NVME_LOG_ERROR);
+}
+
+/* This is called when user enables the controller */
+static void nvmet_mdev_mmio_cntrl_enable(struct nvmet_mdev_vctrl *vctrl)
+{
+	u64 acq, asq;
+
+	lockdep_assert_held(&vctrl->lock);
+
+	/* Controller must be reset from the dead state */
+	if (nvmet_mdev_vctrl_is_dead(vctrl))
+		goto error;
+
+	if (nvmet_cc_ams(vctrl->mmio.cc) != NVME_CC_AMS_RR)
+		goto error;
+
+	/* Check the page size */
+	if (nvmet_cc_mps(vctrl->mmio.cc) != (PAGE_SHIFT - 12))
+		goto error;
+
+	/* Start the admin completion queue */
+	acq = vctrl->mmio.acql | ((u64)vctrl->mmio.acqh << 32);
+	asq = vctrl->mmio.asql | ((u64)vctrl->mmio.asqh << 32);
+
+	if (!nvmet_mdev_vctrl_enable(vctrl, acq, asq, vctrl->mmio.aqa))
+		goto error;
+
+	/* Success! */
+	vctrl->mmio.csts |= NVME_CSTS_RDY;
+	return;
+error:
+	_WARN(vctrl, "MMIO: failure to enable the controller - fatal error\n");
+	nvmet_mdev_mmio_fatal_error(vctrl);
+}
+
+/*
+ * This is called when user sends a notification that controller is
+ * about to be disabled
+ */
+static void nvmet_mdev_mmio_cntrl_shutdown(struct nvmet_mdev_vctrl *vctrl)
+{
+	lockdep_assert_held(&vctrl->lock);
+
+	nvmet_update_cc(vctrl->nvmet_ctrl, vctrl->mmio.cc);
+	/*
+	 * Manually clear shutdown notification bits
+	 * TODO: inherit csts/cc from nvmet core.
+	 */
+	vctrl->mmio.cc &= ~NVME_CC_SHN_MASK;
+
+	if (nvmet_mdev_vctrl_is_dead(vctrl)) {
+		_DBG(vctrl, "MMIO: shutdown notification for dead ctrl\n");
+		return;
+	}
+
+	/* not enabled */
+	if (!(vctrl->mmio.csts & NVME_CSTS_RDY)) {
+		_DBG(vctrl, "MMIO: shutdown notification with CSTS.RDY==0\n");
+		nvmet_mdev_assert_io_not_running(vctrl);
+		return;
+	}
+
+	nvmet_mdev_io_pause(vctrl);
+	nvmet_mdev_vctrl_disable(vctrl);
+	vctrl->mmio.csts |= NVME_CSTS_SHST_CMPLT;
+}
+
+/* MMIO BAR read/write */
+static int nvmet_mdev_mmio_bar_access(struct nvmet_mdev_vctrl *vctrl,
+				      u16 offset, char *buf, u32 count,
+				      bool is_write)
+{
+	u32 val, oldval;
+
+	mutex_lock(&vctrl->lock);
+
+	/*
+	 * Drop non DWORD sized and aligned reads/writes
+	 * (QWORD  read/writes are split by the caller)
+	 */
+	if (count != 4 || (offset & 0x3))
+		goto drop;
+
+	val = is_write ? le32_to_cpu(*(__le32 *)buf) : 0;
+
+	switch (offset) {
+	case NVME_REG_CAP:
+		/* controller capabilities (low 32 bit) */
+		if (is_write)
+			goto drop;
+		store_le32(buf, vctrl->mmio.cap & 0xFFFFFFFF);
+		break;
+	case NVME_REG_CAP + 4:
+		/* controller capabilities (upper 32 bit) */
+		if (is_write)
+			goto drop;
+		store_le32(buf, vctrl->mmio.cap >> 32);
+		break;
+	case NVME_REG_VS:
+		if (is_write)
+			goto drop;
+		store_le32(buf, NVME_MDEV_NVME_VER);
+		break;
+	case NVME_REG_INTMS:
+	case NVME_REG_INTMC:
+		/* Interrupt Mask Set & Clear */
+		goto drop;
+	case NVME_REG_CC:
+		/* Controller Configuration */
+		if (!is_write) {
+			store_le32(buf, vctrl->mmio.cc);
+			break;
+		}
+
+		oldval = vctrl->mmio.cc;
+		vctrl->mmio.cc = val;
+
+		/* drop if reserved bits set */
+		if (vctrl->mmio.cc & 0xFF00000E) {
+			_DBG(vctrl,
+			     "MMIO: reserved bits of CC set - fatal error\n");
+			nvmet_mdev_mmio_fatal_error(vctrl);
+			goto drop;
+		}
+
+		/*
+		 * CSS(command set),MPS(memory page size),AMS(queue arbitration)
+		 * must not be changed while controller is running
+		 */
+		if (vctrl->mmio.csts & NVME_CSTS_RDY) {
+			if ((vctrl->mmio.cc & 0x3FF0) != (oldval & 0x3FF0)) {
+				_DBG(vctrl,
+				     "MMIO: attempt to change setting bits of CC while CC.EN=1 - fatal error\n");
+
+				nvmet_mdev_mmio_fatal_error(vctrl);
+				goto drop;
+			}
+		}
+
+		if (nvmet_cc_shn(vctrl->mmio.cc)) {
+			_DBG(vctrl, "MMIO: CC.SHN != 0 - shutdown\n");
+			nvmet_mdev_mmio_cntrl_shutdown(vctrl);
+		}
+
+		/* change in controller enabled state */
+		if (nvmet_cc_en(val) == nvmet_cc_en(oldval))
+			break;
+
+		if (nvmet_cc_en(vctrl->mmio.cc)) {
+			_DBG(vctrl, "MMIO: CC.EN<=1 - enable the controller\n");
+			nvmet_mdev_mmio_cntrl_enable(vctrl);
+		} else {
+			_DBG(vctrl, "MMIO: CC.EN<=0 - reset controller\n");
+			__nvmet_mdev_vctrl_reset(vctrl, false);
+		}
+
+		break;
+	case NVME_REG_CSTS:
+		/* Controller Status */
+		if (is_write)
+			goto drop;
+		store_le32(buf, vctrl->mmio.csts);
+		break;
+	case NVME_REG_AQA:
+		/* admin queue submission and completion size */
+		if (!is_write)
+			store_le32(buf, vctrl->mmio.aqa);
+		else if (!(vctrl->mmio.csts & NVME_CSTS_RDY))
+			vctrl->mmio.aqa = val;
+		else
+			goto drop;
+		break;
+	case NVME_REG_ASQ:
+		/* admin submission queue address (low 32 bit) */
+		if (!is_write)
+			store_le32(buf, vctrl->mmio.asql);
+		else if (!(vctrl->mmio.csts & NVME_CSTS_RDY))
+			vctrl->mmio.asql = val;
+		else
+			goto drop;
+		break;
+
+	case NVME_REG_ASQ + 4:
+		/* admin submission queue address (high 32 bit) */
+		if (!is_write)
+			store_le32(buf, vctrl->mmio.asqh);
+		else if (!(vctrl->mmio.csts & NVME_CSTS_RDY))
+			vctrl->mmio.asqh = val;
+		else
+			goto drop;
+		break;
+	case NVME_REG_ACQ:
+		/* admin completion queue address (low 32 bit) */
+		if (!is_write)
+			store_le32(buf, vctrl->mmio.acql);
+		else if (!(vctrl->mmio.csts & NVME_CSTS_RDY))
+			vctrl->mmio.acql = val;
+		else
+			goto drop;
+		break;
+	case NVME_REG_ACQ + 4:
+		/* admin completion queue address (high 32 bit) */
+		if (!is_write)
+			store_le32(buf, vctrl->mmio.acqh);
+		else if (!(vctrl->mmio.csts & NVME_CSTS_RDY))
+			vctrl->mmio.acqh = val;
+		else
+			goto drop;
+		break;
+	case NVME_REG_CMBLOC:
+	case NVME_REG_CMBSZ:
+		/* not supported - hardwired to 0 */
+		if (is_write)
+			goto drop;
+		store_le32(buf, 0);
+		break;
+	case NVME_REG_DBS ... (NVME_REG_DBS + DB_AREA_SIZE - 1): {
+		/* completion and submission doorbells */
+		u16 db_offset = offset - NVME_REG_DBS;
+		u16 index = db_offset >> (DB_STRIDE_SHIFT + 2);
+		u16 qid = index >> 1;
+		bool sq = (index & 0x1) == 0;
+
+		if (!is_write || (db_offset & DB_MASK))
+			goto drop;
+
+		if (!(vctrl->mmio.csts & NVME_CSTS_RDY))
+			goto drop;
+
+		if (nvmet_mdev_vctrl_is_dead(vctrl))
+			goto drop;
+
+		sq ? nvmet_mdev_mmio_db_write_sq(vctrl, qid, val) :
+		     nvmet_mdev_mmio_db_write_cq(vctrl, qid, val);
+		break;
+	}
+	default:
+		goto drop;
+	}
+
+	mutex_unlock(&vctrl->lock);
+	return count;
+
+drop:
+	_WARN(vctrl, "MMIO: dropping write at 0x%x\n", offset);
+	mutex_unlock(&vctrl->lock);
+	return 0;
+}
+
+/* Called when the virtual controller is created */
+int nvmet_mdev_mmio_create(struct nvmet_mdev_vctrl *vctrl)
+{
+	struct nvmet_ctrl *ctrl = vctrl->nvmet_ctrl;
+	int ret;
+
+	/* BAR0 */
+	nvmet_mdev_pci_setup_bar(vctrl, PCI_BASE_ADDRESS_0, MMIO_BAR_SIZE,
+				 nvmet_mdev_mmio_bar_access);
+
+	/* CAP has 4 bits for the doorbell stride shift */
+	BUILD_BUG_ON(DB_STRIDE_SHIFT > 0xF);
+
+	/* Shadow doorbell limits doorbells to 1 page */
+	BUILD_BUG_ON(DB_AREA_SIZE > PAGE_SIZE);
+
+	/* Just in case... */
+	BUILD_BUG_ON((PAGE_SHIFT - 12) > 0xF);
+
+	/*
+	 * Inherit nvmet core's defaults and configfs values and then add
+	 * specific settings.
+	 */
+	vctrl->mmio.cap = ctrl->cap;
+
+	vctrl->mmio.cap |=
+		/*
+		 * CQR: physically contiguous queues until we fine a OS
+		 * that needs otherwise.
+		 */
+		(1ULL << 16) |
+		/* DSTRD: doorbell stride */
+		((u64)DB_STRIDE_SHIFT << 32) |
+		/* MPSMIN: Minimum page size supported is PAGE_SIZE */
+		((u64)(PAGE_SHIFT - 12) << 48) |
+		/* MPSMAX: Maximum page size is PAGE_SIZE as well */
+		((u64)(PAGE_SHIFT - 12) << 52);
+
+	/* Create the (regular) doorbell buffers */
+	vctrl->mmio.dbs_page = alloc_pages(__GFP_ZERO, 0);
+
+	ret = -ENOMEM;
+
+	if (!vctrl->mmio.dbs_page)
+		goto error0;
+
+	vctrl->mmio.db_page_kmap = kmap_local_page(vctrl->mmio.dbs_page);
+	if (!vctrl->mmio.db_page_kmap)
+		goto error1;
+
+	vctrl->mmio.fake_eidx_page = alloc_page(__GFP_ZERO);
+	if (!vctrl->mmio.fake_eidx_page)
+		goto error2;
+
+	vctrl->mmio.fake_eidx_kmap = kmap_local_page(vctrl->mmio.fake_eidx_page);
+	if (!vctrl->mmio.fake_eidx_kmap)
+		goto error3;
+	return 0;
+error3:
+	put_page(vctrl->mmio.fake_eidx_kmap);
+error2:
+	kunmap_local(vctrl->mmio.db_page_kmap);
+error1:
+	put_page(vctrl->mmio.dbs_page);
+error0:
+	return ret;
+}
+
+/* Called when the virtual controller is reset */
+void nvmet_mdev_mmio_reset(struct nvmet_mdev_vctrl *vctrl, bool pci_reset)
+{
+	vctrl->mmio.cc = 0;
+	vctrl->mmio.csts = 0;
+
+	if (pci_reset) {
+		vctrl->mmio.aqa  = 0;
+		vctrl->mmio.asql = 0;
+		vctrl->mmio.asqh = 0;
+		vctrl->mmio.acql = 0;
+		vctrl->mmio.acqh = 0;
+	}
+}
+
+/* Called when the virtual controller is opened */
+void nvmet_mdev_mmio_open(struct nvmet_mdev_vctrl *vctrl)
+{
+	if (!vctrl->mmio.shadow_db_supported)
+		nvmet_mdev_vctrl_region_set_mmap(vctrl,
+						 VFIO_PCI_BAR0_REGION_INDEX,
+						 NVME_REG_DBS, PAGE_SIZE,
+						 &nvmet_mdev_mmio_dbs_vm_ops);
+	else
+		nvmet_mdev_vctrl_region_disable_mmap(vctrl,
+						     VFIO_PCI_BAR0_REGION_INDEX);
+}
+
+/* Called when the virtual controller queues are enabled */
+int nvmet_mdev_mmio_enable_dbs(struct nvmet_mdev_vctrl *vctrl)
+{
+	if (WARN_ON(vctrl->mmio.shadow_db_en))
+		return -EINVAL;
+
+	nvmet_mdev_assert_io_not_running(vctrl);
+
+	/* setup normal doorbells and reset them */
+	vctrl->mmio.dbs = vctrl->mmio.db_page_kmap;
+	vctrl->mmio.eidxs = vctrl->mmio.fake_eidx_kmap;
+	memset((void *)vctrl->mmio.dbs, 0, DB_AREA_SIZE);
+	memset((void *)vctrl->mmio.eidxs, 0, DB_AREA_SIZE);
+	return 0;
+}
+
+/* Called when the virtual controller shadow doorbell is enabled */
+int nvmet_mdev_mmio_enable_dbs_shadow(struct nvmet_mdev_vctrl *vctrl,
+				      dma_addr_t sdb_iova,
+				      dma_addr_t eidx_iova)
+{
+	int ret;
+
+	ret = nvmet_mdev_viommu_add(&vctrl->viommu,
+				    VFIO_DMA_MAP_FLAG_READ |
+				    VFIO_DMA_MAP_FLAG_WRITE,
+				    sdb_iova, PAGE_SIZE,
+				    &vctrl->viommu.mem_map_list);
+	if (ret)
+		return ret;
+
+	ret = nvmet_mdev_viommu_create_kmap(&vctrl->viommu, sdb_iova,
+					    &vctrl->mmio.sdb_map);
+	if (ret)
+		goto unmap_sdb;
+
+	ret = nvmet_mdev_viommu_add(&vctrl->viommu,
+				    VFIO_DMA_MAP_FLAG_READ |
+				    VFIO_DMA_MAP_FLAG_WRITE,
+				    eidx_iova, PAGE_SIZE,
+				    &vctrl->viommu.mem_map_list);
+	if (ret)
+		goto kunmap_sdb;
+
+	ret = nvmet_mdev_viommu_create_kmap(&vctrl->viommu, eidx_iova,
+					    &vctrl->mmio.seidx_map);
+	if (ret)
+		goto unmap_eidx;
+
+	vctrl->mmio.sdb_iova = sdb_iova;
+	vctrl->mmio.dbs = vctrl->mmio.sdb_map.kmap;
+
+	vctrl->mmio.eidx_iova = eidx_iova;
+	vctrl->mmio.eidxs = vctrl->mmio.seidx_map.kmap;
+
+	memcpy((void *)vctrl->mmio.dbs, vctrl->mmio.db_page_kmap,
+	       DB_AREA_SIZE);
+	memcpy((void *)vctrl->mmio.eidxs, vctrl->mmio.db_page_kmap,
+	       DB_AREA_SIZE);
+
+	vctrl->mmio.shadow_db_en = true;
+	return 0;
+
+unmap_eidx:
+	nvmet_mdev_vctrl_viommu_unmap(vctrl, eidx_iova, PAGE_SIZE);
+kunmap_sdb:
+	nvmet_mdev_viommu_free_kmap(&vctrl->viommu, &vctrl->mmio.sdb_map);
+unmap_sdb:
+	nvmet_mdev_vctrl_viommu_unmap(vctrl, sdb_iova, PAGE_SIZE);
+	return ret;
+}
+
+/* Disable the doorbells */
+void nvmet_mdev_mmio_disable_dbs(struct nvmet_mdev_vctrl *vctrl)
+{
+	nvmet_mdev_assert_io_not_running(vctrl);
+
+	/* Free the shadow doorbells */
+	nvmet_mdev_viommu_free_kmap(&vctrl->viommu, &vctrl->mmio.sdb_map);
+	nvmet_mdev_viommu_free_kmap(&vctrl->viommu, &vctrl->mmio.seidx_map);
+
+	nvmet_mdev_vctrl_viommu_unmap(vctrl, vctrl->mmio.sdb_iova, PAGE_SIZE);
+	nvmet_mdev_vctrl_viommu_unmap(vctrl, vctrl->mmio.eidx_iova, PAGE_SIZE);
+
+	/* Clear the doorbells */
+	vctrl->mmio.sdb_iova = 0;
+	vctrl->mmio.eidx_iova = 0;
+	vctrl->mmio.dbs = NULL;
+	vctrl->mmio.eidxs = NULL;
+	vctrl->mmio.shadow_db_en = false;
+}
+
+/* Called when the virtual controller is about to be freed */
+void nvmet_mdev_mmio_free(struct nvmet_mdev_vctrl *vctrl)
+{
+	nvmet_mdev_assert_io_not_running(vctrl);
+	kunmap_local(vctrl->mmio.db_page_kmap);
+	put_page(vctrl->mmio.dbs_page);
+	kunmap_local(vctrl->mmio.fake_eidx_kmap);
+	put_page(vctrl->mmio.fake_eidx_page);
+}
diff --git a/drivers/nvme/target/mdev-pci/pci.c b/drivers/nvme/target/mdev-pci/pci.c
new file mode 100644
index 000000000000..dda0eed00854
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/pci.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * NVMe virtual controller minimal PCI/PCIe config space implementation
+ * Copyright (c) 2019 - Maxim Levitsky
+ */
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include "priv.h"
+
+/* setup a 64 bit PCI bar */
+void nvmet_mdev_pci_setup_bar(struct nvmet_mdev_vctrl *vctrl, u8 bar,
+			      unsigned int size, region_access_fn access_fn)
+{
+	nvmet_mdev_vctrl_add_region(vctrl, VFIO_PCI_BAR0_REGION_INDEX +
+				    ((bar - PCI_BASE_ADDRESS_0) >> 2),
+				    size, access_fn);
+
+	store_le32(vctrl->pcicfg.wmask + bar, ~((u64)size - 1));
+	store_le32(vctrl->pcicfg.values + bar,
+		   PCI_BASE_ADDRESS_SPACE_MEMORY |
+		   PCI_BASE_ADDRESS_MEM_TYPE_64);
+}
+
+/* Allocate a pci capability */
+static u8 nvmet_mdev_pci_allocate_cap(struct nvmet_mdev_vctrl *vctrl, u8 id,
+				      u8 size)
+{
+	u8 *cfg = vctrl->pcicfg.values;
+	u8 newcap = vctrl->pcicfg.end;
+	u8 cap = cfg[PCI_CAPABILITY_LIST];
+
+	size = round_up(size, 4);
+	/* only standard cfg space caps for now */
+	WARN_ON(newcap + size > 256);
+
+	if (!cfg[PCI_CAPABILITY_LIST]) {
+		/* special case for first capability */
+		u16 status = load_le16(cfg + PCI_STATUS);
+
+		status |= PCI_STATUS_CAP_LIST;
+		store_le16(cfg + PCI_STATUS, status);
+
+		cfg[PCI_CAPABILITY_LIST] = newcap;
+		goto setupcap;
+	}
+
+	while (cfg[cap + PCI_CAP_LIST_NEXT] != 0)
+		cap = cfg[cap + PCI_CAP_LIST_NEXT];
+
+	cfg[cap + PCI_CAP_LIST_NEXT] = newcap;
+
+setupcap:
+	cfg[newcap + PCI_CAP_LIST_ID] = id;
+	cfg[newcap + PCI_CAP_LIST_NEXT] = 0;
+	vctrl->pcicfg.end += size;
+	return newcap;
+}
+
+static void nvmet_mdev_pci_setup_pm_cap(struct nvmet_mdev_vctrl *vctrl)
+{
+	u8 *cfg  =  vctrl->pcicfg.values;
+	u8 *cfgm =  vctrl->pcicfg.wmask;
+
+	u8 cap = nvmet_mdev_pci_allocate_cap(vctrl, PCI_CAP_ID_PM,
+					     PCI_PM_SIZEOF);
+
+	store_le16(cfg + cap + PCI_PM_PMC, 0x3);
+	store_le16(cfg + cap + PCI_PM_CTRL, PCI_PM_CTRL_NO_SOFT_RESET);
+	store_le16(cfgm + cap + PCI_PM_CTRL, 0x3);
+	vctrl->pcicfg.pmcap = cap;
+}
+
+static void nvmet_mdev_pci_setup_msix_cap(struct nvmet_mdev_vctrl *vctrl)
+{
+	u8 *cfg  =  vctrl->pcicfg.values;
+	u8 *cfgm =  vctrl->pcicfg.wmask;
+	u8  cap = nvmet_mdev_pci_allocate_cap(vctrl, PCI_CAP_ID_MSIX,
+					      PCI_CAP_MSIX_SIZEOF);
+
+	int MSIX_TBL_SIZE = roundup(MAX_VIRTUAL_IRQS * 16, PAGE_SIZE);
+	int MSIX_PBA_SIZE = roundup(DIV_ROUND_UP(MAX_VIRTUAL_IRQS, 8),
+				    PAGE_SIZE);
+
+	store_le16(cfg + cap + PCI_MSIX_FLAGS, MAX_VIRTUAL_IRQS - 1);
+	store_le16(cfgm + cap + PCI_MSIX_FLAGS,
+		   PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE);
+
+	store_le32(cfg + cap + PCI_MSIX_TABLE, 0x2);
+	store_le32(cfg + cap + PCI_MSIX_PBA, MSIX_TBL_SIZE | 0x2);
+
+	nvmet_mdev_pci_setup_bar(vctrl, PCI_BASE_ADDRESS_2,
+				 __roundup_pow_of_two(MSIX_TBL_SIZE +
+						MSIX_PBA_SIZE), NULL);
+	vctrl->pcicfg.msixcap = cap;
+}
+
+static void nvmet_mdev_pci_setup_pcie_cap(struct nvmet_mdev_vctrl *vctrl)
+{
+	u8 *cfg = vctrl->pcicfg.values;
+	u8 cap = nvmet_mdev_pci_allocate_cap(vctrl, PCI_CAP_ID_EXP,
+					     PCI_CAP_EXP_ENDPOINT_SIZEOF_V2);
+
+	store_le16(cfg + cap + PCI_EXP_FLAGS, 0x02 |
+		   (PCI_EXP_TYPE_ENDPOINT << 4));
+
+	store_le32(cfg + cap + PCI_EXP_DEVCAP,
+		   PCI_EXP_DEVCAP_RBER | PCI_EXP_DEVCAP_FLR);
+	store_le32(cfg + cap + PCI_EXP_LNKCAP,
+		   PCI_EXP_LNKCAP_SLS_8_0GB | (4 << 4) /*4x*/);
+	store_le16(cfg + cap + PCI_EXP_LNKSTA,
+		   PCI_EXP_LNKSTA_CLS_8_0GB | (4 << 4) /*4x*/);
+
+	store_le32(cfg + cap + PCI_EXP_LNKCAP2, PCI_EXP_LNKCAP2_SLS_8_0GB);
+	store_le16(cfg + cap + PCI_EXP_LNKCTL2, PCI_EXP_LNKCTL2_TLS_8_0GT);
+	vctrl->pcicfg.pciecap = cap;
+}
+
+/* This is called on PCI config read/write */
+static int nvmet_mdev_pci_cfg_access(struct nvmet_mdev_vctrl *vctrl, u16 offset,
+				     char *buf, u32 count, bool is_write)
+{
+	unsigned int i;
+
+	mutex_lock(&vctrl->lock);
+
+	if (!is_write) {
+		memcpy(buf, (vctrl->pcicfg.values + offset), count);
+		goto out;
+	}
+
+	for (i = 0; i < count; i++) {
+		u8 address = offset + i;
+		u8 value = buf[i];
+		u8 old_value = vctrl->pcicfg.values[address];
+		u8 wmask = vctrl->pcicfg.wmask[address];
+		u8 new_value = (value & wmask) | (old_value & ~wmask);
+
+		/* D3/D0 power control */
+		if (address == vctrl->pcicfg.pmcap + PCI_PM_CTRL) {
+			u8 state = new_value & 0x03;
+
+			if (state != 0 && state != 3)
+				new_value = old_value;
+
+			if (old_value != new_value) {
+				const char *s = state == 3 ? "D3" : "D0";
+
+				if (state == 3)
+					__nvmet_mdev_vctrl_reset(vctrl, true);
+				_DBG(vctrl, "PCI: going to %s\n", s);
+			}
+		}
+
+		/* FLR reset */
+		if (address == vctrl->pcicfg.pciecap + PCI_EXP_DEVCTL + 1)
+			if (value & 0x80) {
+				_DBG(vctrl, "PCI: FLR reset\n");
+				__nvmet_mdev_vctrl_reset(vctrl, true);
+			}
+		vctrl->pcicfg.values[offset + i] = new_value;
+	}
+out:
+	mutex_unlock(&vctrl->lock);
+	return count;
+}
+
+/* setup pci configuration */
+int nvmet_mdev_pci_create(struct nvmet_mdev_vctrl *vctrl)
+{
+	u8 *cfg, *cfgm;
+
+	vctrl->pcicfg.values = kzalloc(NVMET_MDEV_PCI_CFG_SIZE, GFP_KERNEL);
+	if (!vctrl->pcicfg.values)
+		return -ENOMEM;
+
+	vctrl->pcicfg.wmask = kzalloc(NVMET_MDEV_PCI_CFG_SIZE, GFP_KERNEL);
+	if (!vctrl->pcicfg.wmask) {
+		kfree(vctrl->pcicfg.values);
+		return -ENOMEM;
+	}
+
+	cfg = vctrl->pcicfg.values;
+	cfgm = vctrl->pcicfg.wmask;
+
+	nvmet_mdev_vctrl_add_region(vctrl, VFIO_PCI_CONFIG_REGION_INDEX,
+				    NVMET_MDEV_PCI_CFG_SIZE,
+				    nvmet_mdev_pci_cfg_access);
+
+	/* vendor information */
+	store_le16(cfg + PCI_VENDOR_ID, NVME_MDEV_PCI_VENDOR_ID);
+	store_le16(cfg + PCI_DEVICE_ID, NVME_MDEV_PCI_DEVICE_ID);
+
+	/* pci command register */
+	store_le16(cfgm + PCI_COMMAND,
+		   PCI_COMMAND_INTX_DISABLE |
+		   PCI_COMMAND_MEMORY |
+		   PCI_COMMAND_MASTER);
+
+	/* pci status register */
+	store_le16(cfg + PCI_STATUS, PCI_STATUS_CAP_LIST);
+
+	/* subsystem information */
+	store_le16(cfg + PCI_SUBSYSTEM_VENDOR_ID, NVME_MDEV_PCI_SUBVENDOR_ID);
+	store_le16(cfg + PCI_SUBSYSTEM_ID, NVME_MDEV_PCI_SUBDEVICE_ID);
+	store_le8(cfg + PCI_CLASS_REVISION, NVME_MDEV_PCI_REVISION);
+
+	/* Programming Interface (NVM Express) */
+	store_le8(cfg + PCI_CLASS_PROG, 0x02);
+
+	/*
+	 * Device class and subclass (Mass storage controller, Non-Volatile
+	 * memory controller)
+	 */
+	store_le16(cfg + PCI_CLASS_DEVICE, 0x0108);
+
+	/* dummy read/write */
+	store_le8(cfgm + PCI_CACHE_LINE_SIZE, 0xFF);
+
+	/* initial value */
+	store_le8(cfg + PCI_CAPABILITY_LIST, 0);
+	vctrl->pcicfg.end = 0x40;
+
+	nvmet_mdev_pci_setup_pm_cap(vctrl);
+	nvmet_mdev_pci_setup_msix_cap(vctrl);
+	nvmet_mdev_pci_setup_pcie_cap(vctrl);
+
+	/* INTX IRQ number - info only for BIOS */
+	store_le8(cfgm + PCI_INTERRUPT_LINE, 0xFF);
+	store_le8(cfg + PCI_INTERRUPT_PIN, 0x01);
+
+	return 0;
+}
+
+/* teardown pci configuration */
+void nvmet_mdev_pci_free(struct nvmet_mdev_vctrl *vctrl)
+{
+	kfree(vctrl->pcicfg.values);
+	kfree(vctrl->pcicfg.wmask);
+	vctrl->pcicfg.values = NULL;
+	vctrl->pcicfg.wmask = NULL;
+}
diff --git a/drivers/nvme/target/mdev-pci/priv.h b/drivers/nvme/target/mdev-pci/priv.h
new file mode 100644
index 000000000000..9572a182ddc7
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/priv.h
@@ -0,0 +1,487 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Driver private data structures and helper macros
+ * Copyright (c) 2019 - Maxim Levitsky
+ */
+
+#ifndef _MDEV_NVME_PRIV_H
+#define _MDEV_NVME_PRIV_H
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/rbtree.h>
+#include <linux/vfio.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/eventfd.h>
+#include <linux/byteorder/generic.h>
+#include "../../host/nvme.h"
+#include "../nvmet.h"
+
+#define NVME_MDEV_NVME_VER  NVME_VS(0x01, 0x03, 0x00)
+
+#define NVME_MDEV_PCI_VENDOR_ID		PCI_VENDOR_ID_REDHAT_QUMRANET
+#define NVME_MDEV_PCI_DEVICE_ID		0x1234
+#define NVME_MDEV_PCI_SUBVENDOR_ID	PCI_SUBVENDOR_ID_REDHAT_QUMRANET
+#define NVME_MDEV_PCI_SUBDEVICE_ID	0
+#define NVME_MDEV_PCI_REVISION		0x0
+
+#define DB_STRIDE_SHIFT 4 /*4 = 1 cacheline */
+#define NVMET_MDEV_MAX_NR_QUEUES 16
+#define MAX_VIRTUAL_IRQS 128
+
+struct page_map {
+	void *kmap;
+	struct page *page;
+	dma_addr_t iova;
+};
+
+struct user_prplist {
+	/* used by user data iterator */
+	struct page_map page;
+	/* index of current entry */
+	unsigned int index;
+};
+
+struct nvmet_ext_data_iter {
+	/* private */
+	struct nvmet_mdev_viommu *viommu;
+	union {
+		const union nvme_data_ptr *dptr;
+	struct user_prplist uprp;
+	};
+
+	/* user interface */
+	/* number of data pages, yet to be covered */
+	u64		count;
+	/* iterator physical address value */
+	phys_addr_t	physical;
+
+	struct list_head *mem_map_list;
+
+	/* moves iterator to the next item */
+	int (*next)(struct nvmet_ext_data_iter *data_iter);
+	/*
+	 * if != NULL, user should call this when it's done with data
+	 * pointed by the iterator
+	 */
+	void (*release)(struct nvmet_ext_data_iter *data_iter);
+};
+
+/* virtual submission queue */
+struct nvmet_mdev_vsq {
+	struct nvmet_sq nvmet_sq;
+	u16 qid;
+	u16 size;
+	/* next item to read */
+	u16 head;
+
+	/* the queue */
+	struct nvme_command *data;
+	unsigned int data_size;
+	dma_addr_t iova;
+
+	struct nvmet_mdev_vcq *vcq;
+	struct nvmet_mdev_vctrl *vctrl;
+
+	struct nvmet_mdev_req *reqs;
+};
+
+/* virtual completion queue */
+struct nvmet_mdev_vcq {
+	struct nvmet_cq nvmet_cq;
+	/* basic queue settings */
+	u16 qid;
+	u16 size;
+	u16 head;
+	u16 tail;
+	u16 phase;
+
+	volatile struct nvme_completion *data;
+	unsigned int data_size;
+	dma_addr_t iova;
+
+	struct llist_head mreq_list;
+	struct nvmet_mdev_vctrl *vctrl;
+
+	/* IRQ settings */
+	int irq /* -1 if disabled */;
+};
+
+struct nvmet_mdev_req {
+	struct nvmet_req req;
+	struct nvme_completion cqe;
+	struct sg_table sgt;
+	struct nvmet_ext_data_iter data_iter;
+	struct list_head mem_map_list;
+
+	struct nvmet_mdev_vcq *vcq;
+	struct llist_node cq_node;
+};
+
+/* Virtual IOMMU */
+struct nvmet_mdev_viommu {
+	struct vfio_device *vfio_dev;
+
+	/* dma/prp bookkeeping */
+	struct rb_root_cached maps_tree;
+	struct list_head mem_map_list;
+	struct nvmet_mdev_vctrl *vctrl;
+};
+
+struct doorbell {
+	volatile __le32 sqt;
+	u8 rsvd1[(4 << DB_STRIDE_SHIFT) - sizeof(__le32)];
+	volatile __le32 cqh;
+	u8 rsvd2[(4 << DB_STRIDE_SHIFT) - sizeof(__le32)];
+};
+
+/* MMIO state */
+struct nvmet_mdev_user_ctrl_mmio {
+	u32 cc;		/* controller configuration */
+	u32 csts;	/* controller status */
+	u64 cap		/* controller capabilities */;
+
+	/* admin queue location & size */
+	u32 aqa;
+	u32 asql;
+	u32 asqh;
+	u32 acql;
+	u32 acqh;
+
+	bool shadow_db_supported;
+	bool shadow_db_en;
+
+	/* Regular doorbells */
+	struct page *dbs_page;
+	struct page *fake_eidx_page;
+	void *db_page_kmap;
+	void *fake_eidx_kmap;
+
+	/* Shadow doorbells */
+	dma_addr_t sdb_iova;
+	struct page_map sdb_map;
+	dma_addr_t eidx_iova;
+	struct page_map seidx_map;
+
+	/* Current doorbell mappings */
+	volatile struct doorbell *dbs;
+	volatile struct doorbell *eidxs;
+};
+
+/* pci configuration space of the device */
+#define NVMET_MDEV_PCI_CFG_SIZE 4096
+struct nvmet_mdev_pci_cfg_space {
+	u8 *values;
+	u8 *wmask;
+
+	u8 pmcap;
+	u8 pciecap;
+	u8 msixcap;
+	u8 end;
+};
+
+/* IRQ state of the controller */
+struct nvmet_mdev_user_irq {
+	struct eventfd_ctx *trigger;
+	/* IRQ coalescing */
+	bool irq_coalesc_en;
+	ktime_t irq_time;
+	unsigned int irq_pending_cnt;
+};
+
+enum nvmet_mdev_irq_mode {
+	NVME_MDEV_IMODE_NONE,
+	NVME_MDEV_IMODE_INTX,
+	NVME_MDEV_IMODE_MSIX,
+};
+
+struct nvmet_mdev_user_irqs {
+	/* one of VFIO_PCI_{INTX|MSI|MSIX}_IRQ_INDEX */
+	enum nvmet_mdev_irq_mode mode;
+
+	struct nvmet_mdev_user_irq vecs[MAX_VIRTUAL_IRQS];
+	/* user interrupt coalescing settings */
+	u8 irq_coalesc_max;
+	unsigned int irq_coalesc_time_us;
+	/* device removal trigger */
+	struct eventfd_ctx *request_trigger;
+};
+
+/* IO region abstraction (BARs, the PCI config space */
+struct nvmet_mdev_vctrl;
+typedef int (*region_access_fn)(struct nvmet_mdev_vctrl *vctrl, u16 offset,
+				char *buf, u32 size, bool is_write);
+
+struct nvmet_mdev_io_region {
+	unsigned int size;
+	region_access_fn rw;
+
+	/*
+	 * IF != NULL, the mmap_area_start/size specify the mmaped window
+	 * of this region
+	 */
+	const struct vm_operations_struct *mmap_ops;
+	unsigned int mmap_area_start;
+	unsigned int mmap_area_size;
+};
+
+struct nvmet_mdev_type {
+	struct mdev_type type;
+	char *name;
+};
+
+struct nvmet_mdev_port {
+	struct mdev_parent parent;
+	struct device device;
+	struct nvmet_port *nvmet_port;
+	struct mutex mutex;
+
+	int ctrl_count;
+	int type_count;
+	struct nvmet_mdev_type *types;
+	struct mdev_type **mdev_types;
+};
+
+#define vfio_dev_to_nvmet_mdev_vctrl(vfio_dev) \
+	container_of(vfio_dev, struct nvmet_mdev_vctrl, vfio_dev)
+
+/* Virtual NVME controller state */
+struct nvmet_mdev_vctrl {
+	struct vfio_device vfio_dev;
+	struct kref ref;
+	struct mutex lock;
+
+	struct mdev_device *mdev;
+	struct nvmet_ctrl *nvmet_ctrl;
+
+	/* the IO thread */
+	struct task_struct *iothread;
+	bool iothread_parked;
+	bool io_idle;
+	ktime_t now;
+	int expected_responses;
+	int poll_timeout_ms;
+
+	struct nvmet_mdev_io_region regions[VFIO_PCI_NUM_REGIONS];
+
+	/* virtual controller state */
+	struct nvmet_mdev_user_ctrl_mmio mmio;
+	struct nvmet_mdev_pci_cfg_space pcicfg;
+	struct nvmet_mdev_user_irqs irqs;
+
+	/* emulated submission queues */
+	struct nvmet_mdev_vsq vsqs[NVMET_MDEV_MAX_NR_QUEUES];
+	unsigned long vsq_en[BITS_TO_LONGS(NVMET_MDEV_MAX_NR_QUEUES)];
+
+	/* emulated completion queues */
+	unsigned long vcq_en[BITS_TO_LONGS(NVMET_MDEV_MAX_NR_QUEUES)];
+	struct nvmet_mdev_vcq vcqs[NVMET_MDEV_MAX_NR_QUEUES];
+
+	/* Interface to access user memory */
+	struct notifier_block vfio_map_notifier;
+	struct notifier_block vfio_unmap_notifier;
+	struct nvmet_mdev_viommu viommu;
+
+	/* Settings */
+	unsigned int arb_burst_shift;
+	u8 worload_hint;
+	unsigned int iothread_cpu;
+};
+
+/* vctrl.c */
+int nvmet_mdev_vctrl_create(struct nvmet_mdev_vctrl *vctrl,
+			    struct mdev_device *mdev);
+void nvmet_mdev_vctrl_destroy(struct nvmet_mdev_vctrl *vctrl);
+int nvmet_mdev_vctrl_open(struct vfio_device *vfio_dev);
+void nvmet_mdev_vctrl_release(struct vfio_device *vfio_dev);
+bool nvmet_mdev_vctrl_enable(struct nvmet_mdev_vctrl *vctrl, dma_addr_t cqiova,
+			     dma_addr_t sqiova, u32 sizes);
+void nvmet_mdev_vctrl_disable(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_vctrl_reset(struct nvmet_mdev_vctrl *vctrl);
+void __nvmet_mdev_vctrl_reset(struct nvmet_mdev_vctrl *vctrl, bool pci_reset);
+void nvmet_mdev_vctrl_add_region(struct nvmet_mdev_vctrl *vctrl,
+				 unsigned int index, unsigned int size,
+				 region_access_fn access_fn);
+void nvmet_mdev_vctrl_region_set_mmap(struct nvmet_mdev_vctrl *vctrl,
+				      unsigned int index, unsigned int offset,
+				      unsigned int size,
+				      const struct vm_operations_struct *ops);
+void nvmet_mdev_vctrl_region_disable_mmap(struct nvmet_mdev_vctrl *vctrl,
+					  unsigned int index);
+bool nvmet_mdev_vctrl_is_dead(struct nvmet_mdev_vctrl *vctrl);
+int nvmet_mdev_vctrl_viommu_map(struct nvmet_mdev_vctrl *vctrl, u32 flags,
+				dma_addr_t iova, u64 size);
+int nvmet_mdev_vctrl_viommu_unmap(struct nvmet_mdev_vctrl *vctrl,
+				  dma_addr_t iova, u64 size);
+
+/* io.c */
+int nvmet_mdev_io_create(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_io_free(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_io_resume(struct nvmet_mdev_vctrl *vctrl);
+bool nvmet_mdev_io_pause(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_assert_io_not_running(struct nvmet_mdev_vctrl *vctrl);
+bool nvmet_mdev_process_responses(struct nvmet_mdev_vctrl *vctrl,
+				  struct nvmet_mdev_vcq *vcq);
+
+/* mmio.c */
+int nvmet_mdev_mmio_create(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_mmio_open(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_mmio_reset(struct nvmet_mdev_vctrl *vctrl, bool pci_reset);
+void nvmet_mdev_mmio_free(struct nvmet_mdev_vctrl *vctrl);
+
+int nvmet_mdev_mmio_enable_dbs(struct nvmet_mdev_vctrl *vctrl);
+int nvmet_mdev_mmio_enable_dbs_shadow(struct nvmet_mdev_vctrl *vctrl,
+				      dma_addr_t sdb_iova,
+				      dma_addr_t eidx_iova);
+
+void nvmet_mdev_mmio_viommu_update(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_mmio_disable_dbs(struct nvmet_mdev_vctrl *vctrl);
+bool nvmet_mdev_mmio_db_check(struct nvmet_mdev_vctrl *vctrl, u16 qid, u16 size,
+			      u16 db);
+
+/* pci.c */
+int nvmet_mdev_pci_create(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_pci_free(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_pci_setup_bar(struct nvmet_mdev_vctrl *vctrl, u8 bar,
+			      unsigned int size, region_access_fn access_fn);
+
+/* irq.c */
+void nvmet_mdev_irqs_setup(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_irqs_reset(struct nvmet_mdev_vctrl *vctrl);
+int nvmet_mdev_irqs_enable(struct nvmet_mdev_vctrl *vctrl,
+			   enum nvmet_mdev_irq_mode mode);
+void nvmet_mdev_irqs_disable(struct nvmet_mdev_vctrl *vctrl,
+			     enum nvmet_mdev_irq_mode mode);
+int nvmet_mdev_irqs_set_triggers(struct nvmet_mdev_vctrl *vctrl,
+				 int start, int count, int32_t *fds);
+int nvmet_mdev_irqs_set_unplug_trigger(struct nvmet_mdev_vctrl *vctrl,
+				       int32_t fd);
+void nvmet_mdev_irq_raise_unplug_event(struct nvmet_mdev_vctrl *vctrl);
+void nvmet_mdev_irq_raise(struct nvmet_mdev_vctrl *vctrl, unsigned int index);
+void nvmet_mdev_irq_trigger(struct nvmet_mdev_vctrl *vctrl, unsigned int index);
+void nvmet_mdev_irq_cond_trigger(struct nvmet_mdev_vctrl *vctrl,
+				 unsigned int index, ktime_t now);
+void nvmet_mdev_irq_clear(struct nvmet_mdev_vctrl *vctrl, unsigned int index,
+			  ktime_t now);
+
+/* vcq.c */
+int nvmet_mdev_vcq_init(struct nvmet_mdev_vctrl *vctrl, u16 qid,
+			dma_addr_t iova, u16 size, int irq);
+int nvmet_mdev_vcq_viommu_update(struct nvmet_mdev_viommu *viommu,
+				 struct nvmet_mdev_vcq *q);
+void nvmet_mdev_vcq_delete(struct nvmet_mdev_vctrl *vctrl, u16 qid);
+void nvmet_mdev_vcq_process(struct nvmet_mdev_vctrl *vctrl,
+			    struct nvmet_mdev_vcq *q, bool trigger_irqs,
+			    ktime_t now);
+bool nvmet_mdev_vcq_flush(struct nvmet_mdev_vctrl *vctrl,
+			  struct nvmet_mdev_vcq *q, ktime_t now);
+void nvmet_mdev_vcq_write_cqe(struct nvmet_mdev_vctrl *vctrl,
+			      struct nvmet_mdev_vcq *q,
+			      struct nvme_completion *cqe);
+
+/* vsq.c */
+int nvmet_mdev_vsq_init(struct nvmet_mdev_vctrl *vctrl, u16 qid,
+			dma_addr_t iova, u16 size, u16 cqid);
+int nvmet_mdev_vsq_viommu_update(struct nvmet_mdev_viommu *viommu,
+				 struct nvmet_mdev_vsq *q);
+void nvmet_mdev_vsq_delete(struct nvmet_mdev_vctrl *vctrl, u16 qid);
+struct nvme_command *nvmet_mdev_vsq_get_cmd(struct nvmet_mdev_vctrl *vctrl,
+					    struct nvmet_mdev_vsq *q,
+					    u16 *index);
+bool nvmet_mdev_vsq_suspend_io(struct nvmet_mdev_vsq *q);
+
+/* udata.c */
+void nvmet_mdev_udata_iter_setup(struct nvmet_mdev_viommu *viommu,
+				 struct nvmet_ext_data_iter *iter,
+				 struct list_head *maps_list);
+int nvmet_mdev_udata_iter_set_dptr(struct nvmet_ext_data_iter *it,
+				   const union nvme_data_ptr *dptr, u64 size);
+void *nvmet_mdev_udata_update_queue_vmap(struct nvmet_mdev_viommu *viommu,
+					 dma_addr_t iova, void *data,
+					 unsigned int data_size);
+void nvmet_mdev_udata_queue_vunmap(struct nvmet_mdev_viommu *viommu,
+				   dma_addr_t iova, void *data,
+				   unsigned int data_size);
+
+/* viommu.c */
+void nvmet_mdev_viommu_init(struct nvmet_mdev_viommu *viommu,
+			    struct vfio_device *vfio_dev);
+int nvmet_mdev_viommu_add(struct nvmet_mdev_viommu *viommu, u32 flags,
+			  dma_addr_t iova, u64 size,
+			  struct list_head *maps_list);
+int nvmet_mdev_viommu_remove(struct nvmet_mdev_viommu *viommu,
+			     dma_addr_t iova, u64 size);
+int nvmet_mdev_viommu_remove_list(struct nvmet_mdev_viommu *viommu,
+				  struct list_head *remove_list);
+int nvmet_mdev_viommu_translate(struct nvmet_mdev_viommu *viommu,
+				dma_addr_t iova, dma_addr_t *physical);
+int nvmet_mdev_viommu_create_kmap(struct nvmet_mdev_viommu *viommu,
+				  dma_addr_t iova, struct page_map *page);
+void nvmet_mdev_viommu_free_kmap(struct nvmet_mdev_viommu *viommu,
+				 struct page_map *page);
+void nvmet_mdev_viommu_update_kmap(struct nvmet_mdev_viommu *viommu,
+				   struct page_map *page);
+void nvmet_mdev_viommu_reset(struct nvmet_mdev_viommu *viommu);
+
+/* instance.c */
+int nvmet_mdev_register_port(struct nvmet_mdev_port *mport);
+void nvmet_mdev_unregister_port(struct nvmet_mdev_port *mport);
+void nvmet_mdev_remove_ctrl(struct nvmet_mdev_vctrl *vctrl);
+
+/* some utilities*/
+
+#define store_le32(address, value) (*((__le32 *)(address)) = cpu_to_le32(value))
+#define store_le16(address, value) (*((__le16 *)(address)) = cpu_to_le16(value))
+#define store_le8(address, value)  (*((u8 *)(address)) = (value))
+
+#define load_le16(address) le16_to_cpu(*(__le16 *)(address))
+
+#define DNR(e) ((e) | NVME_STATUS_DNR)
+
+#define PAGE_ADDRESS(address) ((address) & PAGE_MASK)
+
+#define _DBG(vctrl, fmt, ...) \
+	dev_dbg(vctrl->vfio_dev.dev, fmt, ##__VA_ARGS__)
+
+#define _INFO(vctrl, fmt, ...) \
+	dev_info(vctrl->vfio_dev.dev, fmt, ##__VA_ARGS__)
+
+#define _WARN(vctrl, fmt, ...) \
+	dev_warn(vctrl->vfio_dev.dev, fmt, ##__VA_ARGS__)
+
+/* Rough translation of internal errors to the NVME errors */
+static inline int nvmet_mdev_translate_error(int error)
+{
+	/* nvme status, including no error (NVME_SC_SUCCESS) */
+	if (error >= 0)
+		return error;
+
+	switch (error) {
+	case -ENOMEM:
+		/* no memory - truly an internal error */
+		return NVME_SC_INTERNAL;
+	case -ENOSPC:
+		/*
+		 * Happens when user sends to large PRP list User shoudn't do
+		 * this since the maximum transfer size is specified in the
+		 * controller caps
+		 */
+		return DNR(NVME_SC_DATA_XFER_ERROR);
+	case -EFAULT:
+		/* Bad memory pointers in the prp lists */
+		return DNR(NVME_SC_DATA_XFER_ERROR);
+	case -EINVAL:
+		/* Bad prp offsets in the prp lists/command */
+		return DNR(NVME_SC_PRP_INVALID_OFFSET);
+	default:
+		/* Shouldn't happen */
+		WARN_ON_ONCE(true);
+		return NVME_SC_INTERNAL;
+	}
+}
+
+extern const struct nvmet_fabrics_ops nvmet_mdev_ops;
+
+#endif // _MDEV_NVME_H
diff --git a/drivers/nvme/target/mdev-pci/target.c b/drivers/nvme/target/mdev-pci/target.c
new file mode 100644
index 000000000000..1b688544db1e
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/target.c
@@ -0,0 +1,284 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * NVMe target callouts
+ * Copyright (c) 2019 - Maxim Levitsky
+ * Copyright (C) 2025 Oracle Corporation
+ */
+#include <linux/device.h>
+#include <linux/nvme.h>
+#include <linux/module.h>
+#include <linux/mdev.h>
+#include <linux/slab.h>
+#include "../nvmet.h"
+#include "../../host/nvme.h"
+#include "../../host/fabrics.h"
+#include "priv.h"
+
+static void nvmet_mdev_delete_ctrl(struct nvmet_ctrl *ctrl)
+{
+	struct nvmet_mdev_port *mport = ctrl->port->priv;
+
+	mutex_lock(&mport->mutex);
+	nvmet_mdev_remove_ctrl(ctrl->drvdata);
+	mutex_unlock(&mport->mutex);
+}
+
+static void nvmet_mdev_remove_port(struct nvmet_port *port)
+{
+	nvmet_mdev_unregister_port(port->priv);
+}
+
+static int nvmet_mdev_count_ctrls(void *priv, struct nvmet_port *port,
+				  struct nvmet_ctrl *ctrl)
+{
+	int *count = priv;
+
+	(*count)++;
+	return 0;
+}
+
+static int nvmet_mdev_add_port(struct nvmet_port *port)
+{
+	struct nvmet_mdev_port *mport;
+	int count = 0;
+	int ret;
+
+	ret = nvmet_for_each_static_ctrl(port, nvmet_mdev_ops.type,
+					 nvmet_mdev_count_ctrls, &count);
+	if (ret)
+		return ret;
+
+	if (!count) {
+		pr_err("Controllers must be added and enabled before enabling port.\n");
+		return -ENODEV;
+	}
+
+	mport = kzalloc(sizeof(*mport), GFP_KERNEL);
+	if (!mport)
+		return -ENOMEM;
+
+	mutex_init(&mport->mutex);
+	port->priv = mport;
+	mport->nvmet_port = port;
+	mport->ctrl_count = count;
+
+	return nvmet_mdev_register_port(mport);
+}
+
+static u16 nvmet_mdev_adm_create_cq(struct nvmet_ctrl *ctrl, u16 cqid,
+				    u16 cq_flags, u16 qsize, u64 prp1, u16 irq)
+{
+	struct nvmet_mdev_vctrl *vctrl = ctrl->drvdata;
+	u16 ret;
+
+	mutex_lock(&vctrl->lock);
+	if (cqid >= NVMET_MDEV_MAX_NR_QUEUES || test_bit(cqid, vctrl->vcq_en)) {
+		ret = DNR(NVME_SC_QID_INVALID);
+		goto unlock;
+	}
+
+	if (!(cq_flags & NVME_QUEUE_PHYS_CONTIG)) {
+		ret = DNR(NVME_SC_INVALID_QUEUE);
+		goto unlock;
+	}
+
+	if (cq_flags & NVME_CQ_IRQ_ENABLED) {
+		if (irq >= MAX_VIRTUAL_IRQS) {
+			ret = DNR(NVME_SC_INVALID_VECTOR);
+			goto unlock;
+		}
+	}
+
+	ret = nvmet_mdev_vcq_init(vctrl, cqid, prp1, qsize + 1, irq);
+unlock:
+	mutex_unlock(&vctrl->lock);
+	return ret;
+}
+
+static u16 nvmet_mdev_adm_delete_cq(struct nvmet_ctrl *ctrl, u16 cqid)
+{
+	struct nvmet_mdev_vctrl *vctrl = ctrl->drvdata;
+	u16 ret = NVME_SC_SUCCESS;
+
+	mutex_lock(&vctrl->lock);
+	if (cqid >= NVMET_MDEV_MAX_NR_QUEUES ||
+	    !test_bit(cqid, vctrl->vcq_en)) {
+		ret = DNR(NVME_SC_QID_INVALID);
+		goto unlock;
+	}
+
+	nvmet_mdev_vcq_delete(vctrl, cqid);
+unlock:
+	mutex_unlock(&vctrl->lock);
+	return ret;
+}
+
+static u16 nvmet_mdev_adm_create_sq(struct nvmet_ctrl *ctrl, u16 sqid,
+				    u16 sq_flags, u16 qsize, u64 prp1)
+{
+	struct nvmet_mdev_vctrl *vctrl = ctrl->drvdata;
+	u16 ret;
+
+	mutex_lock(&vctrl->lock);
+	if (sqid >= NVMET_MDEV_MAX_NR_QUEUES || test_bit(sqid, vctrl->vsq_en)) {
+		ret = DNR(NVME_SC_QID_INVALID);
+		goto unlock;
+	}
+
+	/*
+	 * sqid and cqid are checked they are equal by nvmet before calling
+	 * this
+	 */
+	if (!test_bit(sqid, vctrl->vcq_en)) {
+		ret = DNR(NVME_SC_CQ_INVALID);
+		goto unlock;
+	}
+
+	if (!(sq_flags & NVME_QUEUE_PHYS_CONTIG)) {
+		ret = DNR(NVME_SC_INVALID_QUEUE);
+		goto unlock;
+	}
+
+	ret = nvmet_mdev_vsq_init(vctrl, sqid, prp1, qsize + 1, sqid);
+unlock:
+	mutex_unlock(&vctrl->lock);
+	return ret;
+}
+
+static u16 nvmet_mdev_adm_delete_sq(struct nvmet_ctrl *ctrl, u16 sqid)
+{
+	struct nvmet_mdev_vctrl *vctrl = ctrl->drvdata;
+	u16 ret = NVME_SC_SUCCESS;
+
+	mutex_lock(&vctrl->lock);
+	if (sqid >= NVMET_MDEV_MAX_NR_QUEUES ||
+	    !test_bit(sqid, vctrl->vsq_en)) {
+		ret = DNR(NVME_SC_QID_INVALID);
+		goto unlock;
+	}
+
+	nvmet_mdev_vsq_delete(vctrl, sqid);
+unlock:
+	mutex_unlock(&vctrl->lock);
+	return ret;
+}
+
+static u16 nvmet_mdev_adm_get_features(const struct nvmet_ctrl *ctrl, u8 feat,
+				       void *feat_data)
+{
+	struct nvmet_mdev_vctrl *vctrl = ctrl->drvdata;
+	struct nvmet_feat_arbitration *arb;
+	struct nvmet_feat_irq_coalesce *irqc;
+	struct nvmet_feat_irq_config *irqcfg;
+
+	switch (feat) {
+	case NVME_FEAT_ARBITRATION:
+		arb = feat_data;
+
+		arb->ab = vctrl->arb_burst_shift;
+		break;
+	case NVME_FEAT_IRQ_COALESCE:
+		irqc = feat_data;
+
+		irqc->thr = vctrl->irqs.irq_coalesc_max - 1;
+
+		irqc->time = vctrl->irqs.irq_coalesc_time_us;
+		do_div(irqc->time, 100);
+		break;
+	case NVME_FEAT_IRQ_CONFIG:
+		irqcfg = feat_data;
+
+		if (irqcfg->iv >= MAX_VIRTUAL_IRQS)
+			return DNR(NVME_SC_INVALID_FIELD);
+
+		irqcfg->cd = vctrl->irqs.vecs[irqcfg->iv].irq_coalesc_en;
+		break;
+	default:
+		return DNR(NVME_SC_INVALID_FIELD);
+	}
+
+	return NVME_SC_SUCCESS;
+}
+
+static u16 nvmet_mdev_adm_set_features(const struct nvmet_ctrl *ctrl, u8 feat,
+				       void *feat_data)
+{
+	struct nvmet_mdev_vctrl *vctrl = ctrl->drvdata;
+	struct nvmet_feat_arbitration *arb;
+	struct nvmet_feat_irq_coalesce *irqc;
+	struct nvmet_feat_irq_config *irqcfg;
+
+	switch (feat) {
+	case NVME_FEAT_ARBITRATION:
+		arb = feat_data;
+
+		vctrl->arb_burst_shift = arb->ab;
+		break;
+	case NVME_FEAT_IRQ_COALESCE:
+		irqc = feat_data;
+
+		vctrl->irqs.irq_coalesc_max = irqc->thr + 1;
+		vctrl->irqs.irq_coalesc_time_us = irqc->time * 100;
+		break;
+	case NVME_FEAT_IRQ_CONFIG:
+		irqcfg = feat_data;
+
+		if (irqcfg->iv >= MAX_VIRTUAL_IRQS)
+			return DNR(NVME_SC_INVALID_FIELD);
+
+		vctrl->irqs.vecs[irqcfg->iv].irq_coalesc_en = irqcfg->cd != 0;
+		break;
+	default:
+		return DNR(NVME_SC_INVALID_FIELD);
+	}
+
+	return NVME_SC_SUCCESS;
+}
+
+static u16 nvmet_mdev_adm_set_dbbuf(struct nvmet_ctrl *ctrl, u64 prp1, u64 prp2)
+{
+	struct nvmet_mdev_vctrl *vctrl = ctrl->drvdata;
+	int ret;
+
+	if (!vctrl->mmio.shadow_db_supported)
+		return DNR(NVME_SC_INVALID_OPCODE);
+
+	if (vctrl->mmio.shadow_db_en)
+		return DNR(NVME_SC_INVALID_FIELD);
+
+	if ((offset_in_page(prp1) != 0) || (offset_in_page(prp2) != 0))
+		return DNR(NVME_SC_INVALID_FIELD);
+
+	ret = nvmet_mdev_mmio_enable_dbs_shadow(vctrl, prp1, prp2);
+	return nvmet_mdev_translate_error(ret);
+}
+
+static void nvmet_mdev_queue_response(struct nvmet_req *req)
+{
+	struct nvmet_mdev_req *mreq = container_of(req, struct nvmet_mdev_req,
+						   req);
+
+	llist_add(&mreq->cq_node, &mreq->vcq->mreq_list);
+}
+
+const struct nvmet_fabrics_ops nvmet_mdev_ops = {
+	.owner			= THIS_MODULE,
+	.flags			= NVMF_SGLS_NOT_SUPP | NVMF_STATIC_CTRL,
+	.type			= NVMF_TRTYPE_MDEV_PCI,
+	.delete_ctrl		= nvmet_mdev_delete_ctrl,
+	.add_port		= nvmet_mdev_add_port,
+	.remove_port		= nvmet_mdev_remove_port,
+	.create_cq		= nvmet_mdev_adm_create_cq,
+	.delete_cq		= nvmet_mdev_adm_delete_cq,
+	.create_sq		= nvmet_mdev_adm_create_sq,
+	.delete_sq		= nvmet_mdev_adm_delete_sq,
+	.set_feature		= nvmet_mdev_adm_set_features,
+	.get_feature		= nvmet_mdev_adm_get_features,
+	.set_dbbuf		= nvmet_mdev_adm_set_dbbuf,
+	.queue_response		= nvmet_mdev_queue_response,
+};
+
+MODULE_AUTHOR("Maxim Levitsky <mlevitsk@redhat.com>, "
+	      "Mike Christie <michael.christie@oracle.com>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("nvmet-transport-253"); /* 253 == NVMF_TRTYPE_MDEV_PCI */
diff --git a/drivers/nvme/target/mdev-pci/udata.c b/drivers/nvme/target/mdev-pci/udata.c
new file mode 100644
index 000000000000..9fa9f63e9d23
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/udata.c
@@ -0,0 +1,304 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * User (guest) data access routines
+ * Implementation of PRP iterator in user memory
+ * Copyright (c) 2019 - Maxim Levitsky
+ */
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/highmem.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/mdev.h>
+#include <linux/nvme.h>
+#include "priv.h"
+
+#define MAX_PRP ((PAGE_SIZE / sizeof(__le64)) - 1)
+
+/* Setup up a new PRP iterator */
+void nvmet_mdev_udata_iter_setup(struct nvmet_mdev_viommu *viommu,
+				 struct nvmet_ext_data_iter *iter,
+				 struct list_head *mem_map_list)
+{
+	memset(iter, 0, sizeof(*iter));
+	iter->viommu = viommu;
+	iter->mem_map_list = mem_map_list;
+}
+
+/* Load a new prp list into the iterator. Internal */
+static int nvmet_mdev_udata_iter_load_prplist(struct nvmet_ext_data_iter *iter,
+					      dma_addr_t iova)
+{
+	dma_addr_t  data_iova;
+	int ret;
+	__le64 *map;
+
+	ret = nvmet_mdev_viommu_add(iter->viommu,
+				    VFIO_DMA_MAP_FLAG_READ |
+				    VFIO_DMA_MAP_FLAG_WRITE, iova, PAGE_SIZE,
+				    iter->mem_map_list);
+	if (ret)
+		return ret;
+
+	/* map the prp list */
+	ret = nvmet_mdev_viommu_create_kmap(iter->viommu, PAGE_ADDRESS(iova),
+					    &iter->uprp.page);
+	if (ret)
+		return ret;
+
+	iter->uprp.index = offset_in_page(iova) / (sizeof(__le64));
+
+	/* read its first entry and check its alignment */
+	map = iter->uprp.page.kmap;
+	data_iova = le64_to_cpu(map[iter->uprp.index]);
+
+	if (offset_in_page(data_iova) != 0) {
+		nvmet_mdev_viommu_free_kmap(iter->viommu, &iter->uprp.page);
+		return -EINVAL;
+	}
+
+	ret = nvmet_mdev_viommu_add(iter->viommu,
+				    VFIO_DMA_MAP_FLAG_READ |
+				    VFIO_DMA_MAP_FLAG_WRITE, data_iova,
+				    PAGE_SIZE, iter->mem_map_list);
+	if (ret)
+		return ret;
+
+	/* translate the entry to complete the setup */
+	ret =  nvmet_mdev_viommu_translate(iter->viommu, data_iova,
+					   &iter->physical);
+	if (ret)
+		nvmet_mdev_viommu_free_kmap(iter->viommu, &iter->uprp.page);
+
+	return ret;
+}
+
+/* ->next function when iterator points to prp list */
+static int nvmet_mdev_udata_iter_next_prplist(struct nvmet_ext_data_iter *iter)
+{
+	dma_addr_t iova;
+	int ret;
+	__le64 *map = iter->uprp.page.kmap;
+
+	if (WARN_ON(iter->count <= 0))
+		return 0;
+
+	if (--iter->count == 0) {
+		nvmet_mdev_viommu_free_kmap(iter->viommu, &iter->uprp.page);
+		return 0;
+	}
+
+	iter->uprp.index++;
+
+	if (iter->uprp.index < MAX_PRP || iter->count == 1) {
+		/*
+		 * advance over next pointer in current prp list these
+		 * pointers must be page aligned
+		 */
+		iova = le64_to_cpu(map[iter->uprp.index]);
+		if (offset_in_page(iova) != 0)
+			return -EINVAL;
+
+		ret = nvmet_mdev_viommu_add(iter->viommu,
+					    VFIO_DMA_MAP_FLAG_READ |
+					    VFIO_DMA_MAP_FLAG_WRITE, iova,
+					    PAGE_SIZE, iter->mem_map_list);
+		if (ret)
+			return ret;
+
+		ret  = nvmet_mdev_viommu_translate(iter->viommu, iova,
+						   &iter->physical);
+		if (ret)
+			nvmet_mdev_viommu_free_kmap(iter->viommu,
+						    &iter->uprp.page);
+		return ret;
+	}
+
+	/* switch to next prp list. it must be page aligned as well */
+	iova = le64_to_cpu(map[MAX_PRP]);
+
+	if (offset_in_page(iova) != 0)
+		return -EINVAL;
+
+	nvmet_mdev_viommu_free_kmap(iter->viommu, &iter->uprp.page);
+	return nvmet_mdev_udata_iter_load_prplist(iter, iova);
+}
+
+/* ->next function when iterator points to user data pointer */
+static int nvmet_mdev_udata_iter_next_dptr(struct nvmet_ext_data_iter *iter)
+{
+	dma_addr_t  iova;
+	int ret;
+
+	if (WARN_ON(iter->count <= 0))
+		return 0;
+
+	if (--iter->count == 0)
+		return 0;
+
+	/*
+	 * we will be called only once to deal with the second
+	 * pointer in the data pointer
+	 */
+	iova = le64_to_cpu(iter->dptr->prp2);
+
+	if (iter->count == 1) {
+		/*
+		 * only need to read one more entry, meaning the 2nd entry of
+		 * the dptr. It must be page aligned
+		 */
+		if (offset_in_page(iova) != 0)
+			return -EINVAL;
+
+		/*
+		 * Size may be less than a page but it doesn't matter to
+		 * the viommu as we have to get the entire page either way.
+		 */
+		ret = nvmet_mdev_viommu_add(iter->viommu,
+					   VFIO_DMA_MAP_FLAG_READ |
+					   VFIO_DMA_MAP_FLAG_WRITE, iova,
+					   PAGE_SIZE, iter->mem_map_list);
+		if (ret)
+			return ret;
+
+		return nvmet_mdev_viommu_translate(iter->viommu, iova,
+						   &iter->physical);
+	} else {
+		/*
+		 * Second dptr entry is prp pointer, and it might not
+		 * be page aligned (but QWORD aligned at least)
+		 */
+		if (iova & 0x7ULL)
+			return -EINVAL;
+		iter->next = nvmet_mdev_udata_iter_next_prplist;
+		return nvmet_mdev_udata_iter_load_prplist(iter, iova);
+	}
+}
+
+static void nvmet_mdev_udata_iter_release(struct nvmet_ext_data_iter *iter)
+{
+	nvmet_mdev_viommu_free_kmap(iter->viommu, &iter->uprp.page);
+	nvmet_mdev_viommu_remove_list(iter->viommu, iter->mem_map_list);
+}
+
+/* Set prp list iterator to point to data pointer found in NVME command */
+int nvmet_mdev_udata_iter_set_dptr(struct nvmet_ext_data_iter *iter,
+				   const union nvme_data_ptr *dptr, u64 size)
+{
+	int ret;
+	u64 prp1 = le64_to_cpu(dptr->prp1);
+	dma_addr_t iova = PAGE_ADDRESS(prp1);
+	unsigned int page_offset = offset_in_page(prp1);
+
+	/* first dptr pointer must be at least DWORD aligned */
+	if (page_offset & 0x3)
+		return -EINVAL;
+
+	ret = nvmet_mdev_viommu_add(iter->viommu,
+				   VFIO_DMA_MAP_FLAG_READ |
+				   VFIO_DMA_MAP_FLAG_WRITE, iova, size,
+				   iter->mem_map_list);
+	if (ret)
+		return ret;
+
+	iter->dptr = dptr;
+	iter->next = nvmet_mdev_udata_iter_next_dptr;
+	iter->release = nvmet_mdev_udata_iter_release;
+	iter->count = DIV_ROUND_UP_ULL(size + page_offset, PAGE_SIZE);
+
+	ret = nvmet_mdev_viommu_translate(iter->viommu, iova, &iter->physical);
+	if (ret)
+		goto release;
+
+	iter->physical += page_offset;
+	return 0;
+
+release:
+	nvmet_mdev_udata_iter_release(iter);
+	return ret;
+}
+
+/* Map an SQ/CQ queue (contiguous in guest physical memory) */
+static int
+nvmet_mdev_queue_getpages_contiguous(struct nvmet_mdev_viommu *viommu,
+				     dma_addr_t iova, struct page **pages,
+				     unsigned int npages)
+{
+	dma_addr_t curr_iova = iova;
+	phys_addr_t physical;
+	unsigned int i;
+	int ret;
+
+	for (i = 0 ; i < npages; i++) {
+		ret = nvmet_mdev_viommu_add(viommu,
+					    VFIO_DMA_MAP_FLAG_READ |
+					    VFIO_DMA_MAP_FLAG_WRITE,
+					    curr_iova, PAGE_SIZE,
+					    &viommu->mem_map_list);
+		if (ret)
+			goto remove;
+
+		ret = nvmet_mdev_viommu_translate(viommu, curr_iova, &physical);
+		if (ret)
+			goto remove;
+
+		pages[i] = pfn_to_page(PHYS_PFN(physical));
+		curr_iova += PAGE_SIZE;
+	}
+	return 0;
+
+remove:
+	nvmet_mdev_viommu_remove(viommu, iova, npages * PAGE_SIZE);
+	return ret;
+}
+
+/* map a SQ/CQ queue to host physical memory */
+static void *nvmet_mdev_udata_queue_vmap(struct nvmet_mdev_viommu *viommu,
+					 dma_addr_t iova, unsigned int size)
+{
+	unsigned int npages;
+	struct page **pages;
+	void *map = NULL;
+
+	/* queue must be page aligned */
+	if (offset_in_page(iova) != 0)
+		return NULL;
+
+	npages = DIV_ROUND_UP(size, PAGE_SIZE);
+	pages = kcalloc(npages, sizeof(struct page *), GFP_KERNEL);
+	if (!pages)
+		return NULL;
+
+	if (nvmet_mdev_queue_getpages_contiguous(viommu, iova, pages, npages))
+		goto out;
+
+	map = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
+out:
+	kfree(pages);
+	return map;
+}
+
+void nvmet_mdev_udata_queue_vunmap(struct nvmet_mdev_viommu *viommu,
+				   dma_addr_t iova, void *data,
+				   unsigned int data_size)
+{
+	if (!data)
+		return;
+
+	vunmap(data);
+	nvmet_mdev_viommu_remove(viommu, iova,
+				 DIV_ROUND_UP(data_size, PAGE_SIZE));
+}
+
+void *nvmet_mdev_udata_update_queue_vmap(struct nvmet_mdev_viommu *viommu,
+					 dma_addr_t iova, void *data,
+					 unsigned int data_size)
+{
+	if (!iova)
+		return NULL;
+
+	if (data)
+		vunmap(data);
+
+	return nvmet_mdev_udata_queue_vmap(viommu, iova, data_size);
+}
diff --git a/drivers/nvme/target/mdev-pci/vcq.c b/drivers/nvme/target/mdev-pci/vcq.c
new file mode 100644
index 000000000000..22ac71a67a4c
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/vcq.c
@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Virtual NVMe completion queue implementation
+ * Copyright (c) 2019 - Maxim Levitsky
+ * Copyright (C) 2025 Oracle Corporation
+ */
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include "priv.h"
+
+static int nvmet_mdev_calc_vcq_size(struct nvmet_mdev_vcq *vcq)
+{
+	return round_up(vcq->size * sizeof(struct nvme_completion), PAGE_SIZE);
+}
+
+/* Create new virtual completion queue */
+int nvmet_mdev_vcq_init(struct nvmet_mdev_vctrl *vctrl, u16 qid,
+			dma_addr_t iova, u16 size, int irq)
+{
+	struct nvmet_ctrl *ctrl = vctrl->nvmet_ctrl;
+	struct nvmet_mdev_vcq *q = &vctrl->vcqs[qid];
+
+	lockdep_assert_held(&vctrl->lock);
+
+	q->vctrl = vctrl;
+	q->qid = qid;
+	q->size = size;
+	q->tail = 0;
+	q->phase = 1;
+	q->irq = irq;
+	q->head = 0;
+	q->iova = iova;
+	q->data = NULL;
+	q->data_size = nvmet_mdev_calc_vcq_size(q);
+	init_llist_head(&q->mreq_list);
+
+	q->data = nvmet_mdev_udata_update_queue_vmap(&vctrl->viommu, q->iova,
+						     (void *)q->data,
+						     q->data_size);
+	if (!q->data)
+		goto delete;
+
+	_DBG(vctrl, "VCQ: create qid=%d depth=%d irq=%d\n", qid, size, irq);
+
+	vctrl->mmio.dbs[q->qid].cqh = 0;
+	vctrl->mmio.eidxs[q->qid].cqh = 0;
+
+	if (nvmet_cq_create(ctrl, &q->nvmet_cq, qid, size))
+		goto delete;
+
+	set_bit(qid, vctrl->vcq_en);
+	return NVME_SC_SUCCESS;
+
+delete:
+	nvmet_mdev_vcq_delete(vctrl, qid);
+	return NVME_SC_INTERNAL;
+}
+
+/* Delete a virtual completion queue */
+void nvmet_mdev_vcq_delete(struct nvmet_mdev_vctrl *vctrl, u16 qid)
+{
+	struct nvmet_mdev_vcq *q = &vctrl->vcqs[qid];
+
+	lockdep_assert_held(&vctrl->lock);
+
+	nvmet_mdev_udata_queue_vunmap(&vctrl->viommu, q->iova, (void *)q->data,
+				      q->data_size);
+	q->data = NULL;
+	q->iova = 0;
+	clear_bit(qid, vctrl->vcq_en);
+
+	_DBG(vctrl, "VCQ: delete qid=%d\n", q->qid);
+}
+
+/* Move queue tail one item forward */
+static void nvmet_mdev_vcq_advance_tail(struct nvmet_mdev_vcq *q)
+{
+	if (++q->tail == q->size) {
+		q->tail = 0;
+		q->phase ^= 1;
+	}
+}
+
+/* Move queue head one item forward */
+static void nvmet_mdev_vcq_advance_head(struct nvmet_mdev_vcq *q)
+{
+	q->head++;
+	if (q->head == q->size)
+		q->head = 0;
+}
+
+/* Process a virtual completion queue */
+void nvmet_mdev_vcq_process(struct nvmet_mdev_vctrl *vctrl,
+			    struct nvmet_mdev_vcq *q, bool trigger_irqs,
+			    ktime_t now)
+{
+	u16 new_head;
+	u32 eidx;
+
+	if (!vctrl->mmio.dbs || !vctrl->mmio.eidxs)
+		return;
+
+	new_head = le32_to_cpu(READ_ONCE(vctrl->mmio.dbs[q->qid].cqh));
+
+	if (new_head != q->head) {
+		/* bad tail - can't process */
+		if (!nvmet_mdev_mmio_db_check(vctrl, q->qid, q->size, new_head))
+			return;
+
+		while (q->head != new_head)
+			nvmet_mdev_vcq_advance_head(q);
+
+		eidx = q->head + (q->size >> 1);
+		if (eidx >= q->size)
+			eidx -= q->size;
+		WRITE_ONCE(vctrl->mmio.eidxs[q->qid].cqh, cpu_to_le32(eidx));
+	}
+
+	if (q->irq != -1 && trigger_irqs) {
+		if (q->tail != new_head)
+			nvmet_mdev_irq_cond_trigger(vctrl, q->irq, now);
+		else
+			nvmet_mdev_irq_clear(vctrl, q->irq, now);
+	}
+}
+
+/* flush interrupts on a completion queue */
+bool nvmet_mdev_vcq_flush(struct nvmet_mdev_vctrl *vctrl,
+			  struct nvmet_mdev_vcq *q, ktime_t now)
+{
+	u16 new_head = le32_to_cpu(READ_ONCE(vctrl->mmio.dbs[q->qid].cqh));
+
+	if (new_head == q->tail || q->irq == -1)
+		return false;
+
+	nvmet_mdev_irq_trigger(vctrl, q->irq);
+	nvmet_mdev_irq_clear(vctrl, q->irq, now);
+	return true;
+}
+
+void nvmet_mdev_vcq_write_cqe(struct nvmet_mdev_vctrl *vctrl,
+			      struct nvmet_mdev_vcq *q,
+			      struct nvme_completion *cqe)
+{
+	volatile u64 *qw = (u64 *)&q->data[q->tail];
+	u64 *data = (u64 *)cqe;
+
+	cqe->status = cqe->status | cpu_to_le16(q->phase);
+
+	WRITE_ONCE(qw[0], data[0]);
+	/* ensure that hardware sees the phase bit flip last */
+	wmb();
+	WRITE_ONCE(qw[1], data[1]);
+
+	nvmet_mdev_vcq_advance_tail(q);
+	if (q->irq != -1)
+		nvmet_mdev_irq_raise(vctrl, q->irq);
+}
diff --git a/drivers/nvme/target/mdev-pci/vctrl.c b/drivers/nvme/target/mdev-pci/vctrl.c
new file mode 100644
index 000000000000..0568bec52f56
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/vctrl.c
@@ -0,0 +1,260 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Virtual NVMe controller implementation
+ * Copyright (c) 2019 - Maxim Levitsky
+ */
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/mdev.h>
+#include <linux/nvme.h>
+#include "priv.h"
+
+bool nvmet_mdev_vctrl_is_dead(struct nvmet_mdev_vctrl *vctrl)
+{
+	return (vctrl->mmio.csts & (NVME_CSTS_CFS | NVME_CSTS_SHST_MASK)) != 0;
+}
+
+/* Create a new virtual controller */
+int nvmet_mdev_vctrl_create(struct nvmet_mdev_vctrl *vctrl,
+			   struct mdev_device *mdev)
+{
+	int ret;
+
+	/* Basic init */
+	vctrl->mdev = mdev;
+	vctrl->viommu.vctrl = vctrl;
+
+	kref_init(&vctrl->ref);
+	mutex_init(&vctrl->lock);
+
+	/* default feature values */
+	vctrl->arb_burst_shift = 3;
+	vctrl->mmio.shadow_db_supported = vctrl->nvmet_ctrl->shadow_db;
+
+	ret = nvmet_mdev_pci_create(vctrl);
+	if (ret)
+		return ret;
+
+	ret = nvmet_mdev_mmio_create(vctrl);
+	if (ret)
+		goto free_pci;
+
+	nvmet_mdev_irqs_setup(vctrl);
+
+	ret = nvmet_mdev_io_create(vctrl);
+	if (ret)
+		goto free_mmio;
+
+	_INFO(vctrl, "device created\n");
+	return 0;
+
+free_mmio:
+	nvmet_mdev_mmio_free(vctrl);
+free_pci:
+	nvmet_mdev_pci_free(vctrl);
+	return ret;
+}
+
+/* Try to destroy an vctrl */
+void nvmet_mdev_vctrl_destroy(struct nvmet_mdev_vctrl *vctrl)
+{
+	mutex_unlock(&vctrl->lock);
+
+	mutex_lock(&vctrl->lock); /* only for lockdep checks */
+	nvmet_mdev_io_free(vctrl);
+	__nvmet_mdev_vctrl_reset(vctrl, true);
+
+	nvmet_mdev_pci_free(vctrl);
+	nvmet_mdev_mmio_free(vctrl);
+
+	mutex_unlock(&vctrl->lock);
+
+	_INFO(vctrl, "device is destroyed\n");
+}
+
+/* Called when new mediated device is first opened by a user */
+int nvmet_mdev_vctrl_open(struct vfio_device *vfio_dev)
+{
+	struct nvmet_mdev_vctrl *vctrl = vfio_dev_to_nvmet_mdev_vctrl(vfio_dev);
+
+	_INFO(vctrl, "device is opened\n");
+
+	mutex_lock(&vctrl->lock);
+	nvmet_mdev_viommu_init(&vctrl->viommu, vfio_dev);
+	nvmet_mdev_mmio_open(vctrl);
+	mutex_unlock(&vctrl->lock);
+	return 0;
+}
+
+/* Called when new mediated device is closed (last close of the user) */
+void nvmet_mdev_vctrl_release(struct vfio_device *vfio_dev)
+{
+	struct nvmet_mdev_vctrl *vctrl = vfio_dev_to_nvmet_mdev_vctrl(vfio_dev);
+
+	mutex_lock(&vctrl->lock);
+	nvmet_mdev_io_pause(vctrl);
+	/*
+	 * Remove the guest DMA mappings - new user that will open the
+	 * device might be a different guest
+	 */
+	nvmet_mdev_viommu_reset(&vctrl->viommu);
+
+	/* Reset the controller to a clean state for a new user */
+	__nvmet_mdev_vctrl_reset(vctrl, false);
+	nvmet_mdev_irqs_reset(vctrl);
+	mutex_unlock(&vctrl->lock);
+
+	_INFO(vctrl, "device is released\n");
+}
+
+/* Called each time the controller is reset (CC.EN <= 0 or VM level reset) */
+void __nvmet_mdev_vctrl_reset(struct nvmet_mdev_vctrl *vctrl, bool pci_reset)
+{
+	lockdep_assert_held(&vctrl->lock);
+
+	if ((vctrl->mmio.csts & NVME_CSTS_RDY) &&
+	    !(vctrl->mmio.csts & NVME_CSTS_SHST_MASK)) {
+		_DBG(vctrl, "unsafe reset (CSTS.RDY==1)\n");
+		nvmet_mdev_io_pause(vctrl);
+		nvmet_mdev_vctrl_disable(vctrl);
+	}
+	nvmet_mdev_mmio_reset(vctrl, pci_reset);
+}
+
+/* setups initial admin queues and doorbells */
+bool nvmet_mdev_vctrl_enable(struct nvmet_mdev_vctrl *vctrl, dma_addr_t cqiova,
+			     dma_addr_t sqiova, u32 sizes)
+{
+	struct nvmet_ctrl *ctrl = vctrl->nvmet_ctrl;
+	u16 cqentries, sqentries;
+	int ret;
+
+	nvmet_mdev_assert_io_not_running(vctrl);
+
+	lockdep_assert_held(&vctrl->lock);
+
+	sqentries = (sizes & 0xFFFF) + 1;
+	cqentries = (sizes >> 16) + 1;
+
+	if (cqentries > 4096 || cqentries < 2)
+		return false;
+	if (sqentries > 4096 || sqentries < 2)
+		return false;
+
+	ret = nvmet_mdev_mmio_enable_dbs(vctrl);
+	if (ret)
+		return false;
+
+	ret = nvmet_mdev_vcq_init(vctrl, 0, cqiova, cqentries, 0);
+	if (ret)
+		goto disable_dbs;
+
+	ret = nvmet_mdev_vsq_init(vctrl, 0, sqiova, sqentries, 0);
+	if (ret)
+		goto delete_vcq;
+
+	nvmet_update_cc(ctrl, vctrl->mmio.cc);
+	if (ctrl->csts != NVME_CSTS_RDY)
+		goto delete_vsq;
+
+	if (!vctrl->mmio.shadow_db_supported) {
+		/* start polling right away to support admin queue */
+		vctrl->io_idle = false;
+		nvmet_mdev_io_resume(vctrl);
+	}
+
+	return true;
+
+delete_vsq:
+	nvmet_mdev_vsq_delete(vctrl, 0);
+delete_vcq:
+	nvmet_mdev_vcq_delete(vctrl, 0);
+disable_dbs:
+	nvmet_mdev_mmio_disable_dbs(vctrl);
+	return false;
+}
+
+/* destroy all io/admin queues on the controller */
+void nvmet_mdev_vctrl_disable(struct nvmet_mdev_vctrl *vctrl)
+{
+	u16 sqid, cqid;
+
+	nvmet_mdev_assert_io_not_running(vctrl);
+
+	lockdep_assert_held(&vctrl->lock);
+
+	sqid = 1;
+	for_each_set_bit_from(sqid, vctrl->vsq_en, NVMET_MDEV_MAX_NR_QUEUES)
+		nvmet_mdev_vsq_delete(vctrl, sqid);
+
+	cqid = 1;
+	for_each_set_bit_from(cqid, vctrl->vcq_en, NVMET_MDEV_MAX_NR_QUEUES)
+		nvmet_mdev_vcq_delete(vctrl, cqid);
+
+	nvmet_mdev_vsq_delete(vctrl, 0);
+	nvmet_mdev_vcq_delete(vctrl, 0);
+
+	nvmet_mdev_mmio_disable_dbs(vctrl);
+	vctrl->io_idle = true;
+}
+
+/* External reset */
+void nvmet_mdev_vctrl_reset(struct nvmet_mdev_vctrl *vctrl)
+{
+	mutex_lock(&vctrl->lock);
+	_INFO(vctrl, "reset\n");
+	__nvmet_mdev_vctrl_reset(vctrl, true);
+	mutex_unlock(&vctrl->lock);
+}
+
+/* Add IO region */
+void nvmet_mdev_vctrl_add_region(struct nvmet_mdev_vctrl *vctrl,
+				 unsigned int index, unsigned int size,
+				 region_access_fn access_fn)
+{
+	struct nvmet_mdev_io_region *region = &vctrl->regions[index];
+
+	region->size = size;
+	region->rw = access_fn;
+	region->mmap_ops = NULL;
+}
+
+/* Enable mmap window on an IO region */
+void nvmet_mdev_vctrl_region_set_mmap(struct nvmet_mdev_vctrl *vctrl,
+				      unsigned int index,
+				      unsigned int offset,
+				      unsigned int size,
+				      const struct vm_operations_struct *ops)
+{
+	struct nvmet_mdev_io_region *region = &vctrl->regions[index];
+
+	region->mmap_area_start = offset;
+	region->mmap_area_size = size;
+	region->mmap_ops = ops;
+}
+
+/* Disable mmap window on an IO region */
+void nvmet_mdev_vctrl_region_disable_mmap(struct nvmet_mdev_vctrl *vctrl,
+					  unsigned int index)
+{
+	struct nvmet_mdev_io_region *region = &vctrl->regions[index];
+
+	region->mmap_area_start = 0;
+	region->mmap_area_size = 0;
+	region->mmap_ops = NULL;
+}
+
+/* remove a user memory mapping */
+int nvmet_mdev_vctrl_viommu_unmap(struct nvmet_mdev_vctrl *vctrl,
+				  dma_addr_t iova, u64 size)
+{
+	bool paused;
+	int ret;
+
+	paused = nvmet_mdev_io_pause(vctrl);
+	ret = nvmet_mdev_viommu_remove(&vctrl->viommu, iova, size);
+	if (paused)
+		nvmet_mdev_io_resume(vctrl);
+	return ret;
+}
diff --git a/drivers/nvme/target/mdev-pci/viommu.c b/drivers/nvme/target/mdev-pci/viommu.c
new file mode 100644
index 000000000000..9ef4281a87da
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/viommu.c
@@ -0,0 +1,308 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Virtual IOMMU - mapping user memory to the real device
+ * Copyright (c) 2019 - Maxim Levitsky
+ */
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/highmem.h>
+#include <linux/slab.h>
+#include <linux/mdev.h>
+#include <linux/vmalloc.h>
+#include <linux/nvme.h>
+#include <linux/iommu.h>
+#include <linux/interval_tree_generic.h>
+#include "priv.h"
+
+struct mem_mapping {
+	struct rb_node rb;
+	struct list_head link;
+
+	dma_addr_t __subtree_last;
+	dma_addr_t iova_start; /* first iova in this mapping */
+	dma_addr_t iova_last;  /* last iova in this mapping */
+
+	unsigned long pfn;  /* physical address of this mapping */
+	int refcount;
+};
+
+#define map_len(m) (((m)->iova_last - (m)->iova_start) + 1ULL)
+#define map_pages(m) (map_len(m) >> PAGE_SHIFT)
+#define START(node) ((node)->iova_start)
+#define LAST(node) ((node)->iova_last)
+
+INTERVAL_TREE_DEFINE(struct mem_mapping, rb, dma_addr_t, __subtree_last,
+		     START, LAST, static inline, viommu_int_tree);
+
+static void nvmet_mdev_viommu_dbg_dma_range(struct nvmet_mdev_viommu *viommu,
+					    struct mem_mapping *map,
+					    const char *action)
+{
+	dma_addr_t iova_start  = map->iova_start;
+	dma_addr_t iova_end    = map->iova_start + map_len(map) - 1;
+
+	_DBG(viommu->vctrl, "vIOMMU: %s RW IOVA %pad-%pad",
+	     action, &iova_start, &iova_end);
+}
+
+/* unpin N pages starting at given IOVA */
+static void nvmet_mdev_viommu_unpin_pages(struct nvmet_mdev_viommu *viommu,
+					  dma_addr_t iova, int n)
+{
+	int i, npages;
+
+	for (i = 0; i < n; i += npages) {
+		npages = min_t(int, VFIO_PIN_PAGES_MAX_ENTRIES, n);
+
+		vfio_unpin_pages(viommu->vfio_dev, iova + (i * PAGE_SIZE),
+				 npages);
+	}
+}
+
+/* User memory init code */
+void nvmet_mdev_viommu_init(struct nvmet_mdev_viommu *viommu,
+			    struct vfio_device *vfio_dev)
+{
+	viommu->vfio_dev = vfio_dev;
+	viommu->maps_tree = RB_ROOT_CACHED;
+	INIT_LIST_HEAD(&viommu->mem_map_list);
+}
+
+int nvmet_mdev_viommu_remove_list(struct nvmet_mdev_viommu *viommu,
+				  struct list_head *remove_list)
+{
+	struct mem_mapping *map, *tmp;
+	int count = 0;
+
+	list_for_each_entry_safe(map, tmp, remove_list, link) {
+		list_del(&map->link);
+
+		nvmet_mdev_viommu_unpin_pages(viommu, map->iova_start,
+					      map_pages(map));
+		viommu_int_tree_remove(map, &viommu->maps_tree);
+		kfree(map);
+		count++;
+	}
+	return count;
+}
+
+/* User memory end code */
+void nvmet_mdev_viommu_reset(struct nvmet_mdev_viommu *viommu)
+{
+	nvmet_mdev_viommu_remove_list(viommu, &viommu->mem_map_list);
+	WARN_ON(!list_empty(&viommu->mem_map_list));
+}
+
+/* Adds a new range of user memory */
+int nvmet_mdev_viommu_add(struct nvmet_mdev_viommu *viommu, u32 flags,
+			  dma_addr_t iova, u64 size,
+			  struct list_head *mem_map_list)
+{
+	struct vfio_device *vfio_dev = viommu->vfio_dev;
+	struct nvmet_mdev_vctrl *vctrl = vfio_dev_to_nvmet_mdev_vctrl(vfio_dev);
+	dma_addr_t iova_curr, iova_end, iova_start;
+	struct mem_mapping *map = NULL, *tmp;
+	LIST_HEAD(new_mappings_list);
+	int ret, count = 0;
+
+	/*
+	 * iova/size may not be page aligned if this is for IO. We align
+	 * them here because the viommu requires it.
+	 */
+	iova_start = iova;
+	if (!PAGE_ALIGNED(iova_start)) {
+		iova_start = PAGE_ALIGN_DOWN(iova_start);
+		_DBG(vctrl, "vIOMMU: realign iova %pad -> %pad\n",
+		     &iova, &iova_start);
+	}
+
+	iova_end = iova + size;
+	if (!PAGE_ALIGNED(iova_end)) {
+		iova_end = PAGE_ALIGN(iova_end);
+		_DBG(vctrl,
+		     "vIOMMU: realign size %llu -> %llu\n",
+		     size, PAGE_ALIGN(size));
+	}
+
+	if (!(flags & VFIO_DMA_MAP_FLAG_READ) ||
+	    !(flags & VFIO_DMA_MAP_FLAG_WRITE)) {
+		const char *type = "none";
+
+		if (flags & VFIO_DMA_MAP_FLAG_READ)
+			type = "RO";
+		else if (flags & VFIO_DMA_MAP_FLAG_WRITE)
+			type = "WO";
+
+		_DBG(viommu->vctrl, "vIOMMU: IGN %s IOVA %pad-%pad\n",
+		     type, &iova_start, &iova_end);
+		return 0;
+	}
+
+	/* VFIO pinning all the pages */
+	for (iova_curr = iova_start; iova_curr < iova_end;
+	     iova_curr += PAGE_SIZE) {
+		struct page *page;
+
+		ret = vfio_pin_pages(viommu->vfio_dev, iova_curr, 1,
+				     VFIO_DMA_MAP_FLAG_READ |
+				     VFIO_DMA_MAP_FLAG_WRITE,
+				     &page);
+		if (ret != 1) {
+			_DBG(viommu->vctrl,
+			     "vIOMMU: ADD RW IOVA %pad len: %llu - pin failed %d\n",
+			     &iova, size, ret);
+			goto unwind;
+		}
+
+		/* new mapping needed */
+		if (!map || map->pfn + map_pages(map) != page_to_pfn(page)) {
+			map = kzalloc(sizeof(*map), GFP_KERNEL);
+			if (!map) {
+				vfio_unpin_pages(viommu->vfio_dev, iova_curr,
+						 1);
+				ret = -ENOMEM;
+				goto unwind;
+			}
+			map->iova_start = iova_curr;
+			map->iova_last = iova_curr + PAGE_SIZE - 1ULL;
+			map->pfn = page_to_pfn(page);
+			map->refcount = 1;
+			INIT_LIST_HEAD(&map->link);
+			list_add_tail(&map->link, &new_mappings_list);
+		} else {
+			/* current map can be extended */
+			map->iova_last += PAGE_SIZE;
+		}
+	}
+
+	/* DMA mapping the pages */
+	list_for_each_entry_safe(map, tmp, &new_mappings_list, link) {
+		nvmet_mdev_viommu_dbg_dma_range(viommu, map, "ADD");
+		list_move_tail(&map->link, mem_map_list);
+		viommu_int_tree_insert(map, &viommu->maps_tree);
+		count++;
+	}
+
+	_DBG(viommu->vctrl, "vIOMMU: ADD RW IOVA %pad-%pad len %llu count %d\n",
+	     &iova_start, &iova_end, size, count);
+
+	return 0;
+unwind:
+	list_for_each_entry_safe(map, tmp, &new_mappings_list, link) {
+		nvmet_mdev_viommu_unpin_pages(viommu, map->iova_start,
+					      map_pages(map));
+
+		list_del(&map->link);
+		kfree(map);
+	}
+	return ret;
+}
+
+/* Removes a range of user memory */
+int nvmet_mdev_viommu_remove(struct nvmet_mdev_viommu *viommu, dma_addr_t iova,
+			     u64 size)
+{
+	dma_addr_t last_iova = iova + (size) - 1ULL;
+	struct mem_mapping *map;
+	LIST_HEAD(remove_list);
+	int count = 0;
+
+	/* find out all the relevant ranges */
+	map = viommu_int_tree_iter_first(&viommu->maps_tree, iova, last_iova);
+	while (map) {
+		list_move_tail(&map->link, &remove_list);
+		map = viommu_int_tree_iter_next(map, iova, last_iova);
+	}
+
+	/* remove them */
+	count = nvmet_mdev_viommu_remove_list(viommu, &remove_list);
+	return count;
+}
+
+/* Translate an IOVA to a physical address and read device bus address */
+int nvmet_mdev_viommu_translate(struct nvmet_mdev_viommu *viommu,
+				dma_addr_t iova, dma_addr_t *physical)
+{
+	struct mem_mapping *mapping;
+	u64 offset;
+
+	if (WARN_ON_ONCE(offset_in_page(iova) != 0))
+		return -EINVAL;
+
+	mapping = viommu_int_tree_iter_first(&viommu->maps_tree,
+					     iova, iova + PAGE_SIZE - 1);
+	if (!mapping) {
+		_DBG(viommu->vctrl,
+		     "vIOMMU: translation of IOVA %pad failed\n", &iova);
+		return -EFAULT;
+	}
+
+	WARN_ON(iova > mapping->iova_last);
+	WARN_ON(offset_in_page(mapping->iova_start) != 0);
+
+	offset = iova - mapping->iova_start;
+	*physical = PFN_PHYS(mapping->pfn) + offset;
+	return 0;
+}
+
+/* map an IOVA to kernel address space  */
+int nvmet_mdev_viommu_create_kmap(struct nvmet_mdev_viommu *viommu,
+				  dma_addr_t iova, struct page_map *page)
+{
+	phys_addr_t physical;
+	struct page *new_page;
+	int ret;
+
+	page->iova = iova;
+
+	ret = nvmet_mdev_viommu_translate(viommu, iova, &physical);
+	if (ret)
+		return ret;
+
+	new_page = pfn_to_page(PHYS_PFN(physical));
+
+	page->kmap = kmap_local_page(new_page);
+	if (!page->kmap)
+		return -ENOMEM;
+
+	page->page = new_page;
+	return 0;
+}
+
+/* update IOVA <-> kernel mapping. If fails, removes the previous mapping */
+void nvmet_mdev_viommu_update_kmap(struct nvmet_mdev_viommu *viommu,
+				   struct page_map *page)
+{
+	phys_addr_t physical;
+	struct page *new_page;
+	int ret;
+
+	ret = nvmet_mdev_viommu_translate(viommu, page->iova, &physical);
+	if (ret) {
+		nvmet_mdev_viommu_free_kmap(viommu, page);
+		return;
+	}
+
+	new_page = pfn_to_page(PHYS_PFN(physical));
+	if (new_page == page->page)
+		return;
+
+	nvmet_mdev_viommu_free_kmap(viommu, page);
+
+	page->kmap = kmap_local_page(new_page);
+	if (!page->kmap)
+		return;
+	page->page = new_page;
+}
+
+/* unmap an IOVA to kernel address space  */
+void nvmet_mdev_viommu_free_kmap(struct nvmet_mdev_viommu *viommu,
+				 struct page_map *page)
+{
+	if (page->page) {
+		kunmap_local(page->kmap);
+		page->page = NULL;
+		page->kmap = NULL;
+	}
+}
diff --git a/drivers/nvme/target/mdev-pci/vsq.c b/drivers/nvme/target/mdev-pci/vsq.c
new file mode 100644
index 000000000000..69e82b29cc4a
--- /dev/null
+++ b/drivers/nvme/target/mdev-pci/vsq.c
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Virtual NVMe submission queue implementation
+ * Copyright (c) 2019 - Maxim Levitsky
+ * Copyright (C) 2025 Oracle Corporation
+ */
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include "priv.h"
+
+static int nvmet_mdev_calc_vsq_size(struct nvmet_mdev_vsq *vsq)
+{
+	return round_up(vsq->size * sizeof(struct nvme_command), PAGE_SIZE);
+}
+
+/* Delete an virtual completion queue */
+void nvmet_mdev_vsq_delete(struct nvmet_mdev_vctrl *vctrl, u16 qid)
+{
+	struct nvmet_mdev_vsq *q = &vctrl->vsqs[qid];
+	bool paused;
+
+	lockdep_assert_held(&vctrl->lock);
+	_DBG(vctrl, "VSQ: delete qid=%d\n", q->qid);
+
+	/*
+	 * If this is an unclean shutdown make sure we don't try to start
+	 * new IOs after we free the queue.
+	 */
+	paused = nvmet_mdev_io_pause(vctrl);
+	clear_bit(qid, vctrl->vsq_en);
+	if (paused)
+		nvmet_mdev_io_resume(vctrl);
+
+	if (q->nvmet_sq.ctrl)
+		nvmet_sq_destroy(&q->nvmet_sq);
+
+	/*
+	 * The nvmet sq destruction just waits for the queue_response callout
+	 * to return, so we could still have completions queued. Flush
+	 * them now since we are freeing the queue below.
+	 */
+	if (test_and_clear_bit(qid, vctrl->vcq_en)) {
+		paused = nvmet_mdev_io_pause(vctrl);
+		nvmet_mdev_process_responses(vctrl, q->vcq);
+		if (paused)
+			nvmet_mdev_io_resume(vctrl);
+	}
+
+	kfree(q->reqs);
+
+	nvmet_mdev_udata_queue_vunmap(&vctrl->viommu, q->iova, q->data,
+				      q->data_size);
+	q->data = NULL;
+	q->iova = 0;
+}
+
+/* Create new virtual completion queue */
+int nvmet_mdev_vsq_init(struct nvmet_mdev_vctrl *vctrl, u16 qid,
+			dma_addr_t iova, u16 size, u16 cqid)
+{
+	struct nvmet_ctrl *ctrl = vctrl->nvmet_ctrl;
+	struct nvmet_mdev_vsq *q = &vctrl->vsqs[qid];
+
+	lockdep_assert_held(&vctrl->lock);
+
+	q->vctrl = vctrl;
+	q->qid = qid;
+	q->size = size;
+	q->head = 0;
+	q->vcq = &vctrl->vcqs[cqid];
+	q->data = NULL;
+	q->iova = iova;
+	q->data_size = nvmet_mdev_calc_vsq_size(q);
+
+	_DBG(vctrl, "VSQ: create qid=%d depth=%d cqid=%d\n", qid, size, cqid);
+
+	q->data = nvmet_mdev_udata_update_queue_vmap(&vctrl->viommu, q->iova,
+						     q->data, q->data_size);
+	if (!q->data)
+		goto delete;
+
+	q->reqs = kcalloc(size, sizeof(*q->reqs), GFP_KERNEL);
+	if (!q->reqs)
+		goto delete;
+
+	vctrl->mmio.dbs[q->qid].sqt = 0;
+	vctrl->mmio.eidxs[q->qid].sqt = 0;
+
+	if (nvmet_sq_create(ctrl, &q->nvmet_sq, qid, size))
+		goto delete;
+
+	set_bit(qid, vctrl->vsq_en);
+	return NVME_SC_SUCCESS;
+
+delete:
+	nvmet_mdev_vsq_delete(vctrl, qid);
+	return NVME_SC_INTERNAL;
+}
+
+/* Move queue head one item forward */
+static void nvmet_mdev_vsq_advance_head(struct nvmet_mdev_vsq *q)
+{
+	q->head++;
+	if (q->head == q->size)
+		q->head = 0;
+}
+
+static bool nvmet_mdev_vsq_has_data(struct nvmet_mdev_vctrl *vctrl,
+				    struct nvmet_mdev_vsq *q)
+{
+	u16 tail = le32_to_cpu(READ_ONCE(vctrl->mmio.dbs[q->qid].sqt));
+
+	if (!vctrl->mmio.dbs || !vctrl->mmio.eidxs || !q->data)
+		return false;
+
+	if  (tail == q->head)
+		return false;
+
+	if (!nvmet_mdev_mmio_db_check(vctrl, q->qid, q->size, tail))
+		return false;
+	return true;
+}
+
+/* get one command from a virtual submission queue */
+struct nvme_command *nvmet_mdev_vsq_get_cmd(struct nvmet_mdev_vctrl *vctrl,
+					    struct nvmet_mdev_vsq *q,
+					    u16 *index)
+{
+	u16 oldhead = q->head;
+	u32 eidx;
+
+	if (!nvmet_mdev_vsq_has_data(vctrl, q))
+		return NULL;
+
+	nvmet_mdev_vsq_advance_head(q);
+
+	eidx = q->head + (q->size >> 1);
+	if (eidx >= q->size)
+		eidx -= q->size;
+
+	WRITE_ONCE(vctrl->mmio.eidxs[q->qid].sqt, cpu_to_le32(eidx));
+
+	*index = oldhead;
+	return &q->data[oldhead];
+}
+
+bool nvmet_mdev_vsq_suspend_io(struct nvmet_mdev_vsq *q)
+{
+	struct nvmet_mdev_vctrl *vctrl = q->vctrl;
+	u16 tail = le32_to_cpu(vctrl->mmio.dbs[q->qid].sqt);
+
+	/*
+	 * If the queue is not in working state don't allow the idle code
+	 * to kick in
+	 */
+	if (!vctrl->mmio.dbs || !vctrl->mmio.eidxs || !q->data)
+		return false;
+
+	/* queue has data - refuse idle */
+	if (tail != q->head)
+		return false;
+
+	/* Write eventid to tell the user to ring normal doorbell */
+	vctrl->mmio.eidxs[q->qid].sqt = cpu_to_le32(q->head);
+
+	/* memory barrier to ensure that the user have seen the eidx */
+	mb();
+
+	/* Check that doorbell diddn't move meanwhile */
+	tail = le32_to_cpu(vctrl->mmio.dbs[q->qid].sqt);
+	return (tail == q->head);
+}
