diff mbox series

[Kernel,v19,7/8] vfio iommu: Add migration capability to report supported features

Message ID 1589400279-28522-8-git-send-email-kwankhede@nvidia.com (mailing list archive)
State New, archived
Headers show
Series Add UAPIs to support migration for VFIO devices | expand

Commit Message

Kirti Wankhede May 13, 2020, 8:04 p.m. UTC
Added migration capability in IOMMU info chain.
User application should check IOMMU info chain for migration capability
to use dirty page tracking feature provided by kernel module.
User application must check page sizes supported and maximum dirty
bitmap size returned by this capability structure for ioctls used to get
dirty bitmap.

Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
---
 drivers/vfio/vfio_iommu_type1.c | 24 +++++++++++++++++++++++-
 include/uapi/linux/vfio.h       | 21 +++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

Comments

Alex Williamson May 14, 2020, 5:01 a.m. UTC | #1
On Thu, 14 May 2020 01:34:38 +0530
Kirti Wankhede <kwankhede@nvidia.com> wrote:

> Added migration capability in IOMMU info chain.
> User application should check IOMMU info chain for migration capability
> to use dirty page tracking feature provided by kernel module.
> User application must check page sizes supported and maximum dirty
> bitmap size returned by this capability structure for ioctls used to get
> dirty bitmap.
> 
> Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
> ---
>  drivers/vfio/vfio_iommu_type1.c | 24 +++++++++++++++++++++++-
>  include/uapi/linux/vfio.h       | 21 +++++++++++++++++++++
>  2 files changed, 44 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
> index 4358be26ff80..77351497a9c2 100644
> --- a/drivers/vfio/vfio_iommu_type1.c
> +++ b/drivers/vfio/vfio_iommu_type1.c
> @@ -2389,6 +2389,22 @@ static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,
>  	return ret;
>  }
>  
> +static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,
> +					   struct vfio_info_cap *caps)
> +{
> +	struct vfio_iommu_type1_info_cap_migration cap_mig;
> +
> +	cap_mig.header.id = VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION;
> +	cap_mig.header.version = 1;
> +	cap_mig.flags = VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK;
> +
> +	/* support minimum pgsize */
> +	cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap);
> +	cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX;
> +
> +	return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig));
> +}
> +
>  static long vfio_iommu_type1_ioctl(void *iommu_data,
>  				   unsigned int cmd, unsigned long arg)
>  {
> @@ -2433,10 +2449,16 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
>  		mutex_lock(&iommu->lock);
>  		info.flags = VFIO_IOMMU_INFO_PGSIZES;
>  
> +		vfio_pgsize_bitmap(iommu);


Why is it necessary to rebuild the bitmap here?  The user can't get to
this ioctl until they've added a group to the container and set the
IOMMU model.


>  		info.iova_pgsizes = iommu->pgsize_bitmap;
>  
> -		ret = vfio_iommu_iova_build_caps(iommu, &caps);
> +		ret = vfio_iommu_migration_build_caps(iommu, &caps);
> +
> +		if (!ret)
> +			ret = vfio_iommu_iova_build_caps(iommu, &caps);
> +
>  		mutex_unlock(&iommu->lock);
> +
>  		if (ret)
>  			return ret;
>  
> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> index e3cbf8b78623..c90604322798 100644
> --- a/include/uapi/linux/vfio.h
> +++ b/include/uapi/linux/vfio.h
> @@ -1013,6 +1013,27 @@ struct vfio_iommu_type1_info_cap_iova_range {
>  	struct	vfio_iova_range iova_ranges[];
>  };
>  
> +/*
> + * The migration capability allows to report supported features for migration.
> + *
> + * The structures below define version 1 of this capability.
> + *
> + * pgsize_bitmap: Kernel driver returns supported page sizes bitmap for dirty
> + * page tracking.
> + * max_dirty_bitmap_size: Kernel driver returns maximum supported dirty bitmap
> + * size in bytes to be used by user application for ioctls to get dirty bitmap.
> + */
> +#define VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION  1
> +
> +struct vfio_iommu_type1_info_cap_migration {
> +	struct	vfio_info_cap_header header;
> +	__u32	flags;
> +	/* supports dirty page tracking */
> +#define VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK	(1 << 0)

This flag is a bit redundant to the purpose of this capability, isn't
it?  I think exposing the capability itself is indicating support for
dirty page tracking.  We should probably be explicit in the comment
about exactly what interface this capability implies.  Thanks,

Alex

> +	__u64	pgsize_bitmap;
> +	__u64	max_dirty_bitmap_size;		/* in bytes */
> +};
> +
>  #define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
>  
>  /**
Kirti Wankhede May 14, 2020, 11:55 a.m. UTC | #2
On 5/14/2020 10:31 AM, Alex Williamson wrote:
> On Thu, 14 May 2020 01:34:38 +0530
> Kirti Wankhede <kwankhede@nvidia.com> wrote:
> 
>> Added migration capability in IOMMU info chain.
>> User application should check IOMMU info chain for migration capability
>> to use dirty page tracking feature provided by kernel module.
>> User application must check page sizes supported and maximum dirty
>> bitmap size returned by this capability structure for ioctls used to get
>> dirty bitmap.
>>
>> Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
>> ---
>>   drivers/vfio/vfio_iommu_type1.c | 24 +++++++++++++++++++++++-
>>   include/uapi/linux/vfio.h       | 21 +++++++++++++++++++++
>>   2 files changed, 44 insertions(+), 1 deletion(-)
>>
>> diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
>> index 4358be26ff80..77351497a9c2 100644
>> --- a/drivers/vfio/vfio_iommu_type1.c
>> +++ b/drivers/vfio/vfio_iommu_type1.c
>> @@ -2389,6 +2389,22 @@ static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,
>>   	return ret;
>>   }
>>   
>> +static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,
>> +					   struct vfio_info_cap *caps)
>> +{
>> +	struct vfio_iommu_type1_info_cap_migration cap_mig;
>> +
>> +	cap_mig.header.id = VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION;
>> +	cap_mig.header.version = 1;
>> +	cap_mig.flags = VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK;
>> +
>> +	/* support minimum pgsize */
>> +	cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap);
>> +	cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX;
>> +
>> +	return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig));
>> +}
>> +
>>   static long vfio_iommu_type1_ioctl(void *iommu_data,
>>   				   unsigned int cmd, unsigned long arg)
>>   {
>> @@ -2433,10 +2449,16 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
>>   		mutex_lock(&iommu->lock);
>>   		info.flags = VFIO_IOMMU_INFO_PGSIZES;
>>   
>> +		vfio_pgsize_bitmap(iommu);
> 
> 
> Why is it necessary to rebuild the bitmap here?  The user can't get to
> this ioctl until they've added a group to the container and set the
> IOMMU model.
> 
> 
For mdev device, domain is not added to domain_list so 
vfio_pgsize_bitmap() doesn't get called when there is only mdev device 
attached.
Your concern is right though, vfio_pgsize_bitmap() should get populated 
with attach_group,so fixing it by calling vfio_pgsize_bitmap() for mdev 
device when iommu->external_domain is set.

>>   		info.iova_pgsizes = iommu->pgsize_bitmap;
>>   
>> -		ret = vfio_iommu_iova_build_caps(iommu, &caps);
>> +		ret = vfio_iommu_migration_build_caps(iommu, &caps);
>> +
>> +		if (!ret)
>> +			ret = vfio_iommu_iova_build_caps(iommu, &caps);
>> +
>>   		mutex_unlock(&iommu->lock);
>> +
>>   		if (ret)
>>   			return ret;
>>   
>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
>> index e3cbf8b78623..c90604322798 100644
>> --- a/include/uapi/linux/vfio.h
>> +++ b/include/uapi/linux/vfio.h
>> @@ -1013,6 +1013,27 @@ struct vfio_iommu_type1_info_cap_iova_range {
>>   	struct	vfio_iova_range iova_ranges[];
>>   };
>>   
>> +/*
>> + * The migration capability allows to report supported features for migration.
>> + *
>> + * The structures below define version 1 of this capability.
>> + *
>> + * pgsize_bitmap: Kernel driver returns supported page sizes bitmap for dirty
>> + * page tracking.
>> + * max_dirty_bitmap_size: Kernel driver returns maximum supported dirty bitmap
>> + * size in bytes to be used by user application for ioctls to get dirty bitmap.
>> + */
>> +#define VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION  1
>> +
>> +struct vfio_iommu_type1_info_cap_migration {
>> +	struct	vfio_info_cap_header header;
>> +	__u32	flags;
>> +	/* supports dirty page tracking */
>> +#define VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK	(1 << 0)
> 
> This flag is a bit redundant to the purpose of this capability, isn't
> it?  I think exposing the capability itself is indicating support for
> dirty page tracking.  We should probably be explicit in the comment
> about exactly what interface this capability implies.  Thanks,
>

Capability is added to provide provision for feature flags that kernel 
driver support, that's where we started right?
Later added pgsize_bitmap and max supported bitmap size as you suggested.
I'm confused now, should I keep this flag here?
Even if the flag is removed, 'flags' field is still required so that 
whenever new feature is added, new flag will be added. That's the whole 
purpose we added this capability. Can we add a field which is not used? 
and we don't know when it will be used in future?

Thanks,
Kirti

> Alex
> 
>> +	__u64	pgsize_bitmap;
>> +	__u64	max_dirty_bitmap_size;		/* in bytes */
>> +};
>> +
>>   #define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
>>   
>>   /**
>
Alex Williamson May 14, 2020, 1:39 p.m. UTC | #3
On Thu, 14 May 2020 17:25:10 +0530
Kirti Wankhede <kwankhede@nvidia.com> wrote:

> On 5/14/2020 10:31 AM, Alex Williamson wrote:
> > On Thu, 14 May 2020 01:34:38 +0530
> > Kirti Wankhede <kwankhede@nvidia.com> wrote:
> >   
> >> Added migration capability in IOMMU info chain.
> >> User application should check IOMMU info chain for migration capability
> >> to use dirty page tracking feature provided by kernel module.
> >> User application must check page sizes supported and maximum dirty
> >> bitmap size returned by this capability structure for ioctls used to get
> >> dirty bitmap.
> >>
> >> Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
> >> ---
> >>   drivers/vfio/vfio_iommu_type1.c | 24 +++++++++++++++++++++++-
> >>   include/uapi/linux/vfio.h       | 21 +++++++++++++++++++++
> >>   2 files changed, 44 insertions(+), 1 deletion(-)
> >>
> >> diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
> >> index 4358be26ff80..77351497a9c2 100644
> >> --- a/drivers/vfio/vfio_iommu_type1.c
> >> +++ b/drivers/vfio/vfio_iommu_type1.c
> >> @@ -2389,6 +2389,22 @@ static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,
> >>   	return ret;
> >>   }
> >>   
> >> +static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,
> >> +					   struct vfio_info_cap *caps)
> >> +{
> >> +	struct vfio_iommu_type1_info_cap_migration cap_mig;
> >> +
> >> +	cap_mig.header.id = VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION;
> >> +	cap_mig.header.version = 1;
> >> +	cap_mig.flags = VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK;
> >> +
> >> +	/* support minimum pgsize */
> >> +	cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap);
> >> +	cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX;
> >> +
> >> +	return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig));
> >> +}
> >> +
> >>   static long vfio_iommu_type1_ioctl(void *iommu_data,
> >>   				   unsigned int cmd, unsigned long arg)
> >>   {
> >> @@ -2433,10 +2449,16 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
> >>   		mutex_lock(&iommu->lock);
> >>   		info.flags = VFIO_IOMMU_INFO_PGSIZES;
> >>   
> >> +		vfio_pgsize_bitmap(iommu);  
> > 
> > 
> > Why is it necessary to rebuild the bitmap here?  The user can't get to
> > this ioctl until they've added a group to the container and set the
> > IOMMU model.
> > 
> >   
> For mdev device, domain is not added to domain_list so 
> vfio_pgsize_bitmap() doesn't get called when there is only mdev device 
> attached.
> Your concern is right though, vfio_pgsize_bitmap() should get populated 
> with attach_group,so fixing it by calling vfio_pgsize_bitmap() for mdev 
> device when iommu->external_domain is set.
> 
> >>   		info.iova_pgsizes = iommu->pgsize_bitmap;
> >>   
> >> -		ret = vfio_iommu_iova_build_caps(iommu, &caps);
> >> +		ret = vfio_iommu_migration_build_caps(iommu, &caps);
> >> +
> >> +		if (!ret)
> >> +			ret = vfio_iommu_iova_build_caps(iommu, &caps);
> >> +
> >>   		mutex_unlock(&iommu->lock);
> >> +
> >>   		if (ret)
> >>   			return ret;
> >>   
> >> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> >> index e3cbf8b78623..c90604322798 100644
> >> --- a/include/uapi/linux/vfio.h
> >> +++ b/include/uapi/linux/vfio.h
> >> @@ -1013,6 +1013,27 @@ struct vfio_iommu_type1_info_cap_iova_range {
> >>   	struct	vfio_iova_range iova_ranges[];
> >>   };
> >>   
> >> +/*
> >> + * The migration capability allows to report supported features for migration.
> >> + *
> >> + * The structures below define version 1 of this capability.
> >> + *
> >> + * pgsize_bitmap: Kernel driver returns supported page sizes bitmap for dirty
> >> + * page tracking.
> >> + * max_dirty_bitmap_size: Kernel driver returns maximum supported dirty bitmap
> >> + * size in bytes to be used by user application for ioctls to get dirty bitmap.
> >> + */
> >> +#define VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION  1
> >> +
> >> +struct vfio_iommu_type1_info_cap_migration {
> >> +	struct	vfio_info_cap_header header;
> >> +	__u32	flags;
> >> +	/* supports dirty page tracking */
> >> +#define VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK	(1 << 0)  
> > 
> > This flag is a bit redundant to the purpose of this capability, isn't
> > it?  I think exposing the capability itself is indicating support for
> > dirty page tracking.  We should probably be explicit in the comment
> > about exactly what interface this capability implies.  Thanks,
> >  
> 
> Capability is added to provide provision for feature flags that kernel 
> driver support, that's where we started right?
> Later added pgsize_bitmap and max supported bitmap size as you suggested.
> I'm confused now, should I keep this flag here?
> Even if the flag is removed, 'flags' field is still required so that 
> whenever new feature is added, new flag will be added. That's the whole 
> purpose we added this capability. Can we add a field which is not used? 
> and we don't know when it will be used in future?

We have empty flags fields all over the uapi.  When I look at this
capability, I wonder what it means if it were to be implemented without
VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK set.  For example, what
migration feature would an IOMMU be exposing if it didn't have dirty
page tracking.  All of the extensions we're implementing to support
migration in the IOMMU are related to dirty page tracking.  Therefore
it seems that the existence of the capability itself is expressing the
support for dirty page tracking, right?  Thanks,

Alex

> >> +	__u64	pgsize_bitmap;
> >> +	__u64	max_dirty_bitmap_size;		/* in bytes */
> >> +};
> >> +
> >>   #define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
> >>   
> >>   /**  
> >   
>
diff mbox series

Patch

diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 4358be26ff80..77351497a9c2 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -2389,6 +2389,22 @@  static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,
 	return ret;
 }
 
+static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,
+					   struct vfio_info_cap *caps)
+{
+	struct vfio_iommu_type1_info_cap_migration cap_mig;
+
+	cap_mig.header.id = VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION;
+	cap_mig.header.version = 1;
+	cap_mig.flags = VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK;
+
+	/* support minimum pgsize */
+	cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap);
+	cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX;
+
+	return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig));
+}
+
 static long vfio_iommu_type1_ioctl(void *iommu_data,
 				   unsigned int cmd, unsigned long arg)
 {
@@ -2433,10 +2449,16 @@  static long vfio_iommu_type1_ioctl(void *iommu_data,
 		mutex_lock(&iommu->lock);
 		info.flags = VFIO_IOMMU_INFO_PGSIZES;
 
+		vfio_pgsize_bitmap(iommu);
 		info.iova_pgsizes = iommu->pgsize_bitmap;
 
-		ret = vfio_iommu_iova_build_caps(iommu, &caps);
+		ret = vfio_iommu_migration_build_caps(iommu, &caps);
+
+		if (!ret)
+			ret = vfio_iommu_iova_build_caps(iommu, &caps);
+
 		mutex_unlock(&iommu->lock);
+
 		if (ret)
 			return ret;
 
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index e3cbf8b78623..c90604322798 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -1013,6 +1013,27 @@  struct vfio_iommu_type1_info_cap_iova_range {
 	struct	vfio_iova_range iova_ranges[];
 };
 
+/*
+ * The migration capability allows to report supported features for migration.
+ *
+ * The structures below define version 1 of this capability.
+ *
+ * pgsize_bitmap: Kernel driver returns supported page sizes bitmap for dirty
+ * page tracking.
+ * max_dirty_bitmap_size: Kernel driver returns maximum supported dirty bitmap
+ * size in bytes to be used by user application for ioctls to get dirty bitmap.
+ */
+#define VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION  1
+
+struct vfio_iommu_type1_info_cap_migration {
+	struct	vfio_info_cap_header header;
+	__u32	flags;
+	/* supports dirty page tracking */
+#define VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK	(1 << 0)
+	__u64	pgsize_bitmap;
+	__u64	max_dirty_bitmap_size;		/* in bytes */
+};
+
 #define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
 
 /**