| Message ID | 1584880325-10561-3-git-send-email-yi.l.liu@intel.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | vfio: expose virtual Shared Virtual Addressing to VMs | expand |
Hi Yi,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on vfio/next]
[also build test ERROR on v5.6-rc6 next-20200320]
[if your patch is applied to the wrong git tree, please drop us a note to help
improve the system. BTW, we also suggest to use '--base' option to specify the
base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Liu-Yi-L/vfio-expose-virtual-Shared-Virtual-Addressing-to-VMs/20200322-213259
base: https://github.com/awilliam/linux-vfio.git next
config: arm64-defconfig (attached as .config)
compiler: aarch64-linux-gcc (GCC) 9.2.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
GCC_VERSION=9.2.0 make.cross ARCH=arm64
If you fix the issue, kindly add following tag
Reported-by: kbuild test robot <lkp@intel.com>
All errors (new ones prefixed by >>):
drivers/vfio/vfio.c: In function 'vfio_create_mm':
drivers/vfio/vfio.c:2149:8: error: implicit declaration of function 'ioasid_alloc_set'; did you mean 'ioasid_alloc'? [-Werror=implicit-function-declaration]
2149 | ret = ioasid_alloc_set((struct ioasid_set *) mm,
| ^~~~~~~~~~~~~~~~
| ioasid_alloc
drivers/vfio/vfio.c:2158:13: warning: assignment to 'long long unsigned int' from 'struct mm_struct *' makes integer from pointer without a cast [-Wint-conversion]
2158 | token->val = mm;
| ^
drivers/vfio/vfio.c: In function 'vfio_mm_unlock_and_free':
drivers/vfio/vfio.c:2170:2: error: implicit declaration of function 'ioasid_free_set'; did you mean 'ioasid_free'? [-Werror=implicit-function-declaration]
2170 | ioasid_free_set(vmm->ioasid_sid, true);
| ^~~~~~~~~~~~~~~
| ioasid_free
drivers/vfio/vfio.c: In function 'vfio_mm_pasid_alloc':
>> drivers/vfio/vfio.c:2230:3: error: implicit declaration of function 'ioasid_adjust_set' [-Werror=implicit-function-declaration]
2230 | ioasid_adjust_set(vmm->ioasid_sid, quota);
| ^~~~~~~~~~~~~~~~~
drivers/vfio/vfio.c:2233:26: warning: passing argument 1 of 'ioasid_alloc' makes pointer from integer without a cast [-Wint-conversion]
2233 | pasid = ioasid_alloc(vmm->ioasid_sid, min, max, NULL);
| ~~~^~~~~~~~~~~~
| |
| int
In file included from include/linux/iommu.h:16,
from drivers/vfio/vfio.c:20:
include/linux/ioasid.h:45:56: note: expected 'struct ioasid_set *' but argument is of type 'int'
45 | static inline ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min,
| ~~~~~~~~~~~~~~~~~~~^~~
drivers/vfio/vfio.c: In function 'vfio_mm_pasid_free':
drivers/vfio/vfio.c:2252:25: warning: passing argument 1 of 'ioasid_find' makes pointer from integer without a cast [-Wint-conversion]
2252 | pdata = ioasid_find(vmm->ioasid_sid, pasid, NULL);
| ~~~^~~~~~~~~~~~
| |
| int
In file included from include/linux/iommu.h:16,
from drivers/vfio/vfio.c:20:
include/linux/ioasid.h:55:52: note: expected 'struct ioasid_set *' but argument is of type 'int'
55 | static inline void *ioasid_find(struct ioasid_set *set, ioasid_t ioasid,
| ~~~~~~~~~~~~~~~~~~~^~~
cc1: some warnings being treated as errors
vim +/ioasid_adjust_set +2230 drivers/vfio/vfio.c
2133
2134 /**
2135 * VFIO_MM objects - create, release, get, put, search
2136 * Caller of the function should have held vfio.vfio_mm_lock.
2137 */
2138 static struct vfio_mm *vfio_create_mm(struct mm_struct *mm)
2139 {
2140 struct vfio_mm *vmm;
2141 struct vfio_mm_token *token;
2142 int ret = 0;
2143
2144 vmm = kzalloc(sizeof(*vmm), GFP_KERNEL);
2145 if (!vmm)
2146 return ERR_PTR(-ENOMEM);
2147
2148 /* Per mm IOASID set used for quota control and group operations */
2149 ret = ioasid_alloc_set((struct ioasid_set *) mm,
2150 VFIO_DEFAULT_PASID_QUOTA, &vmm->ioasid_sid);
2151 if (ret) {
2152 kfree(vmm);
2153 return ERR_PTR(ret);
2154 }
2155
2156 kref_init(&vmm->kref);
2157 token = &vmm->token;
> 2158 token->val = mm;
2159 vmm->pasid_quota = VFIO_DEFAULT_PASID_QUOTA;
2160 mutex_init(&vmm->pasid_lock);
2161
2162 list_add(&vmm->vfio_next, &vfio.vfio_mm_list);
2163
2164 return vmm;
2165 }
2166
2167 static void vfio_mm_unlock_and_free(struct vfio_mm *vmm)
2168 {
2169 /* destroy the ioasid set */
2170 ioasid_free_set(vmm->ioasid_sid, true);
2171 mutex_unlock(&vfio.vfio_mm_lock);
2172 kfree(vmm);
2173 }
2174
2175 /* called with vfio.vfio_mm_lock held */
2176 static void vfio_mm_release(struct kref *kref)
2177 {
2178 struct vfio_mm *vmm = container_of(kref, struct vfio_mm, kref);
2179
2180 list_del(&vmm->vfio_next);
2181 vfio_mm_unlock_and_free(vmm);
2182 }
2183
2184 void vfio_mm_put(struct vfio_mm *vmm)
2185 {
2186 kref_put_mutex(&vmm->kref, vfio_mm_release, &vfio.vfio_mm_lock);
2187 }
2188 EXPORT_SYMBOL_GPL(vfio_mm_put);
2189
2190 /* Assume vfio_mm_lock or vfio_mm reference is held */
2191 static void vfio_mm_get(struct vfio_mm *vmm)
2192 {
2193 kref_get(&vmm->kref);
2194 }
2195
2196 struct vfio_mm *vfio_mm_get_from_task(struct task_struct *task)
2197 {
2198 struct mm_struct *mm = get_task_mm(task);
2199 struct vfio_mm *vmm;
2200 unsigned long long val = (unsigned long long) mm;
2201
2202 mutex_lock(&vfio.vfio_mm_lock);
2203 list_for_each_entry(vmm, &vfio.vfio_mm_list, vfio_next) {
2204 if (vmm->token.val == val) {
2205 vfio_mm_get(vmm);
2206 goto out;
2207 }
2208 }
2209
2210 vmm = vfio_create_mm(mm);
2211 if (IS_ERR(vmm))
2212 vmm = NULL;
2213 out:
2214 mutex_unlock(&vfio.vfio_mm_lock);
2215 mmput(mm);
2216 return vmm;
2217 }
2218 EXPORT_SYMBOL_GPL(vfio_mm_get_from_task);
2219
2220 int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int quota, int min, int max)
2221 {
2222 ioasid_t pasid;
2223 int ret = -ENOSPC;
2224
2225 mutex_lock(&vmm->pasid_lock);
2226
2227 /* update quota as it is tunable by admin */
2228 if (vmm->pasid_quota != quota) {
2229 vmm->pasid_quota = quota;
> 2230 ioasid_adjust_set(vmm->ioasid_sid, quota);
2231 }
2232
2233 pasid = ioasid_alloc(vmm->ioasid_sid, min, max, NULL);
2234 if (pasid == INVALID_IOASID) {
2235 ret = -ENOSPC;
2236 goto out_unlock;
2237 }
2238
2239 ret = pasid;
2240 out_unlock:
2241 mutex_unlock(&vmm->pasid_lock);
2242 return ret;
2243 }
2244 EXPORT_SYMBOL_GPL(vfio_mm_pasid_alloc);
2245
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
> From: Liu, Yi L <yi.l.liu@intel.com> > Sent: Sunday, March 22, 2020 8:32 PM > > From: Liu Yi L <yi.l.liu@intel.com> > > This patch adds a module option to make the PASID quota tunable by > administrator. > > TODO: needs to think more on how to make the tuning to be per-process. > > Previous discussions: > https://patchwork.kernel.org/patch/11209429/ > > Cc: Kevin Tian <kevin.tian@intel.com> > CC: Jacob Pan <jacob.jun.pan@linux.intel.com> > Cc: Alex Williamson <alex.williamson@redhat.com> > Cc: Eric Auger <eric.auger@redhat.com> > Cc: Jean-Philippe Brucker <jean-philippe@linaro.org> > Signed-off-by: Liu Yi L <yi.l.liu@intel.com> > --- > drivers/vfio/vfio.c | 8 +++++++- > drivers/vfio/vfio_iommu_type1.c | 7 ++++++- > include/linux/vfio.h | 3 ++- > 3 files changed, 15 insertions(+), 3 deletions(-) > > diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c > index d13b483..020a792 100644 > --- a/drivers/vfio/vfio.c > +++ b/drivers/vfio/vfio.c > @@ -2217,13 +2217,19 @@ struct vfio_mm *vfio_mm_get_from_task(struct > task_struct *task) > } > EXPORT_SYMBOL_GPL(vfio_mm_get_from_task); > > -int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int min, int max) > +int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int quota, int min, int max) > { > ioasid_t pasid; > int ret = -ENOSPC; > > mutex_lock(&vmm->pasid_lock); > > + /* update quota as it is tunable by admin */ > + if (vmm->pasid_quota != quota) { > + vmm->pasid_quota = quota; > + ioasid_adjust_set(vmm->ioasid_sid, quota); > + } > + It's a bit weird to have quota adjusted in the alloc path, since the latter might be initiated by non-privileged users. Why not doing the simple math in vfio_ create_mm to set the quota when the ioasid set is created? even in the future you may allow per-process quota setting, that should come from separate privileged path instead of thru alloc... > pasid = ioasid_alloc(vmm->ioasid_sid, min, max, NULL); > if (pasid == INVALID_IOASID) { > ret = -ENOSPC; > diff --git a/drivers/vfio/vfio_iommu_type1.c > b/drivers/vfio/vfio_iommu_type1.c > index 331ceee..e40afc0 100644 > --- a/drivers/vfio/vfio_iommu_type1.c > +++ b/drivers/vfio/vfio_iommu_type1.c > @@ -60,6 +60,11 @@ module_param_named(dma_entry_limit, > dma_entry_limit, uint, 0644); > MODULE_PARM_DESC(dma_entry_limit, > "Maximum number of user DMA mappings per container > (65535)."); > > +static int pasid_quota = VFIO_DEFAULT_PASID_QUOTA; > +module_param_named(pasid_quota, pasid_quota, uint, 0644); > +MODULE_PARM_DESC(pasid_quota, > + "Quota of user owned PASIDs per vfio-based application > (1000)."); > + > struct vfio_iommu { > struct list_head domain_list; > struct list_head iova_list; > @@ -2200,7 +2205,7 @@ static int vfio_iommu_type1_pasid_alloc(struct > vfio_iommu *iommu, > goto out_unlock; > } > if (vmm) > - ret = vfio_mm_pasid_alloc(vmm, min, max); > + ret = vfio_mm_pasid_alloc(vmm, pasid_quota, min, max); > else > ret = -EINVAL; > out_unlock: > diff --git a/include/linux/vfio.h b/include/linux/vfio.h > index 75f9f7f1..af2ef78 100644 > --- a/include/linux/vfio.h > +++ b/include/linux/vfio.h > @@ -106,7 +106,8 @@ struct vfio_mm { > > extern struct vfio_mm *vfio_mm_get_from_task(struct task_struct *task); > extern void vfio_mm_put(struct vfio_mm *vmm); > -extern int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int min, int max); > +extern int vfio_mm_pasid_alloc(struct vfio_mm *vmm, > + int quota, int min, int max); > extern int vfio_mm_pasid_free(struct vfio_mm *vmm, ioasid_t pasid); > > /* > -- > 2.7.4
> From: Tian, Kevin <kevin.tian@intel.com> > Sent: Monday, March 30, 2020 4:41 PM > To: Liu, Yi L <yi.l.liu@intel.com>; alex.williamson@redhat.com; > Subject: RE: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 parameter for quota > tuning > > > From: Liu, Yi L <yi.l.liu@intel.com> > > Sent: Sunday, March 22, 2020 8:32 PM > > > > From: Liu Yi L <yi.l.liu@intel.com> > > > > This patch adds a module option to make the PASID quota tunable by > > administrator. > > > > TODO: needs to think more on how to make the tuning to be per-process. > > > > Previous discussions: > > https://patchwork.kernel.org/patch/11209429/ > > > > Cc: Kevin Tian <kevin.tian@intel.com> > > CC: Jacob Pan <jacob.jun.pan@linux.intel.com> > > Cc: Alex Williamson <alex.williamson@redhat.com> > > Cc: Eric Auger <eric.auger@redhat.com> > > Cc: Jean-Philippe Brucker <jean-philippe@linaro.org> > > Signed-off-by: Liu Yi L <yi.l.liu@intel.com> > > --- > > drivers/vfio/vfio.c | 8 +++++++- > > drivers/vfio/vfio_iommu_type1.c | 7 ++++++- > > include/linux/vfio.h | 3 ++- > > 3 files changed, 15 insertions(+), 3 deletions(-) > > > > diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c > > index d13b483..020a792 100644 > > --- a/drivers/vfio/vfio.c > > +++ b/drivers/vfio/vfio.c > > @@ -2217,13 +2217,19 @@ struct vfio_mm *vfio_mm_get_from_task(struct > > task_struct *task) > > } > > EXPORT_SYMBOL_GPL(vfio_mm_get_from_task); > > > > -int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int min, int max) > > +int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int quota, int min, int max) > > { > > ioasid_t pasid; > > int ret = -ENOSPC; > > > > mutex_lock(&vmm->pasid_lock); > > > > + /* update quota as it is tunable by admin */ > > + if (vmm->pasid_quota != quota) { > > + vmm->pasid_quota = quota; > > + ioasid_adjust_set(vmm->ioasid_sid, quota); > > + } > > + > > It's a bit weird to have quota adjusted in the alloc path, since the latter might > be initiated by non-privileged users. Why not doing the simple math in vfio_ > create_mm to set the quota when the ioasid set is created? even in the future > you may allow per-process quota setting, that should come from separate > privileged path instead of thru alloc.. The reason is the kernel parameter modification has no event which can be used to adjust the quota. So I chose to adjust it in pasid_alloc path. If it's not good, how about adding one more IOCTL to let user- space trigger a quota adjustment event? Then even non-privileged user could trigger quota adjustment, the quota is actually controlled by privileged user. How about your opinion? Regards, Yi Liu
> From: Liu, Yi L <yi.l.liu@intel.com> > Sent: Monday, March 30, 2020 4:53 PM > > > From: Tian, Kevin <kevin.tian@intel.com> > > Sent: Monday, March 30, 2020 4:41 PM > > To: Liu, Yi L <yi.l.liu@intel.com>; alex.williamson@redhat.com; > > Subject: RE: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 parameter > for quota > > tuning > > > > > From: Liu, Yi L <yi.l.liu@intel.com> > > > Sent: Sunday, March 22, 2020 8:32 PM > > > > > > From: Liu Yi L <yi.l.liu@intel.com> > > > > > > This patch adds a module option to make the PASID quota tunable by > > > administrator. > > > > > > TODO: needs to think more on how to make the tuning to be per-process. > > > > > > Previous discussions: > > > https://patchwork.kernel.org/patch/11209429/ > > > > > > Cc: Kevin Tian <kevin.tian@intel.com> > > > CC: Jacob Pan <jacob.jun.pan@linux.intel.com> > > > Cc: Alex Williamson <alex.williamson@redhat.com> > > > Cc: Eric Auger <eric.auger@redhat.com> > > > Cc: Jean-Philippe Brucker <jean-philippe@linaro.org> > > > Signed-off-by: Liu Yi L <yi.l.liu@intel.com> > > > --- > > > drivers/vfio/vfio.c | 8 +++++++- > > > drivers/vfio/vfio_iommu_type1.c | 7 ++++++- > > > include/linux/vfio.h | 3 ++- > > > 3 files changed, 15 insertions(+), 3 deletions(-) > > > > > > diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c > > > index d13b483..020a792 100644 > > > --- a/drivers/vfio/vfio.c > > > +++ b/drivers/vfio/vfio.c > > > @@ -2217,13 +2217,19 @@ struct vfio_mm > *vfio_mm_get_from_task(struct > > > task_struct *task) > > > } > > > EXPORT_SYMBOL_GPL(vfio_mm_get_from_task); > > > > > > -int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int min, int max) > > > +int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int quota, int min, int > max) > > > { > > > ioasid_t pasid; > > > int ret = -ENOSPC; > > > > > > mutex_lock(&vmm->pasid_lock); > > > > > > + /* update quota as it is tunable by admin */ > > > + if (vmm->pasid_quota != quota) { > > > + vmm->pasid_quota = quota; > > > + ioasid_adjust_set(vmm->ioasid_sid, quota); > > > + } > > > + > > > > It's a bit weird to have quota adjusted in the alloc path, since the latter > might > > be initiated by non-privileged users. Why not doing the simple math in > vfio_ > > create_mm to set the quota when the ioasid set is created? even in the > future > > you may allow per-process quota setting, that should come from separate > > privileged path instead of thru alloc.. > > The reason is the kernel parameter modification has no event which > can be used to adjust the quota. So I chose to adjust it in pasid_alloc > path. If it's not good, how about adding one more IOCTL to let user- > space trigger a quota adjustment event? Then even non-privileged > user could trigger quota adjustment, the quota is actually controlled > by privileged user. How about your opinion? > why do you need an event to adjust? As I said, you can set the quota when the set is created in vfio_create_mm... Thanks Kevin
> From: Tian, Kevin <kevin.tian@intel.com> > Sent: Monday, March 30, 2020 5:20 PM > To: Liu, Yi L <yi.l.liu@intel.com>; alex.williamson@redhat.com; > Subject: RE: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 parameter for quota > tuning > > > From: Liu, Yi L <yi.l.liu@intel.com> > > Sent: Monday, March 30, 2020 4:53 PM > > > > > From: Tian, Kevin <kevin.tian@intel.com> > > > Sent: Monday, March 30, 2020 4:41 PM > > > To: Liu, Yi L <yi.l.liu@intel.com>; alex.williamson@redhat.com; > > > Subject: RE: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 > > > parameter > > for quota > > > tuning > > > > > > > From: Liu, Yi L <yi.l.liu@intel.com> > > > > Sent: Sunday, March 22, 2020 8:32 PM > > > > > > > > From: Liu Yi L <yi.l.liu@intel.com> > > > > > > > > This patch adds a module option to make the PASID quota tunable by > > > > administrator. > > > > > > > > TODO: needs to think more on how to make the tuning to be per-process. > > > > > > > > Previous discussions: > > > > https://patchwork.kernel.org/patch/11209429/ > > > > > > > > Cc: Kevin Tian <kevin.tian@intel.com> > > > > CC: Jacob Pan <jacob.jun.pan@linux.intel.com> > > > > Cc: Alex Williamson <alex.williamson@redhat.com> > > > > Cc: Eric Auger <eric.auger@redhat.com> > > > > Cc: Jean-Philippe Brucker <jean-philippe@linaro.org> > > > > Signed-off-by: Liu Yi L <yi.l.liu@intel.com> > > > > --- > > > > drivers/vfio/vfio.c | 8 +++++++- > > > > drivers/vfio/vfio_iommu_type1.c | 7 ++++++- > > > > include/linux/vfio.h | 3 ++- > > > > 3 files changed, 15 insertions(+), 3 deletions(-) > > > > > > > > diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index > > > > d13b483..020a792 100644 > > > > --- a/drivers/vfio/vfio.c > > > > +++ b/drivers/vfio/vfio.c > > > > @@ -2217,13 +2217,19 @@ struct vfio_mm > > *vfio_mm_get_from_task(struct > > > > task_struct *task) > > > > } > > > > EXPORT_SYMBOL_GPL(vfio_mm_get_from_task); > > > > > > > > -int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int min, int max) > > > > +int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int quota, int min, > > > > +int > > max) > > > > { > > > > ioasid_t pasid; > > > > int ret = -ENOSPC; > > > > > > > > mutex_lock(&vmm->pasid_lock); > > > > > > > > + /* update quota as it is tunable by admin */ > > > > + if (vmm->pasid_quota != quota) { > > > > + vmm->pasid_quota = quota; > > > > + ioasid_adjust_set(vmm->ioasid_sid, quota); > > > > + } > > > > + > > > > > > It's a bit weird to have quota adjusted in the alloc path, since the > > > latter > > might > > > be initiated by non-privileged users. Why not doing the simple math > > > in > > vfio_ > > > create_mm to set the quota when the ioasid set is created? even in > > > the > > future > > > you may allow per-process quota setting, that should come from > > > separate privileged path instead of thru alloc.. > > > > The reason is the kernel parameter modification has no event which can > > be used to adjust the quota. So I chose to adjust it in pasid_alloc > > path. If it's not good, how about adding one more IOCTL to let user- > > space trigger a quota adjustment event? Then even non-privileged user > > could trigger quota adjustment, the quota is actually controlled by > > privileged user. How about your opinion? > > > > why do you need an event to adjust? As I said, you can set the quota when the set is > created in vfio_create_mm... oh, it's to support runtime adjustments. I guess it may be helpful to let per-VM quota tunable even the VM is running. If just set the quota in vfio_create_mm(), it is not able to adjust at runtime. Regards, Yi Liu
> From: Liu, Yi L <yi.l.liu@intel.com> > Sent: Monday, March 30, 2020 5:27 PM > > > From: Tian, Kevin <kevin.tian@intel.com> > > Sent: Monday, March 30, 2020 5:20 PM > > To: Liu, Yi L <yi.l.liu@intel.com>; alex.williamson@redhat.com; > > Subject: RE: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 parameter > for quota > > tuning > > > > > From: Liu, Yi L <yi.l.liu@intel.com> > > > Sent: Monday, March 30, 2020 4:53 PM > > > > > > > From: Tian, Kevin <kevin.tian@intel.com> > > > > Sent: Monday, March 30, 2020 4:41 PM > > > > To: Liu, Yi L <yi.l.liu@intel.com>; alex.williamson@redhat.com; > > > > Subject: RE: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 > > > > parameter > > > for quota > > > > tuning > > > > > > > > > From: Liu, Yi L <yi.l.liu@intel.com> > > > > > Sent: Sunday, March 22, 2020 8:32 PM > > > > > > > > > > From: Liu Yi L <yi.l.liu@intel.com> > > > > > > > > > > This patch adds a module option to make the PASID quota tunable by > > > > > administrator. > > > > > > > > > > TODO: needs to think more on how to make the tuning to be per- > process. > > > > > > > > > > Previous discussions: > > > > > https://patchwork.kernel.org/patch/11209429/ > > > > > > > > > > Cc: Kevin Tian <kevin.tian@intel.com> > > > > > CC: Jacob Pan <jacob.jun.pan@linux.intel.com> > > > > > Cc: Alex Williamson <alex.williamson@redhat.com> > > > > > Cc: Eric Auger <eric.auger@redhat.com> > > > > > Cc: Jean-Philippe Brucker <jean-philippe@linaro.org> > > > > > Signed-off-by: Liu Yi L <yi.l.liu@intel.com> > > > > > --- > > > > > drivers/vfio/vfio.c | 8 +++++++- > > > > > drivers/vfio/vfio_iommu_type1.c | 7 ++++++- > > > > > include/linux/vfio.h | 3 ++- > > > > > 3 files changed, 15 insertions(+), 3 deletions(-) > > > > > > > > > > diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index > > > > > d13b483..020a792 100644 > > > > > --- a/drivers/vfio/vfio.c > > > > > +++ b/drivers/vfio/vfio.c > > > > > @@ -2217,13 +2217,19 @@ struct vfio_mm > > > *vfio_mm_get_from_task(struct > > > > > task_struct *task) > > > > > } > > > > > EXPORT_SYMBOL_GPL(vfio_mm_get_from_task); > > > > > > > > > > -int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int min, int max) > > > > > +int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int quota, int min, > > > > > +int > > > max) > > > > > { > > > > > ioasid_t pasid; > > > > > int ret = -ENOSPC; > > > > > > > > > > mutex_lock(&vmm->pasid_lock); > > > > > > > > > > + /* update quota as it is tunable by admin */ > > > > > + if (vmm->pasid_quota != quota) { > > > > > + vmm->pasid_quota = quota; > > > > > + ioasid_adjust_set(vmm->ioasid_sid, quota); > > > > > + } > > > > > + > > > > > > > > It's a bit weird to have quota adjusted in the alloc path, since the > > > > latter > > > might > > > > be initiated by non-privileged users. Why not doing the simple math > > > > in > > > vfio_ > > > > create_mm to set the quota when the ioasid set is created? even in > > > > the > > > future > > > > you may allow per-process quota setting, that should come from > > > > separate privileged path instead of thru alloc.. > > > > > > The reason is the kernel parameter modification has no event which can > > > be used to adjust the quota. So I chose to adjust it in pasid_alloc > > > path. If it's not good, how about adding one more IOCTL to let user- > > > space trigger a quota adjustment event? Then even non-privileged user > > > could trigger quota adjustment, the quota is actually controlled by > > > privileged user. How about your opinion? > > > > > > > why do you need an event to adjust? As I said, you can set the quota when > the set is > > created in vfio_create_mm... > > oh, it's to support runtime adjustments. I guess it may be helpful to let > per-VM quota tunable even the VM is running. If just set the quota in > vfio_create_mm(), it is not able to adjust at runtime. > ok, I didn't note the module parameter was granted with a write permission. However there is a further problem. We cannot support PASID reclaim now. What about the admin sets a quota smaller than previous value while some IOASID sets already exceed the new quota? I'm not sure how to fail a runtime module parameter change due to that situation. possibly a normal sysfs node better suites the runtime change requirement... Thanks Kevin
On Mon, 30 Mar 2020 11:44:08 +0000 "Tian, Kevin" <kevin.tian@intel.com> wrote: > > From: Liu, Yi L <yi.l.liu@intel.com> > > Sent: Monday, March 30, 2020 5:27 PM > > > > > From: Tian, Kevin <kevin.tian@intel.com> > > > Sent: Monday, March 30, 2020 5:20 PM > > > To: Liu, Yi L <yi.l.liu@intel.com>; alex.williamson@redhat.com; > > > Subject: RE: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 parameter > > for quota > > > tuning > > > > > > > From: Liu, Yi L <yi.l.liu@intel.com> > > > > Sent: Monday, March 30, 2020 4:53 PM > > > > > > > > > From: Tian, Kevin <kevin.tian@intel.com> > > > > > Sent: Monday, March 30, 2020 4:41 PM > > > > > To: Liu, Yi L <yi.l.liu@intel.com>; alex.williamson@redhat.com; > > > > > Subject: RE: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 > > > > > parameter > > > > for quota > > > > > tuning > > > > > > > > > > > From: Liu, Yi L <yi.l.liu@intel.com> > > > > > > Sent: Sunday, March 22, 2020 8:32 PM > > > > > > > > > > > > From: Liu Yi L <yi.l.liu@intel.com> > > > > > > > > > > > > This patch adds a module option to make the PASID quota tunable by > > > > > > administrator. > > > > > > > > > > > > TODO: needs to think more on how to make the tuning to be per- > > process. > > > > > > > > > > > > Previous discussions: > > > > > > https://patchwork.kernel.org/patch/11209429/ > > > > > > > > > > > > Cc: Kevin Tian <kevin.tian@intel.com> > > > > > > CC: Jacob Pan <jacob.jun.pan@linux.intel.com> > > > > > > Cc: Alex Williamson <alex.williamson@redhat.com> > > > > > > Cc: Eric Auger <eric.auger@redhat.com> > > > > > > Cc: Jean-Philippe Brucker <jean-philippe@linaro.org> > > > > > > Signed-off-by: Liu Yi L <yi.l.liu@intel.com> > > > > > > --- > > > > > > drivers/vfio/vfio.c | 8 +++++++- > > > > > > drivers/vfio/vfio_iommu_type1.c | 7 ++++++- > > > > > > include/linux/vfio.h | 3 ++- > > > > > > 3 files changed, 15 insertions(+), 3 deletions(-) > > > > > > > > > > > > diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index > > > > > > d13b483..020a792 100644 > > > > > > --- a/drivers/vfio/vfio.c > > > > > > +++ b/drivers/vfio/vfio.c > > > > > > @@ -2217,13 +2217,19 @@ struct vfio_mm > > > > *vfio_mm_get_from_task(struct > > > > > > task_struct *task) > > > > > > } > > > > > > EXPORT_SYMBOL_GPL(vfio_mm_get_from_task); > > > > > > > > > > > > -int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int min, int max) > > > > > > +int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int quota, int min, > > > > > > +int > > > > max) > > > > > > { > > > > > > ioasid_t pasid; > > > > > > int ret = -ENOSPC; > > > > > > > > > > > > mutex_lock(&vmm->pasid_lock); > > > > > > > > > > > > + /* update quota as it is tunable by admin */ > > > > > > + if (vmm->pasid_quota != quota) { > > > > > > + vmm->pasid_quota = quota; > > > > > > + ioasid_adjust_set(vmm->ioasid_sid, quota); > > > > > > + } > > > > > > + > > > > > > > > > > It's a bit weird to have quota adjusted in the alloc path, since the > > > > > latter > > > > might > > > > > be initiated by non-privileged users. Why not doing the simple math > > > > > in > > > > vfio_ > > > > > create_mm to set the quota when the ioasid set is created? even in > > > > > the > > > > future > > > > > you may allow per-process quota setting, that should come from > > > > > separate privileged path instead of thru alloc.. > > > > > > > > The reason is the kernel parameter modification has no event which can > > > > be used to adjust the quota. So I chose to adjust it in pasid_alloc > > > > path. If it's not good, how about adding one more IOCTL to let user- > > > > space trigger a quota adjustment event? Then even non-privileged user > > > > could trigger quota adjustment, the quota is actually controlled by > > > > privileged user. How about your opinion? > > > > > > > > > > why do you need an event to adjust? As I said, you can set the quota when > > the set is > > > created in vfio_create_mm... > > > > oh, it's to support runtime adjustments. I guess it may be helpful to let > > per-VM quota tunable even the VM is running. If just set the quota in > > vfio_create_mm(), it is not able to adjust at runtime. > > > > ok, I didn't note the module parameter was granted with a write permission. > However there is a further problem. We cannot support PASID reclaim now. > What about the admin sets a quota smaller than previous value while some > IOASID sets already exceed the new quota? I'm not sure how to fail a runtime > module parameter change due to that situation. possibly a normal sysfs > node better suites the runtime change requirement... Yep, making this runtime adjustable seems a bit unpredictable and racy, and it's not clear to me how a user is going to jump in at just the right time for a user and adjust the limit. I'd probably go for a simple non-runtime adjustable module option. It's a safety net at this point anyway afaict. Thanks, Alex
> From: Alex Williamson <alex.williamson@redhat.com> > Sent: Friday, April 3, 2020 1:59 AM > To: Tian, Kevin <kevin.tian@intel.com> > Subject: Re: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 parameter for quota > tuning > > On Mon, 30 Mar 2020 11:44:08 +0000 > "Tian, Kevin" <kevin.tian@intel.com> wrote: > > > > From: Liu, Yi L <yi.l.liu@intel.com> > > > Sent: Monday, March 30, 2020 5:27 PM > > > > > > > From: Tian, Kevin <kevin.tian@intel.com> > > > > Sent: Monday, March 30, 2020 5:20 PM > > > > To: Liu, Yi L <yi.l.liu@intel.com>; alex.williamson@redhat.com; > > > > Subject: RE: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 parameter > > > for quota > > > > tuning > > > > > > > > > From: Liu, Yi L <yi.l.liu@intel.com> > > > > > Sent: Monday, March 30, 2020 4:53 PM > > > > > > > > > > > From: Tian, Kevin <kevin.tian@intel.com> > > > > > > Sent: Monday, March 30, 2020 4:41 PM > > > > > > To: Liu, Yi L <yi.l.liu@intel.com>; alex.williamson@redhat.com; > > > > > > Subject: RE: [PATCH v1 2/8] vfio/type1: Add vfio_iommu_type1 > > > > > > parameter > > > > > for quota > > > > > > tuning > > > > > > > > > > > > > From: Liu, Yi L <yi.l.liu@intel.com> > > > > > > > Sent: Sunday, March 22, 2020 8:32 PM > > > > > > > > > > > > > > From: Liu Yi L <yi.l.liu@intel.com> > > > > > > > > > > > > > > This patch adds a module option to make the PASID quota tunable by > > > > > > > administrator. > > > > > > > > > > > > > > TODO: needs to think more on how to make the tuning to be per- > > > process. > > > > > > > > > > > > > > Previous discussions: > > > > > > > https://patchwork.kernel.org/patch/11209429/ > > > > > > > > > > > > > > Cc: Kevin Tian <kevin.tian@intel.com> > > > > > > > CC: Jacob Pan <jacob.jun.pan@linux.intel.com> > > > > > > > Cc: Alex Williamson <alex.williamson@redhat.com> > > > > > > > Cc: Eric Auger <eric.auger@redhat.com> > > > > > > > Cc: Jean-Philippe Brucker <jean-philippe@linaro.org> > > > > > > > Signed-off-by: Liu Yi L <yi.l.liu@intel.com> > > > > > > > --- > > > > > > > drivers/vfio/vfio.c | 8 +++++++- > > > > > > > drivers/vfio/vfio_iommu_type1.c | 7 ++++++- > > > > > > > include/linux/vfio.h | 3 ++- > > > > > > > 3 files changed, 15 insertions(+), 3 deletions(-) > > > > > > > > > > > > > > diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index > > > > > > > d13b483..020a792 100644 > > > > > > > --- a/drivers/vfio/vfio.c > > > > > > > +++ b/drivers/vfio/vfio.c > > > > > > > @@ -2217,13 +2217,19 @@ struct vfio_mm > > > > > *vfio_mm_get_from_task(struct > > > > > > > task_struct *task) > > > > > > > } > > > > > > > EXPORT_SYMBOL_GPL(vfio_mm_get_from_task); > > > > > > > > > > > > > > -int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int min, int max) > > > > > > > +int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int quota, int min, > > > > > > > +int > > > > > max) > > > > > > > { > > > > > > > ioasid_t pasid; > > > > > > > int ret = -ENOSPC; > > > > > > > > > > > > > > mutex_lock(&vmm->pasid_lock); > > > > > > > > > > > > > > + /* update quota as it is tunable by admin */ > > > > > > > + if (vmm->pasid_quota != quota) { > > > > > > > + vmm->pasid_quota = quota; > > > > > > > + ioasid_adjust_set(vmm->ioasid_sid, quota); > > > > > > > + } > > > > > > > + > > > > > > > > > > > > It's a bit weird to have quota adjusted in the alloc path, since the > > > > > > latter > > > > > might > > > > > > be initiated by non-privileged users. Why not doing the simple math > > > > > > in > > > > > vfio_ > > > > > > create_mm to set the quota when the ioasid set is created? even in > > > > > > the > > > > > future > > > > > > you may allow per-process quota setting, that should come from > > > > > > separate privileged path instead of thru alloc.. > > > > > > > > > > The reason is the kernel parameter modification has no event which can > > > > > be used to adjust the quota. So I chose to adjust it in pasid_alloc > > > > > path. If it's not good, how about adding one more IOCTL to let user- > > > > > space trigger a quota adjustment event? Then even non-privileged user > > > > > could trigger quota adjustment, the quota is actually controlled by > > > > > privileged user. How about your opinion? > > > > > > > > > > > > > why do you need an event to adjust? As I said, you can set the quota when > > > the set is > > > > created in vfio_create_mm... > > > > > > oh, it's to support runtime adjustments. I guess it may be helpful to let > > > per-VM quota tunable even the VM is running. If just set the quota in > > > vfio_create_mm(), it is not able to adjust at runtime. > > > > > > > ok, I didn't note the module parameter was granted with a write permission. > > However there is a further problem. We cannot support PASID reclaim now. > > What about the admin sets a quota smaller than previous value while some > > IOASID sets already exceed the new quota? I'm not sure how to fail a runtime > > module parameter change due to that situation. possibly a normal sysfs > > node better suites the runtime change requirement... > > Yep, making this runtime adjustable seems a bit unpredictable and racy, > and it's not clear to me how a user is going to jump in at just the > right time for a user and adjust the limit. I'd probably go for a > simple non-runtime adjustable module option. It's a safety net at this > point anyway afaict. Thanks, thanks, I can do the changes. Regards, Yi Liu
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index d13b483..020a792 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -2217,13 +2217,19 @@ struct vfio_mm *vfio_mm_get_from_task(struct task_struct *task) } EXPORT_SYMBOL_GPL(vfio_mm_get_from_task); -int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int min, int max) +int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int quota, int min, int max) { ioasid_t pasid; int ret = -ENOSPC; mutex_lock(&vmm->pasid_lock); + /* update quota as it is tunable by admin */ + if (vmm->pasid_quota != quota) { + vmm->pasid_quota = quota; + ioasid_adjust_set(vmm->ioasid_sid, quota); + } + pasid = ioasid_alloc(vmm->ioasid_sid, min, max, NULL); if (pasid == INVALID_IOASID) { ret = -ENOSPC; diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 331ceee..e40afc0 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -60,6 +60,11 @@ module_param_named(dma_entry_limit, dma_entry_limit, uint, 0644); MODULE_PARM_DESC(dma_entry_limit, "Maximum number of user DMA mappings per container (65535)."); +static int pasid_quota = VFIO_DEFAULT_PASID_QUOTA; +module_param_named(pasid_quota, pasid_quota, uint, 0644); +MODULE_PARM_DESC(pasid_quota, + "Quota of user owned PASIDs per vfio-based application (1000)."); + struct vfio_iommu { struct list_head domain_list; struct list_head iova_list; @@ -2200,7 +2205,7 @@ static int vfio_iommu_type1_pasid_alloc(struct vfio_iommu *iommu, goto out_unlock; } if (vmm) - ret = vfio_mm_pasid_alloc(vmm, min, max); + ret = vfio_mm_pasid_alloc(vmm, pasid_quota, min, max); else ret = -EINVAL; out_unlock: diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 75f9f7f1..af2ef78 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -106,7 +106,8 @@ struct vfio_mm { extern struct vfio_mm *vfio_mm_get_from_task(struct task_struct *task); extern void vfio_mm_put(struct vfio_mm *vmm); -extern int vfio_mm_pasid_alloc(struct vfio_mm *vmm, int min, int max); +extern int vfio_mm_pasid_alloc(struct vfio_mm *vmm, + int quota, int min, int max); extern int vfio_mm_pasid_free(struct vfio_mm *vmm, ioasid_t pasid); /*