| Message ID | 20241022111017.946170-3-huangjunxian6@hisilicon.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | RDMA/hns: Bugfixes | expand |
在 2024/10/22 13:10, Junxian Huang 写道: > From: wenglianfa <wenglianfa@huawei.com> > > QP needs to be modified to IB_QPS_ERROR to trigger HW flush cqe. But > when this process races with destroy qp, the destroy-qp process may > modify the QP to IB_QPS_RESET first. In this case flush cqe will fail > since it is invalid to modify qp from IB_QPS_RESET to IB_QPS_ERROR. > > Add lock and bit flag to make sure pending flush cqe work is completed > first and no more new works will be added. > > Fixes: ffd541d45726 ("RDMA/hns: Add the workqueue framework for flush cqe handler") > Signed-off-by: wenglianfa <wenglianfa@huawei.com> > Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com> > --- > drivers/infiniband/hw/hns/hns_roce_device.h | 2 ++ > drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 7 +++++++ > drivers/infiniband/hw/hns/hns_roce_qp.c | 14 ++++++++++++-- > 3 files changed, 21 insertions(+), 2 deletions(-) > > diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h > index 73c78005901e..9b51d5a1533f 100644 > --- a/drivers/infiniband/hw/hns/hns_roce_device.h > +++ b/drivers/infiniband/hw/hns/hns_roce_device.h > @@ -593,6 +593,7 @@ struct hns_roce_dev; > > enum { > HNS_ROCE_FLUSH_FLAG = 0, > + HNS_ROCE_STOP_FLUSH_FLAG = 1, > }; > > struct hns_roce_work { > @@ -656,6 +657,7 @@ struct hns_roce_qp { > enum hns_roce_cong_type cong_type; > u8 tc_mode; > u8 priority; > + spinlock_t flush_lock; spin_lock_init is missing? The spin lock flush_lock should be initialized before used. Zhu Yanjun > }; > > struct hns_roce_ib_iboe { > diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c > index e85c450e1809..aa42c5a9b254 100644 > --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c > +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c > @@ -5598,8 +5598,15 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) > { > struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); > struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); > + unsigned long flags; > int ret; > > + /* Make sure flush_cqe() is completed */ > + spin_lock_irqsave(&hr_qp->flush_lock, flags); > + set_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag); > + spin_unlock_irqrestore(&hr_qp->flush_lock, flags); > + flush_work(&hr_qp->flush_work.work); > + > ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); > if (ret) > ibdev_err(&hr_dev->ib_dev, > diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c > index dcaa370d4a26..3439312b0138 100644 > --- a/drivers/infiniband/hw/hns/hns_roce_qp.c > +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c > @@ -90,11 +90,18 @@ static void flush_work_handle(struct work_struct *work) > void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) > { > struct hns_roce_work *flush_work = &hr_qp->flush_work; > + unsigned long flags; > + > + spin_lock_irqsave(&hr_qp->flush_lock, flags); > + /* Exit directly after destroy_qp() */ > + if (test_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag)) { > + spin_unlock_irqrestore(&hr_qp->flush_lock, flags); > + return; > + } > > - flush_work->hr_dev = hr_dev; > - INIT_WORK(&flush_work->work, flush_work_handle); > refcount_inc(&hr_qp->refcount); > queue_work(hr_dev->irq_workq, &flush_work->work); > + spin_unlock_irqrestore(&hr_qp->flush_lock, flags); > } > > void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp) > @@ -1140,6 +1147,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, > struct ib_udata *udata, > struct hns_roce_qp *hr_qp) > { > + struct hns_roce_work *flush_work = &hr_qp->flush_work; > struct hns_roce_ib_create_qp_resp resp = {}; > struct ib_device *ibdev = &hr_dev->ib_dev; > struct hns_roce_ib_create_qp ucmd = {}; > @@ -1151,6 +1159,8 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, > > hr_qp->state = IB_QPS_RESET; > hr_qp->flush_flag = 0; > + flush_work->hr_dev = hr_dev; > + INIT_WORK(&flush_work->work, flush_work_handle); > > if (init_attr->create_flags) > return -EOPNOTSUPP;
On 2024/10/23 23:13, Zhu Yanjun wrote: > 在 2024/10/22 13:10, Junxian Huang 写道: >> From: wenglianfa <wenglianfa@huawei.com> >> >> QP needs to be modified to IB_QPS_ERROR to trigger HW flush cqe. But >> when this process races with destroy qp, the destroy-qp process may >> modify the QP to IB_QPS_RESET first. In this case flush cqe will fail >> since it is invalid to modify qp from IB_QPS_RESET to IB_QPS_ERROR. >> >> Add lock and bit flag to make sure pending flush cqe work is completed >> first and no more new works will be added. >> >> Fixes: ffd541d45726 ("RDMA/hns: Add the workqueue framework for flush cqe handler") >> Signed-off-by: wenglianfa <wenglianfa@huawei.com> >> Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com> >> --- >> drivers/infiniband/hw/hns/hns_roce_device.h | 2 ++ >> drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 7 +++++++ >> drivers/infiniband/hw/hns/hns_roce_qp.c | 14 ++++++++++++-- >> 3 files changed, 21 insertions(+), 2 deletions(-) >> >> diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h >> index 73c78005901e..9b51d5a1533f 100644 >> --- a/drivers/infiniband/hw/hns/hns_roce_device.h >> +++ b/drivers/infiniband/hw/hns/hns_roce_device.h >> @@ -593,6 +593,7 @@ struct hns_roce_dev; >> enum { >> HNS_ROCE_FLUSH_FLAG = 0, >> + HNS_ROCE_STOP_FLUSH_FLAG = 1, >> }; >> struct hns_roce_work { >> @@ -656,6 +657,7 @@ struct hns_roce_qp { >> enum hns_roce_cong_type cong_type; >> u8 tc_mode; >> u8 priority; >> + spinlock_t flush_lock; > spin_lock_init is missing? > > The spin lock flush_lock should be initialized before used. > Will fix it. Thanks. Junxian > Zhu Yanjun >> }; >> struct hns_roce_ib_iboe { >> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c >> index e85c450e1809..aa42c5a9b254 100644 >> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c >> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c >> @@ -5598,8 +5598,15 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) >> { >> struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); >> struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); >> + unsigned long flags; >> int ret; >> + /* Make sure flush_cqe() is completed */ >> + spin_lock_irqsave(&hr_qp->flush_lock, flags); >> + set_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag); >> + spin_unlock_irqrestore(&hr_qp->flush_lock, flags); >> + flush_work(&hr_qp->flush_work.work); >> + >> ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); >> if (ret) >> ibdev_err(&hr_dev->ib_dev, >> diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c >> index dcaa370d4a26..3439312b0138 100644 >> --- a/drivers/infiniband/hw/hns/hns_roce_qp.c >> +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c >> @@ -90,11 +90,18 @@ static void flush_work_handle(struct work_struct *work) >> void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) >> { >> struct hns_roce_work *flush_work = &hr_qp->flush_work; >> + unsigned long flags; >> + >> + spin_lock_irqsave(&hr_qp->flush_lock, flags); >> + /* Exit directly after destroy_qp() */ >> + if (test_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag)) { >> + spin_unlock_irqrestore(&hr_qp->flush_lock, flags); >> + return; >> + } >> - flush_work->hr_dev = hr_dev; >> - INIT_WORK(&flush_work->work, flush_work_handle); >> refcount_inc(&hr_qp->refcount); >> queue_work(hr_dev->irq_workq, &flush_work->work); >> + spin_unlock_irqrestore(&hr_qp->flush_lock, flags); >> } >> void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp) >> @@ -1140,6 +1147,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, >> struct ib_udata *udata, >> struct hns_roce_qp *hr_qp) >> { >> + struct hns_roce_work *flush_work = &hr_qp->flush_work; >> struct hns_roce_ib_create_qp_resp resp = {}; >> struct ib_device *ibdev = &hr_dev->ib_dev; >> struct hns_roce_ib_create_qp ucmd = {}; >> @@ -1151,6 +1159,8 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, >> hr_qp->state = IB_QPS_RESET; >> hr_qp->flush_flag = 0; >> + flush_work->hr_dev = hr_dev; >> + INIT_WORK(&flush_work->work, flush_work_handle); >> if (init_attr->create_flags) >> return -EOPNOTSUPP; >
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 73c78005901e..9b51d5a1533f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -593,6 +593,7 @@ struct hns_roce_dev; enum { HNS_ROCE_FLUSH_FLAG = 0, + HNS_ROCE_STOP_FLUSH_FLAG = 1, }; struct hns_roce_work { @@ -656,6 +657,7 @@ struct hns_roce_qp { enum hns_roce_cong_type cong_type; u8 tc_mode; u8 priority; + spinlock_t flush_lock; }; struct hns_roce_ib_iboe { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index e85c450e1809..aa42c5a9b254 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -5598,8 +5598,15 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + unsigned long flags; int ret; + /* Make sure flush_cqe() is completed */ + spin_lock_irqsave(&hr_qp->flush_lock, flags); + set_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag); + spin_unlock_irqrestore(&hr_qp->flush_lock, flags); + flush_work(&hr_qp->flush_work.work); + ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); if (ret) ibdev_err(&hr_dev->ib_dev, diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index dcaa370d4a26..3439312b0138 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -90,11 +90,18 @@ static void flush_work_handle(struct work_struct *work) void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { struct hns_roce_work *flush_work = &hr_qp->flush_work; + unsigned long flags; + + spin_lock_irqsave(&hr_qp->flush_lock, flags); + /* Exit directly after destroy_qp() */ + if (test_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag)) { + spin_unlock_irqrestore(&hr_qp->flush_lock, flags); + return; + } - flush_work->hr_dev = hr_dev; - INIT_WORK(&flush_work->work, flush_work_handle); refcount_inc(&hr_qp->refcount); queue_work(hr_dev->irq_workq, &flush_work->work); + spin_unlock_irqrestore(&hr_qp->flush_lock, flags); } void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp) @@ -1140,6 +1147,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, struct ib_udata *udata, struct hns_roce_qp *hr_qp) { + struct hns_roce_work *flush_work = &hr_qp->flush_work; struct hns_roce_ib_create_qp_resp resp = {}; struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_ib_create_qp ucmd = {}; @@ -1151,6 +1159,8 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, hr_qp->state = IB_QPS_RESET; hr_qp->flush_flag = 0; + flush_work->hr_dev = hr_dev; + INIT_WORK(&flush_work->work, flush_work_handle); if (init_attr->create_flags) return -EOPNOTSUPP;