@@ -386,6 +386,23 @@ static void free_dca_mem(struct dca_mem *mem)
spin_unlock(&mem->lock);
}
+int hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
+{
+ struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;
+
+ cfg->buf_id = HNS_DCA_INVALID_BUF_ID;
+
+ return 0;
+}
+
+void hns_roce_disable_dca(struct hns_roce_dev *hr_dev,
+ struct hns_roce_qp *hr_qp)
+{
+ struct hns_roce_dca_cfg *cfg = &hr_qp->dca_cfg;
+
+ cfg->buf_id = HNS_DCA_INVALID_BUF_ID;
+}
+
static inline struct hns_roce_ucontext *
uverbs_attr_to_hr_uctx(struct uverbs_attr_bundle *attrs)
{
@@ -26,4 +26,7 @@ void hns_roce_register_udca(struct hns_roce_dev *hr_dev,
void hns_roce_unregister_udca(struct hns_roce_dev *hr_dev,
struct hns_roce_ucontext *uctx);
+int hns_roce_enable_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
+void hns_roce_disable_dca(struct hns_roce_dev *hr_dev,
+ struct hns_roce_qp *hr_qp);
#endif
@@ -367,6 +367,10 @@ struct hns_roce_mtr {
struct hns_roce_hem_cfg hem_cfg; /* config for hardware addressing */
};
+struct hns_roce_dca_cfg {
+ u32 buf_id;
+};
+
struct hns_roce_mw {
struct ib_mw ibmw;
u32 pdn;
@@ -664,6 +668,7 @@ struct hns_roce_qp {
struct hns_roce_wq sq;
struct hns_roce_mtr mtr;
+ struct hns_roce_dca_cfg dca_cfg;
u32 buff_size;
struct mutex mutex;
@@ -346,6 +346,11 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
return 0;
}
+static inline bool check_qp_dca_enable(struct hns_roce_qp *hr_qp)
+{
+ return !!(hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA);
+}
+
static int check_send_valid(struct hns_roce_dev *hr_dev,
struct hns_roce_qp *hr_qp)
{
@@ -4290,6 +4295,21 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
V2_QPC_BYTE_140_TRRL_BA_S, 0);
+ /* hip09 reused the IRRL_HEAD fileds in hip08 */
+ if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
+ if (check_qp_dca_enable(hr_qp)) {
+ roce_set_bit(context->byte_196_sq_psn,
+ V2_QPC_BYTE_196_DCA_MODE_S, 1);
+ roce_set_bit(qpc_mask->byte_196_sq_psn,
+ V2_QPC_BYTE_196_DCA_MODE_S, 0);
+ }
+ } else {
+ /* reset IRRL_HEAD */
+ roce_set_field(qpc_mask->byte_196_sq_psn,
+ V2_QPC_BYTE_196_IRRL_HEAD_M,
+ V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
+ }
+
context->irrl_ba = cpu_to_le32(irrl_ba >> 6);
qpc_mask->irrl_ba = 0;
roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
@@ -4456,9 +4476,6 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
V2_QPC_BYTE_212_LSN_S, 0);
- roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
- V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
-
return 0;
}
@@ -796,6 +796,8 @@ struct hns_roce_v2_qp_context {
#define V2_QPC_BYTE_196_IRRL_HEAD_S 0
#define V2_QPC_BYTE_196_IRRL_HEAD_M GENMASK(7, 0)
+#define V2_QPC_BYTE_196_DCA_MODE_S 6
+
#define V2_QPC_BYTE_196_SQ_MAX_PSN_S 8
#define V2_QPC_BYTE_196_SQ_MAX_PSN_M GENMASK(31, 8)
@@ -39,6 +39,7 @@
#include "hns_roce_common.h"
#include "hns_roce_device.h"
#include "hns_roce_hem.h"
+#include "hns_roce_dca.h"
static void flush_work_handle(struct work_struct *work)
{
@@ -553,8 +554,21 @@ static int set_user_sq_size(struct hns_roce_dev *hr_dev,
return 0;
}
+static bool check_dca_is_enable(struct hns_roce_dev *hr_dev, bool is_user,
+ unsigned long addr)
+{
+ if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DCA_MODE))
+ return false;
+
+ /* If the user QP's buffer addr is 0, the DCA mode should be enabled */
+ if (is_user)
+ return !addr;
+
+ return false;
+}
+
static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev,
- struct hns_roce_qp *hr_qp,
+ struct hns_roce_qp *hr_qp, bool dca_en,
struct hns_roce_buf_attr *buf_attr)
{
int buf_size;
@@ -598,10 +612,22 @@ static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev,
if (hr_qp->buff_size < 1)
return -EINVAL;
- buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
buf_attr->fixed_page = true;
buf_attr->region_count = idx;
+ if (dca_en) {
+ /*
+ * When enable DCA, there's no need to alloc buffer now, and
+ * the page shift should be fixed to 4K.
+ */
+ buf_attr->mtt_only = true;
+ buf_attr->page_shift = HNS_HW_PAGE_SHIFT;
+ } else {
+ buf_attr->mtt_only = false;
+ buf_attr->page_shift = HNS_HW_PAGE_SHIFT +
+ hr_dev->caps.mtt_buf_pg_sz;
+ }
+
return 0;
}
@@ -700,12 +726,53 @@ static void free_rq_inline_buf(struct hns_roce_qp *hr_qp)
kfree(hr_qp->rq_inl_buf.wqe_list);
}
-static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
+static int alloc_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
+ bool dca_en, struct hns_roce_buf_attr *buf_attr,
+ struct ib_udata *udata, unsigned long addr)
+{
+ struct ib_device *ibdev = &hr_dev->ib_dev;
+ int ret;
+
+ if (dca_en) {
+ /* DCA must be enabled after the buffer size is configured. */
+ ret = hns_roce_enable_dca(hr_dev, hr_qp);
+ if (ret) {
+ ibdev_err(ibdev, "failed to enable DCA, ret = %d.\n",
+ ret);
+ return ret;
+ }
+
+ hr_qp->en_flags |= HNS_ROCE_QP_CAP_DCA;
+ }
+
+ ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, buf_attr,
+ HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
+ udata, addr);
+ if (ret) {
+ ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
+ if (dca_en)
+ hns_roce_disable_dca(hr_dev, hr_qp);
+ }
+
+ return ret;
+}
+
+static void free_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
+ struct ib_udata *udata)
+{
+ hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);
+
+ if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DCA)
+ hns_roce_disable_dca(hr_dev, hr_qp);
+}
+
+static int alloc_qp_wqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
struct ib_qp_init_attr *init_attr,
struct ib_udata *udata, unsigned long addr)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_buf_attr buf_attr = {};
+ bool dca_en;
int ret;
if (!udata && hr_qp->rq_inl_buf.wqe_cnt) {
@@ -720,16 +787,16 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
hr_qp->rq_inl_buf.wqe_list = NULL;
}
- ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr);
+ dca_en = check_dca_is_enable(hr_dev, !!udata, addr);
+ ret = set_wqe_buf_attr(hr_dev, hr_qp, dca_en, &buf_attr);
if (ret) {
- ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);
+ ibdev_err(ibdev, "failed to set WQE attr, ret = %d.\n", ret);
goto err_inline;
}
- ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
- HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
- udata, addr);
+
+ ret = alloc_wqe_buf(hr_dev, hr_qp, dca_en, &buf_attr, udata, addr);
if (ret) {
- ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
+ ibdev_err(ibdev, "failed to alloc WQE buf, ret = %d.\n", ret);
goto err_inline;
}
@@ -740,9 +807,10 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
return ret;
}
-static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
+static void free_qp_wqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
+ struct ib_udata *udata)
{
- hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);
+ free_wqe_buf(hr_dev, hr_qp, udata);
free_rq_inline_buf(hr_qp);
}
@@ -800,7 +868,6 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
goto err_out;
}
hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
- resp->cap_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
}
if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) {
@@ -813,7 +880,6 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
goto err_sdb;
}
hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
- resp->cap_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
}
} else {
/* QP doorbell register address */
@@ -987,22 +1053,22 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
}
}
- ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp);
+ ret = alloc_qpn(hr_dev, hr_qp);
if (ret) {
- ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n",
- ret);
+ ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret);
goto err_wrid;
}
- ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr);
+ ret = alloc_qp_wqe(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr);
if (ret) {
ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret);
- goto err_db;
+ goto err_qpn;
}
- ret = alloc_qpn(hr_dev, hr_qp);
+ ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp);
if (ret) {
- ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret);
+ ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n",
+ ret);
goto err_buf;
}
@@ -1010,7 +1076,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
if (ret) {
ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n",
ret);
- goto err_qpn;
+ goto err_db;
}
ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr);
@@ -1020,6 +1086,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
}
if (udata) {
+ resp.cap_flags = hr_qp->en_flags;
ret = ib_copy_to_udata(udata, &resp,
min(udata->outlen, sizeof(resp)));
if (ret) {
@@ -1045,12 +1112,12 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
hns_roce_qp_remove(hr_dev, hr_qp);
err_qpc:
free_qpc(hr_dev, hr_qp);
-err_qpn:
- free_qpn(hr_dev, hr_qp);
-err_buf:
- free_qp_buf(hr_dev, hr_qp);
err_db:
free_qp_db(hr_dev, hr_qp, udata);
+err_buf:
+ free_qp_wqe(hr_dev, hr_qp, udata);
+err_qpn:
+ free_qpn(hr_dev, hr_qp);
err_wrid:
free_kernel_wrid(hr_qp);
return ret;
@@ -1065,7 +1132,7 @@ void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
free_qpc(hr_dev, hr_qp);
free_qpn(hr_dev, hr_qp);
- free_qp_buf(hr_dev, hr_qp);
+ free_qp_wqe(hr_dev, hr_qp, udata);
free_kernel_wrid(hr_qp);
free_qp_db(hr_dev, hr_qp, udata);
@@ -77,6 +77,7 @@ enum hns_roce_qp_cap_flags {
HNS_ROCE_QP_CAP_RQ_RECORD_DB = 1 << 0,
HNS_ROCE_QP_CAP_SQ_RECORD_DB = 1 << 1,
HNS_ROCE_QP_CAP_OWNER_DB = 1 << 2,
+ HNS_ROCE_QP_CAP_DCA = 1 << 4,
};
struct hns_roce_ib_create_qp_resp {