@@ -2041,7 +2041,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
u32 hdrsize = packet->hlen;
u32 psn = ib_bth_get_psn(packet->ohdr);
u32 pad = packet->pad;
- struct ib_wc wc;
+ struct rvt_wc wc;
u32 pmtu = qp->pmtu;
int diff;
struct ib_reth *reth;
@@ -173,7 +173,7 @@ static void ruc_loopback(struct rvt_qp *sqp)
struct rvt_swqe *wqe;
struct rvt_sge *sge;
unsigned long flags;
- struct ib_wc wc;
+ struct rvt_wc wc;
u64 sdata;
atomic64_t *maddr;
enum ib_wc_status send_status;
@@ -312,7 +312,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet)
u32 hdrsize = packet->hlen;
u32 psn;
u32 pad = packet->pad;
- struct ib_wc wc;
+ struct rvt_wc wc;
u32 pmtu = qp->pmtu;
struct ib_reth *reth;
int ret;
@@ -79,7 +79,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
unsigned long flags;
struct rvt_sge_state ssge;
struct rvt_sge *sge;
- struct ib_wc wc;
+ struct rvt_wc wc;
u32 length;
enum ib_qp_type sqptype, dqptype;
@@ -867,7 +867,7 @@ static int opa_smp_check(struct hfi1_ibport *ibp, u16 pkey, u8 sc5,
void hfi1_ud_rcv(struct hfi1_packet *packet)
{
u32 hdrsize = packet->hlen;
- struct ib_wc wc;
+ struct rvt_wc wc;
u32 src_qp;
u16 pkey;
int mgmt_pkey_idx = -1;
@@ -1744,7 +1744,7 @@ void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
u32 hdrsize;
u32 psn;
u32 pad;
- struct ib_wc wc;
+ struct rvt_wc wc;
u32 pmtu = qp->pmtu;
int diff;
struct ib_reth *reth;
@@ -191,7 +191,7 @@ static void qib_ruc_loopback(struct rvt_qp *sqp)
struct rvt_swqe *wqe;
struct rvt_sge *sge;
unsigned long flags;
- struct ib_wc wc;
+ struct rvt_wc wc;
u64 sdata;
atomic64_t *maddr;
enum ib_wc_status send_status;
@@ -242,7 +242,7 @@ void qib_uc_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
u32 hdrsize;
u32 psn;
u32 pad;
- struct ib_wc wc;
+ struct rvt_wc wc;
u32 pmtu = qp->pmtu;
struct ib_reth *reth;
int ret;
@@ -58,7 +58,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
unsigned long flags;
struct rvt_sge_state ssge;
struct rvt_sge *sge;
- struct ib_wc wc;
+ struct rvt_wc wc;
u32 length;
enum ib_qp_type sqptype, dqptype;
@@ -434,7 +434,7 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
int opcode;
u32 hdrsize;
u32 pad;
- struct ib_wc wc;
+ struct rvt_wc wc;
u32 qkey;
u32 src_qp;
u16 dlid;
@@ -62,10 +62,10 @@
*
* This may be called with qp->s_lock held.
*/
-void rvt_cq_enter(struct rvt_cq *cq, struct ib_wc *entry, bool solicited)
+void rvt_cq_enter(struct rvt_cq *cq, struct rvt_wc *entry, bool solicited)
{
struct ib_uverbs_wc *uqueue = NULL;
- struct ib_wc *kqueue = NULL;
+ struct rvt_wc *kqueue = NULL;
struct rvt_cq_wc *u_wc = NULL;
struct rvt_k_cq_wc *k_wc = NULL;
unsigned long flags;
@@ -230,6 +230,8 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
* numbers of entries.
*/
if (udata && udata->outlen >= sizeof(__u64)) {
+ int err;
+
sz = sizeof(struct ib_uverbs_wc) * (entries + 1);
sz += sizeof(*u_wc);
u_wc = vmalloc_user(sz);
@@ -237,23 +239,11 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
ret = ERR_PTR(-ENOMEM);
goto bail_cq;
}
- } else {
- sz = sizeof(struct ib_wc) * (entries + 1);
- sz += sizeof(*k_wc);
- k_wc = vzalloc_node(sz, rdi->dparms.node);
- if (!k_wc) {
- ret = ERR_PTR(-ENOMEM);
- goto bail_cq;
- }
- }
-
- /*
- * Return the address of the WC as the offset to mmap.
- * See rvt_mmap() for details.
- */
- if (udata && udata->outlen >= sizeof(__u64)) {
- int err;
+ /*
+ * Return the address of the WC as the offset to mmap.
+ * See rvt_mmap() for details.
+ */
cq->ip = rvt_create_mmap_info(rdi, sz, context, u_wc);
if (!cq->ip) {
ret = ERR_PTR(-ENOMEM);
@@ -266,6 +256,14 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
ret = ERR_PTR(err);
goto bail_ip;
}
+ } else {
+ sz = sizeof(struct rvt_wc) * (entries + 1);
+ sz += sizeof(*k_wc);
+ k_wc = vzalloc_node(sz, rdi->dparms.node);
+ if (!k_wc) {
+ ret = ERR_PTR(-ENOMEM);
+ goto bail_cq;
+ }
}
spin_lock_irq(&rdi->n_cqs_lock);
@@ -343,6 +341,7 @@ int rvt_destroy_cq(struct ib_cq *ibcq)
kref_put(&cq->ip->ref, rvt_release_mmap_info);
else
vfree(cq->queue);
+ vfree(cq->kqueue);
kfree(cq);
return 0;
@@ -418,7 +417,7 @@ int rvt_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
if (!u_wc)
return -ENOMEM;
} else {
- sz = sizeof(struct ib_wc) * (cqe + 1);
+ sz = sizeof(struct rvt_wc) * (cqe + 1);
sz += sizeof(*k_wc);
k_wc = vzalloc_node(sz, rdi->dparms.node);
if (!k_wc)
@@ -520,6 +519,24 @@ int rvt_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
return ret;
}
+static void copy_rvt_wc_to_ib_wc(struct ib_wc *ibwc, struct rvt_wc *rvtwc)
+{
+ ibwc->wr_id = rvtwc->wr_id;
+ ibwc->status = rvtwc->status;
+ ibwc->opcode = rvtwc->opcode;
+ ibwc->vendor_err = rvtwc->vendor_err;
+ ibwc->byte_len = rvtwc->byte_len;
+ ibwc->qp = rvtwc->qp;
+ ibwc->ex.invalidate_rkey = rvtwc->ex.invalidate_rkey;
+ ibwc->src_qp = rvtwc->src_qp;
+ ibwc->wc_flags = rvtwc->wc_flags;
+ ibwc->slid = rvtwc->slid;
+ ibwc->pkey_index = rvtwc->pkey_index;
+ ibwc->sl = rvtwc->sl;
+ ibwc->dlid_path_bits = rvtwc->dlid_path_bits;
+ ibwc->port_num = rvtwc->port_num;
+}
+
/**
* rvt_poll_cq - poll for work completion entries
* @ibcq: the completion queue to poll
@@ -554,7 +571,7 @@ int rvt_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
break;
/* The kernel doesn't need a RMB since it has the lock. */
trace_rvt_cq_poll(cq, &wc->kqueue[tail], npolled);
- *entry = wc->kqueue[tail];
+ copy_rvt_wc_to_ib_wc(entry, &wc->kqueue[tail]);
if (tail >= cq->ibcq.cqe)
tail = 0;
else
@@ -1049,7 +1049,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
*/
int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err)
{
- struct ib_wc wc;
+ struct rvt_wc wc;
int ret = 0;
struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
@@ -1573,7 +1573,7 @@ int rvt_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
return -ENOMEM;
}
if (unlikely(qp_err_flush)) {
- struct ib_wc wc;
+ struct rvt_wc wc;
memset(&wc, 0, sizeof(wc));
wc.qp = &qp->ibqp;
@@ -1996,7 +1996,7 @@ int rvt_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe)
{
int i, j, ret;
- struct ib_wc wc;
+ struct rvt_wc wc;
struct rvt_lkey_table *rkt;
struct rvt_pd *pd;
struct rvt_sge_state *ss;
@@ -109,7 +109,7 @@
DECLARE_EVENT_CLASS(
rvt_cq_entry_template,
- TP_PROTO(struct rvt_cq *cq, struct ib_wc *wc, u32 idx),
+ TP_PROTO(struct rvt_cq *cq, struct rvt_wc *wc, u32 idx),
TP_ARGS(cq, wc, idx),
TP_STRUCT__entry(
RDI_DEV_ENTRY(cq->rdi)
@@ -143,12 +143,12 @@
DEFINE_EVENT(
rvt_cq_entry_template, rvt_cq_enter,
- TP_PROTO(struct rvt_cq *cq, struct ib_wc *wc, u32 idx),
+ TP_PROTO(struct rvt_cq *cq, struct rvt_wc *wc, u32 idx),
TP_ARGS(cq, wc, idx));
DEFINE_EVENT(
rvt_cq_entry_template, rvt_cq_poll,
- TP_PROTO(struct rvt_cq *cq, struct ib_wc *wc, u32 idx),
+ TP_PROTO(struct rvt_cq *cq, struct rvt_wc *wc, u32 idx),
TP_ARGS(cq, wc, idx));
#endif /* __RVT_TRACE_CQ_H */
@@ -62,6 +62,30 @@
#include <rdma/rvt-abi.h>
/*
+ * If any fields within struct rvt_wc change, the function
+ * copy_rvt_wc_to_ib_wc() should be updated.
+ */
+struct rvt_wc {
+ u64 wr_id;
+ enum ib_wc_status status;
+ enum ib_wc_opcode opcode;
+ u32 vendor_err;
+ u32 byte_len;
+ struct ib_qp *qp;
+ union {
+ __be32 imm_data;
+ u32 invalidate_rkey;
+ } ex;
+ u32 src_qp;
+ int wc_flags;
+ u32 slid;
+ u16 pkey_index;
+ u8 sl;
+ u8 dlid_path_bits;
+ u8 port_num; /* valid only for DR SMPs onswitches*/
+} ____cacheline_aligned_in_smp;
+
+/*
* This structure is used to contain the head pointer, tail pointer,
* and completion queue entries as a single memory allocation so
* it can be mmap'ed into user space.
@@ -69,7 +93,8 @@
struct rvt_k_cq_wc {
u32 head; /* index of next entry to fill */
u32 tail; /* index of next ib_poll_cq() entry */
- struct ib_wc kqueue[0];
+ /* this is actually size ibcq.cqe + 1 */
+ struct rvt_wc kqueue[0];
};
/*
@@ -93,6 +118,6 @@ struct rvt_cq {
return container_of(ibcq, struct rvt_cq, ibcq);
}
-void rvt_cq_enter(struct rvt_cq *cq, struct ib_wc *entry, bool solicited);
+void rvt_cq_enter(struct rvt_cq *cq, struct rvt_wc *entry, bool solicited);
#endif /* DEF_RDMAVT_INCCQH */
@@ -589,7 +589,7 @@ static inline void rvt_qp_swqe_complete(
if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
(wqe->wr.send_flags & IB_SEND_SIGNALED) ||
status != IB_WC_SUCCESS) {
- struct ib_wc wc;
+ struct rvt_wc wc;
memset(&wc, 0, sizeof(wc));
wc.wr_id = wqe->wr.wr_id;