diff mbox series

[for-next,v6,04/10] RDMA/rxe: Add ib_alloc_mw and ib_dealloc_mw verbs

Message ID 20210429184855.54939-5-rpearson@hpe.com (mailing list archive)
State Superseded
Headers show
Series RDMA/rxe: Implement memory windows | expand

Commit Message

Bob Pearson April 29, 2021, 6:48 p.m. UTC 
Add ib_alloc_mw and ib_dealloc_mw verbs APIs.

Added new file rxe_mw.c focused on MWs.
Changed the 8 bit random key generator.
Added a cleanup routine for MWs.
Added verbs routines to ib_device_ops.

Signed-off-by: Bob Pearson <rpearson@hpe.com>
---
 drivers/infiniband/sw/rxe/Makefile    |  1 +
 drivers/infiniband/sw/rxe/rxe_loc.h   |  6 +++
 drivers/infiniband/sw/rxe/rxe_mr.c    | 20 +++++-----
 drivers/infiniband/sw/rxe/rxe_mw.c    | 53 +++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_pool.c  |  1 +
 drivers/infiniband/sw/rxe/rxe_verbs.c |  3 ++
 drivers/infiniband/sw/rxe/rxe_verbs.h |  2 +
 7 files changed, 75 insertions(+), 11 deletions(-)
 create mode 100644 drivers/infiniband/sw/rxe/rxe_mw.c

Comments

Jason Gunthorpe May 18, 2021, 8:09 p.m. UTC | #1 
On Thu, Apr 29, 2021 at 01:48:49PM -0500, Bob Pearson wrote:
> @@ -1106,6 +1108,7 @@ static const struct ib_device_ops rxe_dev_ops = {
>  
>  	INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
>  	INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
> +	INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
>  	INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
>  	INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
>  	INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),

ib_mw is already listed below:

 	INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),

?

You added it in

 RDMA/rxe: Split MEM into MR and MW

Jason
Zhu Yanjun May 19, 2021, 8:18 a.m. UTC | #2 
Hi, Bob

Based on the latest rdma

https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git
6efb943b8616 (tag: v5.13-rc1, origin/wip/for-testing, origin/linus,
origin/for-next, origin/HEAD) Linux 5.13-rc1

and rdma-core

https://github.com/linux-rdma/rdma-core.git
53d52f54 (HEAD -> master, origin/master, origin/HEAD) Merge pull
request #1003 from yishaih/mlx5_dv

After applying your patch series:

19d8361223ff (HEAD -> for-next) RDMA/rxe: Disallow MR dereg and
invalidate when bound
31f6a56c0039 RDMA/rxe: Implement memory access through MWs
c6684f842a96 RDMA/rxe: Implement invalidate MW operations
ac107e28e8ce RDMA/rxe: Add support for bind MW work requests
a93e19c5bf98 RDMA/rxe: Move local ops to subroutine
ab284077b5a1 RDMA/rxe: Replace WR_REG_MASK by WR_LOCAL_OP_MASK
e7f66570e3a1 RDMA/rxe: Add ib_alloc_mw and ib_dealloc_mw verbs
959efd04eff5 RDMA/rxe: Enable MW object pool
6860a2129427 RDMA/rxe: Return errors for add index and key
4cda80602bfa RDMA/rxe: Add bind MW fields to rxe_send_wr

Then I run "run_tests.py", the following test results. Can you have
time to check it?

======================================================================
ERROR: test_cq_events_rc (tests.test_cq_events.CqEventsTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_cq_events.py", line 54, in test_cq_events_rc
    traffic(client, server, self.iters, self.gid_index, self.ib_port)
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_cq_events_ud (tests.test_cq_events.CqEventsTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_cq_events.py", line 50, in test_cq_events_ud
    traffic(client, server, self.iters, self.gid_index, self.ib_port)
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_rc_traffic_cq_ex (tests.test_cqex.CqExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_cqex.py", line 73, in test_rc_traffic_cq_ex
    u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 567, in poll_cq_ex
    raise PyverbsRDMAErrno('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is 4. Errno:
28, No space left on device

======================================================================
ERROR: test_ud_traffic_cq_ex (tests.test_cqex.CqExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_cqex.py", line 68, in test_ud_traffic_cq_ex
    u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 567, in poll_cq_ex
    raise PyverbsRDMAErrno('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is 4. Errno:
28, No space left on device

======================================================================
ERROR: test_mr_rereg_access_bad_flow (tests.test_mr.MRTest)
Test that cover rereg MR's access with this flow:
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_mr.py", line 129, in
test_mr_rereg_access_bad_flow
    u.rdma_traffic(**self.traffic_args, send_op=e.IBV_WR_RDMA_WRITE)
  File "/root/rdma-core/tests/utils.py", line 869, in rdma_traffic
    poll_cq(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_mr_rereg_pd (tests.test_mr.MRTest)
Test that cover rereg MR's PD with this flow:
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_mr.py", line 145, in test_mr_rereg_pd
    u.traffic(**self.traffic_args)
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_qp_ex_rc_atomic_cmp_swp (tests.test_qpex.QpExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_qpex.py", line 249, in
test_qp_ex_rc_atomic_cmp_swp
    u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 869, in rdma_traffic
    poll_cq(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Remote
access error. Errno: 10, No child processes

======================================================================
ERROR: test_qp_ex_rc_atomic_fetch_add (tests.test_qpex.QpExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_qpex.py", line 261, in
test_qp_ex_rc_atomic_fetch_add
    u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 869, in rdma_traffic
    poll_cq(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Remote
access error. Errno: 10, No child processes

======================================================================
ERROR: test_qp_ex_rc_rdma_read (tests.test_qpex.QpExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_qpex.py", line 237, in
test_qp_ex_rc_rdma_read
    u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 869, in rdma_traffic
    poll_cq(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Remote
access error. Errno: 10, No child processes

======================================================================
ERROR: test_qp_ex_rc_rdma_write (tests.test_qpex.QpExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_qpex.py", line 218, in
test_qp_ex_rc_rdma_write
    u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 869, in rdma_traffic
    poll_cq(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_qp_ex_rc_rdma_write_imm (tests.test_qpex.QpExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_qpex.py", line 227, in
test_qp_ex_rc_rdma_write_imm
    u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_qp_ex_rc_send (tests.test_qpex.QpExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_qpex.py", line 191, in test_qp_ex_rc_send
    u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_qp_ex_rc_send_imm (tests.test_qpex.QpExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_qpex.py", line 205, in test_qp_ex_rc_send_imm
    u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_qp_ex_ud_send (tests.test_qpex.QpExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_qpex.py", line 186, in test_qp_ex_ud_send
    u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_qp_ex_ud_send_imm (tests.test_qpex.QpExTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_qpex.py", line 200, in test_qp_ex_ud_send_imm
    u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_ro_rc_traffic (tests.test_relaxed_ordering.RoTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_relaxed_ordering.py", line 52, in
test_ro_rc_traffic
    traffic(client, server, self.iters, self.gid_index, self.ib_port)
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
ERROR: test_ro_ud_traffic (tests.test_relaxed_ordering.RoTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_relaxed_ordering.py", line 56, in
test_ro_ud_traffic
    traffic(client, server, self.iters, self.gid_index, self.ib_port)
  File "/root/rdma-core/tests/utils.py", line 667, in traffic
    poll(client.cq)
  File "/root/rdma-core/tests/utils.py", line 538, in poll_cq
    raise PyverbsRDMAError('Completion status is {s}'.
pyverbs.pyverbs_error.PyverbsRDMAError: Completion status is Local
protection error. Errno: 4, Interrupted system call

======================================================================
FAIL: test_resize_cq (tests.test_cq.CQTest)
Test resize CQ, start with specific value and then increase and decrease
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/root/rdma-core/tests/test_cq.py", line 132, in test_resize_cq
    self.client.cq.resize(new_cq_size)
AssertionError: PyverbsRDMAError not raised

----------------------------------------------------------------------
Ran 183 tests in 2.071s

FAILED (failures=1, errors=17, skipped=125)

On Wed, May 19, 2021 at 4:09 AM Jason Gunthorpe <jgg@nvidia.com> wrote:
>
> On Thu, Apr 29, 2021 at 01:48:49PM -0500, Bob Pearson wrote:
> > @@ -1106,6 +1108,7 @@ static const struct ib_device_ops rxe_dev_ops = {
> >
> >       INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
> >       INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
> > +     INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
> >       INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
> >       INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
> >       INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
>
> ib_mw is already listed below:
>
>         INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
>
> ?
>
> You added it in
>
>  RDMA/rxe: Split MEM into MR and MW
>
> Jason
diff mbox series

Patch

diff --git a/drivers/infiniband/sw/rxe/Makefile b/drivers/infiniband/sw/rxe/Makefile
index 66af72dca759..1e24673e9318 100644
--- a/drivers/infiniband/sw/rxe/Makefile
+++ b/drivers/infiniband/sw/rxe/Makefile
@@ -15,6 +15,7 @@  rdma_rxe-y := \
 	rxe_qp.o \
 	rxe_cq.o \
 	rxe_mr.o \
+	rxe_mw.o \
 	rxe_opcode.o \
 	rxe_mmap.o \
 	rxe_icrc.o \
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index ef8061d2fbe0..edf575930a98 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -76,6 +76,7 @@  enum copy_direction {
 	from_mr_obj,
 };
 
+u8 rxe_get_next_key(u32 last_key);
 void rxe_mr_init_dma(struct rxe_pd *pd, int access, struct rxe_mr *mr);
 
 int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova,
@@ -106,6 +107,11 @@  void rxe_mr_cleanup(struct rxe_pool_entry *arg);
 
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
 
+/* rxe_mw.c */
+int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
+int rxe_dealloc_mw(struct ib_mw *ibmw);
+void rxe_mw_cleanup(struct rxe_pool_entry *arg);
+
 /* rxe_net.c */
 void rxe_loopback(struct sk_buff *skb);
 int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb);
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index 9f63947bab12..7f2cfc1ce659 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -7,19 +7,17 @@ 
 #include "rxe.h"
 #include "rxe_loc.h"
 
-/*
- * lfsr (linear feedback shift register) with period 255
+/* Return a random 8 bit key value that is
+ * different than the last_key. Set last_key to -1
+ * if this is the first key for an MR or MW
  */
-static u8 rxe_get_key(void)
+u8 rxe_get_next_key(u32 last_key)
 {
-	static u32 key = 1;
-
-	key = key << 1;
-
-	key |= (0 != (key & 0x100)) ^ (0 != (key & 0x10))
-		^ (0 != (key & 0x80)) ^ (0 != (key & 0x40));
+	u8 key;
 
-	key &= 0xff;
+	do {
+		get_random_bytes(&key, 1);
+	} while (key == last_key);
 
 	return key;
 }
@@ -47,7 +45,7 @@  int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 
 static void rxe_mr_init(int access, struct rxe_mr *mr)
 {
-	u32 lkey = mr->pelem.index << 8 | rxe_get_key();
+	u32 lkey = mr->pelem.index << 8 | rxe_get_next_key(-1);
 	u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;
 
 	mr->ibmr.lkey = lkey;
diff --git a/drivers/infiniband/sw/rxe/rxe_mw.c b/drivers/infiniband/sw/rxe/rxe_mw.c
new file mode 100644
index 000000000000..69128e298d44
--- /dev/null
+++ b/drivers/infiniband/sw/rxe/rxe_mw.c
@@ -0,0 +1,53 @@ 
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2020 Hewlett Packard Enterprise, Inc. All rights reserved.
+ */
+
+#include "rxe.h"
+
+int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
+{
+	struct rxe_mw *mw = to_rmw(ibmw);
+	struct rxe_pd *pd = to_rpd(ibmw->pd);
+	struct rxe_dev *rxe = to_rdev(ibmw->device);
+	int ret;
+
+	rxe_add_ref(pd);
+
+	ret = rxe_add_to_pool(&rxe->mw_pool, mw);
+	if (ret) {
+		rxe_drop_ref(pd);
+		return ret;
+	}
+
+	rxe_add_index(mw);
+	ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1);
+	mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?
+			RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;
+	spin_lock_init(&mw->lock);
+
+	return 0;
+}
+
+int rxe_dealloc_mw(struct ib_mw *ibmw)
+{
+	struct rxe_mw *mw = to_rmw(ibmw);
+	struct rxe_pd *pd = to_rpd(ibmw->pd);
+	unsigned long flags;
+
+	spin_lock_irqsave(&mw->lock, flags);
+	mw->state = RXE_MW_STATE_INVALID;
+	spin_unlock_irqrestore(&mw->lock, flags);
+
+	rxe_drop_ref(mw);
+	rxe_drop_ref(pd);
+
+	return 0;
+}
+
+void rxe_mw_cleanup(struct rxe_pool_entry *elem)
+{
+	struct rxe_mw *mw = container_of(elem, typeof(*mw), pelem);
+
+	rxe_drop_index(mw);
+}
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
index df0bec719341..0b8e7c6255a2 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.c
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -65,6 +65,7 @@  struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 		.name		= "rxe-mw",
 		.size		= sizeof(struct rxe_mw),
 		.elem_offset	= offsetof(struct rxe_mw, pelem),
+		.cleanup	= rxe_mw_cleanup,
 		.flags		= RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
 		.max_index	= RXE_MAX_MW_INDEX,
 		.min_index	= RXE_MIN_MW_INDEX,
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index aeb5e232c195..fff81bf78a86 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -1060,6 +1060,7 @@  static const struct ib_device_ops rxe_dev_ops = {
 
 	.alloc_hw_stats = rxe_ib_alloc_hw_stats,
 	.alloc_mr = rxe_alloc_mr,
+	.alloc_mw = rxe_alloc_mw,
 	.alloc_pd = rxe_alloc_pd,
 	.alloc_ucontext = rxe_alloc_ucontext,
 	.attach_mcast = rxe_attach_mcast,
@@ -1069,6 +1070,7 @@  static const struct ib_device_ops rxe_dev_ops = {
 	.create_srq = rxe_create_srq,
 	.create_user_ah = rxe_create_ah,
 	.dealloc_driver = rxe_dealloc,
+	.dealloc_mw = rxe_dealloc_mw,
 	.dealloc_pd = rxe_dealloc_pd,
 	.dealloc_ucontext = rxe_dealloc_ucontext,
 	.dereg_mr = rxe_dereg_mr,
@@ -1106,6 +1108,7 @@  static const struct ib_device_ops rxe_dev_ops = {
 
 	INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
+	INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
 	INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 8d32e3f50813..c8597ae8c833 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -323,6 +323,8 @@  enum rxe_mw_state {
 struct rxe_mw {
 	struct ib_mw ibmw;
 	struct rxe_pool_entry pelem;
+	spinlock_t lock;
+	enum rxe_mw_state state;
 };
 
 struct rxe_mc_grp {