[rdma-core,1/2] pyverbs: Handle CQ events properly

Commit Message

Noa Osherovich Jan. 6, 2020, 3:01 p.m. UTC

When using events, the CQ has to ack all the retrieved events prior to
destruction. Since this call uses a mutex, the events are usually acked
together at the end of the datapath to avoid performance degradation.

In Python, each exception triggers a teardown of all pyverbs' resources,
including CQs (which may haven't acked their events yet). Exception can
be caused by something as trivial as a syntax error.

To avoid that, let CQ object keep track of the number of events waiting
to be acked. This number will be incremented by the completion channel
during get_cq_event() and decremented by the CQ during ack_events().
During close(), if there are still events waiting to be acked, the CQ
will ack them prior to its destruction.

Also add a reference from a CQ object to its completion channel object
to allow users to know if it is connected to one or not during poll.

Signed-off-by: Noa Osherovich <[e-mail]>
Reviewed-by: Edward Srouji <[e-mail]>
---
 pyverbs/cq.pxd |  2 ++
 pyverbs/cq.pyx | 14 +++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

Patch

diff --git a/pyverbs/cq.pxd b/pyverbs/cq.pxd
index 8eeb2e1fd0c2..9704b96f3ff7 100644
--- a/pyverbs/cq.pxd
+++ b/pyverbs/cq.pxd
@@ -20,6 +20,8 @@  cdef class CQ(PyverbsCM):
     cdef add_ref(self, obj)
     cdef object qps
     cdef object srqs
+    cdef object channel
+    cdef object num_events
 
 cdef class CqInitAttrEx(PyverbsObject):
     cdef v.ibv_cq_init_attr_ex attr
diff --git a/pyverbs/cq.pyx b/pyverbs/cq.pyx
index dda47207507f..defb37646034 100755
--- a/pyverbs/cq.pyx
+++ b/pyverbs/cq.pyx
@@ -47,7 +47,7 @@  cdef class CompChannel(PyverbsCM):
     def get_cq_event(self, CQ expected_cq):
         """
         Waits for the next completion event in the completion event channel
-        :param expected_cq: The CQ that got the event
+        :param expected_cq: The CQ that is expected to get the event
         :return: None
         """
         cdef v.ibv_cq *cq
@@ -58,6 +58,7 @@  cdef class CompChannel(PyverbsCM):
             raise PyverbsRDMAErrno('Failed to get CQ event')
         if cq != expected_cq.cq:
             raise PyverbsRDMAErrno('Received event on an unexpected CQ')
+        expected_cq.num_events += 1
 
     cdef add_ref(self, obj):
         if isinstance(obj, CQ) or isinstance(obj, CQEX):
@@ -87,15 +88,18 @@  cdef class CQ(PyverbsCM):
             self.cq = v.ibv_create_cq(context.context, cqe, <void*>cq_context,
                                       channel.cc, comp_vector)
             channel.add_ref(self)
+            self.channel = channel
         else:
             self.cq = v.ibv_create_cq(context.context, cqe, <void*>cq_context,
                                       NULL, comp_vector)
+            self.channel = None
         if self.cq == NULL:
             raise PyverbsRDMAErrno('Failed to create a CQ')
         self.context = context
         context.add_ref(self)
         self.qps = weakref.WeakSet()
         self.srqs = weakref.WeakSet()
+        self.num_events = 0
         self.logger.debug('Created a CQ')
 
     cdef add_ref(self, obj):
@@ -112,12 +116,15 @@  cdef class CQ(PyverbsCM):
     cpdef close(self):
         self.logger.debug('Closing CQ')
         close_weakrefs([self.qps, self.srqs])
+        if self.num_events:
+            self.ack_events(self.num_events)
         if self.cq != NULL:
             rc = v.ibv_destroy_cq(self.cq)
             if rc != 0:
                 raise PyverbsRDMAErrno('Failed to close CQ')
             self.cq = NULL
             self.context = None
+            self.channel = None
 
     def poll(self, num_entries=1):
         """
@@ -166,6 +173,7 @@  cdef class CQ(PyverbsCM):
         :return: None
         """
         v.ibv_ack_cq_events(self.cq, num_events)
+        self.num_events -= num_events
 
     def __str__(self):
         print_format = '{:22}: {:<20}\n'
@@ -173,6 +181,10 @@  cdef class CQ(PyverbsCM):
                print_format.format('Handle', self.cq.handle) +\
                print_format.format('CQEs', self.cq.cqe)
 
+    @property
+    def comp_channel(self):
+        return self.channel
+
 
 cdef class CqInitAttrEx(PyverbsObject):
     def __init__(self, cqe = 100, CompChannel channel = None, comp_vector = 0,