@@ -273,6 +273,7 @@ dapl_udapl_libdaplofa_la_SOURCES = dapl/udapl/dapl_init.c \
dapl/openib_common/cq.c \
dapl/openib_common/qp.c \
dapl/openib_common/util.c \
+ dapl/openib_common/srq.c \
dapl/openib_cma/cm.c \
dapl/openib_cma/device.c $(XPROGRAMS)
@@ -389,6 +390,7 @@ dapl_udapl_libdaploscm_la_SOURCES = dapl/udapl/dapl_init.c \
dapl/openib_common/cq.c \
dapl/openib_common/qp.c \
dapl/openib_common/util.c \
+ dapl/openib_common/srq.c \
dapl/openib_scm/cm.c \
dapl/openib_scm/device.c $(XPROGRAMS)
@@ -505,6 +507,7 @@ dapl_udapl_libdaploucm_la_SOURCES = dapl/udapl/dapl_init.c \
dapl/openib_common/cq.c \
dapl/openib_common/qp.c \
dapl/openib_common/util.c \
+ dapl/openib_common/srq.c \
dapl/openib_ucm/cm.c \
dapl/openib_ucm/device.c $(XPROGRAMS)
@@ -622,6 +625,7 @@ dapl_udapl_libdaplomcm_la_SOURCES = dapl/udapl/dapl_init.c \
dapl/openib_common/cq.c \
dapl/openib_common/qp.c \
dapl/openib_common/util.c \
+ dapl/openib_common/srq.c \
dapl/openib_mcm/cm.c \
dapl/openib_mcm/mix.c \
dapl/openib_mcm/proxy.c \
@@ -263,6 +263,19 @@ dapls_evd_dto_wait (
IN DAPL_EVD *evd_ptr,
IN uint32_t timeout);
+DAT_RETURN
+dapls_ib_srq_alloc (
+ IN DAPL_SRQ *srq_ptr);
+
+DAT_RETURN
+dapls_ib_srq_free (
+ IN DAPL_SRQ *srq_ptr);
+
+DAT_RETURN
+dapls_ib_srq_resize(
+ IN DAPL_SRQ *srq_ptr,
+ IN uint32_t new_max_wr);
+
#ifdef DAT_EXTENSIONS
void
dapls_cqe_to_event_extension(
@@ -366,7 +366,7 @@ dapli_connection_request(IN dp_ib_cm_handle_t ib_cm_handle,
* requestor that we cant help them.
*/
ia_ptr = sp_ptr->header.owner_ia;
- ep_ptr = dapl_ep_alloc(ia_ptr, NULL);
+ ep_ptr = dapl_ep_alloc(ia_ptr, NULL, DAT_FALSE);
if (ep_ptr == NULL) {
dapls_cr_free(cr_ptr);
/* Invoking function will call dapls_ib_cm_reject() */
@@ -237,7 +237,7 @@ dapl_ep_create(IN DAT_IA_HANDLE ia_handle,
}
/* Allocate EP */
- ep_ptr = dapl_ep_alloc(ia_ptr, ep_attr);
+ ep_ptr = dapl_ep_alloc(ia_ptr, ep_attr, DAT_FALSE);
if (ep_ptr == NULL) {
dat_status =
DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY);
@@ -84,6 +84,7 @@ dapl_ep_create_with_srq(IN DAT_IA_HANDLE ia_handle,
DAT_EP_ATTR ep_attr_limit;
DAPL_EVD *evd_ptr;
DAT_RETURN dat_status;
+ *ep_handle = NULL;
dat_status = DAT_SUCCESS;
dapl_dbg_log(DAPL_DBG_TYPE_API,
@@ -146,25 +147,37 @@ dapl_ep_create_with_srq(IN DAT_IA_HANDLE ia_handle,
goto bail;
}
+ if (ep_handle == NULL) {
+ dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG7);
+ goto bail;
+ }
+
+ if (DAPL_BAD_PTR(ep_attr)) {
+ dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG6);
+ goto bail;
+ }
+
/*
* Verify the SRQ handle. It is an error to invoke this call with
* a NULL handle
*/
+ dapl_os_lock(&ia_ptr->header.lock);
if (DAPL_BAD_HANDLE(srq_handle, DAPL_MAGIC_SRQ)) {
dat_status =
DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ);
+ dapl_os_unlock(&ia_ptr->header.lock);
goto bail;
}
-
- if (ep_handle == NULL) {
- dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG7);
+ if (((DAPL_SRQ *) srq_handle)->param.srq_state ==
+ DAT_SRQ_STATE_SHUTDOWN) {
+ dat_status =
+ DAT_ERROR(DAT_INVALID_STATE, DAT_INVALID_STATE_SRQ_SHUTDOWN);
+ dapl_os_unlock(&ia_ptr->header.lock);
goto bail;
- }
- if (DAPL_BAD_PTR(ep_attr)) {
- dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG6);
- goto bail;
}
+ dapl_os_atomic_inc(&((DAPL_SRQ *) srq_handle)->srq_ref_count);
+ dapl_os_unlock(&ia_ptr->header.lock);
/*
* Qualify EP Attributes are legal and make sense. Note that if one
@@ -173,8 +186,7 @@ dapl_ep_create_with_srq(IN DAT_IA_HANDLE ia_handle,
* the respective queue.
*/
if (ep_attr != NULL &&
- (ep_attr->service_type != DAT_SERVICE_TYPE_RC ||
- (recv_evd_handle == DAT_HANDLE_NULL && ep_attr->max_recv_dtos != 0)
+ ((recv_evd_handle == DAT_HANDLE_NULL && ep_attr->max_recv_dtos != 0)
|| (recv_evd_handle != DAT_HANDLE_NULL
&& ep_attr->max_recv_dtos == 0)
|| (request_evd_handle == DAT_HANDLE_NULL
@@ -186,6 +198,7 @@ dapl_ep_create_with_srq(IN DAT_IA_HANDLE ia_handle,
dapl_ep_check_recv_completion_flags(ep_attr->
recv_completion_flags)))) {
dat_status = DAT_INVALID_PARAMETER | DAT_INVALID_ARG6;
+ dapl_os_atomic_dec(&((DAPL_SRQ *) srq_handle)->srq_ref_count);
goto bail;
}
@@ -195,6 +208,7 @@ dapl_ep_create_with_srq(IN DAT_IA_HANDLE ia_handle,
dat_status = dapls_ib_query_hca(ia_ptr->hca_ptr,
NULL, &ep_attr_limit, NULL);
if (dat_status != DAT_SUCCESS) {
+ dapl_os_atomic_dec(&((DAPL_SRQ *) srq_handle)->srq_ref_count);
goto bail;
}
if (ep_attr->max_mtu_size > ep_attr_limit.max_mtu_size ||
@@ -209,6 +223,8 @@ dapl_ep_create_with_srq(IN DAT_IA_HANDLE ia_handle,
ep_attr_limit.max_rdma_read_out)
{
dat_status = DAT_INVALID_PARAMETER | DAT_INVALID_ARG6;
+ dapl_os_atomic_dec(&((DAPL_SRQ *) srq_handle)->
+ srq_ref_count);
goto bail;
}
}
@@ -246,22 +262,12 @@ dapl_ep_create_with_srq(IN DAT_IA_HANDLE ia_handle,
}
}
- dat_status = DAT_NOT_IMPLEMENTED;
-
- /*
- * XXX The rest of the EP code is useful in this case too,
- * XXX but need to complete the SRQ implementation before
- * XXX committing resources
- */
- *ep_handle = ep_ptr = NULL;
- goto bail;
-#ifdef notdef
-
/* Allocate EP */
- ep_ptr = dapl_ep_alloc(ia_ptr, ep_attr);
+ ep_ptr = dapl_ep_alloc(ia_ptr, ep_attr, DAT_TRUE);
if (ep_ptr == NULL) {
dat_status =
DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY);
+ dapl_os_atomic_dec(&((DAPL_SRQ *) srq_handle)->srq_ref_count);
goto bail;
}
@@ -279,6 +285,7 @@ dapl_ep_create_with_srq(IN DAT_IA_HANDLE ia_handle,
ep_ptr->param.recv_evd_handle = recv_evd_handle;
ep_ptr->param.request_evd_handle = request_evd_handle;
ep_ptr->param.connect_evd_handle = connect_evd_handle;
+ ep_ptr->param.srq_handle = srq_handle;
/*
* Make sure we handle the NULL DTO EVDs
@@ -309,6 +316,8 @@ dapl_ep_create_with_srq(IN DAT_IA_HANDLE ia_handle,
if (dat_status != DAT_SUCCESS) {
dapl_os_atomic_dec(&((DAPL_PZ *) pz_handle)->
pz_ref_count);
+ dapl_os_atomic_dec(&((DAPL_SRQ *) srq_handle)->
+ srq_ref_count);
dapl_ep_dealloc(ep_ptr);
goto bail;
}
@@ -340,8 +349,6 @@ dapl_ep_create_with_srq(IN DAT_IA_HANDLE ia_handle,
*ep_handle = ep_ptr;
-#endif /* notdef */
-
bail:
return dat_status;
}
@@ -162,6 +162,11 @@ DAT_RETURN DAT_API dapl_ep_free(IN DAT_EP_HANDLE ep_handle)
evd_ref_count);
param->connect_evd_handle = NULL;
}
+ if (param->srq_handle != NULL) {
+ dapl_os_atomic_dec(&((DAPL_SRQ *) param->srq_handle)->
+ srq_ref_count);
+ param->srq_handle = NULL;
+ }
/*
* Finish tearing everything down.
@@ -90,6 +90,11 @@ dapl_ep_post_recv(IN DAT_EP_HANDLE ep_handle,
}
ep_ptr = (DAPL_EP *) ep_handle;
+ if (ep_ptr->param.srq_handle) {
+ dat_status =
+ DAT_ERROR(DAT_INVALID_STATE, DAT_INVALID_STATE_SRQ_IN_USE);
+ goto bail;
+ }
/*
* Synchronization ok since this buffer is only used for receive
@@ -104,7 +104,8 @@ char *dapl_get_ep_state_str(DAT_EP_STATE state)
* none
*
*/
-DAPL_EP *dapl_ep_alloc(IN DAPL_IA * ia_ptr, IN const DAT_EP_ATTR * ep_attr)
+DAPL_EP *dapl_ep_alloc(IN DAPL_IA * ia_ptr, IN const DAT_EP_ATTR * ep_attr,
+ IN DAT_BOOLEAN using_srq)
{
DAPL_EP *ep_ptr;
@@ -172,13 +173,16 @@ DAPL_EP *dapl_ep_alloc(IN DAPL_IA * ia_ptr, IN const DAT_EP_ATTR * ep_attr)
goto bail;
}
- if (DAT_SUCCESS != dapls_cb_create(&ep_ptr->recv_buffer,
- ep_ptr,
- ep_ptr->param.ep_attr.max_recv_dtos))
- {
- dapl_ep_dealloc(ep_ptr);
- ep_ptr = NULL;
- goto bail;
+ /* SRQ case - don't allocate as we use the SRQ recv buffer */
+ if (using_srq == DAT_FALSE) {
+ if (DAT_SUCCESS != dapls_cb_create(&ep_ptr->recv_buffer,
+ ep_ptr,
+ ep_ptr->param.ep_attr.max_recv_dtos))
+ {
+ dapl_ep_dealloc(ep_ptr);
+ ep_ptr = NULL;
+ goto bail;
+ }
}
dapls_io_trc_alloc(ep_ptr);
@@ -209,7 +213,8 @@ void dapl_ep_dealloc(IN DAPL_EP * ep_ptr)
ep_ptr->header.magic = DAPL_MAGIC_INVALID; /* reset magic to prevent reuse */
dapls_cb_free(&ep_ptr->req_buffer);
- dapls_cb_free(&ep_ptr->recv_buffer);
+ if (ep_ptr->recv_buffer.pool)
+ dapls_cb_free(&ep_ptr->recv_buffer);
if (NULL != ep_ptr->cxn_timer) {
dapl_os_free(ep_ptr->cxn_timer, sizeof(DAPL_OS_TIMER));
@@ -627,8 +632,10 @@ void dapls_ep_flush_cqs(DAPL_EP * ep_ptr)
if (ep_ptr->param.request_evd_handle)
dapli_ep_flush_evd((DAPL_EVD *) ep_ptr->param.request_evd_handle);
if (ep_ptr->param.recv_evd_handle)
- while (dapls_cb_pending(&ep_ptr->recv_buffer))
- dapli_ep_flush_evd((DAPL_EVD *) ep_ptr->param.recv_evd_handle);
+ if (ep_ptr->recv_buffer.pool)
+ while (dapls_cb_pending(&ep_ptr->recv_buffer))
+ dapli_ep_flush_evd((DAPL_EVD *) ep_ptr->param.
+ recv_evd_handle);
}
/*
@@ -46,7 +46,8 @@
extern DAPL_EP *
dapl_ep_alloc (
IN DAPL_IA *ia,
- IN const DAT_EP_ATTR *ep_attr );
+ IN const DAT_EP_ATTR *ep_attr,
+ IN DAT_BOOLEAN using_srq );
extern void
dapl_ep_dealloc (
@@ -67,10 +67,10 @@ dapl_evd_qp_async_error_callback(IN ib_hca_handle_t ib_hca_handle,
DAT_RETURN dat_status;
#ifdef _VENDOR_IBAL_
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, "%s() IB err %s\n",
+ dapl_dbg_log(DAPL_DBG_TYPE_EXCEPTION, "%s() IB err %s\n",
__FUNCTION__, ib_get_async_event_str(cause_ptr->code));
#else
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, "%s() IB async QP err - ctx=%p\n",
+ dapl_dbg_log(DAPL_DBG_TYPE_EXCEPTION, "%s() IB async QP err - ctx=%p\n",
__FUNCTION__, context);
#endif
@@ -85,16 +85,21 @@ dapl_evd_qp_async_error_callback(IN ib_hca_handle_t ib_hca_handle,
async_evd = (DAPL_EVD *) ia_ptr->async_error_evd;
DAPL_CNTR(ia_ptr, DCNT_IA_ASYNC_QP_ERROR);
- dapl_log(DAPL_DBG_TYPE_WARN, " -- %s: ep %p qp %p (%x) state %d\n",
+ dapl_log(DAPL_DBG_TYPE_EXCEPTION, " -- %s: ep %p qp %p (%x) state %d\n",
__FUNCTION__, ep_ptr, ep_ptr->qp_handle,
ep_ptr->qpn, ep_ptr->param.ep_state);
/*
* Transition to ERROR if we are connected; other states need to
- * complete first (e.g. pending states)
+ * complete first (e.g. pending states) unless it's SRQ LAST_WQE_REACHED
+ * which is a regular async event in case of disconnect.
*/
if (ep_ptr->param.ep_state == DAT_EP_STATE_CONNECTED) {
- ep_ptr->param.ep_state = DAT_EP_STATE_ERROR;
+ if (!ep_ptr->param.srq_handle || cause_ptr->event_type != IBV_EVENT_QP_LAST_WQE_REACHED) {
+ dapl_os_lock(&ep_ptr->header.lock);
+ ep_ptr->param.ep_state = DAT_EP_STATE_ERROR;
+ dapl_os_unlock(&ep_ptr->header.lock);
+ }
}
dapl_os_assert(async_evd != NULL);
@@ -738,8 +738,8 @@ dapls_evd_post_async_error_event(IN DAPL_EVD * evd_ptr,
{
DAT_EVENT *event_ptr;
- dapl_log(DAPL_DBG_TYPE_WARN,
- " WARNING: async event - %s evd=%p\n",
+ dapl_log(DAPL_DBG_TYPE_EXCEPTION,
+ " async event - %s evd=%p\n",
dapl_event_str(event_number), evd_ptr);
dapl_os_lock(&evd_ptr->header.lock);
@@ -975,7 +975,7 @@ dapls_evd_post_cr_event_ext(IN DAPL_SP * sp_ptr,
* requestor that we cant help them.
*/
ia_ptr = sp_ptr->header.owner_ia;
- ep_ptr = dapl_ep_alloc(ia_ptr, NULL);
+ ep_ptr = dapl_ep_alloc(ia_ptr, NULL, DAT_FALSE);
if (ep_ptr == NULL) {
dapls_cr_free(cr_ptr);
goto bail;
@@ -1086,6 +1086,7 @@ dapli_evd_cqe_to_event(IN DAPL_EVD * evd_ptr,
DAPL_COOKIE *cookie;
DAT_DTO_COMPLETION_STATUS dto_status;
DAPL_COOKIE_BUFFER *buffer;
+ DAPL_SRQ *srq_ptr = NULL;
/*
* All that can be relied on if the status is bad is the status
@@ -1096,6 +1097,7 @@ dapli_evd_cqe_to_event(IN DAPL_EVD * evd_ptr,
cookie = (DAPL_COOKIE *) (uintptr_t) DAPL_GET_CQE_WRID(cqe_ptr);
dapl_os_assert((NULL != cookie));
+ /* In case of RECV SRQ DTO cookie->ep holds pointer to the SRQ */
ep_ptr = cookie->ep;
dapl_os_assert((NULL != ep_ptr));
@@ -1125,8 +1127,13 @@ dapli_evd_cqe_to_event(IN DAPL_EVD * evd_ptr,
break;
}
#endif
-
- if (DAPL_DTO_TYPE_RECV == cookie->val.dto.type)
+ if (DAPL_DTO_TYPE_RECV_SRQ == cookie->val.dto.type) {
+ /* in SRQ event we do NOT have ep pointer */
+ srq_ptr = (DAPL_SRQ *)ep_ptr;
+ buffer = &srq_ptr->recv_buffer;
+ ep_ptr = NULL;
+ }
+ else if (DAPL_DTO_TYPE_RECV == cookie->val.dto.type)
buffer = &ep_ptr->recv_buffer;
else
buffer = &ep_ptr->req_buffer;
@@ -1151,6 +1158,8 @@ dapli_evd_cqe_to_event(IN DAPL_EVD * evd_ptr,
}
dapls_cookie_dealloc(buffer, cookie);
+ if (srq_ptr)
+ dapl_os_atomic_dec(&srq_ptr->recv_count);
break;
}
@@ -1198,6 +1207,36 @@ dapli_evd_cqe_to_event(IN DAPL_EVD * evd_ptr,
(dto_status != DAT_DTO_ERR_FLUSHED)) {
DAPL_EVD *evd_ptr;
+ /* In SRQ case we need to look up the EP */
+ if (ep_ptr == NULL) {
+ DAT_UINT32 qp_num = DAPL_GET_CQE_QP_NUM(cqe_ptr);
+ DAPL_IA *ia_ptr = (DAPL_IA *)srq_ptr->param.ia_handle;
+
+ dapl_os_lock(&ia_ptr->header.lock);
+ ep_ptr = (dapl_llist_is_empty(&ia_ptr->ep_list_head)
+ ? NULL :
+ dapl_llist_peek_head(&ia_ptr->ep_list_head));
+
+ while (ep_ptr != NULL) {
+ if (ep_ptr->qp_handle->qp->qp_num == qp_num)
+ break;
+
+ ep_ptr =
+ dapl_llist_next_entry(&ia_ptr->ep_list_head,
+ &ep_ptr->header.
+ ia_list_entry);
+ }
+ dapl_os_unlock(&ia_ptr->header.lock);
+
+ if (ep_ptr == NULL ) {
+ dapl_dbg_log(DAPL_DBG_TYPE_DTO_COMP_ERR,
+ " SRQ %p at ia %p - "
+ "EP with qpn %d was not found\n",
+ srq_ptr, ia_ptr, qp_num);
+ return;
+ }
+ }
+
/*
* If we are connected, generate disconnect and generate an
* event. We may be racing with other disconnect ops, so we
@@ -93,13 +93,7 @@ dapl_srq_create(IN DAT_IA_HANDLE ia_handle,
DAPL_CNTR(ia_ptr, DCNT_IA_SRQ_CREATE);
- /*
- * Verify non-required parameters.
- * N.B. Assumption: any parameter that can be
- * modified by dat_ep_modify() is not strictly
- * required when the EP is created
- */
- if (pz_handle != NULL && DAPL_BAD_HANDLE(pz_handle, DAPL_MAGIC_PZ)) {
+ if (DAPL_BAD_HANDLE(pz_handle, DAPL_MAGIC_PZ)) {
dat_status =
DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_PZ);
goto bail;
@@ -114,10 +108,6 @@ dapl_srq_create(IN DAT_IA_HANDLE ia_handle,
goto bail;
}
- /* SRQ provider not implemented */
- dat_status = DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
- goto bail;
-
/* Allocate SRQ */
srq_ptr = dapl_srq_alloc(ia_ptr, srq_attr);
if (srq_ptr == NULL) {
@@ -127,18 +117,25 @@ dapl_srq_create(IN DAT_IA_HANDLE ia_handle,
}
srq_ptr->param.ia_handle = (DAT_IA_HANDLE) ia_ptr;
- srq_ptr->param.srq_state = DAT_SRQ_STATE_OPERATIONAL;
srq_ptr->param.pz_handle = pz_handle;
/*
- * XXX Allocate provider resource here!!!
+ * Get a SRQ from the IB provider
*/
+ dat_status = dapls_ib_srq_alloc(srq_ptr);
+ if (dat_status != DAT_SUCCESS) {
+ dapl_srq_dealloc(srq_ptr);
+ goto bail;
+ }
/* Link it onto the IA */
dapl_ia_link_srq(ia_ptr, srq_ptr);
*srq_handle = srq_ptr;
+ /* Ready */
+ srq_ptr->param.srq_state = DAT_SRQ_STATE_OPERATIONAL;
+
bail:
return dat_status;
}
@@ -72,6 +72,16 @@ DAT_RETURN DAT_API dapl_srq_free(IN DAT_SRQ_HANDLE srq_handle)
srq_ptr = (DAPL_SRQ *) srq_handle;
param = &srq_ptr->param;
+ ia_ptr = srq_ptr->header.owner_ia;
+
+ /*
+ * Verify parameters
+ */
+ if (DAPL_BAD_HANDLE(ia_ptr, DAPL_MAGIC_IA)) {
+ dat_status =
+ DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA);
+ goto bail;
+ }
/*
* Verify parameter & state
@@ -82,6 +92,7 @@ DAT_RETURN DAT_API dapl_srq_free(IN DAT_SRQ_HANDLE srq_handle)
goto bail;
}
+ dapl_os_lock(&ia_ptr->header.lock);
if (dapl_os_atomic_read(&srq_ptr->srq_ref_count) != 0) {
/*
* The DAPL 1.2 spec says to return DAT_SRQ_IN_USE, which does
@@ -89,32 +100,17 @@ DAT_RETURN DAT_API dapl_srq_free(IN DAT_SRQ_HANDLE srq_handle)
*/
dat_status =
DAT_ERROR(DAT_INVALID_STATE, DAT_INVALID_STATE_SRQ_IN_USE);
+ dapl_os_unlock(&ia_ptr->header.lock);
goto bail;
}
-
- ia_ptr = srq_ptr->header.owner_ia;
+ param->srq_state = DAT_SRQ_STATE_SHUTDOWN;
+ dapl_os_unlock(&ia_ptr->header.lock);
DAPL_CNTR(ia_ptr, DCNT_IA_SRQ_FREE);
- /*
- * Do verification of parameters and the state change atomically.
- */
- dapl_os_lock(&srq_ptr->header.lock);
-
- /* Remove the SRQ from the IA */
- dapl_ia_unlink_srq(ia_ptr, srq_ptr);
-
- dapl_os_unlock(&srq_ptr->header.lock);
-
- /*
- * Finish tearing everything down.
- */
-
- /*
- * Take care of the transport resource
- */
+ dapl_ia_unlink_srq(srq_ptr->header.owner_ia, srq_ptr);
- /* XXX Put provider code here!!! */
+ dapls_ib_srq_free(srq_ptr);
/* Free the resource */
dapl_srq_dealloc(srq_ptr);
@@ -95,7 +95,7 @@ dapl_srq_post_recv(IN DAT_SRQ_HANDLE srq_handle,
* app must syncronize access to the SRQ.
*/
dat_status = dapls_dto_cookie_alloc(&srq_ptr->recv_buffer,
- DAPL_DTO_TYPE_RECV,
+ DAPL_DTO_TYPE_RECV_SRQ,
user_cookie, &cookie);
if (DAT_SUCCESS != dat_status) {
goto bail;
@@ -110,8 +110,8 @@ dapl_srq_post_recv(IN DAT_SRQ_HANDLE srq_handle,
/*
* Invoke provider specific routine to post DTO
*/
- /* XXX Put code here XXX */
- /* XXX */ dat_status = DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+ dat_status = dapls_ib_post_srq_recv(srq_ptr, cookie,
+ num_segments, local_iov);
if (dat_status != DAT_SUCCESS) {
dapl_os_atomic_dec(&srq_ptr->recv_count);
@@ -84,11 +84,9 @@ dapl_srq_query(IN DAT_SRQ_HANDLE srq_handle,
srq_ptr = (DAPL_SRQ *) srq_handle;
- /*
- * XXX Need to calculate available_dto_count and outstanding_dto_count
- */
srq_ptr->param.available_dto_count = DAT_VALUE_UNKNOWN;
- srq_ptr->param.outstanding_dto_count = DAT_VALUE_UNKNOWN;
+ srq_ptr->param.outstanding_dto_count =
+ dapl_os_atomic_read(&srq_ptr->recv_count);
*srq_param = srq_ptr->param;
dats_get_ia_handle(srq_ptr->header.owner_ia, &srq_param->ia_handle);
@@ -66,7 +66,6 @@
DAT_RETURN DAT_API
dapl_srq_resize(IN DAT_SRQ_HANDLE srq_handle, IN DAT_COUNT srq_max_recv_dto)
{
- DAPL_IA *ia_ptr;
DAPL_SRQ *srq_ptr;
DAT_RETURN dat_status;
@@ -82,19 +81,17 @@ dapl_srq_resize(IN DAT_SRQ_HANDLE srq_handle, IN DAT_COUNT srq_max_recv_dto)
}
srq_ptr = (DAPL_SRQ *) srq_handle;
- ia_ptr = srq_ptr->header.owner_ia;
/*
* Check for nonsense requests per the spec
*/
- if (srq_max_recv_dto <= srq_ptr->param.low_watermark) {
+ if (srq_max_recv_dto <= srq_ptr->param.low_watermark ||
+ srq_max_recv_dto < dapl_os_atomic_read(&srq_ptr->recv_count)) {
dat_status = DAT_ERROR(DAT_INVALID_STATE, DAT_NO_SUBTYPE);
goto bail;
}
- /* XXX Put implementation here XXX */
-
- /* XXX */ dat_status = DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+ dat_status = dapls_ib_srq_resize(srq_ptr, (uint32_t)srq_max_recv_dto);
bail:
return dat_status;
@@ -82,9 +82,7 @@ dapl_srq_set_lw(IN DAT_SRQ_HANDLE srq_handle, IN DAT_COUNT low_watermark)
srq_ptr = (DAPL_SRQ *) srq_handle;
- /* XXX Put implementation here XXX */
-
- /* XXX */ dat_status = DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+ dat_status = DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
bail:
return dat_status;
@@ -84,7 +84,6 @@ DAPL_SRQ *dapl_srq_alloc(IN DAPL_IA * ia_ptr, IN const DAT_SRQ_ATTR * srq_attr)
/*
* Initialize the body.
- * XXX Assume srq_attrs is required
*/
srq_ptr->param.max_recv_dtos = srq_attr->max_recv_dtos;
srq_ptr->param.max_recv_iov = srq_attr->max_recv_iov;
@@ -122,7 +121,6 @@ void dapl_srq_dealloc(IN DAPL_SRQ * srq_ptr)
dapl_os_assert(srq_ptr->header.magic == DAPL_MAGIC_SRQ);
srq_ptr->header.magic = DAPL_MAGIC_INVALID; /* reset magic to prevent reuse */
- dapl_ia_unlink_srq(srq_ptr->header.owner_ia, srq_ptr);
dapls_cb_free(&srq_ptr->recv_buffer);
dapl_os_lock_destroy(&srq_ptr->header.lock);
@@ -479,6 +479,7 @@ struct dapl_srq
DAPL_ATOMIC srq_ref_count;
DAPL_COOKIE_BUFFER recv_buffer;
DAPL_ATOMIC recv_count;
+ ib_srq_handle_t srq_handle;
};
/* DAPL_PZ maps to DAT_PZ_HANDLE */
@@ -580,6 +581,7 @@ typedef enum dapl_dto_type
{
DAPL_DTO_TYPE_SEND,
DAPL_DTO_TYPE_RECV,
+ DAPL_DTO_TYPE_RECV_SRQ,
DAPL_DTO_TYPE_RDMA_WRITE,
DAPL_DTO_TYPE_RDMA_READ,
#ifdef DAT_EXTENSIONS
@@ -114,7 +114,8 @@ DAT_RETURN dapls_ib_get_async_event(IN ib_error_record_t * err_record,
DAT_RETURN dat_status = DAT_SUCCESS;
int err_code = err_record->event_type;
- dapl_log(DAPL_DBG_TYPE_WARN, " WARNING: %s\n", dapl_ib_async_str(err_code));
+ dapl_log(DAPL_DBG_TYPE_EXCEPTION, " %s %s\n", __FUNCTION__,
+ dapl_ib_async_str(err_code));
switch (err_code)
{
@@ -90,6 +90,7 @@ typedef struct ibv_mw *ib_mw_handle_t;
typedef struct ibv_wc ib_work_completion_t;
typedef struct ibv_ah *ib_ah_handle_t;
typedef union ibv_gid *ib_gid_handle_t;
+typedef struct ibv_srq *ib_srq_handle_t;
/* HCA context type maps to IB verbs */
typedef struct ibv_context *ib_hca_handle_t;
@@ -112,6 +112,53 @@ dapls_ib_post_recv (
}
/*
+ * dapls_ib_post_srq_recv
+ *
+ * Provider specific Post SRQ RECV function
+ */
+STATIC _INLINE_ DAT_RETURN
+dapls_ib_post_srq_recv (
+ IN DAPL_SRQ *srq_ptr,
+ IN DAPL_COOKIE *cookie,
+ IN DAT_COUNT segments,
+ IN DAT_LMR_TRIPLET *local_iov )
+{
+ struct ibv_recv_wr wr;
+ struct ibv_recv_wr *bad_wr;
+ ib_data_segment_t *ds = (ib_data_segment_t *)local_iov;
+ DAT_COUNT i, total_len;
+ int ret;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_SRQ,
+ " post_srq_rcv: srq %p cookie %p segs %d l_iov %p\n",
+ srq_ptr, cookie, segments, local_iov);
+
+ /* setup work request */
+ total_len = 0;
+ wr.next = 0;
+ wr.num_sge = segments;
+ wr.wr_id = (uint64_t)(uintptr_t)cookie;
+ wr.sg_list = ds;
+
+ if (cookie != NULL) {
+ for (i = 0; i < segments; i++) {
+ dapl_dbg_log(DAPL_DBG_TYPE_SRQ,
+ " post_srq_rcv: l_key 0x%x va %p len %d\n",
+ ds->lkey, ds->addr, ds->length );
+ total_len += ds->length;
+ ds++;
+ }
+ cookie->val.dto.size = total_len;
+ }
+
+ ret = ibv_post_srq_recv(srq_ptr->srq_handle, &wr, &bad_wr);
+ if (ret)
+ return(dapl_convert_errno(errno,"ibv_recv"));
+
+ return DAT_SUCCESS;
+}
+
+/*
* dapls_ib_post_send
*
* Provider specific Post SEND function
@@ -504,6 +551,7 @@ STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p)
#define DAPL_GET_CQE_OPTYPE(cqe_p) dapls_cqe_opcode(cqe_p)
#define DAPL_GET_CQE_WRID(cqe_p) ((ib_work_completion_t*)cqe_p)->wr_id
+#define DAPL_GET_CQE_QP_NUM(cqe_p) ((ib_work_completion_t*)cqe_p)->qp_num
#define DAPL_GET_CQE_STATUS(cqe_p) ((ib_work_completion_t*)cqe_p)->status
#define DAPL_GET_CQE_VENDOR_ERR(cqe_p) ((ib_work_completion_t*)cqe_p)->vendor_err
#define DAPL_GET_CQE_BYTESNUM(cqe_p) ((ib_work_completion_t*)cqe_p)->byte_len
@@ -61,8 +61,10 @@ dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr,
{
DAT_EP_ATTR *attr;
DAPL_EVD *rcv_evd, *req_evd;
+ DAPL_SRQ *dapl_srq = (DAPL_SRQ *)ep_ptr->param.srq_handle;
ib_cq_handle_t rcv_cq, req_cq;
ib_pd_handle_t ib_pd_handle;
+ ib_srq_handle_t ib_srq_handle = NULL;
int ret = EINVAL;
int max_inline = ia_ptr->hca_ptr->ib_trans.max_inline_send;
struct ibv_qp_init_attr qp_create;
@@ -72,6 +74,8 @@ dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr,
attr = &ep_ptr->param.ep_attr;
ib_pd_handle = ((DAPL_PZ *) ep_ptr->param.pz_handle)->pd_handle;
+ if (dapl_srq)
+ ib_srq_handle = dapl_srq->srq_handle;
rcv_evd = (DAPL_EVD *) ep_ptr->param.recv_evd_handle;
req_evd = (DAPL_EVD *) ep_ptr->param.request_evd_handle;
@@ -167,6 +171,7 @@ dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr,
qp_create.cap.max_inline_data = max_inline;
qp_create.qp_type = IBV_QPT_RC;
qp_create.qp_context = (void *)ep_ptr;
+ qp_create.srq = ib_srq_handle;
#ifdef DAT_EXTENSIONS
if ((int)attr->service_type == (int)DAT_IB_SERVICE_TYPE_UD) {
new file mode 100644
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2009-2014 Intel Corporation. All rights reserved.
+ *
+ * This Software is licensed under one of the following licenses:
+ *
+ * 1) under the terms of the "Common Public License 1.0" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/cpl.php.
+ *
+ * 2) under the terms of the "The BSD License" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/bsd-license.php.
+ *
+ * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
+ * copy of which is available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/gpl-license.php.
+ *
+ * Licensee has the right to choose one of the above licenses.
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice and one of the license notices.
+ *
+ * Redistributions in binary form must reproduce both the above copyright
+ * notice, one of the license notices in the documentation
+ * and/or other materials provided with the distribution.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "dapl.h"
+#include "dapl_adapter_util.h"
+#include "dapl_ep_util.h"
+
+/*
+ * dapl_ib_srq_alloc
+ *
+ * Alloc a SRQ
+ *
+ * Input:
+ * ia_handle SRQ handle
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ *
+ */
+DAT_RETURN
+dapls_ib_srq_alloc(IN DAPL_SRQ *srq_ptr)
+{
+ struct ibv_srq_init_attr srq_init_attr;
+ ib_pd_handle_t ib_pd_handle;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_SRQ,
+ " srq alloc: srq %p\n", srq_ptr);
+
+ ib_pd_handle = ((DAPL_PZ *) srq_ptr->param.pz_handle)->pd_handle;
+ srq_init_attr.srq_context = NULL;
+ srq_init_attr.attr.max_wr = srq_ptr->param.max_recv_dtos;
+ srq_init_attr.attr.max_sge = srq_ptr->param.max_recv_iov;
+ srq_init_attr.attr.srq_limit = 0;
+
+ srq_ptr->srq_handle = ibv_create_srq(ib_pd_handle, &srq_init_attr);
+ if (!srq_ptr->srq_handle)
+ goto err;
+
+ return DAT_SUCCESS;
+
+err:
+ dapl_log(DAPL_DBG_TYPE_ERR, "ib_srq_alloc ERR %s\n", strerror(errno));
+
+ return dapl_convert_errno(ENOMEM, "srq_allocate" );
+}
+
+/*
+ * dapl_ib_srq_free
+ *
+ * Free a SRQ
+ *
+ * Input:
+ * ia_handle SRQ handle
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_PARAMETER
+ * dapl_convert_errno
+ */
+DAT_RETURN
+dapls_ib_srq_free(IN DAPL_SRQ *srq_ptr)
+{
+
+ dapl_dbg_log(DAPL_DBG_TYPE_SRQ, " srq free: srq %p\n", srq_ptr);
+
+ if (srq_ptr->srq_handle == IB_INVALID_HANDLE)
+ return DAT_INVALID_PARAMETER;
+
+ if (ibv_destroy_srq(srq_ptr->srq_handle)) {
+ dapl_log(DAPL_DBG_TYPE_ERR,
+ " srq_free: ibv_destroy_srq error - %s\n",
+ strerror(errno));
+ return (dapl_convert_errno(errno, "srq_free"));
+ }
+
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapl_ib_srq_resize
+ *
+ *Resize a SRQ
+ *
+ * Input:
+ * SRQ handle
+ * New size
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ *
+ */
+DAT_RETURN
+dapls_ib_srq_resize(IN DAPL_SRQ *srq_ptr, IN uint32_t new_max_wr)
+{
+ struct ibv_srq_attr srq_attr;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_SRQ, " srq resize: srq %p\n", srq_ptr);
+
+ srq_attr.max_wr = new_max_wr;
+
+ if (ibv_modify_srq(srq_ptr->srq_handle, &srq_attr, IBV_SRQ_MAX_WR)) {
+ dapl_log(DAPL_DBG_TYPE_ERR,
+ " srq_resize: ibv_modify_srq error - %s\n",
+ strerror(errno));
+ return (dapl_convert_errno(errno, "srq_resize"));
+ }
+
+ return DAT_SUCCESS;
+}
@@ -638,11 +638,16 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca)
case IBV_EVENT_SRQ_LIMIT_REACHED:
case IBV_EVENT_SQ_DRAINED:
{
+ DAPL_DBG_TYPE dbg_type = DAPL_DBG_TYPE_ERR;
struct dapl_ep *ep_ptr =
event.element.qp->qp_context;
- dapl_log(DAPL_DBG_TYPE_ERR,
- "dapl async_event QP (%p) ERR %d\n",
+ if (event.event_type == IBV_EVENT_QP_LAST_WQE_REACHED &&
+ ep_ptr->param.srq_handle) {
+ dbg_type = DAPL_DBG_TYPE_EVD;
+ }
+
+ dapl_log(dbg_type, "dapl async_event QP (%p) Event %d\n",
ep_ptr, event.event_type);
/* report up if async callback still setup */
@@ -577,8 +577,9 @@ typedef DAT_UINT64 DAT_EP_PARAM_MASK;
typedef enum dat_srq_state
{
+ DAT_SRQ_STATE_ERROR,
DAT_SRQ_STATE_OPERATIONAL,
- DAT_SRQ_STATE_ERROR
+ DAT_SRQ_STATE_SHUTDOWN
} DAT_SRQ_STATE;
#define DAT_VALUE_UNKNOWN (((DAT_COUNT) ~0)-1)
@@ -319,6 +319,7 @@ typedef enum dat_return_subtype
DAT_INVALID_STATE_SRQ_OPERATIONAL,
DAT_INVALID_STATE_SRQ_ERROR,
DAT_INVALID_STATE_SRQ_IN_USE,
+ DAT_INVALID_STATE_SRQ_SHUTDOWN,
/* DAT_LENGTH_ERROR has no subtypes */
@@ -1,10 +1,13 @@
-bin_PROGRAMS = dtest dtestcm
+bin_PROGRAMS = dtest dtestcm dtestsrq
dtest_SOURCES = dtest.c
dtest_CFLAGS = -g -Wall -D_GNU_SOURCE
dtestcm_SOURCES = dtestcm.c
dtestcm_CFLAGS = -g -Wall -D_GNU_SOURCE
+dtestsrq_SOURCES = dtestsrq.c
+dtestsrq_CFLAGS = -g -Wall -D_GNU_SOURCE
+
if EXT_TYPE_IB
bin_PROGRAMS += dtestx
dtestx_SOURCES = dtestx.c
@@ -15,4 +18,5 @@ endif
INCLUDES = -I $(srcdir)/../../dat/include
dtest_LDADD = $(top_builddir)/dat/udat/libdat2.la
dtestcm_LDADD = $(top_builddir)/dat/udat/libdat2.la
+dtestsrq_LDADD = $(top_builddir)/dat/udat/libdat2.la