[ofw] [PATCH] uDAPL v2 scm: private data is not handled properly via CR rejects.
Davis, Arlin R
arlin.r.davis at intel.com
Fri Jun 12 09:40:09 PDT 2009
For both RC and UD connect requests, the private
data is not being received on socket and passed
back via the active side REJECT event.
UD requires new extended reject event type of
DAT_IB_UD_CONNECTION_REJECT_EVENT to distiquish
between RC and UD type rejects.
cr_thread exit/cleanup processing fixed to insure
all items are off the list before exiting.
Signed-off-by: Arlin Davis <arlin.r.davis at intel.com>
---
dapl/openib_scm/dapl_ib_cm.c | 96 +++++++++++++++++++++++----------
dat/include/dat2/dat_ib_extensions.h | 7 ++-
2 files changed, 72 insertions(+), 31 deletions(-)
diff --git a/dapl/openib_scm/dapl_ib_cm.c b/dapl/openib_scm/dapl_ib_cm.c
index db0a25b..d116c62 100644
--- a/dapl/openib_scm/dapl_ib_cm.c
+++ b/dapl/openib_scm/dapl_ib_cm.c
@@ -539,17 +539,6 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
sin_addr));
goto bail;
}
- /* check for consumer reject */
- if (cm_ptr->dst.rej) {
- dapl_log(DAPL_DBG_TYPE_CM,
- " CONN_RTU read: PEER REJ reason=0x%x -> %s\n",
- ntohs(cm_ptr->dst.rej),
- inet_ntoa(((struct sockaddr_in *)
- ep_ptr->param.remote_ia_address_ptr)->
- sin_addr));
- event = IB_CME_DESTINATION_REJECT_PRIVATE_DATA;
- goto bail;
- }
/* convert peer response values to host order */
cm_ptr->dst.port = ntohs(cm_ptr->dst.port);
@@ -601,6 +590,23 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
}
}
+ /* check for consumer reject */
+ if (cm_ptr->dst.rej) {
+ dapl_log(DAPL_DBG_TYPE_CM,
+ " CONN_RTU read: PEER REJ reason=0x%x -> %s\n",
+ ntohs(cm_ptr->dst.rej),
+ inet_ntoa(((struct sockaddr_in *)
+ ep_ptr->param.remote_ia_address_ptr)->
+ sin_addr));
+ event = IB_CME_DESTINATION_REJECT_PRIVATE_DATA;
+#ifdef DAT_EXTENSIONS
+ if (cm_ptr->dst.qp_type == IBV_QPT_UD)
+ goto ud_bail;
+ else
+#endif
+ goto bail;
+ }
+
/* modify QP to RTR and then to RTS with remote info */
dapl_os_lock(&ep_ptr->header.lock);
if (dapls_modify_qp_state(ep_ptr->qp_handle,
@@ -637,9 +643,11 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
/* init cm_handle and post the event with private data */
ep_ptr->cm_handle = cm_ptr;
cm_ptr->state = SCM_CONNECTED;
+ event = IB_CME_CONNECTED;
dapl_dbg_log(DAPL_DBG_TYPE_EP, " ACTIVE: connected!\n");
#ifdef DAT_EXTENSIONS
+ud_bail:
if (cm_ptr->dst.qp_type == IBV_QPT_UD) {
DAT_IB_EXTENSION_EVENT_DATA xevent;
@@ -652,26 +660,35 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
&cm_ptr->dst.ia_address,
sizeof(cm_ptr->dst.ia_address));
+ if (event == IB_CME_CONNECTED)
+ event = DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED;
+ else
+ event = DAT_IB_UD_CONNECTION_REJECT_EVENT;
+
dapls_evd_post_connection_event_ext((DAPL_EVD *) ep_ptr->param.
connect_evd_handle,
- DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED,
+ event,
(DAT_EP_HANDLE) ep_ptr,
- (DAT_COUNT) cm_ptr->dst.
- p_size,
- (DAT_PVOID *) cm_ptr->
- p_data,
- (DAT_PVOID *) & xevent);
+ (DAT_COUNT) cm_ptr->dst.p_size,
+ (DAT_PVOID *) cm_ptr->p_data,
+ (DAT_PVOID *) &xevent);
+
+ /* done with socket, don't destroy cm_ptr, need pdata */
+ closesocket(cm_ptr->socket);
+ cm_ptr->socket = DAPL_INVALID_SOCKET;
} else
#endif
dapl_evd_connection_callback(cm_ptr,
IB_CME_CONNECTED,
cm_ptr->p_data, ep_ptr);
return;
- bail:
- /* close socket, free cm structure and post error event */
- dapli_cm_destroy(cm_ptr);
+
+bail:
+ /* close socket, and post error event */
dapls_ib_reinit_ep(ep_ptr); /* reset QP state */
- dapl_evd_connection_callback(NULL, event, NULL, ep_ptr);
+ closesocket(cm_ptr->socket);
+ cm_ptr->socket = DAPL_INVALID_SOCKET;
+ dapl_evd_connection_callback(NULL, event, cm_ptr->p_data, ep_ptr);
}
/*
@@ -1034,11 +1051,13 @@ void dapli_socket_accept_rtu(dp_ib_cm_handle_t cm_ptr)
param.connect_evd_handle,
DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED,
(DAT_EP_HANDLE) cm_ptr->ep,
- (DAT_COUNT) cm_ptr->dst.
- p_size,
- (DAT_PVOID *) cm_ptr->
- p_data,
- (DAT_PVOID *) & xevent);
+ (DAT_COUNT) cm_ptr->dst.p_size,
+ (DAT_PVOID *) cm_ptr->p_data,
+ (DAT_PVOID *) &xevent);
+
+ /* done with socket, don't destroy cm_ptr, need pdata */
+ closesocket(cm_ptr->socket);
+ cm_ptr->socket = DAPL_INVALID_SOCKET;
} else
#endif
dapls_cr_callback(cm_ptr, IB_CME_CONNECTED, NULL, cm_ptr->sp);
@@ -1295,10 +1314,16 @@ dapls_ib_reject_connection(IN dp_ib_cm_handle_t cm_ptr,
" reject(cm %p reason %x, pdata %p, psize %d)\n",
cm_ptr, reason, pdata, psize);
+ if (psize > IB_MAX_REJ_PDATA_SIZE)
+ return DAT_LENGTH_ERROR;
+
/* write reject data to indicate reject */
if (cm_ptr->socket != DAPL_INVALID_SOCKET) {
cm_ptr->dst.rej = (uint16_t) reason;
cm_ptr->dst.rej = htons(cm_ptr->dst.rej);
+ cm_ptr->dst.p_size = htonl(psize);
+ /* get qp_type from request */
+ cm_ptr->dst.qp_type = ntohs(cm_ptr->dst.qp_type);
iov[0].iov_base = (void *)&cm_ptr->dst;
iov[0].iov_len = sizeof(ib_qp_cm_t);
@@ -1316,7 +1341,7 @@ dapls_ib_reject_connection(IN dp_ib_cm_handle_t cm_ptr,
}
/* cr_thread will destroy CR */
- cm_ptr->state = SCM_REJECTED;
+ cm_ptr->state = SCM_DESTROY;
if (send(cm_ptr->hca->ib_trans.scm[1], "w", sizeof "w", 0) == -1)
dapl_log(DAPL_DBG_TYPE_CM,
" cm_destroy: thread wakeup error = %s\n",
@@ -1555,7 +1580,7 @@ void cr_thread(void *arg)
dapl_os_lock(&hca_ptr->ib_trans.lock);
hca_ptr->ib_trans.cr_state = IB_THREAD_RUN;
- while (hca_ptr->ib_trans.cr_state == IB_THREAD_RUN) {
+ while (1) {
dapl_fd_zero(set);
dapl_fd_set(hca_ptr->ib_trans.scm[0], set, DAPL_FD_READ);
@@ -1574,10 +1599,13 @@ void cr_thread(void *arg)
dapl_llist_remove_entry(&hca_ptr->ib_trans.list,
(DAPL_LLIST_ENTRY *) &
cr->entry);
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " CR FREE: %p ep=%p st=%d sock=%d\n",
+ cr, cr->ep, cr->state, cr->socket);
dapl_os_free(cr, sizeof(*cr));
continue;
}
- if (cr->socket == DAPL_INVALID_SOCKET)
+ if (cr->socket == DAPL_INVALID_SOCKET)
continue;
event = (cr->state == SCM_CONN_PENDING) ?
@@ -1664,6 +1692,11 @@ void cr_thread(void *arg)
dapl_os_lock(&hca_ptr->ib_trans.lock);
}
+ /* set to exit and all resources destroyed */
+ if ((hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) &&
+ (dapl_llist_is_empty(&hca_ptr->ib_trans.list)))
+ break;
+
dapl_os_unlock(&hca_ptr->ib_trans.lock);
dapl_select(set);
@@ -1676,6 +1709,11 @@ void cr_thread(void *arg)
strerror(errno));
}
dapl_os_lock(&hca_ptr->ib_trans.lock);
+
+ /* set to exit and all resources destroyed */
+ if ((hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) &&
+ (dapl_llist_is_empty(&hca_ptr->ib_trans.list)))
+ break;
}
dapl_os_unlock(&hca_ptr->ib_trans.lock);
diff --git a/dat/include/dat2/dat_ib_extensions.h b/dat/include/dat2/dat_ib_extensions.h
index 2c082b1..59df1de 100755
--- a/dat/include/dat2/dat_ib_extensions.h
+++ b/dat/include/dat2/dat_ib_extensions.h
@@ -70,8 +70,10 @@
* 2.0.3 - Add query/print counter support for IA, EP, and EVD's
* dat_query_counters(), dat_print_counters()
*
+ * 2.0.4 - Add DAT_IB_UD_CONNECTION_REJECT_EVENT extended UD event
+ *
*/
-#define DAT_IB_EXTENSION_VERSION 203 /* 2.0.3 */
+#define DAT_IB_EXTENSION_VERSION 204 /* 2.0.4 */
#define DAT_ATTR_COUNTERS "DAT_COUNTERS"
#define DAT_IB_ATTR_FETCH_AND_ADD "DAT_IB_FETCH_AND_ADD"
#define DAT_IB_ATTR_CMP_AND_SWAP "DAT_IB_CMP_AND_SWAP"
@@ -89,7 +91,8 @@ typedef enum dat_ib_event_number
{
DAT_IB_DTO_EVENT = DAT_IB_EXTENSION_RANGE_BASE,
DAT_IB_UD_CONNECTION_REQUEST_EVENT,
- DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED
+ DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED,
+ DAT_IB_UD_CONNECTION_REJECT_EVENT
} DAT_IB_EVENT_NUMBER;
--
1.5.2.5
More information about the ofw
mailing list