<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=us-ascii">
<meta name=Generator content="Microsoft Word 11 (filtered medium)">
<style>
<!--
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman";}
a:link, span.MsoHyperlink
{color:blue;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{color:purple;
text-decoration:underline;}
p.MsoPlainText, li.MsoPlainText, div.MsoPlainText
{margin:0in;
margin-bottom:.0001pt;
font-size:10.0pt;
font-family:"Courier New";}
span.EmailStyle17
{mso-style-type:personal-compose;
font-family:Arial;
color:windowtext;}
@page Section1
{size:8.5in 11.0in;
margin:1.0in 1.25in 1.0in 1.25in;}
div.Section1
{page:Section1;}
-->
</style>
</head>
<body lang=EN-US link=blue vlink=purple>
<div class=Section1>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'>James,<o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'>Here are the changes to support async events. Also
consolidated the uAT,uCM,uCQ threads into one processing thread.<o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'>Thanks,<o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'>-arlin<o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoNormal style='text-autospace:none'><font size=2 face="Courier New"><span
style='font-size:10.0pt;font-family:"Courier New"'>Signed-off-by: Arlin Davis <a
href="mailto:ardavis@ichips.intel.com">ardavis@ichips.intel.com</a><o:p></o:p></span></font></p>
<p class=MsoNormal style='text-autospace:none'><font size=2 face="Courier New"><span
style='font-size:10.0pt;font-family:"Courier New"'><o:p> </o:p></span></font></p>
<p class=MsoPlainText><font size=2 face="Courier New"><span style='font-size:
10.0pt'>Index: dapl/openib/dapl_ib_util.c<br>
===================================================================<br>
--- dapl/openib/dapl_ib_util.c (revision 3293)<br>
+++ dapl/openib/dapl_ib_util.c (working copy)<br>
@@ -55,13 +55,14 @@<br>
<br>
#include <stdlib.h><br>
#include <netinet/tcp.h><br>
-#include <sys/utsname.h><br>
-#include <unistd.h> <br>
-#include <fcntl.h><br>
-#include <strings.h><br>
-<br>
-int g_dapl_loopback_connection = 0;<br>
+#include <sys/poll.h><br>
<br>
+int g_dapl_loopback_connection = 0;<br>
+int g_ib_destroy = 0;<br>
+int g_ib_pipe[2];<br>
+DAPL_OS_THREAD g_ib_thread;<br>
+DAPL_OS_LOCK g_hca_lock;<br>
+struct dapl_llist_entry *g_hca_list; <br>
<br>
/* just get IP address, IPv4 only for now */<br>
int dapli_get_hca_addr( struct dapl_hca *hca_ptr )<br>
@@ -130,7 +131,18 @@<br>
int32_t dapls_ib_init (void)<br>
{ <br>
dapl_dbg_log (DAPL_DBG_TYPE_UTIL, " dapl_ib_init: \n" );<br>
- if (dapli_cm_thread_init() || dapli_at_thread_init()) <br>
+<br>
+ /* initialize hca_list lock */<br>
+ dapl_os_lock_init(&g_hca_lock);<br>
+ <br>
+ /* initialize hca list for CQ events */<br>
+ dapl_llist_init_head(&g_hca_list);<br>
+<br>
+ /* create pipe for waking up work thread */<br>
+ if (pipe(g_ib_pipe))<br>
+ return 1;<br>
+<br>
+ if (dapli_ib_thread_init()) <br>
return 1;<br>
<br>
return 0;<br>
@@ -139,8 +151,7 @@<br>
int32_t dapls_ib_release (void)<br>
{<br>
dapl_dbg_log (DAPL_DBG_TYPE_UTIL, " dapl_ib_release: \n" );<br>
- dapli_at_thread_destroy();<br>
- dapli_cm_thread_destroy();<br>
+ dapli_ib_thread_destroy();<br>
return 0;<br>
}<br>
<br>
@@ -196,6 +207,7 @@<br>
ibv_get_device_name(hca_ptr->ib_trans.ib_dev) );<br>
return DAT_INTERNAL_ERROR;<br>
}<br>
+ hca_ptr->ib_trans.ib_ctx = hca_ptr->ib_hca_handle;<br>
<br>
/* set inline max with enviromment or default, get local lid and gid 0 */<br>
hca_ptr->ib_trans.max_inline_send = <br>
@@ -223,19 +235,22 @@<br>
goto bail;<br>
}<br>
<br>
- /* one thread for each device open */<br>
- if (dapli_cq_thread_init(hca_ptr)) {<br>
- dapl_dbg_log (DAPL_DBG_TYPE_ERR, <br>
- " open_hca: cq_thread_init failed for %s\n", <br>
- ibv_get_device_name(hca_ptr->ib_trans.ib_dev) );<br>
- goto bail;<br>
- }<br>
+ /* initialize hca wait object for uAT event */<br>
+ dapl_os_wait_object_init(&hca_ptr->ib_trans.wait_object);<br>
<br>
- /* initialize cq_lock and wait object */<br>
- dapl_os_lock_init(&hca_ptr->ib_trans.cq_lock);<br>
- dapl_os_wait_object_init (&hca_ptr->ib_trans.wait_object);<br>
- <br>
- dapl_dbg_log (DAPL_DBG_TYPE_UTIL, <br>
+ /* <br>
+ * Put new hca_transport on list for async and CQ event processing <br>
+ * Wakeup work thread to add to polling list<br>
+ */<br>
+ dapl_llist_init_entry((DAPL_LLIST_ENTRY*)&hca_ptr->ib_trans.entry);<br>
+ dapl_os_lock( &g_hca_lock );<br>
+ dapl_llist_add_tail(&g_hca_list, <br>
+ (DAPL_LLIST_ENTRY*)&hca_ptr->ib_trans.entry, <br>
+ &hca_ptr->ib_trans.entry);<br>
+ write(g_ib_pipe[1], "w", sizeof "w");<br>
+ dapl_os_unlock(&g_hca_lock);<br>
+ <br>
+ dapl_dbg_log (DAPL_DBG_TYPE_UTIL, <br>
" open_hca: %s, port %d, %s %d.%d.%d.%d
INLINE_MAX=%d\n", <br>
ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
hca_ptr->port_num,<br>
((struct sockaddr_in
*)&hca_ptr->hca_address)->sin_family == AF_INET ?
"AF_INET":"AF_INET6",<br>
@@ -245,7 +260,6 @@<br>
((struct sockaddr_in
*)&hca_ptr->hca_address)->sin_addr.s_addr >> 24 & 0xff,<br>
hca_ptr->ib_trans.max_inline_send );<br>
<br>
-<br>
return DAT_SUCCESS;<br>
<br>
bail:<br>
@@ -276,16 +290,28 @@<br>
dapl_dbg_log (DAPL_DBG_TYPE_UTIL," close_hca: %p->%p\n",<br>
hca_ptr,hca_ptr->ib_hca_handle);<br>
<br>
- dapli_cq_thread_destroy(hca_ptr);<br>
-<br>
if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) {<br>
if (ibv_close_device(hca_ptr->ib_hca_handle)) <br>
return(dapl_convert_errno(errno,"ib_close_device"));<br>
hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;<br>
}<br>
- <br>
- dapl_os_lock_destroy(&hca_ptr->ib_trans.cq_lock);<br>
<br>
+ /* <br>
+ * Remove hca from async and CQ event processing list<br>
+ * Wakeup work thread to remove from polling list<br>
+ */<br>
+ hca_ptr->ib_trans.destroy = 1;<br>
+ write(g_ib_pipe[1], "w", sizeof "w");<br>
+<br>
+ /* wait for thread to remove HCA references */<br>
+ while (hca_ptr->ib_trans.destroy != 2) {<br>
+ struct timespec sleep, remain;<br>
+ sleep.tv_sec = 0;<br>
+ sleep.tv_nsec = 10000000; /* 10 ms */<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL, <br>
+ " ib_thread_destroy: waiting on hca %p
destroy\n");<br>
+ nanosleep (&sleep, &remain);<br>
+ }<br>
return (DAT_SUCCESS);<br>
}<br>
<br>
@@ -432,31 +458,285 @@<br>
IN void *context )<br>
<br>
{<br>
- ib_hca_transport_t *hca_ptr;<br>
+ ib_hca_transport_t *hca_ptr;<br>
<br>
- dapl_dbg_log (DAPL_DBG_TYPE_UTIL,<br>
- " setup_async_cb: ia %p type %d handle %p cb %p ctx
%p\n",<br>
- ia_ptr, handler_type, evd_ptr, callback, context);<br>
-<br>
- hca_ptr = &ia_ptr->hca_ptr->ib_trans;<br>
- switch(handler_type)<br>
- {<br>
- case DAPL_ASYNC_UNAFILIATED:<br>
- hca_ptr->async_unafiliated = callback;<br>
- break;<br>
- case DAPL_ASYNC_CQ_ERROR:<br>
- hca_ptr->async_cq_error = callback;<br>
- break;<br>
- case DAPL_ASYNC_CQ_COMPLETION:<br>
- hca_ptr->async_cq = callback;<br>
- break;<br>
- case DAPL_ASYNC_QP_ERROR:<br>
- hca_ptr->async_qp_error = callback;<br>
- break;<br>
- default:<br>
- break;<br>
- }<br>
- return DAT_SUCCESS;<br>
+ dapl_dbg_log (DAPL_DBG_TYPE_UTIL,<br>
+ " setup_async_cb: ia %p type %d handle %p cb %p ctx
%p\n",<br>
+ ia_ptr, handler_type, evd_ptr, callback, context);<br>
+<br>
+ hca_ptr = &ia_ptr->hca_ptr->ib_trans;<br>
+ switch(handler_type)<br>
+ {<br>
+ case DAPL_ASYNC_UNAFILIATED:<br>
+ hca_ptr->async_unafiliated = callback;<br>
+ hca_ptr->async_un_ctx = context;<br>
+ break;<br>
+ case DAPL_ASYNC_CQ_ERROR:<br>
+ hca_ptr->async_cq_error = callback;<br>
+ hca_ptr->async_cq_ctx = context;<br>
+ break;<br>
+ case DAPL_ASYNC_CQ_COMPLETION:<br>
+ hca_ptr->async_cq = callback;<br>
+ hca_ptr->async_ctx = context;<br>
+ break;<br>
+ case DAPL_ASYNC_QP_ERROR:<br>
+ hca_ptr->async_qp_error = callback;<br>
+ hca_ptr->async_qp_ctx = context;<br>
+ break;<br>
+ default:<br>
+ break;<br>
+ }<br>
+ return DAT_SUCCESS;<br>
}<br>
<br>
+int dapli_ib_thread_init(void)<br>
+{<br>
+ DAT_RETURN dat_status;<br>
+<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,<br>
+ " ib_thread_init(%d)\n", getpid());<br>
+<br>
+ /* create thread to process inbound connect request */<br>
+ dat_status = dapl_os_thread_create(dapli_thread, NULL, &g_ib_thread);<br>
+ if (dat_status != DAT_SUCCESS)<br>
+ {<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, <br>
+ " ib_thread_init: failed to create thread\n");<br>
+ return 1;<br>
+ }<br>
+ return 0;<br>
+}<br>
+<br>
+void dapli_ib_thread_destroy(void)<br>
+{<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,<br>
+ " ib_thread_destroy(%d)\n", getpid());<br>
+<br>
+ /* destroy ib_thread, wait for termination */<br>
+ g_ib_destroy = 1;<br>
+ write(g_ib_pipe[1], "w", sizeof "w");<br>
+ while (g_ib_destroy != 2) {<br>
+ struct timespec sleep, remain;<br>
+ sleep.tv_sec = 0;<br>
+ sleep.tv_nsec = 10000000; /* 10 ms */<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL, <br>
+ " ib_thread_destroy: waiting for
ib_thread\n");<br>
+ nanosleep(&sleep, &remain);<br>
+ }<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,<br>
+ " ib_thread_destroy(%d) exit\n",getpid());<br>
+}<br>
+<br>
+void dapli_async_event_cb(struct _ib_hca_transport *hca)<br>
+{<br>
+ struct ibv_async_event event;<br>
+ struct pollfd async_fd = {<br>
+ .fd = hca->ib_ctx->async_fd,<br>
+ .events = POLLIN,<br>
+ .revents = 0<br>
+ };<br>
+ <br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapli_async_event_cb(%p)\n",hca);<br>
+<br>
+ if (hca->destroy)<br>
+ return;<br>
+<br>
+ if ((poll(&async_fd, 1, 0)==1) &&<br>
+ (!ibv_get_async_event(hca->ib_ctx, &event))) {<br>
+<br>
+ switch (event.event_type) {<br>
+<br>
+ case IBV_EVENT_CQ_ERR:<br>
+ {<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,<br>
+ " dapli_async_event CQ ERR %d\n",<br>
+ event.event_type); <br>
+ <br>
+ /* report up if async callback still setup */<br>
+ if (hca->async_cq_error)<br>
+ hca->async_cq_error(hca->ib_ctx,<br>
+ &event,<br>
+ hca->async_cq_ctx);<br>
+ break;<br>
+ }<br>
+ case IBV_EVENT_COMM_EST:<br>
+ {<br>
+ /* Received messages on connected QP before RTU */<br>
+ struct dapl_ep *ep_ptr =
event.element.qp->qp_context;<br>
+ <br>
+ /* TODO: cannot process COMM_EST until ibv <br>
+ * guarantees valid QP context for events. <br>
+ * Race conditions exist with QP destroy call. <br>
+ * For now, assume the RTU will arrive.<br>
+ */<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,<br>
+ " dapli_async_event COMM_EST
(qp=%p)\n",<br>
+ event.element.qp); <br>
+<br>
+ if (!DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP) &&<br>
+ ep_ptr->cm_handle != IB_INVALID_HANDLE)<br>
+ ib_cm_establish(ep_ptr->cm_handle->cm_id);<br>
+ <br>
+ break;<br>
+ }<br>
+ case IBV_EVENT_QP_FATAL:<br>
+ case IBV_EVENT_QP_REQ_ERR:<br>
+ case IBV_EVENT_QP_ACCESS_ERR:<br>
+ case IBV_EVENT_QP_LAST_WQE_REACHED:<br>
+ case IBV_EVENT_SRQ_ERR:<br>
+ case IBV_EVENT_SRQ_LIMIT_REACHED:<br>
+ case IBV_EVENT_SQ_DRAINED:<br>
+ {<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,<br>
+ " dapli_async_event QP ERR %d\n",<br>
+ event.event_type); <br>
+ <br>
+ /* report up if async callback still setup */<br>
+ if (hca->async_qp_error)<br>
+ hca->async_qp_error(hca->ib_ctx,<br>
+ &event,<br>
+ hca->async_qp_ctx);<br>
+ break;<br>
+ }<br>
+ case IBV_EVENT_PATH_MIG:<br>
+ case IBV_EVENT_PATH_MIG_ERR:<br>
+ case IBV_EVENT_DEVICE_FATAL:<br>
+ case IBV_EVENT_PORT_ACTIVE:<br>
+ case IBV_EVENT_PORT_ERR:<br>
+ case IBV_EVENT_LID_CHANGE:<br>
+ case IBV_EVENT_PKEY_CHANGE:<br>
+ case IBV_EVENT_SM_CHANGE:<br>
+ {<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,<br>
+ " dapli_async_event DEV ERR %d\n",<br>
+ event.event_type); <br>
+<br>
+ /* report up if async callback still setup */<br>
+ if (hca->async_unafiliated)<br>
+ hca->async_unafiliated( <br>
+ hca->ib_ctx,<br>
+ &event,<br>
+ hca->async_un_ctx);<br>
+ break;<br>
+ }<br>
+ default:<br>
+ {<br>
+ dapl_dbg_log (DAPL_DBG_TYPE_WARN,<br>
+ "--> DsEventCb: UNKNOWN\n");<br>
+ break;<br>
+ }<br>
+ }<br>
+ ibv_put_async_event(&event);<br>
+ }<br>
+}<br>
+<br>
+<br>
+/* work thread for uAT, uCM, CQ, and async events */<br>
+void dapli_thread(void *arg) <br>
+{<br>
+ struct pollfd ufds[__FD_SETSIZE];<br>
+ struct _ib_hca_transport *uhca[__FD_SETSIZE]={NULL};<br>
+ struct _ib_hca_transport *hca;<br>
+ int ret,idx,fds;<br>
+ char rbuf[2];<br>
+<br>
+ dapl_dbg_log (DAPL_DBG_TYPE_UTIL,<br>
+ " ib_thread(%d,0x%x): ENTER: pipe %d cm %d at
%d\n",<br>
+ getpid(), g_ib_thread, <br>
+ g_ib_pipe[0], ib_cm_get_fd(), <br>
+ ib_at_get_fd());<br>
+<br>
+ /* Poll across pipe, CM, AT never changes */<br>
+ dapl_os_lock( &g_hca_lock );<br>
+ <br>
+ ufds[0].fd = g_ib_pipe[0]; /* pipe */<br>
+ ufds[0].events = POLLIN;<br>
+ ufds[1].fd = ib_cm_get_fd(); /* uCM */<br>
+ ufds[1].events = POLLIN;<br>
+ ufds[2].fd = ib_at_get_fd(); /* uAT */<br>
+ ufds[2].events = POLLIN;<br>
+ <br>
+ while (!g_ib_destroy) {<br>
+ <br>
+ /* build ufds after pipe, cm, at events */<br>
+ ufds[0].revents = 0;<br>
+ ufds[1].revents = 0;<br>
+ ufds[2].revents = 0;<br>
+ idx=2;<br>
+<br>
+ /* Walk HCA list and setup async and CQ events */<br>
+ if (!dapl_llist_is_empty(&g_hca_list))<br>
+ hca = dapl_llist_peek_head(&g_hca_list);<br>
+ else<br>
+ hca = NULL;<br>
+ <br>
+ while(hca) {<br>
+ int i;<br>
+ ufds[++idx].fd = hca->ib_ctx->async_fd; /* uASYNC */<br>
+ ufds[idx].events = POLLIN;<br>
+ ufds[idx].revents = 0;<br>
+ uhca[idx] = hca;<br>
+ for (i=0;i<hca->ib_ctx->num_comp;i++) { /* uCQ */<br>
+ ufds[++idx].fd = hca->ib_ctx->cq_fd[i]; <br>
+ ufds[idx].events = POLLIN;<br>
+ ufds[idx].revents = 0;<br>
+ uhca[idx] = hca;<br>
+ }<br>
+ hca = dapl_llist_next_entry(<br>
+ &g_hca_list,<br>
+ (DAPL_LLIST_ENTRY*)&hca->entry);<br>
+ }<br>
+ <br>
+ /* unlock, and setup poll */<br>
+ fds = idx+1;<br>
+ dapl_os_unlock(&g_hca_lock);<br>
+ ret = poll(ufds, fds, -1); <br>
+ if (ret <= 0) {<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,<br>
+ " ib_thread(%d): ERR %s poll\n",<br>
+ getpid(),strerror(errno));<br>
+ dapl_os_lock(&g_hca_lock);<br>
+ continue;<br>
+ }<br>
+<br>
+ /* check and process CQ and ASYNC events, each open device */<br>
+ for(idx=3;idx<fds;idx++) {<br>
+ if (ufds[idx].revents == POLLIN) {<br>
+ dapli_cq_event_cb(uhca[idx]);<br>
+ dapli_async_event_cb(uhca[idx]);<br>
+ }<br>
+ }<br>
+ <br>
+ /* check and process user events */<br>
+ if (ufds[0].revents == POLLIN) {<br>
+<br>
+ read(g_ib_pipe[0], rbuf, 2);<br>
+ <br>
+ /* cleanup any device on list marked for destroy */<br>
+ for(idx=3;idx<fds;idx++) {<br>
+ if(uhca[idx] && uhca[idx]->destroy == 1) {<br>
+ dapl_os_lock(&g_hca_lock);<br>
+ dapl_llist_remove_entry(<br>
+ &g_hca_list, <br>
+ (DAPL_LLIST_ENTRY*)<br>
+ &uhca[idx]->entry);<br>
+ dapl_os_unlock(&g_hca_lock);<br>
+ uhca[idx]->destroy = 2;<br>
+ }<br>
+ }<br>
+ }<br>
+<br>
+ /* CM and AT events */<br>
+ if (ufds[1].revents == POLLIN)<br>
+ dapli_cm_event_cb();<br>
+<br>
+ if (ufds[2].revents == POLLIN)<br>
+ dapli_at_event_cb();<br>
+<br>
+ dapl_os_lock(&g_hca_lock);<br>
+ }<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL," ib_thread(%d)
EXIT\n",getpid());<br>
+ g_ib_destroy = 2;<br>
+ dapl_os_unlock(&g_hca_lock); <br>
+}<br>
<br>
Index: dapl/openib/dapl_ib_cm.c<br>
===================================================================<br>
--- dapl/openib/dapl_ib_cm.c (revision 3293)<br>
+++ dapl/openib/dapl_ib_cm.c (working copy)<br>
@@ -70,90 +70,6 @@<br>
static inline uint64_t cpu_to_be64(uint64_t x) { return x; }<br>
#endif<br>
<br>
-static int g_at_destroy;<br>
-static DAPL_OS_THREAD g_at_thread;<br>
-static int g_cm_destroy;<br>
-static DAPL_OS_THREAD g_cm_thread;<br>
-static DAPL_OS_LOCK g_cm_lock;<br>
-static struct dapl_llist_entry *g_cm_list; <br>
-<br>
-int dapli_cm_thread_init(void)<br>
-{<br>
- DAT_RETURN dat_status;<br>
-<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," cm_thread_init(%d)\n",
getpid());<br>
-<br>
- /* initialize cr_list lock */<br>
- dapl_os_lock_init(&g_cm_lock);<br>
- <br>
- /* initialize CM list for listens on this HCA */<br>
- dapl_llist_init_head(&g_cm_list);<br>
-<br>
- /* create thread to process inbound connect request */<br>
- dat_status = dapl_os_thread_create(cm_thread, NULL, &g_cm_thread);<br>
- if (dat_status != DAT_SUCCESS)<br>
- {<br>
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, <br>
- " cm_thread_init: failed to create thread\n");<br>
- return 1;<br>
- }<br>
- return 0;<br>
-}<br>
-<br>
-void dapli_cm_thread_destroy(void)<br>
-{<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," cm_thread_destroy(%d)\n",
getpid());<br>
-<br>
- /* destroy cr_thread and lock */<br>
- g_cm_destroy = 1;<br>
- pthread_kill( g_cm_thread, SIGUSR1 );<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," cm_thread_destroy(%d) SIGUSR1
sent\n",getpid());<br>
- while (g_cm_destroy) {<br>
- struct timespec sleep, remain;<br>
- sleep.tv_sec = 0;<br>
- sleep.tv_nsec = 10000000; /* 10 ms */<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM, <br>
- " cm_thread_destroy: waiting for
cm_thread\n");<br>
- nanosleep (&sleep, &remain);<br>
- }<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," cm_thread_destroy(%d)
exit\n",getpid());<br>
-}<br>
-<br>
-int dapli_at_thread_init(void)<br>
-{<br>
- DAT_RETURN dat_status;<br>
-<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," at_thread_init(%d)\n",
getpid());<br>
-<br>
- /* create thread to process AT async requests */<br>
- dat_status = dapl_os_thread_create(at_thread, NULL, &g_at_thread);<br>
- if (dat_status != DAT_SUCCESS)<br>
- {<br>
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, <br>
- " at_thread_init: failed to create thread\n");<br>
- return 1;<br>
- }<br>
- return 0;<br>
-}<br>
-<br>
-void dapli_at_thread_destroy(void)<br>
-{<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," at_thread_destroy(%d)\n",
getpid());<br>
-<br>
- /* destroy cr_thread and lock */<br>
- g_at_destroy = 1;<br>
- pthread_kill( g_at_thread, SIGUSR1 );<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," at_thread_destroy(%d) SIGUSR1
sent\n",getpid());<br>
- while (g_at_destroy) {<br>
- struct timespec sleep, remain;<br>
- sleep.tv_sec = 0;<br>
- sleep.tv_nsec = 10000000; /* 10 ms */<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM, <br>
- " at_thread_destroy: waiting for
at_thread\n");<br>
- nanosleep (&sleep, &remain);<br>
- }<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," at_thread_destroy(%d)
exit\n",getpid());<br>
-}<br>
<br>
void dapli_ip_comp_handler(uint64_t req_id, void *context, int rec_num)<br>
{<br>
@@ -348,12 +264,6 @@<br>
if (conn->ep)<br>
conn->ep->cm_handle = IB_INVALID_HANDLE;<br>
<br>
- /* take off the CM thread work queue and free */<br>
- dapl_os_lock( &g_cm_lock );<br>
- dapl_llist_remove_entry(&g_cm_list, <br>
- (DAPL_LLIST_ENTRY*)&conn->entry);<br>
- dapl_os_unlock(&g_cm_lock);<br>
-<br>
dapl_os_free(conn, sizeof(*conn));<br>
}<br>
}<br>
@@ -426,8 +336,8 @@<br>
if (new_conn) {<br>
<br>
(void)dapl_os_memzero(new_conn, sizeof(*new_conn));<br>
- dapl_os_lock_init(&new_conn->lock);<br>
new_conn->cm_id = event->cm_id; /* provided by uCM */<br>
+ event->cm_id->context = new_conn; /* update CM_ID context */<br>
new_conn->sp = conn->sp;<br>
new_conn->hca = conn->hca;<br>
new_conn->service_id = conn->service_id;<br>
@@ -444,13 +354,6 @@<br>
event->param.req_rcvd.primary_path,<br>
sizeof(struct ib_sa_path_rec));<br>
<br>
- /* put new CR on CM thread event work queue */<br>
- dapl_llist_init_entry((DAPL_LLIST_ENTRY*)&new_conn->entry);<br>
- dapl_os_lock( &g_cm_lock );<br>
- dapl_llist_add_tail(&g_cm_list, <br>
- (DAPL_LLIST_ENTRY*)&new_conn->entry, new_conn);<br>
- dapl_os_unlock(&g_cm_lock);<br>
-<br>
dapl_dbg_log(DAPL_DBG_TYPE_CM, " passive_cb: "<br>
"REQ on HCA %p SP %p SID %d L_ID %d new_id %d
p_data %p\n",<br>
new_conn->hca, new_conn->sp, <br>
@@ -521,18 +424,13 @@<br>
if (conn->ep)<br>
conn->ep->cm_handle = IB_INVALID_HANDLE;<br>
<br>
- /* take off the CM thread work queue and free */<br>
- dapl_os_lock( &g_cm_lock );<br>
- dapl_llist_remove_entry(&g_cm_list, <br>
- (DAPL_LLIST_ENTRY*)&conn->entry);<br>
- dapl_os_unlock(&g_cm_lock);<br>
dapl_os_free(conn, sizeof(*conn));<br>
}<br>
return(destroy);<br>
}<br>
<br>
static int dapli_cm_passive_cb(struct dapl_cm_id *conn,<br>
- struct ib_cm_event *event)<br>
+ struct ib_cm_event *event)<br>
{<br>
int destroy;<br>
struct dapl_cm_id *new_conn;<br>
@@ -541,9 +439,6 @@<br>
" passive_cb: conn %p id %d event %d\n",<br>
conn, conn->cm_id, event->event );<br>
<br>
- if (conn->cm_id == 0)<br>
- return 0;<br>
-<br>
dapl_os_lock(&conn->lock);<br>
if (conn->destroy) {<br>
dapl_os_unlock(&conn->lock);<br>
@@ -608,155 +503,11 @@<br>
if (conn->ep)<br>
conn->ep->cm_handle = IB_INVALID_HANDLE;<br>
<br>
- /* take off the CM thread work queue and free */<br>
- dapl_os_lock( &g_cm_lock );<br>
- dapl_llist_remove_entry(&g_cm_list, <br>
- (DAPL_LLIST_ENTRY*)&conn->entry);<br>
- dapl_os_unlock(&g_cm_lock);<br>
-<br>
dapl_os_free(conn, sizeof(*conn));<br>
}<br>
return(destroy);<br>
}<br>
<br>
-/* something to catch the signal */<br>
-static void ib_sig_handler(int signum)<br>
-{<br>
- return;<br>
-}<br>
-<br>
-/* async CM processing thread */<br>
-void cm_thread(void *arg) <br>
-{<br>
- struct dapl_cm_id *conn, *next_conn;<br>
- struct ib_cm_event *event;<br>
- struct pollfd ufds;<br>
- sigset_t sigset;<br>
-<br>
- dapl_dbg_log (DAPL_DBG_TYPE_CM,<br>
- " cm_thread(%d,0x%x): ENTER: cm_fd %d\n",<br>
- getpid(), g_cm_thread, ib_cm_get_fd());<br>
-<br>
- sigemptyset(&sigset);<br>
- sigaddset(&sigset, SIGUSR1);<br>
- pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);<br>
- signal( SIGUSR1, ib_sig_handler); <br>
- <br>
- dapl_os_lock( &g_cm_lock );<br>
- while (!g_cm_destroy) {<br>
- struct ib_cm_id *cm_id;<br>
- int ret;<br>
-<br>
- /* select for CM event, all events process via cm_fd */<br>
- ufds.fd = ib_cm_get_fd();<br>
- ufds.events = POLLIN;<br>
- ufds.revents = 0;<br>
-<br>
- dapl_os_unlock(&g_cm_lock);<br>
- ret = poll(&ufds, 1, -1); <br>
- if (ret <= 0) {<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
- " cm_thread(%d): ERR %s poll\n",<br>
- getpid(),strerror(errno));<br>
- dapl_os_lock(&g_cm_lock);<br>
- continue;<br>
- }<br>
-<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
- " cm_thread: GET EVENT fd=%d n=%d\n",<br>
- ib_cm_get_fd(),ret);<br>
-<br>
- if (ib_cm_event_get_timed(0,&event)) { <br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
- " cm_thread: ERR %s event_get on %d\n", <br>
- strerror(errno), ib_cm_get_fd() );<br>
- dapl_os_lock(&g_cm_lock);<br>
- continue;<br>
- }<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
- " cm_thread: GET EVENT fd=%d
woke\n",ib_cm_get_fd());<br>
- dapl_os_lock(&g_cm_lock);<br>
-<br>
- /* set proper cm_id */<br>
- if (event->event == IB_CM_REQ_RECEIVED ||<br>
- event->event == IB_CM_SIDR_REQ_RECEIVED)<br>
- cm_id = event->param.req_rcvd.listen_id;<br>
- else<br>
- cm_id = event->cm_id;<br>
-<br>
- dapl_dbg_log (DAPL_DBG_TYPE_CM,<br>
- " cm_thread: EVENT event(%d) cm_id=%d (%d)\n",<br>
- event->event, event->cm_id, cm_id );<br>
-<br>
- /* <br>
- * Walk cm_list looking for connection id in event<br>
- * no need to walk if uCM would provide context with event<br>
- */<br>
- if (!dapl_llist_is_empty(&g_cm_list))<br>
- next_conn = dapl_llist_peek_head(&g_cm_list);<br>
- else<br>
- next_conn = NULL;<br>
-<br>
- ret = 0;<br>
- while (next_conn) {<br>
- conn = next_conn;<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
- " cm_thread: LIST cm %p c_id %d e_id %d)\n", <br>
- conn, conn->cm_id, cm_id );<br>
- <br>
- next_conn = dapl_llist_next_entry(<br>
- &g_cm_list,<br>
- (DAPL_LLIST_ENTRY*)&conn->entry );<br>
-<br>
- if (cm_id == conn->cm_id) {<br>
- dapl_os_unlock(&g_cm_lock);<br>
- if (conn->sp)<br>
- ret = dapli_cm_passive_cb(conn,event);<br>
- else<br>
- ret = dapli_cm_active_cb(conn,event);<br>
- dapl_os_lock(&g_cm_lock);<br>
- break;<br>
- }<br>
- } <br>
- ib_cm_event_put(event);<br>
- if (ret) {<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
- " cm_thread: destroy cm_id %d\n",cm_id);<br>
- ib_cm_destroy_id(cm_id);<br>
- }<br>
- }<br>
- dapl_os_unlock(&g_cm_lock); <br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," cm_thread(%d) EXIT,
cm_list=%s\n",<br>
- getpid(),dapl_llist_is_empty(&g_cm_list) ?
"EMPTY":"NOT EMPTY");<br>
- g_cm_destroy = 0;<br>
-}<br>
-<br>
-/* async AT processing thread */<br>
-void at_thread(void *arg) <br>
-{<br>
- sigset_t sigset;<br>
-<br>
- dapl_dbg_log (DAPL_DBG_TYPE_CM,<br>
- " at_thread(%d,0x%x): ENTER: at_fd %d\n",<br>
- getpid(), g_at_thread, ib_at_get_fd());<br>
-<br>
- sigemptyset(&sigset);<br>
- sigaddset(&sigset, SIGUSR1);<br>
- pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);<br>
- signal(SIGUSR1, ib_sig_handler); <br>
- <br>
- while (!g_at_destroy) {<br>
- /* poll forever until callback or signal */<br>
- if (ib_at_callback_get_timed(-1) < 0) { <br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
- " at_thread: SIG? ret=%s, destroy=%d\n", <br>
- strerror(errno), g_at_destroy );<br>
- }<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," at_thread: callback
woke\n");<br>
- }<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," at_thread(%d) EXIT \n",
getpid());<br>
- g_at_destroy = 0;<br>
-}<br>
<br>
/************************ DAPL provider entry points **********************/<br>
<br>
@@ -853,13 +604,6 @@<br>
conn->retries = 0;<br>
dapl_os_memcpy(&conn->r_addr, r_addr, sizeof(DAT_SOCK_ADDR6));<br>
<br>
- /* put on CM thread work queue */<br>
- dapl_llist_init_entry((DAPL_LLIST_ENTRY*)&conn->entry);<br>
- dapl_os_lock( &g_cm_lock );<br>
- dapl_llist_add_tail(&g_cm_list, <br>
- (DAPL_LLIST_ENTRY*)&conn->entry, conn);<br>
- dapl_os_unlock(&g_cm_lock);<br>
-<br>
status = ib_at_route_by_ip(<br>
((struct sockaddr_in *)&conn->r_addr)->sin_addr.s_addr, <br>
((struct sockaddr_in
*)&conn->hca->hca_address)->sin_addr.s_addr, <br>
@@ -1019,13 +763,6 @@<br>
conn->hca = ia_ptr->hca_ptr;<br>
conn->service_id = ServiceID;<br>
<br>
- /* put on CM thread work queue */<br>
- dapl_llist_init_entry((DAPL_LLIST_ENTRY*)&conn->entry);<br>
- dapl_os_lock( &g_cm_lock );<br>
- dapl_llist_add_tail(&g_cm_list, <br>
- (DAPL_LLIST_ENTRY*)&conn->entry, conn);<br>
- dapl_os_unlock(&g_cm_lock);<br>
-<br>
dapl_dbg_log(DAPL_DBG_TYPE_EP,<br>
" setup_listener(conn=%p cm_id=%d)\n",<br>
sp_ptr->cm_srvc_handle,conn->cm_id);<br>
@@ -1345,8 +1082,6 @@<br>
return size;<br>
}<br>
<br>
-#ifndef SOCKET_CM<br>
-<br>
/*<br>
* Map all socket CM event codes to the DAT equivelent.<br>
*/<br>
@@ -1457,7 +1192,44 @@<br>
return ib_cm_event;<br>
}<br>
<br>
-#endif<br>
+void dapli_cm_event_cb()<br>
+{<br>
+ struct ib_cm_event *event;<br>
+ <br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapli_cm_event()\n");<br>
+<br>
+ /* process one CM event, fairness */<br>
+ if(!ib_cm_event_get_timed(0,&event)) {<br>
+ struct dapl_cm_id *conn;<br>
+ int ret;<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
+ " dapli_cm_event: EVENT=%p ID=%p CTX=%p\n",<br>
+ event->event, event->cm_id, <br>
+ event->cm_id->context);<br>
+ <br>
+ /* set proper conn from cm_id context*/<br>
+ conn = (struct dapl_cm_id*)event->cm_id->context;<br>
+<br>
+ if (conn->sp) <br>
+ ret = dapli_cm_passive_cb(conn,event);<br>
+ else <br>
+ ret = dapli_cm_active_cb(conn,event);<br>
+ <br>
+ ib_cm_event_put(event);<br>
+<br>
+ if (ret) <br>
+ ib_cm_destroy_id(conn->cm_id);<br>
+ }<br>
+}<br>
+<br>
+void dapli_at_event_cb()<br>
+{<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapli_at_event_cb()\n");<br>
+<br>
+ /* process one AT event, fairness */<br>
+ ib_at_callback_get_timed(0);<br>
+}<br>
+<br>
<br>
/*<br>
* Local variables:<br>
Index: dapl/openib/dapl_ib_util.h<br>
===================================================================<br>
--- dapl/openib/dapl_ib_util.h (revision 3293)<br>
+++ dapl/openib/dapl_ib_util.h (working copy)<br>
@@ -231,18 +231,22 @@<br>
/* ib_hca_transport_t, specific to this implementation */<br>
typedef struct _ib_hca_transport<br>
{ <br>
- struct ibv_device *ib_dev;<br>
+ struct ib_llist_entry entry;<br>
+ int destroy;<br>
+ struct ibv_device *ib_dev;<br>
+ struct ibv_context *ib_ctx;<br>
ib_cq_handle_t ib_cq_empty;<br>
- DAPL_OS_LOCK cq_lock;<br>
DAPL_OS_WAIT_OBJECT wait_object;<br>
- int cq_destroy;<br>
- DAPL_OS_THREAD cq_thread;<br>
int max_inline_send;<br>
union ibv_gid gid;<br>
ib_async_handler_t async_unafiliated;<br>
+ void *async_un_ctx;<br>
ib_async_handler_t async_cq_error;<br>
+ void *async_ctx;<br>
ib_async_handler_t async_cq;<br>
+ void *async_cq_ctx;<br>
ib_async_handler_t async_qp_error;<br>
+ void *async_qp_ctx;<br>
<br>
} ib_hca_transport_t;<br>
<br>
@@ -252,21 +256,15 @@<br>
/* prototypes */<br>
int32_t dapls_ib_init (void);<br>
int32_t dapls_ib_release (void);<br>
-void cm_thread (void *arg);<br>
-int dapli_cm_thread_init(void);<br>
-void dapli_cm_thread_destroy(void);<br>
-void at_thread (void *arg);<br>
-int dapli_at_thread_init(void);<br>
-void dapli_at_thread_destroy(void);<br>
-void cq_thread (void *arg);<br>
-int dapli_cq_thread_init(struct dapl_hca *hca_ptr);<br>
-void dapli_cq_thread_destroy(struct dapl_hca *hca_ptr);<br>
-<br>
-int dapli_get_lid(struct dapl_hca *hca_ptr, int port, uint16_t *lid);<br>
-int dapli_get_gid(struct dapl_hca *hca_ptr, int port, int index, <br>
- union ibv_gid *gid);<br>
-int dapli_get_hca_addr(struct dapl_hca *hca_ptr);<br>
+void dapli_thread(void *arg);<br>
+int dapli_ib_thread_init(void);<br>
+void dapli_ib_thread_destroy(void);<br>
+int dapli_get_hca_addr(struct dapl_hca *hca_ptr);<br>
void dapli_ip_comp_handler(uint64_t req_id, void *context, int rec_num);<br>
+void dapli_cm_event_cb(void);<br>
+void dapli_at_event_cb(void);<br>
+void dapli_cq_event_cb(struct _ib_hca_transport *hca);<br>
+void dapli_async_event_cb(struct _ib_hca_transport *hca);<br>
<br>
DAT_RETURN<br>
dapls_modify_qp_state ( IN ib_qp_handle_t qp_handle,<br>
Index: dapl/openib/dapl_ib_cq.c<br>
===================================================================<br>
--- dapl/openib/dapl_ib_cq.c (revision 3293)<br>
+++ dapl/openib/dapl_ib_cq.c (working copy)<br>
@@ -52,94 +52,40 @@<br>
#include "dapl_evd_util.h"<br>
#include "dapl_ring_buffer_util.h"<br>
#include <sys/poll.h><br>
-#include <signal.h><br>
<br>
-int dapli_cq_thread_init(struct dapl_hca *hca_ptr)<br>
+void dapli_cq_event_cb(struct _ib_hca_transport *hca)<br>
{<br>
- DAT_RETURN dat_status;<br>
+ int i;<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL," dapli_cq_event_cb(%p)\n",
hca);<br>
<br>
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread_init(%p)\n",
hca_ptr);<br>
-<br>
- /* create thread to process inbound connect request */<br>
- dat_status = dapl_os_thread_create( cq_thread,
(void*)hca_ptr,&hca_ptr->ib_trans.cq_thread);<br>
- if (dat_status != DAT_SUCCESS)<br>
- {<br>
- dapl_dbg_log(DAPL_DBG_TYPE_ERR,<br>
- " cq_thread_init: failed to create
thread\n");<br>
- return 1;<br>
- }<br>
- return 0;<br>
-}<br>
-<br>
-void dapli_cq_thread_destroy(struct dapl_hca *hca_ptr)<br>
-{<br>
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread_destroy(%p)\n",
hca_ptr);<br>
-<br>
- /* destroy cr_thread and lock */<br>
- hca_ptr->ib_trans.cq_destroy = 1;<br>
- pthread_kill(hca_ptr->ib_trans.cq_thread, SIGUSR1);<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM," cq_thread_destroy(%p) SIGUSR1
sent\n",hca_ptr);<br>
- while (hca_ptr->ib_trans.cq_destroy != 2) {<br>
- struct timespec sleep, remain;<br>
- sleep.tv_sec = 0;<br>
- sleep.tv_nsec = 10000000; /* 10 ms */<br>
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL,<br>
- " cq_thread_destroy: waiting for
cq_thread\n");<br>
- nanosleep (&sleep, &remain);<br>
- }<br>
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread_destroy(%d)
exit\n",getpid());<br>
- return;<br>
-}<br>
-<br>
-/* something to catch the signal */<br>
-static void ib_cq_handler(int signum)<br>
-{<br>
- return;<br>
-}<br>
-<br>
-void cq_thread( void *arg )<br>
-{<br>
- struct dapl_hca *hca_ptr = arg;<br>
- struct dapl_evd *evd_ptr;<br>
- struct ibv_cq *ibv_cq = NULL;<br>
- sigset_t sigset;<br>
- int status = 0; <br>
-<br>
- dapl_dbg_log ( DAPL_DBG_TYPE_UTIL," cq_thread: ENTER hca
%p\n",hca_ptr);<br>
- <br>
- sigemptyset(&sigset);<br>
- sigaddset(&sigset,SIGUSR1);<br>
- pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);<br>
- signal(SIGUSR1, ib_cq_handler);<br>
-<br>
- /* wait on DTO event, or signal to abort */<br>
- while (!hca_ptr->ib_trans.cq_destroy) {<br>
-<br>
- struct pollfd cq_poll = {<br>
- .fd = hca_ptr->ib_hca_handle->cq_fd[0],<br>
+ /* check all comp events on this device */<br>
+ for(i=0;i<hca->ib_ctx->num_comp;i++) {<br>
+ struct dapl_evd *evd_ptr = NULL;<br>
+ struct ibv_cq *ibv_cq = NULL;<br>
+ struct pollfd cq_fd = {<br>
+ .fd = hca->ib_ctx->cq_fd[i],<br>
.events = POLLIN,<br>
.revents = 0<br>
};<br>
-<br>
- status = poll(&cq_poll, 1, -1);<br>
- if ((status == 1) &&<br>
- (!ibv_get_cq_event(hca_ptr->ib_hca_handle, 0, &ibv_cq,
(void*)&evd_ptr))) {<br>
- <br>
+ if ((poll(&cq_fd, 1, 0) == 1) &&<br>
+ (!ibv_get_cq_event(hca->ib_ctx, i, <br>
+ &ibv_cq, (void*)&evd_ptr))) {<br>
+<br>
+ /* <br>
+ * TODO: ibv put event to protect against<br>
+ * destroy CQ race conditions?<br>
+ */<br>
if (DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD))<br>
continue;<br>
<br>
/* process DTO event via callback */<br>
- dapl_evd_dto_callback ( evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle,<br>
+ dapl_evd_dto_callback ( hca->ib_ctx,<br>
evd_ptr->ib_cq_handle,<br>
(void*)evd_ptr );<br>
- } else {<br>
-<br>
- }<br>
- } <br>
- hca_ptr->ib_trans.cq_destroy = 2;<br>
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread: EXIT: hca %p \n",
hca_ptr);<br>
- return;<br>
+ } <br>
+ }<br>
}<br>
+<br>
/*<br>
* Map all verbs DTO completion codes to the DAT equivelent.<br>
*<br>
<o:p> </o:p></span></font></p>
<p class=MsoNormal style='text-autospace:none'><font size=2 face="Courier New"><span
style='font-size:10.0pt;font-family:"Courier New"'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
</div>
</body>
</html>