<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:st1="urn:schemas-microsoft-com:office:smarttags" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv=Content-Type content="text/html; charset=us-ascii">
<meta name=Generator content="Microsoft Word 11 (filtered medium)">
<o:SmartTagType namespaceuri="urn:schemas-microsoft-com:office:smarttags"
name="PersonName"/>
<!--[if !mso]>
<style>
st1\:*{behavior:url(#default#ieooui) }
</style>
<![endif]-->
<style>
<!--
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman";}
a:link, span.MsoHyperlink
{color:blue;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{color:purple;
text-decoration:underline;}
p.MsoPlainText, li.MsoPlainText, div.MsoPlainText
{margin:0in;
margin-bottom:.0001pt;
font-size:10.0pt;
font-family:"Courier New";}
span.EmailStyle17
{mso-style-type:personal-compose;
font-family:Arial;
color:windowtext;}
@page Section1
{size:8.5in 11.0in;
margin:1.0in 1.25in 1.0in 1.25in;}
div.Section1
{page:Section1;}
-->
</style>
</head>
<body lang=EN-US link=blue vlink=purple>
<div class=Section1>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'>James,<o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'>Here are some changes to support ib_cm_init_qp_attr() and the
cm event processing on a per device basis.<o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'>Also, added copyright credits for kDAPL cm work that was
used in uDAPL.<o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoPlainText><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'>Attachment included.<o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'>Thanks,<o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'>-arlin<o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoNormal style='text-autospace:none'><font size=2 face="Courier New"><span
style='font-size:10.0pt;font-family:"Courier New"'>Signed-off by: Arlin Davis
<<st1:PersonName w:st="on">ardavis@ichips.intel.com</st1:PersonName>><o:p></o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoPlainText><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
<p class=MsoPlainText><font size=2 face="Courier New"><span style='font-size:
10.0pt'>Index: dapl/openib/TODO<br>
===================================================================<br>
--- dapl/openib/TODO (revision 3459)<br>
+++ dapl/openib/TODO (working copy)<br>
@@ -7,8 +7,6 @@<br>
DAPL:<br>
- reinit EP needs a QP timewait completion notification<br>
- direct cq_wait_object when multi-CQ verbs event support arrives<br>
-- async event support<br>
-- add support for ib_cm_init_qp_attr <br>
- shared receive queue support<br>
<br>
Under discussion:<br>
Index: dapl/openib/dapl_ib_util.c<br>
===================================================================<br>
--- dapl/openib/dapl_ib_util.c (revision
3459)<br>
+++ dapl/openib/dapl_ib_util.c (working
copy)<br>
@@ -56,6 +56,7 @@<br>
#include <stdlib.h><br>
#include <netinet/tcp.h><br>
#include <sys/poll.h><br>
+#include <fcntl.h><br>
<br>
int g_dapl_loopback_connection
= 0;<br>
int g_ib_destroy
= 0;<br>
@@ -85,26 +86,49 @@<br>
at_rec.retries = 0;<br>
<br>
/* call with async_comp until the sync
version works */<br>
- status =
ib_at_ips_by_gid(&hca_ptr->ib_trans.gid,
&ipv4_addr->sin_addr.s_addr, 1, <br>
+ status = ib_at_ips_by_gid(&hca_ptr->ib_trans.gid,
<br>
+
&ipv4_addr->sin_addr.s_addr, 1, <br>
&at_comp, &at_rec.req_id);<br>
<br>
if (status < 0) {<br>
dapl_dbg_log
(DAPL_DBG_TYPE_ERR, <br>
-
" get_hca_addr: ERR ips_by_gid %d %s \n",<br>
+
" ips_by_gid: ERR ips_by_gid %d %s \n",<br>
status,
strerror(errno));<br>
return 1;<br>
}<br>
<br>
dapl_dbg_log (DAPL_DBG_TYPE_UTIL, <br>
-
" get_hca_addr: ips_by_gid ret %d at_rec %p -> id %lld\n",<br>
+
" ips_by_gid: RET %d at_rec %p -> id %lld\n",<br>
status,
&at_rec, at_rec.req_id );<br>
<br>
if (status > 0) { <br>
dapli_ip_comp_handler(at_rec.req_id, (void*)&at_rec, status);<br>
} else {<br>
- dat_status =
dapl_os_wait_object_wait(&hca_ptr->ib_trans.wait_object,500000);<br>
- if (dat_status !=
DAT_SUCCESS)<br>
- ib_at_cancel(at_rec.req_id);<br>
+ /* limit the
resolution and cancel times */<br>
+ dat_status =
dapl_os_wait_object_wait(<br>
+ &hca_ptr->ib_trans.wait_object,<br>
+ 500000);<br>
+ if (dat_status !=
DAT_SUCCESS) {<br>
+ dapl_dbg_log(<br>
+ DAPL_DBG_TYPE_UTIL,
<br>
+ "
ips_by_gid: REQ TIMEOUT, cancel %lld\n",<br>
+ at_rec.req_id);<br>
+ <br>
+ /*
<br>
+
* FIX: indeterministic<br>
+
* AT may or may not provide -EINTR event <br>
+
*/<br>
+ ib_at_cancel(at_rec.req_id);
<br>
+ <br>
+ if
(dapl_os_wait_object_wait(<br>
+ &hca_ptr->ib_trans.wait_object,<br>
+ 500000)
!= DAT_SUCCESS)<br>
+ {<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
<br>
+ "
ips_by_gid: cancel %lld failed\n",<br>
+ at_rec.req_id);<br>
+ }<br>
+ }<br>
}<br>
<br>
if (!ipv4_addr->sin_addr.s_addr) <br>
@@ -130,6 +154,7 @@<br>
*/<br>
int32_t dapls_ib_init (void)<br>
{ <br>
+ long opts;<br>
dapl_dbg_log (DAPL_DBG_TYPE_UTIL, "
dapl_ib_init: \n" );<br>
<br>
/* initialize hca_list lock */<br>
@@ -142,6 +167,11 @@<br>
if (pipe(g_ib_pipe))<br>
return 1;<br>
<br>
+ /* set AT fd to non-blocking */ <br>
+ opts = fcntl(ib_at_get_fd(), F_GETFL);<br>
+ if (fcntl(ib_at_get_fd(), F_SETFL, opts | O_NONBLOCK)
< 0)<br>
+ return 1;<br>
+<br>
if (dapli_ib_thread_init()) <br>
return 1;<br>
<br>
@@ -177,6 +207,8 @@<br>
IN DAPL_HCA *hca_ptr)<br>
{<br>
struct dlist *dev_list;<br>
+ long opts;<br>
+ int i;<br>
<br>
dapl_dbg_log (DAPL_DBG_TYPE_UTIL, <br>
" open_hca: %s - %p\n", hca_name, hca_ptr );<br>
@@ -227,10 +259,10 @@<br>
(unsigned long long)bswap_64(hca_ptr->ib_trans.gid.global.subnet_prefix),<br>
(unsigned long long)bswap_64(hca_ptr->ib_trans.gid.global.interface_id) );<br>
<br>
- /* get the IP address of the device */<br>
+ /* get the IP address of the device using GID */<br>
if (dapli_get_hca_addr(hca_ptr)) {<br>
dapl_dbg_log
(DAPL_DBG_TYPE_ERR, <br>
-
" open_hca: IB get ADDR failed for %s\n", <br>
+
" open_hca: ERR ib_at_ips_by_gid for %s\n", <br>
ibv_get_device_name(hca_ptr->ib_trans.ib_dev) );<br>
goto bail;<br>
}<br>
@@ -238,6 +270,34 @@<br>
/* initialize hca wait object for uAT event */<br>
dapl_os_wait_object_init(&hca_ptr->ib_trans.wait_object);<br>
<br>
+ /* set event FD's to non-blocking */<br>
+ opts = fcntl(hca_ptr->ib_hca_handle->async_fd,
F_GETFL); /* uASYNC */<br>
+ if (opts < 0 ||
fcntl(hca_ptr->ib_hca_handle->async_fd, <br>
+
F_SETFL, opts | O_NONBLOCK) < 0) {<br>
+ dapl_dbg_log
(DAPL_DBG_TYPE_ERR, <br>
+
" open_hca: ERR with async FD\n" );<br>
+ goto bail;<br>
+ }<br>
+ for
(i=0;i<hca_ptr->ib_hca_handle->num_comp;i++) { /* uCQ */<br>
+ opts =
fcntl(hca_ptr->ib_hca_handle->cq_fd[i], F_GETFL);<br>
+ if (opts < 0
|| fcntl(hca_ptr->ib_hca_handle->async_fd, <br>
+
F_SETFL, opts | O_NONBLOCK) < 0) {<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
<br>
+
" open_hca: ERR with CQ FD\n");<br>
+ goto
bail;<br>
+ }<br>
+ } <br>
+ <br>
+ /* Get CM device handle for events, and set to
non-blocking */ <br>
+ hca_ptr->ib_trans.ib_cm =
ib_cm_get_device(hca_ptr->ib_hca_handle);<br>
+ opts = fcntl(hca_ptr->ib_trans.ib_cm->fd, F_GETFL);
/* uCM */<br>
+ if (opts < 0 ||
fcntl(hca_ptr->ib_trans.ib_cm->fd, <br>
+
F_SETFL, opts | O_NONBLOCK) < 0) {<br>
+ dapl_dbg_log
(DAPL_DBG_TYPE_ERR, <br>
+
" open_hca: ERR with CM FD\n" );<br>
+ goto bail;<br>
+ }<br>
+ <br>
/* <br>
* Put new hca_transport on list for async
and CQ event processing <br>
* Wakeup work thread to add to polling
list<br>
@@ -509,20 +569,34 @@<br>
<br>
void dapli_ib_thread_destroy(void)<br>
{<br>
+ int retries = 10;<br>
dapl_dbg_log(DAPL_DBG_TYPE_UTIL,<br>
" ib_thread_destroy(%d)\n", getpid());<br>
<br>
- /* destroy ib_thread, wait for termination */<br>
+ /* <br>
+ * wait for async thread to terminate. <br>
+ * pthread_join would be the correct method<br>
+ * but some applications have some issues<br>
+ */<br>
+ <br>
+ /* destroy ib_thread, wait for termination, if not
already */<br>
+ dapl_os_lock( &g_hca_lock );<br>
g_ib_destroy = 1;<br>
write(g_ib_pipe[1], "w", sizeof
"w");<br>
- while (g_ib_destroy != 2) {<br>
+ <br>
+ while ((g_ib_destroy != 2) && (retries--)) {<br>
struct
timespec sleep, remain;<br>
sleep.tv_sec
= 0;<br>
- sleep.tv_nsec =
10000000; /* 10 ms */<br>
+ sleep.tv_nsec =
20000000; /* 20 ms */<br>
dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
<br>
-
" ib_thread_destroy: waiting for ib_thread\n");<br>
+ "
ib_thread_destroy: waiting for ib_thread\n");<br>
+ write(g_ib_pipe[1],
"w", sizeof "w");<br>
+ dapl_os_unlock(
&g_hca_lock );<br>
nanosleep(&sleep,
&remain);<br>
+ dapl_os_lock(
&g_hca_lock );<br>
}<br>
+ dapl_os_unlock( &g_hca_lock );<br>
+ <br>
dapl_dbg_log(DAPL_DBG_TYPE_UTIL,<br>
" ib_thread_destroy(%d) exit\n",getpid());<br>
}<br>
@@ -639,30 +713,26 @@<br>
struct _ib_hca_transport *hca;<br>
int
ret,idx,fds;<br>
char
rbuf[2];<br>
-<br>
+ <br>
dapl_dbg_log (DAPL_DBG_TYPE_UTIL,<br>
-
" ib_thread(%d,0x%x): ENTER: pipe %d cm %d at %d\n",<br>
+
" ib_thread(%d,0x%x): ENTER: pipe %d at %d\n",<br>
getpid(), g_ib_thread, <br>
-
g_ib_pipe[0], ib_cm_get_fd(), <br>
-
ib_at_get_fd());<br>
+
g_ib_pipe[0], ib_at_get_fd());<br>
<br>
/* Poll across pipe, CM, AT never changes */<br>
dapl_os_lock( &g_hca_lock );<br>
<br>
ufds[0].fd = g_ib_pipe[0]; /*
pipe */<br>
ufds[0].events = POLLIN;<br>
- ufds[1].fd = ib_cm_get_fd(); /* uCM */<br>
+ ufds[1].fd = ib_at_get_fd(); /* uAT */<br>
ufds[1].events = POLLIN;<br>
- ufds[2].fd = ib_at_get_fd(); /* uAT */<br>
- ufds[2].events = POLLIN;<br>
- <br>
+ <br>
while (!g_ib_destroy) {<br>
<br>
- /* build ufds
after pipe, cm, at events */<br>
+ /* build ufds
after pipe, at events */<br>
ufds[0].revents
= 0;<br>
ufds[1].revents
= 0;<br>
- ufds[2].revents =
0;<br>
- idx=2;<br>
+ idx=1;<br>
<br>
/*
Walk HCA list and setup async and CQ events */<br>
if
(!dapl_llist_is_empty(&g_hca_list))<br>
@@ -672,6 +742,10 @@<br>
<br>
while(hca) {<br>
int
i;<br>
+ ufds[++idx].fd
= hca->ib_cm->fd; /* uCM */<br>
+ ufds[idx].events
= POLLIN;<br>
+ ufds[idx].revents
= 0;<br>
+ uhca[idx]
= hca;<br>
ufds[++idx].fd
= hca->ib_ctx->async_fd; /* uASYNC */<br>
ufds[idx].events
= POLLIN;<br>
ufds[idx].revents
= 0;<br>
@@ -699,10 +773,11 @@<br>
continue;<br>
}<br>
<br>
- /* check and
process CQ and ASYNC events, each open device */<br>
- for(idx=3;idx<fds;idx++)
{<br>
+ /* check and
process CM, CQ and ASYNC events, per device */<br>
+ for(idx=2;idx<fds;idx++)
{<br>
if
(ufds[idx].revents == POLLIN) {<br>
dapli_cq_event_cb(uhca[idx]);<br>
+ dapli_cm_event_cb(uhca[idx]);<br>
dapli_async_event_cb(uhca[idx]);<br>
}<br>
}<br>
@@ -726,11 +801,8 @@<br>
}<br>
}<br>
<br>
- /* CM and AT
events */<br>
+ /* AT events */<br>
if
(ufds[1].revents == POLLIN)<br>
- dapli_cm_event_cb();<br>
-<br>
- if
(ufds[2].revents == POLLIN)<br>
dapli_at_event_cb();<br>
<br>
dapl_os_lock(&g_hca_lock);<br>
@@ -738,5 +810,6 @@<br>
dapl_dbg_log(DAPL_DBG_TYPE_UTIL,"
ib_thread(%d) EXIT\n",getpid());<br>
g_ib_destroy = 2;<br>
dapl_os_unlock(&g_hca_lock); <br>
+ pthread_exit(NULL);<br>
}<br>
<br>
Index: dapl/openib/dapl_ib_cm.c<br>
===================================================================<br>
--- dapl/openib/dapl_ib_cm.c (revision 3459)<br>
+++ dapl/openib/dapl_ib_cm.c (working copy)<br>
@@ -42,7 +42,11 @@<br>
*<br>
* $Id: $<br>
*<br>
- * Copyright (c) 2005 Intel Corporation. All rights
reserved.<br>
+ * Copyright (c) 2005 Voltaire Inc. All rights reserved.<br>
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.<br>
+ * Copyright (c) 2004-2005, Mellanox Technologies, Inc. All rights reserved. <br>
+ * Copyright (c) 2003 Topspin Corporation. All rights reserved. <br>
+ * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.<br>
*<br>
**************************************************************************/<br>
<br>
@@ -78,11 +82,11 @@<br>
int status;<br>
<br>
dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
-
" ip_comp_handler: at_rec %p ->id %lld id %lld rec_num %d %x\n",<br>
+
" ip_comp_handler: rec %p ->id %lld id %lld num %d %x\n",<br>
context, at_rec->req_id, req_id, rec_num,<br>
ipv4_addr->sin_addr.s_addr);<br>
<br>
- if (rec_num <= 0) {<br>
+ if (rec_num <= 0) {<br>
struct
ib_at_completion at_comp;<br>
<br>
dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
@@ -91,7 +95,7 @@<br>
<br>
ipv4_addr->sin_addr.s_addr
= 0;<br>
<br>
- if
(++at_rec->retries > IB_MAX_AT_RETRY) <br>
+ if
((++at_rec->retries > IB_MAX_AT_RETRY) || (rec_num == -EINTR))<br>
goto bail;<br>
<br>
at_comp.fn =
dapli_ip_comp_handler;<br>
@@ -103,9 +107,9 @@<br>
if (status
< 0) <br>
goto
bail;<br>
<br>
- dapl_dbg_log
(DAPL_DBG_TYPE_UTIL,<br>
-
" ip_comp_handler: NEW ips_by_gid ret %d at_rec %p -> id %lld\n",<br>
-
status, at_rec, at_rec->req_id );<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,<br>
+ "
ip_comp_handler: ips_by_gid %d rec %p->id %lld\n",<br>
+ status,
at_rec, at_rec->req_id );<br>
} <br>
<br>
if (ipv4_addr->sin_addr.s_addr)<br>
@@ -114,7 +118,7 @@<br>
return;<br>
bail:<br>
dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
-
" ip_comp_handler: ERR: at_rec %p, req_id %lld rec_num %d\n",<br>
+
" ip_comp_handler: ERR: at_rec %p, id %lld num %d\n",<br>
at_rec, req_id, rec_num);<br>
<br>
dapl_os_wait_object_wakeup(at_rec->wait_object);<br>
@@ -130,23 +134,35 @@<br>
" path_comp_handler: ctxt %p, req_id %lld rec_num %d\n",<br>
context, req_id, rec_num);<br>
<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_CM, <br>
+ "
path_comp_handler: SRC GID subnet %016llx id %016llx\n",<br>
+ (unsigned long
long)cpu_to_be64(conn->dapl_rt.sgid.global.subnet_prefix),<br>
+ (unsigned long
long)cpu_to_be64(conn->dapl_rt.sgid.global.interface_id) );<br>
+<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_CM, <br>
+ "
path_comp_handler: DST GID subnet %016llx id %016llx\n",<br>
+ (unsigned long
long)cpu_to_be64(conn->dapl_rt.dgid.global.subnet_prefix),<br>
+ (unsigned long
long)cpu_to_be64(conn->dapl_rt.dgid.global.interface_id) );<br>
+<br>
if (rec_num <= 0) {<br>
dapl_dbg_log(DAPL_DBG_TYPE_CM,
<br>
-
" path_comp_handler: resolution err %d retry %d\n",<br>
+
" path_comp_handler: ERR %d retry %d\n",<br>
rec_num, conn->retries + 1);<br>
if
(++conn->retries > IB_MAX_AT_RETRY) {<br>
dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
-
" path_comp_handler: ep_ptr 0x%p\n",conn->ep);<br>
+ "
path_comp_handler: ERR no PATH (ep=%p)\n",<br>
+ conn->ep);<br>
event
= IB_CME_DESTINATION_UNREACHABLE;<br>
goto
bail;<br>
}<br>
<br>
status =
ib_at_paths_by_route(&conn->dapl_rt, 0,<br>
&conn->dapl_path, 1,<br>
-
&conn->dapl_comp, &conn->dapl_comp.req_id);<br>
+
&conn->dapl_comp, <br>
+
&conn->dapl_comp.req_id);<br>
if (status)
{<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
-
" path_by_route: err %d id %lld\n",<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,<br>
+
" path_by_route: retry ERR %d id %lld\n",<br>
status, conn->dapl_comp.req_id);<br>
event
= IB_CME_LOCAL_FAILURE;<br>
goto
bail;<br>
@@ -185,20 +201,9 @@<br>
" rt_comp_handler: conn %p, req_id %lld rec_num %d\n",<br>
conn, req_id, rec_num);<br>
<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM, <br>
-
" rt_comp_handler: SRC GID subnet %016llx id %016llx\n",<br>
-
(unsigned long long)cpu_to_be64(conn->dapl_rt.sgid.global.subnet_prefix),<br>
-
(unsigned long long)cpu_to_be64(conn->dapl_rt.sgid.global.interface_id) );<br>
-<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM, <br>
-
" rt_comp_handler: DST GID subnet %016llx id %016llx\n",<br>
-
(unsigned long long)cpu_to_be64(conn->dapl_rt.dgid.global.subnet_prefix),<br>
-
(unsigned long long)cpu_to_be64(conn->dapl_rt.dgid.global.interface_id) );<br>
-<br>
-<br>
if (rec_num <= 0) {<br>
dapl_dbg_log(DAPL_DBG_TYPE_ERR,<br>
-
" dapl_rt_comp_handler: rec %d retry %d\n",<br>
+
" dapl_rt_comp_handler: ERROR rec %d retry %d\n",<br>
rec_num, conn->retries+1 ); <br>
<br>
if
(++conn->retries > IB_MAX_AT_RETRY) {<br>
@@ -206,10 +211,11 @@<br>
goto
bail;<br>
}<br>
<br>
- status =
ib_at_route_by_ip(((struct sockaddr_in
*)&conn->r_addr)->sin_addr.s_addr,<br>
-
0, 0, IB_AT_ROUTE_FORCE_ATS, <br>
-
&conn->dapl_rt, <br>
-
&conn->dapl_comp,&conn->dapl_comp.req_id);<br>
+ status =
ib_at_route_by_ip(<br>
+ ((struct
sockaddr_in *)&conn->r_addr)->sin_addr.s_addr,<br>
+ 0,
0, IB_AT_ROUTE_FORCE_ATS, <br>
+ &conn->dapl_rt,
<br>
+ &conn->dapl_comp,&conn->dapl_comp.req_id);<br>
if (status
< 0) {<br>
dapl_dbg_log(DAPL_DBG_TYPE_ERR,
"dapl_rt_comp_handler: "<br>
"ib_at_route_by_ip failed with status %d\n",<br>
@@ -223,9 +229,10 @@<br>
return;<br>
}<br>
<br>
- if (!conn->dapl_rt.dgid.global.subnet_prefix ||
req_id != conn->dapl_comp.req_id) {<br>
+ if (!conn->dapl_rt.dgid.global.subnet_prefix || <br>
+ req_id != conn->dapl_comp.req_id)
{<br>
dapl_dbg_log(DAPL_DBG_TYPE_ERR,<br>
-
" dapl_rt_comp_handler: ERROR: unexpected callback req_id=%d(%d)\n",<br>
+
" dapl_rt_comp_handler: ERROR: cb id=%d(%d)\n",<br>
req_id, conn->dapl_comp.req_id ); <br>
return;<br>
}<br>
@@ -234,11 +241,13 @@<br>
conn->dapl_comp.context = conn;<br>
conn->retries = 0;<br>
status =
ib_at_paths_by_route(&conn->dapl_rt, 0, &conn->dapl_path, 1,<br>
-
&conn->dapl_comp, &conn->dapl_comp.req_id);<br>
+
&conn->dapl_comp, <br>
+
&conn->dapl_comp.req_id);<br>
if (status) {<br>
dapl_dbg_log(DAPL_DBG_TYPE_ERR,<br>
"dapl_rt_comp_handler: ib_at_paths_by_route "<br>
-
"returned %d id %lld\n", status, conn->dapl_comp.req_id);<br>
+
"returned %d id %lld\n", status, <br>
+
conn->dapl_comp.req_id);<br>
event =
IB_CME_LOCAL_FAILURE;<br>
goto bail;<br>
}<br>
@@ -284,19 +293,16 @@<br>
/* move QP state to RTR and RTS */<br>
/* TODO: could use a ib_cm_init_qp_attr() call
here */<br>
dapl_dbg_log(DAPL_DBG_TYPE_CM, <br>
-
" rep_recv: RTR_RTS: cm_id %d r_qp 0x%x r_lid 0x%x r_SID %d\n",<br>
+
" rep_recv: RTR_RTS: id %d rqp %x rlid %x rSID %d\n",<br>
conn->cm_id,event->param.rep_rcvd.remote_qpn,<br>
-
ntohs(conn->req.primary_path->dlid),conn->service_id );<br>
+
ntohs(conn->req.primary_path->dlid),conn->service_id);<br>
<br>
if ( dapls_modify_qp_state(
conn->ep->qp_handle, <br>
-
IBV_QPS_RTR, <br>
-
event->param.rep_rcvd.remote_qpn,<br>
-
ntohs(conn->req.primary_path->dlid),<br>
-
1 ) != DAT_SUCCESS )<br>
+
IBV_QPS_RTR, conn ) != DAT_SUCCESS )<br>
goto disc;<br>
<br>
if ( dapls_modify_qp_state(
conn->ep->qp_handle, <br>
-
IBV_QPS_RTS,0,0,0 ) != DAT_SUCCESS)<br>
+
IBV_QPS_RTS, conn ) != DAT_SUCCESS)<br>
goto disc; <br>
<br>
<br>
@@ -356,10 +362,10 @@<br>
sizeof(struct
ib_sa_path_rec));<br>
<br>
dapl_dbg_log(DAPL_DBG_TYPE_CM,
" passive_cb: "<br>
- "REQ
on HCA %p SP %p SID %d L_ID %d new_id %d p_data %p\n",<br>
- new_conn->hca,
new_conn->sp, <br>
- conn->service_id,
conn->cm_id, new_conn->cm_id, <br>
- event->private_data
);<br>
+ "REQ
on HCA %p SP %p SID %d LID %d new_id %d pd %p\n",<br>
+ new_conn->hca,
new_conn->sp, <br>
+ conn->service_id,
conn->cm_id, new_conn->cm_id, <br>
+ event->private_data
);<br>
<br>
}<br>
return new_conn;<br>
@@ -454,7 +460,8 @@<br>
new_conn =
dapli_req_recv(conn,event);<br>
<br>
if
(new_conn) <br>
- dapls_cr_callback(new_conn,
IB_CME_CONNECTION_REQUEST_PENDING, <br>
+ dapls_cr_callback(new_conn,
<br>
+
IB_CME_CONNECTION_REQUEST_PENDING, <br>
event->private_data, new_conn->sp);<br>
break;<br>
case IB_CM_REP_ERROR:<br>
@@ -468,7 +475,7 @@<br>
case IB_CM_RTU_RECEIVED:<br>
/* move QP
to RTS state */<br>
if (
dapls_modify_qp_state(conn->ep->qp_handle, <br>
-
IBV_QPS_RTS,0,0,0 ) != DAT_SUCCESS) {<br>
+
IBV_QPS_RTS, conn ) != DAT_SUCCESS) {<br>
dapls_cr_callback(conn,
IB_CME_LOCAL_FAILURE, <br>
NULL, conn->sp);<br>
} else {<br>
@@ -556,7 +563,7 @@<br>
ep_ptr = (DAPL_EP*)ep_handle;<br>
qp_ptr = ep_ptr->qp_handle;<br>
<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM, " connect: r_SID
%d, pdata %p, plen %d\n", <br>
+ dapl_dbg_log(DAPL_DBG_TYPE_CM, " connect: rSID
%d, pdata %p, ln %d\n", <br>
r_qual,p_data,p_size);<br>
<br>
/* Allocate CM and initialize lock */<br>
@@ -573,20 +580,21 @@<br>
<br>
conn->ep = ep_ptr;<br>
conn->hca =
ep_ptr->header.owner_ia->hca_ptr;<br>
- status =
ib_cm_create_id(conn->hca->ib_hca_handle, &conn->cm_id, conn);<br>
+ status = ib_cm_create_id(conn->hca->ib_hca_handle,<br>
+
&conn->cm_id, conn);<br>
if (status < 0) {<br>
dat_status =
dapl_convert_errno(errno,"create_cm_id");<br>
dapl_os_free(conn,
sizeof(*conn));<br>
return
dat_status;<br>
}<br>
- conn->ep->cm_handle = conn;<br>
+ ep_ptr->cm_handle = conn;<br>
<br>
/* Setup QP/CM parameters */<br>
(void)dapl_os_memzero(&conn->req,sizeof(conn->req));<br>
conn->service_id = r_qual;<br>
conn->req.qp_num =
ep_ptr->qp_handle->qp_num;<br>
conn->req.qp_type = IBV_QPT_RC;<br>
- conn->req.starting_psn = 1;<br>
+ conn->req.starting_psn = ep_ptr->qp_handle->qp_num;<br>
conn->req.private_data = p_data;<br>
conn->req.private_data_len = p_size;<br>
conn->req.peer_to_peer = 0;<br>
@@ -607,14 +615,14 @@<br>
<br>
status = ib_at_route_by_ip(<br>
((struct
sockaddr_in *)&conn->r_addr)->sin_addr.s_addr, <br>
- ((struct sockaddr_in
*)&conn->hca->hca_address)->sin_addr.s_addr, <br>
- 0,
IB_AT_ROUTE_FORCE_ATS, &conn->dapl_rt, &conn->dapl_comp, <br>
- &conn->dapl_comp.req_id);<br>
-<br>
- dapl_dbg_log(DAPL_DBG_TYPE_CM, " connect:
at_route ret=%d,%s req_id %d GID %016llx %016llx\n", <br>
-
status, strerror(errno), conn->dapl_comp.req_id,<br>
-
(unsigned long long)cpu_to_be64(conn->dapl_rt.dgid.global.subnet_prefix),<br>
-
(unsigned long long)cpu_to_be64(conn->dapl_rt.dgid.global.interface_id) );<br>
+ ((struct
sockaddr_in *)&conn->hca->hca_address)->sin_addr.s_addr,<br>
+ 0, 0,
&conn->dapl_rt, &conn->dapl_comp,
&conn->dapl_comp.req_id);<br>
+<br>
+ dapl_dbg_log(DAPL_DBG_TYPE_CM, <br>
+ " connect:
at_route requested(ret=%d,id=%d): SRC %x DST %x\n", <br>
+ status,
conn->dapl_comp.req_id,<br>
+ ((struct sockaddr_in
*)&conn->hca->hca_address)->sin_addr.s_addr,<br>
+ ((struct sockaddr_in
*)&conn->r_addr)->sin_addr.s_addr);<br>
<br>
if (status < 0) {<br>
dat_status =
dapl_convert_errno(errno,"ib_at_route_by_ip");<br>
@@ -653,7 +661,7 @@<br>
int status;<br>
<br>
dapl_dbg_log (DAPL_DBG_TYPE_CM,<br>
- "
disconnect(ep_handle %p, conn %p, cm_id %d flags %x)\n",<br>
+ "
disconnect(ep %p, conn %p, id %d flags %x)\n",<br>
ep_ptr,conn,
(conn?conn->cm_id:0),close_flags);<br>
<br>
if (conn == IB_INVALID_HANDLE)<br>
@@ -703,7 +711,7 @@<br>
dapls_ib_disconnect(ep_ptr,
DAT_CLOSE_ABRUPT_FLAG);<br>
<br>
if (ep_ptr->qp_handle != IB_INVALID_HANDLE) <br>
- dapls_modify_qp_state(ep_ptr->qp_handle,
IBV_QPS_ERR, 0,0,0);<br>
+ dapls_modify_qp_state(ep_ptr->qp_handle,
IBV_QPS_ERR, 0 );<br>
}<br>
<br>
/*<br>
@@ -749,8 +757,8 @@<br>
return
DAT_INTERNAL_ERROR;<br>
}<br>
<br>
- status =
ib_cm_create_id(ia_ptr->hca_ptr->ib_hca_handle, &conn->cm_id,<br>
-
conn);<br>
+ status =
ib_cm_create_id(ia_ptr->hca_ptr->ib_hca_handle,<br>
+
&conn->cm_id, conn);<br>
if (status < 0) {<br>
dat_status =
dapl_convert_errno(errno,"create_cm_id");<br>
dapl_os_free(conn,
sizeof(*conn));<br>
@@ -758,8 +766,8 @@<br>
}<br>
<br>
dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
-
" setup_listener(ia_ptr %p SID %d sp_ptr %p conn %p cm_id %d)\n",<br>
-
ia_ptr, ServiceID, sp_ptr, conn, conn->cm_id );<br>
+ "
setup_listener(ia_ptr %p SID %d sp %p conn %p id %d)\n",<br>
+ ia_ptr,
ServiceID, sp_ptr, conn, conn->cm_id );<br>
<br>
sp_ptr->cm_srvc_handle = conn;<br>
conn->sp = sp_ptr;<br>
@@ -864,7 +872,7 @@<br>
conn = cr_ptr->ib_cm_handle;<br>
<br>
dapl_dbg_log (DAPL_DBG_TYPE_CM,<br>
-
" accept_connection(cr %p conn %p, cm_id %d, p_data %p, p_sz=%d)\n",<br>
+
" accept(cr %p conn %p, id %d, p_data %p, p_sz=%d)\n",<br>
cr_ptr, conn, conn->cm_id, p_data, p_size );<br>
<br>
/* Obtain size of private data structure &
contents */<br>
@@ -888,13 +896,9 @@<br>
}<br>
}<br>
<br>
- /* move QP to RTR state, TODO fix port setting */<br>
- /* TODO: could use a ib_cm_init_qp_attr() call here
*/<br>
+ /* move QP to RTR state */<br>
dat_status =
dapls_modify_qp_state(ep_ptr->qp_handle, <br>
-
IBV_QPS_RTR, <br>
-
conn->req_rcvd.remote_qpn, <br>
-
ntohs(conn->req_rcvd.primary_path->dlid),<br>
-
1 ); <br>
+
IBV_QPS_RTR, conn); <br>
if (dat_status != DAT_SUCCESS ) {<br>
dapl_dbg_log(DAPL_DBG_TYPE_ERR,<br>
" accept: modify_qp_state failed: %d\n",<br>
@@ -910,6 +914,7 @@<br>
passive_params.private_data = p_data;<br>
passive_params.private_data_len = p_size;<br>
passive_params.qp_num =
ep_ptr->qp_handle->qp_num;<br>
+ passive_params.starting_psn =
ep_ptr->qp_handle->qp_num;<br>
passive_params.responder_resources = IB_TARGET_MAX;<br>
passive_params.initiator_depth =
IB_INITIATOR_DEPTH;<br>
passive_params.rnr_retry_count =
IB_RNR_RETRY_COUNT;<br>
@@ -1156,8 +1161,8 @@<br>
}<br>
}<br>
dapl_dbg_log (DAPL_DBG_TYPE_CALLBACK,<br>
- "dapls_ib_get_dat_event:
event translate(%s) ib=0x%x dat=0x%x\n",<br>
- active ?
"active" : "passive", ib_cm_event, dat_event_num);<br>
+ "dapls_ib_get_dat_event:
event(%s) ib=0x%x dat=0x%x\n",<br>
+ active ?
"active" : "passive", ib_cm_event, dat_event_num);<br>
<br>
return dat_event_num;<br>
}<br>
@@ -1195,14 +1200,15 @@<br>
return ib_cm_event;<br>
}<br>
<br>
-void dapli_cm_event_cb()<br>
+<br>
+void dapli_cm_event_cb(struct _ib_hca_transport *hca)<br>
{<br>
struct ib_cm_event *event;<br>
<br>
dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "
dapli_cm_event()\n");<br>
<br>
/* process one CM event, fairness */<br>
- if(!ib_cm_get_event_timed(0,&event)) {<br>
+ if(!ib_cm_get_event(hca->ib_cm,&event)) {<br>
struct
dapl_cm_id *conn;<br>
int ret;<br>
dapl_dbg_log(DAPL_DBG_TYPE_CM,<br>
Index: dapl/openib/dapl_ib_qp.c<br>
===================================================================<br>
--- dapl/openib/dapl_ib_qp.c (revision 3459)<br>
+++ dapl/openib/dapl_ib_qp.c (working copy)<br>
@@ -112,7 +112,7 @@<br>
qp_create.qp_type = IBV_QPT_RC;<br>
qp_create.qp_context = (void*)ep_ptr;<br>
<br>
- ep_ptr->qp_handle = ibv_create_qp( ib_pd_handle,
&qp_create);<br>
+ ep_ptr->qp_handle = ibv_create_qp(ib_pd_handle,
&qp_create);<br>
if (!ep_ptr->qp_handle) <br>
return(dapl_convert_errno(ENOMEM,
"create_qp"));<br>
<br>
@@ -121,10 +121,10 @@<br>
ep_ptr->qp_handle->qp_num,<br>
qp_create.cap.max_send_wr,qp_create.cap.max_send_sge,<br>
qp_create.cap.max_recv_wr,qp_create.cap.max_recv_sge
);<br>
-<br>
+ <br>
/* Setup QP attributes for INIT state on the way
out */ <br>
if (dapls_modify_qp_state(ep_ptr->qp_handle,<br>
-
IBV_QPS_INIT,0,0,0 ) != DAT_SUCCESS ) {<br>
+
IBV_QPS_INIT, 0) != DAT_SUCCESS ) {<br>
ibv_destroy_qp(ep_ptr->qp_handle); <br>
ep_ptr->qp_handle
= IB_INVALID_HANDLE;<br>
return
DAT_INTERNAL_ERROR;<br>
@@ -161,7 +161,7 @@<br>
<br>
if (ep_ptr->qp_handle != IB_INVALID_HANDLE) {<br>
/* force
error state to flush queue, then destroy */<br>
- dapls_modify_qp_state(ep_ptr->qp_handle,
IBV_QPS_ERR, 0,0,0);<br>
+ dapls_modify_qp_state(ep_ptr->qp_handle,
IBV_QPS_ERR, 0);<br>
<br>
if
(ibv_destroy_qp(ep_ptr->qp_handle)) <br>
return(dapl_convert_errno(errno,"destroy_qp"));<br>
@@ -217,7 +217,7 @@<br>
(ep_ptr->qp_handle->state != IBV_QPS_ERR)) {<br>
ep_ptr->qp_state
= IB_QP_STATE_ERROR;<br>
return
(dapls_modify_qp_state(ep_ptr->qp_handle, <br>
-
IBV_QPS_ERR,0,0,0));<br>
+
IBV_QPS_ERR, 0));<br>
}<br>
<br>
/*<br>
@@ -272,8 +272,8 @@<br>
<br>
if ( ep_ptr->qp_handle != IB_INVALID_HANDLE )
{<br>
/* move to
RESET state and then to INIT */<br>
- dapls_modify_qp_state(ep_ptr->qp_handle,
IBV_QPS_RESET, 0,0,0);<br>
- dapls_modify_qp_state(ep_ptr->qp_handle,
IBV_QPS_INIT, 0,0,0);<br>
+ dapls_modify_qp_state(ep_ptr->qp_handle,
IBV_QPS_RESET, 0);<br>
+ dapls_modify_qp_state(ep_ptr->qp_handle,
IBV_QPS_INIT, 0);<br>
ep_ptr->qp_state
= IB_QP_STATE_INIT;<br>
}<br>
<br>
@@ -283,104 +283,101 @@<br>
}<br>
<br>
/* <br>
- * Generic QP modify for init, reset, error, RTS, RTR<br>
+ * Generic QP modify for reset, error, INIT, RTS, RTR<br>
*/<br>
DAT_RETURN<br>
dapls_modify_qp_state ( IN ib_qp_handle_t qp_handle,<br>
IN
ib_qp_state_t qp_state,<br>
- IN
uint32_t qpn,<br>
- IN
uint16_t lid,<br>
- IN
uint8_t port )<br>
+ IN
struct dapl_cm_id *conn )<br>
{<br>
- struct ibv_qp_attr qp_attr;<br>
- enum ibv_qp_attr_mask mask =
IBV_QP_STATE;<br>
- <br>
- dapl_os_memzero((void*)&qp_attr,
sizeof(qp_attr));<br>
- qp_attr.qp_state = qp_state;<br>
+ struct ibv_qp_attr attr;<br>
+ int mask
= 0;<br>
+ <br>
+ dapl_dbg_log (DAPL_DBG_TYPE_EP, <br>
+
" modify_qp: qp %p, state %d qp_num 0x%x\n", <br>
+
qp_handle, qp_state, qp_handle->qp_num);<br>
+<br>
+ dapl_os_memzero((void*)&attr, sizeof(attr));<br>
+ attr.qp_state = qp_state;<br>
<br>
switch (qp_state) {<br>
- /* additional
attributes with RTR and RTS */<br>
- case IBV_QPS_RTR:<br>
- {<br>
- mask
|=
IBV_QP_AV
|<br>
- IBV_QP_PATH_MTU
|<br>
- IBV_QP_DEST_QPN
|<br>
- IBV_QP_RQ_PSN
|<br>
- IBV_QP_MAX_DEST_RD_ATOMIC
|<br>
- IBV_QP_MIN_RNR_TIMER;<br>
- qp_attr.qp_state
= IBV_QPS_RTR;<br>
- qp_attr.path_mtu
= IBV_MTU_1024;<br>
- qp_attr.dest_qp_num
= qpn;<br>
- qp_attr.rq_psn
=
1;<br>
- qp_attr.max_dest_rd_atomic =
IB_TARGET_MAX;<br>
- qp_attr.min_rnr_timer =
12;<br>
- qp_attr.ah_attr.is_global =
0;<br>
- qp_attr.ah_attr.dlid =
lid;<br>
- qp_attr.ah_attr.sl =
0;<br>
- qp_attr.ah_attr.src_path_bits =
0;<br>
- qp_attr.ah_attr.port_num =
port;<br>
-<br>
- dapl_dbg_log
(DAPL_DBG_TYPE_EP,<br>
-
" modify_qp_RTR: qpn %x lid %x port %x, rq_psn %x\n",<br>
-
qpn,lid,port,ntohl(qp_attr.rq_psn) );<br>
- break;<br>
-<br>
- } <br>
- case IBV_QPS_RTS:
<br>
- {<br>
- mask
|=
IBV_QP_TIMEOUT
|<br>
- IBV_QP_RETRY_CNT
|<br>
- IBV_QP_RNR_RETRY
|<br>
- IBV_QP_SQ_PSN
|<br>
- IBV_QP_MAX_QP_RD_ATOMIC;<br>
- qp_attr.qp_state =
IBV_QPS_RTS;<br>
- qp_attr.timeout =
14;<br>
- qp_attr.retry_cnt =
7;<br>
- qp_attr.rnr_retry =
7;<br>
- qp_attr.sq_psn =
1;<br>
- qp_attr.max_rd_atomic =
IB_TARGET_MAX;<br>
- dapl_dbg_log
(DAPL_DBG_TYPE_EP,<br>
-
" modify_qp_RTS: psn %x or %x\n",<br>
-
ntohl(qp_attr.sq_psn), qp_attr.max_rd_atomic );<br>
- break;<br>
- }<br>
- case
IBV_QPS_INIT: <br>
+ case IBV_QPS_INIT:<br>
{<br>
DAPL_IA *ia_ptr;<br>
DAPL_EP *ep_ptr;
<br>
+ <br>
/*
need to find way back to port num */<br>
ep_ptr
= (DAPL_EP*)qp_handle->qp_context;<br>
if
(ep_ptr)<br>
ia_ptr
= ep_ptr->header.owner_ia;<br>
else<br>
- break;<br>
+ return(dapl_convert_errno(EINVAL,"
qp_CTX"));<br>
<br>
- mask
|= IBV_QP_PKEY_INDEX |<br>
- IBV_QP_PORT |<br>
- IBV_QP_ACCESS_FLAGS;<br>
-<br>
- qp_attr.pkey_index
= 0;<br>
- qp_attr.port_num
= ia_ptr->hca_ptr->port_num;<br>
- qp_attr.qp_access_flags
= <br>
+ /*
<br>
+
* Set qp attributes by hand for INIT state. Allows<br>
+
* consumers to pre-post receives, per uDAPL<br>
+
* specification, before IB has path record info<br>
+
* with connect request processing<br>
+
*/ <br>
+ mask
= IBV_QP_STATE | IBV_QP_PKEY_INDEX |<br>
+ IBV_QP_PORT
| IBV_QP_ACCESS_FLAGS;<br>
+<br>
+ attr.pkey_index
= 0;<br>
+ attr.port_num
= ia_ptr->hca_ptr->port_num;<br>
+ attr.qp_access_flags
= <br>
IBV_ACCESS_LOCAL_WRITE
|<br>
IBV_ACCESS_REMOTE_WRITE
|<br>
IBV_ACCESS_REMOTE_READ
|<br>
IBV_ACCESS_REMOTE_ATOMIC;<br>
<br>
- dapl_dbg_log
(DAPL_DBG_TYPE_EP,<br>
- "
modify_qp_INIT: pi %x port %x acc %x\n",<br>
- qp_attr.pkey_index,
qp_attr.port_num,<br>
- qp_attr.qp_access_flags
);<br>
+ ep_ptr->qp_state
= IB_QP_STATE_INIT;<br>
break;<br>
}<br>
+ case IBV_QPS_RTR:<br>
+ if
(!conn)<br>
+ return(dapl_convert_errno(EINVAL,"
qp_RTR"));<br>
+ /*
<br>
+
* Get pkey_index from CM, move from INIT to INIT<br>
+
* to update index. The initial value was set by hand <br>
+
* to allow consumers to pre-post receives.<br>
+
*/<br>
+ attr.qp_state
= IBV_QPS_INIT;<br>
+
<br>
+ /*
get pkey_index from CM, move from INIT to INIT */ <br>
+ if
(ib_cm_init_qp_attr(conn->cm_id, &attr, &mask)) <br>
+ return(dapl_convert_errno(errno,"
qp_cINIT"));<br>
+ <br>
+ mask
= IBV_QP_PKEY_INDEX; <br>
+ if
(ibv_modify_qp(qp_handle, &attr, mask))<br>
+ return(dapl_convert_errno(errno,"
reINIT"));<br>
+<br>
+ /*
get qp attributes from CM, move to RTR */ <br>
+ attr.qp_state
= IBV_QPS_RTR;<br>
+ if
(ib_cm_init_qp_attr(conn->cm_id, &attr, &mask)) <br>
+ return(dapl_convert_errno(errno,"
qp_cRTR"));<br>
+ <br>
+ attr.path_mtu
= IBV_MTU_1024;<br>
+ attr.rq_psn
= qp_handle->qp_num;<br>
+ break;<br>
+<br>
+ case IBV_QPS_RTS:
<br>
+ if
(!conn)<br>
+ return(dapl_convert_errno(EINVAL,"
qp_RTS"));<br>
+<br>
+ /*
get qp attributes from CM, move to RTS */ <br>
+ if
(ib_cm_init_qp_attr(conn->cm_id, &attr, &mask)) <br>
+ return(dapl_convert_errno(errno,"
qp_cRTS"));<br>
+ <br>
+ break;<br>
+<br>
default:<br>
+ mask
= IBV_QP_STATE;<br>
break;<br>
- <br>
}<br>
-<br>
- if (ibv_modify_qp(qp_handle, &qp_attr, mask))<br>
- return(dapl_convert_errno(errno,"modify_qp_state"));<br>
<br>
+ if (ibv_modify_qp(qp_handle, &attr, mask))<br>
+ return(dapl_convert_errno(errno,"
modify_qp"));<br>
+<br>
return DAT_SUCCESS;<br>
}<br>
<br>
Index: dapl/openib/README<br>
===================================================================<br>
--- dapl/openib/README (revision 3459)<br>
+++ dapl/openib/README (working copy)<br>
@@ -47,5 +47,4 @@<br>
<br>
Known issues:<br>
no memory windows support in ibverbs,
dat_create_rmr fails.<br>
- hard coded modify QP RTR to port 1, waiting for
ib_cm_init_qp_attr call.<br>
<br>
Index: dapl/openib/dapl_ib_util.h<br>
===================================================================<br>
--- dapl/openib/dapl_ib_util.h (revision 3459)<br>
+++ dapl/openib/dapl_ib_util.h (working copy)<br>
@@ -235,6 +235,7 @@<br>
int destroy;<br>
struct ibv_device *ib_dev;<br>
struct ibv_context *ib_ctx;<br>
+ struct ib_cm_device *ib_cm;<br>
ib_cq_handle_t ib_cq_empty;<br>
DAPL_OS_WAIT_OBJECT
wait_object;<br>
int max_inline_send;<br>
@@ -261,7 +262,7 @@<br>
void dapli_ib_thread_destroy(void);<br>
int dapli_get_hca_addr(struct dapl_hca *hca_ptr);<br>
void dapli_ip_comp_handler(uint64_t req_id, void *context, int rec_num);<br>
-void dapli_cm_event_cb(void);<br>
+void dapli_cm_event_cb(struct _ib_hca_transport *hca);<br>
void dapli_at_event_cb(void);<br>
void dapli_cq_event_cb(struct _ib_hca_transport *hca);<br>
void dapli_async_event_cb(struct _ib_hca_transport *hca);<br>
@@ -269,9 +270,7 @@<br>
DAT_RETURN<br>
dapls_modify_qp_state ( IN ib_qp_handle_t qp_handle,<br>
IN
ib_qp_state_t qp_state,<br>
- IN
uint32_t qpn,<br>
- IN
uint16_t lid,<br>
- IN
uint8_t port );<br>
+ IN
struct dapl_cm_id *conn );<br>
<br>
/* inline functions */<br>
STATIC _INLINE_ IB_HCA_NAME dapl_ib_convert_name (IN char *name)<br>
Index: dapl/openib/dapl_ib_cq.c<br>
===================================================================<br>
--- dapl/openib/dapl_ib_cq.c (revision 3459)<br>
+++ dapl/openib/dapl_ib_cq.c (working copy)<br>
@@ -71,8 +71,10 @@<br>
(!ibv_get_cq_event(hca->ib_ctx,
i, <br>
&ibv_cq, (void*)&evd_ptr))) {<br>
<br>
- if
(DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD))<br>
+ if
(DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD)) {<br>
+ ibv_ack_cq_events(ibv_cq,
1);<br>
continue;<br>
+ }<br>
<br>
/*
process DTO event via callback */<br>
dapl_evd_dto_callback
( hca->ib_ctx,<br>
<o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Arial><span style='font-size:10.0pt;
font-family:Arial'><o:p> </o:p></span></font></p>
</div>
</body>
</html>