<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=us-ascii">
<meta name=ProgId content=Word.Document>
<meta name=Generator content="Microsoft Word 10">
<meta name=Originator content="Microsoft Word 10">
<link rel=File-List href="cid:filelist.xml@01C63532.F2A0F6F0">
<!--[if gte mso 9]><xml>
<o:OfficeDocumentSettings>
<o:DoNotRelyOnCSS/>
</o:OfficeDocumentSettings>
</xml><![endif]--><!--[if gte mso 9]><xml>
<w:WordDocument>
<w:SpellingState>Clean</w:SpellingState>
<w:DocumentKind>DocumentEmail</w:DocumentKind>
<w:EnvelopeVis/>
<w:Compatibility>
<w:ApplyBreakingRules/>
</w:Compatibility>
<w:BrowserLevel>MicrosoftInternetExplorer4</w:BrowserLevel>
</w:WordDocument>
</xml><![endif]-->
<style>
<!--
/* Font Definitions */
@font-face
{font-family:Tahoma;
panose-1:2 11 6 4 3 5 4 4 2 4;
mso-font-charset:0;
mso-generic-font-family:swiss;
mso-font-pitch:variable;
mso-font-signature:1627421319 -2147483648 8 0 66047 0;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{mso-style-parent:"";
margin:0in;
margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:12.0pt;
font-family:"Times New Roman";
mso-fareast-font-family:"Times New Roman";}
a:link, span.MsoHyperlink
{color:blue;
text-decoration:underline;
text-underline:single;}
a:visited, span.MsoHyperlinkFollowed
{color:blue;
text-decoration:underline;
text-underline:single;}
span.EmailStyle17
{mso-style-type:personal-reply;
mso-style-noshow:yes;
mso-ansi-font-size:10.0pt;
mso-bidi-font-size:10.0pt;
font-family:Arial;
mso-ascii-font-family:Arial;
mso-hansi-font-family:Arial;
mso-bidi-font-family:Arial;
color:navy;}
@page Section1
{size:8.5in 11.0in;
margin:1.0in 1.25in 1.0in 1.25in;
mso-header-margin:.5in;
mso-footer-margin:.5in;
mso-paper-source:0;}
div.Section1
{page:Section1;}
/* List Definitions */
@list l0
{mso-list-id:48651663;
mso-list-template-ids:-216648792;}
@list l1
{mso-list-id:173693326;
mso-list-template-ids:-519150520;}
@list l2
{mso-list-id:435059093;
mso-list-template-ids:1519425978;}
@list l3
{mso-list-id:653410496;
mso-list-template-ids:-1662982158;}
@list l4
{mso-list-id:702097764;
mso-list-template-ids:1152958516;}
@list l5
{mso-list-id:1167550123;
mso-list-template-ids:-13594910;}
@list l6
{mso-list-id:1493334255;
mso-list-template-ids:1669079000;}
@list l7
{mso-list-id:1501769649;
mso-list-template-ids:-1975117152;}
@list l8
{mso-list-id:1554851424;
mso-list-template-ids:-271308298;}
@list l9
{mso-list-id:1773738612;
mso-list-template-ids:949367998;}
@list l10
{mso-list-id:1815564568;
mso-list-template-ids:-1966183530;}
@list l11
{mso-list-id:2125998480;
mso-list-template-ids:1228965544;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
-->
</style>
<!--[if gte mso 10]>
<style>
/* Style Definitions */
table.MsoNormalTable
{mso-style-name:"Table Normal";
mso-tstyle-rowband-size:0;
mso-tstyle-colband-size:0;
mso-style-noshow:yes;
mso-style-parent:"";
mso-padding-alt:0in 5.4pt 0in 5.4pt;
mso-para-margin:0in;
mso-para-margin-bottom:.0001pt;
mso-pagination:widow-orphan;
font-size:10.0pt;
font-family:"Times New Roman";}
</style>
<![endif]-->
</head>
<body lang=EN-US link=blue vlink=blue style='tab-interval:.5in'>
<div class=Section1>
<p class=MsoNormal><font size=2 color=navy face=Arial><span style='font-size:
10.0pt;font-family:Arial;color:navy'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 color=navy face=Arial><span style='font-size:
10.0pt;font-family:Arial;color:navy'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=2 face=Tahoma><span style='font-size:10.0pt;
font-family:Tahoma'>-----Original Message-----<br>
<b><span style='font-weight:bold'>From:</span></b>
openib-general-bounces@openib.org [mailto:openib-general-bounces@openib.org] <b><span
style='font-weight:bold'>On Behalf Of </span></b>Ian Jiang<br>
<b><span style='font-weight:bold'>Sent:</span></b> Friday, February 17, 2006
9:20 AM<br>
<b><span style='font-weight:bold'>To:</span></b> openib-general<br>
<b><span style='font-weight:bold'>Subject:</span></b> [openib-general]
[VAPI]VAPI_poll_cq: CQ is empty</span></font></p>
<p class=MsoNormal><font size=3 face="Times New Roman"><span style='font-size:
12.0pt'><o:p> </o:p></span></font></p>
<p class=MsoNormal><font size=3 face="Times New Roman"><span style='font-size:
12.0pt'>To get familiar to the IBGD-1.8.0 VAPI, I wrote a program very simple,
according to two examples *hca_per* and *rctp* in IBGD.<br>
A Sender and a Receiver ran on tow different nodes just to complete a
Send/Recv progress.<b><span style='font-weight:bold'> <br>
<br>
Sender<br>
======<br>
(a) Create IB resources:</span></b> <o:p></o:p></span></font></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l5 level1 lfo1;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>List HCAs (only one
HCA in fact)<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l5 level1 lfo1;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Get the handle of
the HCA<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l5 level1 lfo1;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Query the HCA<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l5 level1 lfo1;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Allocate a PD<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l5 level1 lfo1;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Quey Port 1 of the
HCA (only one Port in fact)<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l5 level1 lfo1;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Create Send CQ and
Recv CQ<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l5 level1 lfo1;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Create QP<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'>(b) Modify QP to INIT state:</span></font></b><o:p></o:p></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l0 level1 lfo2;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>qp_move_to_init(¶ms);<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'>(c) Create MRs for Recv and Send
respectively:</span></font></b><o:p></o:p></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l1 level1 lfo3;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>user_mr_create(¶ms.in_mr,
params.mr_sz_req);<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l1 level1 lfo3;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>user_mr_create(¶ms.out_mr,
params.mr_sz_req);<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'>(d) Send parameters to Receiver<br>
<br>
(e) Get ready to transfer:</span></font></b><o:p></o:p></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l7 level1 lfo4;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Modify QP to RTR
state<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l7 level1 lfo4;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Modify QP to RTS
state<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'>(f) Post Send</span></font></b><o:p></o:p></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l10 level1 lfo5;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>post_send_req(¶ms.ib_res,
¶ms.out_mr)<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'>(g) Wait Send to complete</span></font></b><o:p></o:p></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l8 level1 lfo6;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>reap_send_req(¶ms.ib_res,
¶ms.out_mr, 1/* not block*/);<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><font size=3 face="Times New Roman"><span style='font-size:
12.0pt'><br>
<b><span style='font-weight:bold'>Receiver<br>
=======<br>
(a) Wait parameters from Sender<br>
<br>
(b) Create IB resources:</span></b> <o:p></o:p></span></font></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l3 level1 lfo7;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>List HCAs (only one
HCA in fact)<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l3 level1 lfo7;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Get the handle of
the HCA<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l3 level1 lfo7;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Query the HCA<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l3 level1 lfo7;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Allocate a PD<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l3 level1 lfo7;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Quey Port 1 of the
HCA (only one Port in fact)<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l3 level1 lfo7;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Create Send CQ and
Recv CQ<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l3 level1 lfo7;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Create QP <o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'>(b) Modify QP to INIT state:</span></font></b><o:p></o:p></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l9 level1 lfo8;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>qp_move_to_init(¶ms);<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'>(c) Create MRs for Recv and Send
respectively:</span></font></b><o:p></o:p></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l6 level1 lfo9;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>user_mr_create(¶ms.in_mr,
params.mr_sz_req);<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l6 level1 lfo9;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>user_mr_create(¶ms.out_mr,
params.mr_sz_req);<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'>(d) Post Recv</span></font></b><o:p></o:p></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l11 level1 lfo10;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>post_recv_req(¶ms.ib_res,
¶ms.in_mr)<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'>(e) Get ready to transfer:</span></font></b><o:p></o:p></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l4 level1 lfo11;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Modify QP to RTR
state<o:p></o:p></span></font></li>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l4 level1 lfo11;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>Modify QP to RTS
state<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'>(g) Wait Recv to complete</span></font></b><o:p></o:p></p>
<ol start=1 type=1>
<li class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;
mso-list:l2 level1 lfo12;tab-stops:list .5in'><font size=3
face="Times New Roman"><span style='font-size:12.0pt'>reap_recv_req(¶ms.ib_res,
¶ms.in_mr, 1/* not block*/);<o:p></o:p></span></font></li>
</ol>
<p class=MsoNormal><b><font size=3 face="Times New Roman"><span
style='font-size:12.0pt;font-weight:bold'><br>
Problem:<br>
=======<br>
</span></font></b>Both VAPI_poll_cq for Send CQ and Recv CQ returned "CQ
is empty". And I failed to find out where the problem was, so turned to
OpenIB for help. I am afraid that I am not clear enough about the CQ
processing. Any suggestion is appreciated!<br>
<br>
<b><span style='font-weight:bold'><br>
Here are some pieces fo codes:<br>
=========================</span></b><br>
<br>
<b><span style='font-weight:bold'>/*********************************** Create
IB Resources ****************************************/</span></b><br>
int ib_res_create(struct ib_resource *ib_res_p)<br>
{<br>
VAPI_ret_t vapi_ret;<br>
u_int32_t num_of_hcas;<br>
VAPI_hca_id_t inst_hca_id;<br>
VAPI_cqe_num_t
num_of_cqe;<br>
<br>
VAPI_srq_attr_t
srq_props;<br>
VAPI_srq_attr_t
actual_srq_props;<br>
<br>
VAPI_qp_init_attr_t qp_init_attr;<br>
VAPI_qp_init_attr_ext_t qp_ext_attr;<br>
VAPI_qp_prop_t qp_prop;<br>
<br>
<br>
if (ib_res_p == NULL) {<br>
PRINT_ERR("NULL ib_res_p\n");<br>
return -1;<br>
}<br>
<br>
ini_ib_res(ib_res_p);<br>
<br>
/* list HCAs */<br>
vapi_ret = EVAPI_list_hcas(1, &num_of_hcas,
&inst_hca_id);<br>
if ((vapi_ret != VAPI_OK) && (vapi_ret !=
VAPI_EAGAIN)) {<br>
printf("list HCAs failed\n");<br>
VAPIERR(vapi_ret);<br>
return -1;<br>
}<br>
PRINT_TRACE("number of HCAs: %d, HCA ID: %s\n",
num_of_hcas, (char *)inst_hca_id);<br>
switch(num_of_hcas) {<br>
case 0:<br>
printf("No HCAs installed\n");<br>
return -1;<br>
case 1:<br>
strcpy(ib_res_p->hca_id, inst_hca_id);<br>
break;<br>
default:<br>
/* ToDo: deal with multiple HCAs */<br>
printf("ToDo: deal with multiple
HCAs\n");<br>
printf("Use the first HCA\n");<br>
strcpy(ib_res_p->hca_id, inst_hca_id);<br>
}<br>
PRINT_TRACE("HCA to be used: %s\n", (char
*)ib_res_p->hca_id);<br>
<br>
/* get the handle of the HCA */<br>
vapi_ret = EVAPI_get_hca_hndl(ib_res_p->hca_id,
&ib_res_p->hca_hndl);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("HCA not open\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
<br>
/* query the HCA */<br>
vapi_ret = VAPI_query_hca_cap(ib_res_p->hca_hndl, <br>
&ib_res_p->hca_vendor, <br>
&ib_res_p->hca_cap);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Query HCA failed\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
PRINT_HCA_CAP(&ib_res_p->hca_vendor,
&ib_res_p->hca_cap);<br>
<br>
/* allocate PD */<br>
//vapi_ret = EVAPI_alloc_pd(ib_res_p->hca_hndl,
MAX_NUM_AVS, &ib_res_p->pd_hndl);<br>
vapi_ret = VAPI_alloc_pd(ib_res_p->hca_hndl,
&ib_res_p->pd_hndl);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Allocate PA
failed\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
PRINT_TRACE("PD allocated: %ld\n",
ib_res_p->pd_hndl);<br>
<br>
/* query Port */<br>
vapi_ret = VAPI_query_hca_port_prop(ib_res_p->hca_hndl, <br>
DEFAULT_PORT_NUM, <br>
&ib_res_p->hca_port);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Query Port %d
failed\n", DEFAULT_PORT_NUM);<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
PRINT_PORT_PROP(&ib_res_p->hca_port);<br>
<br>
/* send CQ */<br>
vapi_ret = VAPI_create_cq(ib_res_p->hca_hndl, <br>
MIN_SEND_CQE_NUM,<br>
&ib_res_p->s_cq_hndl,<br>
&num_of_cqe);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Create CQ for send
failed\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
PRINT_TRACE("CQ for send created. CQE NUM: %d\n",
num_of_cqe);<br>
<br>
/* receive CQ */<br>
vapi_ret = VAPI_create_cq(ib_res_p->hca_hndl, <br>
MIN_SEND_CQE_NUM,<br>
&ib_res_p->r_cq_hndl,<br>
&num_of_cqe);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Create CQ for send
failed\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
PRINT_TRACE("CQ for receive created. CQE NUM:
%d\n", num_of_cqe);<br>
<br>
/* QP */<br>
qp_init_attr.rq_cq_hndl =
ib_res_p->r_cq_hndl;<br>
qp_init_attr.sq_cq_hndl =
ib_res_p->s_cq_hndl;<br>
<br>
qp_init_attr.cap.max_oust_wr_rq = QP_INI_MAX_OUST_WR_RQ_NUM;<br>
qp_init_attr.cap.max_oust_wr_sq = QP_INI_MAX_OUST_WR_SQ_NUM;<br>
qp_init_attr.cap.max_sg_size_rq = QP_INI_MAX_SG_SIZE_RQ_NUM;<br>
qp_init_attr.cap.max_sg_size_sq = QP_INI_MAX_SG_SIZE_SQ_NUM;<br>
<br>
qp_init_attr.pd_hndl
= ib_res_p->pd_hndl;<br>
qp_init_attr.rdd_hndl
= 0;<br>
qp_init_attr.sq_sig_type =
VAPI_SIGNAL_REQ_WR;<br>
qp_init_attr.rq_sig_type =
VAPI_SIGNAL_ALL_WR;<br>
<br>
qp_init_attr.ts_type
= VAPI_TS_RC;<br>
<br>
vapi_ret = VAPI_create_qp_ext(ib_res_p->hca_hndl, <br>
&qp_init_attr, <br>
&qp_ext_attr,<br>
&ib_res_p->qp_entry.qp_hndl, <br>
&qp_prop);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Create QP
failed\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
ib_res_p->qp_entry.qp_num = qp_prop.qp_num;<br>
ib_res_p->qp_entry.srq_hndl = ib_res_p->srq_hndl;<br>
PRINT_TRACE("QP created\n");<br>
PRINT_QP_PROP(&qp_prop);<br>
<br>
return 0;<br>
<br>
clean_exit:<br>
clean_ib_res(ib_res_p);<br>
return -1;<br>
}<br>
<br>
<b><span style='font-weight:bold'>/*****************************Modify QP state
***************************************/<br>
</span></b>int qp_move_to_init(test_params_t *param_p)<br>
{<br>
VAPI_qp_attr_mask_t qp_attr_mask;<br>
VAPI_qp_attr_t qp_attr;<br>
VAPI_qp_cap_t qp_cap;<br>
VAPI_ret_t
res;<br>
<br>
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);<br>
qp_attr.qp_state = VAPI_INIT;<br>
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);<br>
qp_attr.pkey_ix = 0;<br>
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);<br>
qp_attr.port = DEFAULT_PORT_NUM;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_PORT);<br>
qp_attr.remote_atomic_flags = <br>
VAPI_EN_REM_WRITE |
VAPI_EN_REM_READ | VAPI_EN_REM_ATOMIC_OP;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_REMOTE_ATOMIC_FLAGS);<br>
<br>
res = VAPI_modify_qp(param_p->ib_res.hca_hndl, <br>
param_p->ib_res.qp_entry.qp_hndl,
<br>
&qp_attr,
&qp_attr_mask, &qp_cap);<br>
if (res != VAPI_OK) {<br>
printf("Error: Modifying
QP to INIT: %s\n", VAPI_strerror(res));<br>
return -1;<br>
}<br>
PRINT_TRACE("Modified QP to INIT\n");<br>
print_qp_cap(&qp_cap);<br>
<br>
return 0;<br>
}<br>
<br>
int qp_move_to_rtr(test_params_t *param_p)<br>
{<br>
VAPI_qp_attr_mask_t qp_attr_mask;<br>
VAPI_qp_attr_t qp_attr;<br>
VAPI_qp_cap_t qp_cap;<br>
VAPI_ret_t
res;<br>
<br>
<br>
param_p->mtu = (param_p->ib_res.hca_vendor.vendor_part_id
== 23108) ? <br>
MTU1024 : MTU2048;<br>
<br>
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);<br>
qp_attr.qp_state = VAPI_RTR;<br>
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);<br>
qp_attr.av.sl
= 0; /*USED_SL*/<br>
qp_attr.av.grh_flag = FALSE;<br>
qp_attr.av.dlid =
param_p->dst_msg.lid;<br>
qp_attr.av.static_rate = 2; /* 1x */<br>
qp_attr.av.src_path_bits = 0;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_AV);<br>
qp_attr.path_mtu =
param_p->mtu;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_PATH_MTU);<br>
qp_attr.rq_psn =
START_PSN;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RQ_PSN);<br>
qp_attr.qp_ous_rd_atom = QP_OUS_RD_ATOM;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_OUS_RD_ATOM);<br>
qp_attr.dest_qp_num = param_p->dst_msg.qp_num;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_DEST_QP_NUM);<br>
qp_attr.min_rnr_timer = 0;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_MIN_RNR_TIMER);<br>
<br>
res = VAPI_modify_qp(param_p->ib_res.hca_hndl, <br>
param_p->ib_res.qp_entry.qp_hndl,
<br>
&qp_attr,
&qp_attr_mask, &qp_cap);<br>
if (res != VAPI_OK) {<br>
printf("Error: Modifying
QP to RTR: %s\n", VAPI_strerror(res)); <br>
return -1/*(RET_ERR)*/;<br>
}<br>
PRINT_TRACE("Modified QP to RTR\n");<br>
print_qp_cap(&qp_cap);<br>
<br>
return 0;<br>
}<br>
<br>
int qp_move_to_rts(test_params_t *param_p)<br>
{<br>
VAPI_qp_attr_mask_t qp_attr_mask;<br>
VAPI_qp_attr_t qp_attr;<br>
VAPI_qp_cap_t qp_cap;<br>
VAPI_ret_t
res;<br>
<br>
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);<br>
qp_attr.qp_state = VAPI_RTS; <br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_STATE);<br>
qp_attr.sq_psn = START_PSN;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_SQ_PSN);<br>
qp_attr.timeout = 18;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_TIMEOUT);<br>
qp_attr.retry_count = 6;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RETRY_COUNT);<br>
qp_attr.rnr_retry = 6;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RNR_RETRY);<br>
qp_attr.ous_dst_rd_atom = QP_OUS_RD_ATOM;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_OUS_DST_RD_ATOM);<br>
<br>
res = VAPI_modify_qp(param_p->ib_res.hca_hndl, <br>
param_p->ib_res.qp_entry.qp_hndl,<br>
&qp_attr,
&qp_attr_mask, &qp_cap);<br>
if (res != VAPI_OK) {<br>
printf("Error: Modifying
QP to RTS: %s\n", VAPI_strerror(res));<br>
return /*(RET_ERR)*/-1;<br>
}<br>
PRINT_TRACE("Modified QP to RTS\n");<br>
print_qp_cap(&qp_cap);<br>
<br>
return 0;<br>
}<br>
<b><span style='font-weight:bold'>/************************** Recv/Send
requests ******************************/</span></b><br>
/*<br>
* post receive request<br>
*/<br>
int post_recv_req(struct ib_resource *ib_res_p, struct user_mr *u_mr_p)<br>
{<br>
VAPI_ret_t
res;<br>
VAPI_rr_desc_t rr;<br>
VAPI_sg_lst_entry_t sg_entry_r;<br>
VAPI_hca_hndl_t hca_hndl;<br>
VAPI_qp_hndl_t qp_hndl;<br>
VAPI_srq_hndl_t srq_hndl;<br>
<br>
if (ib_res_p == NULL) {<br>
PRINT_ERR("NULL ib_res_p\n");<br>
return -1;<br>
}<br>
if (u_mr_p == NULL) {<br>
PRINT_ERR("NULL user mr
pointer\n");<br>
return -1;<br>
}<br>
<br>
hca_hndl = ib_res_p->hca_hndl;<br>
qp_hndl = ib_res_p->qp_entry.qp_hndl;<br>
hca_hndl = ib_res_p->srq_hndl;<br>
<br>
rr.opcode = VAPI_RECEIVE;<br>
rr.comp_type = VAPI_SIGNALED;<br>
rr.sg_lst_len = 1;<br>
sg_entry_r.lkey = u_mr_p->mrw_rep.l_key;<br>
sg_entry_r.len = u_mr_p->mrw_req.size;<br>
sg_entry_r.addr = (VAPI_virt_addr_t)(MT_virt_addr_t)u_mr_p->user_buf;<br>
rr.sg_lst_p = &sg_entry_r;<br>
<a href="http://rr.id">rr.id</a> = sg_entry_r.addr;<br>
<br>
PRINT_RECV_REQ(&rr);<br>
<br>
res = VAPI_post_rr(hca_hndl, qp_hndl, &rr);<br>
if (res != VAPI_OK) {<br>
printf("VAPI post Recv Req
failed\n");<br>
VAPIERR(res);<br>
return -1;<br>
}<br>
<br>
return 0;<br>
}<br>
<br>
/*<br>
* post send request<br>
*/<br>
int post_send_req(struct ib_resource *ib_res_p, struct user_mr *u_mr_p)<br>
{<br>
VAPI_ret_t
res;<br>
VAPI_sr_desc_t
sr;<br>
VAPI_sg_lst_entry_t sg_entry_s;<br>
VAPI_hca_hndl_t hca_hndl;<br>
VAPI_qp_hndl_t qp_hndl;<br>
<br>
if (ib_res_p == NULL) {<br>
PRINT_ERR("NULL ib_res_p\n");<br>
return -1;<br>
}<br>
if (u_mr_p == NULL) {<br>
PRINT_ERR("NULL user mr
pointer\n");<br>
return -1;<br>
}<br>
<br>
hca_hndl = ib_res_p->hca_hndl;<br>
qp_hndl = ib_res_p->qp_entry.qp_hndl;<br>
<br>
sr.comp_type = VAPI_SIGNALED;<br>
sr.set_se = FALSE;<br>
sr.opcode = VAPI_SEND;<br>
sr.remote_qkey = 0;<br>
sr.sg_lst_len = 1;<br>
sg_entry_s.lkey = u_mr_p->mrw_rep.l_key;<br>
sg_entry_s.len = u_mr_p->mrw_req.size;<br>
sg_entry_s.addr =
(VAPI_virt_addr_t)(MT_virt_addr_t)u_mr_p->user_buf;<br>
sr.sg_lst_p = &sg_entry_s;<br>
<a href="http://sr.id">sr.id</a> = sg_entry_s.addr;<br>
PRINT_SEND_REQ(&sr);<br>
<br>
res = VAPI_post_sr(hca_hndl, qp_hndl, &sr);<br>
if (res != VAPI_OK) {<br>
printf("VAPI post Send Req
failed\n");<br>
VAPIERR(res);<br>
return -1;<br>
}<br>
<br>
return 0;<br>
}<br>
<br>
int reap_send_req(struct ib_resource *ib_res_p, struct user_mr *u_mr_p, int
block)<br>
{<br>
VAPI_ret_t res;<br>
VAPI_wc_desc_t wc_desc;<br>
VAPI_hca_hndl_t hca_hndl;<br>
VAPI_cq_hndl_t s_cq_hndl;<br>
<br>
int poll_cnt = 0;<br>
<br>
if (ib_res_p == NULL) {<br>
PRINT_ERR("NULL ib_res_p\n");<br>
return -1;<br>
}<br>
if (u_mr_p == NULL) {<br>
PRINT_ERR("NULL user mr
pointer\n");<br>
return -1;<br>
}<br>
<br>
hca_hndl = ib_res_p->hca_hndl;<br>
s_cq_hndl = ib_res_p->s_cq_hndl;<br>
<br>
if (block) {<br>
do { poll_cnt++;<br>
MTPERF_TIME_START(VAPI_poll_cq);<br>
res = VAPI_poll_cq(hca_hndl, s_cq_hndl,
&wc_desc);<br>
//res = EVAPI_poll_cq_block(hca_hndl,
s_cq_hndl, REAP_REQ_WAIT_TIME, &wc_desc);<br>
MTPERF_TIME_END(VAPI_poll_cq);<br>
if (res != VAPI_OK && res !=
VAPI_CQ_EMPTY) {<br>
PRINT_ERR("Poll
CQ block failed\n");<br>
VAPIERR(res);<br>
return -1;<br>
}<br>
show_qp_state(hca_hndl,
ib_res_p->qp_entry.qp_hndl, ib_res_p->qp_entry.qp_num);<br>
VAPI_RET(res);<br>
} while(res == VAPI_CQ_EMPTY && poll_cnt < 10);<br>
if (wc_desc.status != VAPI_SUCCESS) {<br>
PRINT_ERR("Req
unsuccess: %s\n", VAPI_wc_status_sym(wc_desc.status));<br>
PRINT_WC_DESC(&wc_desc);<br>
return -1;<br>
}<br>
} else {<br>
printf("ToDo: %s for
unblock\n", __func__);<br>
}<br>
<br>
PRINT_TRACE("Req success\n");<br>
PRINT_WC_DESC(&wc_desc);<br>
return 0;<br>
}<br>
<br>
<br>
int reap_recv_req(struct ib_resource *ib_res_p, struct user_mr *u_mr_p, int
block)<br>
{<br>
VAPI_ret_t res;<br>
VAPI_wc_desc_t wc_desc;<br>
VAPI_hca_hndl_t hca_hndl;<br>
VAPI_cq_hndl_t r_cq_hndl;<br>
<br>
int poll_cnt = 0;<br>
<br>
if (ib_res_p == NULL) {<br>
PRINT_ERR("NULL ib_res_p\n");<br>
return -1;<br>
}<br>
if (u_mr_p == NULL) {<br>
PRINT_ERR("NULL user mr
pointer\n");<br>
return -1;<br>
}<br>
<br>
hca_hndl = ib_res_p->hca_hndl;<br>
r_cq_hndl = ib_res_p->r_cq_hndl;<br>
<br>
if (block) {<br>
do { poll_cnt++;<br>
res = VAPI_poll_cq(hca_hndl,
r_cq_hndl,&wc_desc);<br>
if (res != VAPI_OK && res !=
VAPI_CQ_EMPTY) {<br>
PRINT_ERR("Poll
CQ block failed\n");<br>
VAPIERR(res);<br>
return -1;<br>
}<br>
sleep(1);<br>
} while(res == VAPI_CQ_EMPTY && poll_cnt < 20);<br>
if (wc_desc.status != VAPI_SUCCESS) {<br>
PRINT_ERR("Req
failed: %s\n", VAPI_wc_status_sym(wc_desc.status));<br>
PRINT_WC_DESC(&wc_desc);<br>
return -1;<br>
}<br>
} else {<br>
printf("ToDo: %s for
unblock\n", __func__);<br>
}<br>
<br>
PRINT_TRACE("Req success\n");<br>
PRINT_WC_DESC(&wc_desc);<br>
return 0;<br>
}<br>
<b><span style='font-weight:bold'><br clear=all>
</span></b><br>
-- <br>
Ian Jiang<br>
<a href="mailto:ianjiang.ict@gmail.com">ianjiang.ict@gmail.com</a><br>
<br>
Laboratory of Spatial Information Technology<br>
Division of System Architecture<br>
Institute of Computing Technology <br>
Chinese Academy of Sciences <o:p></o:p></p>
</div>
</body>
</html>