To get familiar to the IBGD-1.8.0 VAPI, I wrote a program very simple, according to two examples *hca_per* and *rctp* in IBGD.<br>
A Sender and a Receiver ran on tow different nodes just to complete a Send/Recv progress.<span style="font-weight: bold;"><span style="font-weight: bold;"><span style="font-weight: bold;"><span style="font-weight: bold;">
<span style="font-weight: bold;"><span style="font-weight: bold;"><span style="font-weight: bold;"></span><br>
<br>
</span></span>Sender<br>
======<br>
</span></span>(a) </span>Create IB resources:</span>
<ol>
<li>List HCAs (only one HCA in fact)</li>
<li>Get the handle of the HCA</li>
<li>Query the HCA</li>
<li>Allocate a PD</li>
<li>Quey Port 1 of the HCA (only one Port in fact)</li>
<li>Create Send CQ and Recv CQ</li>
<li>Create QP</li>
</ol>
<span style="font-weight: bold;"><span style="font-weight: bold;">(b) </span>Modify QP to INIT state:</span><br>
<ol>
<li>qp_move_to_init(¶ms);</li>
</ol>
<span style="font-weight: bold;"><span style="font-weight: bold;">(c) </span></span><span style="font-weight: bold;"><span style="font-weight: bold;"></span></span><span style="font-weight: bold;"><span style="font-weight: bold;">
</span>Create MRs for Recv and Send respectively:</span><br>
<ol>
<li>user_mr_create(¶ms.in_mr, params.mr_sz_req);</li>
<li>user_mr_create(¶ms.out_mr, params.mr_sz_req);</li>
</ol>
<span style="font-weight: bold;">(d) Send parameters to Receiver<br>
<br style="font-weight: bold;">
</span><span style="font-weight: bold;">(e) Get ready to transfer:</span><br>
<ol>
<li>Modify QP to RTR state</li>
<li>Modify QP to RTS state</li>
</ol>
<span style="font-weight: bold;"><span style="font-weight: bold;">(f) </span>Post Send<br>
</span>
<ol>
<li>post_send_req(¶ms.ib_res, ¶ms.out_mr)</li>
</ol>
<span style="font-weight: bold;"><span style="font-weight: bold;">(g) </span>Wait Send to complete<br>
</span>
<ol>
<li>reap_send_req(¶ms.ib_res, ¶ms.out_mr, 1/* not block*/);</li>
</ol>
<br>
<span style="font-weight: bold;"><span style="font-weight: bold;"><span style="font-weight: bold;"><span style="font-weight: bold;"><span style="font-weight: bold;">Receiver<br>
<span style="font-weight: bold;">=======<br>
(a) Wait parameters from Sender<br>
<br>
</span></span>
</span></span>(b) </span>Create IB resources:</span>
<ol>
<li>List HCAs (only one HCA in fact)</li><li>Get the handle of the HCA</li><li>Query the HCA</li><li>Allocate a PD</li><li>Quey Port 1 of the HCA (only one Port in fact)</li><li>Create Send CQ and Recv CQ</li><li>Create QP
</li>
</ol>
<span style="font-weight: bold;"><span style="font-weight: bold;">(b) </span>Modify QP to INIT state:</span><br>
<ol>
<li>qp_move_to_init(¶ms);</li>
</ol>
<span style="font-weight: bold;"><span style="font-weight: bold;">(c) </span></span><span style="font-weight: bold;"><span style="font-weight: bold;"></span></span><span style="font-weight: bold;"><span style="font-weight: bold;">
</span>Create MRs for Recv and Send respectively:</span><br>
<ol>
<li>user_mr_create(¶ms.in_mr, params.mr_sz_req);</li><li>user_mr_create(¶ms.out_mr, params.mr_sz_req);</li>
</ol>
<span style="font-weight: bold;">(d) Post Recv<br>
</span>
<ol>
<li>post_recv_req(¶ms.ib_res, ¶ms.in_mr)</li>
</ol>
<span style="font-weight: bold;">
</span><span style="font-weight: bold;">(e) Get ready to transfer:</span><br>
<ol>
<li>Modify QP to RTR state</li><li>Modify QP to RTS state</li>
</ol>
<span style="font-weight: bold;"><span style="font-weight: bold;"></span></span><span style="font-weight: bold;"><span style="font-weight: bold;">(g) </span>Wait Recv to complete<br>
</span>
<ol>
<li>reap_recv_req(¶ms.ib_res, ¶ms.in_mr, 1/* not block*/);</li>
</ol>
<br style="font-weight: bold;">
<span style="font-weight: bold;">Problem:<br>
<span style="font-weight: bold;">=======<br>
</span></span>Both VAPI_poll_cq for Send CQ and Recv CQ returned "CQ is
empty". And I failed to find out where the problem was, so turned to
OpenIB for help. I am afraid that I am not clear enough about the CQ
processing. Any suggestion is appreciated!<br>
<br>
<br style="font-weight: bold;">
<span style="font-weight: bold;">Here are some pieces fo codes:</span><br style="font-weight: bold;">
<span style="font-weight: bold;">=========================</span><br>
<br>
<span style="font-weight: bold;">/*********************************** Create IB Resources ****************************************/</span><br>
int ib_res_create(struct ib_resource *ib_res_p)<br>
{<br>
VAPI_ret_t vapi_ret;<br>
u_int32_t num_of_hcas;<br>
VAPI_hca_id_t inst_hca_id;<br>
VAPI_cqe_num_t num_of_cqe;<br>
<br>
VAPI_srq_attr_t srq_props;<br>
VAPI_srq_attr_t actual_srq_props;<br>
<br>
VAPI_qp_init_attr_t qp_init_attr;<br>
VAPI_qp_init_attr_ext_t qp_ext_attr;<br>
VAPI_qp_prop_t qp_prop;<br>
<br>
<br>
if (ib_res_p == NULL) {<br>
PRINT_ERR("NULL ib_res_p\n");<br>
return -1;<br>
}<br>
<br>
ini_ib_res(ib_res_p);<br>
<br>
/* list HCAs */<br>
vapi_ret = EVAPI_list_hcas(1, &num_of_hcas, &inst_hca_id);<br>
if ((vapi_ret != VAPI_OK) && (vapi_ret != VAPI_EAGAIN)) {<br>
printf("list HCAs failed\n");<br>
VAPIERR(vapi_ret);<br>
return -1;<br>
}<br>
PRINT_TRACE("number of HCAs: %d, HCA ID: %s\n", num_of_hcas, (char *)inst_hca_id);<br>
switch(num_of_hcas) {<br>
case 0:<br>
printf("No HCAs installed\n");<br>
return -1;<br>
case 1:<br>
strcpy(ib_res_p->hca_id, inst_hca_id);<br>
break;<br>
default:<br>
/* ToDo: deal with multiple HCAs */<br>
printf("ToDo: deal with multiple HCAs\n");<br>
printf("Use the first HCA\n");<br>
strcpy(ib_res_p->hca_id, inst_hca_id);<br>
}<br>
PRINT_TRACE("HCA to be used: %s\n", (char *)ib_res_p->hca_id);<br>
<br>
/* get the handle of the HCA */<br>
vapi_ret = EVAPI_get_hca_hndl(ib_res_p->hca_id, &ib_res_p->hca_hndl);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("HCA not open\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
<br>
/* query the HCA */<br>
vapi_ret = VAPI_query_hca_cap(ib_res_p->hca_hndl, <br>
&ib_res_p->hca_vendor, <br>
&ib_res_p->hca_cap);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Query HCA failed\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
PRINT_HCA_CAP(&ib_res_p->hca_vendor, &ib_res_p->hca_cap);<br>
<br>
/* allocate PD */<br>
//vapi_ret = EVAPI_alloc_pd(ib_res_p->hca_hndl, MAX_NUM_AVS, &ib_res_p->pd_hndl);<br>
vapi_ret = VAPI_alloc_pd(ib_res_p->hca_hndl, &ib_res_p->pd_hndl);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Allocate PA failed\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
PRINT_TRACE("PD allocated: %ld\n", ib_res_p->pd_hndl);<br>
<br>
/* query Port */<br>
vapi_ret = VAPI_query_hca_port_prop(ib_res_p->hca_hndl, <br>
DEFAULT_PORT_NUM, <br>
&ib_res_p->hca_port);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Query Port %d failed\n", DEFAULT_PORT_NUM);<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
PRINT_PORT_PROP(&ib_res_p->hca_port);<br>
<br>
/* send CQ */<br>
vapi_ret = VAPI_create_cq(ib_res_p->hca_hndl, <br>
MIN_SEND_CQE_NUM,<br>
&ib_res_p->s_cq_hndl,<br>
&num_of_cqe);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Create CQ for send failed\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
PRINT_TRACE("CQ for send created. CQE NUM: %d\n", num_of_cqe);<br>
<br>
/* receive CQ */<br>
vapi_ret = VAPI_create_cq(ib_res_p->hca_hndl, <br>
MIN_SEND_CQE_NUM,<br>
&ib_res_p->r_cq_hndl,<br>
&num_of_cqe);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Create CQ for send failed\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
PRINT_TRACE("CQ for receive created. CQE NUM: %d\n", num_of_cqe);<br>
<br>
/* QP */<br>
qp_init_attr.rq_cq_hndl = ib_res_p->r_cq_hndl;<br>
qp_init_attr.sq_cq_hndl = ib_res_p->s_cq_hndl;<br>
<br>
qp_init_attr.cap.max_oust_wr_rq = QP_INI_MAX_OUST_WR_RQ_NUM;<br>
qp_init_attr.cap.max_oust_wr_sq = QP_INI_MAX_OUST_WR_SQ_NUM;<br>
qp_init_attr.cap.max_sg_size_rq = QP_INI_MAX_SG_SIZE_RQ_NUM;<br>
qp_init_attr.cap.max_sg_size_sq = QP_INI_MAX_SG_SIZE_SQ_NUM;<br>
<br>
qp_init_attr.pd_hndl
= ib_res_p->pd_hndl;<br>
qp_init_attr.rdd_hndl = 0;<br>
qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR;<br>
qp_init_attr.rq_sig_type = VAPI_SIGNAL_ALL_WR;<br>
<br>
qp_init_attr.ts_type = VAPI_TS_RC;<br>
<br>
vapi_ret = VAPI_create_qp_ext(ib_res_p->hca_hndl, <br>
&qp_init_attr, <br>
&qp_ext_attr,<br>
&ib_res_p->qp_entry.qp_hndl, <br>
&qp_prop);<br>
if (vapi_ret != VAPI_OK) {<br>
printf("Create QP failed\n");<br>
VAPIERR(vapi_ret);<br>
goto clean_exit;<br>
}<br>
ib_res_p->qp_entry.qp_num = qp_prop.qp_num;<br>
ib_res_p->qp_entry.srq_hndl = ib_res_p->srq_hndl;<br>
PRINT_TRACE("QP created\n");<br>
PRINT_QP_PROP(&qp_prop);<br>
<br>
return 0;<br>
<br>
clean_exit:<br>
clean_ib_res(ib_res_p);<br>
return -1;<br>
}<br>
<br>
<span style="font-weight: bold;">/*****************************Modify QP state ***************************************/<br>
</span><span style="font-weight: bold;"><span style="font-weight: bold;"></span></span>int qp_move_to_init(test_params_t *param_p)<br>
{<br>
VAPI_qp_attr_mask_t qp_attr_mask;<br>
VAPI_qp_attr_t qp_attr;<br>
VAPI_qp_cap_t qp_cap;<br>
VAPI_ret_t res;<br>
<br>
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);<br>
qp_attr.qp_state = VAPI_INIT;<br>
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);<br>
qp_attr.pkey_ix = 0;<br>
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);<br>
qp_attr.port = DEFAULT_PORT_NUM;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_PORT);<br>
qp_attr.remote_atomic_flags = <br>
VAPI_EN_REM_WRITE | VAPI_EN_REM_READ | VAPI_EN_REM_ATOMIC_OP;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_REMOTE_ATOMIC_FLAGS);<br>
<br>
res = VAPI_modify_qp(param_p->ib_res.hca_hndl, <br>
param_p->ib_res.qp_entry.qp_hndl, <br>
&qp_attr, &qp_attr_mask, &qp_cap);<br>
if (res != VAPI_OK) {<br>
printf("Error: Modifying QP to INIT: %s\n", VAPI_strerror(res));<br>
return -1;<br>
}<br>
PRINT_TRACE("Modified QP to INIT\n");<br>
print_qp_cap(&qp_cap);<br>
<br>
return 0;<br>
}<br>
<br>
int qp_move_to_rtr(test_params_t *param_p)<br>
{<br>
VAPI_qp_attr_mask_t qp_attr_mask;<br>
VAPI_qp_attr_t qp_attr;<br>
VAPI_qp_cap_t qp_cap;<br>
VAPI_ret_t res;<br>
<br>
<br>
param_p->mtu = (param_p->ib_res.hca_vendor.vendor_part_id == 23108) ? <br>
MTU1024 : MTU2048;<br>
<br>
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);<br>
qp_attr.qp_state = VAPI_RTR;<br>
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);<br>
qp_attr.av.sl = 0; /*USED_SL*/<br>
qp_attr.av.grh_flag = FALSE;<br>
qp_attr.av.dlid = param_p->dst_msg.lid;<br>
qp_attr.av.static_rate = 2; /* 1x */<br>
qp_attr.av.src_path_bits = 0;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_AV);<br>
qp_attr.path_mtu = param_p->mtu;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_PATH_MTU);<br>
qp_attr.rq_psn = START_PSN;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RQ_PSN);<br>
qp_attr.qp_ous_rd_atom = QP_OUS_RD_ATOM;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_OUS_RD_ATOM);<br>
qp_attr.dest_qp_num = param_p->dst_msg.qp_num;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_DEST_QP_NUM);<br>
qp_attr.min_rnr_timer = 0;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_MIN_RNR_TIMER);<br>
<br>
res = VAPI_modify_qp(param_p->ib_res.hca_hndl, <br>
param_p->ib_res.qp_entry.qp_hndl, <br>
&qp_attr, &qp_attr_mask, &qp_cap);<br>
if (res != VAPI_OK) {<br>
printf("Error:
Modifying QP to RTR: %s\n",
VAPI_strerror(res)); <br>
return -1/*(RET_ERR)*/;<br>
}<br>
PRINT_TRACE("Modified QP to RTR\n");<br>
print_qp_cap(&qp_cap);<br>
<br>
return 0;<br>
}<br>
<br>
int qp_move_to_rts(test_params_t *param_p)<br>
{<br>
VAPI_qp_attr_mask_t qp_attr_mask;<br>
VAPI_qp_attr_t qp_attr;<br>
VAPI_qp_cap_t qp_cap;<br>
VAPI_ret_t res;<br>
<br>
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);<br>
qp_attr.qp_state = VAPI_RTS; <br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_STATE);<br>
qp_attr.sq_psn = START_PSN;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_SQ_PSN);<br>
qp_attr.timeout = 18;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_TIMEOUT);<br>
qp_attr.retry_count = 6;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RETRY_COUNT);<br>
qp_attr.rnr_retry = 6;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RNR_RETRY);<br>
qp_attr.ous_dst_rd_atom = QP_OUS_RD_ATOM;<br>
QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_OUS_DST_RD_ATOM);<br>
<br>
res = VAPI_modify_qp(param_p->ib_res.hca_hndl, <br>
param_p->ib_res.qp_entry.qp_hndl,<br>
&qp_attr, &qp_attr_mask, &qp_cap);<br>
if (res != VAPI_OK) {<br>
printf("Error: Modifying QP to RTS: %s\n", VAPI_strerror(res));<br>
return /*(RET_ERR)*/-1;<br>
}<br>
PRINT_TRACE("Modified QP to RTS\n");<br>
print_qp_cap(&qp_cap);<br>
<br>
return 0;<br>
}<br>
<span style="font-weight: bold;">/************************** Recv/Send requests ******************************/</span><br>
/*<br>
* post receive request<br>
*/<br>
int post_recv_req(struct ib_resource *ib_res_p, struct user_mr *u_mr_p)<br>
{<br>
VAPI_ret_t res;<br>
VAPI_rr_desc_t rr;<br>
VAPI_sg_lst_entry_t sg_entry_r;<br>
VAPI_hca_hndl_t hca_hndl;<br>
VAPI_qp_hndl_t qp_hndl;<br>
VAPI_srq_hndl_t srq_hndl;<br>
<br>
if (ib_res_p == NULL) {<br>
PRINT_ERR("NULL ib_res_p\n");<br>
return -1;<br>
}<br>
if (u_mr_p == NULL) {<br>
PRINT_ERR("NULL user mr pointer\n");<br>
return -1;<br>
}<br>
<br>
hca_hndl = ib_res_p->hca_hndl;<br>
qp_hndl = ib_res_p->qp_entry.qp_hndl;<br>
hca_hndl = ib_res_p->srq_hndl;<br>
<br>
rr.opcode = VAPI_RECEIVE;<br>
rr.comp_type = VAPI_SIGNALED;<br>
rr.sg_lst_len = 1;<br>
sg_entry_r.lkey = u_mr_p->mrw_rep.l_key;<br>
sg_entry_r.len = u_mr_p->mrw_req.size;<br>
sg_entry_r.addr = (VAPI_virt_addr_t)(MT_virt_addr_t)u_mr_p->user_buf;<br>
rr.sg_lst_p = &sg_entry_r;<br>
<a href="http://rr.id">rr.id</a> = sg_entry_r.addr;<br>
<br>
PRINT_RECV_REQ(&rr);<br>
<br>
res = VAPI_post_rr(hca_hndl, qp_hndl, &rr);<br>
if (res != VAPI_OK) {<br>
printf("VAPI post Recv Req failed\n");<br>
VAPIERR(res);<br>
return -1;<br>
}<br>
<br>
return 0;<br>
}<br>
<br>
/*<br>
* post send request<br>
*/<br>
int post_send_req(struct ib_resource *ib_res_p, struct user_mr *u_mr_p)<br>
{<br>
VAPI_ret_t res;<br>
VAPI_sr_desc_t sr;<br>
VAPI_sg_lst_entry_t sg_entry_s;<br>
VAPI_hca_hndl_t hca_hndl;<br>
VAPI_qp_hndl_t qp_hndl;<br>
<br>
if (ib_res_p == NULL) {<br>
PRINT_ERR("NULL ib_res_p\n");<br>
return -1;<br>
}<br>
if (u_mr_p == NULL) {<br>
PRINT_ERR("NULL user mr pointer\n");<br>
return -1;<br>
}<br>
<br>
hca_hndl = ib_res_p->hca_hndl;<br>
qp_hndl = ib_res_p->qp_entry.qp_hndl;<br>
<br>
sr.comp_type = VAPI_SIGNALED;<br>
sr.set_se = FALSE;<br>
sr.opcode = VAPI_SEND;<br>
sr.remote_qkey = 0;<br>
sr.sg_lst_len = 1;<br>
sg_entry_s.lkey = u_mr_p->mrw_rep.l_key;<br>
sg_entry_s.len = u_mr_p->mrw_req.size;<br>
sg_entry_s.addr = (VAPI_virt_addr_t)(MT_virt_addr_t)u_mr_p->user_buf;<br>
sr.sg_lst_p = &sg_entry_s;<br>
<a href="http://sr.id">sr.id</a> = sg_entry_s.addr;<br>
PRINT_SEND_REQ(&sr);<br>
<br>
res = VAPI_post_sr(hca_hndl, qp_hndl, &sr);<br>
if (res != VAPI_OK) {<br>
printf("VAPI post Send Req failed\n");<br>
VAPIERR(res);<br>
return -1;<br>
}<br>
<br>
return 0;<br>
}<br>
<br>
int reap_send_req(struct ib_resource *ib_res_p, struct user_mr *u_mr_p, int block)<br>
{<br>
VAPI_ret_t res;<br>
VAPI_wc_desc_t wc_desc;<br>
VAPI_hca_hndl_t hca_hndl;<br>
VAPI_cq_hndl_t s_cq_hndl;<br>
<br>
int poll_cnt = 0;<br>
<br>
if (ib_res_p == NULL) {<br>
PRINT_ERR("NULL ib_res_p\n");<br>
return -1;<br>
}<br>
if (u_mr_p == NULL) {<br>
PRINT_ERR("NULL user mr pointer\n");<br>
return -1;<br>
}<br>
<br>
hca_hndl = ib_res_p->hca_hndl;<br>
s_cq_hndl = ib_res_p->s_cq_hndl;<br>
<br>
if (block) {<br>
do { poll_cnt++;<br>
MTPERF_TIME_START(VAPI_poll_cq);<br>
res = VAPI_poll_cq(hca_hndl, s_cq_hndl, &wc_desc);<br>
//res = EVAPI_poll_cq_block(hca_hndl, s_cq_hndl, REAP_REQ_WAIT_TIME, &wc_desc);<br>
MTPERF_TIME_END(VAPI_poll_cq);<br>
if (res != VAPI_OK && res != VAPI_CQ_EMPTY) {<br>
PRINT_ERR("Poll CQ block failed\n");<br>
VAPIERR(res);<br>
return -1;<br>
}<br>
show_qp_state(hca_hndl, ib_res_p->qp_entry.qp_hndl, ib_res_p->qp_entry.qp_num);<br>
VAPI_RET(res);<br>
} while(res == VAPI_CQ_EMPTY && poll_cnt < 10);<br>
if (wc_desc.status != VAPI_SUCCESS) {<br>
PRINT_ERR("Req unsuccess: %s\n", VAPI_wc_status_sym(wc_desc.status));<br>
PRINT_WC_DESC(&wc_desc);<br>
return -1;<br>
}<br>
} else {<br>
printf("ToDo: %s for unblock\n", __func__);<br>
}<br>
<br>
PRINT_TRACE("Req success\n");<br>
PRINT_WC_DESC(&wc_desc);<br>
return 0;<br>
}<br>
<br>
<br>
int reap_recv_req(struct ib_resource *ib_res_p, struct user_mr *u_mr_p, int block)<br>
{<br>
VAPI_ret_t res;<br>
VAPI_wc_desc_t wc_desc;<br>
VAPI_hca_hndl_t hca_hndl;<br>
VAPI_cq_hndl_t r_cq_hndl;<br>
<br>
int poll_cnt = 0;<br>
<br>
if (ib_res_p == NULL) {<br>
PRINT_ERR("NULL ib_res_p\n");<br>
return -1;<br>
}<br>
if (u_mr_p == NULL) {<br>
PRINT_ERR("NULL user mr pointer\n");<br>
return -1;<br>
}<br>
<br>
hca_hndl = ib_res_p->hca_hndl;<br>
r_cq_hndl = ib_res_p->r_cq_hndl;<br>
<br>
if (block) {<br>
do { poll_cnt++;<br>
res = VAPI_poll_cq(hca_hndl, r_cq_hndl,&wc_desc);<br>
if (res != VAPI_OK && res != VAPI_CQ_EMPTY) {<br>
PRINT_ERR("Poll CQ block failed\n");<br>
VAPIERR(res);<br>
return -1;<br>
}<br>
sleep(1);<br>
} while(res == VAPI_CQ_EMPTY && poll_cnt < 20);<br>
if (wc_desc.status != VAPI_SUCCESS) {<br>
PRINT_ERR("Req failed: %s\n", VAPI_wc_status_sym(wc_desc.status));<br>
PRINT_WC_DESC(&wc_desc);<br>
return -1;<br>
}<br>
} else {<br>
printf("ToDo: %s for unblock\n", __func__);<br>
}<br>
<br>
PRINT_TRACE("Req success\n");<br>
PRINT_WC_DESC(&wc_desc);<br>
return 0;<br>
}<br>
<span style="font-weight: bold;"><span style="font-weight: bold;"><br clear="all">
</span></span><br>-- <br>Ian Jiang<br><a href="mailto:ianjiang.ict@gmail.com">ianjiang.ict@gmail.com</a><br><br>Laboratory of Spatial Information Technology<br>Division of System Architecture<br>Institute of Computing Technology
<br>Chinese Academy of Sciences