I am trying to use VAPI in kernel space and have a question about the registration of physical memory.<br><br>"<a href="http://11.2.8.3">11.2.8.3</a> REGISTER PHYSICAL MEMORY REGION" of IB spec. 1.2 says<br>Actual size of the PBL resources allocated. The actual size of the
<br>PBL resources allocated shall be greater than or equal to the size<br>of the PBL resources passed in by the Consumer as an input<br>modifier.<br>But the pbuf_len in the structure of type VAPI_mrw_t returned bye VAPI_register_mr() is zero in my test.
<br>I am not sure if this correct and whether that is what caused the "VAPI_CQ_EMPTY" problem.<br>Any suggestion is appreciated and thanks a lot!<br><br clear="all"><br>======================<br>codes related to registration
<br>======================<br>struct user_mr {<br> u_int8_t *user_buf;<br> buf_type_t buf_type;<br> int align;<br> unsigned int user_buf_len;<br> VAPI_hca_hndl_t hca_hndl;<br> VAPI_pd_hndl_t pd_hndl;
<br> VAPI_mrw_t mrw_req; /* requested memory region properties */<br> VAPI_mrw_t mrw_rep; /* responded memory region properties */<br> VAPI_mr_hndl_t mr_hndl;<br>};<br><br>int user_pmr_create(struct user_mr *user_mr_p, unsigned int sz)
<br>{<br> VAPI_ret_t vapi_ret;<br> VAPI_phy_buf_t *pbuf_list_p = NULL;<br> unsigned int pbuf_list_len = 0;<br> unsigned int pages;<br> u_int64_t offset, len;<br> void * addr;<br> int i;<br><br> if (user_mr_p == NULL) {
<br> PRINT_ERR("NULL user_mr_p\n");<br> return -1;<br> }<br><br> ini_user_mr(user_mr_p);<br><br> /* allocate */<br> /* TODO: deal with large MR size */<br> if (sz <0 || sz > (128 * 1024)) {
<br> PRINT_ERR("invalid MR size: %d\n", sz);<br> return -1;<br> }<br> user_mr_p->user_buf = kmalloc(sz, GFP_ATOMIC);<br> if (user_mr_p->user_buf == NULL) {<br> PRINT_ERR("kmalloc failed. size: %u\n", sz);
<br> return -1;<br> }<br> user_mr_p->user_buf_len = sz;<br> PRINT_TRACE("user buffer 0x%p, len: %u\n", user_mr_p->user_buf, sz);<br><br> /* set buffer vectors for physical memory */<br> offset = offset_in_page(user_mr_p->user_buf);
<br> len = PAGE_ALIGN(sz + offset);<br> pages = len / PAGE_SIZE;<br> pbuf_list_p = kmalloc(pages * sizeof(VAPI_phy_buf_t), GFP_ATOMIC);<br> if (pbuf_list_p == NULL) {<br> PRINT_ERR("kmalloc for pbuf_list failed. size: %d * %d\n", pages,
<br> (int)sizeof(VAPI_phy_buf_t));<br> return -1;<br> }<br> pbuf_list_len = pages;<br> addr = user_mr_p->user_buf - offset;<br> for (i = 0; i < pages; i++) {<br> pbuf_list_p[i].start = virt_to_phys(addr);
<br> pbuf_list_p[i].size = PAGE_SIZE;<br> addr += PAGE_SIZE;<br>#ifdef KVAPITEST_TRACE_PBUF_LIST<br> MY_PRINT("\tpbuf_list[0x%x]: 0x%llx\n", i, pbuf_list_p[i].start);<br>#endif<br> }<br>
<br> /* register */<br> user_mr_p->mrw_req.pbuf_list_len = pbuf_list_len;<br> user_mr_p->mrw_req.pbuf_list_p = pbuf_list_p;<br> user_mr_p->mrw_req.iova_offset = offset;<br> user_mr_p->mrw_req.type = VAPI_MPR;
<br> user_mr_p->mrw_req.pd_hndl = user_mr_p->pd_hndl;<br> user_mr_p->mrw_req.start = (VAPI_virt_addr_t)(MT_virt_addr_t)<br> user_mr_p->user_buf;<br> user_mr_p->mrw_req.acl = VAPI_EN_LOCAL_WRITE
<br> | VAPI_EN_REMOTE_WRITE <br> | VAPI_EN_REMOTE_READ;<br> user_mr_p->mrw_req.size = user_mr_p->user_buf_len;<br> PRINT_MR_PROP(&user_mr_p->mrw_req);<br> vapi_ret = VAPI_register_mr(user_mr_p->hca_hndl,
<br> &user_mr_p->mrw_req, <br> &user_mr_p->mr_hndl, <br> &user_mr_p->mrw_rep);<br> if (vapi_ret != VAPI_OK) {<br> MY_PRINT("MR register failed\n");
<br> VAPIERR(vapi_ret);<br> goto reg_err;<br> }<br> PRINT_TRACE("MR registered. MR handle: %d\n", user_mr_p->mr_hndl);<br> PRINT_MR_PROP(&user_mr_p->mrw_rep);<br> if (pbuf_list_p != NULL) {
<br> kfree(pbuf_list_p);<br> }<br> <br> return 0;<br><br>reg_err:<br> if (user_mr_p->user_buf != NULL) {<br> kfree(user_mr_p->user_buf);<br> user_mr_p->user_buf_len = 0;<br> }<br>
if (pbuf_list_p != NULL) {<br> kfree(pbuf_list_p);<br> }<br> return -1;<br>}<br><br>===================<br>log messages of Receiver<br>===================<br><br>Mar 4 05:03:16 linux2 kernel: ==== MSG ====
<br>Mar 4 05:03:16 linux2 kernel: test type: Send/Recv<br>Mar 4 05:03:16 linux2 kernel: LID: 0x169<br>Mar 4 05:03:16 linux2 kernel: QP NUM: 0xa0405<br>Mar 4 05:03:16 linux2 kernel: R_Key: 0x0<br>Mar 4 05:03:16 linux2 kernel: VA: 0x0
<br>Mar 4 05:03:16 linux2 kernel: required size: 10000<br>Mar 4 05:03:16 linux2 kernel: align to PAGE_SZ: 1<br>Mar 4 05:03:16 linux2 kernel: iterate times: 1<br>Mar 4 05:03:16 linux2 kernel: communicate_params: [Receiver]Send Message
<br>Mar 4 05:03:16 linux2 kernel: ==== MSG ====<br>Mar 4 05:03:16 linux2 kernel: test type: Send/Recv<br>Mar 4 05:03:16 linux2 kernel: LID: 0x161<br>Mar 4 05:03:16 linux2 kernel: QP NUM: 0xa0405<br>Mar 4 05:03:16 linux2 kernel: R_Key: 0x0
<br>Mar 4 05:03:16 linux2 kernel: VA: 0x0<br>Mar 4 05:03:16 linux2 kernel: required size: 10000<br>Mar 4 05:03:16 linux2 kernel: align to PAGE_SZ: 1<br>Mar 4 05:03:16 linux2 kernel: iterate times: 1<br>Mar 4 05:03:16 linux2 kernel: kvapitest_init: get ready to transfer
<br>Mar 4 05:03:16 linux2 kernel: ready_to_trans: move to RTR...<br>Mar 4 05:03:16 linux2 kernel: qp_move_to_rtr: Modified QP to RTR<br>Mar 4 05:03:16 linux2 kernel: max_oust_wr_sq = 255<br>Mar 4 05:03:16 linux2 kernel: max_oust_wr_rq = 255
<br>Mar 4 05:03:16 linux2 kernel: max_sg_size_sq = 5<br>Mar 4 05:03:16 linux2 kernel: max_sg_size_rq = 3<br>Mar 4 05:03:16 linux2 kernel: ==== QP attr to rtr ====<br>Mar 4 05:03:16 linux2 kernel: .qp_state: 2<br>Mar 4 05:03:16 linux2 kernel: .av.sl: 0
<br>Mar 4 05:03:16 linux2 kernel: .av.grh_flag: 0<br>Mar 4 05:03:16 linux2 kernel: .av.dlid: 0x169<br>Mar 4 05:03:16 linux2 kernel: .av.static_rate: 2<br>Mar 4 05:03:16 linux2 kernel: .av.src_path_bits: 0<br>Mar 4 05:03:16 linux2 kernel: .path_mtu: 4
<br>Mar 4 05:03:16 linux2 kernel: .dst_qp_num: 0xa0405<br>Mar 4 05:03:16 linux2 kernel: ready_to_trans: move to RTS...<br>Mar 4 05:03:16 linux2 kernel: qp_move_to_rts: Modified QP to RTS<br>Mar 4 05:03:16 linux2 kernel: test_sr_lat: test 0, [Receiver]
<br>Mar 4 05:03:16 linux2 kernel: test_sr_lat: create in_mr<br>Mar 4 05:03:16 linux2 kernel: user_pmr_create: user buffer 0x0000010029b64000, len: 10000<br>Mar 4 05:03:16 linux2 kernel: ==== MR ==== (requested)<br>
Mar 4 05:03:16 linux2 kernel: VAPI_mrw_type: 2<br>Mar 4 05:03:16 linux2 kernel: VAPI_lkey: 0<br>Mar 4 05:03:16 linux2 kernel: VAPI_rkey: 0<br>Mar 4 05:03:16 linux2 kernel: VAPI_virt_addr: 0x10029b64000<br>Mar 4 05:03:16 linux2 kernel: VAPI_size: 10000
<br>Mar 4 05:03:16 linux2 kernel: VAPI_pd_hndl: 1100297486048<br>Mar 4 05:03:16 linux2 kernel: VAPI_mrw_acl: 7<br>Mar 4 05:03:16 linux2 kernel: pbuf_len: 3<br>Mar 4 05:03:16 linux2 kernel: VAPI_phy_buf: 0x000001002c8e4580
<br>Mar 4 05:03:16 linux2 kernel: pbuf_list[0x0]: 0x29b64000<br>Mar 4 05:03:16 linux2 kernel: pbuf_list[0x1]: 0x29b65000<br>Mar 4 05:03:16 linux2 kernel: pbuf_list[0x2]: 0x29b66000<br>Mar 4 05:03:16 linux2 kernel: iova_offset: 0x0
<br>Mar 4 05:03:16 linux2 kernel: user_pmr_create: MR registered. MR handle: 3<br>Mar 4 05:03:16 linux2 kernel: ==== MR ==== (returned)<br>Mar 4 05:03:16 linux2 kernel: VAPI_mrw_type: 2<br>Mar 4 05:03:16 linux2 kernel: VAPI_lkey: 1631554304
<br>Mar 4 05:03:16 linux2 kernel: VAPI_rkey: 1631554304<br>Mar 4 05:03:16 linux2 kernel: VAPI_virt_addr: 0x10029b64000<br>Mar 4 05:03:16 linux2 kernel: VAPI_size: 10000<br>Mar 4 05:03:16 linux2 kernel: VAPI_pd_hndl: 1100297486048
<br>Mar 4 05:03:16 linux2 kernel: VAPI_mrw_acl: 7<br>Mar 4 05:03:16 linux2 kernel: pbuf_len: 0<br>Mar 4 05:03:16 linux2 kernel: VAPI_phy_buf: 0x0000000000000000<br>Mar 4 05:03:16 linux2 kernel: iova_offset: 0x0<br>Mar 4 05:03:16 linux2 kernel: test_sr_lat: post_recv
<br>Mar 4 05:03:16 linux2 kernel: VAPI_post_rr with<br>Mar 4 05:03:16 linux2 kernel: HCA: 0x0<br>Mar 4 05:03:16 linux2 kernel: QP: 0x1002c8ef088<br>Mar 4 05:03:16 linux2 kernel: send_req ID: 1100211437568<br>Mar 4 05:03:16 linux2 kernel: opcode: VAPI_RECEIVE
<br>Mar 4 05:03:16 linux2 kernel: comp_type: 0 (0-SIGNALED, 1-UNSIGNALED)<br>Mar 4 05:03:16 linux2 kernel: SG list len: 1 SG_lst_entty[0] addr:0x10029b64000, len:10000, lkey: 1631554304<br>Mar 4 05:03:16 linux2 kernel: test_sr_lat: reap_recv
<br>Mar 4 05:03:16 linux2 kernel: to VAPI_poll_cq recv for the 1 time<br>Mar 4 05:03:16 linux2 kernel: === QP handle 0x2c8ef088, QP num 0xa0405: state 3<br>Mar 4 05:03:16 linux2 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
<br>Mar 4 05:03:16 linux2 kernel: to VAPI_poll_cq recv for the 2 time<br>Mar 4 05:03:16 linux2 kernel: === QP handle 0x2c8ef088, QP num 0xa0405: state 3<br>Mar 4 05:03:16 linux2 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
<br>Mar 4 05:03:16 linux2 kernel: to VAPI_poll_cq recv for the 3 time<br>Mar 4 05:03:16 linux2 kernel: === QP handle 0x2c8ef088, QP num 0xa0405: state 3<br>Mar 4 05:03:16 linux2 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
<br>Mar 4 05:03:16 linux2 kernel: to VAPI_poll_cq recv for the 4 time<br>Mar 4 05:03:16 linux2 kernel: === QP handle 0x2c8ef088, QP num 0xa0405: state 3<br>Mar 4 05:03:16 linux2 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
<br>Mar 4 05:03:16 linux2 kernel: to VAPI_poll_cq recv for the 5 time<br>Mar 4 05:03:16 linux2 kernel: === QP handle 0x2c8ef088, QP num 0xa0405: state 3<br>Mar 4 05:03:16 linux2 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
<br>Mar 4 05:03:16 linux2 kernel: reap_recv_req: VAPI_poll_cq 5 time(s)<br>Mar 4 05:03:16 linux2 kernel: Error in reap_recv_req: Req failed: UnKnown<br>Mar 4 05:03:16 linux2 kernel: status: 743692440<br>Mar 4 05:03:16 linux2 kernel: id: 16
<br>Mar 4 05:03:16 linux2 kernel: opcode: VAPI_CQE_SQ_SEND_DATA<br>Mar 4 05:03:16 linux2 kernel: Num. of bytes transferred: 0<br>Mar 4 05:03:16 linux2 kernel: ...<br>Mar 4 05:03:16 linux2 kernel: Error in test_sr_lat: reap_recv_req failed. ret: -1
<br><br><br>==================<br>
log messages of Sender<br>
==================<br>Mar 3 21:33:08 linux3 kernel: ==== MSG ====<br>Mar 3 21:33:08 linux3 kernel: test type: Send/Recv<br>Mar 3 21:33:08 linux3 kernel: LID: 0x169<br>Mar 3 21:33:08 linux3 kernel: QP NUM: 0xa0405<br>
Mar 3 21:33:08 linux3 kernel: R_Key: 0x0<br>Mar 3 21:33:08 linux3 kernel: VA: 0x0<br>Mar 3 21:33:08 linux3 kernel: required size: 10000<br>Mar 3 21:33:08 linux3 kernel: align to PAGE_SZ: 1<br>Mar 3 21:33:08 linux3 kernel: iterate times: 1
<br>Mar 3 21:33:08 linux3 kernel: communicate_params: [Sender]to receive Message<br>Mar 3 21:33:08 linux3 kernel: communicate_params: [Sender] received message<br>Mar 3 21:33:08 linux3 kernel: ==== MSG ====<br>Mar 3 21:33:08 linux3 kernel: test type: Send/Recv
<br>Mar 3 21:33:08 linux3 kernel: LID: 0x161<br>Mar 3 21:33:08 linux3 kernel: QP NUM: 0xa0405<br>Mar 3 21:33:08 linux3 kernel: R_Key: 0x0<br>Mar 3 21:33:08 linux3 kernel: VA: 0x0<br>Mar 3 21:33:08 linux3 kernel: required size: 10000
<br>Mar 3 21:33:08 linux3 kernel: align to PAGE_SZ: 1<br>Mar 3 21:33:08 linux3 kernel: iterate times: 1<br>Mar 3 21:33:08 linux3 kernel: kvapitest_init: get ready to transfer<br>Mar 3 21:33:08 linux3 kernel: ready_to_trans: move to RTR...
<br>Mar 3 21:33:08 linux3 kernel: qp_move_to_rtr: Modified QP to RTR<br>Mar 3 21:33:08 linux3 kernel: max_oust_wr_sq = 255<br>Mar 3 21:33:08 linux3 kernel: max_oust_wr_rq = 255<br>Mar 3 21:33:08 linux3 kernel: max_sg_size_sq = 5
<br>Mar 3 21:33:08 linux3 kernel: max_sg_size_rq = 3<br>Mar 3 21:33:08 linux3 kernel: ==== QP attr to rtr ====<br>Mar 3 21:33:08 linux3 kernel: .qp_state: 2<br>Mar 3 21:33:08 linux3 kernel: .av.sl: 0<br>Mar 3 21:33:08 linux3 kernel: .av.grh_flag: 0
<br>Mar 3 21:33:08 linux3 kernel: .av.dlid: 0x161<br>Mar 3 21:33:08 linux3 kernel: .av.static_rate: 2<br>Mar 3 21:33:08 linux3 kernel: .av.src_path_bits: 0<br>Mar 3 21:33:08 linux3 kernel: .path_mtu: 4<br>Mar 3 21:33:08 linux3 kernel: .dst_qp_num: 0xa0405
<br>Mar 3 21:33:08 linux3 kernel: ready_to_trans: move to RTS...<br>Mar 3 21:33:08 linux3 kernel: qp_move_to_rts: Modified QP to RTS<br>Mar 3 21:33:08 linux3 kernel: test_sr_lat: test 0, [Sender]<br>Mar 3 21:33:08 linux3 kernel: test_sr_lat: wait to start...
<br>Mar 3 21:33:08 linux3 kernel: test_sr_lat: create out_mr<br>Mar 3 21:33:08 linux3 kernel: user_pmr_create: user buffer 0x00000100335bc000, len: 10000<br>Mar 3 21:33:08 linux3 kernel: ==== MR ==== (requested)<br>
Mar 3 21:33:08 linux3 kernel: VAPI_mrw_type: 2<br>Mar 3 21:33:08 linux3 kernel: VAPI_lkey: 0<br>Mar 3 21:33:08 linux3 kernel: VAPI_rkey: 0<br>Mar 3 21:33:08 linux3 kernel: VAPI_virt_addr: 0x100335bc000<br>Mar 3 21:33:08 linux3 kernel: VAPI_size: 10000
<br>Mar 3 21:33:08 linux3 kernel: VAPI_pd_hndl: 1100433669576<br>Mar 3 21:33:08 linux3 kernel: VAPI_mrw_acl: 7<br>Mar 3 21:33:08 linux3 kernel: pbuf_len: 3<br>Mar 3 21:33:08 linux3 kernel: VAPI_phy_buf: 0x000001000e8e83c8
<br>Mar 3 21:33:08 linux3 kernel: pbuf_list[0x0]: 0x335bc000<br>Mar 3 21:33:08 linux3 kernel: pbuf_list[0x1]: 0x335bd000<br>Mar 3 21:33:08 linux3 kernel: pbuf_list[0x2]: 0x335be000<br>Mar 3 21:33:08 linux3 kernel: iova_offset: 0x0
<br>Mar 3 21:33:08 linux3 kernel: user_pmr_create: MR registered. MR handle: 3<br>Mar 3 21:33:08 linux3 kernel: ==== MR ==== (returned)<br>Mar 3 21:33:08 linux3 kernel: VAPI_mrw_type: 2<br>Mar 3 21:33:08 linux3 kernel: VAPI_lkey: 1497336576
<br>Mar 3 21:33:08 linux3 kernel: VAPI_rkey: 1497336576<br>Mar 3 21:33:08 linux3 kernel: VAPI_virt_addr: 0x100335bc000<br>Mar 3 21:33:08 linux3 kernel: VAPI_size: 10000<br>Mar 3 21:33:08 linux3 kernel: VAPI_pd_hndl: 1100433669576
<br>Mar 3 21:33:08 linux3 kernel: VAPI_mrw_acl: 7<br>Mar 3 21:33:08 linux3 kernel: pbuf_len: 0<br>Mar 3 21:33:08 linux3 kernel: VAPI_phy_buf: 0x0000000000000000<br>Mar 3 21:33:08 linux3 kernel: iova_offset: 0x0<br>Mar 3 21:33:08 linux3 kernel: test_sr_lat: post_send
<br>Mar 3 21:33:08 linux3 kernel: VAPI_post_sr with<br>Mar 3 21:33:08 linux3 kernel: HCA: 0x0<br>Mar 3 21:33:08 linux3 kernel: QP: 0x10007cdc2b8<br>Mar 3 21:33:08 linux3 kernel: send_req ID: 1100373278720<br>Mar 3 21:33:08 linux3 kernel: opcode: VAPI_SEND
<br>Mar 3 21:33:08 linux3 kernel: comp_type: 0 (0-SIGNALED, 1-UNSIGNALED)<br>Mar 3 21:33:08 linux3 kernel: SG list len: 1 SG_lst_entty[0] addr:0x100335bc000, len:10000, lkey: 1497336576<br>Mar 3 21:33:08 linux3 kernel: imm_data: 256
<br>Mar 3 21:33:08 linux3 kernel: fence: 72(1-True, 0-False)<br>Mar 3 21:33:08 linux3 kernel: set_se: 0(1-True, 0-False)<br>Mar 3 21:33:08 linux3 kernel: test_sr_lat: reap_send<br>Mar 3 21:33:08 linux3 kernel: to VAPI_poll_cq send for the 1 time
<br>Mar 3 21:33:08 linux3 kernel: === QP handle 0x7cdc2b8, QP num 0xa0405: state 3<br>Mar 3 21:33:08 linux3 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty<br>Mar 3 21:33:08 linux3 kernel: to VAPI_poll_cq send for the 2 time
<br>Mar 3 21:33:09 linux3 kernel: === QP handle 0x7cdc2b8, QP num 0xa0405: state 3<br>Mar 3 21:33:09 linux3 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty<br>Mar 3 21:33:09 linux3 kernel: to VAPI_poll_cq send for the 3 time
<br>Mar 3 21:33:09 linux3 kernel: === QP handle 0x7cdc2b8, QP num 0xa0405: state 3<br>Mar 3 21:33:09 linux3 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty<br>Mar 3 21:33:09 linux3 kernel: to VAPI_poll_cq send for the 4 time
<br>Mar 3 21:33:09 linux3 kernel: === QP handle 0x7cdc2b8, QP num 0xa0405: state 3<br>Mar 3 21:33:09 linux3 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty<br>Mar 3 21:33:09 linux3 kernel: to VAPI_poll_cq send for the 5 time
<br>Mar 3 21:33:09 linux3 kernel: === QP handle 0x7cdc2b8, QP num 0xa0405: state 3<br>Mar 3 21:33:09 linux3 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty<br>Mar 3 21:33:09 linux3 kernel: reap_send_req: VAPI_poll_cq 5 time(s)
<br>Mar 3 21:33:09 linux3 kernel: Error in reap_send_req: Req unsuccess: UnKnown<br>Mar 3 21:33:09 linux3 kernel: status: 861650944<br>Mar 3 21:33:09 linux3 kernel: id: 2<br>Mar 3 21:33:09 linux3 kernel: opcode: Unknown
<br>Mar 3 21:33:09 linux3 kernel: Num. of bytes transferred: 1<br>Mar 3 21:33:09 linux3 kernel: ...<br>Mar 3 21:33:09 linux3 kernel: Error in test_sr_lat: reap_send_req failed. ret: -1<br><br><br>-- <br>Ian Jiang<br><a href="mailto:ianjiang.ict@gmail.com">
ianjiang.ict@gmail.com</a><br><br>Laboratory of Spatial Information Technology<br>Division of System Architecture<br>Institute of Computing Technology<br>Chinese Academy of Sciences