[openib-general] [VAPI]register physical memory in kernel space
Ian Jiang
ianjiang.ict at gmail.com
Fri Mar 3 05:58:36 PST 2006
I am trying to use VAPI in kernel space and have a question about the
registration of physical memory.
"11.2.8.3 REGISTER PHYSICAL MEMORY REGION" of IB spec. 1.2 says
Actual size of the PBL resources allocated. The actual size of the
PBL resources allocated shall be greater than or equal to the size
of the PBL resources passed in by the Consumer as an input
modifier.
But the pbuf_len in the structure of type VAPI_mrw_t returned bye
VAPI_register_mr() is zero in my test.
I am not sure if this correct and whether that is what caused the
"VAPI_CQ_EMPTY" problem.
Any suggestion is appreciated and thanks a lot!
======================
codes related to registration
======================
struct user_mr {
u_int8_t *user_buf;
buf_type_t buf_type;
int align;
unsigned int user_buf_len;
VAPI_hca_hndl_t hca_hndl;
VAPI_pd_hndl_t pd_hndl;
VAPI_mrw_t mrw_req; /* requested memory region properties */
VAPI_mrw_t mrw_rep; /* responded memory region properties */
VAPI_mr_hndl_t mr_hndl;
};
int user_pmr_create(struct user_mr *user_mr_p, unsigned int sz)
{
VAPI_ret_t vapi_ret;
VAPI_phy_buf_t *pbuf_list_p = NULL;
unsigned int pbuf_list_len = 0;
unsigned int pages;
u_int64_t offset, len;
void * addr;
int i;
if (user_mr_p == NULL) {
PRINT_ERR("NULL user_mr_p\n");
return -1;
}
ini_user_mr(user_mr_p);
/* allocate */
/* TODO: deal with large MR size */
if (sz <0 || sz > (128 * 1024)) {
PRINT_ERR("invalid MR size: %d\n", sz);
return -1;
}
user_mr_p->user_buf = kmalloc(sz, GFP_ATOMIC);
if (user_mr_p->user_buf == NULL) {
PRINT_ERR("kmalloc failed. size: %u\n", sz);
return -1;
}
user_mr_p->user_buf_len = sz;
PRINT_TRACE("user buffer 0x%p, len: %u\n", user_mr_p->user_buf, sz);
/* set buffer vectors for physical memory */
offset = offset_in_page(user_mr_p->user_buf);
len = PAGE_ALIGN(sz + offset);
pages = len / PAGE_SIZE;
pbuf_list_p = kmalloc(pages * sizeof(VAPI_phy_buf_t), GFP_ATOMIC);
if (pbuf_list_p == NULL) {
PRINT_ERR("kmalloc for pbuf_list failed. size: %d * %d\n", pages,
(int)sizeof(VAPI_phy_buf_t));
return -1;
}
pbuf_list_len = pages;
addr = user_mr_p->user_buf - offset;
for (i = 0; i < pages; i++) {
pbuf_list_p[i].start = virt_to_phys(addr);
pbuf_list_p[i].size = PAGE_SIZE;
addr += PAGE_SIZE;
#ifdef KVAPITEST_TRACE_PBUF_LIST
MY_PRINT("\tpbuf_list[0x%x]: 0x%llx\n", i, pbuf_list_p[i].start);
#endif
}
/* register */
user_mr_p->mrw_req.pbuf_list_len = pbuf_list_len;
user_mr_p->mrw_req.pbuf_list_p = pbuf_list_p;
user_mr_p->mrw_req.iova_offset = offset;
user_mr_p->mrw_req.type = VAPI_MPR;
user_mr_p->mrw_req.pd_hndl = user_mr_p->pd_hndl;
user_mr_p->mrw_req.start = (VAPI_virt_addr_t)(MT_virt_addr_t)
user_mr_p->user_buf;
user_mr_p->mrw_req.acl = VAPI_EN_LOCAL_WRITE
| VAPI_EN_REMOTE_WRITE
| VAPI_EN_REMOTE_READ;
user_mr_p->mrw_req.size = user_mr_p->user_buf_len;
PRINT_MR_PROP(&user_mr_p->mrw_req);
vapi_ret = VAPI_register_mr(user_mr_p->hca_hndl,
&user_mr_p->mrw_req,
&user_mr_p->mr_hndl,
&user_mr_p->mrw_rep);
if (vapi_ret != VAPI_OK) {
MY_PRINT("MR register failed\n");
VAPIERR(vapi_ret);
goto reg_err;
}
PRINT_TRACE("MR registered. MR handle: %d\n", user_mr_p->mr_hndl);
PRINT_MR_PROP(&user_mr_p->mrw_rep);
if (pbuf_list_p != NULL) {
kfree(pbuf_list_p);
}
return 0;
reg_err:
if (user_mr_p->user_buf != NULL) {
kfree(user_mr_p->user_buf);
user_mr_p->user_buf_len = 0;
}
if (pbuf_list_p != NULL) {
kfree(pbuf_list_p);
}
return -1;
}
===================
log messages of Receiver
===================
Mar 4 05:03:16 linux2 kernel: ==== MSG ====
Mar 4 05:03:16 linux2 kernel: test type: Send/Recv
Mar 4 05:03:16 linux2 kernel: LID: 0x169
Mar 4 05:03:16 linux2 kernel: QP NUM: 0xa0405
Mar 4 05:03:16 linux2 kernel: R_Key: 0x0
Mar 4 05:03:16 linux2 kernel: VA: 0x0
Mar 4 05:03:16 linux2 kernel: required size: 10000
Mar 4 05:03:16 linux2 kernel: align to PAGE_SZ: 1
Mar 4 05:03:16 linux2 kernel: iterate times: 1
Mar 4 05:03:16 linux2 kernel: communicate_params: [Receiver]Send Message
Mar 4 05:03:16 linux2 kernel: ==== MSG ====
Mar 4 05:03:16 linux2 kernel: test type: Send/Recv
Mar 4 05:03:16 linux2 kernel: LID: 0x161
Mar 4 05:03:16 linux2 kernel: QP NUM: 0xa0405
Mar 4 05:03:16 linux2 kernel: R_Key: 0x0
Mar 4 05:03:16 linux2 kernel: VA: 0x0
Mar 4 05:03:16 linux2 kernel: required size: 10000
Mar 4 05:03:16 linux2 kernel: align to PAGE_SZ: 1
Mar 4 05:03:16 linux2 kernel: iterate times: 1
Mar 4 05:03:16 linux2 kernel: kvapitest_init: get ready to transfer
Mar 4 05:03:16 linux2 kernel: ready_to_trans: move to RTR...
Mar 4 05:03:16 linux2 kernel: qp_move_to_rtr: Modified QP to RTR
Mar 4 05:03:16 linux2 kernel: max_oust_wr_sq = 255
Mar 4 05:03:16 linux2 kernel: max_oust_wr_rq = 255
Mar 4 05:03:16 linux2 kernel: max_sg_size_sq = 5
Mar 4 05:03:16 linux2 kernel: max_sg_size_rq = 3
Mar 4 05:03:16 linux2 kernel: ==== QP attr to rtr ====
Mar 4 05:03:16 linux2 kernel: .qp_state: 2
Mar 4 05:03:16 linux2 kernel: .av.sl: 0
Mar 4 05:03:16 linux2 kernel: .av.grh_flag: 0
Mar 4 05:03:16 linux2 kernel: .av.dlid: 0x169
Mar 4 05:03:16 linux2 kernel: .av.static_rate: 2
Mar 4 05:03:16 linux2 kernel: .av.src_path_bits: 0
Mar 4 05:03:16 linux2 kernel: .path_mtu: 4
Mar 4 05:03:16 linux2 kernel: .dst_qp_num: 0xa0405
Mar 4 05:03:16 linux2 kernel: ready_to_trans: move to RTS...
Mar 4 05:03:16 linux2 kernel: qp_move_to_rts: Modified QP to RTS
Mar 4 05:03:16 linux2 kernel: test_sr_lat: test 0, [Receiver]
Mar 4 05:03:16 linux2 kernel: test_sr_lat: create in_mr
Mar 4 05:03:16 linux2 kernel: user_pmr_create: user buffer
0x0000010029b64000, len: 10000
Mar 4 05:03:16 linux2 kernel: ==== MR ==== (requested)
Mar 4 05:03:16 linux2 kernel: VAPI_mrw_type: 2
Mar 4 05:03:16 linux2 kernel: VAPI_lkey: 0
Mar 4 05:03:16 linux2 kernel: VAPI_rkey: 0
Mar 4 05:03:16 linux2 kernel: VAPI_virt_addr: 0x10029b64000
Mar 4 05:03:16 linux2 kernel: VAPI_size: 10000
Mar 4 05:03:16 linux2 kernel: VAPI_pd_hndl: 1100297486048
Mar 4 05:03:16 linux2 kernel: VAPI_mrw_acl: 7
Mar 4 05:03:16 linux2 kernel: pbuf_len: 3
Mar 4 05:03:16 linux2 kernel: VAPI_phy_buf: 0x000001002c8e4580
Mar 4 05:03:16 linux2 kernel: pbuf_list[0x0]: 0x29b64000
Mar 4 05:03:16 linux2 kernel: pbuf_list[0x1]: 0x29b65000
Mar 4 05:03:16 linux2 kernel: pbuf_list[0x2]: 0x29b66000
Mar 4 05:03:16 linux2 kernel: iova_offset: 0x0
Mar 4 05:03:16 linux2 kernel: user_pmr_create: MR registered. MR handle: 3
Mar 4 05:03:16 linux2 kernel: ==== MR ==== (returned)
Mar 4 05:03:16 linux2 kernel: VAPI_mrw_type: 2
Mar 4 05:03:16 linux2 kernel: VAPI_lkey: 1631554304
Mar 4 05:03:16 linux2 kernel: VAPI_rkey: 1631554304
Mar 4 05:03:16 linux2 kernel: VAPI_virt_addr: 0x10029b64000
Mar 4 05:03:16 linux2 kernel: VAPI_size: 10000
Mar 4 05:03:16 linux2 kernel: VAPI_pd_hndl: 1100297486048
Mar 4 05:03:16 linux2 kernel: VAPI_mrw_acl: 7
Mar 4 05:03:16 linux2 kernel: pbuf_len: 0
Mar 4 05:03:16 linux2 kernel: VAPI_phy_buf: 0x0000000000000000
Mar 4 05:03:16 linux2 kernel: iova_offset: 0x0
Mar 4 05:03:16 linux2 kernel: test_sr_lat: post_recv
Mar 4 05:03:16 linux2 kernel: VAPI_post_rr with
Mar 4 05:03:16 linux2 kernel: HCA: 0x0
Mar 4 05:03:16 linux2 kernel: QP: 0x1002c8ef088
Mar 4 05:03:16 linux2 kernel: send_req ID: 1100211437568
Mar 4 05:03:16 linux2 kernel: opcode: VAPI_RECEIVE
Mar 4 05:03:16 linux2 kernel: comp_type: 0 (0-SIGNALED, 1-UNSIGNALED)
Mar 4 05:03:16 linux2 kernel: SG list len: 1 SG_lst_entty[0]
addr:0x10029b64000, len:10000, lkey: 1631554304
Mar 4 05:03:16 linux2 kernel: test_sr_lat: reap_recv
Mar 4 05:03:16 linux2 kernel: to VAPI_poll_cq recv for the 1 time
Mar 4 05:03:16 linux2 kernel: === QP handle 0x2c8ef088, QP num 0xa0405:
state 3
Mar 4 05:03:16 linux2 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
Mar 4 05:03:16 linux2 kernel: to VAPI_poll_cq recv for the 2 time
Mar 4 05:03:16 linux2 kernel: === QP handle 0x2c8ef088, QP num 0xa0405:
state 3
Mar 4 05:03:16 linux2 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
Mar 4 05:03:16 linux2 kernel: to VAPI_poll_cq recv for the 3 time
Mar 4 05:03:16 linux2 kernel: === QP handle 0x2c8ef088, QP num 0xa0405:
state 3
Mar 4 05:03:16 linux2 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
Mar 4 05:03:16 linux2 kernel: to VAPI_poll_cq recv for the 4 time
Mar 4 05:03:16 linux2 kernel: === QP handle 0x2c8ef088, QP num 0xa0405:
state 3
Mar 4 05:03:16 linux2 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
Mar 4 05:03:16 linux2 kernel: to VAPI_poll_cq recv for the 5 time
Mar 4 05:03:16 linux2 kernel: === QP handle 0x2c8ef088, QP num 0xa0405:
state 3
Mar 4 05:03:16 linux2 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
Mar 4 05:03:16 linux2 kernel: reap_recv_req: VAPI_poll_cq 5 time(s)
Mar 4 05:03:16 linux2 kernel: Error in reap_recv_req: Req failed: UnKnown
Mar 4 05:03:16 linux2 kernel: status: 743692440
Mar 4 05:03:16 linux2 kernel: id: 16
Mar 4 05:03:16 linux2 kernel: opcode: VAPI_CQE_SQ_SEND_DATA
Mar 4 05:03:16 linux2 kernel: Num. of bytes transferred: 0
Mar 4 05:03:16 linux2 kernel: ...
Mar 4 05:03:16 linux2 kernel: Error in test_sr_lat: reap_recv_req failed.
ret: -1
==================
log messages of Sender
==================
Mar 3 21:33:08 linux3 kernel: ==== MSG ====
Mar 3 21:33:08 linux3 kernel: test type: Send/Recv
Mar 3 21:33:08 linux3 kernel: LID: 0x169
Mar 3 21:33:08 linux3 kernel: QP NUM: 0xa0405
Mar 3 21:33:08 linux3 kernel: R_Key: 0x0
Mar 3 21:33:08 linux3 kernel: VA: 0x0
Mar 3 21:33:08 linux3 kernel: required size: 10000
Mar 3 21:33:08 linux3 kernel: align to PAGE_SZ: 1
Mar 3 21:33:08 linux3 kernel: iterate times: 1
Mar 3 21:33:08 linux3 kernel: communicate_params: [Sender]to receive
Message
Mar 3 21:33:08 linux3 kernel: communicate_params: [Sender] received message
Mar 3 21:33:08 linux3 kernel: ==== MSG ====
Mar 3 21:33:08 linux3 kernel: test type: Send/Recv
Mar 3 21:33:08 linux3 kernel: LID: 0x161
Mar 3 21:33:08 linux3 kernel: QP NUM: 0xa0405
Mar 3 21:33:08 linux3 kernel: R_Key: 0x0
Mar 3 21:33:08 linux3 kernel: VA: 0x0
Mar 3 21:33:08 linux3 kernel: required size: 10000
Mar 3 21:33:08 linux3 kernel: align to PAGE_SZ: 1
Mar 3 21:33:08 linux3 kernel: iterate times: 1
Mar 3 21:33:08 linux3 kernel: kvapitest_init: get ready to transfer
Mar 3 21:33:08 linux3 kernel: ready_to_trans: move to RTR...
Mar 3 21:33:08 linux3 kernel: qp_move_to_rtr: Modified QP to RTR
Mar 3 21:33:08 linux3 kernel: max_oust_wr_sq = 255
Mar 3 21:33:08 linux3 kernel: max_oust_wr_rq = 255
Mar 3 21:33:08 linux3 kernel: max_sg_size_sq = 5
Mar 3 21:33:08 linux3 kernel: max_sg_size_rq = 3
Mar 3 21:33:08 linux3 kernel: ==== QP attr to rtr ====
Mar 3 21:33:08 linux3 kernel: .qp_state: 2
Mar 3 21:33:08 linux3 kernel: .av.sl: 0
Mar 3 21:33:08 linux3 kernel: .av.grh_flag: 0
Mar 3 21:33:08 linux3 kernel: .av.dlid: 0x161
Mar 3 21:33:08 linux3 kernel: .av.static_rate: 2
Mar 3 21:33:08 linux3 kernel: .av.src_path_bits: 0
Mar 3 21:33:08 linux3 kernel: .path_mtu: 4
Mar 3 21:33:08 linux3 kernel: .dst_qp_num: 0xa0405
Mar 3 21:33:08 linux3 kernel: ready_to_trans: move to RTS...
Mar 3 21:33:08 linux3 kernel: qp_move_to_rts: Modified QP to RTS
Mar 3 21:33:08 linux3 kernel: test_sr_lat: test 0, [Sender]
Mar 3 21:33:08 linux3 kernel: test_sr_lat: wait to start...
Mar 3 21:33:08 linux3 kernel: test_sr_lat: create out_mr
Mar 3 21:33:08 linux3 kernel: user_pmr_create: user buffer
0x00000100335bc000, len: 10000
Mar 3 21:33:08 linux3 kernel: ==== MR ==== (requested)
Mar 3 21:33:08 linux3 kernel: VAPI_mrw_type: 2
Mar 3 21:33:08 linux3 kernel: VAPI_lkey: 0
Mar 3 21:33:08 linux3 kernel: VAPI_rkey: 0
Mar 3 21:33:08 linux3 kernel: VAPI_virt_addr: 0x100335bc000
Mar 3 21:33:08 linux3 kernel: VAPI_size: 10000
Mar 3 21:33:08 linux3 kernel: VAPI_pd_hndl: 1100433669576
Mar 3 21:33:08 linux3 kernel: VAPI_mrw_acl: 7
Mar 3 21:33:08 linux3 kernel: pbuf_len: 3
Mar 3 21:33:08 linux3 kernel: VAPI_phy_buf: 0x000001000e8e83c8
Mar 3 21:33:08 linux3 kernel: pbuf_list[0x0]: 0x335bc000
Mar 3 21:33:08 linux3 kernel: pbuf_list[0x1]: 0x335bd000
Mar 3 21:33:08 linux3 kernel: pbuf_list[0x2]: 0x335be000
Mar 3 21:33:08 linux3 kernel: iova_offset: 0x0
Mar 3 21:33:08 linux3 kernel: user_pmr_create: MR registered. MR handle: 3
Mar 3 21:33:08 linux3 kernel: ==== MR ==== (returned)
Mar 3 21:33:08 linux3 kernel: VAPI_mrw_type: 2
Mar 3 21:33:08 linux3 kernel: VAPI_lkey: 1497336576
Mar 3 21:33:08 linux3 kernel: VAPI_rkey: 1497336576
Mar 3 21:33:08 linux3 kernel: VAPI_virt_addr: 0x100335bc000
Mar 3 21:33:08 linux3 kernel: VAPI_size: 10000
Mar 3 21:33:08 linux3 kernel: VAPI_pd_hndl: 1100433669576
Mar 3 21:33:08 linux3 kernel: VAPI_mrw_acl: 7
Mar 3 21:33:08 linux3 kernel: pbuf_len: 0
Mar 3 21:33:08 linux3 kernel: VAPI_phy_buf: 0x0000000000000000
Mar 3 21:33:08 linux3 kernel: iova_offset: 0x0
Mar 3 21:33:08 linux3 kernel: test_sr_lat: post_send
Mar 3 21:33:08 linux3 kernel: VAPI_post_sr with
Mar 3 21:33:08 linux3 kernel: HCA: 0x0
Mar 3 21:33:08 linux3 kernel: QP: 0x10007cdc2b8
Mar 3 21:33:08 linux3 kernel: send_req ID: 1100373278720
Mar 3 21:33:08 linux3 kernel: opcode: VAPI_SEND
Mar 3 21:33:08 linux3 kernel: comp_type: 0 (0-SIGNALED, 1-UNSIGNALED)
Mar 3 21:33:08 linux3 kernel: SG list len: 1 SG_lst_entty[0]
addr:0x100335bc000, len:10000, lkey: 1497336576
Mar 3 21:33:08 linux3 kernel: imm_data: 256
Mar 3 21:33:08 linux3 kernel: fence: 72(1-True, 0-False)
Mar 3 21:33:08 linux3 kernel: set_se: 0(1-True, 0-False)
Mar 3 21:33:08 linux3 kernel: test_sr_lat: reap_send
Mar 3 21:33:08 linux3 kernel: to VAPI_poll_cq send for the 1 time
Mar 3 21:33:08 linux3 kernel: === QP handle 0x7cdc2b8, QP num 0xa0405:
state 3
Mar 3 21:33:08 linux3 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
Mar 3 21:33:08 linux3 kernel: to VAPI_poll_cq send for the 2 time
Mar 3 21:33:09 linux3 kernel: === QP handle 0x7cdc2b8, QP num 0xa0405:
state 3
Mar 3 21:33:09 linux3 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
Mar 3 21:33:09 linux3 kernel: to VAPI_poll_cq send for the 3 time
Mar 3 21:33:09 linux3 kernel: === QP handle 0x7cdc2b8, QP num 0xa0405:
state 3
Mar 3 21:33:09 linux3 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
Mar 3 21:33:09 linux3 kernel: to VAPI_poll_cq send for the 4 time
Mar 3 21:33:09 linux3 kernel: === QP handle 0x7cdc2b8, QP num 0xa0405:
state 3
Mar 3 21:33:09 linux3 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
Mar 3 21:33:09 linux3 kernel: to VAPI_poll_cq send for the 5 time
Mar 3 21:33:09 linux3 kernel: === QP handle 0x7cdc2b8, QP num 0xa0405:
state 3
Mar 3 21:33:09 linux3 kernel: VAPI ret: -212, VAPI_CQ_EMPTY: CQ is empty
Mar 3 21:33:09 linux3 kernel: reap_send_req: VAPI_poll_cq 5 time(s)
Mar 3 21:33:09 linux3 kernel: Error in reap_send_req: Req unsuccess:
UnKnown
Mar 3 21:33:09 linux3 kernel: status: 861650944
Mar 3 21:33:09 linux3 kernel: id: 2
Mar 3 21:33:09 linux3 kernel: opcode: Unknown
Mar 3 21:33:09 linux3 kernel: Num. of bytes transferred: 1
Mar 3 21:33:09 linux3 kernel: ...
Mar 3 21:33:09 linux3 kernel: Error in test_sr_lat: reap_send_req failed.
ret: -1
--
Ian Jiang
ianjiang.ict at gmail.com
Laboratory of Spatial Information Technology
Division of System Architecture
Institute of Computing Technology
Chinese Academy of Sciences
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/general/attachments/20060303/cacbbd1b/attachment.html>
More information about the general
mailing list