[ofw] Patch: ib_send_bw add the option to also run with events instead of polling.
Tzachi Dar
tzachid at mellanox.co.il
Sun Jun 29 07:00:02 PDT 2008
Index: perf_defs.h
===================================================================
--- perf_defs.h (revision 1304)
+++ perf_defs.h (working copy)
@@ -114,6 +114,7 @@
volatile char *poll_buf;
int *scnt,*ccnt;
+ cl_waitobj_handle_t cq_waitobj;
};
Index: send_bw/send_bw.c
===================================================================
--- send_bw/send_bw.c (revision 1304)
+++ send_bw/send_bw.c (working copy)
@@ -51,7 +51,7 @@
int iters;
int tx_depth;
int duplex;
- int use_event;
+ int use_event;
};
static int page_size;
@@ -62,7 +62,7 @@
void
pp_cq_comp_cb(
- IN const ib_cq_handle_t h_cq,
+ IN const ib_cq_handle_t h_cq,
IN void *cq_context )
{
UNUSED_PARAM( h_cq );
@@ -72,7 +72,8 @@
-static struct pingpong_context *pp_init_ctx(unsigned size,int port,
struct user_parameters *user_parm)
+static struct pingpong_context *pp_init_ctx(unsigned size,int port,
+ struct user_parameters *user_parm)
{
struct pingpong_context *ctx;
@@ -101,14 +102,14 @@
ctx->tx_depth = user_parm->tx_depth;
/* in case of UD need space for the GRH */
if (user_parm->connection_type==UD) {
- ctx->buf = malloc(( size + 40 ) * 2); //PORTED ALINGED
+ ctx->buf = malloc(( size + 40 ) * 2);
if (!ctx->buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
return NULL;
}
memset(ctx->buf, 0, ( size + 40 ) * 2);
} else {
- ctx->buf = malloc( size * 2); //PORTED ALINGED
+ ctx->buf = malloc( size * 2);
if (!ctx->buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
return NULL;
@@ -199,17 +200,9 @@
}
}
}
-
- if (user_parm->use_event) {
-//PORTED ctx->channel = ibv_create_comp_channel(ctx->context);
- ctx->channel = NULL;//remove when PORTED
- if (!ctx->channel) {
- fprintf(stderr, "Couldn't create completion channel\n");
- return NULL;
- }
- } else
- ctx->channel = NULL;
+ ctx->channel = NULL;
+
ib_status = ib_alloc_pd(ctx->ca ,
IB_PDT_NORMAL,
ctx, //pd_context
@@ -241,16 +234,42 @@
return NULL;
}
+ if (user_parm->use_event) {
+ cl_status_t cl_status;
+
+ cl_status = cl_waitobj_create( FALSE, &ctx->cq_waitobj );
+ if( cl_status != CL_SUCCESS ) {
+ ctx->cq_waitobj = NULL;
+ fprintf(stderr, "cl_waitobj_create() returned %s\n",
CL_STATUS_MSG(cl_status) );
+ return NULL;
+ }
+
+ cq_create.h_wait_obj = ctx->cq_waitobj;
+ cq_create.pfn_comp_cb = NULL;
+ } else {
+ cq_create.h_wait_obj = NULL;
+ cq_create.pfn_comp_cb = pp_cq_comp_cb;
+ }
+
cq_create.size = user_parm->tx_depth*2;
- cq_create.h_wait_obj = NULL;
- cq_create.pfn_comp_cb = pp_cq_comp_cb;
ib_status = ib_create_cq(ctx->ca,&cq_create ,ctx, NULL, &ctx->scq);
if (ib_status != IB_SUCCESS) {
fprintf(stderr, "Couldn't create CQ\n");
+ fprintf(stderr, "ib_status = %d\n", ib_status);
return NULL;
}
+
+ if (user_parm->use_event) {
+ ib_status = ib_rearm_cq( ctx->scq, FALSE );
+ if( ib_status )
+ {
+ ib_destroy_cq( ctx->scq, NULL );
+ fprintf(stderr,"ib_rearm_cq returned %s\n", ib_get_err_str(
ib_status ));
+ return NULL;
+ }
+ }
}
-
+
{
ib_qp_create_t qp_create;
memset(&qp_create, 0, sizeof(ib_qp_create_t));
@@ -579,7 +598,7 @@
struct pingpong_dest *rem_dest, int size)
{
- ib_qp_handle_t qp;
+ ib_qp_handle_t qp;
int scnt, ccnt, rcnt;
ib_recv_wr_t *bad_wr_recv;
ib_api_status_t ib_status;
@@ -648,24 +667,19 @@
p_wc_free = &wc;
p_wc_free->p_next = NULL;
p_wc_done = NULL;
-#if PORTED
+
if (user_param->use_event) {
- struct ibv_cq *ev_cq;
- void *ev_ctx;
- if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
- fprintf(stderr, "Failed to get cq_event\n");
+ cl_status_t cl_status;
+
+ PERF_DEBUG("%s:%d IN cl_waitobj_wait_on", __FUNCTION__, __LINE__);
+ cl_status = cl_waitobj_wait_on( ctx->cq_waitobj, EVENT_NO_TIMEOUT,
TRUE );
+ if( cl_status != CL_SUCCESS )
+ {
+ fprintf(stderr, "cl_waitobj_wait_on() (%d)\n", cl_status);
return 1;
- }
- if (ev_cq != ctx->cq) {
- fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
- return 1;
}
- if (ibv_req_notify_cq(ctx->cq, 0)) {
- fprintf(stderr, "Couldn't request CQ notification\n");
- return 1;
- }
- }
-#endif
+ }
+
do {
ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done);
if (ib_status == IB_SUCCESS ) {
@@ -678,6 +692,7 @@
scnt, ccnt);
return 1;
}
+
switch ((int) p_wc_done->wr_id) {
case PINGPONG_SEND_WRID:
tcompleted[ccnt] = get_cycles();
@@ -716,6 +731,15 @@
return 1;
}
+ if (user_param->use_event) {
+ ib_status = ib_rearm_cq( ctx->scq, FALSE );
+ if( ib_status )
+ {
+ ib_destroy_cq( ctx->scq, NULL );
+ fprintf(stderr,"ib_rearm_cq returned %s\n", ib_get_err_str(
ib_status ));
+ return 1;
+ }
+ }
}
}
return(0);
@@ -726,7 +750,7 @@
struct pingpong_dest *rem_dest, int size)
{
- ib_qp_handle_t qp;
+ ib_qp_handle_t qp;
int scnt, ccnt, rcnt;
ib_recv_wr_t *bad_wr_recv;
ib_api_status_t ib_status;
@@ -770,25 +794,21 @@
p_wc_free->p_next = NULL;
/*Server is polling on recieve first */
-#if PORTED
if (user_param->use_event) {
- struct ibv_cq *ev_cq;
- void *ev_ctx;
- if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
- fprintf(stderr, "Failed to get cq_event\n");
+ cl_status_t cl_status;
+
+ PERF_DEBUG("%s:%d IN cl_waitobj_wait_on\n", __FUNCTION__,
__LINE__);
+ cl_status = cl_waitobj_wait_on( ctx->cq_waitobj, EVENT_NO_TIMEOUT,
TRUE );
+ if( cl_status != CL_SUCCESS )
+ {
+ fprintf(stderr, "cl_waitobj_wait_on() (%d)\n", cl_status);
return 1;
- }
- if (ev_cq != ctx->cq) {
- fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
- return 1;
}
- if (ibv_req_notify_cq(ctx->cq, 0)) {
- fprintf(stderr, "Couldn't request CQ notification\n");
- return 1;
- }
- }
-#endif
+ PERF_DEBUG("%s:%d OUT cl_waitobj_wait_on\n", __FUNCTION__,
__LINE__);
+ }
+
do {
+
ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done);
if (ib_status == IB_SUCCESS) {
tcompleted[ccnt] = get_cycles();
@@ -818,20 +838,29 @@
p_wc_free->p_next = NULL;
p_wc_done = NULL;
}
-
-
} while (ib_status == IB_SUCCESS);
if (ib_status != IB_NOT_FOUND) {
fprintf(stderr, "Poll Recieve CQ failed %d\n", ib_status);
return 12;
}
+ if (user_param->use_event) {
+ ib_status = ib_rearm_cq( ctx->scq, FALSE );
+ if( ib_status )
+ {
+ ib_destroy_cq( ctx->scq, NULL );
+ fprintf(stderr,"ib_rearm_cq returned %s\n", ib_get_err_str(
ib_status ));
+ return 1;
+ }
+ }
+
}
} else {
/* client is posting and not receiving. */
while (scnt < user_param->iters || ccnt < user_param->iters) {
while (scnt < user_param->iters && (scnt - ccnt) <
user_param->tx_depth ) {
ib_send_wr_t *bad_wr;
+
tposted[scnt] = get_cycles();
ib_status = ib_post_send(qp, &ctx->wr, &bad_wr);
if (ib_status != IB_SUCCESS) {
@@ -850,25 +879,20 @@
p_wc_free = &wc;
p_wc_free->p_next = NULL;
p_wc_done = NULL;
-
-#if PORTED
- if (user_param->use_event) {
- struct ibv_cq *ev_cq;
- void *ev_ctx;
- if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
- fprintf(stderr, "Failed to get cq_event\n");
+
+ if ( (user_param->use_event) ) {
+ cl_status_t cl_status;
+
+ PERF_DEBUG("%s:%d IN cl_waitobj_wait_on\n", __FUNCTION__,
__LINE__);
+ cl_status = cl_waitobj_wait_on( ctx->cq_waitobj, EVENT_NO_TIMEOUT,
TRUE );
+ if( cl_status != CL_SUCCESS )
+ {
+ fprintf(stderr, "cl_waitobj_wait_on() (%d)\n", cl_status);
return 1;
- }
- if (ev_cq != ctx->cq) {
- fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
- return 1;
}
- if (ibv_req_notify_cq(ctx->cq, 0)) {
- fprintf(stderr, "Couldn't request CQ notification\n");
- return 1;
- }
+ PERF_DEBUG("%s:%d OUT cl_waitobj_wait_on\n", __FUNCTION__,
__LINE__);
}
-#endif
+
do {
ib_status = ib_poll_cq(ctx->scq, &p_wc_free, &p_wc_done);
if (ib_status == IB_SUCCESS ) {
@@ -887,13 +911,22 @@
p_wc_free->p_next = NULL;
p_wc_done = NULL;
}
-
- } while (ib_status == IB_SUCCESS );
+ } while (ib_status == IB_SUCCESS );
if (ib_status != IB_NOT_FOUND) {
fprintf(stderr, "poll CQ failed %d\n", ib_status);
return 1;
}
+ if ( (user_param->use_event) ) {
+ ib_status = ib_rearm_cq( ctx->scq, FALSE );
+ if( ib_status )
+ {
+ ib_destroy_cq( ctx->scq, NULL );
+ fprintf(stderr,"ib_rearm_cq returned %s\n", ib_get_err_str(
ib_status ));
+ return 1;
+ }
+ }
+
PERF_DEBUG("ccnt = %d \n",ccnt);
}
}
@@ -904,21 +937,20 @@
int __cdecl main(int argc, char *argv[])
{
- struct pingpong_context *ctx;
- struct pingpong_dest *my_dest;
- struct pingpong_dest *rem_dest;
- struct user_parameters user_param;
+ struct pingpong_context *ctx;
+ struct pingpong_dest *my_dest;
+ struct pingpong_dest *rem_dest;
+ struct user_parameters user_param;
char *ib_devname = NULL;
int port = 18515;
int ib_port = 1;
unsigned size = 65536;
- SOCKET sockfd = INVALID_SOCKET;
+ SOCKET sockfd = INVALID_SOCKET;
int i = 0;
int size_max_pow = 24;
- WSADATA wsaData;
+ WSADATA wsaData;
int iResult;
-
/* init default values to user's parameters */
memset(&user_param, 0, sizeof(struct user_parameters));
@@ -985,7 +1017,7 @@
break;
case 'i':
ib_port = strtol(optarg, NULL, 0);
- if (ib_port < 0) {
+ if (ib_port <= 0) {
usage(argv[0]);
return 1;
}
@@ -1081,15 +1113,10 @@
if (sockfd == INVALID_SOCKET)
return 9;
-#if PORTED
if (user_param.use_event) {
printf("Test with events.\n");
- if (ibv_req_notify_cq(ctx->cq, 0)) {
- fprintf(stderr, "Couldn't request CQ notification\n");
- return 1;
- }
}
-#endif
+
printf("----------------------------------------------------------------
--\n");
printf(" #bytes #iterations BW peak[MB/sec] BW average[MB/sec]
\n");
@@ -1164,7 +1191,18 @@
goto end;
-end:
+end:
+ if (user_param.use_event) {
+ cl_status_t cl_status;
+
+ cl_status = cl_waitobj_destroy( ctx->cq_waitobj );
+ if( cl_status != CL_SUCCESS )
+ {
+ fprintf (stderr,
+ "cl_waitobj_destroy() returned %s\n", CL_STATUS_MSG(cl_status));
+ }
+ }
+
WSACleanup();
return 0;
}
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20080629/e9fb770f/attachment.html>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: send_bw.patch
Type: application/octet-stream
Size: 10652 bytes
Desc: send_bw.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20080629/e9fb770f/attachment.obj>
More information about the ofw
mailing list