[ofw] patch: Allow send_bw to work with grh.
Tzachi Dar
tzachid at mellanox.co.il
Mon Nov 22 09:58:25 PST 2010
This checkin is used to allow the send_bw to work with RoCE which requires GRH.
If anyone has idea what I should use for ib_grh_set_ver_class_flow() I'll be happy to know.
Thanks
Tzachi
Index: user/perf_defs.h
===================================================================
--- user/perf_defs.h (revision 6853)
+++ user/perf_defs.h (working copy)
@@ -55,6 +55,13 @@
#define KEY_MSG_SIZE (sizeof "0000:000000:000000:00000000:0000000000000000")
#define KEY_PRINT_FMT "%04x:%06x:%06x:%08x:%016I64x"
+
+// The Format of the message we pass through sockets (With Gid).
+#define KEY_PRINT_FMT_GID "%04x:%06x:%06x:%08x:%016I64x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x"
+
+#define KEY_MSG_SIZE_GID 98 // Message size with gid (MGID as well).
+
+
#define KEY_SCAN_FMT "%x:%x:%x:%x:%x"
#define VERSION 2.0
@@ -122,8 +129,9 @@
ib_net16_t lid;
ib_net32_t qpn;
ib_net32_t psn;
- uint32_t rkey;
- uint64_t vaddr;
+ uint32_t rkey;
+ uint64_t vaddr;
+ ib_gid_t gid;
};
Index: user/perf_utils.c
===================================================================
--- user/perf_utils.c (revision 6853)
+++ user/perf_utils.c (working copy)
@@ -74,10 +74,18 @@
static int pp_write_keys(SOCKET sockfd, const struct pingpong_dest *my_dest)
{
- char msg[KEY_MSG_SIZE];
+ char msg[KEY_MSG_SIZE_GID];
PERF_ENTER;
- sprintf(msg, KEY_PRINT_FMT,cl_hton16(my_dest->lid), cl_hton32(my_dest->qpn),
- cl_hton32(my_dest->psn), cl_hton32(my_dest->rkey), my_dest->vaddr);
+ sprintf(msg, KEY_PRINT_FMT_GID,cl_hton16(my_dest->lid), cl_hton32(my_dest->qpn),
+ cl_hton32(my_dest->psn), cl_hton32(my_dest->rkey), my_dest->vaddr,
+ my_dest->gid.raw[0],my_dest->gid.raw[1],
+ my_dest->gid.raw[2],my_dest->gid.raw[3],
+ my_dest->gid.raw[4],my_dest->gid.raw[5],
+ my_dest->gid.raw[6],my_dest->gid.raw[7],
+ my_dest->gid.raw[8],my_dest->gid.raw[9],
+ my_dest->gid.raw[10],my_dest->gid.raw[11],
+ my_dest->gid.raw[12],my_dest->gid.raw[13],
+ my_dest->gid.raw[14],my_dest->gid.raw[15]);
if (send(sockfd, msg, sizeof msg,0) != sizeof msg) {
perror("pp_write_keys");
@@ -93,7 +101,7 @@
{
int parsed;
- char msg[KEY_MSG_SIZE];
+ char msg[KEY_MSG_SIZE_GID];
PERF_ENTER;
if (recv(sockfd, msg, sizeof msg, 0) != sizeof msg) {
perror("pp_read_keys");
@@ -101,14 +109,23 @@
return -1;
}
- parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn,
- &rem_dest->psn,&rem_dest->rkey, &rem_dest->vaddr);
+ parsed = sscanf(msg, KEY_PRINT_FMT_GID, &rem_dest->lid, &rem_dest->qpn,
+ &rem_dest->psn,&rem_dest->rkey, &rem_dest->vaddr,
+ &rem_dest->gid.raw[0],&rem_dest->gid.raw[1],
+ &rem_dest->gid.raw[2],&rem_dest->gid.raw[3],
+ &rem_dest->gid.raw[4],&rem_dest->gid.raw[5],
+ &rem_dest->gid.raw[6],&rem_dest->gid.raw[7],
+ &rem_dest->gid.raw[8],&rem_dest->gid.raw[9],
+ &rem_dest->gid.raw[10],&rem_dest->gid.raw[11],
+ &rem_dest->gid.raw[12],&rem_dest->gid.raw[13],
+ &rem_dest->gid.raw[14],&rem_dest->gid.raw[15]);
+
rem_dest->lid = cl_ntoh16(rem_dest->lid);
rem_dest->qpn = cl_ntoh32(rem_dest->qpn);
rem_dest->psn = cl_ntoh32(rem_dest->psn);
rem_dest->rkey = cl_ntoh32(rem_dest->rkey);
- if (parsed != 5) {
+ if (parsed != 21) {
fprintf(stderr, "Couldn't parse line <%.*s > parsed = %d %s (%x)\n",
(int)sizeof msg, msg,parsed,sock_get_error_str(), WSAGetLastError());
return -1;
@@ -242,5 +259,3 @@
-
-
Index: user/send_bw/send_bw.c
===================================================================
--- user/send_bw/send_bw.c (revision 6853)
+++ user/send_bw/send_bw.c (working copy)
@@ -53,6 +53,7 @@
int tx_depth;
int duplex;
int use_event;
+ int use_grh;
};
static int page_size;
@@ -402,6 +403,13 @@
IB_MOD_QP_RESP_RES |
IB_MOD_QP_PRIMARY_AV;
+ if(user_parm->use_grh)
+ {
+ attr.state.rtr.primary_av.grh_valid = 1;
+ attr.state.rtr.primary_av.grh.hop_limit = 1;
+ attr.state.rtr.primary_av.grh.dest_gid =dest->gid;
+ attr.state.rtr.primary_av.grh.src_gid = ctx->ca_attr->p_port_attr->p_gid_table[0];
+ }
ib_status = ib_modify_qp(ctx->qp[0], &attr);
if(ib_status != IB_SUCCESS){
@@ -412,7 +420,17 @@
if (user_parm->connection_type == UD) {
ib_av_attr_t av_attr;
- av_attr.grh_valid = 0;
+ if(user_parm->use_grh){
+ av_attr.grh_valid = 1;
+ av_attr.grh.ver_class_flow = ib_grh_set_ver_class_flow(6, 0 ,0);
+ av_attr.grh.resv1 = 0;
+ av_attr.grh.resv2 = 0;
+ av_attr.grh.hop_limit = 1;
+ av_attr.grh.src_gid = ctx->ca_attr->p_port_attr->p_gid_table[0];
+ av_attr.grh.dest_gid = dest->gid;
+ } else {
+ av_attr.grh_valid = 0;
+ }
av_attr.dlid = dest->lid;
av_attr.sl = 0;
av_attr.path_bits = 0;
@@ -534,6 +552,7 @@
/* TBD this should be changed inot VA and different key to each qp */
my_dest[i].rkey = ctx->rkey;
my_dest[i].vaddr = (uintptr_t)ctx->buf + ctx->size;
+ my_dest[i].gid = ctx->ca_attr->p_port_attr->p_gid_table[0];
printf(" local address: LID %#04x, QPN %#06x, PSN %#06x, "
"RKey %#08x VAddr %#016Lx\n",
@@ -581,6 +600,7 @@
printf(" -t, --tx-depth=<dep> size of tx queue (default 300)\n");
printf(" -n, --iters=<iters> number of exchanges (at least 2, default 1000)\n");
printf(" -b, --bidirectional measure bidirectional bandwidth (default unidirectional)\n");
+ printf(" -g, --grh Use GRH with packets (mandatory for RoCE)\n");
printf(" -V, --version display version number\n");
}
@@ -1005,10 +1025,11 @@
{ "bidirectional", 0, NULL, 'b' },
{ "version", 0, NULL, 'V' },
{ "events", 0, NULL, 'e' },
+ { "grh", 0, NULL, 'g' },
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:ebaVh", long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:ebaVhg", long_options, NULL);
if (c == -1)
break;
@@ -1077,6 +1098,11 @@
case 'b':
user_param.duplex = 1;
break;
+
+ case 'g':
+ user_param.use_grh = 1;
+ break;
+
case 'h':
default:
usage(argv[0]);
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20101122/0a2047e8/attachment.html>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: perf_utils.diff
Type: application/octet-stream
Size: 5847 bytes
Desc: perf_utils.diff
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20101122/0a2047e8/attachment.obj>
More information about the ofw
mailing list