<div>Hi,</div>
<div> </div>
<div>I'm resending this hoping that second time is the charm :-) Thanks.</div>
<div> </div>
<div>-- Hal<br></div>
<div class="gmail_quote">On Mon, Jul 20, 2009 at 10:09 AM, Hal Rosenstock <span dir="ltr"><<a href="mailto:hal.rosenstock@gmail.com">hal.rosenstock@gmail.com</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="PADDING-LEFT: 1ex; MARGIN: 0px 0px 0px 0.8ex; BORDER-LEFT: #ccc 1px solid">Two questions/comments:<br><br>If these are now no longer IB specific, should the executable names be<br>changed from ib_xxx to rdma_xxx ?<br>
<br>Isn't the GID parameter also applicable to IB and iWARP ? If so, will<br>it work for those configurations too ? It looks to me like some things<br>use this to determine RDMAoE.<br><font color="#888888"><br>-- Hal<br>
</font>
<div>
<div></div>
<div class="h5"><br>On Mon, Jul 20, 2009 at 4:48 AM, orenmeron<<a href="mailto:orenmeron@dev.mellanox.co.il">orenmeron@dev.mellanox.co.il</a>> wrote:<br>> --- a/read_bw.c<br>> +++ b/read_bw.c<br>> @@ -71,6 +71,7 @@ struct user_parameters {<br>
> int max_out_read;<br>> int use_event;<br>> int qp_timeout;<br>> + int gid_index; /* if value not negative, we use gid AND gid_index=value */<br>> };<br>> static int sl = 0;<br>
> static int page_size;<br>> @@ -88,6 +89,7 @@ struct pingpong_context {<br>> int tx_depth;<br>> struct ibv_sge list;<br>> struct ibv_send_wr wr;<br>> + union ibv_gid dgid;<br>
> };<br>><br>> struct pingpong_dest {<br>> @@ -96,6 +98,7 @@ struct pingpong_dest {<br>> int psn;<br>> unsigned rkey;<br>> unsigned long long vaddr;<br>> + union ibv_gid dgid;<br>
> };<br>><br>><br>> @@ -150,14 +153,20 @@ static int pp_client_connect(const char *servername, int port)<br>> }<br>><br>> struct pingpong_dest * pp_client_exch_dest(int sockfd,<br>> - const struct pingpong_dest *my_dest)<br>
> + const struct pingpong_dest *my_dest, struct user_parameters *user_parm)<br>> {<br>> struct pingpong_dest *rem_dest = NULL;<br>> - char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];<br>
> + char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];<br>> int parsed;<br>><br>> - sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,<br>
> - my_dest->psn,my_dest->rkey,my_dest->vaddr);<br>> + sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",<br>
> + my_dest->lid, my_dest->qpn, my_dest->psn,my_dest->rkey,my_dest->vaddr,<br>> + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],<br>> + my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],<br>
> + my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],<br>> + my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],<br>> + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],<br>
> + my_dest->dgid.raw[15]);<br>> if (write(sockfd, msg, sizeof msg) != sizeof msg) {<br>> perror("client write");<br>> fprintf(stderr, "Couldn't send local address\n");<br>
> @@ -174,15 +183,59 @@ struct pingpong_dest * pp_client_exch_dest(int sockfd,<br>> if (!rem_dest)<br>> goto out;<br>><br>> - parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>
> - &rem_dest->psn,&rem_dest->rkey,&rem_dest->vaddr);<br>> -<br>> - if (parsed != 5) {<br>> - fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,<br>
> - msg);<br>> - free(rem_dest);<br>> - rem_dest = NULL;<br>> - goto out;<br>> + if (user_parm->gid_index < 0) {<br>> + parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>
> + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>> + if (parsed != 5) {<br>> + fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);<br>
> + free(rem_dest);<br>> + rem_dest = NULL;<br>> + goto out;<br>> + }<br>> + }else{<br>> + char *pstr = msg, *term;<br>
> + char tmp[20];<br>> + int i;<br>> +<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>
> + tmp[term - pstr] = 0;<br>> + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN<br>> +<br>> + pstr += term - pstr + 1;<br>
> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY<br>
> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA<br>> +<br>> + for (i = 0; i < 15; ++i) {<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);<br>> + }<br>
> + pstr += term - pstr + 1;<br>> + strcpy(tmp, pstr);<br>> + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);<br>> }<br>> out:<br>> return rem_dest;<br>
> @@ -244,9 +297,9 @@ int pp_server_connect(int port)<br>> return connfd;<br>> }<br>><br>> -static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest)<br>> +static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm)<br>
> {<br>> - char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];<br>> + char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];<br>
> struct pingpong_dest *rem_dest = NULL;<br>> int parsed;<br>> int n;<br>> @@ -262,18 +315,69 @@ static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpo<br>> if (!rem_dest)<br>
> goto out;<br>><br>> - parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>> - &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>
> - if (parsed != 5) {<br>> - fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,<br>> - msg);<br>> - free(rem_dest);<br>
> - rem_dest = NULL;<br>> - goto out;<br>> + if (user_parm->gid_index < 0) {<br>> + parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>
> + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>> + if (parsed != 5) {<br>> + fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);<br>
> + free(rem_dest);<br>> + rem_dest = NULL;<br>> + goto out;<br>> + }<br>> + }else{<br>> + char *pstr = msg, *term;<br>
> + char tmp[20];<br>> + int i;<br>> +<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>
> + tmp[term - pstr] = 0;<br>> + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN<br>> +<br>> + pstr += term - pstr + 1;<br>
> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY<br>
> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA<br>> +<br>> + for (i = 0; i < 15; ++i) {<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);<br>> + }<br>
> + pstr += term - pstr + 1;<br>> + strcpy(tmp, pstr);<br>> + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);<br>> }<br>><br>> - sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,<br>
> - my_dest->psn, my_dest->rkey, my_dest->vaddr);<br>> + sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",<br>
> + my_dest->lid, my_dest->qpn, my_dest->psn, my_dest->rkey, my_dest->vaddr,<br>> + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],<br>> + my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],<br>
> + my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],<br>> + my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],<br>> + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],<br>
> + my_dest->dgid.raw[15]);<br>> if (write(connfd, msg, sizeof msg) != sizeof msg) {<br>> perror("server write");<br>> fprintf(stderr, "Couldn't send local address\n");<br>
> @@ -318,7 +422,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev,<br>> fprintf(stderr, "Failed to query device props");<br>> return NULL;<br>
> }<br>> - if (device_attr.vendor_part_id == 23108)<br>> + if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1)<br>> user_parm->mtu = 1024;<br>
> else<br>> user_parm->mtu = 2048;<br>> @@ -421,9 +525,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,<br>> attr.rq_psn = dest->psn;<br>
> attr.max_dest_rd_atomic = user_parm->max_out_read;<br>> attr.min_rnr_timer = 12;<br>> - attr.ah_attr.is_global = 0;<br>> - attr.ah_attr.dlid = dest->lid;<br>
> - <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = sl;<br>> + if (user_parm->gid_index<0) {<br>> + attr.ah_attr.is_global = 0;<br>> + attr.ah_attr.dlid = dest->lid;<br>
> + <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = sl;<br>> + } else {<br>> + attr.ah_attr.is_global = 1;<br>> + attr.ah_attr.grh.dgid = dest->dgid;<br>
> + attr.ah_attr.grh.hop_limit = 1;<br>> + <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = 0;<br>> + }<br>> attr.ah_attr.src_path_bits = 0;<br>
> attr.ah_attr.port_num = port;<br>> if (ibv_modify_qp(ctx->qp, &attr,<br>> @@ -474,6 +585,7 @@ static void usage(const char *argv0)<br>> printf(" -n, --iters=<iters> number of exchanges (at least 2, default 1000)\n");<br>
> printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n");<br>> printf(" -S, --sl=<sl> SL (default 0)\n");<br>> + printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n");<br>
> printf(" -b, --bidirectional measure bidirectional bandwidth (default unidirectional)\n");<br>> printf(" -V, --version display version number\n");<br>> printf(" -e, --events sleep on CQ events (default poll)\n");<br>
> @@ -609,6 +721,7 @@ int main(int argc, char *argv[])<br>> int duplex = 0;<br>> int i = 0;<br>> int no_cpu_freq_fail = 0;<br>
> + union ibv_gid gid;<br>><br>> /* init default values to user's parameters */<br>> memset(&user_param, 0, sizeof(struct user_parameters));<br>> @@ -619,6 +732,7 @@ int main(int argc, char *argv[])<br>
> user_param.use_event = 0;<br>> user_param.max_out_read = 4; /* the device capability on gen2 */<br>> user_param.qp_timeout = 14;<br>> + user_param.gid_index = -1; /*gid will not be used*/<br>
> /* Parameter parsing. */<br>> while (1) {<br>> int c;<br>> @@ -634,6 +748,7 @@ int main(int argc, char *argv[])<br>> { .name = "tx-depth", .has_arg = 1, .val = 't' },<br>
> { .name = "qp-timeout", .has_arg = 1, .val = 'u' },<br>> { .name = "sl", .has_arg = 1, .val = 'S' },<br>> + { .name = "gid-index", .has_arg = 1, .val = 'x' },<br>
> { .name = "all", .has_arg = 0, .val = 'a' },<br>> { .name = "bidirectional", .has_arg = 0, .val = 'b' },<br>> { .name = "version", .has_arg = 0, .val = 'V' },<br>
> @@ -642,7 +757,7 @@ int main(int argc, char *argv[])<br>> { 0 }<br>> };<br>><br>> - c = getopt_long(argc, argv, "p:d:i:m:o:s:n:t:u:S:abVeF", long_options, NULL);<br>
> + c = getopt_long(argc, argv, "p:d:i:m:o:s:n:t:u:S:x:abVeF", long_options, NULL);<br>> if (c == -1)<br>> break;<br>><br>> @@ -721,6 +836,14 @@ int main(int argc, char *argv[])<br>
> if (sl > 15) { usage(argv[0]); return 1; }<br>> break;<br>><br>> + case 'x':<br>> + user_param.gid_index = strtol(optarg, NULL, 0);<br>
> + if (user_param.gid_index > 63) {<br>> + usage(argv[0]);<br>> + return 1;<br>> + }<br>> + break;<br>
> +<br>> default:<br>> usage(argv[0]);<br>> return 1;<br>> @@ -740,6 +863,10 @@ int main(int argc, char *argv[])<br>> printf(" RDMA_Read BW Test\n");<br>
><br>> printf("Connection type : RC\n");<br>> + if (user_param.gid_index > -1) {<br>> + printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n");<br>
> + }<br>> +<br>> /* Done with parameter parsing. Perform setup. */<br>> if (user_param.all == ALL)<br>> /*since we run all sizes */<br>> @@ -771,16 +898,28 @@ int main(int argc, char *argv[])<br>
> if (!ctx)<br>> return 1;<br>><br>> + if (user_param.gid_index != -1) {<br>> + int err=0;<br>> + err = ibv_query_gid (ctx->context, ib_port, user_param.gid_index, &gid);<br>
> + if (err) {<br>> + return -1;<br>> + }<br>> + ctx->dgid=gid;<br>> + }<br>> +<br>> /* Create connection between client and server.<br>
> * We do it by exchanging data over a TCP socket connection. */<br>><br>> my_dest.lid = pp_get_local_lid(ctx, ib_port);<br>> my_dest.qpn = ctx->qp->qp_num;<br>> my_dest.psn = lrand48() & 0xffffff;<br>
> - if (!my_dest.lid) {<br>> - fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");<br>> - return 1;<br>> + if (user_param.gid_index < 0) {/*We do not fail test upon lid in RDMA0E/Eth conf*/<br>
> + if (!my_dest.lid) {<br>> + fprintf(stderr, "Local lid 0x0 detected. Is an SM running? If you are running on an RMDAoE interface you must use GIDs\n");<br>
> + return 1;<br>> + }<br>> }<br>> + my_dest.dgid = gid;<br>> my_dest.rkey = ctx->mr->rkey;<br>> my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;<br>
><br>> @@ -788,17 +927,26 @@ int main(int argc, char *argv[])<br>> "RKey %#08x VAddr %#016Lx\n",<br>> my_dest.lid, my_dest.qpn, my_dest.psn,<br>> my_dest.rkey, my_dest.vaddr);<br>
> + if (user_param.gid_index > -1) {<br>> + printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",<br>> + my_dest.dgid.raw[0],my_dest.dgid.raw[1],<br>
> + my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4],<br>> + my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7],<br>> + my_dest.dgid.raw[8], my_dest.dgid.raw[9], my_dest.dgid.raw[10],<br>
> + my_dest.dgid.raw[11], my_dest.dgid.raw[12], my_dest.dgid.raw[13],<br>> + my_dest.dgid.raw[14], my_dest.dgid.raw[15]);<br>> + }<br>><br>> if (user_param.servername) {<br>
> sockfd = pp_client_connect(user_param.servername, port);<br>> if (sockfd < 0)<br>> return 1;<br>> - rem_dest = pp_client_exch_dest(sockfd, &my_dest);<br>
> + rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);<br>> } else {<br>> sockfd = pp_server_connect(port);<br>> if (sockfd < 0)<br>> return 1;<br>
> - rem_dest = pp_server_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);<br>> }<br>><br>> if (!rem_dest)<br>
> @@ -808,6 +956,15 @@ int main(int argc, char *argv[])<br>> "RKey %#08x VAddr %#016Lx\n",<br>> rem_dest->lid, rem_dest->qpn, rem_dest->psn,<br>> rem_dest->rkey, rem_dest->vaddr);<br>
> + if (user_param.gid_index > -1) {<br>> + printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",<br>> + rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],<br>
> + rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4],<br>> + rem_dest->dgid.raw[5], rem_dest->dgid.raw[6], rem_dest->dgid.raw[7],<br>> + rem_dest->dgid.raw[8], rem_dest->dgid.raw[9], rem_dest->dgid.raw[10],<br>
> + rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13],<br>> + rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);<br>> + }<br>><br>> if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest, &user_param))<br>
> return 1;<br>> @@ -815,9 +972,9 @@ int main(int argc, char *argv[])<br>> /* An additional handshake is required *after* moving qp to RTR.<br>> Arbitrarily reuse exch_dest for this purpose. */<br>
> if (user_param.servername)<br>> - rem_dest = pp_client_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);<br>> else<br>
> - rem_dest = pp_server_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);<br>><br>> if (!rem_dest)<br>> return 1;<br>
> @@ -826,7 +983,7 @@ int main(int argc, char *argv[])<br>> /* For half duplex tests, server just waits for client to exit */<br>><br>> if (!user_param.servername && !duplex) {<br>> - rem_dest = pp_server_exch_dest(sockfd, &my_dest);<br>
> + rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);<br>> if (write(sockfd, "done", sizeof "done") != sizeof "done"){<br>> perror("server write");<br>
> fprintf(stderr, "Couldn't write to socket\n");<br>> @@ -873,9 +1030,9 @@ int main(int argc, char *argv[])<br>> }<br>><br>> if (user_param.servername)<br>
> - rem_dest = pp_client_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);<br>> else<br>> - rem_dest = pp_server_exch_dest(sockfd, &my_dest);<br>
> + rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);<br>><br>> if (write(sockfd, "done", sizeof "done") != sizeof "done"){<br>> perror("server write");<br>
><br>> --- a/read_lat.c<br>> +++ b/read_lat.c<br>> @@ -75,6 +75,7 @@ struct user_parameters {<br>> int max_out_read;<br>> int use_event;<br>> int qp_timeout;<br>> + int gid_index; /* if value not negative, we use gid AND gid_index=value */<br>
><br>> };<br>> struct report_options {<br>> @@ -97,6 +98,7 @@ struct pingpong_context {<br>> int tx_depth;<br>> struct ibv_sge list;<br>> struct ibv_send_wr wr;<br>> + union ibv_gid dgid;<br>
> };<br>><br>> struct pingpong_dest {<br>> @@ -105,6 +107,7 @@ struct pingpong_dest {<br>> int psn;<br>> unsigned rkey;<br>> unsigned long long vaddr;<br>> + union ibv_gid dgid;<br>
> };<br>><br>><br>> @@ -140,14 +143,33 @@ static struct ibv_device *pp_find_dev(const char *ib_devname) {<br>><br>> #define KEY_MSG_SIZE (sizeof "0000:000000:000000:00000000:0000000000000000")<br>
> #define KEY_PRINT_FMT "%04x:%06x:%06x:%08x:%016Lx"<br>> +#define KEY_MSG_SIZE_GID (sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00")<br>> +#define KEY_PRINT_FMT_GID "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x"<br>
><br>> -static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest)<br>> +static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm)<br>> {<br>> - char msg[KEY_MSG_SIZE];<br>
> + if (user_parm->gid_index < 0) {<br>> + char msg[KEY_MSG_SIZE];<br>><br>> - sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,<br>> - my_dest->psn, my_dest->rkey, my_dest->vaddr);<br>
> + sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,<br>> + my_dest->psn, my_dest->rkey, my_dest->vaddr);<br>><br>> + if (write(sockfd, msg, sizeof msg) != sizeof msg) {<br>
> + perror("client write");<br>> + fprintf(stderr, "Couldn't send local address\n");<br>> + return -1;<br>> + }<br>
> +<br>> + return 0;<br>> + } else {<br>> + char msg[KEY_MSG_SIZE_GID];<br>> +<br>> + sprintf(msg, KEY_PRINT_FMT_GID, my_dest->lid, my_dest->qpn,<br>> + my_dest->psn, my_dest->rkey, my_dest->vaddr,<br>
> + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2], my_dest->dgid.raw[3],<br>> + my_dest->dgid.raw[4], my_dest->dgid.raw[5], my_dest->dgid.raw[6], my_dest->dgid.raw[7],<br>
> + my_dest->dgid.raw[8], my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],<br>> + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14], my_dest->dgid.raw[15]);<br>
> if (write(sockfd, msg, sizeof msg) != sizeof msg) {<br>> perror("client write");<br>> fprintf(stderr, "Couldn't send local address\n");<br>> @@ -155,30 +177,84 @@ static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest)<br>
> }<br>><br>> return 0;<br>> + }<br>> }<br>><br>> static int pp_read_keys(int sockfd, const struct pingpong_dest *my_dest,<br>> - struct pingpong_dest *rem_dest)<br>
> + struct pingpong_dest *rem_dest, struct user_parameters *user_parm)<br>> {<br>> - int parsed;<br>> - char msg[KEY_MSG_SIZE];<br>> + if (user_parm->gid_index < 0) {<br>
> + int parsed;<br>> + char msg[KEY_MSG_SIZE];<br>> +<br>> + if (read(sockfd, msg, sizeof msg) != sizeof msg) {<br>> + perror("pp_read_keys");<br>
> + fprintf(stderr, "Couldn't read remote address\n");<br>> + return -1;<br>> + }<br>><br>> - if (read(sockfd, msg, sizeof msg) != sizeof msg) {<br>
> - perror("pp_read_keys");<br>> - fprintf(stderr, "Couldn't read remote address\n");<br>> - return -1;<br>> - }<br>> + parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn,<br>
> + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>><br>> - parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn,<br>> - &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>
> + if (parsed != 5) {<br>> + fprintf(stderr, "Couldn't parse line <%.*s>\n",<br>> + (int)sizeof msg, msg);<br>> + return -1;<br>
> + }<br>><br>> - if (parsed != 5) {<br>> - fprintf(stderr, "Couldn't parse line <%.*s>\n",<br>> - (int)sizeof msg, msg);<br>> - return -1;<br>
> + return 0;<br>> + } else {<br>> + char msg[KEY_MSG_SIZE_GID];<br>> + if (read(sockfd, msg, sizeof msg) != sizeof msg) {<br>> + perror("pp_read_keys");<br>
> + fprintf(stderr, "Couldn't read remote address\n");<br>> + return -1;<br>> + }<br>> + char *pstr = msg, *term;<br>> + char tmp[20];<br>
> + int i;<br>> +<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID<br>
> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>
> + tmp[term - pstr] = 0;<br>> + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY<br>> +<br>> + pstr += term - pstr + 1;<br>
> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA<br>
> +<br>> + for (i = 0; i < 15; ++i) {<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>
> + tmp[term - pstr] = 0;<br>> + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);<br>> + }<br>> + pstr += term - pstr + 1;<br>
> + strcpy(tmp, pstr);<br>> + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);<br>> + return 0;<br>> }<br>> -<br>> - return 0;<br>
> }<br>><br>> static int pp_client_connect(const char *servername, int port)<br>> @@ -223,12 +299,12 @@ static int pp_client_connect(const char *servername, int port)<br>> }<br>><br>> static int pp_client_exch_dest(int sockfd, const struct pingpong_dest *my_dest,<br>
> - struct pingpong_dest *rem_dest)<br>> + struct pingpong_dest *rem_dest, struct user_parameters *user_parm)<br>> {<br>> - if (pp_write_keys(sockfd, my_dest))<br>
> + if (pp_write_keys(sockfd, my_dest, user_parm))<br>> return -1;<br>><br>> - return pp_read_keys(sockfd, my_dest, rem_dest);<br>> + return pp_read_keys(sockfd, my_dest, rem_dest, user_parm);<br>
> }<br>><br>> static int pp_server_connect(int port)<br>> @@ -288,13 +364,13 @@ static int pp_server_connect(int port)<br>> }<br>><br>> static int pp_server_exch_dest(int sockfd, const struct pingpong_dest *my_dest,<br>
> - struct pingpong_dest* rem_dest)<br>> + struct pingpong_dest* rem_dest, struct user_parameters *user_parm)<br>> {<br>><br>> - if (pp_read_keys(sockfd, my_dest, rem_dest))<br>
> + if (pp_read_keys(sockfd, my_dest, rem_dest, user_parm))<br>> return -1;<br>><br>> - return pp_write_keys(sockfd, my_dest);<br>> + return pp_write_keys(sockfd, my_dest, user_parm);<br>
> }<br>><br>> static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,<br>> @@ -332,7 +408,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,<br>> fprintf(stderr, "Failed to query device props");<br>
> return NULL;<br>> }<br>> - if (device_attr.vendor_part_id == 23108) {<br>> + if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) {<br>
> user_parm->mtu = 1024;<br>> } else {<br>> user_parm->mtu = 2048;<br>> @@ -444,9 +520,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,<br>
> attr.rq_psn = dest->psn;<br>> attr.max_dest_rd_atomic = user_parm->max_out_read;<br>> attr.min_rnr_timer = 12;<br>> - attr.ah_attr.is_global = 0;<br>
> - attr.ah_attr.dlid = dest->lid;<br>> - <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = sl;<br>> + if (user_parm->gid_index < 0) {<br>> + attr.ah_attr.is_global = 0;<br>
> + attr.ah_attr.dlid = dest->lid;<br>> + <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = sl;<br>> + } else {<br>> + attr.ah_attr.is_global = 1;<br>
> + attr.ah_attr.grh.dgid = dest->dgid;<br>> + attr.ah_attr.grh.hop_limit = 1;<br>> + <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = 0;<br>
> + }<br>> attr.ah_attr.src_path_bits = 0;<br>> attr.ah_attr.port_num = port;<br>> if (ibv_modify_qp(ctx->qp, &attr,<br>> @@ -497,23 +580,45 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,<br>
> char addr_fmt[] = "%8s address: LID %#04x QPN %#06x PSN %#06x RKey %#08x VAddr %#016Lx\n";<br>> int sockfd;<br>> int rc;<br>> + union ibv_gid gid;<br>
><br>><br>> /* Create connection between client and server.<br>> * We do it by exchanging data over a TCP socket connection. */<br>><br>> + if (user_parm->gid_index != -1) {<br>> + int err=0;<br>
> + err = ibv_query_gid (ctx->context, ib_port, user_parm->gid_index, &gid);<br>> + if (err) {<br>> + return -1;<br>> + }<br>> + ctx->dgid=gid;<br>
> + }<br>> +<br>> my_dest.lid = pp_get_local_lid(ctx, ib_port);<br>> + my_dest.dgid = gid;<br>> my_dest.qpn = ctx->qp->qp_num;<br>> my_dest.psn = lrand48() & 0xffffff;<br>
> - if (!my_dest.lid) {<br>> - fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");<br>> - return -1;<br>> + if (user_parm->gid_index < 0) {/*We do not fail test upon lid in RDMAoE/Eth conf*/<br>
> + if (!my_dest.lid) {<br>> + fprintf(stderr, "Local lid 0x0 detected. Is an SM running? If you are running on an RMDAoE interface you must use GIDs\n");<br>
> + return 1;<br>> + }<br>> }<br>> my_dest.rkey = ctx->mr->rkey;<br>> my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;<br>><br>> printf(addr_fmt, "local", my_dest.lid, my_dest.qpn, my_dest.psn,<br>
> my_dest.rkey, my_dest.vaddr);<br>> + if (user_parm->gid_index > -1) {<br>> + printf(" GID: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",<br>
> + my_dest.dgid.raw[0],my_dest.dgid.raw[1],<br>> + my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4],<br>> + my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7],<br>
> + my_dest.dgid.raw[8], my_dest.dgid.raw[9], my_dest.dgid.raw[10],<br>> + my_dest.dgid.raw[11], my_dest.dgid.raw[12], my_dest.dgid.raw[13],<br>> + my_dest.dgid.raw[14], my_dest.dgid.raw[15]);<br>
> + }<br>><br>> sockfd = servername ? pp_client_connect(servername, port) :<br>> pp_server_connect(port);<br>> @@ -524,13 +629,22 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,<br>
> return sockfd;<br>> }<br>><br>> - rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest) :<br>> - pp_server_exch_dest(sockfd, &my_dest, rem_dest);<br>
> + rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest, user_parm) :<br>> + pp_server_exch_dest(sockfd, &my_dest, rem_dest, user_parm);<br>> if (rc)<br>> return rc;<br>
><br>> printf(addr_fmt, "remote", rem_dest->lid, rem_dest->qpn, rem_dest->psn,<br>> rem_dest->rkey, rem_dest->vaddr);<br>> + if (user_parm->gid_index > -1) {<br>
> + printf(" GID: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",<br>> + rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],<br>
> + rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4],<br>> + rem_dest->dgid.raw[5], rem_dest->dgid.raw[6], rem_dest->dgid.raw[7],<br>> + rem_dest->dgid.raw[8], rem_dest->dgid.raw[9], rem_dest->dgid.raw[10],<br>
> + rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13],<br>> + rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);<br>> + }<br>><br>> if ((rc = pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest,user_parm)))<br>
> return rc;<br>> @@ -539,8 +653,8 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,<br>> * Arbitrarily reuse exch_dest for this purpose.<br>> */<br>
><br>> - rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest) :<br>> - pp_server_exch_dest(sockfd, &my_dest, rem_dest);<br>> + rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest, user_parm) :<br>
> + pp_server_exch_dest(sockfd, &my_dest, rem_dest, user_parm);<br>><br>> if (rc)<br>> return rc;<br>> @@ -565,6 +679,8 @@ static void usage(const char *argv0)<br>> printf(" -n, --iters=<iters> number of exchanges (at least 2, default 1000)\n");<br>
> printf(" -o, --outs=<num> num of outstanding read/atom(default 4)\n");<br>> printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n");<br>
> + printf(" -S, --sl=<sl> SL (default 0)\n");<br>> + printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n");<br>
> printf(" -a, --all Run sizes from 2 till 2^23\n");<br>> printf(" -C, --report-cycles report times in cpu cycle units (default microseconds)\n");<br>> printf(" -H, --report-histogram print out all results (default print summary only)\n");<br>
> @@ -754,6 +870,7 @@ int main(int argc, char *argv[])<br>> user_param.use_event = 0;<br>> user_param.max_out_read = 4; /* the device capability on gen2 */<br>> user_param.qp_timeout = 14;<br>
> + user_param.gid_index = -1; /*gid will not be used*/<br>> /* Parameter parsing. */<br>> while (1) {<br>> int c;<br>> @@ -770,6 +887,7 @@ int main(int argc, char *argv[])<br>
> { .name = "tx-depth", .has_arg = 1, .val = 't' },<br>> { .name = "qp-timeout", .has_arg = 1, .val = 'u' },<br>> { .name = "sl", .has_arg = 1, .val = 'S' },<br>
> + { .name = "gid-index", .has_arg = 1, .val = 'x' },<br>> { .name = "all", .has_arg = 0, .val = 'a' },<br>> { .name = "report-cycles", .has_arg = 0, .val = 'C' },<br>
> { .name = "report-histogram",.has_arg = 0, .val = 'H' },<br>> @@ -780,7 +898,7 @@ int main(int argc, char *argv[])<br>> { 0 }<br>> };<br>
><br>> - c = getopt_long(argc, argv, "p:c:m:d:i:s:o:n:t:u:S:aeHUVF", long_options, NULL);<br>> + c = getopt_long(argc, argv, "p:c:m:d:i:s:o:n:t:u:S:x:aeHUVF", long_options, NULL);<br>
> if (c == -1)<br>> break;<br>><br>> @@ -874,6 +992,14 @@ int main(int argc, char *argv[])<br>> if (sl > 15) { usage(argv[0]); return 5; }<br>
> break;<br>><br>> + case 'x':<br>> + user_param.gid_index = strtol(optarg, NULL, 0);<br>> + if (user_param.gid_index > 63) {<br>
> + usage(argv[0]);<br>> + return 1;<br>> + }<br>> + break;<br>> +<br>> default:<br>
> usage(argv[0]);<br>> return 6;<br>> @@ -899,6 +1025,9 @@ int main(int argc, char *argv[])<br>> printf(" RDMA_Read Latency Test\n");<br>
> printf("Connection type : RC\n");<br>> /* anyway make sure the connection is RC */<br>> + if (user_param.gid_index > -1) {<br>> + printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n");<br>
> + }<br>> tmp_size = size;<br>> if (user_param.all == ALL) {<br>> /*since we run all sizes */<br>> @@ -956,7 +1085,7 @@ int main(int argc, char *argv[])<br>> /* done close sockets */<br>
> if(user_param.servername) {<br>> /*Signal client is finished */<br>> - pp_client_exch_dest(user_param.sockfd, &my_dest, &rem_dest);<br>> + pp_client_exch_dest(user_param.sockfd, &my_dest, &rem_dest, &user_param);<br>
> if (write(user_param.sockfd, "done", sizeof "done") != sizeof "done"){<br>> perror("client write");<br>> fprintf(stderr, "Couldn't write to socket\n");<br>
> @@ -965,7 +1094,7 @@ int main(int argc, char *argv[])<br>> close(user_param.sockfd);<br>> } else {<br>> /*Server is finished wait for client */<br>> - pp_server_exch_dest(user_param.sockfd, &my_dest, &rem_dest);<br>
> + pp_server_exch_dest(user_param.sockfd, &my_dest, &rem_dest, &user_param);<br>> if (write(user_param.sockfd, "done", sizeof "done") != sizeof "done"){<br>
> perror("server write");<br>> fprintf(stderr, "Couldn't write to socket\n");<br>> --- a/send_bw.c<br>> +++ b/send_bw.c<br>> @@ -84,6 +84,7 @@ struct user_parameters {<br>
> int use_mcg;<br>> int inline_size;<br>> int qp_timeout;<br>> + int gid_index; /* if value not negative, we use gid AND gid_index=value */<br>> };<br>> static int sl = 0;<br>
> static int page_size;<br>> @@ -106,6 +107,7 @@ struct pingpong_context {<br>> struct ibv_send_wr wr;<br>> struct ibv_recv_wr rwr;<br>> struct ibv_ah *ah;<br>> + union ibv_gid dgid;<br>
> };<br>><br>> struct pingpong_dest {<br>> @@ -114,6 +116,7 @@ struct pingpong_dest {<br>> int psn;<br>> unsigned rkey;<br>> unsigned long long vaddr;<br>> + union ibv_gid dgid;<br>
> };<br>><br>> static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)<br>> @@ -167,14 +170,20 @@ static int pp_client_connect(const char *servername, int port)<br>> }<br>><br>> struct pingpong_dest * pp_client_exch_dest(int sockfd,<br>
> - const struct pingpong_dest *my_dest)<br>> + const struct pingpong_dest *my_dest, struct user_parameters *user_parm)<br>> {<br>
> struct pingpong_dest *rem_dest = NULL;<br>> - char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];<br>> + char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];<br>
> int parsed;<br>><br>> - sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,<br>> - my_dest->psn,my_dest->rkey,my_dest->vaddr);<br>> + sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",<br>
> + my_dest->lid, my_dest->qpn, my_dest->psn,my_dest->rkey,my_dest->vaddr,<br>> + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],<br>> + my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],<br>
> + my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],<br>> + my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],<br>> + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],<br>
> + my_dest->dgid.raw[15]);<br>> if (write(sockfd, msg, sizeof msg) != sizeof msg) {<br>> perror("client write");<br>> fprintf(stderr, "Couldn't send local address\n");<br>
> @@ -191,15 +200,59 @@ struct pingpong_dest * pp_client_exch_dest(int sockfd,<br>> if (!rem_dest)<br>> goto out;<br>><br>> - parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>
> - &rem_dest->psn,&rem_dest->rkey,&rem_dest->vaddr);<br>> + if (user_parm->gid_index < 0) {<br>> + parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>
> + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>> + if (parsed != 5) {<br>> + fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);<br>
> + free(rem_dest);<br>> + rem_dest = NULL;<br>> + goto out;<br>> + }<br>> + }else{<br>> + char *pstr = msg, *term;<br>
> + char tmp[20];<br>> + int i;<br>><br>> - if (parsed != 5) {<br>> - fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,<br>> - msg);<br>
> - free(rem_dest);<br>> - rem_dest = NULL;<br>> - goto out;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>
> + tmp[term - pstr] = 0;<br>> + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN<br>> +<br>> + pstr += term - pstr + 1;<br>
> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN<br>
> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>
> + tmp[term - pstr] = 0;<br>> + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA<br>> +<br>> + for (i = 0; i < 15; ++i) {<br>> + pstr += term - pstr + 1;<br>
> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);<br>
> + }<br>> + pstr += term - pstr + 1;<br>> + strcpy(tmp, pstr);<br>> + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);<br>> }<br>
> out:<br>> return rem_dest;<br>> @@ -261,9 +314,9 @@ int pp_server_connect(int port)<br>> return connfd;<br>> }<br>><br>> -static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest)<br>
> +static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm)<br>> {<br>> - char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];<br>
> + char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];<br>> struct pingpong_dest *rem_dest = NULL;<br>> int parsed;<br>> int n;<br>
> @@ -279,18 +332,69 @@ static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpo<br>> if (!rem_dest)<br>> goto out;<br>><br>> - parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>
> - &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>> - if (parsed != 5) {<br>> - fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,<br>
> - msg);<br>> - free(rem_dest);<br>> - rem_dest = NULL;<br>> - goto out;<br>> + if (user_parm->gid_index < 0) {<br>> + parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>
> + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>> + if (parsed != 5) {<br>> + fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);<br>
> + free(rem_dest);<br>> + rem_dest = NULL;<br>> + goto out;<br>> + }<br>> + }else{<br>> + char *pstr = msg, *term;<br>
> + char tmp[20];<br>> + int i;<br>> +<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>
> + tmp[term - pstr] = 0;<br>> + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN<br>> +<br>> + pstr += term - pstr + 1;<br>
> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY<br>
> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA<br>> +<br>> + for (i = 0; i < 15; ++i) {<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);<br>> + }<br>
> + pstr += term - pstr + 1;<br>> + strcpy(tmp, pstr);<br>> + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);<br>> }<br>><br>> - sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,<br>
> - my_dest->psn, my_dest->rkey, my_dest->vaddr);<br>> + sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",<br>
> + my_dest->lid, my_dest->qpn, my_dest->psn, my_dest->rkey, my_dest->vaddr,<br>> + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],<br>> + my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],<br>
> + my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],<br>> + my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],<br>> + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],<br>
> + my_dest->dgid.raw[15]);<br>> if (write(connfd, msg, sizeof msg) != sizeof msg) {<br>> perror("server write");<br>> fprintf(stderr, "Couldn't send local address\n");<br>
> @@ -346,7 +450,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev,<br>> fprintf(stderr, "Failed to query device props");<br>> return NULL;<br>
> }<br>> - if (device_attr.vendor_part_id == 23108) {<br>> + if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) {<br>> user_parm->mtu = 1024;<br>
> } else {<br>> user_parm->mtu = 2048;<br>> @@ -504,9 +608,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,<br>> attr.max_dest_rd_atomic = 1;<br>
> attr.min_rnr_timer = 12;<br>> }<br>> - attr.ah_attr.is_global = 0;<br>> - attr.ah_attr.dlid = dest->lid;<br>> - <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = sl;<br>
> + if (user_parm->gid_index < 0) {<br>> + attr.ah_attr.is_global = 0;<br>> + attr.ah_attr.dlid = dest->lid;<br>> + <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = sl;<br>
> + } else {<br>> + attr.ah_attr.is_global = 1;<br>> + attr.ah_attr.grh.dgid = dest->dgid;<br>> + attr.ah_attr.grh.hop_limit = 1;<br>> + <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = 0;<br>
> + }<br>> attr.ah_attr.src_path_bits = 0;<br>> attr.ah_attr.port_num = port;<br>> if ((user_parm->connection_type==UD) && (user_parm->use_mcg)) {<br>> @@ -518,9 +629,6 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,<br>
> attr.ah_attr.is_global = 1;<br>> attr.ah_attr.grh.sgid_index = 0;<br>> memcpy(attr.ah_attr.grh.dgid.raw, mcg_gid, 16);<br>> - } else {<br>> - attr.ah_attr.dlid = dest->lid;<br>
> - attr.ah_attr.is_global = 0;<br>> }<br>> if (user_parm->connection_type == RC) {<br>> if (ibv_modify_qp(ctx->qp, &attr,<br>> @@ -632,6 +740,7 @@ static void usage(const char *argv0)<br>
> printf(" -I, --inline_size=<size> max size of message to be sent in inline mode (default 400)\n");<br>> printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n");<br>
> printf(" -S, --sl=<sl> SL (default 0)\n");<br>> + printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n");<br>
> printf(" -b, --bidirectional measure bidirectional bandwidth (default unidirectional)\n");<br>> printf(" -V, --version display version number\n");<br>> printf(" -e, --events sleep on CQ events (default poll)\n");<br>
> @@ -680,8 +789,13 @@ int run_iter_bi(struct pingpong_context *ctx, struct user_parameters *user_param<br>> int scnt, ccnt, rcnt;<br>> struct ibv_recv_wr *bad_wr_recv;<br>
> if (user_param->connection_type == UD) {<br>> - if (size > 2048)<br>> - size = 2048;<br>> + if (size > 2048) {<br>> + if (user_param->gid_index < 0) {<br>
> + size = 2048;<br>> + } else {<br>> + size = 1024;<br>> + }<br>> + }<br>> }<br>
> /*********************************************<br>> * Important note :<br>> @@ -799,8 +913,13 @@ int run_iter_uni(struct pingpong_context *ctx, struct user_parameters *user_para<br>> struct ibv_recv_wr *bad_wr_recv;<br>
><br>> if (user_param->connection_type == UD) {<br>> - if (size > 2048)<br>> - size = 2048;<br>> + if (size > 2048) {<br>> + if (user_param->gid_index < 0) {<br>
> + size = 2048;<br>> + } else {<br>> + size = 1024;<br>> + }<br>> + }<br>> }<br>
><br>> if (user_param->connection_type == UD)<br>> @@ -947,6 +1066,7 @@ int main(int argc, char *argv[])<br>> int inline_given_in_cmd = 0;<br>> struct ibv_context *context;<br>
> int no_cpu_freq_fail = 0;<br>> + union ibv_gid gid;<br>> /* init default values to user's parameters */<br>> memset(&user_param, 0, sizeof(struct user_parameters));<br>
> user_param.mtu = 0;<br>> @@ -957,6 +1077,7 @@ int main(int argc, char *argv[])<br>> user_param.duplex = 0;<br>> user_param.inline_size = MAX_INLINE;<br>> user_param.qp_timeout = 14;<br>
> + user_param.gid_index = -1; /*gid will not be used*/<br>> /* Parameter parsing. */<br>> while (1) {<br>> int c;<br>> @@ -974,6 +1095,7 @@ int main(int argc, char *argv[])<br>
> { .name = "rx-depth", .has_arg = 1, .val = 'r' },<br>> { .name = "qp-timeout", .has_arg = 1, .val = 'u' },<br>> { .name = "sl", .has_arg = 1, .val = 'S' },<br>
> + { .name = "gid-index", .has_arg = 1, .val = 'x' },<br>> { .name = "all", .has_arg = 0, .val = 'a' },<br>> { .name = "bidirectional", .has_arg = 0, .val = 'b' },<br>
> { .name = "version", .has_arg = 0, .val = 'V' },<br>> @@ -984,7 +1106,7 @@ int main(int argc, char *argv[])<br>> { 0 }<br>> };<br>
><br>> - c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:I:r:u:S:ebaVgNF", long_options, NULL);<br>> + c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:I:r:u:S:x:ebaVgNF", long_options, NULL);<br>
> if (c == -1)<br>> break;<br>><br>> @@ -1038,6 +1160,14 @@ int main(int argc, char *argv[])<br>><br>> break;<br>><br>> + case 'x':<br>
> + user_param.gid_index = strtol(optarg, NULL, 0);<br>> + if (user_param.gid_index > 63) {<br>> + usage(argv[0]);<br>> + return 1;<br>
> + }<br>> + break;<br>> +<br>> case 't':<br>> user_param.tx_depth = strtol(optarg, NULL, 0);<br>> if (user_param.tx_depth < 1) { usage(argv[0]); return 1; }<br>
> @@ -1117,6 +1247,9 @@ int main(int argc, char *argv[])<br>> else{<br>> printf("Connection type : UD\n");<br>> }<br>> + if (user_param.gid_index > -1) {<br>> + printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n");<br>
> + }<br>><br>> /* Done with parameter parsing. Perform setup. */<br>> if (user_param.all == ALL)<br>> @@ -1126,6 +1259,10 @@ int main(int argc, char *argv[])<br>> printf("Max msg size in UD is 2048 changing to 2048\n");<br>
> size = 2048;<br>> }<br>> + if (user_param.connection_type == UD && user_param.gid_index > -1 && size > 1024) {<br>> + printf("Max msg size in UD RDMAoE is 1024. changing to 1024\n");<br>
> + size = 1024;<br>> + }<br>><br>> srand48(getpid() * time(NULL));<br>><br>> @@ -1174,25 +1311,46 @@ int main(int argc, char *argv[])<br>> my_dest.lid = pp_get_local_lid(ctx, ib_port);<br>
> my_dest.qpn = ctx->qp->qp_num;<br>> my_dest.psn = lrand48() & 0xffffff;<br>> - if (!my_dest.lid) {<br>> - fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");<br>
> - return 1;<br>> + if (user_param.gid_index != -1) {<br>> + int err=0;<br>> + err = ibv_query_gid (ctx->context, ib_port, user_param.gid_index, &gid);<br>
> + if (err) {<br>> + return -1;<br>> + }<br>> + ctx->dgid=gid;<br>> + }<br>> +<br>> + if (user_param.gid_index < 0) {/*We do not fail test upon lid in RDMA0E/Eth conf*/<br>
> + if (!my_dest.lid) {<br>> + fprintf(stderr, "Local lid 0x0 detected. Is an SM running? If you are running on an RMDAoE interface you must use GIDs\n");<br>
> + return 1;<br>> + }<br>> }<br>> + my_dest.dgid = gid;<br>> my_dest.rkey = ctx->mr->rkey;<br>> my_dest.vaddr = (uintptr_t)ctx->buf + size;<br>
> printf(" local address: LID %#04x, QPN %#06x, PSN %#06x\n",<br>> my_dest.lid, my_dest.qpn, my_dest.psn);<br>> + if (user_param.gid_index > -1) {<br>> + printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",<br>
> + my_dest.dgid.raw[0],my_dest.dgid.raw[1],<br>> + my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4],<br>> + my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7],<br>
> + my_dest.dgid.raw[8], my_dest.dgid.raw[9], my_dest.dgid.raw[10],<br>> + my_dest.dgid.raw[11], my_dest.dgid.raw[12], my_dest.dgid.raw[13],<br>> + my_dest.dgid.raw[14], my_dest.dgid.raw[15]);<br>
> + }<br>><br>> if (user_param.servername) {<br>> sockfd = pp_client_connect(user_param.servername, port);<br>> if (sockfd < 0)<br>> return 1;<br>
> - rem_dest = pp_client_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);<br>> } else {<br>> sockfd = pp_server_connect(port);<br>
> if (sockfd < 0)<br>> return 1;<br>> - rem_dest = pp_server_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);<br>
> }<br>><br>> if (!rem_dest)<br>> @@ -1200,6 +1358,15 @@ int main(int argc, char *argv[])<br>><br>> printf(" remote address: LID %#04x, QPN %#06x, PSN %#06x\n",<br>> rem_dest->lid, rem_dest->qpn, rem_dest->psn);<br>
> + if (user_param.gid_index > -1) {<br>> + printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",<br>> + rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],<br>
> + rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4],<br>> + rem_dest->dgid.raw[5], rem_dest->dgid.raw[6], rem_dest->dgid.raw[7],<br>> + rem_dest->dgid.raw[8], rem_dest->dgid.raw[9], rem_dest->dgid.raw[10],<br>
> + rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13],<br>> + rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);<br>> + }<br>><br>> if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest, &user_param))<br>
> return 1;<br>> @@ -1207,9 +1374,9 @@ int main(int argc, char *argv[])<br>> /* An additional handshake is required *after* moving qp to RTR.<br>> Arbitrarily reuse exch_dest for this purpose. */<br>
> if (user_param.servername) {<br>> - rem_dest = pp_client_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);<br>> } else {<br>
> - rem_dest = pp_server_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);<br>> }<br>> if (user_param.use_event) {<br>
> printf("Test with events.\n");<br>> @@ -1263,8 +1430,13 @@ int main(int argc, char *argv[])<br>> ctx->recv_list.lkey = ctx->mr->lkey;<br>><br>> if (user_param.all == ALL) {<br>
> - if (user_param.connection_type == UD)<br>> - size_max_pow = 12;<br>> + if (user_param.connection_type == UD) {<br>> + if (user_param.gid_index < 0) {<br>
> + size_max_pow = 12;<br>> + } else {<br>> + size_max_pow = 11;<br>> + }<br>> + }<br>><br>
> for (i = 1; i < size_max_pow ; ++i) {<br>> size = 1 << i;<br>> @@ -1278,9 +1450,9 @@ int main(int argc, char *argv[])<br>> if (user_param.servername) {<br>
> print_report(user_param.iters, size, user_param.duplex, tposted, tcompleted, noPeak, no_cpu_freq_fail);<br>> /* sync again for the sake of UC/UC */<br>
> - rem_dest = pp_client_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);<br>> } else<br>
> - rem_dest = pp_server_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);<br>> }<br>
> } else {<br>> if (user_param.duplex) {<br>> @@ -1298,9 +1470,9 @@ int main(int argc, char *argv[])<br>><br>> /* close sockets */<br>> if (user_param.servername)<br>> - rem_dest = pp_client_exch_dest(sockfd, &my_dest);<br>
> + rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);<br>> else<br>> - rem_dest = pp_server_exch_dest(sockfd, &my_dest);<br>> + rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);<br>
><br>> if (write(sockfd, "done", sizeof "done") != sizeof "done"){<br>> perror("write");<br>><br>> --- a/send_lat.c<br>> +++ b/send_lat.c<br>
> @@ -82,6 +82,7 @@ struct user_parameters {<br>> int inline_size;<br>> int use_mcg;<br>> int qp_timeout;<br>> + int gid_index; /* if value not negative, we use gid AND gid_index=value */<br>
> };<br>><br>> struct report_options {<br>> @@ -109,6 +110,7 @@ struct pingpong_context {<br>> volatile char *poll_buf;<br>> int size;<br>> int tx_depth;<br>
> + union ibv_gid dgid;<br>> };<br>><br>> struct pingpong_dest {<br>> @@ -117,6 +119,7 @@ struct pingpong_dest {<br>> int qpn;<br>> int psn;<br>> unsigned int rkey;<br>
> + union ibv_gid dgid;<br>> };<br>><br>><br>> @@ -152,13 +155,33 @@ static struct ibv_device *pp_find_dev(const char *ib_devname) {<br>><br>> #define KEY_MSG_SIZE (sizeof "0000:000000:000000:00000000:0000000000000000")<br>
> #define KEY_PRINT_FMT "%04x:%06x:%06x:%08x:%016Lx"<br>> +#define KEY_MSG_SIZE_GID (sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00")<br>> +#define KEY_PRINT_FMT_GID "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x"<br>
><br>> -static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest)<br>> +static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm)<br>> {<br>> - char msg[KEY_MSG_SIZE];<br>
> + if (user_parm->gid_index < 0) {<br>> + char msg[KEY_MSG_SIZE];<br>><br>> - sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,<br>> - my_dest->psn, my_dest->rkey, my_dest->vaddr);<br>
> + sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,<br>> + my_dest->psn, my_dest->rkey, my_dest->vaddr);<br>> +<br>> + if (write(sockfd, msg, sizeof msg) != sizeof msg) {<br>
> + perror("client write");<br>> + fprintf(stderr, "Couldn't send local address\n");<br>> + return -1;<br>> + }<br>
> +<br>> + return 0;<br>> + } else {<br>> + char msg[KEY_MSG_SIZE_GID];<br>> +<br>> + sprintf(msg, KEY_PRINT_FMT_GID, my_dest->lid, my_dest->qpn,<br>> + my_dest->psn, my_dest->rkey, my_dest->vaddr,<br>
> + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2], my_dest->dgid.raw[3],<br>> + my_dest->dgid.raw[4], my_dest->dgid.raw[5], my_dest->dgid.raw[6], my_dest->dgid.raw[7],<br>
> + my_dest->dgid.raw[8], my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],<br>> + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14], my_dest->dgid.raw[15]);<br>
><br>> if (write(sockfd, msg, sizeof msg) != sizeof msg) {<br>> perror("client write");<br>> @@ -167,30 +190,84 @@ static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest)<br>
> }<br>><br>> return 0;<br>> + }<br>> }<br>><br>> static int pp_read_keys(int sockfd, const struct pingpong_dest *my_dest,<br>> - struct pingpong_dest *rem_dest)<br>
> + struct pingpong_dest *rem_dest, struct user_parameters *user_parm)<br>> {<br>> - int parsed;<br>> - char msg[KEY_MSG_SIZE];<br>> + if (user_parm->gid_index < 0) {<br>
> + int parsed;<br>> + char msg[KEY_MSG_SIZE];<br>> +<br>> + if (read(sockfd, msg, sizeof msg) != sizeof msg) {<br>> + perror("pp_read_keys");<br>
> + fprintf(stderr, "Couldn't read remote address\n");<br>> + return -1;<br>> + }<br>><br>> - if (read(sockfd, msg, sizeof msg) != sizeof msg) {<br>
> - perror("pp_read_keys");<br>> - fprintf(stderr, "Couldn't read remote address\n");<br>> - return -1;<br>> - }<br>> + parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn,<br>
> + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>><br>> - parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn,<br>> - &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>
> + if (parsed != 5) {<br>> + fprintf(stderr, "Couldn't parse line <%.*s>\n",<br>> + (int)sizeof msg, msg);<br>> + return -1;<br>
> + }<br>><br>> - if (parsed != 5) {<br>> - fprintf(stderr, "Couldn't parse line <%.*s>\n",<br>> - (int)sizeof msg, msg);<br>> - return -1;<br>
> - }<br>> + return 0;<br>> + } else {<br>> + char msg[KEY_MSG_SIZE_GID];<br>> + if (read(sockfd, msg, sizeof msg) != sizeof msg) {<br>> + perror("pp_read_keys");<br>
> + fprintf(stderr, "Couldn't read remote address\n");<br>> + return -1;<br>> + }<br>> + char *pstr = msg, *term;<br>> + char tmp[20];<br>
> + int i;<br>><br>> - return 0;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>
> + tmp[term - pstr] = 0;<br>> + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN<br>> +<br>> + pstr += term - pstr + 1;<br>
> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY<br>
> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA<br>> +<br>> + for (i = 0; i < 15; ++i) {<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);<br>> + }<br>
> + pstr += term - pstr + 1;<br>> + strcpy(tmp, pstr);<br>> + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);<br>> + return 0;<br>> + }<br>
> }<br>><br>> static int pp_client_connect(const char *servername, int port)<br>> @@ -235,12 +312,12 @@ static int pp_client_connect(const char *servername, int port)<br>> }<br>><br>> static int pp_client_exch_dest(int sockfd, const struct pingpong_dest *my_dest,<br>
> - struct pingpong_dest *rem_dest)<br>> + struct pingpong_dest *rem_dest, struct user_parameters *user_parm)<br>> {<br>> - if (pp_write_keys(sockfd, my_dest))<br>
> + if (pp_write_keys(sockfd, my_dest, user_parm))<br>> return -1;<br>><br>> - return pp_read_keys(sockfd, my_dest, rem_dest);<br>> + return pp_read_keys(sockfd, my_dest, rem_dest, user_parm);<br>
> }<br>><br>> static int pp_server_connect(int port)<br>> @@ -300,13 +377,13 @@ static int pp_server_connect(int port)<br>> }<br>><br>> static int pp_server_exch_dest(int sockfd, const struct pingpong_dest *my_dest,<br>
> - struct pingpong_dest* rem_dest)<br>> + struct pingpong_dest* rem_dest, struct user_parameters *user_parm)<br>> {<br>><br>> - if (pp_read_keys(sockfd, my_dest, rem_dest))<br>
> + if (pp_read_keys(sockfd, my_dest, rem_dest, user_parm))<br>> return -1;<br>><br>> - return pp_write_keys(sockfd, my_dest);<br>> + return pp_write_keys(sockfd, my_dest, user_parm);<br>
> }<br>><br>> static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,<br>> @@ -351,7 +428,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,<br>> fprintf(stderr, "Failed to query device props");<br>
> return NULL;<br>> }<br>> - if (device_attr.vendor_part_id == 23108) {<br>> + if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) {<br>
> user_parm->mtu = 1024;<br>> } else {<br>> user_parm->mtu = 2048;<br>> @@ -522,9 +599,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,<br>
> attr.min_rnr_timer = 12;<br>> }<br>><br>> - attr.ah_attr.is_global = 0;<br>> - attr.ah_attr.dlid = dest->lid;<br>> - <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = sl;<br>
> + if (user_parm->gid_index < 0) {<br>> + attr.ah_attr.is_global = 0;<br>> + attr.ah_attr.dlid = dest->lid;<br>> + <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = sl;<br>
> + } else {<br>> + attr.ah_attr.is_global = 1;<br>> + attr.ah_attr.grh.dgid = dest->dgid;<br>> + attr.ah_attr.grh.hop_limit = 1;<br>> + <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = 0;<br>
> + }<br>> attr.ah_attr.src_path_bits = 0;<br>> attr.ah_attr.port_num = port;<br>> if ((user_parm->connection_type==UD) && (user_parm->use_mcg)) {<br>> @@ -538,9 +622,6 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,<br>
> attr.ah_attr.is_global = 1;<br>> attr.ah_attr.grh.sgid_index = 0;<br>> memcpy(attr.ah_attr.grh.dgid.raw, mcg_gid, 16);<br>> - } else {<br>> - attr.ah_attr.dlid = dest->lid;<br>
> - attr.ah_attr.is_global = 0;<br>> }<br>><br>> if (user_parm->connection_type==RC) {<br>> @@ -637,6 +718,7 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,<br>
> struct pingpong_dest my_dest;<br>> int sockfd;<br>> int rc;<br>> + union ibv_gid gid;<br>><br>><br>> /* Create connection between client and server.<br>
> @@ -645,14 +727,34 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,<br>> my_dest.lid = pp_get_local_lid(ctx, ib_port);<br>> my_dest.qpn = ctx->qp->qp_num;<br>
> my_dest.psn = lrand48() & 0xffffff;<br>> - if (!my_dest.lid) {<br>> - fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");<br>> - return -1;<br>
> + if (user_parm->gid_index < 0) {/*We do not fail test upon lid in RDMA0E/Eth conf*/<br>> + if (!my_dest.lid) {<br>> + fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");<br>
> + return -1;<br>> + }<br>> }<br>> + if (user_parm->gid_index != -1) {<br>> + int err=0;<br>> + err = ibv_query_gid (ctx->context, ib_port, user_parm->gid_index, &gid);<br>
> + if (err) {<br>> + return -1;<br>> + }<br>> + ctx->dgid=gid;<br>> + }<br>> + my_dest.dgid = gid;<br>> my_dest.rkey = ctx->mr->rkey;<br>
> my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;<br>><br>> printf(addr_fmt, "local", my_dest.lid, my_dest.qpn, my_dest.psn);<br>> + if (user_parm->gid_index > -1) {<br>
> + printf(" GID: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",<br>> + my_dest.dgid.raw[0],my_dest.dgid.raw[1],<br>> + my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4],<br>
> + my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7],<br>> + my_dest.dgid.raw[8], my_dest.dgid.raw[9], my_dest.dgid.raw[10],<br>> + my_dest.dgid.raw[11], my_dest.dgid.raw[12], my_dest.dgid.raw[13],<br>
> + my_dest.dgid.raw[14], my_dest.dgid.raw[15]);<br>> + }<br>><br>> sockfd = servername ? pp_client_connect(servername, port) :<br>> pp_server_connect(port);<br>> @@ -663,13 +765,22 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,<br>
> return sockfd;<br>> }<br>><br>> - rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest) :<br>> - pp_server_exch_dest(sockfd, &my_dest, rem_dest);<br>
> + rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest, user_parm) :<br>> + pp_server_exch_dest(sockfd, &my_dest, rem_dest, user_parm);<br>> if (rc)<br>> return rc;<br>
><br>> printf(addr_fmt, "remote", rem_dest->lid, rem_dest->qpn, rem_dest->psn,<br>> rem_dest->rkey, rem_dest->vaddr);<br>> + if (user_parm->gid_index > -1) {<br>
> + printf(" GID: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",<br>> + rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],<br>
> + rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4],<br>> + rem_dest->dgid.raw[5], rem_dest->dgid.raw[6], rem_dest->dgid.raw[7],<br>> + rem_dest->dgid.raw[8], rem_dest->dgid.raw[9], rem_dest->dgid.raw[10],<br>
> + rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13],<br>> + rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);<br>> + }<br>><br>> if ((rc = pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest,user_parm)))<br>
> return rc;<br>> @@ -678,8 +789,8 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,<br>> * Arbitrarily reuse exch_dest for this purpose.<br>> */<br>
><br>> - rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest) :<br>> - pp_server_exch_dest(sockfd, &my_dest, rem_dest);<br>> + rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest, user_parm) :<br>
> + pp_server_exch_dest(sockfd, &my_dest, rem_dest, user_parm);<br>><br>> if (rc)<br>> return rc;<br>> @@ -714,6 +825,7 @@ static void usage(const char *argv0)<br>> printf(" -I, --inline_size=<size> max size of message to be sent in inline mode (default 400)\n");<br>
> printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n");<br>> printf(" -S, --sl=<sl> SL (default 0)\n");<br>
> + printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n");<br>> printf(" -C, --report-cycles report times in cpu cycle units (default microseconds)\n");<br>
> printf(" -H, --report-histogram print out all results (default print summary only)\n");<br>> printf(" -U, --report-unsorted (implies -H) print out unsorted results (default sorted)\n");<br>
> @@ -814,7 +926,11 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,<br>><br>> if (user_param->connection_type==UD) {<br>> if (size > 2048) {<br>> - size = 2048;<br>
> + if (user_param->gid_index < 0) {<br>> + size = 2048;<br>> + } else {<br>> + size = 1024;<br>> + }<br>
> }<br>> }<br>><br>> @@ -1011,6 +1127,7 @@ int main(int argc, char *argv[])<br>> user_param.inline_size = MAX_INLINE;<br>> user_param.signal_comp = 0;<br>> user_param.qp_timeout = 14;<br>
> + user_param.gid_index = -1; /*gid will not be used*/<br>> /* Parameter parsing. */<br>> while (1) {<br>> int c;<br>> @@ -1027,6 +1144,7 @@ int main(int argc, char *argv[])<br>
> { .name = "inline_size", .has_arg = 1, .val = 'I' },<br>> { .name = "qp-timeout", .has_arg = 1, .val = 'u' },<br>> { .name = "sl", .has_arg = 1, .val = 'S' },<br>
> + { .name = "gid-index", .has_arg = 1, .val = 'x' },<br>> { .name = "signal", .has_arg = 0, .val = 'l' },<br>> { .name = "all", .has_arg = 0, .val = 'a' },<br>
> { .name = "report-cycles", .has_arg = 0, .val = 'C' },<br>> @@ -1038,7 +1156,7 @@ int main(int argc, char *argv[])<br>> { .name = "CPU-freq", .has_arg = 0, .val = 'F' },<br>
> { 0 }<br>> };<br>> - c = getopt_long(argc, argv, "p:c:m:d:i:s:n:t:I:u:S:laeCHUVgF", long_options, NULL);<br>> + c = getopt_long(argc, argv, "p:c:m:d:i:s:n:t:I:u:S:x:laeCHUVgF", long_options, NULL);<br>
> if (c == -1)<br>> break;<br>><br>> @@ -1095,6 +1213,14 @@ int main(int argc, char *argv[])<br>> }<br>> break;<br>><br>
> + case 'x':<br>> + user_param.gid_index = strtol(optarg, NULL, 0);<br>> + if (user_param.gid_index > 63) {<br>> + usage(argv[0]);<br>
> + return 1;<br>> + }<br>> + break;<br>> +<br>> case 't':<br>> user_param.tx_depth = strtol(optarg, NULL, 0);<br>
> if (user_param.tx_depth < 1) {<br>> @@ -1179,10 +1305,21 @@ int main(int argc, char *argv[])<br>> } else {<br>> printf("Connection type : UD\n");<br>
> }<br>> + if (user_param.gid_index > -1) {<br>> + printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n");<br>> + }<br>
> if (user_param.all == 1) {<br>> /*since we run all sizes lets allocate big enough buffer */<br>> size = 8388608; /*2^23 */<br>> }<br>> + if (user_param.connection_type == UD && size > 2048) {<br>
> + printf("Max msg size in UD is 2048 changing to 2048\n");<br>> + size = 2048;<br>> + }<br>> + if (user_param.connection_type == UD && user_param.gid_index > -1 && size > 1024) {<br>
> + printf("Max msg size in UD RDMAoE is 1024. changing to 1024\n");<br>> + size = 1024;<br>> + }<br>><br>> srand48(getpid() * time(NULL));<br>> page_size = sysconf(_SC_PAGESIZE);<br>
> @@ -1214,7 +1351,11 @@ int main(int argc, char *argv[])<br>><br>> if (user_param.all == 1) {<br>> if (user_param.connection_type==UD) {<br>> - size_max_pow = 12;<br>
> + if (user_param.gid_index < 0) {<br>> + size_max_pow = 12;<br>> + } else {<br>> + size_max_pow = 11;<br>
> + }<br>> }<br>> for (i = 1; i < size_max_pow ; ++i) {<br>> size = 1 << i;<br>><br>> --- a/write_bw_postlist.c<br>> +++ b/write_bw_postlist.c<br>
> @@ -75,6 +75,7 @@ struct user_parameters {<br>> int maxpostsofqpiniteration;<br>> int inline_size;<br>> int qp_timeout;<br>> + int gid_index; /* if value not negative, we use gid AND gid_index=value */<br>
> };<br>> struct extended_qp {<br>> struct ibv_qp *qp;<br>> @@ -98,6 +99,7 @@ struct pingpong_context {<br>> struct ibv_send_wr wr;<br>> int *scnt;<br>> int *ccnt ;<br>
> + union ibv_gid dgid;<br>> };<br>><br>> struct pingpong_dest {<br>> @@ -106,6 +108,7 @@ struct pingpong_dest {<br>> int psn;<br>> unsigned rkey;<br>> unsigned long long vaddr;<br>
> + union ibv_gid dgid;<br>> };<br>><br>><br>> @@ -160,14 +163,20 @@ static int pp_client_connect(const char *servername, int port)<br>> }<br>><br>> struct pingpong_dest * pp_client_exch_dest(int sockfd,<br>
> - const struct pingpong_dest *my_dest)<br>> + const struct pingpong_dest *my_dest, struct user_parameters *user_parm)<br>> {<br>
> struct pingpong_dest *rem_dest = NULL;<br>> - char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];<br>> + char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];<br>
> int parsed;<br>><br>> - sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,<br>> - my_dest->psn,my_dest->rkey,my_dest->vaddr);<br>> + sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",<br>
> + my_dest->lid, my_dest->qpn, my_dest->psn,my_dest->rkey,my_dest->vaddr,<br>> + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],<br>> + my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],<br>
> + my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],<br>> + my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],<br>> + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],<br>
> + my_dest->dgid.raw[15]);<br>> if (write(sockfd, msg, sizeof msg) != sizeof msg) {<br>> perror("client write");<br>> fprintf(stderr, "Couldn't send local address\n");<br>
> @@ -184,15 +193,59 @@ struct pingpong_dest * pp_client_exch_dest(int sockfd,<br>> if (!rem_dest)<br>> goto out;<br>><br>> - parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>
> - &rem_dest->psn,&rem_dest->rkey,&rem_dest->vaddr);<br>> -<br>> - if (parsed != 5) {<br>> - fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,<br>
> - msg);<br>> - free(rem_dest);<br>> - rem_dest = NULL;<br>> - goto out;<br>> + if (user_parm->gid_index < 0) {<br>> + parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>
> + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>> + if (parsed != 5) {<br>> + fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);<br>
> + free(rem_dest);<br>> + rem_dest = NULL;<br>> + goto out;<br>> + }<br>> + }else{<br>> + char *pstr = msg, *term;<br>
> + char tmp[20];<br>> + int i;<br>> +<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>
> + tmp[term - pstr] = 0;<br>> + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN<br>> +<br>> + pstr += term - pstr + 1;<br>
> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY<br>
> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA<br>> +<br>> + for (i = 0; i < 15; ++i) {<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);<br>> + }<br>
> + pstr += term - pstr + 1;<br>> + strcpy(tmp, pstr);<br>> + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);<br>> }<br>> out:<br>> return rem_dest;<br>
> @@ -254,9 +307,9 @@ int pp_server_connect(int port)<br>> return connfd;<br>> }<br>><br>> -static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest)<br>> +static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm)<br>
> {<br>> - char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];<br>> + char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];<br>
> struct pingpong_dest *rem_dest = NULL;<br>> int parsed;<br>> int n;<br>> @@ -272,18 +325,69 @@ static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpo<br>> if (!rem_dest)<br>
> goto out;<br>><br>> - parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>> - &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>
> - if (parsed != 5) {<br>> - fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,<br>> - msg);<br>> - free(rem_dest);<br>
> - rem_dest = NULL;<br>> - goto out;<br>> + if (user_parm->gid_index < 0) {<br>> + parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,<br>
> + &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);<br>> + if (parsed != 5) {<br>> + fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);<br>
> + free(rem_dest);<br>> + rem_dest = NULL;<br>> + goto out;<br>> + }<br>> + }else{<br>> + char *pstr = msg, *term;<br>
> + char tmp[20];<br>> + int i;<br>> +<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>
> + tmp[term - pstr] = 0;<br>> + rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN<br>> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN<br>> +<br>> + pstr += term - pstr + 1;<br>
> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY<br>
> +<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>
> + rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA<br>> +<br>> + for (i = 0; i < 15; ++i) {<br>> + pstr += term - pstr + 1;<br>> + term = strpbrk(pstr, ":");<br>
> + memcpy(tmp, pstr, term - pstr);<br>> + tmp[term - pstr] = 0;<br>> + rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);<br>> + }<br>
> + pstr += term - pstr + 1;<br>> + strcpy(tmp, pstr);<br>> + rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);<br>> }<br>><br>> - sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,<br>
> - my_dest->psn, my_dest->rkey, my_dest->vaddr);<br>> + sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",<br>
> + my_dest->lid, my_dest->qpn, my_dest->psn, my_dest->rkey, my_dest->vaddr,<br>> + my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],<br>> + my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],<br>
> + my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],<br>> + my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],<br>> + my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],<br>
> + my_dest->dgid.raw[15]);<br>> if (write(connfd, msg, sizeof msg) != sizeof msg) {<br>> perror("server write");<br>> fprintf(stderr, "Couldn't send local address\n");<br>
> @@ -341,7 +445,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev,<br>> fprintf(stderr, "Failed to query device props");<br>> return NULL;<br>
> }<br>> - if (device_attr.vendor_part_id == 23108) {<br>> + if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) {<br>> user_parm->mtu = 1024;<br>
> } else {<br>> user_parm->mtu = 2048;<br>> @@ -444,9 +548,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,<br>> attr.max_dest_rd_atomic = 1;<br>
> attr.min_rnr_timer = 12;<br>> }<br>> - attr.ah_attr.is_global = 0;<br>> - attr.ah_attr.dlid = dest->lid;<br>> - <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = sl;<br>
> + if (user_parm->gid_index < 0) {<br>> + attr.ah_attr.is_global = 0;<br>> + attr.ah_attr.dlid = dest->lid;<br>> + <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = sl;<br>
> + } else {<br>> + attr.ah_attr.is_global = 1;<br>> + attr.ah_attr.grh.dgid = dest->dgid;<br>> + attr.ah_attr.grh.hop_limit = 1;<br>> + <a href="http://attr.ah_attr.sl/" target="_blank">attr.ah_attr.sl</a> = 0;<br>
> + }<br>> attr.ah_attr.src_path_bits = 0;<br>> attr.ah_attr.port_num = port;<br>> if (user_parm->connection_type == RC) {<br>> @@ -524,6 +635,7 @@ static void usage(const char *argv0)<br>
> printf(" -I, --inline_size=<size> max size of message to be sent in inline mode (default 400)\n");<br>> printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n");<br>
> printf(" -S, --sl=<sl> SL (default 0)\n");<br>> + printf(" -x, --gid-inde</div></div></blockquote></div><br>