[ewg] [PATCH] Perftest: support RDMAoE
orenmeron
orenmeron at dev.mellanox.co.il
Mon Jul 20 01:48:08 PDT 2009
--- a/read_bw.c
+++ b/read_bw.c
@@ -71,6 +71,7 @@ struct user_parameters {
int max_out_read;
int use_event;
int qp_timeout;
+ int gid_index; /* if value not negative, we use gid AND gid_index=value */
};
static int sl = 0;
static int page_size;
@@ -88,6 +89,7 @@ struct pingpong_context {
int tx_depth;
struct ibv_sge list;
struct ibv_send_wr wr;
+ union ibv_gid dgid;
};
struct pingpong_dest {
@@ -96,6 +98,7 @@ struct pingpong_dest {
int psn;
unsigned rkey;
unsigned long long vaddr;
+ union ibv_gid dgid;
};
@@ -150,14 +153,20 @@ static int pp_client_connect(const char *servername, int port)
}
struct pingpong_dest * pp_client_exch_dest(int sockfd,
- const struct pingpong_dest *my_dest)
+ const struct pingpong_dest *my_dest, struct user_parameters *user_parm)
{
struct pingpong_dest *rem_dest = NULL;
- char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
+ char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
int parsed;
- sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,
- my_dest->psn,my_dest->rkey,my_dest->vaddr);
+ sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
+ my_dest->lid, my_dest->qpn, my_dest->psn,my_dest->rkey,my_dest->vaddr,
+ my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],
+ my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],
+ my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],
+ my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],
+ my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],
+ my_dest->dgid.raw[15]);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
perror("client write");
fprintf(stderr, "Couldn't send local address\n");
@@ -174,15 +183,59 @@ struct pingpong_dest * pp_client_exch_dest(int sockfd,
if (!rem_dest)
goto out;
- parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
- &rem_dest->psn,&rem_dest->rkey,&rem_dest->vaddr);
-
- if (parsed != 5) {
- fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,
- msg);
- free(rem_dest);
- rem_dest = NULL;
- goto out;
+ if (user_parm->gid_index < 0) {
+ parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
+ &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
+ if (parsed != 5) {
+ fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);
+ free(rem_dest);
+ rem_dest = NULL;
+ goto out;
+ }
+ }else{
+ char *pstr = msg, *term;
+ char tmp[20];
+ int i;
+
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
+
+ for (i = 0; i < 15; ++i) {
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);
+ }
+ pstr += term - pstr + 1;
+ strcpy(tmp, pstr);
+ rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);
}
out:
return rem_dest;
@@ -244,9 +297,9 @@ int pp_server_connect(int port)
return connfd;
}
-static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest)
+static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm)
{
- char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
+ char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
struct pingpong_dest *rem_dest = NULL;
int parsed;
int n;
@@ -262,18 +315,69 @@ static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpo
if (!rem_dest)
goto out;
- parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
- &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
- if (parsed != 5) {
- fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,
- msg);
- free(rem_dest);
- rem_dest = NULL;
- goto out;
+ if (user_parm->gid_index < 0) {
+ parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
+ &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
+ if (parsed != 5) {
+ fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);
+ free(rem_dest);
+ rem_dest = NULL;
+ goto out;
+ }
+ }else{
+ char *pstr = msg, *term;
+ char tmp[20];
+ int i;
+
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
+
+ for (i = 0; i < 15; ++i) {
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);
+ }
+ pstr += term - pstr + 1;
+ strcpy(tmp, pstr);
+ rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);
}
- sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,
- my_dest->psn, my_dest->rkey, my_dest->vaddr);
+ sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
+ my_dest->lid, my_dest->qpn, my_dest->psn, my_dest->rkey, my_dest->vaddr,
+ my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],
+ my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],
+ my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],
+ my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],
+ my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],
+ my_dest->dgid.raw[15]);
if (write(connfd, msg, sizeof msg) != sizeof msg) {
perror("server write");
fprintf(stderr, "Couldn't send local address\n");
@@ -318,7 +422,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev,
fprintf(stderr, "Failed to query device props");
return NULL;
}
- if (device_attr.vendor_part_id == 23108)
+ if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1)
user_parm->mtu = 1024;
else
user_parm->mtu = 2048;
@@ -421,9 +525,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
attr.rq_psn = dest->psn;
attr.max_dest_rd_atomic = user_parm->max_out_read;
attr.min_rnr_timer = 12;
- attr.ah_attr.is_global = 0;
- attr.ah_attr.dlid = dest->lid;
- attr.ah_attr.sl = sl;
+ if (user_parm->gid_index<0) {
+ attr.ah_attr.is_global = 0;
+ attr.ah_attr.dlid = dest->lid;
+ attr.ah_attr.sl = sl;
+ } else {
+ attr.ah_attr.is_global = 1;
+ attr.ah_attr.grh.dgid = dest->dgid;
+ attr.ah_attr.grh.hop_limit = 1;
+ attr.ah_attr.sl = 0;
+ }
attr.ah_attr.src_path_bits = 0;
attr.ah_attr.port_num = port;
if (ibv_modify_qp(ctx->qp, &attr,
@@ -474,6 +585,7 @@ static void usage(const char *argv0)
printf(" -n, --iters=<iters> number of exchanges (at least 2, default 1000)\n");
printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n");
printf(" -S, --sl=<sl> SL (default 0)\n");
+ printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n");
printf(" -b, --bidirectional measure bidirectional bandwidth (default unidirectional)\n");
printf(" -V, --version display version number\n");
printf(" -e, --events sleep on CQ events (default poll)\n");
@@ -609,6 +721,7 @@ int main(int argc, char *argv[])
int duplex = 0;
int i = 0;
int no_cpu_freq_fail = 0;
+ union ibv_gid gid;
/* init default values to user's parameters */
memset(&user_param, 0, sizeof(struct user_parameters));
@@ -619,6 +732,7 @@ int main(int argc, char *argv[])
user_param.use_event = 0;
user_param.max_out_read = 4; /* the device capability on gen2 */
user_param.qp_timeout = 14;
+ user_param.gid_index = -1; /*gid will not be used*/
/* Parameter parsing. */
while (1) {
int c;
@@ -634,6 +748,7 @@ int main(int argc, char *argv[])
{ .name = "tx-depth", .has_arg = 1, .val = 't' },
{ .name = "qp-timeout", .has_arg = 1, .val = 'u' },
{ .name = "sl", .has_arg = 1, .val = 'S' },
+ { .name = "gid-index", .has_arg = 1, .val = 'x' },
{ .name = "all", .has_arg = 0, .val = 'a' },
{ .name = "bidirectional", .has_arg = 0, .val = 'b' },
{ .name = "version", .has_arg = 0, .val = 'V' },
@@ -642,7 +757,7 @@ int main(int argc, char *argv[])
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:m:o:s:n:t:u:S:abVeF", long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:m:o:s:n:t:u:S:x:abVeF", long_options, NULL);
if (c == -1)
break;
@@ -721,6 +836,14 @@ int main(int argc, char *argv[])
if (sl > 15) { usage(argv[0]); return 1; }
break;
+ case 'x':
+ user_param.gid_index = strtol(optarg, NULL, 0);
+ if (user_param.gid_index > 63) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+
default:
usage(argv[0]);
return 1;
@@ -740,6 +863,10 @@ int main(int argc, char *argv[])
printf(" RDMA_Read BW Test\n");
printf("Connection type : RC\n");
+ if (user_param.gid_index > -1) {
+ printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n");
+ }
+
/* Done with parameter parsing. Perform setup. */
if (user_param.all == ALL)
/*since we run all sizes */
@@ -771,16 +898,28 @@ int main(int argc, char *argv[])
if (!ctx)
return 1;
+ if (user_param.gid_index != -1) {
+ int err=0;
+ err = ibv_query_gid (ctx->context, ib_port, user_param.gid_index, &gid);
+ if (err) {
+ return -1;
+ }
+ ctx->dgid=gid;
+ }
+
/* Create connection between client and server.
* We do it by exchanging data over a TCP socket connection. */
my_dest.lid = pp_get_local_lid(ctx, ib_port);
my_dest.qpn = ctx->qp->qp_num;
my_dest.psn = lrand48() & 0xffffff;
- if (!my_dest.lid) {
- fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
- return 1;
+ if (user_param.gid_index < 0) {/*We do not fail test upon lid in RDMA0E/Eth conf*/
+ if (!my_dest.lid) {
+ fprintf(stderr, "Local lid 0x0 detected. Is an SM running? If you are running on an RMDAoE interface you must use GIDs\n");
+ return 1;
+ }
}
+ my_dest.dgid = gid;
my_dest.rkey = ctx->mr->rkey;
my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;
@@ -788,17 +927,26 @@ int main(int argc, char *argv[])
"RKey %#08x VAddr %#016Lx\n",
my_dest.lid, my_dest.qpn, my_dest.psn,
my_dest.rkey, my_dest.vaddr);
+ if (user_param.gid_index > -1) {
+ printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ my_dest.dgid.raw[0],my_dest.dgid.raw[1],
+ my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4],
+ my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7],
+ my_dest.dgid.raw[8], my_dest.dgid.raw[9], my_dest.dgid.raw[10],
+ my_dest.dgid.raw[11], my_dest.dgid.raw[12], my_dest.dgid.raw[13],
+ my_dest.dgid.raw[14], my_dest.dgid.raw[15]);
+ }
if (user_param.servername) {
sockfd = pp_client_connect(user_param.servername, port);
if (sockfd < 0)
return 1;
- rem_dest = pp_client_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);
} else {
sockfd = pp_server_connect(port);
if (sockfd < 0)
return 1;
- rem_dest = pp_server_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);
}
if (!rem_dest)
@@ -808,6 +956,15 @@ int main(int argc, char *argv[])
"RKey %#08x VAddr %#016Lx\n",
rem_dest->lid, rem_dest->qpn, rem_dest->psn,
rem_dest->rkey, rem_dest->vaddr);
+ if (user_param.gid_index > -1) {
+ printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],
+ rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4],
+ rem_dest->dgid.raw[5], rem_dest->dgid.raw[6], rem_dest->dgid.raw[7],
+ rem_dest->dgid.raw[8], rem_dest->dgid.raw[9], rem_dest->dgid.raw[10],
+ rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13],
+ rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);
+ }
if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest, &user_param))
return 1;
@@ -815,9 +972,9 @@ int main(int argc, char *argv[])
/* An additional handshake is required *after* moving qp to RTR.
Arbitrarily reuse exch_dest for this purpose. */
if (user_param.servername)
- rem_dest = pp_client_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);
else
- rem_dest = pp_server_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);
if (!rem_dest)
return 1;
@@ -826,7 +983,7 @@ int main(int argc, char *argv[])
/* For half duplex tests, server just waits for client to exit */
if (!user_param.servername && !duplex) {
- rem_dest = pp_server_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);
if (write(sockfd, "done", sizeof "done") != sizeof "done"){
perror("server write");
fprintf(stderr, "Couldn't write to socket\n");
@@ -873,9 +1030,9 @@ int main(int argc, char *argv[])
}
if (user_param.servername)
- rem_dest = pp_client_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);
else
- rem_dest = pp_server_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);
if (write(sockfd, "done", sizeof "done") != sizeof "done"){
perror("server write");
--- a/read_lat.c
+++ b/read_lat.c
@@ -75,6 +75,7 @@ struct user_parameters {
int max_out_read;
int use_event;
int qp_timeout;
+ int gid_index; /* if value not negative, we use gid AND gid_index=value */
};
struct report_options {
@@ -97,6 +98,7 @@ struct pingpong_context {
int tx_depth;
struct ibv_sge list;
struct ibv_send_wr wr;
+ union ibv_gid dgid;
};
struct pingpong_dest {
@@ -105,6 +107,7 @@ struct pingpong_dest {
int psn;
unsigned rkey;
unsigned long long vaddr;
+ union ibv_gid dgid;
};
@@ -140,14 +143,33 @@ static struct ibv_device *pp_find_dev(const char *ib_devname) {
#define KEY_MSG_SIZE (sizeof "0000:000000:000000:00000000:0000000000000000")
#define KEY_PRINT_FMT "%04x:%06x:%06x:%08x:%016Lx"
+#define KEY_MSG_SIZE_GID (sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00")
+#define KEY_PRINT_FMT_GID "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x"
-static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest)
+static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm)
{
- char msg[KEY_MSG_SIZE];
+ if (user_parm->gid_index < 0) {
+ char msg[KEY_MSG_SIZE];
- sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,
- my_dest->psn, my_dest->rkey, my_dest->vaddr);
+ sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,
+ my_dest->psn, my_dest->rkey, my_dest->vaddr);
+ if (write(sockfd, msg, sizeof msg) != sizeof msg) {
+ perror("client write");
+ fprintf(stderr, "Couldn't send local address\n");
+ return -1;
+ }
+
+ return 0;
+ } else {
+ char msg[KEY_MSG_SIZE_GID];
+
+ sprintf(msg, KEY_PRINT_FMT_GID, my_dest->lid, my_dest->qpn,
+ my_dest->psn, my_dest->rkey, my_dest->vaddr,
+ my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2], my_dest->dgid.raw[3],
+ my_dest->dgid.raw[4], my_dest->dgid.raw[5], my_dest->dgid.raw[6], my_dest->dgid.raw[7],
+ my_dest->dgid.raw[8], my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],
+ my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14], my_dest->dgid.raw[15]);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
perror("client write");
fprintf(stderr, "Couldn't send local address\n");
@@ -155,30 +177,84 @@ static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest)
}
return 0;
+ }
}
static int pp_read_keys(int sockfd, const struct pingpong_dest *my_dest,
- struct pingpong_dest *rem_dest)
+ struct pingpong_dest *rem_dest, struct user_parameters *user_parm)
{
- int parsed;
- char msg[KEY_MSG_SIZE];
+ if (user_parm->gid_index < 0) {
+ int parsed;
+ char msg[KEY_MSG_SIZE];
+
+ if (read(sockfd, msg, sizeof msg) != sizeof msg) {
+ perror("pp_read_keys");
+ fprintf(stderr, "Couldn't read remote address\n");
+ return -1;
+ }
- if (read(sockfd, msg, sizeof msg) != sizeof msg) {
- perror("pp_read_keys");
- fprintf(stderr, "Couldn't read remote address\n");
- return -1;
- }
+ parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn,
+ &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
- parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn,
- &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
+ if (parsed != 5) {
+ fprintf(stderr, "Couldn't parse line <%.*s>\n",
+ (int)sizeof msg, msg);
+ return -1;
+ }
- if (parsed != 5) {
- fprintf(stderr, "Couldn't parse line <%.*s>\n",
- (int)sizeof msg, msg);
- return -1;
+ return 0;
+ } else {
+ char msg[KEY_MSG_SIZE_GID];
+ if (read(sockfd, msg, sizeof msg) != sizeof msg) {
+ perror("pp_read_keys");
+ fprintf(stderr, "Couldn't read remote address\n");
+ return -1;
+ }
+ char *pstr = msg, *term;
+ char tmp[20];
+ int i;
+
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
+
+ for (i = 0; i < 15; ++i) {
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);
+ }
+ pstr += term - pstr + 1;
+ strcpy(tmp, pstr);
+ rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);
+ return 0;
}
-
- return 0;
}
static int pp_client_connect(const char *servername, int port)
@@ -223,12 +299,12 @@ static int pp_client_connect(const char *servername, int port)
}
static int pp_client_exch_dest(int sockfd, const struct pingpong_dest *my_dest,
- struct pingpong_dest *rem_dest)
+ struct pingpong_dest *rem_dest, struct user_parameters *user_parm)
{
- if (pp_write_keys(sockfd, my_dest))
+ if (pp_write_keys(sockfd, my_dest, user_parm))
return -1;
- return pp_read_keys(sockfd, my_dest, rem_dest);
+ return pp_read_keys(sockfd, my_dest, rem_dest, user_parm);
}
static int pp_server_connect(int port)
@@ -288,13 +364,13 @@ static int pp_server_connect(int port)
}
static int pp_server_exch_dest(int sockfd, const struct pingpong_dest *my_dest,
- struct pingpong_dest* rem_dest)
+ struct pingpong_dest* rem_dest, struct user_parameters *user_parm)
{
- if (pp_read_keys(sockfd, my_dest, rem_dest))
+ if (pp_read_keys(sockfd, my_dest, rem_dest, user_parm))
return -1;
- return pp_write_keys(sockfd, my_dest);
+ return pp_write_keys(sockfd, my_dest, user_parm);
}
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
@@ -332,7 +408,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
fprintf(stderr, "Failed to query device props");
return NULL;
}
- if (device_attr.vendor_part_id == 23108) {
+ if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) {
user_parm->mtu = 1024;
} else {
user_parm->mtu = 2048;
@@ -444,9 +520,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
attr.rq_psn = dest->psn;
attr.max_dest_rd_atomic = user_parm->max_out_read;
attr.min_rnr_timer = 12;
- attr.ah_attr.is_global = 0;
- attr.ah_attr.dlid = dest->lid;
- attr.ah_attr.sl = sl;
+ if (user_parm->gid_index < 0) {
+ attr.ah_attr.is_global = 0;
+ attr.ah_attr.dlid = dest->lid;
+ attr.ah_attr.sl = sl;
+ } else {
+ attr.ah_attr.is_global = 1;
+ attr.ah_attr.grh.dgid = dest->dgid;
+ attr.ah_attr.grh.hop_limit = 1;
+ attr.ah_attr.sl = 0;
+ }
attr.ah_attr.src_path_bits = 0;
attr.ah_attr.port_num = port;
if (ibv_modify_qp(ctx->qp, &attr,
@@ -497,23 +580,45 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,
char addr_fmt[] = "%8s address: LID %#04x QPN %#06x PSN %#06x RKey %#08x VAddr %#016Lx\n";
int sockfd;
int rc;
+ union ibv_gid gid;
/* Create connection between client and server.
* We do it by exchanging data over a TCP socket connection. */
+ if (user_parm->gid_index != -1) {
+ int err=0;
+ err = ibv_query_gid (ctx->context, ib_port, user_parm->gid_index, &gid);
+ if (err) {
+ return -1;
+ }
+ ctx->dgid=gid;
+ }
+
my_dest.lid = pp_get_local_lid(ctx, ib_port);
+ my_dest.dgid = gid;
my_dest.qpn = ctx->qp->qp_num;
my_dest.psn = lrand48() & 0xffffff;
- if (!my_dest.lid) {
- fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
- return -1;
+ if (user_parm->gid_index < 0) {/*We do not fail test upon lid in RDMAoE/Eth conf*/
+ if (!my_dest.lid) {
+ fprintf(stderr, "Local lid 0x0 detected. Is an SM running? If you are running on an RMDAoE interface you must use GIDs\n");
+ return 1;
+ }
}
my_dest.rkey = ctx->mr->rkey;
my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;
printf(addr_fmt, "local", my_dest.lid, my_dest.qpn, my_dest.psn,
my_dest.rkey, my_dest.vaddr);
+ if (user_parm->gid_index > -1) {
+ printf(" GID: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ my_dest.dgid.raw[0],my_dest.dgid.raw[1],
+ my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4],
+ my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7],
+ my_dest.dgid.raw[8], my_dest.dgid.raw[9], my_dest.dgid.raw[10],
+ my_dest.dgid.raw[11], my_dest.dgid.raw[12], my_dest.dgid.raw[13],
+ my_dest.dgid.raw[14], my_dest.dgid.raw[15]);
+ }
sockfd = servername ? pp_client_connect(servername, port) :
pp_server_connect(port);
@@ -524,13 +629,22 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,
return sockfd;
}
- rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest) :
- pp_server_exch_dest(sockfd, &my_dest, rem_dest);
+ rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest, user_parm) :
+ pp_server_exch_dest(sockfd, &my_dest, rem_dest, user_parm);
if (rc)
return rc;
printf(addr_fmt, "remote", rem_dest->lid, rem_dest->qpn, rem_dest->psn,
rem_dest->rkey, rem_dest->vaddr);
+ if (user_parm->gid_index > -1) {
+ printf(" GID: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],
+ rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4],
+ rem_dest->dgid.raw[5], rem_dest->dgid.raw[6], rem_dest->dgid.raw[7],
+ rem_dest->dgid.raw[8], rem_dest->dgid.raw[9], rem_dest->dgid.raw[10],
+ rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13],
+ rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);
+ }
if ((rc = pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest,user_parm)))
return rc;
@@ -539,8 +653,8 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,
* Arbitrarily reuse exch_dest for this purpose.
*/
- rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest) :
- pp_server_exch_dest(sockfd, &my_dest, rem_dest);
+ rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest, user_parm) :
+ pp_server_exch_dest(sockfd, &my_dest, rem_dest, user_parm);
if (rc)
return rc;
@@ -565,6 +679,8 @@ static void usage(const char *argv0)
printf(" -n, --iters=<iters> number of exchanges (at least 2, default 1000)\n");
printf(" -o, --outs=<num> num of outstanding read/atom(default 4)\n");
printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n");
+ printf(" -S, --sl=<sl> SL (default 0)\n");
+ printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n");
printf(" -a, --all Run sizes from 2 till 2^23\n");
printf(" -C, --report-cycles report times in cpu cycle units (default microseconds)\n");
printf(" -H, --report-histogram print out all results (default print summary only)\n");
@@ -754,6 +870,7 @@ int main(int argc, char *argv[])
user_param.use_event = 0;
user_param.max_out_read = 4; /* the device capability on gen2 */
user_param.qp_timeout = 14;
+ user_param.gid_index = -1; /*gid will not be used*/
/* Parameter parsing. */
while (1) {
int c;
@@ -770,6 +887,7 @@ int main(int argc, char *argv[])
{ .name = "tx-depth", .has_arg = 1, .val = 't' },
{ .name = "qp-timeout", .has_arg = 1, .val = 'u' },
{ .name = "sl", .has_arg = 1, .val = 'S' },
+ { .name = "gid-index", .has_arg = 1, .val = 'x' },
{ .name = "all", .has_arg = 0, .val = 'a' },
{ .name = "report-cycles", .has_arg = 0, .val = 'C' },
{ .name = "report-histogram",.has_arg = 0, .val = 'H' },
@@ -780,7 +898,7 @@ int main(int argc, char *argv[])
{ 0 }
};
- c = getopt_long(argc, argv, "p:c:m:d:i:s:o:n:t:u:S:aeHUVF", long_options, NULL);
+ c = getopt_long(argc, argv, "p:c:m:d:i:s:o:n:t:u:S:x:aeHUVF", long_options, NULL);
if (c == -1)
break;
@@ -874,6 +992,14 @@ int main(int argc, char *argv[])
if (sl > 15) { usage(argv[0]); return 5; }
break;
+ case 'x':
+ user_param.gid_index = strtol(optarg, NULL, 0);
+ if (user_param.gid_index > 63) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+
default:
usage(argv[0]);
return 6;
@@ -899,6 +1025,9 @@ int main(int argc, char *argv[])
printf(" RDMA_Read Latency Test\n");
printf("Connection type : RC\n");
/* anyway make sure the connection is RC */
+ if (user_param.gid_index > -1) {
+ printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n");
+ }
tmp_size = size;
if (user_param.all == ALL) {
/*since we run all sizes */
@@ -956,7 +1085,7 @@ int main(int argc, char *argv[])
/* done close sockets */
if(user_param.servername) {
/*Signal client is finished */
- pp_client_exch_dest(user_param.sockfd, &my_dest, &rem_dest);
+ pp_client_exch_dest(user_param.sockfd, &my_dest, &rem_dest, &user_param);
if (write(user_param.sockfd, "done", sizeof "done") != sizeof "done"){
perror("client write");
fprintf(stderr, "Couldn't write to socket\n");
@@ -965,7 +1094,7 @@ int main(int argc, char *argv[])
close(user_param.sockfd);
} else {
/*Server is finished wait for client */
- pp_server_exch_dest(user_param.sockfd, &my_dest, &rem_dest);
+ pp_server_exch_dest(user_param.sockfd, &my_dest, &rem_dest, &user_param);
if (write(user_param.sockfd, "done", sizeof "done") != sizeof "done"){
perror("server write");
fprintf(stderr, "Couldn't write to socket\n");
--- a/send_bw.c
+++ b/send_bw.c
@@ -84,6 +84,7 @@ struct user_parameters {
int use_mcg;
int inline_size;
int qp_timeout;
+ int gid_index; /* if value not negative, we use gid AND gid_index=value */
};
static int sl = 0;
static int page_size;
@@ -106,6 +107,7 @@ struct pingpong_context {
struct ibv_send_wr wr;
struct ibv_recv_wr rwr;
struct ibv_ah *ah;
+ union ibv_gid dgid;
};
struct pingpong_dest {
@@ -114,6 +116,7 @@ struct pingpong_dest {
int psn;
unsigned rkey;
unsigned long long vaddr;
+ union ibv_gid dgid;
};
static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
@@ -167,14 +170,20 @@ static int pp_client_connect(const char *servername, int port)
}
struct pingpong_dest * pp_client_exch_dest(int sockfd,
- const struct pingpong_dest *my_dest)
+ const struct pingpong_dest *my_dest, struct user_parameters *user_parm)
{
struct pingpong_dest *rem_dest = NULL;
- char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
+ char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
int parsed;
- sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,
- my_dest->psn,my_dest->rkey,my_dest->vaddr);
+ sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
+ my_dest->lid, my_dest->qpn, my_dest->psn,my_dest->rkey,my_dest->vaddr,
+ my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],
+ my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],
+ my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],
+ my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],
+ my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],
+ my_dest->dgid.raw[15]);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
perror("client write");
fprintf(stderr, "Couldn't send local address\n");
@@ -191,15 +200,59 @@ struct pingpong_dest * pp_client_exch_dest(int sockfd,
if (!rem_dest)
goto out;
- parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
- &rem_dest->psn,&rem_dest->rkey,&rem_dest->vaddr);
+ if (user_parm->gid_index < 0) {
+ parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
+ &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
+ if (parsed != 5) {
+ fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);
+ free(rem_dest);
+ rem_dest = NULL;
+ goto out;
+ }
+ }else{
+ char *pstr = msg, *term;
+ char tmp[20];
+ int i;
- if (parsed != 5) {
- fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,
- msg);
- free(rem_dest);
- rem_dest = NULL;
- goto out;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
+
+ for (i = 0; i < 15; ++i) {
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);
+ }
+ pstr += term - pstr + 1;
+ strcpy(tmp, pstr);
+ rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);
}
out:
return rem_dest;
@@ -261,9 +314,9 @@ int pp_server_connect(int port)
return connfd;
}
-static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest)
+static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm)
{
- char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
+ char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
struct pingpong_dest *rem_dest = NULL;
int parsed;
int n;
@@ -279,18 +332,69 @@ static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpo
if (!rem_dest)
goto out;
- parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
- &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
- if (parsed != 5) {
- fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,
- msg);
- free(rem_dest);
- rem_dest = NULL;
- goto out;
+ if (user_parm->gid_index < 0) {
+ parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
+ &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
+ if (parsed != 5) {
+ fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);
+ free(rem_dest);
+ rem_dest = NULL;
+ goto out;
+ }
+ }else{
+ char *pstr = msg, *term;
+ char tmp[20];
+ int i;
+
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
+
+ for (i = 0; i < 15; ++i) {
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);
+ }
+ pstr += term - pstr + 1;
+ strcpy(tmp, pstr);
+ rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);
}
- sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,
- my_dest->psn, my_dest->rkey, my_dest->vaddr);
+ sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
+ my_dest->lid, my_dest->qpn, my_dest->psn, my_dest->rkey, my_dest->vaddr,
+ my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],
+ my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],
+ my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],
+ my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],
+ my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],
+ my_dest->dgid.raw[15]);
if (write(connfd, msg, sizeof msg) != sizeof msg) {
perror("server write");
fprintf(stderr, "Couldn't send local address\n");
@@ -346,7 +450,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev,
fprintf(stderr, "Failed to query device props");
return NULL;
}
- if (device_attr.vendor_part_id == 23108) {
+ if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) {
user_parm->mtu = 1024;
} else {
user_parm->mtu = 2048;
@@ -504,9 +608,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
attr.max_dest_rd_atomic = 1;
attr.min_rnr_timer = 12;
}
- attr.ah_attr.is_global = 0;
- attr.ah_attr.dlid = dest->lid;
- attr.ah_attr.sl = sl;
+ if (user_parm->gid_index < 0) {
+ attr.ah_attr.is_global = 0;
+ attr.ah_attr.dlid = dest->lid;
+ attr.ah_attr.sl = sl;
+ } else {
+ attr.ah_attr.is_global = 1;
+ attr.ah_attr.grh.dgid = dest->dgid;
+ attr.ah_attr.grh.hop_limit = 1;
+ attr.ah_attr.sl = 0;
+ }
attr.ah_attr.src_path_bits = 0;
attr.ah_attr.port_num = port;
if ((user_parm->connection_type==UD) && (user_parm->use_mcg)) {
@@ -518,9 +629,6 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.sgid_index = 0;
memcpy(attr.ah_attr.grh.dgid.raw, mcg_gid, 16);
- } else {
- attr.ah_attr.dlid = dest->lid;
- attr.ah_attr.is_global = 0;
}
if (user_parm->connection_type == RC) {
if (ibv_modify_qp(ctx->qp, &attr,
@@ -632,6 +740,7 @@ static void usage(const char *argv0)
printf(" -I, --inline_size=<size> max size of message to be sent in inline mode (default 400)\n");
printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n");
printf(" -S, --sl=<sl> SL (default 0)\n");
+ printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n");
printf(" -b, --bidirectional measure bidirectional bandwidth (default unidirectional)\n");
printf(" -V, --version display version number\n");
printf(" -e, --events sleep on CQ events (default poll)\n");
@@ -680,8 +789,13 @@ int run_iter_bi(struct pingpong_context *ctx, struct user_parameters *user_param
int scnt, ccnt, rcnt;
struct ibv_recv_wr *bad_wr_recv;
if (user_param->connection_type == UD) {
- if (size > 2048)
- size = 2048;
+ if (size > 2048) {
+ if (user_param->gid_index < 0) {
+ size = 2048;
+ } else {
+ size = 1024;
+ }
+ }
}
/*********************************************
* Important note :
@@ -799,8 +913,13 @@ int run_iter_uni(struct pingpong_context *ctx, struct user_parameters *user_para
struct ibv_recv_wr *bad_wr_recv;
if (user_param->connection_type == UD) {
- if (size > 2048)
- size = 2048;
+ if (size > 2048) {
+ if (user_param->gid_index < 0) {
+ size = 2048;
+ } else {
+ size = 1024;
+ }
+ }
}
if (user_param->connection_type == UD)
@@ -947,6 +1066,7 @@ int main(int argc, char *argv[])
int inline_given_in_cmd = 0;
struct ibv_context *context;
int no_cpu_freq_fail = 0;
+ union ibv_gid gid;
/* init default values to user's parameters */
memset(&user_param, 0, sizeof(struct user_parameters));
user_param.mtu = 0;
@@ -957,6 +1077,7 @@ int main(int argc, char *argv[])
user_param.duplex = 0;
user_param.inline_size = MAX_INLINE;
user_param.qp_timeout = 14;
+ user_param.gid_index = -1; /*gid will not be used*/
/* Parameter parsing. */
while (1) {
int c;
@@ -974,6 +1095,7 @@ int main(int argc, char *argv[])
{ .name = "rx-depth", .has_arg = 1, .val = 'r' },
{ .name = "qp-timeout", .has_arg = 1, .val = 'u' },
{ .name = "sl", .has_arg = 1, .val = 'S' },
+ { .name = "gid-index", .has_arg = 1, .val = 'x' },
{ .name = "all", .has_arg = 0, .val = 'a' },
{ .name = "bidirectional", .has_arg = 0, .val = 'b' },
{ .name = "version", .has_arg = 0, .val = 'V' },
@@ -984,7 +1106,7 @@ int main(int argc, char *argv[])
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:I:r:u:S:ebaVgNF", long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:I:r:u:S:x:ebaVgNF", long_options, NULL);
if (c == -1)
break;
@@ -1038,6 +1160,14 @@ int main(int argc, char *argv[])
break;
+ case 'x':
+ user_param.gid_index = strtol(optarg, NULL, 0);
+ if (user_param.gid_index > 63) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+
case 't':
user_param.tx_depth = strtol(optarg, NULL, 0);
if (user_param.tx_depth < 1) { usage(argv[0]); return 1; }
@@ -1117,6 +1247,9 @@ int main(int argc, char *argv[])
else{
printf("Connection type : UD\n");
}
+ if (user_param.gid_index > -1) {
+ printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n");
+ }
/* Done with parameter parsing. Perform setup. */
if (user_param.all == ALL)
@@ -1126,6 +1259,10 @@ int main(int argc, char *argv[])
printf("Max msg size in UD is 2048 changing to 2048\n");
size = 2048;
}
+ if (user_param.connection_type == UD && user_param.gid_index > -1 && size > 1024) {
+ printf("Max msg size in UD RDMAoE is 1024. changing to 1024\n");
+ size = 1024;
+ }
srand48(getpid() * time(NULL));
@@ -1174,25 +1311,46 @@ int main(int argc, char *argv[])
my_dest.lid = pp_get_local_lid(ctx, ib_port);
my_dest.qpn = ctx->qp->qp_num;
my_dest.psn = lrand48() & 0xffffff;
- if (!my_dest.lid) {
- fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
- return 1;
+ if (user_param.gid_index != -1) {
+ int err=0;
+ err = ibv_query_gid (ctx->context, ib_port, user_param.gid_index, &gid);
+ if (err) {
+ return -1;
+ }
+ ctx->dgid=gid;
+ }
+
+ if (user_param.gid_index < 0) {/*We do not fail test upon lid in RDMA0E/Eth conf*/
+ if (!my_dest.lid) {
+ fprintf(stderr, "Local lid 0x0 detected. Is an SM running? If you are running on an RMDAoE interface you must use GIDs\n");
+ return 1;
+ }
}
+ my_dest.dgid = gid;
my_dest.rkey = ctx->mr->rkey;
my_dest.vaddr = (uintptr_t)ctx->buf + size;
printf(" local address: LID %#04x, QPN %#06x, PSN %#06x\n",
my_dest.lid, my_dest.qpn, my_dest.psn);
+ if (user_param.gid_index > -1) {
+ printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ my_dest.dgid.raw[0],my_dest.dgid.raw[1],
+ my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4],
+ my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7],
+ my_dest.dgid.raw[8], my_dest.dgid.raw[9], my_dest.dgid.raw[10],
+ my_dest.dgid.raw[11], my_dest.dgid.raw[12], my_dest.dgid.raw[13],
+ my_dest.dgid.raw[14], my_dest.dgid.raw[15]);
+ }
if (user_param.servername) {
sockfd = pp_client_connect(user_param.servername, port);
if (sockfd < 0)
return 1;
- rem_dest = pp_client_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);
} else {
sockfd = pp_server_connect(port);
if (sockfd < 0)
return 1;
- rem_dest = pp_server_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);
}
if (!rem_dest)
@@ -1200,6 +1358,15 @@ int main(int argc, char *argv[])
printf(" remote address: LID %#04x, QPN %#06x, PSN %#06x\n",
rem_dest->lid, rem_dest->qpn, rem_dest->psn);
+ if (user_param.gid_index > -1) {
+ printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],
+ rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4],
+ rem_dest->dgid.raw[5], rem_dest->dgid.raw[6], rem_dest->dgid.raw[7],
+ rem_dest->dgid.raw[8], rem_dest->dgid.raw[9], rem_dest->dgid.raw[10],
+ rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13],
+ rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);
+ }
if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest, &user_param))
return 1;
@@ -1207,9 +1374,9 @@ int main(int argc, char *argv[])
/* An additional handshake is required *after* moving qp to RTR.
Arbitrarily reuse exch_dest for this purpose. */
if (user_param.servername) {
- rem_dest = pp_client_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);
} else {
- rem_dest = pp_server_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);
}
if (user_param.use_event) {
printf("Test with events.\n");
@@ -1263,8 +1430,13 @@ int main(int argc, char *argv[])
ctx->recv_list.lkey = ctx->mr->lkey;
if (user_param.all == ALL) {
- if (user_param.connection_type == UD)
- size_max_pow = 12;
+ if (user_param.connection_type == UD) {
+ if (user_param.gid_index < 0) {
+ size_max_pow = 12;
+ } else {
+ size_max_pow = 11;
+ }
+ }
for (i = 1; i < size_max_pow ; ++i) {
size = 1 << i;
@@ -1278,9 +1450,9 @@ int main(int argc, char *argv[])
if (user_param.servername) {
print_report(user_param.iters, size, user_param.duplex, tposted, tcompleted, noPeak, no_cpu_freq_fail);
/* sync again for the sake of UC/UC */
- rem_dest = pp_client_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);
} else
- rem_dest = pp_server_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);
}
} else {
if (user_param.duplex) {
@@ -1298,9 +1470,9 @@ int main(int argc, char *argv[])
/* close sockets */
if (user_param.servername)
- rem_dest = pp_client_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);
else
- rem_dest = pp_server_exch_dest(sockfd, &my_dest);
+ rem_dest = pp_server_exch_dest(sockfd, &my_dest, &user_param);
if (write(sockfd, "done", sizeof "done") != sizeof "done"){
perror("write");
--- a/send_lat.c
+++ b/send_lat.c
@@ -82,6 +82,7 @@ struct user_parameters {
int inline_size;
int use_mcg;
int qp_timeout;
+ int gid_index; /* if value not negative, we use gid AND gid_index=value */
};
struct report_options {
@@ -109,6 +110,7 @@ struct pingpong_context {
volatile char *poll_buf;
int size;
int tx_depth;
+ union ibv_gid dgid;
};
struct pingpong_dest {
@@ -117,6 +119,7 @@ struct pingpong_dest {
int qpn;
int psn;
unsigned int rkey;
+ union ibv_gid dgid;
};
@@ -152,13 +155,33 @@ static struct ibv_device *pp_find_dev(const char *ib_devname) {
#define KEY_MSG_SIZE (sizeof "0000:000000:000000:00000000:0000000000000000")
#define KEY_PRINT_FMT "%04x:%06x:%06x:%08x:%016Lx"
+#define KEY_MSG_SIZE_GID (sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00")
+#define KEY_PRINT_FMT_GID "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x"
-static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest)
+static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm)
{
- char msg[KEY_MSG_SIZE];
+ if (user_parm->gid_index < 0) {
+ char msg[KEY_MSG_SIZE];
- sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,
- my_dest->psn, my_dest->rkey, my_dest->vaddr);
+ sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,
+ my_dest->psn, my_dest->rkey, my_dest->vaddr);
+
+ if (write(sockfd, msg, sizeof msg) != sizeof msg) {
+ perror("client write");
+ fprintf(stderr, "Couldn't send local address\n");
+ return -1;
+ }
+
+ return 0;
+ } else {
+ char msg[KEY_MSG_SIZE_GID];
+
+ sprintf(msg, KEY_PRINT_FMT_GID, my_dest->lid, my_dest->qpn,
+ my_dest->psn, my_dest->rkey, my_dest->vaddr,
+ my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2], my_dest->dgid.raw[3],
+ my_dest->dgid.raw[4], my_dest->dgid.raw[5], my_dest->dgid.raw[6], my_dest->dgid.raw[7],
+ my_dest->dgid.raw[8], my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],
+ my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14], my_dest->dgid.raw[15]);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
perror("client write");
@@ -167,30 +190,84 @@ static int pp_write_keys(int sockfd, const struct pingpong_dest *my_dest)
}
return 0;
+ }
}
static int pp_read_keys(int sockfd, const struct pingpong_dest *my_dest,
- struct pingpong_dest *rem_dest)
+ struct pingpong_dest *rem_dest, struct user_parameters *user_parm)
{
- int parsed;
- char msg[KEY_MSG_SIZE];
+ if (user_parm->gid_index < 0) {
+ int parsed;
+ char msg[KEY_MSG_SIZE];
+
+ if (read(sockfd, msg, sizeof msg) != sizeof msg) {
+ perror("pp_read_keys");
+ fprintf(stderr, "Couldn't read remote address\n");
+ return -1;
+ }
- if (read(sockfd, msg, sizeof msg) != sizeof msg) {
- perror("pp_read_keys");
- fprintf(stderr, "Couldn't read remote address\n");
- return -1;
- }
+ parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn,
+ &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
- parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid, &rem_dest->qpn,
- &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
+ if (parsed != 5) {
+ fprintf(stderr, "Couldn't parse line <%.*s>\n",
+ (int)sizeof msg, msg);
+ return -1;
+ }
- if (parsed != 5) {
- fprintf(stderr, "Couldn't parse line <%.*s>\n",
- (int)sizeof msg, msg);
- return -1;
- }
+ return 0;
+ } else {
+ char msg[KEY_MSG_SIZE_GID];
+ if (read(sockfd, msg, sizeof msg) != sizeof msg) {
+ perror("pp_read_keys");
+ fprintf(stderr, "Couldn't read remote address\n");
+ return -1;
+ }
+ char *pstr = msg, *term;
+ char tmp[20];
+ int i;
- return 0;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
+
+ for (i = 0; i < 15; ++i) {
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);
+ }
+ pstr += term - pstr + 1;
+ strcpy(tmp, pstr);
+ rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);
+ return 0;
+ }
}
static int pp_client_connect(const char *servername, int port)
@@ -235,12 +312,12 @@ static int pp_client_connect(const char *servername, int port)
}
static int pp_client_exch_dest(int sockfd, const struct pingpong_dest *my_dest,
- struct pingpong_dest *rem_dest)
+ struct pingpong_dest *rem_dest, struct user_parameters *user_parm)
{
- if (pp_write_keys(sockfd, my_dest))
+ if (pp_write_keys(sockfd, my_dest, user_parm))
return -1;
- return pp_read_keys(sockfd, my_dest, rem_dest);
+ return pp_read_keys(sockfd, my_dest, rem_dest, user_parm);
}
static int pp_server_connect(int port)
@@ -300,13 +377,13 @@ static int pp_server_connect(int port)
}
static int pp_server_exch_dest(int sockfd, const struct pingpong_dest *my_dest,
- struct pingpong_dest* rem_dest)
+ struct pingpong_dest* rem_dest, struct user_parameters *user_parm)
{
- if (pp_read_keys(sockfd, my_dest, rem_dest))
+ if (pp_read_keys(sockfd, my_dest, rem_dest, user_parm))
return -1;
- return pp_write_keys(sockfd, my_dest);
+ return pp_write_keys(sockfd, my_dest, user_parm);
}
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
@@ -351,7 +428,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
fprintf(stderr, "Failed to query device props");
return NULL;
}
- if (device_attr.vendor_part_id == 23108) {
+ if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) {
user_parm->mtu = 1024;
} else {
user_parm->mtu = 2048;
@@ -522,9 +599,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
attr.min_rnr_timer = 12;
}
- attr.ah_attr.is_global = 0;
- attr.ah_attr.dlid = dest->lid;
- attr.ah_attr.sl = sl;
+ if (user_parm->gid_index < 0) {
+ attr.ah_attr.is_global = 0;
+ attr.ah_attr.dlid = dest->lid;
+ attr.ah_attr.sl = sl;
+ } else {
+ attr.ah_attr.is_global = 1;
+ attr.ah_attr.grh.dgid = dest->dgid;
+ attr.ah_attr.grh.hop_limit = 1;
+ attr.ah_attr.sl = 0;
+ }
attr.ah_attr.src_path_bits = 0;
attr.ah_attr.port_num = port;
if ((user_parm->connection_type==UD) && (user_parm->use_mcg)) {
@@ -538,9 +622,6 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.sgid_index = 0;
memcpy(attr.ah_attr.grh.dgid.raw, mcg_gid, 16);
- } else {
- attr.ah_attr.dlid = dest->lid;
- attr.ah_attr.is_global = 0;
}
if (user_parm->connection_type==RC) {
@@ -637,6 +718,7 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,
struct pingpong_dest my_dest;
int sockfd;
int rc;
+ union ibv_gid gid;
/* Create connection between client and server.
@@ -645,14 +727,34 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,
my_dest.lid = pp_get_local_lid(ctx, ib_port);
my_dest.qpn = ctx->qp->qp_num;
my_dest.psn = lrand48() & 0xffffff;
- if (!my_dest.lid) {
- fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
- return -1;
+ if (user_parm->gid_index < 0) {/*We do not fail test upon lid in RDMA0E/Eth conf*/
+ if (!my_dest.lid) {
+ fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
+ return -1;
+ }
}
+ if (user_parm->gid_index != -1) {
+ int err=0;
+ err = ibv_query_gid (ctx->context, ib_port, user_parm->gid_index, &gid);
+ if (err) {
+ return -1;
+ }
+ ctx->dgid=gid;
+ }
+ my_dest.dgid = gid;
my_dest.rkey = ctx->mr->rkey;
my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;
printf(addr_fmt, "local", my_dest.lid, my_dest.qpn, my_dest.psn);
+ if (user_parm->gid_index > -1) {
+ printf(" GID: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ my_dest.dgid.raw[0],my_dest.dgid.raw[1],
+ my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4],
+ my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7],
+ my_dest.dgid.raw[8], my_dest.dgid.raw[9], my_dest.dgid.raw[10],
+ my_dest.dgid.raw[11], my_dest.dgid.raw[12], my_dest.dgid.raw[13],
+ my_dest.dgid.raw[14], my_dest.dgid.raw[15]);
+ }
sockfd = servername ? pp_client_connect(servername, port) :
pp_server_connect(port);
@@ -663,13 +765,22 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,
return sockfd;
}
- rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest) :
- pp_server_exch_dest(sockfd, &my_dest, rem_dest);
+ rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest, user_parm) :
+ pp_server_exch_dest(sockfd, &my_dest, rem_dest, user_parm);
if (rc)
return rc;
printf(addr_fmt, "remote", rem_dest->lid, rem_dest->qpn, rem_dest->psn,
rem_dest->rkey, rem_dest->vaddr);
+ if (user_parm->gid_index > -1) {
+ printf(" GID: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],
+ rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4],
+ rem_dest->dgid.raw[5], rem_dest->dgid.raw[6], rem_dest->dgid.raw[7],
+ rem_dest->dgid.raw[8], rem_dest->dgid.raw[9], rem_dest->dgid.raw[10],
+ rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13],
+ rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);
+ }
if ((rc = pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest,user_parm)))
return rc;
@@ -678,8 +789,8 @@ static int pp_open_port(struct pingpong_context *ctx, const char * servername,
* Arbitrarily reuse exch_dest for this purpose.
*/
- rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest) :
- pp_server_exch_dest(sockfd, &my_dest, rem_dest);
+ rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest, user_parm) :
+ pp_server_exch_dest(sockfd, &my_dest, rem_dest, user_parm);
if (rc)
return rc;
@@ -714,6 +825,7 @@ static void usage(const char *argv0)
printf(" -I, --inline_size=<size> max size of message to be sent in inline mode (default 400)\n");
printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n");
printf(" -S, --sl=<sl> SL (default 0)\n");
+ printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n");
printf(" -C, --report-cycles report times in cpu cycle units (default microseconds)\n");
printf(" -H, --report-histogram print out all results (default print summary only)\n");
printf(" -U, --report-unsorted (implies -H) print out unsorted results (default sorted)\n");
@@ -814,7 +926,11 @@ int run_iter(struct pingpong_context *ctx, struct user_parameters *user_param,
if (user_param->connection_type==UD) {
if (size > 2048) {
- size = 2048;
+ if (user_param->gid_index < 0) {
+ size = 2048;
+ } else {
+ size = 1024;
+ }
}
}
@@ -1011,6 +1127,7 @@ int main(int argc, char *argv[])
user_param.inline_size = MAX_INLINE;
user_param.signal_comp = 0;
user_param.qp_timeout = 14;
+ user_param.gid_index = -1; /*gid will not be used*/
/* Parameter parsing. */
while (1) {
int c;
@@ -1027,6 +1144,7 @@ int main(int argc, char *argv[])
{ .name = "inline_size", .has_arg = 1, .val = 'I' },
{ .name = "qp-timeout", .has_arg = 1, .val = 'u' },
{ .name = "sl", .has_arg = 1, .val = 'S' },
+ { .name = "gid-index", .has_arg = 1, .val = 'x' },
{ .name = "signal", .has_arg = 0, .val = 'l' },
{ .name = "all", .has_arg = 0, .val = 'a' },
{ .name = "report-cycles", .has_arg = 0, .val = 'C' },
@@ -1038,7 +1156,7 @@ int main(int argc, char *argv[])
{ .name = "CPU-freq", .has_arg = 0, .val = 'F' },
{ 0 }
};
- c = getopt_long(argc, argv, "p:c:m:d:i:s:n:t:I:u:S:laeCHUVgF", long_options, NULL);
+ c = getopt_long(argc, argv, "p:c:m:d:i:s:n:t:I:u:S:x:laeCHUVgF", long_options, NULL);
if (c == -1)
break;
@@ -1095,6 +1213,14 @@ int main(int argc, char *argv[])
}
break;
+ case 'x':
+ user_param.gid_index = strtol(optarg, NULL, 0);
+ if (user_param.gid_index > 63) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+
case 't':
user_param.tx_depth = strtol(optarg, NULL, 0);
if (user_param.tx_depth < 1) {
@@ -1179,10 +1305,21 @@ int main(int argc, char *argv[])
} else {
printf("Connection type : UD\n");
}
+ if (user_param.gid_index > -1) {
+ printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n");
+ }
if (user_param.all == 1) {
/*since we run all sizes lets allocate big enough buffer */
size = 8388608; /*2^23 */
}
+ if (user_param.connection_type == UD && size > 2048) {
+ printf("Max msg size in UD is 2048 changing to 2048\n");
+ size = 2048;
+ }
+ if (user_param.connection_type == UD && user_param.gid_index > -1 && size > 1024) {
+ printf("Max msg size in UD RDMAoE is 1024. changing to 1024\n");
+ size = 1024;
+ }
srand48(getpid() * time(NULL));
page_size = sysconf(_SC_PAGESIZE);
@@ -1214,7 +1351,11 @@ int main(int argc, char *argv[])
if (user_param.all == 1) {
if (user_param.connection_type==UD) {
- size_max_pow = 12;
+ if (user_param.gid_index < 0) {
+ size_max_pow = 12;
+ } else {
+ size_max_pow = 11;
+ }
}
for (i = 1; i < size_max_pow ; ++i) {
size = 1 << i;
--- a/write_bw_postlist.c
+++ b/write_bw_postlist.c
@@ -75,6 +75,7 @@ struct user_parameters {
int maxpostsofqpiniteration;
int inline_size;
int qp_timeout;
+ int gid_index; /* if value not negative, we use gid AND gid_index=value */
};
struct extended_qp {
struct ibv_qp *qp;
@@ -98,6 +99,7 @@ struct pingpong_context {
struct ibv_send_wr wr;
int *scnt;
int *ccnt ;
+ union ibv_gid dgid;
};
struct pingpong_dest {
@@ -106,6 +108,7 @@ struct pingpong_dest {
int psn;
unsigned rkey;
unsigned long long vaddr;
+ union ibv_gid dgid;
};
@@ -160,14 +163,20 @@ static int pp_client_connect(const char *servername, int port)
}
struct pingpong_dest * pp_client_exch_dest(int sockfd,
- const struct pingpong_dest *my_dest)
+ const struct pingpong_dest *my_dest, struct user_parameters *user_parm)
{
struct pingpong_dest *rem_dest = NULL;
- char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
+ char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
int parsed;
- sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,
- my_dest->psn,my_dest->rkey,my_dest->vaddr);
+ sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
+ my_dest->lid, my_dest->qpn, my_dest->psn,my_dest->rkey,my_dest->vaddr,
+ my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],
+ my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],
+ my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],
+ my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],
+ my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],
+ my_dest->dgid.raw[15]);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
perror("client write");
fprintf(stderr, "Couldn't send local address\n");
@@ -184,15 +193,59 @@ struct pingpong_dest * pp_client_exch_dest(int sockfd,
if (!rem_dest)
goto out;
- parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
- &rem_dest->psn,&rem_dest->rkey,&rem_dest->vaddr);
-
- if (parsed != 5) {
- fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,
- msg);
- free(rem_dest);
- rem_dest = NULL;
- goto out;
+ if (user_parm->gid_index < 0) {
+ parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
+ &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
+ if (parsed != 5) {
+ fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);
+ free(rem_dest);
+ rem_dest = NULL;
+ goto out;
+ }
+ }else{
+ char *pstr = msg, *term;
+ char tmp[20];
+ int i;
+
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
+
+ for (i = 0; i < 15; ++i) {
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);
+ }
+ pstr += term - pstr + 1;
+ strcpy(tmp, pstr);
+ rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);
}
out:
return rem_dest;
@@ -254,9 +307,9 @@ int pp_server_connect(int port)
return connfd;
}
-static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest)
+static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpong_dest *my_dest, struct user_parameters *user_parm)
{
- char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
+ char msg[sizeof "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
struct pingpong_dest *rem_dest = NULL;
int parsed;
int n;
@@ -272,18 +325,69 @@ static struct pingpong_dest *pp_server_exch_dest(int connfd, const struct pingpo
if (!rem_dest)
goto out;
- parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
- &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
- if (parsed != 5) {
- fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg,
- msg);
- free(rem_dest);
- rem_dest = NULL;
- goto out;
+ if (user_parm->gid_index < 0) {
+ parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid, &rem_dest->qpn,
+ &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr);
+ if (parsed != 5) {
+ fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);
+ free(rem_dest);
+ rem_dest = NULL;
+ goto out;
+ }
+ }else{
+ char *pstr = msg, *term;
+ char tmp[20];
+ int i;
+
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
+
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
+
+ for (i = 0; i < 15; ++i) {
+ pstr += term - pstr + 1;
+ term = strpbrk(pstr, ":");
+ memcpy(tmp, pstr, term - pstr);
+ tmp[term - pstr] = 0;
+ rem_dest->dgid.raw[i] = (unsigned char)strtoll(tmp, NULL, 16);
+ }
+ pstr += term - pstr + 1;
+ strcpy(tmp, pstr);
+ rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp, NULL, 16);
}
- sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid, my_dest->qpn,
- my_dest->psn, my_dest->rkey, my_dest->vaddr);
+ sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
+ my_dest->lid, my_dest->qpn, my_dest->psn, my_dest->rkey, my_dest->vaddr,
+ my_dest->dgid.raw[0], my_dest->dgid.raw[1], my_dest->dgid.raw[2],
+ my_dest->dgid.raw[3], my_dest->dgid.raw[4], my_dest->dgid.raw[5],
+ my_dest->dgid.raw[6], my_dest->dgid.raw[7], my_dest->dgid.raw[8],
+ my_dest->dgid.raw[9], my_dest->dgid.raw[10], my_dest->dgid.raw[11],
+ my_dest->dgid.raw[12], my_dest->dgid.raw[13], my_dest->dgid.raw[14],
+ my_dest->dgid.raw[15]);
if (write(connfd, msg, sizeof msg) != sizeof msg) {
perror("server write");
fprintf(stderr, "Couldn't send local address\n");
@@ -341,7 +445,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev,
fprintf(stderr, "Failed to query device props");
return NULL;
}
- if (device_attr.vendor_part_id == 23108) {
+ if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) {
user_parm->mtu = 1024;
} else {
user_parm->mtu = 2048;
@@ -444,9 +548,16 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
attr.max_dest_rd_atomic = 1;
attr.min_rnr_timer = 12;
}
- attr.ah_attr.is_global = 0;
- attr.ah_attr.dlid = dest->lid;
- attr.ah_attr.sl = sl;
+ if (user_parm->gid_index < 0) {
+ attr.ah_attr.is_global = 0;
+ attr.ah_attr.dlid = dest->lid;
+ attr.ah_attr.sl = sl;
+ } else {
+ attr.ah_attr.is_global = 1;
+ attr.ah_attr.grh.dgid = dest->dgid;
+ attr.ah_attr.grh.hop_limit = 1;
+ attr.ah_attr.sl = 0;
+ }
attr.ah_attr.src_path_bits = 0;
attr.ah_attr.port_num = port;
if (user_parm->connection_type == RC) {
@@ -524,6 +635,7 @@ static void usage(const char *argv0)
printf(" -I, --inline_size=<size> max size of message to be sent in inline mode (default 400)\n");
printf(" -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec * 2 ^(timeout), default 14\n");
printf(" -S, --sl=<sl> SL (default 0)\n");
+ printf(" -x, --gid-index=<index> test uses GID with GID index taken from command line (for RDMAoE index should be 0)\n");
printf(" -b, --bidirectional measure bidirectional bandwidth (default unidirectional)\n");
printf(" -V, --version display version number\n");
printf(" -F, --CPU-freq do not fail even if cpufreq_ondemand module is loaded\n");
@@ -685,6 +797,7 @@ int main(int argc, char *argv[])
int inline_given_in_cmd = 0;
struct ibv_context *context;
int no_cpu_freq_fail = 0;
+ union ibv_gid gid;
/* init default values to user's parameters */
memset(&user_param, 0, sizeof(struct user_parameters));
@@ -696,6 +809,7 @@ int main(int argc, char *argv[])
user_param.maxpostsofqpiniteration = 100;
user_param.inline_size = MAX_INLINE;
user_param.qp_timeout = 14;
+ user_param.gid_index = -1; /*gid will not be used*/
/* Parameter parsing. */
while (1) {
int c;
@@ -714,6 +828,7 @@ int main(int argc, char *argv[])
{ .name = "inline_size", .has_arg = 1, .val = 'I' },
{ .name = "qp-timeout", .has_arg = 1, .val = 'u' },
{ .name = "sl", .has_arg = 1, .val = 'S' },
+ { .name = "gid-index", .has_arg = 1, .val = 'x' },
{ .name = "all", .has_arg = 0, .val = 'a' },
{ .name = "bidirectional", .has_arg = 0, .val = 'b' },
{ .name = "version", .has_arg = 0, .val = 'V' },
@@ -721,7 +836,7 @@ int main(int argc, char *argv[])
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:m:q:g:c:s:n:t:I:u:S:baVF", long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:m:q:g:c:s:n:t:I:u:S:x:baVF", long_options, NULL);
if (c == -1)
break;
@@ -815,6 +930,14 @@ int main(int argc, char *argv[])
if (sl > 15) { usage(argv[0]); return 1; }
break;
+ case 'x':
+ user_param.gid_index = strtol(optarg, NULL, 0);
+ if (user_param.gid_index > 63) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+
default:
usage(argv[0]);
return 1;
@@ -847,6 +970,9 @@ int main(int argc, char *argv[])
} else {
printf("Each Qp will post %d messages each time\n",user_param.maxpostsofqpiniteration);
}
+ if (user_param.gid_index > -1) {
+ printf("Using GID to support RDMAoE configuration. Refer to port type as Ethernet, default MTU 1024B\n");
+ }
/* Done with parameter parsing. Perform setup. */
if (user_param.all == ALL) {
/*since we run all sizes */
@@ -892,6 +1018,15 @@ int main(int argc, char *argv[])
if (!ctx)
return 1;
+ if (user_param.gid_index != -1) {
+ int err=0;
+ err = ibv_query_gid (ctx->context, ib_port, user_param.gid_index, &gid);
+ if (err) {
+ return -1;
+ }
+ ctx->dgid=gid;
+ }
+
if (user_param.servername) {
sockfd = pp_client_connect(user_param.servername, port);
@@ -911,10 +1046,13 @@ int main(int argc, char *argv[])
* We do it by exchanging data over a TCP socket connection. */
my_dest[i].lid = pp_get_local_lid(ctx, ib_port);
my_dest[i].psn = lrand48() & 0xffffff;
- if (!my_dest[i].lid) {
- fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
- return 1;
+ if (user_param.gid_index < 0) {/*We do not fail test upon lid in RDMA0E/Eth conf*/
+ if (!my_dest[i].lid) {
+ fprintf(stderr, "Local lid 0x0 detected. Is an SM running? If you are running on an RMDAoE interface you must use GIDs\n");
+ return 1;
+ }
}
+ my_dest[i].dgid = gid;
my_dest[i].qpn = ctx->qp[i]->qp_num;
/* TBD this should be changed into VA and diffreent key to each qp */
my_dest[i].rkey = ctx->mr->rkey;
@@ -924,10 +1062,19 @@ int main(int argc, char *argv[])
"RKey %#08x VAddr %#016Lx\n",
my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn,
my_dest[i].rkey, my_dest[i].vaddr);
+ if (user_param.gid_index > -1) {
+ printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ my_dest[i].dgid.raw[0],my_dest[i].dgid.raw[1],
+ my_dest[i].dgid.raw[2], my_dest[i].dgid.raw[3], my_dest[i].dgid.raw[4],
+ my_dest[i].dgid.raw[5], my_dest[i].dgid.raw[6], my_dest[i].dgid.raw[7],
+ my_dest[i].dgid.raw[8], my_dest[i].dgid.raw[9], my_dest[i].dgid.raw[10],
+ my_dest[i].dgid.raw[11], my_dest[i].dgid.raw[12], my_dest[i].dgid.raw[13],
+ my_dest[i].dgid.raw[14], my_dest[i].dgid.raw[15]);
+ }
if (user_param.servername) {
- rem_dest[i] = pp_client_exch_dest(sockfd, &my_dest[i]);
+ rem_dest[i] = pp_client_exch_dest(sockfd, &my_dest[i], &user_param);
} else {
- rem_dest[i] = pp_server_exch_dest(sockfd, &my_dest[i]);
+ rem_dest[i] = pp_server_exch_dest(sockfd, &my_dest[i], &user_param);
}
if (!rem_dest[i])
return 1;
@@ -935,15 +1082,24 @@ int main(int argc, char *argv[])
"RKey %#08x VAddr %#016Lx\n",
rem_dest[i]->lid, rem_dest[i]->qpn, rem_dest[i]->psn,
rem_dest[i]->rkey, rem_dest[i]->vaddr);
+ if (user_param.gid_index > -1) {
+ printf(" GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ rem_dest[i]->dgid.raw[0],rem_dest[i]->dgid.raw[1],
+ rem_dest[i]->dgid.raw[2], rem_dest[i]->dgid.raw[3], rem_dest[i]->dgid.raw[4],
+ rem_dest[i]->dgid.raw[5], rem_dest[i]->dgid.raw[6], rem_dest[i]->dgid.raw[7],
+ rem_dest[i]->dgid.raw[8], rem_dest[i]->dgid.raw[9], rem_dest[i]->dgid.raw[10],
+ rem_dest[i]->dgid.raw[11], rem_dest[i]->dgid.raw[12], rem_dest[i]->dgid.raw[13],
+ rem_dest[i]->dgid.raw[14], rem_dest[i]->dgid.raw[15]);
+ }
if (pp_connect_ctx(ctx, ib_port, my_dest[i].psn, rem_dest[i], &user_param, i))
return 1;
/* An additional handshake is required *after* moving qp to RTR.
Arbitrarily reuse exch_dest for this purpose. */
if (user_param.servername) {
- rem_dest[i] = pp_client_exch_dest(sockfd, &my_dest[i]);
+ rem_dest[i] = pp_client_exch_dest(sockfd, &my_dest[i], &user_param);
} else {
- rem_dest[i] = pp_server_exch_dest(sockfd, &my_dest[i]);
+ rem_dest[i] = pp_server_exch_dest(sockfd, &my_dest[i], &user_param);
}
}
@@ -952,7 +1108,7 @@ int main(int argc, char *argv[])
/* For half duplex tests, server just waits for client to exit */
/* the 0th place is arbitrary to signal finish ... */
if (!user_param.servername && !duplex) {
- rem_dest[0] = pp_server_exch_dest(sockfd, &my_dest[0]);
+ rem_dest[0] = pp_server_exch_dest(sockfd, &my_dest[0], &user_param);
if (write(sockfd, "done", sizeof "done") != sizeof "done"){
perror("server write");
fprintf(stderr, "Couldn't write to socket\n");
@@ -990,9 +1146,9 @@ int main(int argc, char *argv[])
}
/* the 0th place is arbitrary to signal finish ... */
if (user_param.servername) {
- rem_dest[0] = pp_client_exch_dest(sockfd, &my_dest[0]);
+ rem_dest[0] = pp_client_exch_dest(sockfd, &my_dest[0], &user_param);
} else {
- rem_dest[0] = pp_server_exch_dest(sockfd, &my_dest[0]);
+ rem_dest[0] = pp_server_exch_dest(sockfd, &my_dest[0], &user_param);
}
if (write(sockfd, "done", sizeof "done") != sizeof "done"){
More information about the ewg
mailing list