[ewg] [PATCH] Perftest: support RDMAoE

Hal Rosenstock hal.rosenstock at gmail.com
Tue Aug 4 03:54:08 PDT 2009


Hi,

I'm resending this hoping that second time is the charm :-) Thanks.

-- Hal
On Mon, Jul 20, 2009 at 10:09 AM, Hal Rosenstock
<hal.rosenstock at gmail.com>wrote:

> Two questions/comments:
>
> If these are now no longer IB specific, should the executable names be
> changed from ib_xxx to rdma_xxx ?
>
> Isn't the GID parameter also applicable to IB and iWARP ? If so, will
> it work for those configurations too ? It looks to me like some things
> use this to determine RDMAoE.
>
> -- Hal
>
> On Mon, Jul 20, 2009 at 4:48 AM, orenmeron<orenmeron at dev.mellanox.co.il>
> wrote:
> > --- a/read_bw.c
> > +++ b/read_bw.c
> > @@ -71,6 +71,7 @@ struct user_parameters {
> >        int max_out_read;
> >        int use_event;
> >        int qp_timeout;
> > +       int gid_index; /* if value not negative, we use gid AND
> gid_index=value */
> >  };
> >  static int sl = 0;
> >  static int page_size;
> > @@ -88,6 +89,7 @@ struct pingpong_context {
> >        int                 tx_depth;
> >        struct ibv_sge      list;
> >        struct ibv_send_wr  wr;
> > +       union ibv_gid       dgid;
> >  };
> >
> >  struct pingpong_dest {
> > @@ -96,6 +98,7 @@ struct pingpong_dest {
> >        int psn;
> >        unsigned rkey;
> >        unsigned long long vaddr;
> > +       union ibv_gid       dgid;
> >  };
> >
> >
> > @@ -150,14 +153,20 @@ static int pp_client_connect(const char
> *servername, int port)
> >  }
> >
> >  struct pingpong_dest * pp_client_exch_dest(int sockfd,
> > -                                          const struct pingpong_dest
> *my_dest)
> > +                                          const struct pingpong_dest
> *my_dest, struct user_parameters *user_parm)
> >  {
> >        struct pingpong_dest *rem_dest = NULL;
> > -       char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
> > +       char msg[sizeof
> "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
> >        int parsed;
> >
> > -       sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid,
> my_dest->qpn,
> > -               my_dest->psn,my_dest->rkey,my_dest->vaddr);
> > +       sprintf(msg,
> "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
> > +               my_dest->lid, my_dest->qpn,
> my_dest->psn,my_dest->rkey,my_dest->vaddr,
> > +               my_dest->dgid.raw[0], my_dest->dgid.raw[1],
> my_dest->dgid.raw[2],
> > +               my_dest->dgid.raw[3], my_dest->dgid.raw[4],
> my_dest->dgid.raw[5],
> > +               my_dest->dgid.raw[6], my_dest->dgid.raw[7],
> my_dest->dgid.raw[8],
> > +               my_dest->dgid.raw[9], my_dest->dgid.raw[10],
> my_dest->dgid.raw[11],
> > +               my_dest->dgid.raw[12], my_dest->dgid.raw[13],
> my_dest->dgid.raw[14],
> > +               my_dest->dgid.raw[15]);
> >        if (write(sockfd, msg, sizeof msg) != sizeof msg) {
> >                perror("client write");
> >                fprintf(stderr, "Couldn't send local address\n");
> > @@ -174,15 +183,59 @@ struct pingpong_dest * pp_client_exch_dest(int
> sockfd,
> >        if (!rem_dest)
> >                goto out;
> >
> > -       parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > -                       &rem_dest->psn,&rem_dest->rkey,&rem_dest->vaddr);
> > -
> > -       if (parsed != 5) {
> > -               fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg,
> > -                       msg);
> > -               free(rem_dest);
> > -               rem_dest = NULL;
> > -               goto out;
> > +       if (user_parm->gid_index < 0) {
> > +               parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > +                               &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > +               if (parsed != 5) {
> > +                       fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg, msg);
> > +                       free(rem_dest);
> > +                       rem_dest = NULL;
> > +                       goto out;
> > +               }
> > +       }else{
> > +               char *pstr = msg, *term;
> > +               char tmp[20];
> > +               int i;
> > +
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
> > +
> > +               for (i = 0; i < 15; ++i) {
> > +                       pstr += term - pstr + 1;
> > +                       term = strpbrk(pstr, ":");
> > +                       memcpy(tmp, pstr, term - pstr);
> > +                       tmp[term - pstr] = 0;
> > +                       rem_dest->dgid.raw[i] = (unsigned
> char)strtoll(tmp, NULL, 16);
> > +               }
> > +               pstr += term - pstr + 1;
> > +               strcpy(tmp, pstr);
> > +               rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp,
> NULL, 16);
> >        }
> >  out:
> >        return rem_dest;
> > @@ -244,9 +297,9 @@ int pp_server_connect(int port)
> >        return connfd;
> >  }
> >
> > -static struct pingpong_dest *pp_server_exch_dest(int connfd, const
> struct pingpong_dest *my_dest)
> > +static struct pingpong_dest *pp_server_exch_dest(int connfd, const
> struct pingpong_dest *my_dest, struct user_parameters *user_parm)
> >  {
> > -       char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
> > +       char msg[sizeof
> "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
> >        struct pingpong_dest *rem_dest = NULL;
> >        int parsed;
> >        int n;
> > @@ -262,18 +315,69 @@ static struct pingpong_dest
> *pp_server_exch_dest(int connfd, const struct pingpo
> >        if (!rem_dest)
> >                goto out;
> >
> > -       parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > -                       &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > -       if (parsed != 5) {
> > -               fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg,
> > -                       msg);
> > -               free(rem_dest);
> > -               rem_dest = NULL;
> > -               goto out;
> > +       if (user_parm->gid_index < 0) {
> > +               parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > +                               &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > +               if (parsed != 5) {
> > +                       fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg, msg);
> > +                       free(rem_dest);
> > +                       rem_dest = NULL;
> > +                       goto out;
> > +               }
> > +       }else{
> > +               char *pstr = msg, *term;
> > +               char tmp[20];
> > +               int i;
> > +
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
> > +
> > +               for (i = 0; i < 15; ++i) {
> > +                       pstr += term - pstr + 1;
> > +                       term = strpbrk(pstr, ":");
> > +                       memcpy(tmp, pstr, term - pstr);
> > +                       tmp[term - pstr] = 0;
> > +                       rem_dest->dgid.raw[i] = (unsigned
> char)strtoll(tmp, NULL, 16);
> > +                       }
> > +               pstr += term - pstr + 1;
> > +               strcpy(tmp, pstr);
> > +               rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp,
> NULL, 16);
> >        }
> >
> > -       sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid,
> my_dest->qpn,
> > -               my_dest->psn, my_dest->rkey, my_dest->vaddr);
> > +       sprintf(msg,
> "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
> > +               my_dest->lid, my_dest->qpn, my_dest->psn, my_dest->rkey,
> my_dest->vaddr,
> > +               my_dest->dgid.raw[0], my_dest->dgid.raw[1],
> my_dest->dgid.raw[2],
> > +               my_dest->dgid.raw[3], my_dest->dgid.raw[4],
> my_dest->dgid.raw[5],
> > +               my_dest->dgid.raw[6], my_dest->dgid.raw[7],
> my_dest->dgid.raw[8],
> > +               my_dest->dgid.raw[9], my_dest->dgid.raw[10],
> my_dest->dgid.raw[11],
> > +               my_dest->dgid.raw[12], my_dest->dgid.raw[13],
> my_dest->dgid.raw[14],
> > +               my_dest->dgid.raw[15]);
> >        if (write(connfd, msg, sizeof msg) != sizeof msg) {
> >                perror("server write");
> >                fprintf(stderr, "Couldn't send local address\n");
> > @@ -318,7 +422,7 @@ static struct pingpong_context *pp_init_ctx(struct
> ibv_device *ib_dev,
> >                        fprintf(stderr, "Failed to query device props");
> >                        return NULL;
> >                }
> > -               if (device_attr.vendor_part_id == 23108)
> > +               if (device_attr.vendor_part_id == 23108 ||
> user_parm->gid_index > -1)
> >                        user_parm->mtu = 1024;
> >                else
> >                        user_parm->mtu = 2048;
> > @@ -421,9 +525,16 @@ static int pp_connect_ctx(struct pingpong_context
> *ctx, int port, int my_psn,
> >        attr.rq_psn             = dest->psn;
> >        attr.max_dest_rd_atomic     = user_parm->max_out_read;
> >        attr.min_rnr_timer          = 12;
> > -       attr.ah_attr.is_global  = 0;
> > -       attr.ah_attr.dlid       = dest->lid;
> > -       attr.ah_attr.sl         = sl;
> > +       if (user_parm->gid_index<0) {
> > +               attr.ah_attr.is_global  = 0;
> > +               attr.ah_attr.dlid       = dest->lid;
> > +               attr.ah_attr.sl         = sl;
> > +       } else {
> > +               attr.ah_attr.is_global  = 1;
> > +               attr.ah_attr.grh.dgid   = dest->dgid;
> > +               attr.ah_attr.grh.hop_limit = 1;
> > +               attr.ah_attr.sl         = 0;
> > +       }
> >        attr.ah_attr.src_path_bits = 0;
> >        attr.ah_attr.port_num   = port;
> >        if (ibv_modify_qp(ctx->qp, &attr,
> > @@ -474,6 +585,7 @@ static void usage(const char *argv0)
> >        printf("  -n, --iters=<iters>    number of exchanges (at least 2,
> default 1000)\n");
> >        printf("  -u, --qp-timeout=<timeout> QP timeout, timeout value is
> 4 usec * 2 ^(timeout), default 14\n");
> >        printf("  -S, --sl=<sl>          SL (default 0)\n");
> > +       printf("  -x, --gid-index=<index>   test uses GID with GID index
> taken from command line (for RDMAoE index should be 0)\n");
> >        printf("  -b, --bidirectional    measure bidirectional bandwidth
> (default unidirectional)\n");
> >        printf("  -V, --version          display version number\n");
> >        printf("  -e, --events           sleep on CQ events (default
> poll)\n");
> > @@ -609,6 +721,7 @@ int main(int argc, char *argv[])
> >        int                      duplex = 0;
> >        int                      i = 0;
> >        int                      no_cpu_freq_fail = 0;
> > +       union ibv_gid            gid;
> >
> >        /* init default values to user's parameters */
> >        memset(&user_param, 0, sizeof(struct user_parameters));
> > @@ -619,6 +732,7 @@ int main(int argc, char *argv[])
> >        user_param.use_event = 0;
> >        user_param.max_out_read = 4; /* the device capability on gen2 */
> >        user_param.qp_timeout = 14;
> > +       user_param.gid_index = -1; /*gid will not be used*/
> >        /* Parameter parsing. */
> >        while (1) {
> >                int c;
> > @@ -634,6 +748,7 @@ int main(int argc, char *argv[])
> >                        { .name = "tx-depth",       .has_arg = 1, .val =
> 't' },
> >                        { .name = "qp-timeout",     .has_arg = 1, .val =
> 'u' },
> >                        { .name = "sl",             .has_arg = 1, .val =
> 'S' },
> > +                       { .name = "gid-index",      .has_arg = 1, .val =
> 'x' },
> >                        { .name = "all",            .has_arg = 0, .val =
> 'a' },
> >                        { .name = "bidirectional",  .has_arg = 0, .val =
> 'b' },
> >                        { .name = "version",        .has_arg = 0, .val =
> 'V' },
> > @@ -642,7 +757,7 @@ int main(int argc, char *argv[])
> >                        { 0 }
> >                };
> >
> > -               c = getopt_long(argc, argv, "p:d:i:m:o:s:n:t:u:S:abVeF",
> long_options, NULL);
> > +               c = getopt_long(argc, argv,
> "p:d:i:m:o:s:n:t:u:S:x:abVeF", long_options, NULL);
> >                if (c == -1)
> >                        break;
> >
> > @@ -721,6 +836,14 @@ int main(int argc, char *argv[])
> >                        if (sl > 15) { usage(argv[0]); return 1; }
> >                        break;
> >
> > +               case 'x':
> > +                       user_param.gid_index = strtol(optarg, NULL, 0);
> > +                       if (user_param.gid_index > 63) {
> > +                               usage(argv[0]);
> > +                               return 1;
> > +                       }
> > +                       break;
> > +
> >                default:
> >                        usage(argv[0]);
> >                        return 1;
> > @@ -740,6 +863,10 @@ int main(int argc, char *argv[])
> >                printf("                    RDMA_Read BW Test\n");
> >
> >        printf("Connection type : RC\n");
> > +       if (user_param.gid_index > -1) {
> > +               printf("Using GID to support RDMAoE configuration. Refer
> to port type as Ethernet, default MTU 1024B\n");
> > +       }
> > +
> >        /* Done with parameter parsing. Perform setup. */
> >        if (user_param.all == ALL)
> >                /*since we run all sizes */
> > @@ -771,16 +898,28 @@ int main(int argc, char *argv[])
> >        if (!ctx)
> >                return 1;
> >
> > +       if (user_param.gid_index != -1) {
> > +               int err=0;
> > +               err = ibv_query_gid (ctx->context, ib_port,
> user_param.gid_index, &gid);
> > +               if (err) {
> > +                       return -1;
> > +               }
> > +               ctx->dgid=gid;
> > +               }
> > +
> >        /* Create connection between client and server.
> >         * We do it by exchanging data over a TCP socket connection. */
> >
> >        my_dest.lid = pp_get_local_lid(ctx, ib_port);
> >        my_dest.qpn = ctx->qp->qp_num;
> >        my_dest.psn = lrand48() & 0xffffff;
> > -       if (!my_dest.lid) {
> > -               fprintf(stderr, "Local lid 0x0 detected. Is an SM
> running?\n");
> > -               return 1;
> > +       if (user_param.gid_index < 0) {/*We do not fail test upon lid in
> RDMA0E/Eth conf*/
> > +                       if (!my_dest.lid) {
> > +                               fprintf(stderr, "Local lid 0x0 detected.
> Is an SM running? If you are running on an RMDAoE interface you must use
> GIDs\n");
> > +                       return 1;
> > +               }
> >        }
> > +       my_dest.dgid = gid;
> >        my_dest.rkey = ctx->mr->rkey;
> >        my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;
> >
> > @@ -788,17 +927,26 @@ int main(int argc, char *argv[])
> >               "RKey %#08x VAddr %#016Lx\n",
> >               my_dest.lid, my_dest.qpn, my_dest.psn,
> >               my_dest.rkey, my_dest.vaddr);
> > +       if (user_param.gid_index > -1) {
> > +               printf("                  GID
> %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
> > +               my_dest.dgid.raw[0],my_dest.dgid.raw[1],
> > +               my_dest.dgid.raw[2], my_dest.dgid.raw[3],
> my_dest.dgid.raw[4],
> > +               my_dest.dgid.raw[5], my_dest.dgid.raw[6],
> my_dest.dgid.raw[7],
> > +               my_dest.dgid.raw[8], my_dest.dgid.raw[9],
> my_dest.dgid.raw[10],
> > +               my_dest.dgid.raw[11], my_dest.dgid.raw[12],
> my_dest.dgid.raw[13],
> > +               my_dest.dgid.raw[14], my_dest.dgid.raw[15]);
> > +       }
> >
> >        if (user_param.servername) {
> >                sockfd = pp_client_connect(user_param.servername, port);
> >                if (sockfd < 0)
> >                        return 1;
> > -               rem_dest = pp_client_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_client_exch_dest(sockfd, &my_dest,
> &user_param);
> >        } else {
> >                sockfd = pp_server_connect(port);
> >                if (sockfd < 0)
> >                        return 1;
> > -               rem_dest = pp_server_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_server_exch_dest(sockfd, &my_dest,
> &user_param);
> >        }
> >
> >        if (!rem_dest)
> > @@ -808,6 +956,15 @@ int main(int argc, char *argv[])
> >               "RKey %#08x VAddr %#016Lx\n",
> >               rem_dest->lid, rem_dest->qpn, rem_dest->psn,
> >               rem_dest->rkey, rem_dest->vaddr);
> > +       if (user_param.gid_index > -1) {
> > +               printf("                  GID
> %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
> > +               rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],
> > +               rem_dest->dgid.raw[2], rem_dest->dgid.raw[3],
> rem_dest->dgid.raw[4],
> > +               rem_dest->dgid.raw[5], rem_dest->dgid.raw[6],
> rem_dest->dgid.raw[7],
> > +               rem_dest->dgid.raw[8], rem_dest->dgid.raw[9],
> rem_dest->dgid.raw[10],
> > +               rem_dest->dgid.raw[11], rem_dest->dgid.raw[12],
> rem_dest->dgid.raw[13],
> > +               rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);
> > +       }
> >
> >        if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest,
> &user_param))
> >                return 1;
> > @@ -815,9 +972,9 @@ int main(int argc, char *argv[])
> >        /* An additional handshake is required *after* moving qp to RTR.
> >           Arbitrarily reuse exch_dest for this purpose. */
> >        if (user_param.servername)
> > -               rem_dest = pp_client_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_client_exch_dest(sockfd, &my_dest,
> &user_param);
> >        else
> > -               rem_dest = pp_server_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_server_exch_dest(sockfd, &my_dest,
> &user_param);
> >
> >        if (!rem_dest)
> >                return 1;
> > @@ -826,7 +983,7 @@ int main(int argc, char *argv[])
> >        /* For half duplex tests, server just waits for client to exit */
> >
> >        if (!user_param.servername && !duplex) {
> > -               rem_dest = pp_server_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_server_exch_dest(sockfd, &my_dest,
> &user_param);
> >                if (write(sockfd, "done", sizeof "done") != sizeof
> "done"){
> >                        perror("server write");
> >                        fprintf(stderr, "Couldn't write to socket\n");
> > @@ -873,9 +1030,9 @@ int main(int argc, char *argv[])
> >        }
> >
> >        if (user_param.servername)
> > -               rem_dest = pp_client_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_client_exch_dest(sockfd, &my_dest,
> &user_param);
> >        else
> > -               rem_dest = pp_server_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_server_exch_dest(sockfd, &my_dest,
> &user_param);
> >
> >        if (write(sockfd, "done", sizeof "done") != sizeof "done"){
> >                perror("server write");
> >
> > --- a/read_lat.c
> > +++ b/read_lat.c
> > @@ -75,6 +75,7 @@ struct user_parameters {
> >        int max_out_read;
> >        int use_event;
> >        int qp_timeout;
> > +       int gid_index; /* if value not negative, we use gid AND
> gid_index=value */
> >
> >  };
> >  struct report_options {
> > @@ -97,6 +98,7 @@ struct pingpong_context {
> >        int tx_depth;
> >        struct ibv_sge list;
> >        struct ibv_send_wr wr;
> > +       union ibv_gid       dgid;
> >  };
> >
> >  struct pingpong_dest {
> > @@ -105,6 +107,7 @@ struct pingpong_dest {
> >        int psn;
> >        unsigned rkey;
> >        unsigned long long vaddr;
> > +       union ibv_gid       dgid;
> >  };
> >
> >
> > @@ -140,14 +143,33 @@ static struct ibv_device *pp_find_dev(const char
> *ib_devname) {
> >
> >  #define KEY_MSG_SIZE (sizeof
> "0000:000000:000000:00000000:0000000000000000")
> >  #define KEY_PRINT_FMT "%04x:%06x:%06x:%08x:%016Lx"
> > +#define KEY_MSG_SIZE_GID (sizeof
> "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00")
> > +#define KEY_PRINT_FMT_GID
> "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x"
> >
> > -static int pp_write_keys(int sockfd, const struct pingpong_dest
> *my_dest)
> > +static int pp_write_keys(int sockfd, const struct pingpong_dest
> *my_dest, struct user_parameters *user_parm)
> >  {
> > -       char msg[KEY_MSG_SIZE];
> > +       if (user_parm->gid_index < 0) {
> > +               char msg[KEY_MSG_SIZE];
> >
> > -       sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,
> > -               my_dest->psn, my_dest->rkey, my_dest->vaddr);
> > +               sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,
> > +                       my_dest->psn, my_dest->rkey, my_dest->vaddr);
> >
> > +               if (write(sockfd, msg, sizeof msg) != sizeof msg) {
> > +                       perror("client write");
> > +                       fprintf(stderr, "Couldn't send local address\n");
> > +                       return -1;
> > +               }
> > +
> > +               return 0;
> > +       } else {
> > +       char msg[KEY_MSG_SIZE_GID];
> > +
> > +               sprintf(msg, KEY_PRINT_FMT_GID, my_dest->lid,
> my_dest->qpn,
> > +                       my_dest->psn, my_dest->rkey, my_dest->vaddr,
> > +                       my_dest->dgid.raw[0], my_dest->dgid.raw[1],
> my_dest->dgid.raw[2], my_dest->dgid.raw[3],
> > +                       my_dest->dgid.raw[4], my_dest->dgid.raw[5],
> my_dest->dgid.raw[6], my_dest->dgid.raw[7],
> > +                       my_dest->dgid.raw[8], my_dest->dgid.raw[9],
> my_dest->dgid.raw[10], my_dest->dgid.raw[11],
> > +                       my_dest->dgid.raw[12], my_dest->dgid.raw[13],
> my_dest->dgid.raw[14], my_dest->dgid.raw[15]);
> >        if (write(sockfd, msg, sizeof msg) != sizeof msg) {
> >                perror("client write");
> >                fprintf(stderr, "Couldn't send local address\n");
> > @@ -155,30 +177,84 @@ static int pp_write_keys(int sockfd, const struct
> pingpong_dest *my_dest)
> >        }
> >
> >        return 0;
> > +       }
> >  }
> >
> >  static int pp_read_keys(int sockfd, const struct pingpong_dest *my_dest,
> > -                       struct pingpong_dest *rem_dest)
> > +                       struct pingpong_dest *rem_dest, struct
> user_parameters *user_parm)
> >  {
> > -       int parsed;
> > -       char msg[KEY_MSG_SIZE];
> > +       if (user_parm->gid_index < 0) {
> > +               int parsed;
> > +               char msg[KEY_MSG_SIZE];
> > +
> > +               if (read(sockfd, msg, sizeof msg) != sizeof msg) {
> > +                       perror("pp_read_keys");
> > +                       fprintf(stderr, "Couldn't read remote
> address\n");
> > +                       return -1;
> > +               }
> >
> > -       if (read(sockfd, msg, sizeof msg) != sizeof msg) {
> > -               perror("pp_read_keys");
> > -               fprintf(stderr, "Couldn't read remote address\n");
> > -               return -1;
> > -       }
> > +               parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid,
> &rem_dest->qpn,
> > +                               &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> >
> > -       parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid,
> &rem_dest->qpn,
> > -                       &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > +               if (parsed != 5) {
> > +                       fprintf(stderr, "Couldn't parse line <%.*s>\n",
> > +                               (int)sizeof msg, msg);
> > +                       return -1;
> > +               }
> >
> > -       if (parsed != 5) {
> > -               fprintf(stderr, "Couldn't parse line <%.*s>\n",
> > -                       (int)sizeof msg, msg);
> > -               return -1;
> > +               return 0;
> > +       } else {
> > +               char msg[KEY_MSG_SIZE_GID];
> > +               if (read(sockfd, msg, sizeof msg) != sizeof msg) {
> > +                       perror("pp_read_keys");
> > +                       fprintf(stderr, "Couldn't read remote
> address\n");
> > +                       return -1;
> > +               }
> > +               char *pstr = msg, *term;
> > +               char tmp[20];
> > +               int i;
> > +
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
> > +
> > +               for (i = 0; i < 15; ++i) {
> > +                       pstr += term - pstr + 1;
> > +                       term = strpbrk(pstr, ":");
> > +                       memcpy(tmp, pstr, term - pstr);
> > +                       tmp[term - pstr] = 0;
> > +                       rem_dest->dgid.raw[i] = (unsigned
> char)strtoll(tmp, NULL, 16);
> > +                       }
> > +               pstr += term - pstr + 1;
> > +               strcpy(tmp, pstr);
> > +               rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp,
> NULL, 16);
> > +               return 0;
> >        }
> > -
> > -       return 0;
> >  }
> >
> >  static int pp_client_connect(const char *servername, int port)
> > @@ -223,12 +299,12 @@ static int pp_client_connect(const char
> *servername, int port)
> >  }
> >
> >  static int pp_client_exch_dest(int sockfd, const struct pingpong_dest
> *my_dest,
> > -                              struct pingpong_dest *rem_dest)
> > +                              struct pingpong_dest *rem_dest, struct
> user_parameters *user_parm)
> >  {
> > -       if (pp_write_keys(sockfd, my_dest))
> > +       if (pp_write_keys(sockfd, my_dest, user_parm))
> >                return -1;
> >
> > -       return pp_read_keys(sockfd, my_dest, rem_dest);
> > +       return pp_read_keys(sockfd, my_dest, rem_dest, user_parm);
> >  }
> >
> >  static int pp_server_connect(int port)
> > @@ -288,13 +364,13 @@ static int pp_server_connect(int port)
> >  }
> >
> >  static int pp_server_exch_dest(int sockfd, const struct pingpong_dest
> *my_dest,
> > -                              struct pingpong_dest* rem_dest)
> > +                              struct pingpong_dest* rem_dest, struct
> user_parameters *user_parm)
> >  {
> >
> > -       if (pp_read_keys(sockfd, my_dest, rem_dest))
> > +       if (pp_read_keys(sockfd, my_dest, rem_dest, user_parm))
> >                return -1;
> >
> > -       return pp_write_keys(sockfd, my_dest);
> > +       return pp_write_keys(sockfd, my_dest, user_parm);
> >  }
> >
> >  static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev,
> int size,
> > @@ -332,7 +408,7 @@ static struct pingpong_context *pp_init_ctx(struct
> ibv_device *ib_dev, int size,
> >                        fprintf(stderr, "Failed to query device props");
> >                        return NULL;
> >                }
> > -               if (device_attr.vendor_part_id == 23108) {
> > +               if (device_attr.vendor_part_id == 23108 ||
> user_parm->gid_index > -1) {
> >                        user_parm->mtu = 1024;
> >                } else {
> >                        user_parm->mtu = 2048;
> > @@ -444,9 +520,16 @@ static int pp_connect_ctx(struct pingpong_context
> *ctx, int port, int my_psn,
> >        attr.rq_psn                 = dest->psn;
> >        attr.max_dest_rd_atomic     = user_parm->max_out_read;
> >        attr.min_rnr_timer          = 12;
> > -       attr.ah_attr.is_global      = 0;
> > -       attr.ah_attr.dlid           = dest->lid;
> > -       attr.ah_attr.sl             = sl;
> > +       if (user_parm->gid_index < 0) {
> > +               attr.ah_attr.is_global      = 0;
> > +               attr.ah_attr.dlid           = dest->lid;
> > +               attr.ah_attr.sl             = sl;
> > +       } else {
> > +               attr.ah_attr.is_global      = 1;
> > +               attr.ah_attr.grh.dgid       = dest->dgid;
> > +               attr.ah_attr.grh.hop_limit  = 1;
> > +               attr.ah_attr.sl             = 0;
> > +       }
> >        attr.ah_attr.src_path_bits  = 0;
> >        attr.ah_attr.port_num       = port;
> >        if (ibv_modify_qp(ctx->qp, &attr,
> > @@ -497,23 +580,45 @@ static int pp_open_port(struct pingpong_context
> *ctx, const char * servername,
> >        char addr_fmt[] = "%8s address: LID %#04x QPN %#06x PSN %#06x RKey
> %#08x VAddr %#016Lx\n";
> >        int                   sockfd;
> >        int                   rc;
> > +       union ibv_gid         gid;
> >
> >
> >        /* Create connection between client and server.
> >         * We do it by exchanging data over a TCP socket connection. */
> >
> > +       if (user_parm->gid_index != -1) {
> > +               int err=0;
> > +               err = ibv_query_gid (ctx->context, ib_port,
> user_parm->gid_index, &gid);
> > +               if (err) {
> > +                       return -1;
> > +               }
> > +               ctx->dgid=gid;
> > +               }
> > +
> >        my_dest.lid = pp_get_local_lid(ctx, ib_port);
> > +       my_dest.dgid = gid;
> >        my_dest.qpn = ctx->qp->qp_num;
> >        my_dest.psn = lrand48() & 0xffffff;
> > -       if (!my_dest.lid) {
> > -               fprintf(stderr, "Local lid 0x0 detected. Is an SM
> running?\n");
> > -               return -1;
> > +       if (user_parm->gid_index < 0) {/*We do not fail test upon lid in
> RDMAoE/Eth conf*/
> > +                       if (!my_dest.lid) {
> > +                               fprintf(stderr, "Local lid 0x0 detected.
> Is an SM running? If you are running on an RMDAoE interface you must use
> GIDs\n");
> > +                       return 1;
> > +               }
> >        }
> >        my_dest.rkey = ctx->mr->rkey;
> >        my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;
> >
> >        printf(addr_fmt, "local", my_dest.lid, my_dest.qpn, my_dest.psn,
> >               my_dest.rkey, my_dest.vaddr);
> > +       if (user_parm->gid_index > -1) {
> > +               printf("                  GID:
> %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
> > +               my_dest.dgid.raw[0],my_dest.dgid.raw[1],
> > +               my_dest.dgid.raw[2], my_dest.dgid.raw[3],
> my_dest.dgid.raw[4],
> > +               my_dest.dgid.raw[5], my_dest.dgid.raw[6],
> my_dest.dgid.raw[7],
> > +               my_dest.dgid.raw[8], my_dest.dgid.raw[9],
> my_dest.dgid.raw[10],
> > +               my_dest.dgid.raw[11], my_dest.dgid.raw[12],
> my_dest.dgid.raw[13],
> > +               my_dest.dgid.raw[14], my_dest.dgid.raw[15]);
> > +       }
> >
> >        sockfd = servername ? pp_client_connect(servername, port) :
> >                pp_server_connect(port);
> > @@ -524,13 +629,22 @@ static int pp_open_port(struct pingpong_context
> *ctx, const char * servername,
> >                return sockfd;
> >        }
> >
> > -       rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest)
> :
> > -               pp_server_exch_dest(sockfd, &my_dest, rem_dest);
> > +       rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest,
> user_parm) :
> > +               pp_server_exch_dest(sockfd, &my_dest, rem_dest,
> user_parm);
> >        if (rc)
> >                return rc;
> >
> >        printf(addr_fmt, "remote", rem_dest->lid, rem_dest->qpn,
> rem_dest->psn,
> >               rem_dest->rkey, rem_dest->vaddr);
> > +       if (user_parm->gid_index > -1) {
> > +               printf("                  GID:
> %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
> > +               rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],
> > +               rem_dest->dgid.raw[2], rem_dest->dgid.raw[3],
> rem_dest->dgid.raw[4],
> > +               rem_dest->dgid.raw[5], rem_dest->dgid.raw[6],
> rem_dest->dgid.raw[7],
> > +               rem_dest->dgid.raw[8], rem_dest->dgid.raw[9],
> rem_dest->dgid.raw[10],
> > +               rem_dest->dgid.raw[11], rem_dest->dgid.raw[12],
> rem_dest->dgid.raw[13],
> > +               rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);
> > +       }
> >
> >        if ((rc = pp_connect_ctx(ctx, ib_port, my_dest.psn,
> rem_dest,user_parm)))
> >                return rc;
> > @@ -539,8 +653,8 @@ static int pp_open_port(struct pingpong_context *ctx,
> const char * servername,
> >         * Arbitrarily reuse exch_dest for this purpose.
> >         */
> >
> > -       rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest)
> :
> > -               pp_server_exch_dest(sockfd, &my_dest, rem_dest);
> > +       rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest,
> user_parm) :
> > +               pp_server_exch_dest(sockfd, &my_dest, rem_dest,
> user_parm);
> >
> >        if (rc)
> >                return rc;
> > @@ -565,6 +679,8 @@ static void usage(const char *argv0)
> >        printf("  -n, --iters=<iters>          number of exchanges (at
> least 2, default 1000)\n");
> >        printf("  -o, --outs=<num>             num of outstanding
> read/atom(default 4)\n");
> >        printf("  -u, --qp-timeout=<timeout> QP timeout, timeout value is
> 4 usec * 2 ^(timeout), default 14\n");
> > +       printf("  -S, --sl=<sl>                SL (default 0)\n");
> > +       printf("  -x, --gid-index=<index>      test uses GID with GID
> index taken from command line (for RDMAoE index should be 0)\n");
> >        printf("  -a, --all                    Run sizes from 2 till
> 2^23\n");
> >        printf("  -C, --report-cycles          report times in cpu cycle
> units (default microseconds)\n");
> >        printf("  -H, --report-histogram       print out all results
> (default print summary only)\n");
> > @@ -754,6 +870,7 @@ int main(int argc, char *argv[])
> >        user_param.use_event = 0;
> >        user_param.max_out_read = 4; /* the device capability on gen2 */
> >        user_param.qp_timeout = 14;
> > +       user_param.gid_index = -1; /*gid will not be used*/
> >        /* Parameter parsing. */
> >        while (1) {
> >                int c;
> > @@ -770,6 +887,7 @@ int main(int argc, char *argv[])
> >                        { .name = "tx-depth",       .has_arg = 1, .val =
> 't' },
> >                        { .name = "qp-timeout",     .has_arg = 1, .val =
> 'u' },
> >                        { .name = "sl",             .has_arg = 1, .val =
> 'S' },
> > +                       { .name = "gid-index",      .has_arg = 1, .val =
> 'x' },
> >                        { .name = "all",            .has_arg = 0, .val =
> 'a' },
> >                        { .name = "report-cycles",  .has_arg = 0, .val =
> 'C' },
> >                        { .name = "report-histogram",.has_arg = 0, .val =
> 'H' },
> > @@ -780,7 +898,7 @@ int main(int argc, char *argv[])
> >                        { 0 }
> >                };
> >
> > -               c = getopt_long(argc, argv,
> "p:c:m:d:i:s:o:n:t:u:S:aeHUVF", long_options, NULL);
> > +               c = getopt_long(argc, argv,
> "p:c:m:d:i:s:o:n:t:u:S:x:aeHUVF", long_options, NULL);
> >                if (c == -1)
> >                        break;
> >
> > @@ -874,6 +992,14 @@ int main(int argc, char *argv[])
> >                        if (sl > 15) { usage(argv[0]); return 5; }
> >                        break;
> >
> > +               case 'x':
> > +                       user_param.gid_index = strtol(optarg, NULL, 0);
> > +                       if (user_param.gid_index > 63) {
> > +                               usage(argv[0]);
> > +                               return 1;
> > +                       }
> > +                       break;
> > +
> >                default:
> >                        usage(argv[0]);
> >                        return 6;
> > @@ -899,6 +1025,9 @@ int main(int argc, char *argv[])
> >        printf("                    RDMA_Read Latency Test\n");
> >        printf("Connection type : RC\n");
> >        /* anyway make sure the connection is RC */
> > +       if (user_param.gid_index > -1) {
> > +               printf("Using GID to support RDMAoE configuration. Refer
> to port type as Ethernet, default MTU 1024B\n");
> > +       }
> >        tmp_size = size;
> >        if (user_param.all == ALL) {
> >                /*since we run all sizes */
> > @@ -956,7 +1085,7 @@ int main(int argc, char *argv[])
> >        /* done close sockets */
> >        if(user_param.servername) {
> >                /*Signal client is finished */
> > -               pp_client_exch_dest(user_param.sockfd, &my_dest,
> &rem_dest);
> > +               pp_client_exch_dest(user_param.sockfd, &my_dest,
> &rem_dest, &user_param);
> >                if (write(user_param.sockfd, "done", sizeof "done") !=
> sizeof "done"){
> >                        perror("client write");
> >                        fprintf(stderr, "Couldn't write to socket\n");
> > @@ -965,7 +1094,7 @@ int main(int argc, char *argv[])
> >                close(user_param.sockfd);
> >        } else {
> >                /*Server is finished wait for client */
> > -               pp_server_exch_dest(user_param.sockfd, &my_dest,
> &rem_dest);
> > +               pp_server_exch_dest(user_param.sockfd, &my_dest,
> &rem_dest, &user_param);
> >                if (write(user_param.sockfd, "done", sizeof "done") !=
> sizeof "done"){
> >                        perror("server write");
> >                        fprintf(stderr, "Couldn't write to socket\n");
> > --- a/send_bw.c
> > +++ b/send_bw.c
> > @@ -84,6 +84,7 @@ struct user_parameters {
> >        int use_mcg;
> >        int inline_size;
> >        int qp_timeout;
> > +       int gid_index; /* if value not negative, we use gid AND
> gid_index=value */
> >  };
> >  static int sl = 0;
> >  static int page_size;
> > @@ -106,6 +107,7 @@ struct pingpong_context {
> >        struct ibv_send_wr  wr;
> >        struct ibv_recv_wr  rwr;
> >        struct ibv_ah           *ah;
> > +       union ibv_gid       dgid;
> >  };
> >
> >  struct pingpong_dest {
> > @@ -114,6 +116,7 @@ struct pingpong_dest {
> >        int psn;
> >        unsigned rkey;
> >        unsigned long long vaddr;
> > +       union ibv_gid       dgid;
> >  };
> >
> >  static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
> > @@ -167,14 +170,20 @@ static int pp_client_connect(const char
> *servername, int port)
> >  }
> >
> >  struct pingpong_dest * pp_client_exch_dest(int sockfd,
> > -                                          const struct pingpong_dest
> *my_dest)
> > +                                          const struct pingpong_dest
> *my_dest, struct user_parameters *user_parm)
> >  {
> >        struct pingpong_dest *rem_dest = NULL;
> > -       char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
> > +       char msg[sizeof
> "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
> >        int parsed;
> >
> > -       sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid,
> my_dest->qpn,
> > -               my_dest->psn,my_dest->rkey,my_dest->vaddr);
> > +       sprintf(msg,
> "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
> > +               my_dest->lid, my_dest->qpn,
> my_dest->psn,my_dest->rkey,my_dest->vaddr,
> > +               my_dest->dgid.raw[0], my_dest->dgid.raw[1],
> my_dest->dgid.raw[2],
> > +               my_dest->dgid.raw[3], my_dest->dgid.raw[4],
> my_dest->dgid.raw[5],
> > +               my_dest->dgid.raw[6], my_dest->dgid.raw[7],
> my_dest->dgid.raw[8],
> > +               my_dest->dgid.raw[9], my_dest->dgid.raw[10],
> my_dest->dgid.raw[11],
> > +               my_dest->dgid.raw[12], my_dest->dgid.raw[13],
> my_dest->dgid.raw[14],
> > +               my_dest->dgid.raw[15]);
> >        if (write(sockfd, msg, sizeof msg) != sizeof msg) {
> >                perror("client write");
> >                fprintf(stderr, "Couldn't send local address\n");
> > @@ -191,15 +200,59 @@ struct pingpong_dest * pp_client_exch_dest(int
> sockfd,
> >        if (!rem_dest)
> >                goto out;
> >
> > -       parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > -                       &rem_dest->psn,&rem_dest->rkey,&rem_dest->vaddr);
> > +       if (user_parm->gid_index < 0) {
> > +               parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > +                               &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > +               if (parsed != 5) {
> > +                       fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg, msg);
> > +                       free(rem_dest);
> > +                       rem_dest = NULL;
> > +                       goto out;
> > +               }
> > +       }else{
> > +               char *pstr = msg, *term;
> > +               char tmp[20];
> > +               int i;
> >
> > -       if (parsed != 5) {
> > -               fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg,
> > -                       msg);
> > -               free(rem_dest);
> > -               rem_dest = NULL;
> > -               goto out;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
> > +
> > +               for (i = 0; i < 15; ++i) {
> > +                       pstr += term - pstr + 1;
> > +                       term = strpbrk(pstr, ":");
> > +                       memcpy(tmp, pstr, term - pstr);
> > +                       tmp[term - pstr] = 0;
> > +                       rem_dest->dgid.raw[i] = (unsigned
> char)strtoll(tmp, NULL, 16);
> > +               }
> > +               pstr += term - pstr + 1;
> > +               strcpy(tmp, pstr);
> > +               rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp,
> NULL, 16);
> >        }
> >  out:
> >        return rem_dest;
> > @@ -261,9 +314,9 @@ int pp_server_connect(int port)
> >        return connfd;
> >  }
> >
> > -static struct pingpong_dest *pp_server_exch_dest(int connfd, const
> struct pingpong_dest *my_dest)
> > +static struct pingpong_dest *pp_server_exch_dest(int connfd, const
> struct pingpong_dest *my_dest, struct user_parameters *user_parm)
> >  {
> > -       char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
> > +       char msg[sizeof
> "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
> >        struct pingpong_dest *rem_dest = NULL;
> >        int parsed;
> >        int n;
> > @@ -279,18 +332,69 @@ static struct pingpong_dest
> *pp_server_exch_dest(int connfd, const struct pingpo
> >        if (!rem_dest)
> >                goto out;
> >
> > -       parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > -                       &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > -       if (parsed != 5) {
> > -               fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg,
> > -                       msg);
> > -               free(rem_dest);
> > -               rem_dest = NULL;
> > -               goto out;
> > +       if (user_parm->gid_index < 0) {
> > +               parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > +                               &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > +               if (parsed != 5) {
> > +                       fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg, msg);
> > +                       free(rem_dest);
> > +                       rem_dest = NULL;
> > +                       goto out;
> > +               }
> > +       }else{
> > +               char *pstr = msg, *term;
> > +               char tmp[20];
> > +               int i;
> > +
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
> > +
> > +               for (i = 0; i < 15; ++i) {
> > +                       pstr += term - pstr + 1;
> > +                       term = strpbrk(pstr, ":");
> > +                       memcpy(tmp, pstr, term - pstr);
> > +                       tmp[term - pstr] = 0;
> > +                       rem_dest->dgid.raw[i] = (unsigned
> char)strtoll(tmp, NULL, 16);
> > +                       }
> > +               pstr += term - pstr + 1;
> > +               strcpy(tmp, pstr);
> > +               rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp,
> NULL, 16);
> >        }
> >
> > -       sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid,
> my_dest->qpn,
> > -               my_dest->psn, my_dest->rkey, my_dest->vaddr);
> > +       sprintf(msg,
> "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
> > +               my_dest->lid, my_dest->qpn, my_dest->psn, my_dest->rkey,
> my_dest->vaddr,
> > +               my_dest->dgid.raw[0], my_dest->dgid.raw[1],
> my_dest->dgid.raw[2],
> > +               my_dest->dgid.raw[3], my_dest->dgid.raw[4],
> my_dest->dgid.raw[5],
> > +               my_dest->dgid.raw[6], my_dest->dgid.raw[7],
> my_dest->dgid.raw[8],
> > +               my_dest->dgid.raw[9], my_dest->dgid.raw[10],
> my_dest->dgid.raw[11],
> > +               my_dest->dgid.raw[12], my_dest->dgid.raw[13],
> my_dest->dgid.raw[14],
> > +               my_dest->dgid.raw[15]);
> >        if (write(connfd, msg, sizeof msg) != sizeof msg) {
> >                perror("server write");
> >                fprintf(stderr, "Couldn't send local address\n");
> > @@ -346,7 +450,7 @@ static struct pingpong_context *pp_init_ctx(struct
> ibv_device *ib_dev,
> >                        fprintf(stderr, "Failed to query device props");
> >                        return NULL;
> >                }
> > -               if (device_attr.vendor_part_id == 23108) {
> > +               if (device_attr.vendor_part_id == 23108 ||
> user_parm->gid_index > -1) {
> >                        user_parm->mtu = 1024;
> >                } else {
> >                        user_parm->mtu = 2048;
> > @@ -504,9 +608,16 @@ static int pp_connect_ctx(struct pingpong_context
> *ctx, int port, int my_psn,
> >                attr.max_dest_rd_atomic     = 1;
> >                attr.min_rnr_timer          = 12;
> >        }
> > -       attr.ah_attr.is_global  = 0;
> > -       attr.ah_attr.dlid       = dest->lid;
> > -       attr.ah_attr.sl         = sl;
> > +       if (user_parm->gid_index < 0) {
> > +               attr.ah_attr.is_global  = 0;
> > +               attr.ah_attr.dlid       = dest->lid;
> > +               attr.ah_attr.sl         = sl;
> > +       } else {
> > +               attr.ah_attr.is_global  = 1;
> > +               attr.ah_attr.grh.dgid   = dest->dgid;
> > +               attr.ah_attr.grh.hop_limit = 1;
> > +               attr.ah_attr.sl         = 0;
> > +       }
> >        attr.ah_attr.src_path_bits = 0;
> >        attr.ah_attr.port_num   = port;
> >        if ((user_parm->connection_type==UD) && (user_parm->use_mcg)) {
> > @@ -518,9 +629,6 @@ static int pp_connect_ctx(struct pingpong_context
> *ctx, int port, int my_psn,
> >                attr.ah_attr.is_global  = 1;
> >                attr.ah_attr.grh.sgid_index = 0;
> >                memcpy(attr.ah_attr.grh.dgid.raw, mcg_gid, 16);
> > -       } else {
> > -               attr.ah_attr.dlid       = dest->lid;
> > -               attr.ah_attr.is_global  = 0;
> >        }
> >        if (user_parm->connection_type == RC) {
> >                if (ibv_modify_qp(ctx->qp, &attr,
> > @@ -632,6 +740,7 @@ static void usage(const char *argv0)
> >        printf("  -I, --inline_size=<size>    max size of message to be
> sent in inline mode (default 400)\n");
> >        printf("  -u, --qp-timeout=<timeout> QP timeout, timeout value is
> 4 usec * 2 ^(timeout), default 14\n");
> >        printf("  -S, --sl=<sl>               SL (default 0)\n");
> > +       printf("  -x, --gid-index=<index>   test uses GID with GID index
> taken from command line (for RDMAoE index should be 0)\n");
> >        printf("  -b, --bidirectional         measure bidirectional
> bandwidth (default unidirectional)\n");
> >        printf("  -V, --version               display version number\n");
> >        printf("  -e, --events                sleep on CQ events (default
> poll)\n");
> > @@ -680,8 +789,13 @@ int run_iter_bi(struct pingpong_context *ctx, struct
> user_parameters *user_param
> >        int                      scnt, ccnt, rcnt;
> >        struct ibv_recv_wr      *bad_wr_recv;
> >        if (user_param->connection_type == UD) {
> > -               if (size > 2048)
> > -                       size = 2048;
> > +               if (size > 2048) {
> > +                       if (user_param->gid_index < 0) {
> > +                               size = 2048;
> > +                       } else {
> > +                               size = 1024;
> > +                       }
> > +               }
> >        }
> >        /*********************************************
> >         * Important note :
> > @@ -799,8 +913,13 @@ int run_iter_uni(struct pingpong_context *ctx,
> struct user_parameters *user_para
> >        struct ibv_recv_wr      *bad_wr_recv;
> >
> >        if (user_param->connection_type == UD) {
> > -               if (size > 2048)
> > -                       size = 2048;
> > +               if (size > 2048) {
> > +                       if (user_param->gid_index < 0) {
> > +                               size = 2048;
> > +                       } else {
> > +                               size = 1024;
> > +                       }
> > +               }
> >        }
> >
> >        if (user_param->connection_type == UD)
> > @@ -947,6 +1066,7 @@ int main(int argc, char *argv[])
> >        int                      inline_given_in_cmd = 0;
> >        struct ibv_context       *context;
> >        int                      no_cpu_freq_fail = 0;
> > +       union ibv_gid            gid;
> >        /* init default values to user's parameters */
> >        memset(&user_param, 0, sizeof(struct user_parameters));
> >        user_param.mtu = 0;
> > @@ -957,6 +1077,7 @@ int main(int argc, char *argv[])
> >        user_param.duplex = 0;
> >        user_param.inline_size = MAX_INLINE;
> >        user_param.qp_timeout = 14;
> > +       user_param.gid_index = -1; /*gid will not be used*/
> >        /* Parameter parsing. */
> >        while (1) {
> >                int c;
> > @@ -974,6 +1095,7 @@ int main(int argc, char *argv[])
> >                        { .name = "rx-depth",       .has_arg = 1, .val =
> 'r' },
> >                        { .name = "qp-timeout",     .has_arg = 1, .val =
> 'u' },
> >                        { .name = "sl",             .has_arg = 1, .val =
> 'S' },
> > +                       { .name = "gid-index",      .has_arg = 1, .val =
> 'x' },
> >                        { .name = "all",            .has_arg = 0, .val =
> 'a' },
> >                        { .name = "bidirectional",  .has_arg = 0, .val =
> 'b' },
> >                        { .name = "version",        .has_arg = 0, .val =
> 'V' },
> > @@ -984,7 +1106,7 @@ int main(int argc, char *argv[])
> >                        { 0 }
> >                };
> >
> > -               c = getopt_long(argc, argv,
> "p:d:i:m:c:s:n:t:I:r:u:S:ebaVgNF", long_options, NULL);
> > +               c = getopt_long(argc, argv,
> "p:d:i:m:c:s:n:t:I:r:u:S:x:ebaVgNF", long_options, NULL);
> >                if (c == -1)
> >                        break;
> >
> > @@ -1038,6 +1160,14 @@ int main(int argc, char *argv[])
> >
> >                        break;
> >
> > +               case 'x':
> > +                       user_param.gid_index = strtol(optarg, NULL, 0);
> > +                       if (user_param.gid_index > 63) {
> > +                               usage(argv[0]);
> > +                               return 1;
> > +                       }
> > +                       break;
> > +
> >                case 't':
> >                        user_param.tx_depth = strtol(optarg, NULL, 0);
> >                        if (user_param.tx_depth < 1) { usage(argv[0]);
> return 1; }
> > @@ -1117,6 +1247,9 @@ int main(int argc, char *argv[])
> >        else{
> >                printf("Connection type : UD\n");
> >        }
> > +       if (user_param.gid_index > -1) {
> > +               printf("Using GID to support RDMAoE configuration. Refer
> to port type as Ethernet, default MTU 1024B\n");
> > +       }
> >
> >        /* Done with parameter parsing. Perform setup. */
> >        if (user_param.all == ALL)
> > @@ -1126,6 +1259,10 @@ int main(int argc, char *argv[])
> >                printf("Max msg size in UD is 2048 changing to 2048\n");
> >                size = 2048;
> >        }
> > +       if (user_param.connection_type == UD && user_param.gid_index > -1
> && size > 1024) {
> > +               printf("Max msg size in UD RDMAoE is 1024. changing to
> 1024\n");
> > +               size = 1024;
> > +       }
> >
> >        srand48(getpid() * time(NULL));
> >
> > @@ -1174,25 +1311,46 @@ int main(int argc, char *argv[])
> >        my_dest.lid = pp_get_local_lid(ctx, ib_port);
> >        my_dest.qpn = ctx->qp->qp_num;
> >        my_dest.psn = lrand48() & 0xffffff;
> > -       if (!my_dest.lid) {
> > -               fprintf(stderr, "Local lid 0x0 detected. Is an SM
> running?\n");
> > -               return 1;
> > +       if (user_param.gid_index != -1) {
> > +               int err=0;
> > +               err = ibv_query_gid (ctx->context, ib_port,
> user_param.gid_index, &gid);
> > +               if (err) {
> > +                       return -1;
> > +               }
> > +               ctx->dgid=gid;
> > +       }
> > +
> > +       if (user_param.gid_index < 0) {/*We do not fail test upon lid in
> RDMA0E/Eth conf*/
> > +                       if (!my_dest.lid) {
> > +                               fprintf(stderr, "Local lid 0x0 detected.
> Is an SM running? If you are running on an RMDAoE interface you must use
> GIDs\n");
> > +                       return 1;
> > +               }
> >        }
> > +       my_dest.dgid = gid;
> >        my_dest.rkey = ctx->mr->rkey;
> >        my_dest.vaddr = (uintptr_t)ctx->buf + size;
> >        printf("  local address:  LID %#04x, QPN %#06x, PSN %#06x\n",
> >               my_dest.lid, my_dest.qpn, my_dest.psn);
> > +       if (user_param.gid_index > -1) {
> > +               printf("                  GID
> %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
> > +               my_dest.dgid.raw[0],my_dest.dgid.raw[1],
> > +               my_dest.dgid.raw[2], my_dest.dgid.raw[3],
> my_dest.dgid.raw[4],
> > +               my_dest.dgid.raw[5], my_dest.dgid.raw[6],
> my_dest.dgid.raw[7],
> > +               my_dest.dgid.raw[8], my_dest.dgid.raw[9],
> my_dest.dgid.raw[10],
> > +               my_dest.dgid.raw[11], my_dest.dgid.raw[12],
> my_dest.dgid.raw[13],
> > +               my_dest.dgid.raw[14], my_dest.dgid.raw[15]);
> > +       }
> >
> >        if (user_param.servername) {
> >                sockfd = pp_client_connect(user_param.servername, port);
> >                if (sockfd < 0)
> >                        return 1;
> > -               rem_dest = pp_client_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_client_exch_dest(sockfd, &my_dest,
> &user_param);
> >        } else {
> >                sockfd = pp_server_connect(port);
> >                if (sockfd < 0)
> >                        return 1;
> > -               rem_dest = pp_server_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_server_exch_dest(sockfd, &my_dest,
> &user_param);
> >        }
> >
> >        if (!rem_dest)
> > @@ -1200,6 +1358,15 @@ int main(int argc, char *argv[])
> >
> >        printf("  remote address: LID %#04x, QPN %#06x, PSN %#06x\n",
> >               rem_dest->lid, rem_dest->qpn, rem_dest->psn);
> > +       if (user_param.gid_index > -1) {
> > +               printf("                  GID
> %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
> > +               rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],
> > +               rem_dest->dgid.raw[2], rem_dest->dgid.raw[3],
> rem_dest->dgid.raw[4],
> > +               rem_dest->dgid.raw[5], rem_dest->dgid.raw[6],
> rem_dest->dgid.raw[7],
> > +               rem_dest->dgid.raw[8], rem_dest->dgid.raw[9],
> rem_dest->dgid.raw[10],
> > +               rem_dest->dgid.raw[11], rem_dest->dgid.raw[12],
> rem_dest->dgid.raw[13],
> > +               rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);
> > +       }
> >
> >        if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest,
> &user_param))
> >                return 1;
> > @@ -1207,9 +1374,9 @@ int main(int argc, char *argv[])
> >        /* An additional handshake is required *after* moving qp to RTR.
> >           Arbitrarily reuse exch_dest for this purpose. */
> >        if (user_param.servername) {
> > -               rem_dest = pp_client_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_client_exch_dest(sockfd, &my_dest,
> &user_param);
> >        } else {
> > -               rem_dest = pp_server_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_server_exch_dest(sockfd, &my_dest,
> &user_param);
> >        }
> >        if (user_param.use_event) {
> >                printf("Test with events.\n");
> > @@ -1263,8 +1430,13 @@ int main(int argc, char *argv[])
> >        ctx->recv_list.lkey = ctx->mr->lkey;
> >
> >        if (user_param.all == ALL) {
> > -               if (user_param.connection_type == UD)
> > -                       size_max_pow = 12;
> > +               if (user_param.connection_type == UD) {
> > +                       if (user_param.gid_index < 0) {
> > +                               size_max_pow = 12;
> > +                       } else {
> > +                               size_max_pow = 11;
> > +                       }
> > +               }
> >
> >                for (i = 1; i < size_max_pow ; ++i) {
> >                        size = 1 << i;
> > @@ -1278,9 +1450,9 @@ int main(int argc, char *argv[])
> >                        if (user_param.servername) {
> >                                print_report(user_param.iters, size,
> user_param.duplex, tposted, tcompleted, noPeak, no_cpu_freq_fail);
> >                                /* sync again for the sake of UC/UC */
> > -                               rem_dest = pp_client_exch_dest(sockfd,
> &my_dest);
> > +                               rem_dest = pp_client_exch_dest(sockfd,
> &my_dest, &user_param);
> >                        } else
> > -                               rem_dest = pp_server_exch_dest(sockfd,
> &my_dest);
> > +                               rem_dest = pp_server_exch_dest(sockfd,
> &my_dest, &user_param);
> >                }
> >        } else {
> >                if (user_param.duplex) {
> > @@ -1298,9 +1470,9 @@ int main(int argc, char *argv[])
> >
> >        /* close sockets */
> >        if (user_param.servername)
> > -               rem_dest = pp_client_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_client_exch_dest(sockfd, &my_dest,
> &user_param);
> >        else
> > -               rem_dest = pp_server_exch_dest(sockfd, &my_dest);
> > +               rem_dest = pp_server_exch_dest(sockfd, &my_dest,
> &user_param);
> >
> >        if (write(sockfd, "done", sizeof "done") != sizeof "done"){
> >                perror("write");
> >
> > --- a/send_lat.c
> > +++ b/send_lat.c
> > @@ -82,6 +82,7 @@ struct user_parameters {
> >        int inline_size;
> >        int use_mcg;
> >        int qp_timeout;
> > +       int gid_index; /* if value not negative, we use gid AND
> gid_index=value */
> >  };
> >
> >  struct report_options {
> > @@ -109,6 +110,7 @@ struct pingpong_context {
> >        volatile char      *poll_buf;
> >        int                 size;
> >        int                 tx_depth;
> > +       union ibv_gid       dgid;
> >  };
> >
> >  struct pingpong_dest {
> > @@ -117,6 +119,7 @@ struct pingpong_dest {
> >        int qpn;
> >        int psn;
> >        unsigned int rkey;
> > +       union ibv_gid       dgid;
> >  };
> >
> >
> > @@ -152,13 +155,33 @@ static struct ibv_device *pp_find_dev(const char
> *ib_devname) {
> >
> >  #define KEY_MSG_SIZE (sizeof
> "0000:000000:000000:00000000:0000000000000000")
> >  #define KEY_PRINT_FMT "%04x:%06x:%06x:%08x:%016Lx"
> > +#define KEY_MSG_SIZE_GID (sizeof
> "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00")
> > +#define KEY_PRINT_FMT_GID
> "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x"
> >
> > -static int pp_write_keys(int sockfd, const struct pingpong_dest
> *my_dest)
> > +static int pp_write_keys(int sockfd, const struct pingpong_dest
> *my_dest, struct user_parameters *user_parm)
> >  {
> > -       char msg[KEY_MSG_SIZE];
> > +       if (user_parm->gid_index < 0) {
> > +               char msg[KEY_MSG_SIZE];
> >
> > -       sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,
> > -               my_dest->psn, my_dest->rkey, my_dest->vaddr);
> > +               sprintf(msg, KEY_PRINT_FMT, my_dest->lid, my_dest->qpn,
> > +                       my_dest->psn, my_dest->rkey, my_dest->vaddr);
> > +
> > +               if (write(sockfd, msg, sizeof msg) != sizeof msg) {
> > +                       perror("client write");
> > +                       fprintf(stderr, "Couldn't send local address\n");
> > +                       return -1;
> > +               }
> > +
> > +               return 0;
> > +       } else {
> > +       char msg[KEY_MSG_SIZE_GID];
> > +
> > +               sprintf(msg, KEY_PRINT_FMT_GID, my_dest->lid,
> my_dest->qpn,
> > +                       my_dest->psn, my_dest->rkey, my_dest->vaddr,
> > +                       my_dest->dgid.raw[0], my_dest->dgid.raw[1],
> my_dest->dgid.raw[2], my_dest->dgid.raw[3],
> > +                       my_dest->dgid.raw[4], my_dest->dgid.raw[5],
> my_dest->dgid.raw[6], my_dest->dgid.raw[7],
> > +                       my_dest->dgid.raw[8], my_dest->dgid.raw[9],
> my_dest->dgid.raw[10], my_dest->dgid.raw[11],
> > +                       my_dest->dgid.raw[12], my_dest->dgid.raw[13],
> my_dest->dgid.raw[14], my_dest->dgid.raw[15]);
> >
> >        if (write(sockfd, msg, sizeof msg) != sizeof msg) {
> >                perror("client write");
> > @@ -167,30 +190,84 @@ static int pp_write_keys(int sockfd, const struct
> pingpong_dest *my_dest)
> >        }
> >
> >        return 0;
> > +       }
> >  }
> >
> >  static int pp_read_keys(int sockfd, const struct pingpong_dest *my_dest,
> > -                       struct pingpong_dest *rem_dest)
> > +                       struct pingpong_dest *rem_dest, struct
> user_parameters *user_parm)
> >  {
> > -       int parsed;
> > -       char msg[KEY_MSG_SIZE];
> > +       if (user_parm->gid_index < 0) {
> > +               int parsed;
> > +               char msg[KEY_MSG_SIZE];
> > +
> > +               if (read(sockfd, msg, sizeof msg) != sizeof msg) {
> > +                       perror("pp_read_keys");
> > +                       fprintf(stderr, "Couldn't read remote
> address\n");
> > +                       return -1;
> > +               }
> >
> > -       if (read(sockfd, msg, sizeof msg) != sizeof msg) {
> > -               perror("pp_read_keys");
> > -               fprintf(stderr, "Couldn't read remote address\n");
> > -               return -1;
> > -       }
> > +               parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid,
> &rem_dest->qpn,
> > +                               &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> >
> > -       parsed = sscanf(msg, KEY_PRINT_FMT, &rem_dest->lid,
> &rem_dest->qpn,
> > -                       &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > +               if (parsed != 5) {
> > +                       fprintf(stderr, "Couldn't parse line <%.*s>\n",
> > +                               (int)sizeof msg, msg);
> > +                       return -1;
> > +               }
> >
> > -       if (parsed != 5) {
> > -               fprintf(stderr, "Couldn't parse line <%.*s>\n",
> > -                       (int)sizeof msg, msg);
> > -               return -1;
> > -       }
> > +               return 0;
> > +       } else {
> > +               char msg[KEY_MSG_SIZE_GID];
> > +               if (read(sockfd, msg, sizeof msg) != sizeof msg) {
> > +                       perror("pp_read_keys");
> > +                       fprintf(stderr, "Couldn't read remote
> address\n");
> > +                       return -1;
> > +               }
> > +               char *pstr = msg, *term;
> > +               char tmp[20];
> > +               int i;
> >
> > -       return 0;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
> > +
> > +               for (i = 0; i < 15; ++i) {
> > +                       pstr += term - pstr + 1;
> > +                       term = strpbrk(pstr, ":");
> > +                       memcpy(tmp, pstr, term - pstr);
> > +                       tmp[term - pstr] = 0;
> > +                       rem_dest->dgid.raw[i] = (unsigned
> char)strtoll(tmp, NULL, 16);
> > +                       }
> > +               pstr += term - pstr + 1;
> > +               strcpy(tmp, pstr);
> > +               rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp,
> NULL, 16);
> > +               return 0;
> > +       }
> >  }
> >
> >  static int pp_client_connect(const char *servername, int port)
> > @@ -235,12 +312,12 @@ static int pp_client_connect(const char
> *servername, int port)
> >  }
> >
> >  static int pp_client_exch_dest(int sockfd, const struct pingpong_dest
> *my_dest,
> > -                              struct pingpong_dest *rem_dest)
> > +                              struct pingpong_dest *rem_dest, struct
> user_parameters *user_parm)
> >  {
> > -       if (pp_write_keys(sockfd, my_dest))
> > +       if (pp_write_keys(sockfd, my_dest, user_parm))
> >                return -1;
> >
> > -       return pp_read_keys(sockfd, my_dest, rem_dest);
> > +       return pp_read_keys(sockfd, my_dest, rem_dest, user_parm);
> >  }
> >
> >  static int pp_server_connect(int port)
> > @@ -300,13 +377,13 @@ static int pp_server_connect(int port)
> >  }
> >
> >  static int pp_server_exch_dest(int sockfd, const struct pingpong_dest
> *my_dest,
> > -                              struct pingpong_dest* rem_dest)
> > +                              struct pingpong_dest* rem_dest, struct
> user_parameters *user_parm)
> >  {
> >
> > -       if (pp_read_keys(sockfd, my_dest, rem_dest))
> > +       if (pp_read_keys(sockfd, my_dest, rem_dest, user_parm))
> >                return -1;
> >
> > -       return pp_write_keys(sockfd, my_dest);
> > +       return pp_write_keys(sockfd, my_dest, user_parm);
> >  }
> >
> >  static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev,
> int size,
> > @@ -351,7 +428,7 @@ static struct pingpong_context *pp_init_ctx(struct
> ibv_device *ib_dev, int size,
> >                        fprintf(stderr, "Failed to query device props");
> >                        return NULL;
> >                }
> > -               if (device_attr.vendor_part_id == 23108) {
> > +               if (device_attr.vendor_part_id == 23108 ||
> user_parm->gid_index > -1) {
> >                        user_parm->mtu = 1024;
> >                } else {
> >                        user_parm->mtu = 2048;
> > @@ -522,9 +599,16 @@ static int pp_connect_ctx(struct pingpong_context
> *ctx, int port, int my_psn,
> >                attr.min_rnr_timer          = 12;
> >        }
> >
> > -       attr.ah_attr.is_global      = 0;
> > -       attr.ah_attr.dlid           = dest->lid;
> > -       attr.ah_attr.sl             = sl;
> > +       if (user_parm->gid_index < 0) {
> > +               attr.ah_attr.is_global      = 0;
> > +               attr.ah_attr.dlid           = dest->lid;
> > +               attr.ah_attr.sl             = sl;
> > +       } else {
> > +               attr.ah_attr.is_global      = 1;
> > +               attr.ah_attr.grh.dgid       = dest->dgid;
> > +               attr.ah_attr.grh.hop_limit  = 1;
> > +               attr.ah_attr.sl             = 0;
> > +       }
> >        attr.ah_attr.src_path_bits  = 0;
> >        attr.ah_attr.port_num       = port;
> >        if ((user_parm->connection_type==UD) && (user_parm->use_mcg)) {
> > @@ -538,9 +622,6 @@ static int pp_connect_ctx(struct pingpong_context
> *ctx, int port, int my_psn,
> >                attr.ah_attr.is_global  = 1;
> >                attr.ah_attr.grh.sgid_index = 0;
> >                memcpy(attr.ah_attr.grh.dgid.raw, mcg_gid, 16);
> > -       } else {
> > -               attr.ah_attr.dlid       = dest->lid;
> > -               attr.ah_attr.is_global  = 0;
> >        }
> >
> >        if (user_parm->connection_type==RC) {
> > @@ -637,6 +718,7 @@ static int pp_open_port(struct pingpong_context *ctx,
> const char * servername,
> >        struct pingpong_dest  my_dest;
> >        int                   sockfd;
> >        int                   rc;
> > +       union ibv_gid         gid;
> >
> >
> >        /* Create connection between client and server.
> > @@ -645,14 +727,34 @@ static int pp_open_port(struct pingpong_context
> *ctx, const char * servername,
> >        my_dest.lid = pp_get_local_lid(ctx, ib_port);
> >        my_dest.qpn = ctx->qp->qp_num;
> >        my_dest.psn = lrand48() & 0xffffff;
> > -       if (!my_dest.lid) {
> > -               fprintf(stderr, "Local lid 0x0 detected. Is an SM
> running?\n");
> > -               return -1;
> > +       if (user_parm->gid_index < 0) {/*We do not fail test upon lid in
> RDMA0E/Eth conf*/
> > +               if (!my_dest.lid) {
> > +                       fprintf(stderr, "Local lid 0x0 detected. Is an SM
> running?\n");
> > +                       return -1;
> > +               }
> >        }
> > +       if (user_parm->gid_index != -1) {
> > +               int err=0;
> > +               err = ibv_query_gid (ctx->context, ib_port,
> user_parm->gid_index, &gid);
> > +               if (err) {
> > +                       return -1;
> > +               }
> > +               ctx->dgid=gid;
> > +       }
> > +       my_dest.dgid = gid;
> >        my_dest.rkey = ctx->mr->rkey;
> >        my_dest.vaddr = (uintptr_t)ctx->buf + ctx->size;
> >
> >        printf(addr_fmt, "local", my_dest.lid, my_dest.qpn, my_dest.psn);
> > +       if (user_parm->gid_index > -1) {
> > +               printf("                  GID:
> %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
> > +               my_dest.dgid.raw[0],my_dest.dgid.raw[1],
> > +               my_dest.dgid.raw[2], my_dest.dgid.raw[3],
> my_dest.dgid.raw[4],
> > +               my_dest.dgid.raw[5], my_dest.dgid.raw[6],
> my_dest.dgid.raw[7],
> > +               my_dest.dgid.raw[8], my_dest.dgid.raw[9],
> my_dest.dgid.raw[10],
> > +               my_dest.dgid.raw[11], my_dest.dgid.raw[12],
> my_dest.dgid.raw[13],
> > +               my_dest.dgid.raw[14], my_dest.dgid.raw[15]);
> > +       }
> >
> >        sockfd = servername ? pp_client_connect(servername, port) :
> >                pp_server_connect(port);
> > @@ -663,13 +765,22 @@ static int pp_open_port(struct pingpong_context
> *ctx, const char * servername,
> >                return sockfd;
> >        }
> >
> > -       rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest)
> :
> > -               pp_server_exch_dest(sockfd, &my_dest, rem_dest);
> > +       rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest,
> user_parm) :
> > +               pp_server_exch_dest(sockfd, &my_dest, rem_dest,
> user_parm);
> >        if (rc)
> >                return rc;
> >
> >        printf(addr_fmt, "remote", rem_dest->lid, rem_dest->qpn,
> rem_dest->psn,
> >               rem_dest->rkey, rem_dest->vaddr);
> > +       if (user_parm->gid_index > -1) {
> > +               printf("                  GID:
> %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
> > +               rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],
> > +               rem_dest->dgid.raw[2], rem_dest->dgid.raw[3],
> rem_dest->dgid.raw[4],
> > +               rem_dest->dgid.raw[5], rem_dest->dgid.raw[6],
> rem_dest->dgid.raw[7],
> > +               rem_dest->dgid.raw[8], rem_dest->dgid.raw[9],
> rem_dest->dgid.raw[10],
> > +               rem_dest->dgid.raw[11], rem_dest->dgid.raw[12],
> rem_dest->dgid.raw[13],
> > +               rem_dest->dgid.raw[14], rem_dest->dgid.raw[15]);
> > +       }
> >
> >        if ((rc = pp_connect_ctx(ctx, ib_port, my_dest.psn,
> rem_dest,user_parm)))
> >                return rc;
> > @@ -678,8 +789,8 @@ static int pp_open_port(struct pingpong_context *ctx,
> const char * servername,
> >         * Arbitrarily reuse exch_dest for this purpose.
> >         */
> >
> > -       rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest)
> :
> > -               pp_server_exch_dest(sockfd, &my_dest, rem_dest);
> > +       rc = servername ? pp_client_exch_dest(sockfd, &my_dest, rem_dest,
> user_parm) :
> > +               pp_server_exch_dest(sockfd, &my_dest, rem_dest,
> user_parm);
> >
> >        if (rc)
> >                return rc;
> > @@ -714,6 +825,7 @@ static void usage(const char *argv0)
> >        printf("  -I, --inline_size=<size>     max size of message to be
> sent in inline mode (default 400)\n");
> >        printf("  -u, --qp-timeout=<timeout> QP timeout, timeout value is
> 4 usec * 2 ^(timeout), default 14\n");
> >        printf("  -S, --sl=<sl>                SL (default 0)\n");
> > +       printf("  -x, --gid-index=<index>   test uses GID with GID index
> taken from command line (for RDMAoE index should be 0)\n");
> >        printf("  -C, --report-cycles          report times in cpu cycle
> units (default microseconds)\n");
> >        printf("  -H, --report-histogram       print out all results
> (default print summary only)\n");
> >        printf("  -U, --report-unsorted        (implies -H) print out
> unsorted results (default sorted)\n");
> > @@ -814,7 +926,11 @@ int run_iter(struct pingpong_context *ctx, struct
> user_parameters *user_param,
> >
> >        if (user_param->connection_type==UD) {
> >                if (size > 2048) {
> > -                       size = 2048;
> > +                       if (user_param->gid_index < 0) {
> > +                               size = 2048;
> > +                       } else {
> > +                               size = 1024;
> > +                       }
> >                }
> >        }
> >
> > @@ -1011,6 +1127,7 @@ int main(int argc, char *argv[])
> >        user_param.inline_size = MAX_INLINE;
> >        user_param.signal_comp = 0;
> >        user_param.qp_timeout = 14;
> > +       user_param.gid_index = -1; /*gid will not be used*/
> >        /* Parameter parsing. */
> >        while (1) {
> >                int c;
> > @@ -1027,6 +1144,7 @@ int main(int argc, char *argv[])
> >                        { .name = "inline_size",    .has_arg = 1, .val =
> 'I' },
> >                        { .name = "qp-timeout",     .has_arg = 1, .val =
> 'u' },
> >                        { .name = "sl",             .has_arg = 1, .val =
> 'S' },
> > +                       { .name = "gid-index",      .has_arg = 1, .val =
> 'x' },
> >                        { .name = "signal",         .has_arg = 0, .val =
> 'l' },
> >                        { .name = "all",            .has_arg = 0, .val =
> 'a' },
> >                        { .name = "report-cycles",  .has_arg = 0, .val =
> 'C' },
> > @@ -1038,7 +1156,7 @@ int main(int argc, char *argv[])
> >                        { .name = "CPU-freq",       .has_arg = 0, .val =
> 'F' },
> >                        { 0 }
> >                };
> > -               c = getopt_long(argc, argv,
> "p:c:m:d:i:s:n:t:I:u:S:laeCHUVgF", long_options, NULL);
> > +               c = getopt_long(argc, argv,
> "p:c:m:d:i:s:n:t:I:u:S:x:laeCHUVgF", long_options, NULL);
> >                if (c == -1)
> >                        break;
> >
> > @@ -1095,6 +1213,14 @@ int main(int argc, char *argv[])
> >                        }
> >                        break;
> >
> > +               case 'x':
> > +                       user_param.gid_index = strtol(optarg, NULL, 0);
> > +                       if (user_param.gid_index > 63) {
> > +                               usage(argv[0]);
> > +                               return 1;
> > +                       }
> > +                       break;
> > +
> >                case 't':
> >                        user_param.tx_depth = strtol(optarg, NULL, 0);
> >                        if (user_param.tx_depth < 1) {
> > @@ -1179,10 +1305,21 @@ int main(int argc, char *argv[])
> >        } else {
> >                printf("Connection type : UD\n");
> >        }
> > +       if (user_param.gid_index > -1) {
> > +               printf("Using GID to support RDMAoE configuration. Refer
> to port type as Ethernet, default MTU 1024B\n");
> > +       }
> >        if (user_param.all == 1) {
> >                /*since we run all sizes lets allocate big enough buffer
> */
> >                size = 8388608; /*2^23 */
> >        }
> > +       if (user_param.connection_type == UD && size > 2048) {
> > +               printf("Max msg size in UD is 2048 changing to 2048\n");
> > +               size = 2048;
> > +       }
> > +       if (user_param.connection_type == UD && user_param.gid_index > -1
> && size > 1024) {
> > +               printf("Max msg size in UD RDMAoE is 1024. changing to
> 1024\n");
> > +               size = 1024;
> > +       }
> >
> >        srand48(getpid() * time(NULL));
> >        page_size = sysconf(_SC_PAGESIZE);
> > @@ -1214,7 +1351,11 @@ int main(int argc, char *argv[])
> >
> >        if (user_param.all == 1) {
> >                if (user_param.connection_type==UD) {
> > -                       size_max_pow = 12;
> > +                       if (user_param.gid_index < 0) {
> > +                               size_max_pow = 12;
> > +                       } else {
> > +                               size_max_pow = 11;
> > +                       }
> >                }
> >                for (i = 1; i < size_max_pow ; ++i) {
> >                        size = 1 << i;
> >
> > --- a/write_bw_postlist.c
> > +++ b/write_bw_postlist.c
> > @@ -75,6 +75,7 @@ struct user_parameters {
> >     int maxpostsofqpiniteration;
> >     int inline_size;
> >        int qp_timeout;
> > +       int gid_index; /* if value not negative, we use gid AND
> gid_index=value */
> >  };
> >  struct extended_qp {
> >   struct ibv_qp           *qp;
> > @@ -98,6 +99,7 @@ struct pingpong_context {
> >     struct ibv_send_wr  wr;
> >     int                 *scnt;
> >     int                 *ccnt ;
> > +       union ibv_gid       dgid;
> >  };
> >
> >  struct pingpong_dest {
> > @@ -106,6 +108,7 @@ struct pingpong_dest {
> >        int psn;
> >        unsigned rkey;
> >        unsigned long long vaddr;
> > +       union ibv_gid       dgid;
> >  };
> >
> >
> > @@ -160,14 +163,20 @@ static int pp_client_connect(const char
> *servername, int port)
> >  }
> >
> >  struct pingpong_dest * pp_client_exch_dest(int sockfd,
> > -                                          const struct pingpong_dest
> *my_dest)
> > +                                          const struct pingpong_dest
> *my_dest, struct user_parameters *user_parm)
> >  {
> >        struct pingpong_dest *rem_dest = NULL;
> > -       char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
> > +       char msg[sizeof
> "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
> >        int parsed;
> >
> > -       sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid,
> my_dest->qpn,
> > -               my_dest->psn,my_dest->rkey,my_dest->vaddr);
> > +       sprintf(msg,
> "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
> > +               my_dest->lid, my_dest->qpn,
> my_dest->psn,my_dest->rkey,my_dest->vaddr,
> > +               my_dest->dgid.raw[0], my_dest->dgid.raw[1],
> my_dest->dgid.raw[2],
> > +               my_dest->dgid.raw[3], my_dest->dgid.raw[4],
> my_dest->dgid.raw[5],
> > +               my_dest->dgid.raw[6], my_dest->dgid.raw[7],
> my_dest->dgid.raw[8],
> > +               my_dest->dgid.raw[9], my_dest->dgid.raw[10],
> my_dest->dgid.raw[11],
> > +               my_dest->dgid.raw[12], my_dest->dgid.raw[13],
> my_dest->dgid.raw[14],
> > +               my_dest->dgid.raw[15]);
> >        if (write(sockfd, msg, sizeof msg) != sizeof msg) {
> >                perror("client write");
> >                fprintf(stderr, "Couldn't send local address\n");
> > @@ -184,15 +193,59 @@ struct pingpong_dest * pp_client_exch_dest(int
> sockfd,
> >        if (!rem_dest)
> >                goto out;
> >
> > -       parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > -                       &rem_dest->psn,&rem_dest->rkey,&rem_dest->vaddr);
> > -
> > -       if (parsed != 5) {
> > -               fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg,
> > -                       msg);
> > -               free(rem_dest);
> > -               rem_dest = NULL;
> > -               goto out;
> > +       if (user_parm->gid_index < 0) {
> > +               parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > +                               &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > +               if (parsed != 5) {
> > +                       fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg, msg);
> > +                       free(rem_dest);
> > +                       rem_dest = NULL;
> > +                       goto out;
> > +               }
> > +       }else{
> > +               char *pstr = msg, *term;
> > +               char tmp[20];
> > +               int i;
> > +
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
> > +
> > +               for (i = 0; i < 15; ++i) {
> > +                       pstr += term - pstr + 1;
> > +                       term = strpbrk(pstr, ":");
> > +                       memcpy(tmp, pstr, term - pstr);
> > +                       tmp[term - pstr] = 0;
> > +                       rem_dest->dgid.raw[i] = (unsigned
> char)strtoll(tmp, NULL, 16);
> > +               }
> > +               pstr += term - pstr + 1;
> > +               strcpy(tmp, pstr);
> > +               rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp,
> NULL, 16);
> >        }
> >  out:
> >        return rem_dest;
> > @@ -254,9 +307,9 @@ int pp_server_connect(int port)
> >        return connfd;
> >  }
> >
> > -static struct pingpong_dest *pp_server_exch_dest(int connfd, const
> struct pingpong_dest *my_dest)
> > +static struct pingpong_dest *pp_server_exch_dest(int connfd, const
> struct pingpong_dest *my_dest, struct user_parameters *user_parm)
> >  {
> > -       char msg[sizeof "0000:000000:000000:00000000:0000000000000000"];
> > +       char msg[sizeof
> "0000:000000:000000:00000000:0000000000000000:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"];
> >        struct pingpong_dest *rem_dest = NULL;
> >        int parsed;
> >        int n;
> > @@ -272,18 +325,69 @@ static struct pingpong_dest
> *pp_server_exch_dest(int connfd, const struct pingpo
> >        if (!rem_dest)
> >                goto out;
> >
> > -       parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > -                       &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > -       if (parsed != 5) {
> > -               fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg,
> > -                       msg);
> > -               free(rem_dest);
> > -               rem_dest = NULL;
> > -               goto out;
> > +       if (user_parm->gid_index < 0) {
> > +               parsed = sscanf(msg, "%x:%x:%x:%x:%Lx", &rem_dest->lid,
> &rem_dest->qpn,
> > +                               &rem_dest->psn, &rem_dest->rkey,
> &rem_dest->vaddr);
> > +               if (parsed != 5) {
> > +                       fprintf(stderr, "Couldn't parse line
> <%.*s>\n",(int)sizeof msg, msg);
> > +                       free(rem_dest);
> > +                       rem_dest = NULL;
> > +                       goto out;
> > +               }
> > +       }else{
> > +               char *pstr = msg, *term;
> > +               char tmp[20];
> > +               int i;
> > +
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->lid = (int)strtol(tmp, NULL, 16); // LID
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->qpn = (int)strtol(tmp, NULL, 16); // QPN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->psn = (int)strtol(tmp, NULL, 16); // PSN
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->rkey = (unsigned)strtol(tmp, NULL, 16); // RKEY
> > +
> > +               pstr += term - pstr + 1;
> > +               term = strpbrk(pstr, ":");
> > +               memcpy(tmp, pstr, term - pstr);
> > +               tmp[term - pstr] = 0;
> > +               rem_dest->vaddr = strtoull(tmp, NULL, 16); // VA
> > +
> > +               for (i = 0; i < 15; ++i) {
> > +                       pstr += term - pstr + 1;
> > +                       term = strpbrk(pstr, ":");
> > +                       memcpy(tmp, pstr, term - pstr);
> > +                       tmp[term - pstr] = 0;
> > +                       rem_dest->dgid.raw[i] = (unsigned
> char)strtoll(tmp, NULL, 16);
> > +                       }
> > +               pstr += term - pstr + 1;
> > +               strcpy(tmp, pstr);
> > +               rem_dest->dgid.raw[15] = (unsigned char)strtoll(tmp,
> NULL, 16);
> >        }
> >
> > -       sprintf(msg, "%04x:%06x:%06x:%08x:%016Lx", my_dest->lid,
> my_dest->qpn,
> > -               my_dest->psn, my_dest->rkey, my_dest->vaddr);
> > +       sprintf(msg,
> "%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
> > +               my_dest->lid, my_dest->qpn, my_dest->psn, my_dest->rkey,
> my_dest->vaddr,
> > +               my_dest->dgid.raw[0], my_dest->dgid.raw[1],
> my_dest->dgid.raw[2],
> > +               my_dest->dgid.raw[3], my_dest->dgid.raw[4],
> my_dest->dgid.raw[5],
> > +               my_dest->dgid.raw[6], my_dest->dgid.raw[7],
> my_dest->dgid.raw[8],
> > +               my_dest->dgid.raw[9], my_dest->dgid.raw[10],
> my_dest->dgid.raw[11],
> > +               my_dest->dgid.raw[12], my_dest->dgid.raw[13],
> my_dest->dgid.raw[14],
> > +               my_dest->dgid.raw[15]);
> >        if (write(connfd, msg, sizeof msg) != sizeof msg) {
> >                perror("server write");
> >                fprintf(stderr, "Couldn't send local address\n");
> > @@ -341,7 +445,7 @@ static struct pingpong_context *pp_init_ctx(struct
> ibv_device *ib_dev,
> >                        fprintf(stderr, "Failed to query device props");
> >                        return NULL;
> >                }
> > -               if (device_attr.vendor_part_id == 23108) {
> > +               if (device_attr.vendor_part_id == 23108 ||
> user_parm->gid_index > -1) {
> >                        user_parm->mtu = 1024;
> >                } else {
> >                        user_parm->mtu = 2048;
> > @@ -444,9 +548,16 @@ static int pp_connect_ctx(struct pingpong_context
> *ctx, int port, int my_psn,
> >                attr.max_dest_rd_atomic     = 1;
> >                attr.min_rnr_timer          = 12;
> >        }
> > -       attr.ah_attr.is_global  = 0;
> > -       attr.ah_attr.dlid       = dest->lid;
> > -       attr.ah_attr.sl         = sl;
> > +       if (user_parm->gid_index < 0) {
> > +               attr.ah_attr.is_global  = 0;
> > +               attr.ah_attr.dlid       = dest->lid;
> > +               attr.ah_attr.sl         = sl;
> > +       } else {
> > +               attr.ah_attr.is_global  = 1;
> > +               attr.ah_attr.grh.dgid   = dest->dgid;
> > +               attr.ah_attr.grh.hop_limit = 1;
> > +               attr.ah_attr.sl         = 0;
> > +       }
> >        attr.ah_attr.src_path_bits = 0;
> >        attr.ah_attr.port_num   = port;
> >        if (user_parm->connection_type == RC) {
> > @@ -524,6 +635,7 @@ static void usage(const char *argv0)
> >        printf("  -I, --inline_size=<size>  max size of message to be sent
> in inline mode (default 400)\n");
> >        printf("  -u, --qp-timeout=<timeout> QP timeout, timeout value is
> 4 usec * 2 ^(timeout), default 14\n");
> >        printf("  -S, --sl=<sl>             SL (default 0)\n");
> > +       printf("  -x, --gid-inde
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ewg/attachments/20090804/f63cfbd8/attachment.html>


More information about the ewg mailing list