[ofw] patch 2/2 Add support for RDMAoEth to the low level driver
Tzachi Dar
tzachid at mellanox.co.il
Wed Dec 2 08:29:39 PST 2009
This patch contains the tests that I have been using to check that
RDMAoEth works.
It includes two tests that have been changes to support RDMAoEth :
ibv_rc_pingpong.exe and ib_send_Bw.
Both tests have been using GRH.
A typical command line looks like:
running the rc_pingpong:
sw328: ibv_rc_pingpong.exe -g fe800000000000000202c9fffe039cf0 -n
100000 -s 1
sw327: ibv_rc_pingpong.exe -g fe800000000000000202c9fffe039d18 -h
10.4.12.128 -n 100000 -s 1
sw327: ib_send_Bw -a -g fe800000000000000202c9fffe039d18 10.4.12.128
sw328: ib_send_bw -g fe800000000000000202c9fffe039cf0 -a
Please note that the GRH is being created from the mac addresses which
in my case is:
sw327 - 00-02-C9-03-9C-F0
sw328 - 00-02-C9-03-9D-18
Thanks
Tzachi
Index: core/al/kernel/al_cm_cep.c
===================================================================
--- core/al/kernel/al_cm_cep.c (revision 2617)
+++ core/al/kernel/al_cm_cep.c (working copy)
@@ -4390,6 +4390,10 @@
p_av->attr.grh_valid = !ib_gid_is_link_local( &p_path->dgid );
+ // make this same as Linux code: p_path->hop_flow_raw >= 2 means grh
present.
+ if(cl_hton32(p_path->hop_flow_raw) >= 2)
+ p_av->attr.grh_valid = 1;
+
p_av->attr.static_rate = ib_path_rec_rate( p_path );
p_av->attr.path_bits = (uint8_t)(p_path->slid - p_port_cep->base_lid);
Index: tools/perftests/user/perf_defs.h
===================================================================
--- tools/perftests/user/perf_defs.h (revision 2617)
+++ tools/perftests/user/perf_defs.h (working copy)
@@ -154,4 +154,6 @@
pp_server_exch_dest(SOCKET sockfd, const struct pingpong_dest *my_dest,
struct pingpong_dest* rem_dest);
+void str2gid(char *grh, ib_gid_t *gid);
+
#endif
Index: tools/perftests/user/perf_utils.c
===================================================================
--- tools/perftests/user/perf_utils.c (revision 2617)
+++ tools/perftests/user/perf_utils.c (working copy)
@@ -238,9 +238,29 @@
return pp_write_keys(sockfd, my_dest);
}
+void str2gid(char *grh, ib_gid_t *gid)
+{
+ char tmp;
+ tmp = grh[8];
+ grh[8] = 0;
+ (*((ULONG*)&gid->raw[0])) = htonl(strtoul(grh, NULL, 16));
+ grh[8] = tmp;
+ tmp = grh[16];
+ grh[16] = 0;
+ (*((ULONG*)&gid->raw[4])) = htonl(strtoul(grh + 8, NULL, 16));
+ grh[16] = tmp;
+ tmp = grh[24];
+ grh[24] = 0;
+ (*((ULONG*)&gid->raw[8])) = htonl(strtoul(grh + 16, NULL, 16));
+ grh[24] = tmp;
+ (*((ULONG*)&gid->raw[12])) = htonl(strtoul(grh + 24, NULL, 16));
+}
+
+
+
Index: tools/perftests/user/send_bw/send_bw.c
===================================================================
--- tools/perftests/user/send_bw/send_bw.c (revision 2617)
+++ tools/perftests/user/send_bw/send_bw.c (working copy)
@@ -339,6 +339,8 @@
}
+char *grh = NULL;
+
static int pp_connect_ctx(struct pingpong_context *ctx, int port, int
my_psn,
struct pingpong_dest *dest, struct user_parameters *user_parm,int
index)
{
@@ -379,6 +381,15 @@
IB_MOD_QP_RESP_RES |
IB_MOD_QP_PRIMARY_AV;
+ if(grh)
+ {
+ ib_gid_t dgid;
+ str2gid(grh, &dgid);
+ attr.state.rtr.primary_av.grh_valid = 1;
+ attr.state.rtr.primary_av.grh.hop_limit = 1;
+ attr.state.rtr.primary_av.grh.dest_gid = dgid;
+ attr.state.rtr.primary_av.grh.src_gid =
ctx->ca_attr->p_port_attr->p_gid_table[0];
+ }
ib_status = ib_modify_qp(ctx->qp[0], &attr);
if(ib_status != IB_SUCCESS){
@@ -505,7 +516,7 @@
my_dest[i].psn = cl_hton32(rand() & 0xffffff);
if (!my_dest[i].lid) {
fprintf(stderr, "Local lid 0x0 detected. Is an SM running?\n");
- return 1;
+//?? return 1;
}
my_dest[i].qpn = ctx->qp_attr[i].num;
/* TBD this should be changed inot VA and different key to each qp */
@@ -977,10 +988,11 @@
{ "bidirectional", 0, NULL, 'b' },
{ "version", 0, NULL, 'V' },
{ "events", 0, NULL, 'e' },
+ { "grh", 1, NULL, 'g' },
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:ebaV", long_options,
NULL);
+ c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:g:ebaV", long_options,
NULL);
if (c == -1)
break;
@@ -1049,6 +1061,11 @@
case 'b':
user_param.duplex = 1;
break;
+
+ case 'g':
+ grh = _strdup(optarg);
+ break;
+
case 'h':
default:
usage(argv[0]);
Index: ulp/libibverbs/examples/pingpong.c
===================================================================
--- ulp/libibverbs/examples/pingpong.c (revision 2617)
+++ ulp/libibverbs/examples/pingpong.c (working copy)
@@ -50,3 +50,25 @@
return attr.lid;
}
+
+void str2gid(char *grh, union ibv_gid *gid)
+{
+ char tmp;
+
+ tmp = grh[8];
+ grh[8] = 0;
+ (*((ULONG*)&gid->raw[0])) = htonl(strtoul(grh, NULL, 16));
+ grh[8] = tmp;
+
+ tmp = grh[16];
+ grh[16] = 0;
+ (*((ULONG*)&gid->raw[4])) = htonl(strtoul(grh + 8, NULL, 16));
+ grh[16] = tmp;
+
+ tmp = grh[24];
+ grh[24] = 0;
+ (*((ULONG*)&gid->raw[8])) = htonl(strtoul(grh + 16, NULL, 16));
+ grh[24] = tmp;
+
+ (*((ULONG*)&gid->raw[12])) = htonl(strtoul(grh + 24, NULL, 16));
+}
Index: ulp/libibverbs/examples/pingpong.h
===================================================================
--- ulp/libibverbs/examples/pingpong.h (revision 2617)
+++ ulp/libibverbs/examples/pingpong.h (working copy)
@@ -34,5 +34,6 @@
enum ibv_mtu pp_mtu_to_enum(int mtu);
uint16_t pp_get_local_lid(struct ibv_context *context, int port);
+void str2gid(char *grh, union ibv_gid *gid);
#endif /* IBV_PINGPONG_H */
Index: ulp/libibverbs/examples/rc_pingpong/rc_pingpong.c
===================================================================
--- ulp/libibverbs/examples/rc_pingpong/rc_pingpong.c (revision 2617)
+++ ulp/libibverbs/examples/rc_pingpong/rc_pingpong.c (working copy)
@@ -44,6 +44,8 @@
PINGPONG_SEND_WRID = 2,
};
+char *grh = NULL;
+
struct pingpong_context {
struct ibv_context *context;
struct ibv_comp_channel *channel;
@@ -55,6 +57,8 @@
int size;
int rx_depth;
int pending;
+ struct ibv_port_attr portinfo;
+ union ibv_gid dgid;
};
struct pingpong_dest {
@@ -69,6 +73,7 @@
{
struct ibv_qp_attr attr;
+ attr.port_num = (u8)port;
attr.qp_state = IBV_QPS_RTR;
attr.path_mtu = mtu;
attr.dest_qp_num = dest->qpn;
@@ -81,6 +86,15 @@
attr.ah_attr.src_path_bits = 0;
attr.ah_attr.port_num = (uint8_t) port;
+ if(grh)
+ {
+ str2gid(grh, &ctx->dgid);
+ attr.ah_attr.is_global = 1;
+ attr.ah_attr.grh.hop_limit = 1;
+ attr.ah_attr.grh.dgid = ctx->dgid;
+ attr.ah_attr.grh.sgid_index = 0;
+ }
+
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_AV |
@@ -499,7 +513,7 @@
while (1) {
int c;
- c = getopt(argc, argv, "h:p:d:i:s:m:r:n:l:e");
+ c = getopt(argc, argv, "h:p:d:i:s:m:r:n:l:g:e");
if (c == -1)
break;
@@ -555,6 +569,10 @@
servername = _strdup(optarg);
break;
+ case 'g':
+ grh = _strdup(optarg);
+ break;
+
default:
usage(argv[0]);
return 1;
@@ -606,7 +624,7 @@
my_dest.psn = rand() & 0xffffff;
if (!my_dest.lid) {
fprintf(stderr, "Couldn't get local LID\n");
- return 1;
+ //return 1;
}
printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n",
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20091202/ab6c7042/attachment.html>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: lle_tests_ofed.patch
Type: application/octet-stream
Size: 6539 bytes
Desc: lle_tests_ofed.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20091202/ab6c7042/attachment.obj>
More information about the ofw
mailing list