From Line.Holen at Sun.COM Wed Jul 1 00:32:25 2009 From: Line.Holen at Sun.COM (Line.Holen at Sun.COM) Date: Wed, 01 Jul 2009 09:32:25 +0200 Subject: [ofa-general] [PATCH] osm_ucast_ftree.c Count number of hops instead of calculating it Message-ID: <4A4B1109.9010709@Sun.COM> The ftree algorithm is currently calculating number of hops based on switch ranks and reverse hops. This patch replaces this with a simple counter. Signed-off-by: Frank Olaf Sem-Jacobsen Signed-off-by: Line Holen --- This patch is based on master with Nicolas latest patch applied ("opensm/osm_ucast_ftree.c: Fixed issue with reverse_hops ....") diff --git a/opensm/opensm/osm_ucast_ftree.c b/opensm/opensm/osm_ucast_ftree.c index 5a9eeff..d654310 100644 --- a/opensm/opensm/osm_ucast_ftree.c +++ b/opensm/opensm/osm_ucast_ftree.c @@ -1,4 +1,5 @@ /* + * Copyright (c) 2009 Simula Research Laboratory. All rights reserved. * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. * Copyright (c) 2002-2007 Mellanox Technologies LTD. All rights reserved. @@ -2058,12 +2059,11 @@ fabric_route_upgoing_by_going_down(IN ftree_fabric_t * p_ftree, IN ftree_sw_t * p_sw, IN ftree_sw_t * p_prev_sw, IN uint16_t target_lid, - IN uint8_t target_rank, IN boolean_t is_real_lid, IN boolean_t is_main_path, IN boolean_t is_target_a_sw, - IN uint8_t highest_rank_in_route, - IN uint16_t reverse_hops) + IN uint16_t reverse_hops, + IN uint8_t current_hops) { ftree_sw_t *p_remote_sw; uint16_t ports_num; @@ -2074,6 +2074,7 @@ fabric_route_upgoing_by_going_down(IN ftree_fabric_t * p_ftree, uint16_t k; boolean_t created_route = FALSE; boolean_t routed = 0; + uint8_t least_hops; /* we shouldn't enter here if both real_lid and main_path are false */ CL_ASSERT(is_real_lid || is_main_path); @@ -2122,23 +2123,21 @@ fabric_route_upgoing_by_going_down(IN ftree_fabric_t * p_ftree, Set on the remote switch how to get to the target_lid - set LFT(target_lid) on the remote switch to the remote port */ p_remote_sw = p_group->remote_hca_or_sw.p_sw; + least_hops = sw_get_least_hops(p_remote_sw, target_lid); - if (sw_get_least_hops(p_remote_sw, target_lid) != OSM_NO_PATH) { + if (least_hops != OSM_NO_PATH) { /* Loop in the fabric - we already routed the remote switch on our way UP, and now we see it again on our way DOWN */ OSM_LOG(&p_ftree->p_osm->log, OSM_LOG_DEBUG, "Loop of lenght %d in the fabric:\n " "Switch %s (LID %u) closes loop through switch %s (LID %u)\n", - (p_remote_sw->rank - highest_rank_in_route) * 2, + current_hops, tuple_to_str(p_remote_sw->tuple), p_group->base_lid, tuple_to_str(p_sw->tuple), p_group->remote_base_lid); /* We skip only if we have come through a longer path */ - if (((target_rank - highest_rank_in_route) + - (p_remote_sw->rank - highest_rank_in_route) + - 2 * reverse_hops) >= sw_get_least_hops(p_remote_sw, - target_lid)) + if (current_hops + 1 >= least_hops) continue; } @@ -2182,12 +2181,7 @@ fabric_route_upgoing_by_going_down(IN ftree_fabric_t * p_ftree, set hops for ALL the ports in the remote group. */ set_hops_on_remote_sw(p_group, target_lid, - ((target_rank - - highest_rank_in_route) + - (p_remote_sw->rank - - highest_rank_in_route) + - reverse_hops * 2), - is_target_a_sw); + current_hops + 1, is_target_a_sw); } /* The number of upgoing routes is tracked in the @@ -2203,11 +2197,11 @@ fabric_route_upgoing_by_going_down(IN ftree_fabric_t * p_ftree, routed = fabric_route_upgoing_by_going_down(p_ftree, p_remote_sw, /* remote switch - used as a route-upgoing alg. start point */ NULL, /* prev. position - NULL to mark that we went down and not up */ target_lid, /* LID that we're routing to */ - target_rank, /* rank of the LID that we're routing to */ is_real_lid, /* whether the target LID is real or dummy */ is_main_path, /* whether this is path to HCA that should by tracked by counters */ is_target_a_sw, /* Wheter target lid is a switch or not */ - highest_rank_in_route, reverse_hops); /* highest visited point in the tree before going down */ + reverse_hops, + current_hops + 1); created_route |= routed; if (routed) { p_min_port->counter_up++; @@ -2245,12 +2239,12 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, IN ftree_sw_t * p_sw, IN ftree_sw_t * p_prev_sw, IN uint16_t target_lid, - IN uint8_t target_rank, IN boolean_t is_real_lid, IN boolean_t is_main_path, IN boolean_t is_target_a_sw, IN uint16_t reverse_hop_credit, - IN uint16_t reverse_hops) + IN uint16_t reverse_hops, + IN uint8_t current_hops) { ftree_sw_t *p_remote_sw; uint16_t ports_num; @@ -2270,12 +2264,11 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, created_route = fabric_route_upgoing_by_going_down(p_ftree, p_sw, /* local switch - used as a route-upgoing alg. start point */ p_prev_sw, /* switch that we went up from (NULL means that we went down) */ target_lid, /* LID that we're routing to */ - target_rank, /* rank of the LID that we're routing to */ is_real_lid, /* whether this target LID is real or dummy */ is_main_path, /* whether this path to HCA should by tracked by counters */ is_target_a_sw, /* Wheter target lid is a switch or not */ - p_sw->rank, /* the highest visited point in the tree before going down */ - reverse_hops); /* Number of reverse_hops done up to this point */ + reverse_hops, /* Number of reverse_hops done up to this point */ + current_hops); /* recursion stop condition - if it's a root switch, */ if (p_sw->rank == 0) { @@ -2300,12 +2293,14 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, created_route |= fabric_route_downgoing_by_going_up(p_ftree, p_remote_sw, /* remote switch - used as a route-downgoing alg. next step point */ p_sw, /* this switch - prev. position switch for the function */ target_lid, /* LID that we're routing to */ - target_rank, /* rank of the LID that we're routing to */ is_real_lid, /* whether this target LID is real or dummy */ is_main_path, /* whether this is path to HCA that should by tracked by counters */ is_target_a_sw, /* Wheter target lid is a switch or not */ reverse_hop_credit - 1, /* Remaining reverse_hops allowed */ - reverse_hops + 1); /* Number of reverse_hops done up to this point */ + reverse_hops + 1, /* Number of reverse_hops done up to this point */ + current_hops + + + 1); } } @@ -2404,8 +2399,7 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, ((p_remote_sw->p_osm_sw->new_lft[target_lid] != OSM_NO_PATH) && - ((target_rank - p_remote_sw->rank + - 2 * reverse_hops) < + (current_hops + 1 < sw_get_least_hops(p_remote_sw, target_lid)))) { p_remote_sw->p_osm_sw->new_lft[target_lid] = @@ -2420,9 +2414,7 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, set hops for ALL the ports in the remote group. */ set_hops_on_remote_sw(p_min_group, target_lid, - target_rank - - p_remote_sw->rank + - 2 * reverse_hops, + current_hops + 1, is_target_a_sw); /* Recursion step: @@ -2430,12 +2422,14 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, created_route |= fabric_route_downgoing_by_going_up(p_ftree, p_remote_sw, /* remote switch - used as a route-downgoing alg. next step point */ p_sw, /* this switch - prev. position switch for the function */ target_lid, /* LID that we're routing to */ - target_rank, /* rank of the LID that we're routing to */ is_real_lid, /* whether this target LID is real or dummy */ is_main_path, /* whether this is path to HCA that should by tracked by counters */ is_target_a_sw, /* Wheter target lid is a switch or not */ reverse_hop_credit, /* Remaining reverse_hops allowed */ - reverse_hops); /* Number of reverse_hops done up to this point */ + reverse_hops, /* Number of reverse_hops done up to this point */ + current_hops + + + 1); } } } @@ -2481,9 +2475,8 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, /* skip if target lid has been already set on remote switch fwd tbl (with a bigger hop count) */ if (p_remote_sw->p_osm_sw->new_lft[target_lid] != OSM_NO_PATH) - if ((target_rank - p_remote_sw->rank + - 2 * reverse_hops) >= sw_get_least_hops(p_remote_sw, - target_lid)) + if (current_hops + 1 >= + sw_get_least_hops(p_remote_sw, target_lid)) continue; if (p_sw->is_leaf) { @@ -2524,20 +2517,19 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, set hops for ALL the ports in the remote group. */ set_hops_on_remote_sw(p_group, target_lid, - target_rank - p_remote_sw->rank + - 2 * reverse_hops, is_target_a_sw); + current_hops + 1, is_target_a_sw); /* Recursion step: Assign downgoing ports by stepping up, starting on REMOTE switch. */ routed = fabric_route_downgoing_by_going_up(p_ftree, p_remote_sw, /* remote switch - used as a route-downgoing alg. next step point */ p_sw, /* this switch - prev. position switch for the function */ target_lid, /* LID that we're routing to */ - target_rank, /* rank of the LID that we're routing to */ TRUE, /* whether the target LID is real or dummy */ FALSE, /* whether this is path to HCA that should by tracked by counters */ is_target_a_sw, /* Wheter target lid is a switch or not */ reverse_hop_credit, /* Remaining reverse_hops allowed */ - reverse_hops); /* Number of reverse_hops done up to this point */ + reverse_hops, /* Number of reverse_hops done up to this point */ + current_hops + 1); created_route |= routed; } @@ -2562,12 +2554,13 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, created_route |= fabric_route_downgoing_by_going_up(p_ftree, p_remote_sw, /* remote switch - used as a route-downgoing alg. next step point */ p_sw, /* this switch - prev. position switch for the function */ target_lid, /* LID that we're routing to */ - target_rank, /* rank of the LID that we're routing to */ TRUE, /* whether the target LID is real or dummy */ TRUE, /* whether this is path to HCA that should by tracked by counters */ is_target_a_sw, /* Wheter target lid is a switch or not */ reverse_hop_credit - 1, /* Remaining reverse_hops allowed */ - reverse_hops + 1); /* Number of reverse_hops done up to this point */ + reverse_hops + 1, /* Number of reverse_hops done up to this point */ + current_hops + + 1); } return created_route; @@ -2652,12 +2645,12 @@ static void fabric_route_to_cns(IN ftree_fabric_t * p_ftree) fabric_route_downgoing_by_going_up(p_ftree, p_sw, /* local switch - used as a route-downgoing alg. start point */ NULL, /* prev. position switch */ hca_lid, /* LID that we're routing to */ - p_sw->rank + 1, /* rank of the LID that we're routing to */ TRUE, /* whether this HCA LID is real or dummy */ TRUE, /* whether this path to HCA should by tracked by counters */ FALSE, /* wheter target lid is a switch or not */ 0, /* Number of reverse hops allowed */ - 0); /* Number of reverse hops done yet */ + 0, /* Number of reverse hops done yet */ + 1); /* Number of hops done yet */ /* count how many real targets have been routed from this leaf switch */ routed_targets_on_leaf++; @@ -2680,12 +2673,12 @@ static void fabric_route_to_cns(IN ftree_fabric_t * p_ftree) fabric_route_downgoing_by_going_up(p_ftree, p_sw, /* local switch - used as a route-downgoing alg. start point */ NULL, /* prev. position switch */ 0, /* LID that we're routing to - ignored for dummy HCA */ - 0, /* rank of the LID that we're routing to - ignored for dummy HCA */ FALSE, /* whether this HCA LID is real or dummy */ TRUE, /* whether this path to HCA should by tracked by counters */ FALSE, /* Wheter the target LID is a switch or not */ 0, /* Number of reverse hops allowed */ - 0); /* Number of reverse hops done yet */ + 0, /* Number of reverse hops done yet */ + 1); /* Number of hops done yet */ } } } @@ -2767,12 +2760,12 @@ static void fabric_route_to_non_cns(IN ftree_fabric_t * p_ftree) fabric_route_downgoing_by_going_up(p_ftree, p_sw, /* local switch - used as a route-downgoing alg. start point */ NULL, /* prev. position switch */ hca_lid, /* LID that we're routing to */ - p_sw->rank + 1, /* rank of the LID that we're routing to */ TRUE, /* whether this HCA LID is real or dummy */ TRUE, /* whether this path to HCA should by tracked by counters */ FALSE, /* Wheter the target LID is a switch or not */ p_hca_port_group->is_io ? p_ftree->p_osm->subn.opt.max_reverse_hops : 0, /* Number or reverse hops allowed */ - 0); /* Number or reverse hops done yet */ + 0, /* Number or reverse hops done yet */ + 1); /* Number of hops done yet */ } /* done with all the port groups of this HCA - go to next HCA */ } @@ -2820,12 +2813,12 @@ static void fabric_route_to_switches(IN ftree_fabric_t * p_ftree) fabric_route_downgoing_by_going_up(p_ftree, p_sw, /* local switch - used as a route-downgoing alg. start point */ NULL, /* prev. position switch */ p_sw->base_lid, /* LID that we're routing to */ - p_sw->rank, /* rank of the LID that we're routing to */ TRUE, /* whether the target LID is a real or dummy */ FALSE, /* whether this path to HCA should by tracked by counters */ TRUE, /* Wheter the target LID is a switch or not */ 0, /* Number of reverse hops allowed */ - 0); /* Number of reverse hops done yet */ + 0, /* Number of reverse hops done yet */ + 0); /* Number of hops done yet */ } OSM_LOG_EXIT(&p_ftree->p_osm->log); From vlad at lists.openfabrics.org Wed Jul 1 02:22:27 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Wed, 1 Jul 2009 02:22:27 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090701-0200 daily build status Message-ID: <20090701092227.8A391E304D5@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-8.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-8.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-53.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-53.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-53.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.20_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.20_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.27_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1383: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1402: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1383: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1402: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_ia64_check/drivers/infiniband/core/addr.c:36: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_ia64_check/kernel_addons/backport/2.6.22/include/asm/bitops.h: In function 'clear_bit_unlock': /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_ia64_check/kernel_addons/backport/2.6.22/include/asm/bitops.h:8: error: implicit declaration of function 'smp_mb' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_ia64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_ia64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.23_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.23_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18-8.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.18-8.el5_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18-8.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ppc64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ppc64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090701-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From sumit.panchasara at einfochips.com Wed Jul 1 03:11:25 2009 From: sumit.panchasara at einfochips.com (Sumit Panchasara) Date: Wed, 1 Jul 2009 15:41:25 +0530 Subject: [ofa-general] ib_rdma_bw - memory leaks? Message-ID: <000b01c9fa34$4b407a50$e1c16ef0$@panchasara@einfochips.com> I am seeing Memory Leaks in ib_rdma_bw application at below places when ran with valgrind tool: 1. Memory allocated by aspintf() is not freed, as it goes on heap, caller should call "free" after done with the string. 2. Memory allocated by remote destination filed (struct pp_data *) is not freed. 3. rdma_create_event_channel() calls ucma_init() but rdma_destroy_event_channel() does not call ucma_cleanup(), this results into memory leak at provider's library since it does not call ibv_close_device() and thus unable to do *->free_context(). 4. Memory allocated for (struct cma_id_private *) at RDMA_CM_EVENT_CONNECT_REQUEST event is not getting freed. It is calling rdma_destroy_id but that was for rdma_create_id. Thus one block of cm_id_private still remains allocated. This accounts to ~533 bytes in each iteration. Am I observing correctly? Commands used: server: ib_rdma_bw -p 5001 -s 65536 -t 500 -n 5000 -c client: ib_rdma_bw -p 5001 -s 65536 -t 500 -n 5000 -c Thanks, Sumit Panchasara Embedded Engineer - eInfochips Ltd., India. www.einfochips.com -- _____________________________________________________________________ Disclaimer: This e-mail message and all attachments transmitted with it are intended solely for the use of the addressee and may contain legally privileged and confidential information. If the reader of this message is not the intended recipient, or an employee or agent responsible for delivering this message to the intended recipient, you are hereby notified that any dissemination, distribution, copying, or other use of this message or its attachments is strictly prohibited. If you have received this message in error, please notify the sender immediately by replying to this message and please delete it from your computer. Any views expressed in this message are those of the individual sender unless otherwise stated.Company has taken enough precautions to prevent the spread of viruses. However the company accepts no liability for any damage caused by any virus transmitted by this email. _____________________________________________________________________ From hnrose at comcast.net Wed Jul 1 04:13:17 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Wed, 1 Jul 2009 07:13:17 -0400 Subject: [ofa-general] [PATCH] opensm/osmtest: Fix some typos Message-ID: <20090701111317.GA12638@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/opensm/osmtest/osmt_inform.c b/opensm/osmtest/osmt_inform.c index c252a4c..d3558ce 100644 --- a/opensm/osmtest/osmt_inform.c +++ b/opensm/osmtest/osmt_inform.c @@ -725,7 +725,7 @@ ib_api_status_t osmt_run_trap64_65_flow(IN osmtest_t * const p_osmt) } /*--------------------- PREP -------------------------*/ - if (osmt_mtl_mad_post_recv_bufs(&qp_ctx.qp_bind_hndl, qp_ctx.p_recv_buf, 1, /* we need to reveive the report */ + if (osmt_mtl_mad_post_recv_bufs(&qp_ctx.qp_bind_hndl, qp_ctx.p_recv_buf, 1, /* we need to receive the report */ GRH_LEN + MAD_BLOCK_SIZE, 1) != 1) { OSM_LOG(&p_osmt->log, OSM_LOG_ERROR, "ERR 0127: " "Error posting recv bufs for trap 65\n"); diff --git a/opensm/osmtest/osmt_multicast.c b/opensm/osmtest/osmt_multicast.c index 165457c..8a3342c 100644 --- a/opensm/osmtest/osmt_multicast.c +++ b/opensm/osmtest/osmt_multicast.c @@ -2069,7 +2069,7 @@ ib_api_status_t osmt_run_mcast_flow(IN osmtest_t * const p_osmt) if ((status != IB_REMOTE_ERROR) || (res_sa_mad.status != IB_SA_MAD_STATUS_REQ_INVALID)) { OSM_LOG(&p_osmt->log, OSM_LOG_ERROR, "ERR 02C6: " - "Failed to catch BAD RATE joining an exiting MGID: %s/%s\n", + "Failed to catch BAD RATE joining an existing MGID: %s/%s\n", ib_get_err_str(status), ib_get_mad_status_str((ib_mad_t *) (&res_sa_mad))); status = IB_ERROR; @@ -2078,7 +2078,7 @@ ib_api_status_t osmt_run_mcast_flow(IN osmtest_t * const p_osmt) /* Try MTU that does not exist in any MCG */ OSM_LOG(&p_osmt->log, OSM_LOG_INFO, - "Checking BAD MTU (higher them max) when connecting to " + "Checking BAD MTU (higher than max) when connecting to " "existing MGID (o15.0.1.13)...\n"); OSM_LOG(&p_osmt->log, OSM_LOG_ERROR, EXPECTING_ERRORS_START "\n"); @@ -2094,7 +2094,7 @@ ib_api_status_t osmt_run_mcast_flow(IN osmtest_t * const p_osmt) if ((status != IB_REMOTE_ERROR) || (res_sa_mad.status != IB_SA_MAD_STATUS_REQ_INVALID)) { OSM_LOG(&p_osmt->log, OSM_LOG_ERROR, "ERR 02C7: " - "Failed to catch BAD RATE (higher them max) joining an exiting MGID: %s/%s\n", + "Failed to catch BAD RATE (higher than max) joining an existing MGID: %s/%s\n", ib_get_err_str(status), ib_get_mad_status_str((ib_mad_t *) (&res_sa_mad))); status = IB_ERROR; @@ -2119,7 +2119,7 @@ ib_api_status_t osmt_run_mcast_flow(IN osmtest_t * const p_osmt) if ((status != IB_REMOTE_ERROR) || (res_sa_mad.status != IB_SA_MAD_STATUS_REQ_INVALID)) { OSM_LOG(&p_osmt->log, OSM_LOG_ERROR, "ERR 02C8: " - "Failed to catch BAD RATE (less them min) joining an exiting MGID: %s/%s\n", + "Failed to catch BAD RATE (less than min) joining an existing MGID: %s/%s\n", ib_get_err_str(status), ib_get_mad_status_str((ib_mad_t *) (&res_sa_mad))); status = IB_ERROR; @@ -2531,8 +2531,8 @@ ib_api_status_t osmt_run_mcast_flow(IN osmtest_t * const p_osmt) if ((res_sa_mad. status & IB_SMP_STATUS_MASK) == IB_SA_MAD_STATUS_NO_RESOURCES) { - /* You can quitly exit the loop since no available mlid in SA DB - i.e. reached the maximum valiad avalable mlid */ + /* You can quietly exit the loop since no available mlid in SA DB + i.e. reached the maximum valid avalable mlid */ ReachedMlidLimit = TRUE; } } else { From lars.ellenberg at linbit.com Wed Jul 1 05:52:26 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Wed, 1 Jul 2009 14:52:26 +0200 Subject: [ofa-general] [patch] fix SDP page leak in sdp_bz_cleanup Message-ID: <20090701125226.GD9112@soda.linbit> Hi there. Please direct me to the right place for posting patches and contacting developers, in case this is not it. A few days ago I started to try using SDP as DRBD replication transport. (http://www.drbd.org for those of you that do not know us yet) As block traffic naturally is mostly large messages (several pages), I noticed pretty quickly a large memory leak in SDP. I think I can pin it down on sdp_bz_cleanup not doing the put_page(). patch against current ofed 1.5, also applies to the 1.4 branch. All the best, Lars from git://git.openfabrics.org/ofed_1_5/linux-2.6.git diff --git a/drivers/infiniband/ulp/sdp/sdp_main.c b/drivers/infiniband/ulp/sdp/sdp_main.c index 8e1a6d7..1740408 100644 --- a/drivers/infiniband/ulp/sdp/sdp_main.c +++ b/drivers/infiniband/ulp/sdp/sdp_main.c @@ -1368,7 +1368,8 @@ static inline struct bzcopy_state *sdp_bz_cleanup(struct bzcopy_state *bz) } if (bz->pages) { - for (i = bz->cur_page; i < bz->page_cnt; i++) + /* match get_page() calls in sdp_bz_setup() */ + for (i = 0; i < bz->page_cnt; i++) put_page(bz->pages[i]); kfree(bz->pages); -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. From amirv at mellanox.co.il Wed Jul 1 06:02:17 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Wed, 01 Jul 2009 16:02:17 +0300 Subject: [ofa-general] Re: [patch] fix SDP page leak in sdp_bz_cleanup In-Reply-To: <20090701125226.GD9112@soda.linbit> References: <20090701125226.GD9112@soda.linbit> Message-ID: <4A4B5E59.2030001@mellanox.co.il> Hi Lars, This is the right place for posting patches. I will commit it ASAP into both branches. -- Amir Vadai Software Eng. Mellanox Technologies mailto: amirv at mellanox.co.il Tel +972-3-6259539 On 07/01/2009 03:52 PM, Lars Ellenberg wrote: > Hi there. > > Please direct me to the right place for posting patches > and contacting developers, in case this is not it. > > A few days ago I started to try using SDP as DRBD replication transport. > (http://www.drbd.org for those of you that do not know us yet) > > As block traffic naturally is mostly large messages (several pages), > I noticed pretty quickly a large memory leak in SDP. > > I think I can pin it down on sdp_bz_cleanup not doing the put_page(). > > patch against current ofed 1.5, > also applies to the 1.4 branch. > > All the best, > > Lars > > from git://git.openfabrics.org/ofed_1_5/linux-2.6.git > > diff --git a/drivers/infiniband/ulp/sdp/sdp_main.c b/drivers/infiniband/ulp/sdp/sdp_main.c > index 8e1a6d7..1740408 100644 > --- a/drivers/infiniband/ulp/sdp/sdp_main.c > +++ b/drivers/infiniband/ulp/sdp/sdp_main.c > @@ -1368,7 +1368,8 @@ static inline struct bzcopy_state *sdp_bz_cleanup(struct bzcopy_state *bz) > } > > if (bz->pages) { > - for (i = bz->cur_page; i < bz->page_cnt; i++) > + /* match get_page() calls in sdp_bz_setup() */ > + for (i = 0; i < bz->page_cnt; i++) > put_page(bz->pages[i]); > > kfree(bz->pages); > > > > From fenkes at de.ibm.com Wed Jul 1 06:34:56 2009 From: fenkes at de.ibm.com (Joachim Fenkes) Date: Wed, 1 Jul 2009 15:34:56 +0200 Subject: [ofa-general] [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: <20090630180003.GD2982@obsidianresearch.com> References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> Message-ID: <200907011534.59787.fenkes@de.ibm.com> Previously, libibmad reacted to GSI MAD responses with a "redirect" status by throwing an error. IBM eHCA adapters use redirection, so most infiniband_diags tools didn't work against eHCA. Fix: Modify mad_rpc() so that it resends the request to the redirection target if a "redirect" GS response is received. This is repeated until no "redirect" response is received, allowing for multiple levels of indirection. The dport argument is updated with the redirection target, so subsequent MADs will not go through the redirection process again but reach the target directly. Tested using perfquery between ehca, mlx4 and mthca in all possible combinations. Signed-off-by: Joachim Fenkes --- Hi, Hal and Jason, here's an updated patch that will bail on GID-routed redirection. Also, I moved the redirection itself into its own function so it can easily be included into RMPP as well. Of course, I tested this again using ehca, mthca and mlx4. If you have nothing to add to this patch, please queue it for OFED 1.5. Thanks and regards, Joachim libibmad/include/infiniband/mad.h | 9 +++++ libibmad/src/gs.c | 6 ++- libibmad/src/rpc.c | 65 ++++++++++++++++++++++++++++-------- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/libibmad/include/infiniband/mad.h b/libibmad/include/infiniband/mad.h index aa27eb5..bdf5158 100644 --- a/libibmad/include/infiniband/mad.h +++ b/libibmad/include/infiniband/mad.h @@ -115,6 +115,8 @@ enum MAD_ATTR_ID { enum MAD_STATUS { IB_MAD_STS_OK = (0 << 2), + IB_MAD_STS_BUSY = (1 << 0), + IB_MAD_STS_REDIRECT = (1 << 1), IB_MAD_STS_BAD_BASE_VER_OR_CLASS = (1 << 2), IB_MAD_STS_METHOD_NOT_SUPPORTED = (2 << 2), IB_MAD_STS_METHOD_ATTR_NOT_SUPPORTED = (3 << 2), @@ -783,8 +785,15 @@ MAD_EXPORT int madrpc_set_timeout(int timeout); MAD_EXPORT struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port, int *mgmt_classes, int num_classes); MAD_EXPORT void mad_rpc_close_port(struct ibmad_port *srcport); + +/* + * On redirection, the dport argument is updated with the redirection target, + * so subsequent MADs will not go through the redirection process again but + * reach the target directly. + */ MAD_EXPORT void *mad_rpc(const struct ibmad_port *srcport, ib_rpc_t * rpc, ib_portid_t * dport, void *payload, void *rcvdata); + MAD_EXPORT void *mad_rpc_rmpp(const struct ibmad_port *srcport, ib_rpc_t * rpc, ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, void *data); diff --git a/libibmad/src/gs.c b/libibmad/src/gs.c index f3d245e..c7e4ff6 100644 --- a/libibmad/src/gs.c +++ b/libibmad/src/gs.c @@ -70,7 +70,8 @@ uint8_t *pma_query_via(void *rcvbuf, ib_portid_t * dest, int port, rpc.datasz = IB_PC_DATA_SZ; rpc.dataoffs = IB_PC_DATA_OFFS; - dest->qp = 1; + if (!dest->qp) + dest->qp = 1; if (!dest->qkey) dest->qkey = IB_DEFAULT_QP1_QKEY; @@ -109,7 +110,8 @@ uint8_t *performance_reset_via(void *rcvbuf, ib_portid_t * dest, rpc.timeout = timeout; rpc.datasz = IB_PC_DATA_SZ; rpc.dataoffs = IB_PC_DATA_OFFS; - dest->qp = 1; + if (!dest->qp) + dest->qp = 1; if (!dest->qkey) dest->qkey = IB_DEFAULT_QP1_QKEY; diff --git a/libibmad/src/rpc.c b/libibmad/src/rpc.c index 07b623d..7364940 100644 --- a/libibmad/src/rpc.c +++ b/libibmad/src/rpc.c @@ -183,33 +183,68 @@ _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, return -1; } +static int redirect_port(ib_portid_t *port, uint8_t *mad) +{ + port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); + if (!port->lid) { + IBWARN("GID-based redirection is not supported"); + return -1; + } + + port->qp = mad_get_field(mad, 64, IB_CPI_REDIRECT_QP_F); + port->qkey = mad_get_field(mad, 64, IB_CPI_REDIRECT_QKEY_F); + port->sl = mad_get_field(mad, 64, IB_CPI_REDIRECT_SL_F); + + /* TODO: Reverse map redirection P_Key to P_Key index */ + + if (ibdebug) + IBWARN("redirected to lid 0x%x, qp 0x%x, qkey 0x%x, sl 0x%x", + port->lid, port->qp, port->qkey, port->sl); + + return 0; +} + void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc, ib_portid_t * dport, void *payload, void *rcvdata) { int status, len; uint8_t sndbuf[1024], rcvbuf[1024], *mad; int timeout, retries; + int redirect = 1; - len = 0; - memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); + while (redirect) { + len = 0; + memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); - if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) - return 0; + if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) + return 0; - timeout = rpc->timeout ? rpc->timeout : - port->timeout ? port->timeout : madrpc_timeout; - retries = port->retries ? port->retries : madrpc_retries; + timeout = rpc->timeout ? rpc->timeout : + port->timeout ? port->timeout : madrpc_timeout; + retries = port->retries ? port->retries : madrpc_retries; - if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf, - port->class_agents[rpc->mgtclass], - len, timeout, retries)) < 0) { - IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); - return 0; - } + if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf, + port->class_agents[rpc->mgtclass], + len, timeout, retries)) < 0) { + IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); + return 0; + } - mad = umad_get_mad(rcvbuf); + mad = umad_get_mad(rcvbuf); + status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F); + + /* check for exact match instead of only the redirect bit; + * that way, weird statuses cause an error, too */ + if (status == IB_MAD_STS_REDIRECT) { + /* update dport for next request and retry */ + /* bail if redirection fails */ + if (redirect_port(dport, mad)) + redirect = 0; + } else + redirect = 0; + } - if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F)) != 0) { + if (status != 0) { ERRS("MAD completed with error status 0x%x; dport (%s)", status, portid2str(dport)); return 0; -- 1.5.5 From lars.ellenberg at linbit.com Wed Jul 1 06:36:52 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Wed, 1 Jul 2009 15:36:52 +0200 Subject: [ofa-general] using SDP for block device traffic: several problems Message-ID: <20090701133652.GG9112@soda.linbit> On Wed, Jul 01, 2009 at 04:02:17PM +0300, Amir Vadai wrote: Subject: Re: [patch] fix SDP page leak in sdp_bz_cleanup In-Reply-To: <4A4B5E59.2030001 at mellanox.co.il> > Hi Lars, > > This is the right place for posting patches. > > I will commit it ASAP into both branches. Thanks for that one. now, let me summarize some other findings. == off-by-one error, data corruption == I think that "sometimes" you lose the last byte of a fragment. situation: multi core, mlx4_ib driver, IPoIB configured, SDP configured, more details on request ;) do large message traffic on several streaming sockets at the same time, using as much bandwidth as possible, some on IPoIB, some on SDP. "sometimes" (typically within a couple of minutes), when receiving the stream, the last byte of some fragment is missing, or replaced by the first byte of the next fragment (if any). This has been noticed when using SDP from kernel space (for DRBD), and reproduced in userland. I will provide two simple perl scripts (server and client) today or tomorrow, so you should be able to reproduce this yourself in userland. It does not occur (within my patience time span) if there is not much load, or if I only use one stream, or even if I only use SDP (and not simultaneously also IPoIB streams). It only happens on SDP streams. I'm not sure if this off-by-one happens during send or recv. I'm open for suggestions to aid in tracking it down. == module count imbalance == after modprobe, module usage count of ib_sdp is 0, as it should be. starting to use it with some streaming sockest, module count goes up. once the streams start disconnecting, being interrupted from the other side, reconnect and similar stuff, module count quickly drops below zero, manifesting in lsmod showing a module count of 4.2 millon ;) I'm still trying to track this down, I'm not yet sure if it is a double module_put, or a missing (try_)module_get ... more when I find more. Cheers, -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. From hal.rosenstock at gmail.com Wed Jul 1 06:59:41 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 1 Jul 2009 09:59:41 -0400 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: <200907011534.59787.fenkes@de.ibm.com> References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> Message-ID: On Wed, Jul 1, 2009 at 9:34 AM, Joachim Fenkes wrote: > Previously, libibmad reacted to GSI MAD responses with a "redirect" status > by throwing an error. IBM eHCA adapters use redirection, so most > infiniband_diags tools didn't work against eHCA. > > Fix: Modify mad_rpc() so that it resends the request to the redirection > target if a "redirect" GS response is received. This is repeated until no > "redirect" response is received, allowing for multiple levels of > indirection. > > The dport argument is updated with the redirection target, so subsequent > MADs will not go through the redirection process again but reach the target > directly. > > Tested using perfquery between ehca, mlx4 and mthca in all possible > combinations. > > Signed-off-by: Joachim Fenkes > --- > > Hi, Hal and Jason, > > here's an updated patch that will bail on GID-routed redirection. Also, I > moved the redirection itself into its own function so it can easily be > included into RMPP as well. > > Of course, I tested this again using ehca, mthca and mlx4. > > If you have nothing to add to this patch, please queue it for OFED 1.5. > > Thanks and regards, >  Joachim > > >  libibmad/include/infiniband/mad.h |    9 +++++ >  libibmad/src/gs.c                 |    6 ++- >  libibmad/src/rpc.c                |   65 ++++++++++++++++++++++++++++-------- >  3 files changed, 63 insertions(+), 17 deletions(-) > > diff --git a/libibmad/include/infiniband/mad.h b/libibmad/include/infiniband/mad.h > index aa27eb5..bdf5158 100644 > --- a/libibmad/include/infiniband/mad.h > +++ b/libibmad/include/infiniband/mad.h > @@ -115,6 +115,8 @@ enum MAD_ATTR_ID { > >  enum MAD_STATUS { >        IB_MAD_STS_OK                        = (0 << 2), > +       IB_MAD_STS_BUSY                      = (1 << 0), > +       IB_MAD_STS_REDIRECT                  = (1 << 1), >        IB_MAD_STS_BAD_BASE_VER_OR_CLASS     = (1 << 2), >        IB_MAD_STS_METHOD_NOT_SUPPORTED      = (2 << 2), >        IB_MAD_STS_METHOD_ATTR_NOT_SUPPORTED = (3 << 2), > @@ -783,8 +785,15 @@ MAD_EXPORT int madrpc_set_timeout(int timeout); >  MAD_EXPORT struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port, >                        int *mgmt_classes, int num_classes); >  MAD_EXPORT void mad_rpc_close_port(struct ibmad_port *srcport); > + > +/* > + * On redirection, the dport argument is updated with the redirection target, > + * so subsequent MADs will not go through the redirection process again but > + * reach the target directly. > + */ >  MAD_EXPORT void *mad_rpc(const struct ibmad_port *srcport, ib_rpc_t * rpc, >                        ib_portid_t * dport, void *payload, void *rcvdata); > + >  MAD_EXPORT void *mad_rpc_rmpp(const struct ibmad_port *srcport, ib_rpc_t * rpc, >                              ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, >                              void *data); > diff --git a/libibmad/src/gs.c b/libibmad/src/gs.c > index f3d245e..c7e4ff6 100644 > --- a/libibmad/src/gs.c > +++ b/libibmad/src/gs.c > @@ -70,7 +70,8 @@ uint8_t *pma_query_via(void *rcvbuf, ib_portid_t * dest, int port, >        rpc.datasz = IB_PC_DATA_SZ; >        rpc.dataoffs = IB_PC_DATA_OFFS; > > -       dest->qp = 1; > +       if (!dest->qp) > +               dest->qp = 1; >        if (!dest->qkey) >                dest->qkey = IB_DEFAULT_QP1_QKEY; > > @@ -109,7 +110,8 @@ uint8_t *performance_reset_via(void *rcvbuf, ib_portid_t * dest, >        rpc.timeout = timeout; >        rpc.datasz = IB_PC_DATA_SZ; >        rpc.dataoffs = IB_PC_DATA_OFFS; > -       dest->qp = 1; > +       if (!dest->qp) > +               dest->qp = 1; >        if (!dest->qkey) >                dest->qkey = IB_DEFAULT_QP1_QKEY; > > diff --git a/libibmad/src/rpc.c b/libibmad/src/rpc.c > index 07b623d..7364940 100644 > --- a/libibmad/src/rpc.c > +++ b/libibmad/src/rpc.c > @@ -183,33 +183,68 @@ _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, >        return -1; >  } > > +static int redirect_port(ib_portid_t *port, uint8_t *mad) > +{ > +       port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); > +       if (!port->lid) { > +               IBWARN("GID-based redirection is not supported"); > +               return -1; > +       } Sorry for the confusion: determination of GID redirection should be based on a comparison of the RedirectGID to 0. It's valid to supply both a non zero RedirectGID and RedirectLID. > + > +       port->qp = mad_get_field(mad, 64, IB_CPI_REDIRECT_QP_F); > +       port->qkey = mad_get_field(mad, 64, IB_CPI_REDIRECT_QKEY_F); > +       port->sl = mad_get_field(mad, 64, IB_CPI_REDIRECT_SL_F); > + > +       /* TODO: Reverse map redirection P_Key to P_Key index */ > + > +       if (ibdebug) > +               IBWARN("redirected to lid 0x%x, qp 0x%x, qkey 0x%x, sl 0x%x", > +                      port->lid, port->qp, port->qkey, port->sl); Unicast LIDs should be displayed in decimal rather than hex. -- Hal > + > +       return 0; > +} > + >  void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc, >              ib_portid_t * dport, void *payload, void *rcvdata) >  { >        int status, len; >        uint8_t sndbuf[1024], rcvbuf[1024], *mad; >        int timeout, retries; > +       int redirect = 1; > > -       len = 0; > -       memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); > +       while (redirect) { > +               len = 0; > +               memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); > > -       if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) > -               return 0; > +               if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) > +                       return 0; > > -       timeout = rpc->timeout ? rpc->timeout : > -           port->timeout ? port->timeout : madrpc_timeout; > -       retries = port->retries ? port->retries : madrpc_retries; > +               timeout = rpc->timeout ? rpc->timeout : > +                       port->timeout ? port->timeout : madrpc_timeout; > +               retries = port->retries ? port->retries : madrpc_retries; > > -       if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf, > -                             port->class_agents[rpc->mgtclass], > -                             len, timeout, retries)) < 0) { > -               IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); > -               return 0; > -       } > +               if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf, > +                                     port->class_agents[rpc->mgtclass], > +                                     len, timeout, retries)) < 0) { > +                       IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); > +                       return 0; > +               } > > -       mad = umad_get_mad(rcvbuf); > +               mad = umad_get_mad(rcvbuf); > +               status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F); > + > +               /* check for exact match instead of only the redirect bit; > +                * that way, weird statuses cause an error, too */ > +               if (status == IB_MAD_STS_REDIRECT) { > +                       /* update dport for next request and retry */ > +                       /* bail if redirection fails */ > +                       if (redirect_port(dport, mad)) > +                               redirect = 0; > +               } else > +                       redirect = 0; > +       } > > -       if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F)) != 0) { > +       if (status != 0) { >                ERRS("MAD completed with error status 0x%x; dport (%s)", >                     status, portid2str(dport)); >                return 0; > -- > 1.5.5 > > > > From jgunthorpe at obsidianresearch.com Wed Jul 1 08:14:25 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Wed, 1 Jul 2009 09:14:25 -0600 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> Message-ID: <20090701151425.GD20745@obsidianresearch.com> On Wed, Jul 01, 2009 at 09:59:41AM -0400, Hal Rosenstock wrote: > > +static int redirect_port(ib_portid_t *port, uint8_t *mad) > > +{ > > + ?? ?? ?? port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); > > + ?? ?? ?? if (!port->lid) { > > + ?? ?? ?? ?? ?? ?? ?? IBWARN("GID-based redirection is not supported"); > > + ?? ?? ?? ?? ?? ?? ?? return -1; > > + ?? ?? ?? } > > Sorry for the confusion: determination of GID redirection should be > based on a comparison of the RedirectGID to 0. It's valid to supply > both a non zero RedirectGID and RedirectLID. ?? The above is correct. As I said, RedirectGID is not allowed to be 0. Jason From FENKES at de.ibm.com Wed Jul 1 08:17:17 2009 From: FENKES at de.ibm.com (Joachim Fenkes) Date: Wed, 1 Jul 2009 17:17:17 +0200 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> Message-ID: Hal Rosenstock wrote on 01.07.2009 15:59:41: > > +static int redirect_port(ib_portid_t *port, uint8_t *mad) > > +{ > > + port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); > > + if (!port->lid) { > > + IBWARN("GID-based redirection is not supported"); > > + return -1; > > + } > > Sorry for the confusion: determination of GID redirection should be > based on a comparison of the RedirectGID to 0. It's valid to supply > both a non zero RedirectGID and RedirectLID. Are you sure? About the Redirection GID, the spec says "If redirection is not being performed, this shall be set to zero", so if redirection _is_ being performed, the GID may or may not be zero without any explicit implication. For the LID, it says "If this value is zero, the redirect requires the requester to use the supplied RedirectGID to request further path resolution from subnet administration." To me, this explicitly states that a zero LID means that the GID must be used. If both LID and GID are non-zero, it is not specified whether the requester should use the LID or the GID, so I choose to always use the LID as long as it's non-zero, because that's what the code supports. Am I talking crazy or does this make sense to you? > > + > > + port->qp = mad_get_field(mad, 64, IB_CPI_REDIRECT_QP_F); > > + port->qkey = mad_get_field(mad, 64, IB_CPI_REDIRECT_QKEY_F); > > + port->sl = mad_get_field(mad, 64, IB_CPI_REDIRECT_SL_F); > > + > > + /* TODO: Reverse map redirection P_Key to P_Key index */ > > + > > + if (ibdebug) > > + IBWARN("redirected to lid 0x%x, qp 0x%x, qkey 0x%x, sl 0x%x", > > + port->lid, port->qp, port->qkey, port->sl); > > Unicast LIDs should be displayed in decimal rather than hex. Couldn't you have noticed this in the first patch? ;) I'll change it. Cheers, Joachim From hal.rosenstock at gmail.com Wed Jul 1 08:54:13 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 1 Jul 2009 11:54:13 -0400 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: <20090701151425.GD20745@obsidianresearch.com> References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> <20090701151425.GD20745@obsidianresearch.com> Message-ID: On Wed, Jul 1, 2009 at 11:14 AM, Jason Gunthorpe wrote: > On Wed, Jul 01, 2009 at 09:59:41AM -0400, Hal Rosenstock wrote: > >> > +static int redirect_port(ib_portid_t *port, uint8_t *mad) >> > +{ >> > + ?? ?? ?? port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); >> > + ?? ?? ?? if (!port->lid) { >> > + ?? ?? ?? ?? ?? ?? ?? IBWARN("GID-based redirection is not supported"); >> > + ?? ?? ?? ?? ?? ?? ?? return -1; >> > + ?? ?? ?? } >> >> Sorry for the confusion: determination of GID redirection should be >> based on a comparison of the RedirectGID to 0. It's valid to supply >> both a non zero RedirectGID and RedirectLID. > > ?? The above is correct. As I said, RedirectGID is not allowed to be 0. I think it depends on the interpretation of "If redirection is not being performed, this shall be set to zero." in the RedirectGID description as to whether it is referring to redirection in general or just GID redirection. Futhermore, RedirectLID can be non zero but GID redirection is still being used as indicated by the RedirectLID description indicating that a non zero RedirectLID will in general not be valid. -- Hal > > Jason > From hal.rosenstock at gmail.com Wed Jul 1 08:54:41 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 1 Jul 2009 11:54:41 -0400 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> Message-ID: On Wed, Jul 1, 2009 at 11:17 AM, Joachim Fenkes wrote: > Hal Rosenstock wrote on 01.07.2009 15:59:41: > >> > +static int redirect_port(ib_portid_t *port, uint8_t *mad) >> > +{ >> > +       port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); >> > +       if (!port->lid) { >> > +               IBWARN("GID-based redirection is not supported"); >> > +               return -1; >> > +       } >> >> Sorry for the confusion: determination of GID redirection should be >> based on a comparison of the RedirectGID to 0. It's valid to supply >> both a non zero RedirectGID and RedirectLID. > > Are you sure? No; I'm not sure. See previous post to Jason. > About the Redirection GID, the spec says "If redirection is not being > performed, this shall be set to zero", so if redirection _is_ being > performed, the GID may or may not be zero without any explicit > implication. Agreed but Jason doesn't appear to agree. > For the LID, it says "If this value is zero, the redirect requires the > requester to use the supplied RedirectGID to request further path > resolution > from subnet administration." To me, this explicitly states that a zero LID > means that the GID must be used. > > If both LID and GID are non-zero, it is not specified whether the > requester > should use the LID or the GID, so I choose to always use the LID as long > as it's non-zero, because that's what the code supports. It does say that the LID might not be valid even though non-zero. I'm thinking of the more general case (future) rather than just IBM eHCA usage. > Am I talking crazy or does this make sense to you? >> > + >> > +       port->qp = mad_get_field(mad, 64, IB_CPI_REDIRECT_QP_F); >> > +       port->qkey = mad_get_field(mad, 64, IB_CPI_REDIRECT_QKEY_F); >> > +       port->sl = mad_get_field(mad, 64, IB_CPI_REDIRECT_SL_F); >> > + >> > +       /* TODO: Reverse map redirection P_Key to P_Key index */ >> > + >> > +       if (ibdebug) >> > +               IBWARN("redirected to lid 0x%x, qp 0x%x, qkey 0x%x, sl > 0x%x", >> > +                      port->lid, port->qp, port->qkey, port->sl); >> >> Unicast LIDs should be displayed in decimal rather than hex. > > Couldn't you have noticed this in the first patch? ;) Somehow I missed it :-( Sorry. >I'll change it. Thanks. -- Hal > Cheers, >  Joachim > From sean.hefty at intel.com Wed Jul 1 09:05:31 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Wed, 1 Jul 2009 09:05:31 -0700 Subject: [ofa-general] ib_rdma_bw - memory leaks? In-Reply-To: <000b01c9fa34$4b407a50$e1c16ef0$@panchasara@einfochips.com> References: <000b01c9fa34$4b407a50$e1c16ef0$@panchasara@einfochips.com> Message-ID: <785E3F61D0AF4EA48288A3C56CD7B11F@amr.corp.intel.com> >3. rdma_create_event_channel() calls ucma_init() but >rdma_destroy_event_channel() does not call ucma_cleanup(), this results into >memory leak at provider's library since it does not call ibv_close_device() >and thus unable to do *->free_context(). This is the correct behavior. ucma_init() is called from several routines to ensure that the library performs proper initialization. Once initialized, it remains initialized until the library is no longer used. The cleanup is done in rdma_cma_fini(). - Sean From FENKES at de.ibm.com Wed Jul 1 09:20:28 2009 From: FENKES at de.ibm.com (Joachim Fenkes) Date: Wed, 1 Jul 2009 18:20:28 +0200 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> Message-ID: Hal Rosenstock wrote on 01.07.2009 17:54:41: > It does say that the LID might not be valid even though non-zero. Can you elaborate on that? I can't seem to find that in the spec. What I do find, though, is this: "If this value is non-zero, it is the DLID a requester _shall_ use to access the class services." -- v1.2, p736, line 7 Which sounds to me like: If it's non-zero, it's valid and you must use it. > I'm > thinking of the more general case (future) rather than just IBM eHCA > usage. I stopped thinking "eHCA only" two patches ago, don't worry ;) BTW, I'm going to be out of the office starting now and returning next Tuesday, so let's continue this discussion next week ;) Cheers, Joachim From amirv at mellanox.co.il Wed Jul 1 09:45:00 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Wed, 1 Jul 2009 19:45:00 +0300 Subject: [ofa-general] [PATCH] sdp: Fix memory leak in bzcopy In-Reply-To: <1246466700-6400-1-git-send-email-amirv@mellanox.co.il> References: <1246466700-6400-1-git-send-email-amirv@mellanox.co.il> Message-ID: <1246466700-6400-2-git-send-email-amirv@mellanox.co.il> Thanks to Lars Ellenberg for finding the leak Signed-off-by: Amir Vadai --- drivers/infiniband/ulp/sdp/sdp_main.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/drivers/infiniband/ulp/sdp/sdp_main.c b/drivers/infiniband/ulp/sdp/sdp_main.c index f238a0f..862878c 100644 --- a/drivers/infiniband/ulp/sdp/sdp_main.c +++ b/drivers/infiniband/ulp/sdp/sdp_main.c @@ -1294,7 +1294,7 @@ static inline struct bzcopy_state *sdp_bz_cleanup(struct bzcopy_state *bz) } if (bz->pages) { - for (i = bz->cur_page; i < bz->page_cnt; i++) + for (i = 0; i < bz->cur_page; i++) put_page(bz->pages[i]); kfree(bz->pages); -- 1.5.3.7 From amirv at mellanox.co.il Wed Jul 1 09:44:59 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Wed, 1 Jul 2009 19:44:59 +0300 Subject: [ofa-general] [PATCH] sdp: fix bad credits advertised when connection initiated Message-ID: <1246466700-6400-1-git-send-email-amirv@mellanox.co.il> This enables removing the ugly post credits after connection establishment Signed-off-by: Amir Vadai --- drivers/infiniband/ulp/sdp/sdp.h | 1 - drivers/infiniband/ulp/sdp/sdp_bcopy.c | 15 ---- drivers/infiniband/ulp/sdp/sdp_cma.c | 8 +-- .../2.6.16/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.16_sles10/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.17/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.18-EL5.1/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.18-EL5.2/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.18-EL5.3/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.18/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.18_FC6/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.19/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.20/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.21/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.22/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.23/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.24/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.9_U4/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.9_U5/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.9_U6/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- .../2.6.9_U7/sdp_0090_revert_to_2_6_24.patch | 77 +++++++++----------- 25 files changed, 751 insertions(+), 967 deletions(-) diff --git a/drivers/infiniband/ulp/sdp/sdp.h b/drivers/infiniband/ulp/sdp/sdp.h index f9c2421..7f197d2 100644 --- a/drivers/infiniband/ulp/sdp/sdp.h +++ b/drivers/infiniband/ulp/sdp/sdp.h @@ -334,7 +334,6 @@ void sdp_reset(struct sock *sk); void sdp_reset_sk(struct sock *sk, int rc); void sdp_completion_handler(struct ib_cq *cq, void *cq_context); void sdp_work(struct work_struct *work); -int sdp_post_credits(struct sdp_sock *ssk); void sdp_post_send(struct sdp_sock *ssk, struct sk_buff *skb, u8 mid); void sdp_post_recvs(struct sdp_sock *ssk); int sdp_poll_cq(struct sdp_sock *ssk, struct ib_cq *cq); diff --git a/drivers/infiniband/ulp/sdp/sdp_bcopy.c b/drivers/infiniband/ulp/sdp/sdp_bcopy.c index 475d7c3..a090868 100644 --- a/drivers/infiniband/ulp/sdp/sdp_bcopy.c +++ b/drivers/infiniband/ulp/sdp/sdp_bcopy.c @@ -488,21 +488,6 @@ static inline int sdp_nagle_off(struct sdp_sock *ssk, struct sk_buff *skb) (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_PSH); } -int sdp_post_credits(struct sdp_sock *ssk) -{ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; - skb = sdp_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) - return -ENOMEM; - sdp_post_send(ssk, skb, SDP_MID_DATA); - } - return 0; -} - void sdp_post_sends(struct sdp_sock *ssk, int nonagle) { /* TODO: nonagle? */ diff --git a/drivers/infiniband/ulp/sdp/sdp_cma.c b/drivers/infiniband/ulp/sdp/sdp_cma.c index 96d65bd..aba9113 100644 --- a/drivers/infiniband/ulp/sdp/sdp_cma.c +++ b/drivers/infiniband/ulp/sdp/sdp_cma.c @@ -412,11 +412,11 @@ int sdp_cma_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) sdp_sk(sk)->rx_tail; memset(&hh, 0, sizeof hh); hh.bsdh.mid = SDP_MID_HELLO; - hh.bsdh.bufs = htons(sdp_sk(sk)->remote_credits); hh.bsdh.len = htonl(sizeof(struct sdp_bsdh) + SDP_HH_SIZE); hh.max_adverts = 1; hh.majv_minv = SDP_MAJV_MINV; sdp_init_buffers(sdp_sk(sk), rcvbuf_initial_size); + hh.bsdh.bufs = htons(sdp_sk(sk)->rx_head - sdp_sk(sk)->rx_tail); hh.localrcvsz = hh.desremrcvsz = htonl(sdp_sk(sk)->recv_frags * PAGE_SIZE + SDP_HEAD_SIZE); hh.max_adverts = 0x1; @@ -446,7 +446,8 @@ int sdp_cma_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) sdp_sk(child)->rx_tail; memset(&hah, 0, sizeof hah); hah.bsdh.mid = SDP_MID_HELLO_ACK; - hah.bsdh.bufs = htons(sdp_sk(child)->remote_credits); + hah.bsdh.bufs = + htons(sdp_sk(child)->rx_head - sdp_sk(child)->rx_tail); hah.bsdh.len = htonl(sizeof(struct sdp_bsdh) + SDP_HAH_SIZE); hah.majv_minv = SDP_MAJV_MINV; hah.ext_max_adverts = 1; /* Doesn't seem to be mandated by spec, @@ -473,9 +474,6 @@ int sdp_cma_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) rdma_reject(id, NULL, 0); else rc = rdma_accept(id, NULL); - - if (!rc) - rc = sdp_post_credits(sdp_sk(sk)); break; case RDMA_CM_EVENT_CONNECT_ERROR: sdp_dbg(sk, "RDMA_CM_EVENT_CONNECT_ERROR\n"); diff --git a/kernel_patches/backport/2.6.16/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.16/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.16/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.16/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.16_sles10/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.16_sles10/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.16_sles10/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.16_sles10/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.16_sles10_sp1/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.16_sles10_sp1/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.16_sles10_sp1/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.16_sles10_sp1/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.16_sles10_sp2/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.16_sles10_sp2/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.16_sles10_sp2/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.16_sles10_sp2/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.17/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.17/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.17/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.17/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.18-EL5.1/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.18-EL5.1/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.18-EL5.1/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.18-EL5.1/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.18-EL5.2/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.18-EL5.2/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.18-EL5.2/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.18-EL5.2/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.18-EL5.3/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.18-EL5.3/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.18-EL5.3/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.18-EL5.3/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.18/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.18/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.18/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.18/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.18_FC6/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.18_FC6/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.18_FC6/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.18_FC6/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.18_suse10_2/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.18_suse10_2/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.18_suse10_2/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.18_suse10_2/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.19/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.19/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.19/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.19/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.20/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.20/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.20/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.20/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.21/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.21/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.21/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.21/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.22/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.22/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.22/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.22/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.22_suse10_3/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.22_suse10_3/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.22_suse10_3/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.22_suse10_3/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.23/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.23/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.23/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.23/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.24/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.24/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.24/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.24/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.9_U4/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.9_U4/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.9_U4/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.9_U4/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.9_U5/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.9_U5/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.9_U5/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.9_U5/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.9_U6/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.9_U6/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.9_U6/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.9_U6/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); diff --git a/kernel_patches/backport/2.6.9_U7/sdp_0090_revert_to_2_6_24.patch b/kernel_patches/backport/2.6.9_U7/sdp_0090_revert_to_2_6_24.patch index 893db9b..e0d6686 100644 --- a/kernel_patches/backport/2.6.9_U7/sdp_0090_revert_to_2_6_24.patch +++ b/kernel_patches/backport/2.6.9_U7/sdp_0090_revert_to_2_6_24.patch @@ -5,13 +5,13 @@ drivers/infiniband/ulp/sdp/sdp_main.c | 19 +++++++++---------- 4 files changed, 21 insertions(+), 50 deletions(-) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp.h -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h -@@ -317,30 +317,4 @@ - void sdp_start_keepalive_timer(struct sock *sk); +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp.h ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp.h +@@ -354,30 +354,4 @@ void sdp_start_keepalive_timer(struct so void sdp_bzcopy_write_space(struct sdp_sock *ssk); + int sdp_init_sock(struct sock *sk); -static inline struct sk_buff *sdp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp) -{ @@ -40,11 +40,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp.h - - #endif -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c -@@ -139,7 +139,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_bcopy.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_bcopy.c +@@ -147,7 +147,7 @@ static void sdp_fin(struct sock *sk) } @@ -53,7 +53,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c if (!sock_flag(sk, SOCK_DEAD)) { sk->sk_state_change(sk); -@@ -190,7 +190,7 @@ +@@ -198,7 +198,7 @@ void sdp_post_send(struct sdp_sock *ssk, struct ib_send_wr *bad_wr; h->mid = mid; @@ -62,7 +62,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c h->flags = SDP_OOB_PRES | SDP_OOB_PEND; else h->flags = 0; -@@ -234,7 +234,7 @@ +@@ -242,7 +242,7 @@ void sdp_post_send(struct sdp_sock *ssk, ssk->tx_wr.num_sge = frags + 1; ssk->tx_wr.opcode = IB_WR_SEND; ssk->tx_wr.send_flags = IB_SEND_SIGNALED; @@ -71,7 +71,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c ssk->tx_wr.send_flags |= IB_SEND_SOLICITED; rc = ib_post_send(ssk->qp, &ssk->tx_wr, &bad_wr); ++ssk->tx_head; -@@ -304,11 +304,11 @@ +@@ -312,11 +312,11 @@ static void sdp_post_recv(struct sdp_soc /* TODO: allocate from cache */ if (unlikely(ssk->isk.sk.sk_allocation)) { @@ -85,16 +85,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c GFP_KERNEL); gfp_page = GFP_HIGHUSER; } -@@ -476,7 +476,7 @@ - if (likely(ssk->bufs > 1) && - likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE)) { - struct sk_buff *skb; -- skb = sdp_stream_alloc_skb(&ssk->isk.sk, -+ skb = sk_stream_alloc_skb(&ssk->isk.sk, - sizeof(struct sdp_bsdh), - GFP_KERNEL); - if (!skb) -@@ -514,7 +514,7 @@ +@@ -516,7 +516,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *resp_size; ssk->recv_request = 0; @@ -103,7 +94,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*resp_size), gfp_page); -@@ -539,7 +539,7 @@ +@@ -541,7 +541,7 @@ void sdp_post_sends(struct sdp_sock *ssk ssk->tx_head > ssk->sent_request_head + SDP_RESIZE_WAIT && ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) { struct sdp_chrecvbuf *req_size; @@ -112,7 +103,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh) + sizeof(*req_size), gfp_page); -@@ -561,7 +561,7 @@ +@@ -563,7 +563,7 @@ void sdp_post_sends(struct sdp_sock *ssk likely(ssk->tx_head - ssk->tx_tail < SDP_TX_SIZE) && likely((1 << ssk->isk.sk.sk_state) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1))) { @@ -121,7 +112,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), GFP_KERNEL); /* FIXME */ -@@ -573,7 +573,7 @@ +@@ -575,7 +575,7 @@ void sdp_post_sends(struct sdp_sock *ssk !ssk->isk.sk.sk_send_head && ssk->bufs > (ssk->remote_credits >= ssk->rx_head - ssk->rx_tail)) { ssk->sdp_disconnect = 0; @@ -130,7 +121,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c sizeof(struct sdp_bsdh), gfp_page); /* FIXME */ -@@ -778,7 +778,7 @@ +@@ -789,7 +789,7 @@ static int sdp_handle_send_comp(struct s } out: @@ -139,7 +130,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c return 0; } -@@ -864,7 +864,7 @@ +@@ -875,7 +875,7 @@ void sdp_work(struct work_struct *work) sdp_poll_cq(ssk, cq); release_sock(sk); @@ -148,11 +139,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_bcopy.c lock_sock(sk); cq = ssk->cq; if (unlikely(!cq)) -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_cma.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c -@@ -161,8 +161,6 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_cma.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_cma.c +@@ -161,8 +161,6 @@ static int sdp_init_qp(struct sock *sk, goto err_cq; } @@ -161,11 +152,11 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_cma.c qp_init_attr.send_cq = qp_init_attr.recv_cq = cq; rc = rdma_create_qp(id, pd, &qp_init_attr); -Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c +Index: ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c =================================================================== ---- ofed_1_4.orig/drivers/infiniband/ulp/sdp/sdp_main.c -+++ ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c -@@ -509,7 +509,7 @@ +--- ofed_kernel-2.6.16.orig/drivers/infiniband/ulp/sdp/sdp_main.c ++++ ofed_kernel-2.6.16/drivers/infiniband/ulp/sdp/sdp_main.c +@@ -521,7 +521,7 @@ static void sdp_close(struct sock *sk, l __kfree_skb(skb); } @@ -174,7 +165,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section * 3.10, we send a RST here because data was lost. To -@@ -1200,7 +1200,7 @@ +@@ -1243,7 +1243,7 @@ static inline void sdp_mark_urg(struct s { if (unlikely(flags & MSG_OOB)) { struct sk_buff *skb = sk->sk_write_queue.prev; @@ -183,7 +174,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c } } -@@ -1217,8 +1217,7 @@ +@@ -1260,8 +1260,7 @@ static inline void skb_entail(struct soc { skb_header_release(skb); __skb_queue_tail(&sk->sk_write_queue, skb); @@ -193,7 +184,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!sk->sk_send_head) sk->sk_send_head = skb; if (ssk->nonagle & TCP_NAGLE_PUSH) -@@ -1382,7 +1381,7 @@ +@@ -1432,7 +1431,7 @@ static inline int sdp_bcopy_get(struct s if (copy > PAGE_SIZE - off) copy = PAGE_SIZE - off; @@ -202,7 +193,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c return SDP_DO_WAIT_MEM; if (!page) { -@@ -1454,7 +1453,7 @@ +@@ -1504,7 +1503,7 @@ static inline int sdp_bzcopy_get(struct if (left <= this_page) this_page = left; @@ -210,8 +201,8 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c + if (!sk_stream_wmem_schedule(sk, copy)) return SDP_DO_WAIT_MEM; - skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, -@@ -1662,8 +1661,8 @@ + get_page(bz->pages[bz->cur_page]); +@@ -1720,8 +1719,8 @@ new_segment: goto wait_for_sndbuf; } @@ -222,7 +213,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c if (!skb) goto wait_for_memory; -@@ -1687,7 +1686,7 @@ +@@ -1745,7 +1744,7 @@ new_segment: /* OOB data byte should be the last byte of the data payload */ @@ -231,7 +222,7 @@ Index: ofed_1_4/drivers/infiniband/ulp/sdp/sdp_main.c !(flags & MSG_OOB)) { sdp_mark_push(ssk, skb); goto new_segment; -@@ -1763,7 +1762,7 @@ +@@ -1821,7 +1820,7 @@ do_fault: if (sk->sk_send_head == skb) sk->sk_send_head = NULL; __skb_unlink(skb, &sk->sk_write_queue); -- 1.5.3.7 From jgunthorpe at obsidianresearch.com Wed Jul 1 09:50:36 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Wed, 1 Jul 2009 10:50:36 -0600 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> <20090701151425.GD20745@obsidianresearch.com> Message-ID: <20090701165036.GE20745@obsidianresearch.com> On Wed, Jul 01, 2009 at 11:54:13AM -0400, Hal Rosenstock wrote: > I think it depends on the interpretation of "If redirection is not being > performed, this shall be set to zero." in the RedirectGID description > as to whether it is referring to redirection in general or just GID > redirection. ClassPortInfo is used for alot of things, I take that to mean that when it is used in non-redirection contexts that RedirectGID can be 0. Clearly the only sane way this can work is if the GID is always filled in for the redirection case. > Futhermore, RedirectLID can be non zero but GID redirection is still > being used as indicated by the RedirectLID description indicating that > a non zero RedirectLID will in general not be valid. The spec says if it is not zero the requester shall use it. I don't see an ambiguity here. Jason From amirv at mellanox.co.il Wed Jul 1 09:53:16 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Wed, 01 Jul 2009 19:53:16 +0300 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <20090701133652.GG9112@soda.linbit> References: <20090701133652.GG9112@soda.linbit> Message-ID: <4A4B947C.1030607@mellanox.co.il> I understand that you're using ofed-1.5. We should check if those bugs exist in 1.4.1 too - since there were many changes in SDP in ofed-1.5. Please open bugs in bugzilla here: https://bugs.openfabrics.org Thanks, Amir On 07/01/2009 04:36 PM, Lars Ellenberg wrote: > On Wed, Jul 01, 2009 at 04:02:17PM +0300, Amir Vadai wrote: > Subject: Re: [patch] fix SDP page leak in sdp_bz_cleanup > In-Reply-To: <4A4B5E59.2030001 at mellanox.co.il> >> Hi Lars, >> >> This is the right place for posting patches. >> >> I will commit it ASAP into both branches. > > Thanks for that one. > > now, let me summarize some other findings. > > == off-by-one error, data corruption == > > I think that "sometimes" you lose the last byte of a fragment. > > situation: multi core, mlx4_ib driver, > IPoIB configured, SDP configured, more details on request ;) > > do large message traffic on several streaming sockets > at the same time, using as much bandwidth as possible, > some on IPoIB, some on SDP. > > "sometimes" (typically within a couple of minutes), when receiving the > stream, the last byte of some fragment is missing, or replaced by the > first byte of the next fragment (if any). > > This has been noticed when using SDP from kernel space (for DRBD), > and reproduced in userland. > > I will provide two simple perl scripts (server and client) today or > tomorrow, so you should be able to reproduce this yourself in userland. > > It does not occur (within my patience time span) if there is not much > load, or if I only use one stream, or even if I only use SDP (and not > simultaneously also IPoIB streams). It only happens on SDP streams. > > I'm not sure if this off-by-one happens during send or recv. > > I'm open for suggestions to aid in tracking it down. > > > == module count imbalance == > > after modprobe, module usage count of ib_sdp is 0, as it should be. > starting to use it with some streaming sockest, module count goes up. > > once the streams start disconnecting, being interrupted from the other > side, reconnect and similar stuff, module count quickly drops below > zero, manifesting in lsmod showing a module count of 4.2 millon ;) > > I'm still trying to track this down, I'm not yet sure if it is a double > module_put, or a missing (try_)module_get ... > > > more when I find more. > > Cheers, > -- Amir Vadai Software Eng. Mellanox Technologies mailto: amirv at mellanox.co.il Tel +972-3-6259539 From hal.rosenstock at gmail.com Wed Jul 1 12:38:50 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 1 Jul 2009 15:38:50 -0400 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> Message-ID: On Wed, Jul 1, 2009 at 12:20 PM, Joachim Fenkes wrote: > Hal Rosenstock wrote on 01.07.2009 17:54:41: > >> It does say that the LID might not be valid even though non-zero. > > Can you elaborate on that? I can't seem to find that in the spec. > What I do find, though, is this: > > "If this value is non-zero, it is the DLID a requester _shall_ use to > access the > class services." -- v1.2, p736, line 7 Yes, that's same as v1.2.1 p.743 line 14. > > Which sounds to me like: If it's non-zero, it's valid and you must use it. v1.2.1 p.743 line 17 goes on to say: "The RedirectGID, the RedirectQP and RedirectP_Key from this redirect response are all valid, but the RedirectSL, RedirectFL, RedirectTC, and RedirectLID will in general not be valid; they must be replaced using a Path- Record obtained from the SA." Doesn't look like that was a change from v1.2 as there are no change bars. -- Hal >> I'm >> thinking of the more general case (future) rather than just IBM eHCA >> usage. > > I stopped thinking "eHCA only" two patches ago, don't worry ;) > > BTW, I'm going to be out of the office starting now and returning next > Tuesday, > so let's continue this discussion next week ;) > > Cheers, >  Joachim > From hal.rosenstock at gmail.com Wed Jul 1 12:39:01 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 1 Jul 2009 15:39:01 -0400 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: <20090701165036.GE20745@obsidianresearch.com> References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> <20090701151425.GD20745@obsidianresearch.com> <20090701165036.GE20745@obsidianresearch.com> Message-ID: On Wed, Jul 1, 2009 at 12:50 PM, Jason Gunthorpe wrote: > On Wed, Jul 01, 2009 at 11:54:13AM -0400, Hal Rosenstock wrote: > >> I think it depends on the interpretation of "If redirection is not being >> performed, this shall be set to zero." in the RedirectGID description >> as to whether it is referring to redirection in general or just GID >> redirection. > > ClassPortInfo is used for alot of things, Of course. > I take that to mean that > when it is used in non-redirection contexts that RedirectGID can be 0. I took it to mean differently as there's some conflicting text in RedirectLID. > Clearly the only sane way this can work is if the GID is always > filled in for the redirection case. Why is that ? Why must the redirector provide GRH info when it's not required for subnet local cases ? >> Futhermore, RedirectLID can be non zero but GID redirection is still >> being used as indicated by the RedirectLID description indicating that >> a non zero RedirectLID will in general not be valid. > > The spec says if it is not zero the requester shall use it. I don't > see an ambiguity here. To me, the ambiguity is several lines below it where it states that the RedirectLID might not be valid and says to obtain a PathRecord when RedirectGID is supplied rather than relying on the RedirectLID is non zero. -- Hal > Jason From jgunthorpe at obsidianresearch.com Wed Jul 1 13:00:43 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Wed, 1 Jul 2009 14:00:43 -0600 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> <20090701151425.GD20745@obsidianresearch.com> <20090701165036.GE20745@obsidianresearch.com> Message-ID: <20090701200043.GF20745@obsidianresearch.com> On Wed, Jul 01, 2009 at 03:39:01PM -0400, Hal Rosenstock wrote: > > Clearly the only sane way this can work is if the GID is always > > filled in for the redirection case. > > Why is that ? Why must the redirector provide GRH info when it's not > required for subnet local cases ? Because the redirector doesn't know what the initiator will do. It could include a GRH, or maybe not. It must include the GID to cover both cases. > >> Futhermore, RedirectLID can be non zero but GID redirection is still > >> being used as indicated by the RedirectLID description indicating that > >> a non zero RedirectLID will in general not be valid. > > > > The spec says if it is not zero the requester shall use it. I don't > > see an ambiguity here. > > To me, the ambiguity is several lines below it where it states that > the RedirectLID might not be valid and says to obtain a PathRecord > when RedirectGID is supplied rather than relying on the RedirectLID is > non zero. Whoever authored this should not have mixed 'will in general not be valid' and 'they must be replaced' in the same sentance - but I think the meaning is still clear. With a 0 RedirectLID only the RedirectGID, QP and P_Key are to be used by the receiver. When RedirectLID is not 0 then all of the Redirect fields must contain correct data and should be used as necessary by the receiver. It never says to obtain a Path Record when a GID is supplied. It says to obtain a path record with RedirectLID is 0. Jason From worleys at gmail.com Wed Jul 1 13:16:32 2009 From: worleys at gmail.com (Chris Worley) Date: Wed, 1 Jul 2009 14:16:32 -0600 Subject: [ofa-general] Library routine for enumerating IB cards and GUIDS Message-ID: I need a programmatic way to enumerate all IB cards in the system, i.e. their port GUIDs. What library/routines might I use, and which docs should I read concerning exposed routines? Thanks, Chris From sumeet.lahorani at oracle.com Wed Jul 1 14:26:49 2009 From: sumeet.lahorani at oracle.com (Sumeet Lahorani) Date: Wed, 01 Jul 2009 14:26:49 -0700 Subject: [ofa-general] RH 5.3 : ifup-ib does not setup connected mode on slaves Message-ID: <4A4BD499.1000600@oracle.com> Hi, I'm running into a problem with ifup-ib on RH 5.3. I've setup a bonded interfaces over 2 ports of a HCA as follows # cat ifcfg-bond0 DEVICE=bond0 USERCTL=no BOOTPROTO=none ONBOOT=yes IPADDR=192.168.33.219 NETMASK=255.255.254.0 NETWORK=192.168.32.0 BROADCAST=192.168.33.255 BONDING_OPTS="mode=active-backup miimon=100 downdelay=5000 updelay=5000" IPV6INIT=no MTU=65520 # cat ifcfg-ib1 DEVICE=ib1 USERCTL=no ONBOOT=yes MASTER=bond0 SLAVE=yes BOOTPROTO=none HOTPLUG=no CONNECTED_MODE=yes MTU=65520 # cat ifcfg-ib0 DEVICE=ib0 USERCTL=no ONBOOT=yes MASTER=bond0 SLAVE=yes BOOTPROTO=none HOTPLUG=no CONNECTED_MODE=yes MTU=65520 When the network service comes up, it executes ifup-ib to bring up ib0 & ib1. However, this script has the following which causes it to exit before it configures these interfaces to run in connected mode. 69 # slave device? 70 if [ "${SLAVE}" = yes -a "${ISALIAS}" = no -a "${MASTER}" != "" ]; then 71 /sbin/ip link set dev ${DEVICE} down 72 echo "+${DEVICE}" > /sys/class/net/${MASTER}/bonding/slaves 2>/dev/n ull 73 74 if [ -n "$ETHTOOL_OPTS" ] ; then 75 /sbin/ethtool -s ${REALDEVICE} $ETHTOOL_OPTS 76 fi 77 78 exit 0 79 fi ... 120 # We have to set connected mode before we set the MTU since connected mo de 121 # changes our maximum allowed MTU 122 if [ "${CONNECTED_MODE}" = yes -a -e /sys/class/net/${DEVICE}/mode ]; th en 123 echo connected > /sys/class/net/${DEVICE}/mode 124 fi 125 126 if [ -n "${MTU}" ]; then 127 ip link set dev ${DEVICE} mtu ${MTU} 128 fi Is this a bug or is there a different way to bond together 2 connected mode interfaces? # rpm -qf /etc/sysconfig/network-scripts/ifup-ib openib-1.3.2-0.20080728.0355.3.el5 - Sumeet From hal.rosenstock at gmail.com Wed Jul 1 14:23:39 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 1 Jul 2009 17:23:39 -0400 Subject: [ofa-general] Library routine for enumerating IB cards and GUIDS In-Reply-To: References: Message-ID: On Wed, Jul 1, 2009 at 4:16 PM, Chris Worley wrote: > I need a programmatic way to enumerate all IB cards in the system, > i.e. their port GUIDs. > > What library/routines might I use, and which docs should I read > concerning exposed routines? Is /sys/class/infiniband/*/ports/*/gids/0 sufficient ? The port GUIDs are the low 64 bits. That's one way. -- Hal > Thanks, > > Chris > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From hal.rosenstock at gmail.com Wed Jul 1 14:18:30 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 1 Jul 2009 17:18:30 -0400 Subject: [ofa-general] Re: [PATCH v3] libibmad: Handle MAD redirection In-Reply-To: <20090701200043.GF20745@obsidianresearch.com> References: <200906291410.33477.fenkes@de.ibm.com> <200906301404.05895.fenkes@de.ibm.com> <20090630180003.GD2982@obsidianresearch.com> <200907011534.59787.fenkes@de.ibm.com> <20090701151425.GD20745@obsidianresearch.com> <20090701165036.GE20745@obsidianresearch.com> <20090701200043.GF20745@obsidianresearch.com> Message-ID: On Wed, Jul 1, 2009 at 4:00 PM, Jason Gunthorpe wrote: > On Wed, Jul 01, 2009 at 03:39:01PM -0400, Hal Rosenstock wrote: > >> > Clearly the only sane way this can work is if the GID is always >> > filled in for the redirection case. >> >> Why is that ? Why must the redirector provide GRH info when it's not >> required for subnet local cases ? > > Because the redirector doesn't know what the initiator will do. It > could include a GRH, or maybe not. It must include the GID to cover > both cases. It could restrict what the initiator can do by doing this. Nothing wrong with that AFAIT. I agree that this is not what you'd want if the requester were not subnet local. I'm only talking about the subnet local case. >> >> Futhermore, RedirectLID can be non zero but GID redirection is still >> >> being used as indicated by the RedirectLID description indicating that >> >> a non zero RedirectLID will in general not be valid. >> > >> > The spec says if it is not zero the requester shall use it. I don't >> > see an ambiguity here. >> >> To me, the ambiguity is several lines below it where it states that >> the RedirectLID might not be valid and says to obtain a PathRecord >> when RedirectGID is supplied rather than relying on the RedirectLID is >> non zero. > > Whoever authored this should not have mixed 'will in general not be > valid' and 'they must be replaced' in the same sentance - but I think > the meaning is still clear. With a 0 RedirectLID only the RedirectGID, > QP and P_Key are to be used by the receiver. When RedirectLID is not 0 > then all of the Redirect fields must contain correct data and should > be used as necessary by the receiver. > > It never says to obtain a Path Record when a GID is supplied. It says > to obtain a path record with RedirectLID is 0. In looking at this some more, I agree with you on this part now since all that text is part of the RedirectLID 0 paragraph. -- Hal > Jason > From worleys at gmail.com Wed Jul 1 14:42:45 2009 From: worleys at gmail.com (Chris Worley) Date: Wed, 1 Jul 2009 15:42:45 -0600 Subject: [ofa-general] Library routine for enumerating IB cards and GUIDS In-Reply-To: References: Message-ID: On Wed, Jul 1, 2009 at 3:23 PM, Hal Rosenstock wrote: > On Wed, Jul 1, 2009 at 4:16 PM, Chris Worley wrote: >> I need a programmatic way to enumerate all IB cards in the system, >> i.e. their port GUIDs. >> >> What library/routines might I use, and which docs should I read >> concerning exposed routines? > > Is /sys/class/infiniband/*/ports/*/gids/0 sufficient ? The port GUIDs > are the low 64 bits. That's one way. I can definitely write scripts to collect the data... I was looking for "C" routines, other than "system" calls to scripts. Thanks, Chris > > -- Hal > >> Thanks, >> >> Chris >> _______________________________________________ >> general mailing list >> general at lists.openfabrics.org >> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >> >> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general >> > From dledford at redhat.com Wed Jul 1 14:56:38 2009 From: dledford at redhat.com (Doug Ledford) Date: Wed, 1 Jul 2009 17:56:38 -0400 Subject: [ofa-general] RH 5.3 : ifup-ib does not setup connected mode on slaves In-Reply-To: <4A4BD499.1000600@oracle.com> References: <4A4BD499.1000600@oracle.com> Message-ID: <6ECADFFD-2C4F-4A27-AA2B-FB1CB81BE4AC@redhat.com> On Jul 1, 2009, at 5:26 PM, Sumeet Lahorani wrote: > Is this a bug or is there a different way to bond together 2 > connected mode interfaces? > # rpm -qf /etc/sysconfig/network-scripts/ifup-ib > openib-1.3.2-0.20080728.0355.3.el5 Bug, I'll fix it in the next release. -- Doug Ledford GPG KeyID: CFBFF194 http://people.redhat.com/dledford InfiniBand Specific RPMS http://people.redhat.com/dledford/Infiniband -------------- next part -------------- A non-text attachment was scrubbed... Name: PGP.sig Type: application/pgp-signature Size: 203 bytes Desc: This is a digitally signed message part URL: From robert.j.woodruff at intel.com Wed Jul 1 15:05:38 2009 From: robert.j.woodruff at intel.com (Woodruff, Robert J) Date: Wed, 1 Jul 2009 15:05:38 -0700 Subject: [ofa-general] Library routine for enumerating IB cards and GUIDS In-Reply-To: References: Message-ID: <382A478CAD40FA4FB46605CF81FE39F43568408C@orsmsx507.amr.corp.intel.com> You might look at ibv_get_devices(). IBV_GET_DEVICE_LIST(3) Libibverbs Programmer's Manual IBV_GET_DEVICE_LIST(3) NAME ibv_get_device_list, ibv_free_device_list - get and release list of available RDMA devices SYNOPSIS #include struct ibv_device **ibv_get_device_list(int *num_devices); void ibv_free_device_list(struct ibv_device **list); DESCRIPTION ibv_get_device_list() returns a NULL-terminated array of RDMA devices currently available. The argument num_devices is optional; if not NULL, it is set to the number of devices returned in the array. ibv_free_device_list() frees the array of devices list returned by ibv_get_device_list(). RETURN VALUE ibv_get_device_list() returns the array of available RDMA devices, or NULL if the request fails. ibv_free_device_list() returns no value. NOTES Client code should open all the devices it intends to use with ibv_open_device() before calling ibv_free_device_list(). Once it frees the array with ibv_free_device_list(), it will be able to use only the open devices; pointers to unopened devices will no longer be valid. SEE ALSO ibv_fork_init(3), ibv_get_device_name(3), ibv_get_device_guid(3), ibv_open_device(3) -----Original Message----- From: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Chris Worley Sent: Wednesday, July 01, 2009 1:17 PM To: OpenIB Subject: [ofa-general] Library routine for enumerating IB cards and GUIDS I need a programmatic way to enumerate all IB cards in the system, i.e. their port GUIDs. What library/routines might I use, and which docs should I read concerning exposed routines? Thanks, Chris _______________________________________________ general mailing list general at lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From weiny2 at llnl.gov Wed Jul 1 17:57:12 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Wed, 1 Jul 2009 17:57:12 -0700 Subject: [ofa-general] Library routine for enumerating IB cards and GUIDS In-Reply-To: References: Message-ID: <20090701175712.55bd33f0.weiny2@llnl.gov> On Wed, 1 Jul 2009 15:42:45 -0600 Chris Worley wrote: > On Wed, Jul 1, 2009 at 3:23 PM, Hal Rosenstock wrote: > > On Wed, Jul 1, 2009 at 4:16 PM, Chris Worley wrote: > >> I need a programmatic way to enumerate all IB cards in the system, > >> i.e. their port GUIDs. > >> > >> What library/routines might I use, and which docs should I read > >> concerning exposed routines? > > > > Is /sys/class/infiniband/*/ports/*/gids/0 sufficient ? The port GUIDs > > are the low 64 bits. That's one way. > > I can definitely write scripts to collect the data... I was looking > for "C" routines, other than "system" calls to scripts. When you say system do you mean a single node (computer) or the entire IB fabric? The former has been answered. If you mean the latter there is a new libibnetdisc library in the infiniband-diags which provides C interfaces to scan the fabric. Ira From abenjamin at sgi.com Wed Jul 1 19:46:21 2009 From: abenjamin at sgi.com (Arputham Benjamin) Date: Wed, 01 Jul 2009 19:46:21 -0700 Subject: [ofa-general] [PATCH v2] mthca: Distinguish multiple IB cards in /proc/interrupts Message-ID: <4A4C1F7D.9040101@sgi.com> When the mthca driver calls request_irq() to allocate interrupt resources, it uses the fixed device name string "ib_mthca". When multiple IB cards are present in the system, every instance of the resource is named "ib_mthca" in /proc/interrupts. This can make it very confusing trying to work out exactly where IB interrupts are going and why. The mthca driver has been modified to use the PCI name of the IB card for the purpose of allocating interrupt resources. Signed-off-by: Arputham Benjamin --- diff -rup a/drivers/infiniband/hw/mthca/mthca_dev.h b/drivers/infiniband/hw/mthca/mthca_dev.h --- a/drivers/infiniband/hw/mthca/mthca_dev.h 2009-06-29 18:03:30.640012711 -0700 +++ b/drivers/infiniband/hw/mthca/mthca_dev.h 2009-06-29 18:05:45.871951427 -0700 @@ -356,6 +356,7 @@ struct mthca_dev { struct ib_ah *sm_ah[MTHCA_MAX_PORTS]; spinlock_t sm_lock; u8 rate[MTHCA_MAX_PORTS]; + char irq_name[MTHCA_NUM_EQ][IB_DEVICE_NAME_MAX]; }; #ifdef CONFIG_INFINIBAND_MTHCA_DEBUG diff -rup a/drivers/infiniband/hw/mthca/mthca_eq.c b/drivers/infiniband/hw/mthca/mthca_eq.c --- a/drivers/infiniband/hw/mthca/mthca_eq.c 2009-06-29 18:06:20.016944802 -0700 +++ b/drivers/infiniband/hw/mthca/mthca_eq.c 2009-06-30 17:05:17.953297035 -0700 @@ -864,21 +864,27 @@ int mthca_init_eq_table(struct mthca_dev }; for (i = 0; i < MTHCA_NUM_EQ; ++i) { + snprintf(&dev->irq_name[i][0], IB_DEVICE_NAME_MAX, + "%s at pci:%s", eq_name[i], + pci_name(dev->pdev)); err = request_irq(dev->eq_table.eq[i].msi_x_vector, mthca_is_memfree(dev) ? mthca_arbel_msi_x_interrupt : mthca_tavor_msi_x_interrupt, - 0, eq_name[i], dev->eq_table.eq + i); + 0, &dev->irq_name[i][0], + dev->eq_table.eq + i); if (err) goto err_out_cmd; dev->eq_table.eq[i].have_irq = 1; } } else { + snprintf(&dev->irq_name, IB_DEVICE_NAME_MAX, + DRV_NAME "@pci:%s", pci_name(dev->pdev)); err = request_irq(dev->pdev->irq, mthca_is_memfree(dev) ? mthca_arbel_interrupt : mthca_tavor_interrupt, - IRQF_SHARED, DRV_NAME, dev); + IRQF_SHARED, &dev->irq_name, dev); if (err) goto err_out_cmd; dev->eq_table.have_irq = 1; From abenjamin at sgi.com Wed Jul 1 20:04:57 2009 From: abenjamin at sgi.com (Arputham Benjamin) Date: Wed, 01 Jul 2009 20:04:57 -0700 Subject: [ofa-general] [PATCH ] mlx4_core: Distinguish multiple IB cards in /proc/interrupts Message-ID: <4A4C23D9.1030400@sgi.com> When the mlx4_core driver calls request_irq() to allocate interrupt resources, it uses the fixed device name string "mlx4_core". When multiple IB cards are present in the system, every instance of the resource is named "mlx4_core" in /proc/interrupts. This can make it very confusing trying to work out exactly where IB interrupts are going and why. The mlx4_core driver has been modified to use the PCI name of the IB card for the purpose of allocating interrupt resources. Signed-off-by: Arputham Benjamin --- diff -rup a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c --- a/drivers/net/mlx4/eq.c 2009-06-24 16:28:12.273861234 -0700 +++ b/drivers/net/mlx4/eq.c 2009-06-30 16:38:58.200069975 -0700 @@ -609,14 +609,21 @@ int mlx4_init_eq_table(struct mlx4_dev * for (i = 0; i < MLX4_EQ_COMP_CPU0 + dev->caps.num_comp_vectors; ++i) { if (i == 0) - snprintf(eq_name[0], 20, DRV_NAME "(async)"); + snprintf(&priv->irq_name[i][0], + DEVICE_NAME_SIZE, + DRV_NAME "(async)" "@pci:%s", + pci_name(dev->pdev)); else - snprintf(eq_name[i], 20, "eth-mlx4-%d", - i - 1); + snprintf(&priv->irq_name[i][0], + DEVICE_NAME_SIZE, + "eth-mlx4-%d" "@pci:%s", + i - 1, + pci_name(dev->pdev)); err = request_irq(priv->eq_table.eq[i].irq, mlx4_msi_x_interrupt, - 0, eq_name[i], priv->eq_table.eq + i); + 0, &priv->irq_name[i][0], + priv->eq_table.eq + i); if (err) goto err_out_async; @@ -624,8 +631,10 @@ int mlx4_init_eq_table(struct mlx4_dev * } } else { + snprintf(&priv->irq_name, DEVICE_NAME_SIZE, + DRV_NAME "@pci:%s", pci_name(dev->pdev)); err = request_irq(dev->pdev->irq, mlx4_interrupt, - IRQF_SHARED, DRV_NAME, dev); + IRQF_SHARED, &priv->irq_name, dev); if (err) goto err_out_async; diff -rup a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h --- a/drivers/net/mlx4/mlx4.h 2009-06-24 16:22:33.600623178 -0700 +++ b/drivers/net/mlx4/mlx4.h 2009-06-30 16:16:54.532981963 -0700 @@ -334,6 +334,7 @@ struct mlx4_priv { int changed_ports; struct mlx4_sense sense; struct mutex port_mutex; + char irq_name[MLX4_NUM_EQ][DEVICE_NAME_SIZE]; }; static inline struct mlx4_priv *mlx4_priv(struct mlx4_dev *dev) From sumit.panchasara at einfochips.com Thu Jul 2 01:08:16 2009 From: sumit.panchasara at einfochips.com (Sumit Panchasara) Date: Thu, 2 Jul 2009 13:38:16 +0530 Subject: [ofa-general] ib_rdma_bw - memory leaks? In-Reply-To: <785E3F61D0AF4EA48288A3C56CD7B11F@amr.corp.intel.com> References: <000b01c9fa34$4b407a50$e1c16ef0$@panchasara@einfochips.com> <785E3F61D0AF4EA48288A3C56CD7B11F@amr.corp.intel.com> Message-ID: <003d01c9faec$414f4630$c3edd290$@panchasara@einfochips.com> Yes, rdma_cma_fini() calls the ucma_cleanup() on lib unload (usually after main() is returned) as it is declared with "__attribute__((destructor))". I observed its address written inside .dtors section in objdump. As mentioned in my previous email, there are other 3 places of memory leaks, should I proceed and fix them up in rdma_bw.c file? Thanks, Sumit -----Original Message----- From: Sean Hefty [mailto:sean.hefty at intel.com] Sent: Wednesday, July 01, 2009 9:36 PM To: 'Sumit Panchasara'; ewg at lists.openfabrics.org; general at lists.openfabrics.org Subject: RE: [ofa-general] ib_rdma_bw - memory leaks? >3. rdma_create_event_channel() calls ucma_init() but >rdma_destroy_event_channel() does not call ucma_cleanup(), this results into >memory leak at provider's library since it does not call ibv_close_device() >and thus unable to do *->free_context(). This is the correct behavior. ucma_init() is called from several routines to ensure that the library performs proper initialization. Once initialized, it remains initialized until the library is no longer used. The cleanup is done in rdma_cma_fini(). - Sean Email Scanned for Virus & Dangerous Content by : www.CleanMailGateway.com -- _____________________________________________________________________ Disclaimer: This e-mail message and all attachments transmitted with it are intended solely for the use of the addressee and may contain legally privileged and confidential information. If the reader of this message is not the intended recipient, or an employee or agent responsible for delivering this message to the intended recipient, you are hereby notified that any dissemination, distribution, copying, or other use of this message or its attachments is strictly prohibited. If you have received this message in error, please notify the sender immediately by replying to this message and please delete it from your computer. Any views expressed in this message are those of the individual sender unless otherwise stated.Company has taken enough precautions to prevent the spread of viruses. However the company accepts no liability for any damage caused by any virus transmitted by this email. _____________________________________________________________________ From lars.ellenberg at linbit.com Thu Jul 2 01:14:45 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Thu, 2 Jul 2009 10:14:45 +0200 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <4A4B947C.1030607@mellanox.co.il> References: <20090701133652.GG9112@soda.linbit> <4A4B947C.1030607@mellanox.co.il> Message-ID: <20090702081445.GA9118@soda.linbit> On Wed, Jul 01, 2009 at 07:53:16PM +0300, Amir Vadai wrote: > I understand that you're using ofed-1.5. > We should check if those bugs exist in 1.4.1 too - since there were many changes in SDP in ofed-1.5. actually I did my tests mostly on ofed 1.4, as the ofed 1.5 snapshot I tried did not even compile at that point in time ;) so they definetely exist in 1.4, we need to verify on 1.5. > Please open bugs in bugzilla here: https://bugs.openfabrics.org will do. -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. From amirv at mellanox.co.il Thu Jul 2 01:30:25 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Thu, 02 Jul 2009 11:30:25 +0300 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <20090702081445.GA9118@soda.linbit> References: <20090701133652.GG9112@soda.linbit> <4A4B947C.1030607@mellanox.co.il> <20090702081445.GA9118@soda.linbit> Message-ID: <4A4C7021.2090506@mellanox.co.il> Please attach the perl script to reproduce and I will check it. As to the second problem - I did notice such behavior but couldn't find a scenario to reproduce it. I guess it happen when the socket is closed due to error. Tell me if you notice any message in dmesg. P.S please set the mailing list in the CC and my direct mail in the TO... - Amir On 07/02/2009 11:14 AM, Lars Ellenberg wrote: > On Wed, Jul 01, 2009 at 07:53:16PM +0300, Amir Vadai wrote: >> I understand that you're using ofed-1.5. >> We should check if those bugs exist in 1.4.1 too - since there were many changes in SDP in ofed-1.5. > > actually I did my tests mostly on ofed 1.4, as the ofed 1.5 snapshot I > tried did not even compile at that point in time ;) > > so they definetely exist in 1.4, we need to verify on 1.5. > >> Please open bugs in bugzilla here: https://bugs.openfabrics.org > > will do. > -- Amir Vadai Software Eng. Mellanox Technologies mailto: amirv at mellanox.co.il Tel +972-3-6259539 From yevgenyp at mellanox.co.il Thu Jul 2 02:44:10 2009 From: yevgenyp at mellanox.co.il (Yevgeny Petrilin) Date: Thu, 02 Jul 2009 12:44:10 +0300 Subject: [ofa-general][PATCH 2/2] mlx4: ConnectX multi functional device support In-Reply-To: <4A3A34F5.9030800@mellanox.co.il> References: <4A3A34F5.9030800@mellanox.co.il> Message-ID: <4A4C816A.2040506@mellanox.co.il> Roland, Following our discussion on these patches, Is there any update, will they make it to 2.6.31? Thanks, Yevgeny From vlad at lists.openfabrics.org Thu Jul 2 02:23:58 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Thu, 2 Jul 2009 02:23:58 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090702-0200 daily build status Message-ID: <20090702092358.9E0BF10202CD@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-8.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-8.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-53.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-53.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-53.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.20_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.20_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.27_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1383: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1402: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1383: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1402: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_ia64_check/drivers/infiniband/core/addr.c:36: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_ia64_check/kernel_addons/backport/2.6.22/include/asm/bitops.h: In function 'clear_bit_unlock': /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_ia64_check/kernel_addons/backport/2.6.22/include/asm/bitops.h:8: error: implicit declaration of function 'smp_mb' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_ia64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_ia64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.23_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.23_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18-8.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.18-8.el5_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18-8.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ppc64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ppc64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090702-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From lars.ellenberg at linbit.com Thu Jul 2 06:23:04 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Thu, 2 Jul 2009 15:23:04 +0200 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <4A4C7021.2090506@mellanox.co.il> References: <20090701133652.GG9112@soda.linbit> <4A4B947C.1030607@mellanox.co.il> <20090702081445.GA9118@soda.linbit> <4A4C7021.2090506@mellanox.co.il> Message-ID: <20090702132304.GD9305@soda.linbit> On Thu, Jul 02, 2009 at 11:30:25AM +0300, Amir Vadai wrote: > Please attach the perl script to reproduce and I will check it. > As to the second problem - I did notice such behavior but couldn't > find a scenario to reproduce it. I guess it happen when the socket is > closed due to error. > > Tell me if you notice any message in dmesg. My former test cluster has been reassigned. The new test hardware will be available earliest tomorrow. But I don't think there was anything relevant in dmesg. I'll try to reproduce on the new test hardware then, and will get back to you as soon as possible. -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. From rdreier at cisco.com Thu Jul 2 07:35:06 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 02 Jul 2009 07:35:06 -0700 Subject: [ofa-general][PATCH 2/2] mlx4: ConnectX multi functional device support In-Reply-To: <4A4C816A.2040506@mellanox.co.il> (Yevgeny Petrilin's message of "Thu, 02 Jul 2009 12:44:10 +0300") References: <4A3A34F5.9030800@mellanox.co.il> <4A4C816A.2040506@mellanox.co.il> Message-ID: > Is there any update, will they make it to 2.6.31? Not sure... need to evaluate the risk of regression From tziporet at dev.mellanox.co.il Thu Jul 2 08:23:03 2009 From: tziporet at dev.mellanox.co.il (Tziporet Koren) Date: Thu, 02 Jul 2009 18:23:03 +0300 Subject: [ofa-general][PATCH 2/2] mlx4: ConnectX multi functional device support In-Reply-To: References: <4A3A34F5.9030800@mellanox.co.il> <4A4C816A.2040506@mellanox.co.il> Message-ID: <4A4CD0D7.6070409@mellanox.co.il> Roland Dreier wrote: > > Is there any update, will they make it to 2.6.31? > > Not sure... need to evaluate the risk of regression > The risk is very low, however without it our new device coming in July will not work on the new kernel :-( And if not on 2.6.31 can you at list queue it on your future work tree? Tziporet From rdreier at cisco.com Thu Jul 2 10:05:23 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 02 Jul 2009 10:05:23 -0700 Subject: [ofa-general][PATCH 2/2] mlx4: ConnectX multi functional device support In-Reply-To: <4A4CD0D7.6070409@mellanox.co.il> (Tziporet Koren's message of "Thu, 02 Jul 2009 18:23:03 +0300") References: <4A3A34F5.9030800@mellanox.co.il> <4A4C816A.2040506@mellanox.co.il> <4A4CD0D7.6070409@mellanox.co.il> Message-ID: > The risk is very low, however without it our new device coming in July > will not work on the new kernel :-( > And if not on 2.6.31 can you at list queue it on your future work tree? Well, I need to think over the patch a little more. The risk is certainly not zero, since we are changing device initialization for all mlx4 devices. And I'm not sure if missing 2.6.31 support is that big a deal for you, is it? How many users do you have building their own upstream kernels? You're just going to tell everyone to use OFED anyway, right? - R. From sean.hefty at intel.com Thu Jul 2 15:49:12 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Thu, 2 Jul 2009 15:49:12 -0700 Subject: [ofa-general] ib_rdma_bw - memory leaks? In-Reply-To: <003d01c9faec$414f4630$c3edd290$@panchasara@einfochips.com> References: <000b01c9fa34$4b407a50$e1c16ef0$@panchasara@einfochips.com> <785E3F61D0AF4EA48288A3C56CD7B11F@amr.corp.intel.com> <003d01c9faec$414f4630$c3edd290$@panchasara@einfochips.com> Message-ID: <0E8DEF82D0614EB992D6AA8F62453AEA@amr.corp.intel.com> >As mentioned in my previous email, there are other 3 places of memory leaks, >should I proceed and fix them up in rdma_bw.c file? I think that makes sense; I was only commenting on the code that I maintain. Based on looking at the git trees, it appears that Owen Meron is the maintainer of ib_rdma_bw. - Sean From ranjit.pandit.ib at gmail.com Thu Jul 2 16:17:33 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Thu, 2 Jul 2009 16:17:33 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp Message-ID: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> Hi, I have noticed that cmatose test fails when running between two nodes on iWarp (Chelsio) whereas rping works fine. I'm using OFED 1.4.1. Looks like it's getting a RDMA_CM_EVENT_CONNECT_ERROR. I'm wondering if anybody else has seen this behavior. Is cmatose expected to work on iWarp? It is successfully getting ROUTE_RESOLVED but something is going bad after calling rdma_connect in route_handler. In rping's case, rdma_connect is being called in the main thread. Could that be the reason it works? Here is the error with ucmatose. [root at lv2 examples]# ./ucmatose -s 192.168.10.30 cmatose: starting client cmatose: connecting cmatose: event: RDMA_CM_EVENT_CONNECT_ERROR, error: -22 receiving data transfers sending replies data transfers complete test complete return status 0 thanks, Ranjit From sean.hefty at intel.com Thu Jul 2 16:40:32 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Thu, 2 Jul 2009 16:40:32 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> Message-ID: <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> >I'm wondering if anybody else has seen this behavior. >Is cmatose expected to work on iWarp? It's intended to work on iWarp. >[root at lv2 examples]# ./ucmatose -s 192.168.10.30 >cmatose: starting client >cmatose: connecting >cmatose: event: RDMA_CM_EVENT_CONNECT_ERROR, error: -22 This looks like an asynchronous error occurring while trying to connect. I don't see anything obvious in cmatose.c that would lead to a connect error. Does anything occur on the server side? - Sean From ranjit.pandit.ib at gmail.com Thu Jul 2 16:50:18 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Thu, 2 Jul 2009 16:50:18 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> Message-ID: <96f8e60e0907021650w64253c8ap4d577f5384a5350c@mail.gmail.com> I get this on the server side while the client errors out. [root at lv1 examples]# ./ucmatose cmatose: starting server initiating data transfers completing sends On Thu, Jul 2, 2009 at 4:40 PM, Sean Hefty wrote: >>I'm wondering if anybody else has seen this behavior. >>Is cmatose expected to work on iWarp? > > It's intended to work on iWarp. > >>[root at lv2 examples]# ./ucmatose -s 192.168.10.30 >>cmatose: starting client >>cmatose: connecting >>cmatose: event: RDMA_CM_EVENT_CONNECT_ERROR, error: -22 > > This looks like an asynchronous error occurring while trying to connect.  I > don't see anything obvious in cmatose.c that would lead to a connect error. > Does anything occur on the server side? > > - Sean > > From ranjit.pandit.ib at gmail.com Thu Jul 2 16:58:46 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Thu, 2 Jul 2009 16:58:46 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <96f8e60e0907021650w64253c8ap4d577f5384a5350c@mail.gmail.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907021650w64253c8ap4d577f5384a5350c@mail.gmail.com> Message-ID: <96f8e60e0907021658j3dc3811al2733d71f48c25b48@mail.gmail.com> Forgot to mention that the server side hangs while the client errors out. On IB, the server side also completes successfully. On Thu, Jul 2, 2009 at 4:50 PM, pandit ib wrote: > I get this on the server side while the client errors out. > > [root at lv1 examples]# ./ucmatose > cmatose: starting server > initiating data transfers > completing sends > > > > On Thu, Jul 2, 2009 at 4:40 PM, Sean Hefty wrote: >>>I'm wondering if anybody else has seen this behavior. >>>Is cmatose expected to work on iWarp? >> >> It's intended to work on iWarp. >> >>>[root at lv2 examples]# ./ucmatose -s 192.168.10.30 >>>cmatose: starting client >>>cmatose: connecting >>>cmatose: event: RDMA_CM_EVENT_CONNECT_ERROR, error: -22 >> >> This looks like an asynchronous error occurring while trying to connect.  I >> don't see anything obvious in cmatose.c that would lead to a connect error. >> Does anything occur on the server side? >> >> - Sean >> >> > From arlin.r.davis at intel.com Thu Jul 2 18:03:02 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Thu, 2 Jul 2009 18:03:02 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> Message-ID: >>I'm wondering if anybody else has seen this behavior. >>Is cmatose expected to work on iWarp? > >It's intended to work on iWarp. If this test sends data from server side first you could be running into the iWARP requirement of sending from client first. -arlin From sean.hefty at intel.com Thu Jul 2 23:44:18 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Thu, 2 Jul 2009 23:44:18 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> Message-ID: >If this test sends data from server side first you could >be running into the iWARP requirement of sending from >client first. This was my thought as well. I think Chelsio supports sending from the server side first, but I'm not sure, or if it's enabled by default. - Sean From devel-ofed at morey-chaisemartin.com Fri Jul 3 01:04:12 2009 From: devel-ofed at morey-chaisemartin.com (Nicolas Morey-Chaisemartin) Date: Fri, 03 Jul 2009 10:04:12 +0200 Subject: [ofa-general] [PATCH] opensm: Added support for select counters (xmit_wait) Message-ID: <4A4DBB7C.5000000@morey-chaisemartin.com> Support for xmit_wait counters was missing in the perfmgr though it was read from the mad and event plugin interface already handles it. This patch adds support for it (tested and working with an event plugin) Signed-off-by: Nicolas Morey-Chaisemartin --- opensm/include/opensm/osm_perfmgr_db.h | 23 ++++++- opensm/opensm/osm_perfmgr.c | 30 +++++++- opensm/opensm/osm_perfmgr_db.c | 124 +++++++++++++++++++++++++++++-- 3 files changed, 166 insertions(+), 11 deletions(-) diff --git a/opensm/include/opensm/osm_perfmgr_db.h b/opensm/include/opensm/osm_perfmgr_db.h index 42a47bd..35b5ac3 100644 --- a/opensm/include/opensm/osm_perfmgr_db.h +++ b/opensm/include/opensm/osm_perfmgr_db.h @@ -109,6 +109,14 @@ typedef struct { } perfmgr_db_data_cnt_reading_t; /** ========================================================================= + * Port select count reading + */ +typedef struct { + uint64_t xmit_wait; + time_t time; +} perfmgr_db_sel_reading_t; + +/** ========================================================================= * Dump output options */ typedef enum { @@ -125,6 +133,8 @@ typedef struct db_port { perfmgr_db_err_reading_t err_previous; perfmgr_db_data_cnt_reading_t dc_total; perfmgr_db_data_cnt_reading_t dc_previous; + perfmgr_db_sel_reading_t ps_total; + perfmgr_db_sel_reading_t ps_previous; time_t last_reset; } db_port_t; @@ -179,7 +189,16 @@ perfmgr_db_err_t perfmgr_db_get_prev_dc(perfmgr_db_t * db, uint64_t guid, reading); perfmgr_db_err_t perfmgr_db_clear_prev_dc(perfmgr_db_t * db, uint64_t guid, uint8_t port); - +perfmgr_db_err_t perfmgr_db_add_ps_reading(perfmgr_db_t * db, uint64_t guid, + uint8_t port, + perfmgr_db_sel_reading_t * + reading); +perfmgr_db_err_t perfmgr_db_get_prev_ps(perfmgr_db_t * db, uint64_t guid, + uint8_t port, + perfmgr_db_sel_reading_t * + reading); +perfmgr_db_err_t perfmgr_db_clear_prev_ps(perfmgr_db_t * db, uint64_t guid, + uint8_t port); void perfmgr_db_clear_counters(perfmgr_db_t * db); perfmgr_db_err_t perfmgr_db_dump(perfmgr_db_t * db, char *file, perfmgr_db_dump_t dump_type); @@ -196,6 +215,8 @@ void perfmgr_db_fill_data_cnt_read_pc(ib_port_counters_t * wire_read, perfmgr_db_data_cnt_reading_t * reading); void perfmgr_db_fill_data_cnt_read_epc(ib_port_counters_ext_t * wire_read, perfmgr_db_data_cnt_reading_t * reading); +void perfmgr_db_fill_sel_read(ib_port_counters_t * wire_read, + perfmgr_db_sel_reading_t * reading); END_C_DECLS diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c index ecfdbda..8a9eb12 100644 --- a/opensm/opensm/osm_perfmgr.c +++ b/opensm/opensm/osm_perfmgr.c @@ -853,10 +853,12 @@ void osm_perfmgr_destroy(osm_perfmgr_t * pm) static void perfmgr_check_oob_clear(osm_perfmgr_t * pm, monitored_node_t * mon_node, uint8_t port, perfmgr_db_err_reading_t * cr, - perfmgr_db_data_cnt_reading_t * dc) + perfmgr_db_data_cnt_reading_t * dc, + perfmgr_db_sel_reading_t * ps) { perfmgr_db_err_reading_t prev_err; perfmgr_db_data_cnt_reading_t prev_dc; + perfmgr_db_sel_reading_t prev_ps; if (perfmgr_db_get_prev_err(pm->db, mon_node->guid, port, &prev_err) != PERFMGR_EVENT_DB_SUCCESS) { @@ -905,6 +907,23 @@ static void perfmgr_check_oob_clear(osm_perfmgr_t * pm, mon_node->name, mon_node->guid, port); perfmgr_db_clear_prev_dc(pm->db, mon_node->guid, port); } + + if (perfmgr_db_get_prev_ps(pm->db, mon_node->guid, port, &prev_ps) + != PERFMGR_EVENT_DB_SUCCESS) { + OSM_LOG(pm->log, OSM_LOG_VERBOSE, + "Failed to find previous select count " + "reading for %s (0x%" PRIx64 ") port %u\n", + mon_node->name, mon_node->guid, port); + return; + } + + if (ps->xmit_wait < prev_ps.xmit_wait) { + OSM_LOG(pm->log, OSM_LOG_ERROR, + "PerfMgr: ERR 4C17: Detected an out of band select counter " + "clear on node %s (0x%" PRIx64 ") port %u\n", + mon_node->name, mon_node->guid, port); + perfmgr_db_clear_prev_ps(pm->db, mon_node->guid, port); + } } /********************************************************************** @@ -1062,6 +1081,8 @@ static void pc_recv_process(void *context, void *data) uint8_t port = mad_context->perfmgr_context.port; perfmgr_db_err_reading_t err_reading; perfmgr_db_data_cnt_reading_t data_reading; + perfmgr_db_sel_reading_t select_reading; + cl_map_item_t *p_node; monitored_node_t *p_mon_node; @@ -1148,10 +1169,12 @@ static void pc_recv_process(void *context, void *data) */ perfmgr_db_fill_data_cnt_read_pc(wire_read, &data_reading); + perfmgr_db_fill_sel_read(wire_read, &select_reading); + /* detect an out of band clear on the port */ if (mad_context->perfmgr_context.mad_method != IB_MAD_METHOD_SET) perfmgr_check_oob_clear(pm, p_mon_node, port, &err_reading, - &data_reading); + &data_reading, &select_reading); /* log any critical events from this reading */ perfmgr_log_events(pm, p_mon_node, port, &err_reading); @@ -1161,9 +1184,12 @@ static void pc_recv_process(void *context, void *data) &err_reading); perfmgr_db_add_dc_reading(pm->db, node_guid, port, &data_reading); + perfmgr_db_add_ps_reading(pm->db, node_guid, port, + &select_reading); } else { perfmgr_db_clear_prev_err(pm->db, node_guid, port); perfmgr_db_clear_prev_dc(pm->db, node_guid, port); + perfmgr_db_clear_prev_ps(pm->db, node_guid, port); } perfmgr_check_overflow(pm, p_mon_node, port, wire_read); diff --git a/opensm/opensm/osm_perfmgr_db.c b/opensm/opensm/osm_perfmgr_db.c index e5dfc19..132c2fb 100644 --- a/opensm/opensm/osm_perfmgr_db.c +++ b/opensm/opensm/osm_perfmgr_db.c @@ -486,6 +486,102 @@ Exit: return (rc); } +static inline void +debug_dump_ps_reading(perfmgr_db_t * db, uint64_t guid, uint8_t port_num, + db_port_t * port, perfmgr_db_sel_reading_t * cur) +{ + osm_log_t *log = db->perfmgr->log; + if (!osm_log_is_active(log, OSM_LOG_DEBUG)) + return; + + osm_log(log, OSM_LOG_DEBUG, + "xd %" PRIu64 " <-- %" PRIu64 " (%" PRIu64 ")\n", + cur->xmit_wait, port->ps_previous.xmit_wait, + port->ps_total.xmit_wait); +} + +/********************************************************************** + * perfmgr_db_sel_reading_t functions + **********************************************************************/ +perfmgr_db_err_t +perfmgr_db_add_ps_reading(perfmgr_db_t * db, uint64_t guid, uint8_t port, + perfmgr_db_sel_reading_t * reading) +{ + db_port_t *p_port = NULL; + db_node_t *node = NULL; + perfmgr_db_sel_reading_t *previous = NULL; + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; + osm_epi_ps_event_t epi_ps_data; + + cl_plock_excl_acquire(&db->lock); + node = get(db, guid); + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) + goto Exit; + + p_port = &node->ports[port]; + previous = &node->ports[port].ps_previous; + + debug_dump_ps_reading(db, guid, port, p_port, reading); + + epi_ps_data.time_diff_s = reading->time - previous->time; + osm_epi_create_port_id(&epi_ps_data.port_id, guid, port, + node->node_name); + + /* calculate changes from previous reading */ + epi_ps_data.xmit_wait = reading->xmit_wait - previous->xmit_wait; + p_port->ps_total.xmit_wait += epi_ps_data.xmit_wait; + + p_port->ps_previous = *reading; + osm_opensm_report_event(db->perfmgr->osm, + OSM_EVENT_ID_PORT_SELECT, &epi_ps_data); + +Exit: + cl_plock_release(&db->lock); + return (rc); +} + +perfmgr_db_err_t perfmgr_db_get_prev_ps(perfmgr_db_t * db, uint64_t guid, + uint8_t port, + perfmgr_db_sel_reading_t * reading) +{ + db_node_t *node = NULL; + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; + + cl_plock_acquire(&db->lock); + + node = get(db, guid); + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) + goto Exit; + + *reading = node->ports[port].ps_previous; + +Exit: + cl_plock_release(&db->lock); + return (rc); +} + +perfmgr_db_err_t +perfmgr_db_clear_prev_ps(perfmgr_db_t * db, uint64_t guid, uint8_t port) +{ + db_node_t *node = NULL; + perfmgr_db_sel_reading_t *previous = NULL; + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; + + cl_plock_excl_acquire(&db->lock); + node = get(db, guid); + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) + goto Exit; + + previous = &node->ports[port].ps_previous; + + memset(previous, 0, sizeof(*previous)); + node->ports[port].ps_previous.time = time(NULL); + +Exit: + cl_plock_release(&db->lock); + return (rc); +} + static void clear_counters(cl_map_item_t * const p_map_item, void *context) { db_node_t *node = (db_node_t *) p_map_item; @@ -517,6 +613,8 @@ static void clear_counters(cl_map_item_t * const p_map_item, void *context) node->ports[i].dc_total.multicast_rcv_pkts = 0; node->ports[i].dc_total.time = ts; + node->ports[i].ps_total.xmit_wait = 0; + node->ports[i].last_reset = ts; } } @@ -546,7 +644,7 @@ static void dump_node_mr(db_node_t * node, FILE * fp) "%s\t%s\t" "%s\t%s\t%s\t%s\t%s\t%s\t%s\t" "%s\t%s\t%s\t%s\t%s\t%s\t%s\t" - "%s\t%s\t%s\t%s\n", + "%s\t%s\t%s\t%s\t%s\n", "symbol_err_cnt", "link_err_recover", "link_downed", @@ -565,8 +663,7 @@ static void dump_node_mr(db_node_t * node, FILE * fp) "rcv_pkts", "unicast_xmit_pkts", "unicast_rcv_pkts", - "multicast_xmit_pkts", - "multicast_rcv_pkts"); + "multicast_xmit_pkts", "multicast_rcv_pkts", "xmit_wait"); for (i = (node->esp0) ? 0 : 1; i < node->num_ports; i++) { char *since = ctime(&node->ports[i].last_reset); since[strlen(since) - 1] = '\0'; /* remove \n */ @@ -577,8 +674,8 @@ static void dump_node_mr(db_node_t * node, FILE * fp) "%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 - "\t%" PRIu64 "\t%" PRIu64 "\n", node->node_name, - node->node_guid, i, since, + "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\n", + node->node_name, node->node_guid, i, since, node->ports[i].err_total.symbol_err_cnt, node->ports[i].err_total.link_err_recover, node->ports[i].err_total.link_downed, @@ -598,7 +695,8 @@ static void dump_node_mr(db_node_t * node, FILE * fp) node->ports[i].dc_total.unicast_xmit_pkts, node->ports[i].dc_total.unicast_rcv_pkts, node->ports[i].dc_total.multicast_xmit_pkts, - node->ports[i].dc_total.multicast_rcv_pkts); + node->ports[i].dc_total.multicast_rcv_pkts, + node->ports[i].ps_total.xmit_wait); } } @@ -634,7 +732,8 @@ static void dump_node_hr(db_node_t * node, FILE * fp) " unicast_xmit_pkts : %" PRIu64 "\n" " unicast_rcv_pkts : %" PRIu64 "\n" " multicast_xmit_pkts : %" PRIu64 "\n" - " multicast_rcv_pkts : %" PRIu64 "\n", + " multicast_rcv_pkts : %" PRIu64 "\n" + " xmit_wait : %" PRIu64 "\n", node->node_name, node->node_guid, i, @@ -658,7 +757,8 @@ static void dump_node_hr(db_node_t * node, FILE * fp) node->ports[i].dc_total.unicast_xmit_pkts, node->ports[i].dc_total.unicast_rcv_pkts, node->ports[i].dc_total.multicast_xmit_pkts, - node->ports[i].dc_total.multicast_rcv_pkts); + node->ports[i].dc_total.multicast_rcv_pkts, + node->ports[i].ps_total.xmit_wait); } } @@ -809,4 +909,12 @@ perfmgr_db_fill_data_cnt_read_epc(ib_port_counters_ext_t * wire_read, reading->multicast_rcv_pkts = cl_ntoh64(wire_read->multicast_rcv_pkts); reading->time = time(NULL); } + +void +perfmgr_db_fill_sel_read(ib_port_counters_t * wire_read, + perfmgr_db_sel_reading_t * reading) +{ + reading->xmit_wait = cl_ntoh32(wire_read->xmit_wait); + reading->time = time(NULL); +} #endif /* ENABLE_OSM_PERF_MGR */ From vlad at lists.openfabrics.org Fri Jul 3 02:21:33 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Fri, 3 Jul 2009 02:21:33 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090703-0200 daily build status Message-ID: <20090703092133.460501020255@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-53.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-53.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-53.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-8.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-8.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.20_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.20_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.27_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1383: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1402: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1383: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1402: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_ia64_check/drivers/infiniband/core/addr.c:36: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_ia64_check/kernel_addons/backport/2.6.22/include/asm/bitops.h: In function 'clear_bit_unlock': /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_ia64_check/kernel_addons/backport/2.6.22/include/asm/bitops.h:8: error: implicit declaration of function 'smp_mb' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_ia64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_ia64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.23_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.23_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18-8.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.18-8.el5_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18-8.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ppc64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ppc64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090703-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From max.lapan at gmail.com Fri Jul 3 04:47:24 2009 From: max.lapan at gmail.com (Max Lapan) Date: Fri, 3 Jul 2009 15:47:24 +0400 Subject: [ofa-general] Problem with ib_dma_map_single Message-ID: Hello, currently, I'm trying to write small sample kernel module which performs verbs communication in UD mode. Almost everything is complete, but when I am trying to map allocated memory for DMA, I get GPF: general protection fault: 0000 [1] SMP last sysfs file: /sys/devices/pci0000:80/0000:80:11.0/0000:86:01.1/irq CPU 0 Modules linked in: verbs(N+) bridge stp netbk blkbk blktap xenbus_be rdma_ucm(N) ib_ucm(N) ib_srp(N) scsi_transport_srp scsi_tgt ib_sdp(N) rdma_cm(N) iw_cm(N) ib_addr(N) ib_ipoib(N) ib_cm(N) ib_sa(N) ipv6 ib_uverbs(N) ib_umad(N) ib_ipath(N) mlx4_ib(N) ib_mad(N) ib_core(N) fuse ext2 loop sg mptctl e1000 rtc_cmos rtc_core rtc_lib serio_raw pcspkr shpchp k8temp(N) pci_hotplug mlx4_core(N) forcedeth joydev 8250_pnp cfi_cmdset_0001(N) cfi_util(N) i2c_nforce2 jedec_probe(N) i2c_core cfi_probe(N) gen_probe(N) button ck804xrom(N) mtd chipreg(N) map_funcs(N) 8250 serial_core usbhid hid ff_memless linear ohci_hcd sd_mod crc_t10dif ehci_hcd usbcore dm_snapshot xenblk cdrom xennet edd dm_mod ext3 mbcache jbd fan ide_pci_generic amd74xx ide_core ata_generic pata_amd libata dock mptsas mptscsih mptbase scsi_transport_sas scsi_mod thermal processor thermal_sys hwmon Supported: No Pid: 4027, comm: insmod Tainted: G 2.6.27.23-0.1-xen #1 RIP: e030:[] [] verbs_add_device+0x497/0xd9e [verbs] RSP: e02b:ffff880070ea7c28 EFLAGS: 00010286 RAX: ec83485355544155 RBX: ffff880073556000 RCX: 0000000000000002 RDX: 0000000000010000 RSI: ffff8800693f0000 RDI: ffff880073556000 RBP: ffffffffa04a48c0 R08: ffff880074803ae0 R09: 0000000074ac82e0 R10: 00008d8d00000000 R11: 000200d08068c8e8 R12: ffffffffa0200000 R13: 0000000000000000 R14: 00007fff478f978e R15: 0000000000000002 FS: 00007f753f8e56f0(0000) GS:ffffffff80761080(0000) knlGS:0000000000000000 CS: e033 DS: 0000 ES: 0000 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process insmod (pid: 4027, threadinfo ffff880070ea6000, task ffff8800693a0640) Stack: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0000000000000000 Call Trace: [] ib_register_client+0x5b/0x88 [ib_core] [] verbs_init+0x6e/0x9e [verbs] [] _stext+0x42/0x14a [] sys_init_module+0xa0/0x1ba [] system_call_fastpath+0x16/0x1b [<00007f753f45576a>] 0x7f753f45576a Code: 00 48 8b 80 98 02 00 00 48 8b 40 08 8b 8c 24 3c 01 00 00 48 8b 94 24 40 01 00 00 48 8b b4 24 48 01 00 00 48 8b bc 24 50 01 00 00 d0 48 89 44 24 08 e9 8b 01 00 00 8b 94 24 3c 01 00 00 48 8b RIP [] verbs_add_device+0x497/0xd9e [verbs] RSP ---[ end trace 0975eb2c30b98904 ]--- This error caused by this piece of code: len = 2048; buf = kzalloc (len, GFP_KERNEL); key = ib_dma_map_single (dev, buf, len, DMA_TO_DEVICE); I've tried to use different GFP_XX flags, but without success. The similar code in ipoib and srp drivers works well, but, maybe I am missing something. I also tried to use ib_dma_alloc_coherent, but got the same error. Are there any restrictions for ib_dma_map_single call? Must I initialize something to call it? On this machine I have SLES11 with OFED 1.4.1 installed. All IB stuff works well (IPoIB, SRP, verbs benchmarks, etc). -- wbr, Max Lapan From hnrose at comcast.net Thu Jul 2 20:23:54 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Thu, 2 Jul 2009 23:23:54 -0400 Subject: [ofa-general] [PATCH] opensm/osm_sa_(path multipath)_record.c: Fix typo in a couple of log messages Message-ID: <20090703032354.GA28557@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_sa_multipath_record.c b/opensm/opensm/osm_sa_multipath_record.c index 737d892..6ea5ba0 100644 --- a/opensm/opensm/osm_sa_multipath_record.c +++ b/opensm/opensm/osm_sa_multipath_record.c @@ -660,7 +660,7 @@ static ib_api_status_t mpr_rcv_get_path_parms(IN osm_sa_t * sa, if (p_qos_level && p_qos_level->sl_set && p_qos_level->sl != required_sl) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 451E: " - "QoS constaraints: required MultiPathRecord SL (%u) " + "QoS constraints: required MultiPathRecord SL (%u) " "doesn't match QoS policy SL (%u)\n", required_sl, p_qos_level->sl); status = IB_NOT_FOUND; diff --git a/opensm/opensm/osm_sa_path_record.c b/opensm/opensm/osm_sa_path_record.c index 9b50deb..75d9516 100644 --- a/opensm/opensm/osm_sa_path_record.c +++ b/opensm/opensm/osm_sa_path_record.c @@ -663,7 +663,7 @@ static ib_api_status_t pr_rcv_get_path_parms(IN osm_sa_t * sa, if (p_qos_level && p_qos_level->sl_set && (p_qos_level->sl != sl)) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 1F1F: " - "QoS constaraints: required PathRecord SL (%u) " + "QoS constraints: required PathRecord SL (%u) " "doesn't match QoS policy SL (%u)\n", sl, p_qos_level->sl); status = IB_NOT_FOUND; From hnrose at comcast.net Fri Jul 3 06:18:05 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Fri, 3 Jul 2009 09:18:05 -0400 Subject: [ofa-general] [PATCH] opensm/osm_ucast_ftree.c: Made error numbers unique in some log messages Message-ID: <20090703131805.GA29743@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_ucast_ftree.c b/opensm/opensm/osm_ucast_ftree.c index 6c695de..bd61002 100644 --- a/opensm/opensm/osm_ucast_ftree.c +++ b/opensm/opensm/osm_ucast_ftree.c @@ -3617,7 +3617,7 @@ static int fabric_read_guid_files(IN ftree_fabric_t * p_ftree) if (!cl_qmap_count(&p_ftree->cn_guid_tbl)) { OSM_LOG(&p_ftree->p_osm->log, OSM_LOG_ERROR, - "ERR AB23: " + "ERR AB27: " "Compute node guids file has no valid guids\n"); status = -1; goto Exit; @@ -3633,14 +3633,14 @@ static int fabric_read_guid_files(IN ftree_fabric_t * p_ftree) add_guid_item_to_map, &p_ftree->io_guid_tbl)) { OSM_LOG(&p_ftree->p_osm->log, OSM_LOG_ERROR, - "ERR AB23: Problem parsing I/O guid file\n"); + "ERR AB28: Problem parsing I/O guid file\n"); status = -1; goto Exit; } if (!cl_qmap_count(&p_ftree->io_guid_tbl)) { OSM_LOG(&p_ftree->p_osm->log, OSM_LOG_ERROR, - "ERR AB23: " + "ERR AB29: " "I/O node guids file has no valid guids\n"); status = -1; goto Exit; From hnrose at comcast.net Thu Jul 2 12:48:40 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Thu, 2 Jul 2009 15:48:40 -0400 Subject: [ofa-general] [PATCH] opensm/osm_ucast_mgr.c: Add error numbers to some error log messages Message-ID: <20090702194840.GA27859@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c index d616c62..78a7031 100644 --- a/opensm/opensm/osm_ucast_mgr.c +++ b/opensm/opensm/osm_ucast_mgr.c @@ -168,7 +168,7 @@ static void ucast_mgr_process_neighbor(IN osm_ucast_mgr_t * p_mgr, osm_switch_get_hop_count(p_this_sw, lid_ho, port_num)) { if (osm_switch_set_hops (p_this_sw, lid_ho, port_num, hops) != 0) - OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR, + OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR, "ERR 3A03: " "cannot set hops for lid %u at switch 0x%" PRIx64 "\n", lid_ho, cl_ntoh64(osm_node_get_node_guid @@ -645,8 +645,8 @@ int osm_ucast_mgr_build_lid_matrices(IN osm_ucast_mgr_t * p_mgr) p_mgr->p_subn->opt.hop_weights_file); if (parse_node_map(p_mgr->p_subn->opt.hop_weights_file, set_hop_wf, p_mgr)) { - OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR, "ERR : cannot " - "parse hop_weights_file \'%s\'\n", + OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR, "ERR 3A05: " + "cannot parse hop_weights_file \'%s\'\n", p_mgr->p_subn->opt.hop_weights_file); } } From hnrose at comcast.net Fri Jul 3 06:34:41 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Fri, 3 Jul 2009 09:34:41 -0400 Subject: [ofa-general] [PATCHv2] opensm/osm_mesh.c: Improve VL utilization Message-ID: <20090703133440.GA31663@comcast.net> opensm/osm_mesh.c: Improve VL utilization Add steps to the algorithm to further condition the netlist of switches that is handed to lash to analyze. (1) The previous version sorted out all the links out of switches into dimension order’in the order the dimensions were found which is driven by the arbitrary choice of links out of the seed’switch. This takes the additional step of waiting until after the lengths along each dimension are measured then reordering the links out of each switch so that dimensions with the largest lengths are ordered first. i.e. if the mesh analysis discovers that the mesh is a 5x8x6 torus then the coordinates are reordered so that based on the order of links on each switch the mesh is an 8x6x5 torus. (2) The previous version left the order of switches in the switch list the same as when they were discovered by the SMPs which is some sort of depth first tree walk modified by the random return of MADs if multiples are outstanding. This takes the additional step of reordering the switches so that they are presented in odometer’order again with the dimension with the largest lengths changing fastest. With this algorithm, we always see what we believe is the optimal result for each size of toroidal mesh. As a footnote, when we later studied the effect of broken or imperfect meshes, we found that the results are dependent on the choice of origin for the mesh which clearly doesn't matter for a perfect torus with nodes sorted as described here. Thus there is some additional work required to try to understand how to select the optimal origin for meshes with defects. We saw that the number of VLs required was higher when the origin was near a defect so choosing one as far as possible from the defects seems to be the right idea. This still needs to be done since it will remove some fairly rare defect cases which now require 9 VLs to route. Signed-off-by: Robert Pearson Signed-off-by: Hal Rosenstock --- Changes from v1: Move dim_order array into mesh struct Remove unneeded dim_reverse array Pointed out by Sasha diff --git a/opensm/opensm/osm_mesh.c b/opensm/opensm/osm_mesh.c index 1867876..23fad87 100644 --- a/opensm/opensm/osm_mesh.c +++ b/opensm/opensm/osm_mesh.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008,2009 System Fabric Works, Inc. + * Copyright (c) 2008,2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -182,6 +182,7 @@ typedef struct _mesh { int *class_count; /* population of each class */ int dimension; /* mesh dimension */ int *size; /* an array to hold size of mesh */ + int dim_order[MAX_DIMENSION]; } mesh_t; /* @@ -1027,11 +1028,11 @@ static inline int ltmag(int a, int b) } /* - * reorder_links + * reorder_node_links * * reorder the links out of a switch in sign/dimension order */ -static int reorder_links(lash_t *p_lash, int sw) +static int reorder_node_links(lash_t *p_lash, mesh_t *mesh, int sw) { osm_log_t *p_log = &p_lash->p_osm->log; switch_t *s = p_lash->switches[sw]; @@ -1039,9 +1040,10 @@ static int reorder_links(lash_t *p_lash, int sw) int n = node->num_links; link_t **links; int *axes; - int i, j; + int i, j, k, l; int c; int next = 0; + int dimension = mesh->dimension; if (!(links = calloc(n, sizeof(link_t *)))) { OSM_LOG(p_log, OSM_LOG_ERROR, "Failed allocating temp array - out of memory\n"); @@ -1057,19 +1059,23 @@ static int reorder_links(lash_t *p_lash, int sw) /* * find the links with axes */ - for (j = 1; j <= 2*node->dimension; j++) { - c = j; - if (node->coord[(c-1)/2] > 0) - c = opposite(s, c); + for (i = 0; i < dimension; i++) { + j = mesh->dim_order[i]; + for (k = 1; k <= 2; k++) { + c = 2*j + k; - for (i = 0; i < n; i++) { - if (!node->links[i]) - continue; - if (node->axes[i] == c) { - links[next] = node->links[i]; - axes[next] = node->axes[i]; - node->links[i] = NULL; - next++; + if (node->coord[j] > 0) + c = opposite(s, c); + + for (l = 0; l < n; l++) { + if (!node->links[l]) + continue; + if (node->axes[l] == c) { + links[next] = node->links[l]; + axes[next] = node->axes[l]; + node->links[l] = NULL; + next++; + } } } } @@ -1099,9 +1105,9 @@ static int reorder_links(lash_t *p_lash, int sw) } /* - * measure geometry + * make_coord */ -static int measure_geometry(lash_t *p_lash, mesh_t *mesh, int seed) +static int make_coord(lash_t *p_lash, mesh_t *mesh, int seed) { osm_log_t *p_log = &p_lash->p_osm->log; int i, j, k; @@ -1111,8 +1117,6 @@ static int measure_geometry(lash_t *p_lash, mesh_t *mesh, int seed) int dimension = mesh->dimension; int num_switches = p_lash->num_switches; int assigned_axes = 0, unassigned_axes = 0; - int max[MAX_DIMENSION]; - int min[MAX_DIMENSION]; OSM_LOG_ENTER(p_log); @@ -1120,6 +1124,13 @@ static int measure_geometry(lash_t *p_lash, mesh_t *mesh, int seed) s = p_lash->switches[sw]; s->node->coord = calloc(dimension, sizeof(int)); + if (!s->node->coord) { + OSM_LOG(p_log, OSM_LOG_ERROR, + "Failed allocating coord - out of memory\n"); + OSM_LOG_EXIT(p_log); + return -1; + } + for (i = 0; i < dimension; i++) s->node->coord[i] = (sw == seed) ? 0 : LARGE; @@ -1163,14 +1174,36 @@ static int measure_geometry(lash_t *p_lash, mesh_t *mesh, int seed) } } while (change); - for (sw = 0; sw < num_switches; sw++) { - if (reorder_links(p_lash, sw)) { - OSM_LOG_EXIT(p_log); - return -1; - } - } + OSM_LOG_EXIT(p_log); + return 0; +} + +/* + * measure geometry + */ +static int measure_geometry(lash_t *p_lash, mesh_t *mesh) +{ + osm_log_t *p_log = &p_lash->p_osm->log; + int i, j; + int sw; + switch_t *s; + int dimension = mesh->dimension; + int num_switches = p_lash->num_switches; + int max[MAX_DIMENSION]; + int min[MAX_DIMENSION]; + int size[MAX_DIMENSION]; + int max_size; + int max_index; + + OSM_LOG_ENTER(p_log); mesh->size = calloc(dimension, sizeof(int)); + if (!mesh->size) { + OSM_LOG(p_log, OSM_LOG_ERROR, + "Failed allocating size - out of memory\n"); + OSM_LOG_EXIT(p_log); + return -1; + } for (i = 0; i < dimension; i++) { max[i] = -LARGE; @@ -1191,7 +1224,48 @@ static int measure_geometry(lash_t *p_lash, mesh_t *mesh, int seed) } for (i = 0; i < dimension; i++) - mesh->size[i] = max[i] - min[i] + 1; + mesh->size[i] = size[i] = max[i] - min[i] + 1; + + /* + * find an order of dimensions that places largest + * sizes first since this seems to work best with LASH + */ + for (j = 0; j < dimension; j++) { + max_size = -1; + max_index = -1; + + for (i = 0; i < dimension; i++) { + if (size[i] > max_size) { + max_size = size[i]; + max_index = i; + } + } + + mesh->dim_order[j] = max_index; + size[max_index] = -1; + } + + OSM_LOG_EXIT(p_log); + return 0; +} + +/* + * reorder links + */ +static int reorder_links(lash_t *p_lash, mesh_t *mesh) +{ + osm_log_t *p_log = &p_lash->p_osm->log; + int sw; + int num_switches = p_lash->num_switches; + + OSM_LOG_ENTER(p_log); + + for (sw = 0; sw < num_switches; sw++) { + if (reorder_node_links(p_lash, mesh, sw)) { + OSM_LOG_EXIT(p_log); + return -1; + } + } OSM_LOG_EXIT(p_log); return 0; @@ -1387,7 +1461,13 @@ int osm_do_mesh_analysis(lash_t *p_lash) if (s->node->type) { make_geometry(p_lash, max_class_type); - if (measure_geometry(p_lash, mesh, max_class_type)) + if (make_coord(p_lash, mesh, max_class_type)) + goto err; + + if (measure_geometry(p_lash, mesh)) + goto err; + + if (reorder_links(p_lash, mesh)) goto err; p = buf; From nicolas at morey-chaisemartin.com Fri Jul 3 01:03:31 2009 From: nicolas at morey-chaisemartin.com (Nicolas Morey-Chaisemartin) Date: Fri, 03 Jul 2009 10:03:31 +0200 Subject: [ofa-general] [PATCH] opensm: Added support for select counters (xmit_wait) Message-ID: <4A4DBB53.7080702@morey-chaisemartin.com> Support for xmit_wait counters was missing in the perfmgr though it was read from the mad and event plugin interface already handles it. This patch adds support for it (tested and working with an event plugin) Signed-off-by: Nicolas Morey-Chaisemartin --- opensm/include/opensm/osm_perfmgr_db.h | 23 ++++++- opensm/opensm/osm_perfmgr.c | 30 +++++++- opensm/opensm/osm_perfmgr_db.c | 124 +++++++++++++++++++++++++++++-- 3 files changed, 166 insertions(+), 11 deletions(-) diff --git a/opensm/include/opensm/osm_perfmgr_db.h b/opensm/include/opensm/osm_perfmgr_db.h index 42a47bd..35b5ac3 100644 --- a/opensm/include/opensm/osm_perfmgr_db.h +++ b/opensm/include/opensm/osm_perfmgr_db.h @@ -109,6 +109,14 @@ typedef struct { } perfmgr_db_data_cnt_reading_t; /** ========================================================================= + * Port select count reading + */ +typedef struct { + uint64_t xmit_wait; + time_t time; +} perfmgr_db_sel_reading_t; + +/** ========================================================================= * Dump output options */ typedef enum { @@ -125,6 +133,8 @@ typedef struct db_port { perfmgr_db_err_reading_t err_previous; perfmgr_db_data_cnt_reading_t dc_total; perfmgr_db_data_cnt_reading_t dc_previous; + perfmgr_db_sel_reading_t ps_total; + perfmgr_db_sel_reading_t ps_previous; time_t last_reset; } db_port_t; @@ -179,7 +189,16 @@ perfmgr_db_err_t perfmgr_db_get_prev_dc(perfmgr_db_t * db, uint64_t guid, reading); perfmgr_db_err_t perfmgr_db_clear_prev_dc(perfmgr_db_t * db, uint64_t guid, uint8_t port); - +perfmgr_db_err_t perfmgr_db_add_ps_reading(perfmgr_db_t * db, uint64_t guid, + uint8_t port, + perfmgr_db_sel_reading_t * + reading); +perfmgr_db_err_t perfmgr_db_get_prev_ps(perfmgr_db_t * db, uint64_t guid, + uint8_t port, + perfmgr_db_sel_reading_t * + reading); +perfmgr_db_err_t perfmgr_db_clear_prev_ps(perfmgr_db_t * db, uint64_t guid, + uint8_t port); void perfmgr_db_clear_counters(perfmgr_db_t * db); perfmgr_db_err_t perfmgr_db_dump(perfmgr_db_t * db, char *file, perfmgr_db_dump_t dump_type); @@ -196,6 +215,8 @@ void perfmgr_db_fill_data_cnt_read_pc(ib_port_counters_t * wire_read, perfmgr_db_data_cnt_reading_t * reading); void perfmgr_db_fill_data_cnt_read_epc(ib_port_counters_ext_t * wire_read, perfmgr_db_data_cnt_reading_t * reading); +void perfmgr_db_fill_sel_read(ib_port_counters_t * wire_read, + perfmgr_db_sel_reading_t * reading); END_C_DECLS diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c index ecfdbda..8a9eb12 100644 --- a/opensm/opensm/osm_perfmgr.c +++ b/opensm/opensm/osm_perfmgr.c @@ -853,10 +853,12 @@ void osm_perfmgr_destroy(osm_perfmgr_t * pm) static void perfmgr_check_oob_clear(osm_perfmgr_t * pm, monitored_node_t * mon_node, uint8_t port, perfmgr_db_err_reading_t * cr, - perfmgr_db_data_cnt_reading_t * dc) + perfmgr_db_data_cnt_reading_t * dc, + perfmgr_db_sel_reading_t * ps) { perfmgr_db_err_reading_t prev_err; perfmgr_db_data_cnt_reading_t prev_dc; + perfmgr_db_sel_reading_t prev_ps; if (perfmgr_db_get_prev_err(pm->db, mon_node->guid, port, &prev_err) != PERFMGR_EVENT_DB_SUCCESS) { @@ -905,6 +907,23 @@ static void perfmgr_check_oob_clear(osm_perfmgr_t * pm, mon_node->name, mon_node->guid, port); perfmgr_db_clear_prev_dc(pm->db, mon_node->guid, port); } + + if (perfmgr_db_get_prev_ps(pm->db, mon_node->guid, port, &prev_ps) + != PERFMGR_EVENT_DB_SUCCESS) { + OSM_LOG(pm->log, OSM_LOG_VERBOSE, + "Failed to find previous select count " + "reading for %s (0x%" PRIx64 ") port %u\n", + mon_node->name, mon_node->guid, port); + return; + } + + if (ps->xmit_wait < prev_ps.xmit_wait) { + OSM_LOG(pm->log, OSM_LOG_ERROR, + "PerfMgr: ERR 4C17: Detected an out of band select counter " + "clear on node %s (0x%" PRIx64 ") port %u\n", + mon_node->name, mon_node->guid, port); + perfmgr_db_clear_prev_ps(pm->db, mon_node->guid, port); + } } /********************************************************************** @@ -1062,6 +1081,8 @@ static void pc_recv_process(void *context, void *data) uint8_t port = mad_context->perfmgr_context.port; perfmgr_db_err_reading_t err_reading; perfmgr_db_data_cnt_reading_t data_reading; + perfmgr_db_sel_reading_t select_reading; + cl_map_item_t *p_node; monitored_node_t *p_mon_node; @@ -1148,10 +1169,12 @@ static void pc_recv_process(void *context, void *data) */ perfmgr_db_fill_data_cnt_read_pc(wire_read, &data_reading); + perfmgr_db_fill_sel_read(wire_read, &select_reading); + /* detect an out of band clear on the port */ if (mad_context->perfmgr_context.mad_method != IB_MAD_METHOD_SET) perfmgr_check_oob_clear(pm, p_mon_node, port, &err_reading, - &data_reading); + &data_reading, &select_reading); /* log any critical events from this reading */ perfmgr_log_events(pm, p_mon_node, port, &err_reading); @@ -1161,9 +1184,12 @@ static void pc_recv_process(void *context, void *data) &err_reading); perfmgr_db_add_dc_reading(pm->db, node_guid, port, &data_reading); + perfmgr_db_add_ps_reading(pm->db, node_guid, port, + &select_reading); } else { perfmgr_db_clear_prev_err(pm->db, node_guid, port); perfmgr_db_clear_prev_dc(pm->db, node_guid, port); + perfmgr_db_clear_prev_ps(pm->db, node_guid, port); } perfmgr_check_overflow(pm, p_mon_node, port, wire_read); diff --git a/opensm/opensm/osm_perfmgr_db.c b/opensm/opensm/osm_perfmgr_db.c index e5dfc19..132c2fb 100644 --- a/opensm/opensm/osm_perfmgr_db.c +++ b/opensm/opensm/osm_perfmgr_db.c @@ -486,6 +486,102 @@ Exit: return (rc); } +static inline void +debug_dump_ps_reading(perfmgr_db_t * db, uint64_t guid, uint8_t port_num, + db_port_t * port, perfmgr_db_sel_reading_t * cur) +{ + osm_log_t *log = db->perfmgr->log; + if (!osm_log_is_active(log, OSM_LOG_DEBUG)) + return; + + osm_log(log, OSM_LOG_DEBUG, + "xd %" PRIu64 " <-- %" PRIu64 " (%" PRIu64 ")\n", + cur->xmit_wait, port->ps_previous.xmit_wait, + port->ps_total.xmit_wait); +} + +/********************************************************************** + * perfmgr_db_sel_reading_t functions + **********************************************************************/ +perfmgr_db_err_t +perfmgr_db_add_ps_reading(perfmgr_db_t * db, uint64_t guid, uint8_t port, + perfmgr_db_sel_reading_t * reading) +{ + db_port_t *p_port = NULL; + db_node_t *node = NULL; + perfmgr_db_sel_reading_t *previous = NULL; + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; + osm_epi_ps_event_t epi_ps_data; + + cl_plock_excl_acquire(&db->lock); + node = get(db, guid); + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) + goto Exit; + + p_port = &node->ports[port]; + previous = &node->ports[port].ps_previous; + + debug_dump_ps_reading(db, guid, port, p_port, reading); + + epi_ps_data.time_diff_s = reading->time - previous->time; + osm_epi_create_port_id(&epi_ps_data.port_id, guid, port, + node->node_name); + + /* calculate changes from previous reading */ + epi_ps_data.xmit_wait = reading->xmit_wait - previous->xmit_wait; + p_port->ps_total.xmit_wait += epi_ps_data.xmit_wait; + + p_port->ps_previous = *reading; + osm_opensm_report_event(db->perfmgr->osm, + OSM_EVENT_ID_PORT_SELECT, &epi_ps_data); + +Exit: + cl_plock_release(&db->lock); + return (rc); +} + +perfmgr_db_err_t perfmgr_db_get_prev_ps(perfmgr_db_t * db, uint64_t guid, + uint8_t port, + perfmgr_db_sel_reading_t * reading) +{ + db_node_t *node = NULL; + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; + + cl_plock_acquire(&db->lock); + + node = get(db, guid); + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) + goto Exit; + + *reading = node->ports[port].ps_previous; + +Exit: + cl_plock_release(&db->lock); + return (rc); +} + +perfmgr_db_err_t +perfmgr_db_clear_prev_ps(perfmgr_db_t * db, uint64_t guid, uint8_t port) +{ + db_node_t *node = NULL; + perfmgr_db_sel_reading_t *previous = NULL; + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; + + cl_plock_excl_acquire(&db->lock); + node = get(db, guid); + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) + goto Exit; + + previous = &node->ports[port].ps_previous; + + memset(previous, 0, sizeof(*previous)); + node->ports[port].ps_previous.time = time(NULL); + +Exit: + cl_plock_release(&db->lock); + return (rc); +} + static void clear_counters(cl_map_item_t * const p_map_item, void *context) { db_node_t *node = (db_node_t *) p_map_item; @@ -517,6 +613,8 @@ static void clear_counters(cl_map_item_t * const p_map_item, void *context) node->ports[i].dc_total.multicast_rcv_pkts = 0; node->ports[i].dc_total.time = ts; + node->ports[i].ps_total.xmit_wait = 0; + node->ports[i].last_reset = ts; } } @@ -546,7 +644,7 @@ static void dump_node_mr(db_node_t * node, FILE * fp) "%s\t%s\t" "%s\t%s\t%s\t%s\t%s\t%s\t%s\t" "%s\t%s\t%s\t%s\t%s\t%s\t%s\t" - "%s\t%s\t%s\t%s\n", + "%s\t%s\t%s\t%s\t%s\n", "symbol_err_cnt", "link_err_recover", "link_downed", @@ -565,8 +663,7 @@ static void dump_node_mr(db_node_t * node, FILE * fp) "rcv_pkts", "unicast_xmit_pkts", "unicast_rcv_pkts", - "multicast_xmit_pkts", - "multicast_rcv_pkts"); + "multicast_xmit_pkts", "multicast_rcv_pkts", "xmit_wait"); for (i = (node->esp0) ? 0 : 1; i < node->num_ports; i++) { char *since = ctime(&node->ports[i].last_reset); since[strlen(since) - 1] = '\0'; /* remove \n */ @@ -577,8 +674,8 @@ static void dump_node_mr(db_node_t * node, FILE * fp) "%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 - "\t%" PRIu64 "\t%" PRIu64 "\n", node->node_name, - node->node_guid, i, since, + "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\n", + node->node_name, node->node_guid, i, since, node->ports[i].err_total.symbol_err_cnt, node->ports[i].err_total.link_err_recover, node->ports[i].err_total.link_downed, @@ -598,7 +695,8 @@ static void dump_node_mr(db_node_t * node, FILE * fp) node->ports[i].dc_total.unicast_xmit_pkts, node->ports[i].dc_total.unicast_rcv_pkts, node->ports[i].dc_total.multicast_xmit_pkts, - node->ports[i].dc_total.multicast_rcv_pkts); + node->ports[i].dc_total.multicast_rcv_pkts, + node->ports[i].ps_total.xmit_wait); } } @@ -634,7 +732,8 @@ static void dump_node_hr(db_node_t * node, FILE * fp) " unicast_xmit_pkts : %" PRIu64 "\n" " unicast_rcv_pkts : %" PRIu64 "\n" " multicast_xmit_pkts : %" PRIu64 "\n" - " multicast_rcv_pkts : %" PRIu64 "\n", + " multicast_rcv_pkts : %" PRIu64 "\n" + " xmit_wait : %" PRIu64 "\n", node->node_name, node->node_guid, i, @@ -658,7 +757,8 @@ static void dump_node_hr(db_node_t * node, FILE * fp) node->ports[i].dc_total.unicast_xmit_pkts, node->ports[i].dc_total.unicast_rcv_pkts, node->ports[i].dc_total.multicast_xmit_pkts, - node->ports[i].dc_total.multicast_rcv_pkts); + node->ports[i].dc_total.multicast_rcv_pkts, + node->ports[i].ps_total.xmit_wait); } } @@ -809,4 +909,12 @@ perfmgr_db_fill_data_cnt_read_epc(ib_port_counters_ext_t * wire_read, reading->multicast_rcv_pkts = cl_ntoh64(wire_read->multicast_rcv_pkts); reading->time = time(NULL); } + +void +perfmgr_db_fill_sel_read(ib_port_counters_t * wire_read, + perfmgr_db_sel_reading_t * reading) +{ + reading->xmit_wait = cl_ntoh32(wire_read->xmit_wait); + reading->time = time(NULL); +} #endif /* ENABLE_OSM_PERF_MGR */ -- 1.6.2-rc2.GIT From yosefe at voltaire.com Fri Jul 3 07:43:25 2009 From: yosefe at voltaire.com (Yossi Etigin) Date: Fri, 03 Jul 2009 17:43:25 +0300 Subject: [ofa-general] A question about tx lock in ipoib_flush_paths Message-ID: <4A4E190D.7010004@voltaire.com> In ipoib_flush_paths(), we take the netif_tx_lock to remove a path My question is - what data does this lock protect? It isn't path->list and path->rb_node, because priv->lock is enough to protect them. It might be neigh and neigh->ah, to avoid freeing the neighbour and its address handle while ipoib_start_xmit() is using it, but this particular part is done *outside* the tx lock. Unless I'm missing something - shouldn't the code: spin_unlock_irqrestore(&priv->lock, flags); netif_tx_unlock_bh(dev); wait_for_completion(&path->done); release >> path_free(dev, path); lock >> netif_tx_lock_bh(dev); spin_lock_irqsave(&priv->lock, flags); Be like this: spin_unlock_irqrestore(&priv->lock, flags); netif_tx_unlock_bh(dev); wait_for_completion(&path->done); lock >> netif_tx_lock_bh(dev); release >> path_free(dev, path); spin_lock_irqsave(&priv->lock, flags); -Yossi From lars.ellenberg at linbit.com Fri Jul 3 07:48:24 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Fri, 3 Jul 2009 16:48:24 +0200 Subject: [ofa-general] upstream stable merge request for OFED 1.4 Message-ID: <20090703144824.GA24820@barkeeper1-xen.linbit> maybe I'm doing things wrong. but the last merge with upstream for git://git.openfabrics.org/ofed_1_4/linux-2.6.git appears to have been 2.6.27. there is a reason for the kernel.org stable branches. if you do not plan to upgrade 2.6.28 -> 2.6.31 in the near future, please consider to merge with git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-2.6.27.y.git current upstream 2.6.27 stable is v2.6.27.26 958 files changed, 13481 insertions(+), 6784 deletions(-) Thanks, -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. From felix at chelsio.com Fri Jul 3 09:47:12 2009 From: felix at chelsio.com (Felix Marti) Date: Fri, 3 Jul 2009 09:47:12 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com><19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> Message-ID: <8A71B368A89016469F72CD08050AD33405BE9465@maui.asicdesigners.com> Yes, Chelsio supports peer2peer mode but I'm not sure if it is enabled by default; I'm sure Steve will respond to this but he's out for a few days. Maybe modinfo on the iw_cxgb3 module will reveal a peer2peer related parameters? -----Original Message----- From: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Sean Hefty Sent: Thursday, July 02, 2009 11:44 PM To: Davis, Arlin R; 'pandit ib'; general at lists.openfabrics.org Subject: RE: [ofa-general] cmatose fails whereas rping passes on iWarp >If this test sends data from server side first you could >be running into the iWARP requirement of sending from >client first. This was my thought as well. I think Chelsio supports sending from the server side first, but I'm not sure, or if it's enabled by default. - Sean _______________________________________________ general mailing list general at lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From bart.vanassche at gmail.com Fri Jul 3 10:34:32 2009 From: bart.vanassche at gmail.com (Bart Van Assche) Date: Fri, 3 Jul 2009 19:34:32 +0200 Subject: [ofa-general] upstream stable merge request for OFED 1.4 In-Reply-To: <20090703144824.GA24820@barkeeper1-xen.linbit> References: <20090703144824.GA24820@barkeeper1-xen.linbit> Message-ID: On Fri, Jul 3, 2009 at 4:48 PM, Lars Ellenberg wrote: > maybe I'm doing things wrong. > but the last merge with upstream for > git://git.openfabrics.org/ofed_1_4/linux-2.6.git > appears to have been 2.6.27. > > there is a reason for the kernel.org stable branches. > > if you do not plan to upgrade 2.6.28 -> 2.6.31 in the near future, > please consider to merge with > git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-2.6.27.y.git > > current upstream 2.6.27 stable is v2.6.27.26 >  958 files changed, 13481 insertions(+), 6784 deletions(-) Hello Lars, As you probably know any patch submitted for inclusion in a kernel.org stable branch must have been accepted first for inclusion in the latest vanilla kernel. So the above question is equivalent to whether the development of the OFED kernel components should continue as a separate distribution or whether all patches should be submitted for inclusion in the latest Linux kernel. This topic has been discussed several times on this mailing list. You can find the most recent discussion of this topic here: http://lists.openfabrics.org/pipermail/general/2009-April/059080.html. Bart. From arlin.r.davis at intel.com Fri Jul 3 12:03:22 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Fri, 3 Jul 2009 12:03:22 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> Message-ID: >-----Original Message----- >From: Hefty, Sean >Sent: Thursday, July 02, 2009 11:44 PM >To: Davis, Arlin R; 'pandit ib'; general at lists.openfabrics.org >Subject: RE: [ofa-general] cmatose fails whereas rping passes on iWarp > >>If this test sends data from server side first you could >>be running into the iWARP requirement of sending from >>client first. > >This was my thought as well. I think Chelsio supports sending >from the server >side first, but I'm not sure, or if it's enabled by default. I don't believe it is enabled by default. To enable: echo 1 > /sys/module/iw_cxgb3/parameters/peer2peer or add following /etc/modprobe.conf driver option: options iw_cxgb3 peer2peer=1 From vlad at lists.openfabrics.org Sat Jul 4 02:21:37 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Sat, 4 Jul 2009 02:21:37 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090704-0200 daily build status Message-ID: <20090704092137.6B92910202B9@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-53.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-53.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-53.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-8.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-8.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.20_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.20_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.27_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1383: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1402: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1383: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1402: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_ia64_check/drivers/infiniband/core/addr.c:36: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_ia64_check/kernel_addons/backport/2.6.22/include/asm/bitops.h: In function 'clear_bit_unlock': /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_ia64_check/kernel_addons/backport/2.6.22/include/asm/bitops.h:8: error: implicit declaration of function 'smp_mb' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_ia64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_ia64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.23_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.23_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18-8.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.18-8.el5_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18-8.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ppc64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ppc64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090704-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From jackm at dev.mellanox.co.il Sat Jul 4 23:52:28 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Sun, 5 Jul 2009 09:52:28 +0300 Subject: [ofa-general] Re: Write combining on PPC64 In-Reply-To: <200906281221.52028.jackm@dev.mellanox.co.il> References: <200906281221.52028.jackm@dev.mellanox.co.il> Message-ID: <200907050952.28880.jackm@dev.mellanox.co.il> Resending, adding Hoang-Nam Nguyen and Christoph Raisch of IBM. Please see the questions below. Also, who is the person at IBM who does Linux kernel devleopment for the PPC? Thanks! -Jack On Sunday 28 June 2009 12:21, Jack Morgenstein wrote: > Hi Shirley, > > I was reviewing write-combining for the PPC on kernel 2.6.30, and noticed the following > in file arch/powerpc/include/asm/pgtable.h: > > #define _PAGE_CACHE_CTL (_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \ > _PAGE_WRITETHRU) > ... > #define pgprot_noncached_wc(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ > _PAGE_NO_CACHE)) > > It seems to me that this gives the same result as the fix I put in OFED 1.5 for the PPC: > (kernel_patches/fixes/mlx4_0010_add_wc.patch): > > +pgprot_t pgprot_wc(pgprot_t _prot) > +{ > + return __pgprot((pgprot_val(_prot) | _PAGE_NO_CACHE) & > + ~(pgprot_t)_PAGE_GUARDED); > +} > > ( That is, PAGE_NO_CACHE set, and _PAGE_GUARDED cleared). > > Differences are that _PAGE_COHERENT and _PAGE_WRITETHRU are also cleared in pgprot_noncached_wc() > and the patch I put in is only for PPC64, not all PPCs. > > Questions: > 1. Can I use the pgprot_noncached_wc() macro to implement write-combining on all PPCs? (it is defined > thus in arch/powerpc/include/asm/pgtable.h for all PPC platforms) > > 2. Is there any reason that there is no define: > > #define pgprot_writecombine pgprot_noncached_wc > > in file arch/powerpc/include/asm/pgtable.h (so that the general > pgprot_writecombine() call can be used for PPCs as well). > > -Jack > From tziporet at dev.mellanox.co.il Sun Jul 5 01:10:13 2009 From: tziporet at dev.mellanox.co.il (Tziporet Koren) Date: Sun, 05 Jul 2009 11:10:13 +0300 Subject: [ofa-general][PATCH 2/2] mlx4: ConnectX multi functional device support In-Reply-To: References: <4A3A34F5.9030800@mellanox.co.il> <4A4C816A.2040506@mellanox.co.il> <4A4CD0D7.6070409@mellanox.co.il> Message-ID: <4A505FE5.4040108@mellanox.co.il> Roland Dreier wrote: > > Well, I need to think over the patch a little more. The risk is > certainly not zero, since we are changing device initialization for all > mlx4 devices. And I'm not sure if missing 2.6.31 support is that big a > deal for you, is it? How many users do you have building their own > upstream kernels? You're just going to tell everyone to use OFED > anyway, right? > This is NOT related to OFED at all. This is our 10G NIC driver. See the description: /* MT26468 ConnectX EN 10GigE, PCIe, 2.0 5Gt/s */ We have several customer that take our 10G driver from kernel.org, and once the HW is out we will have a problem. Also to include this code in next Redhat & Novell updates we need to have it in the kernel too. I know the risk is not zero, but you must admit its not high either, and we already tested it here thoroughly here with all device IDs Since we sent the patches on time for inclusion not clear why they are being declined now. Please see what can be done. Thanks, Tziporet From vlad at lists.openfabrics.org Sun Jul 5 02:22:09 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Sun, 5 Jul 2009 02:22:09 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090705-0200 daily build status Message-ID: <20090705092209.81A73102022A@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-8.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-8.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-53.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-53.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-53.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-53.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.20_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.20_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_x86_64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.27_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1383: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1402: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1383: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1402: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_ia64_check/drivers/infiniband/core/addr.c:36: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_ia64_check/kernel_addons/backport/2.6.22/include/asm/bitops.h: In function 'clear_bit_unlock': /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_ia64_check/kernel_addons/backport/2.6.22/include/asm/bitops.h:8: error: implicit declaration of function 'smp_mb' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_ia64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_ia64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.23_ia64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.23_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2624: error: implicit declaration of function 'pid_nr' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c: In function 'ipath_signal_procs': /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.c:2682: error: implicit declaration of function 'kill_pid' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath/ipath_driver.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/ipath] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18-8.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.18-8.el5_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18-8.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: from /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:59: /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ppc64_check/drivers/net/cxgb3/adapter.h:194: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ppc64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090705-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From monis at Voltaire.COM Sun Jul 5 08:05:35 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Sun, 05 Jul 2009 18:05:35 +0300 Subject: [ofa-general] [PATCH v2 RESEND] rdma_cm: Add debugfs entries to monitor rdma_cm connections In-Reply-To: <4A3E45D3.3040405@Voltaire.COM> References: <4A3E45D3.3040405@Voltaire.COM> Message-ID: <4A50C13F.6070506@Voltaire.COM> Sean. Roland It seems like a dead and and there won't be a consensus on how to implement this. Without disrespecting Jason's opinion I still would like to see this patch get in. What's the convention in such cases? I think that this feature adds a lot to the debugging capabilities of the InfiniBand/iWARP stack and users would like to see it in. thanks MoniS From vlad at lists.openfabrics.org Mon Jul 6 02:28:00 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Mon, 6 Jul 2009 02:28:00 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090706-0200 daily build status Message-ID: <20090706092800.44533102020C@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_probe': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/nes/nes.c:591: error: implicit declaration of function 'pcie_get_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/nes/nes_hw.h:1172: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_probe': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/nes/nes.c:591: error: implicit declaration of function 'pcie_get_readrq' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/nes/nes_hw.h:1172: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_probe': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/nes/nes.c:591: error: implicit declaration of function 'pcie_get_readrq' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_probe': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/nes/nes.c:591: error: implicit declaration of function 'pcie_get_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.27_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1411: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1430: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1411: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1430: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/nes/nes_hw.h:1172: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_probe': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_ia64_check/drivers/infiniband/hw/nes/nes.c:591: error: implicit declaration of function 'pcie_get_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090706-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From devesh28 at gmail.com Mon Jul 6 04:39:55 2009 From: devesh28 at gmail.com (Devesh Sharma) Date: Mon, 6 Jul 2009 17:09:55 +0530 Subject: [ofa-general] Performance evaluation of Opensm Message-ID: <309a667c0907060439u1c90cc85h3bc8aa43d74d9e31@mail.gmail.com> Hello list. Is there any tool or method available to get some performance numbers related to Opensm? What are the major performance parameters of OpenSM which can be used for performance measurement analaysis? Thanks in advance regards Devesh Sharma - From swise at opengridcomputing.com Mon Jul 6 07:11:12 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Mon, 06 Jul 2009 09:11:12 -0500 Subject: [ofa-general] [PATCH v2 RESEND] rdma_cm: Add debugfs entries to monitor rdma_cm connections In-Reply-To: <4A50C13F.6070506@Voltaire.COM> References: <4A3E45D3.3040405@Voltaire.COM> <4A50C13F.6070506@Voltaire.COM> Message-ID: <4A520600.2080004@opengridcomputing.com> I agree that this is useful for debugging. Moni Shoua wrote: > Sean. Roland > > It seems like a dead and and there won't be a consensus on how to implement this. > Without disrespecting Jason's opinion I still would like to see this patch get in. > What's the convention in such cases? > > I think that this feature adds a lot to the debugging capabilities of the InfiniBand/iWARP stack > and users would like to see it in. > > > thanks > MoniS > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From jackm at dev.mellanox.co.il Mon Jul 6 07:53:40 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Mon, 6 Jul 2009 17:53:40 +0300 Subject: [ofa-general] Write combining support in OFED 1.5 Message-ID: <200907061753.40760.jackm@dev.mellanox.co.il> Hi Roland, I've been looking at the write-combining support in the 2.6.30 kernel, and it looks good. There is also a good solution for PPC write combining support in the kernel (adding #define pgprot_writecombine pgprot_noncached_wc to file arch/powerpc/include/asm/pgtable.h, per e-mail correspondence with kernel ppc people in ibm). However, I am still missing some means of determining whether or not write_combining is supported on a given host. From the write-combining support in OFED 1.4: static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(ibdev); struct mlx4_ib_ucontext *context; struct mlx4_ib_alloc_ucontext_resp resp; int err; resp.qp_tab_size = dev->dev->caps.num_qps; ===> if (mlx4_wc_enabled()) { resp.bf_reg_size = dev->dev->caps.bf_reg_size; resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; } else { resp.bf_reg_size = 0; resp.bf_regs_per_page = 0; } The issue is that if a given platform does not support write-combining, we do not wish to enable use of blueflame in userspace for ConnectX. I have not yet come up with any good, consistent, way of determining if write-combining is supported. I thought of possibly comparing the result returned by pgprot_writecombine(prot) to that returned by pgprot_noncached(prot) -- if they are identical, then assume that write-combining is not supported. (pgprot_noncached() is the default mapping of pgprot_writecombine if it is not defined under the arch directory -- see file include/linux/pgtable.h). This has a problem, however, in that I have no way of determining what value of "prot" to use when doing this comparison -- there may be some architectures which use bits of the prot structure to determine per specific call whether or not to use write-combining (i.e., pgprot_writecombine(prot) could invoke pgprot_noncached(prot) if certain bits were set in the prot structure, or return a write-combining prot value if those bits are not set). Using a zeroed-out pgprot structure in the comparison, for example, may not be appropriate. (we may be allowing blueflame when it should not be, or preventing blueflame when it should be allowed). Do you have any ideas for how to determine if in fact write-combining is available? -Jack From cap at nsc.liu.se Mon Jul 6 07:55:11 2009 From: cap at nsc.liu.se (Peter Kjellstrom) Date: Mon, 6 Jul 2009 16:55:11 +0200 Subject: [ofa-general] Cannot bring up mellanox HCAs on HP Nahalem machines running RHEL 5 In-Reply-To: References: Message-ID: <200907061655.15037.cap@nsc.liu.se> On Friday 26 June 2009, Mike Heinz wrote: > We've open an OFED PR ( https://bugs.openfabrics.org/show_bug.cgi?id=1663 ) > on this issue, but I'm wondering if anyone else has seen this: We just got > some brand new HP "gen6" machines - and we cannot get any Mellanox HCA, DDR > or QDR to come up on those machines - if they are running RHEL 5.3. > Machines running SLES11 appear to work fine. > > The same HCAs work fine on slower machines and on machines running SLES11 - > has anyone else seen this? > > We're running OFED 1.4.1 GA. I had similar problems. Upgrading the firmware to 2.6.0 (if I remember the version correctly) fixed this. /Peter -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 189 bytes Desc: This is a digitally signed message part. URL: From cap at nsc.liu.se Mon Jul 6 08:01:12 2009 From: cap at nsc.liu.se (Peter Kjellstrom) Date: Mon, 6 Jul 2009 17:01:12 +0200 Subject: [ofa-general] verb level interoperability between vendor's hcas In-Reply-To: <4A47B05E.3000206@ucla.edu> References: <4A456EB0.5050102@ucla.edu> <4A47B05E.3000206@ucla.edu> Message-ID: <200907061701.13049.cap@nsc.liu.se> On Sunday 28 June 2009, Scott A. Friedman wrote: > We have had several tickets submitted by users since we have started > adding Qlogic 7240 cards into our cluster which is mostly Mellanox (we > have a couple different cards). I did some testing with this and concluded that the 7240 HCAs performed so much better with PSM that verbs was out of the question (or performed very bad with verbs if one wants to look at it that way). Essentially, they were ok as Qlogic-HCAs but not Infiniband-HCAs (IMHO). Using conservative OpenMPI settings (essentially treating all the HCAs like they were very old/simple) I did get a limited set of simple tests through ok. /Peter -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 189 bytes Desc: This is a digitally signed message part. URL: From cap at nsc.liu.se Mon Jul 6 08:03:17 2009 From: cap at nsc.liu.se (Peter Kjellstrom) Date: Mon, 6 Jul 2009 17:03:17 +0200 Subject: [ofa-general] Performance evaluation of Opensm In-Reply-To: <309a667c0907060439u1c90cc85h3bc8aa43d74d9e31@mail.gmail.com> References: <309a667c0907060439u1c90cc85h3bc8aa43d74d9e31@mail.gmail.com> Message-ID: <200907061703.17711.cap@nsc.liu.se> On Monday 06 July 2009, Devesh Sharma wrote: > Hello list. > > Is there any tool or method available to get some performance numbers > related to Opensm? What are the major performance parameters of OpenSM > which can > be used for performance measurement analaysis? Do you refer to the performance of the fabric as setup by OpenSM or the performance of OpenSM itself? /Peter > Thanks in advance > regards > Devesh Sharma -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 189 bytes Desc: This is a digitally signed message part. URL: From monis at Voltaire.COM Mon Jul 6 08:25:56 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Mon, 06 Jul 2009 18:25:56 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins Message-ID: <4A521784.5090304@Voltaire.COM> Whenever an illegal multicast address is passed to IPoIB for it to join it stops all subsequent requests from being joined. That happens because IPoIB joins to multicast addresses in the order they arrived and doesn't handle the next group's join until the current join finishes with success. This phenomena happens a lot when a bonding interface enslaves IPoIB devices. Before enslaving IPoIB interfaces the bonding device acts like an Ethernet device, including the way it translates muticast IP addresses to HW addresses. When it comes up without slaves it translates the group 224.0.0.1 (all hosts) as if it were an Ethernet device and when it enslaves IPoIB devices this is the address that they get for joining (which is a garbage for them) This patch moves the multicast address to the end of the list after a join attempt. Even if the join fails then the next attempt will be for a different address. Signed-off-by: Moni Shoua -- drivers/infiniband/ulp/ipoib/ipoib_multicast.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c index a0e9753..024fd18 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c @@ -379,6 +379,7 @@ static int ipoib_mcast_join_complete(int status, struct ipoib_mcast *mcast = multicast->context; struct net_device *dev = mcast->dev; struct ipoib_dev_priv *priv = netdev_priv(dev); + struct ipoib_mcast *next_mcast; ipoib_dbg_mcast(priv, "join completion for %pI6 (status %d)\n", mcast->mcmember.mgid.raw, status); @@ -427,9 +428,16 @@ static int ipoib_mcast_join_complete(int status, mutex_lock(&mcast_mutex); spin_lock_irq(&priv->lock); - if (test_bit(IPOIB_MCAST_RUN, &priv->flags)) - queue_delayed_work(ipoib_workqueue, &priv->mcast_task, - mcast->backoff * HZ); + if (test_bit(IPOIB_MCAST_RUN, &priv->flags)) { + list_for_each_entry(next_mcast, &priv->multicast_list, list) { + if (!test_bit(IPOIB_MCAST_FLAG_SENDONLY, &next_mcast->flags) + && !test_bit(IPOIB_MCAST_FLAG_BUSY, &next_mcast->flags) + && !test_bit(IPOIB_MCAST_FLAG_ATTACHED, &next_mcast->flags)) + break; + } + queue_delayed_work(ipoib_workqueue, &priv->mcast_join_task, + next_mcast->backoff * HZ); + } spin_unlock_irq(&priv->lock); mutex_unlock(&mcast_mutex); @@ -577,6 +585,9 @@ void ipoib_mcast_join_task(struct work_struct *work) break; } + if (!list_is_last(&mcast->list, &priv->multicast_list)) + list_move_tail(&mcast->list, &priv->multicast_list); + ipoib_mcast_join(dev, mcast, 1); return; } From michael.heinz at qlogic.com Mon Jul 6 08:25:02 2009 From: michael.heinz at qlogic.com (Mike Heinz) Date: Mon, 6 Jul 2009 10:25:02 -0500 Subject: [ofa-general] Cannot bring up mellanox HCAs on HP Nahalem machines running RHEL 5 In-Reply-To: <200907061655.15037.cap@nsc.liu.se> References: <200907061655.15037.cap@nsc.liu.se> Message-ID: <4C2744E8AD2982428C5BFE523DF8CDCB453E27DCE3@MNEXMB1.qlogic.org> Peter, Thanks for that information. I think we're already using 2.6, but I'll pass it on to our test group and verify what firmware revision we're using. -- Michael Heinz Principal Engineer, Qlogic Corporation King of Prussia, Pennsylvania -----Original Message----- From: Peter Kjellstrom [mailto:cap at nsc.liu.se] Sent: Monday, July 06, 2009 10:55 AM To: general at lists.openfabrics.org Cc: Mike Heinz Subject: Re: [ofa-general] Cannot bring up mellanox HCAs on HP Nahalem machines running RHEL 5 On Friday 26 June 2009, Mike Heinz wrote: > We've open an OFED PR ( > https://bugs.openfabrics.org/show_bug.cgi?id=1663 ) on this issue, but > I'm wondering if anyone else has seen this: We just got some brand new > HP "gen6" machines - and we cannot get any Mellanox HCA, DDR or QDR to come up on those machines - if they are running RHEL 5.3. > Machines running SLES11 appear to work fine. > > The same HCAs work fine on slower machines and on machines running > SLES11 - has anyone else seen this? > > We're running OFED 1.4.1 GA. I had similar problems. Upgrading the firmware to 2.6.0 (if I remember the version correctly) fixed this. /Peter From hal.rosenstock at gmail.com Mon Jul 6 08:53:59 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 6 Jul 2009 11:53:59 -0400 Subject: [ofa-general] Re: [PATCH] opensm/lash: Set minimum VL for LASH to use In-Reply-To: <20090623214928.GG4324@me> References: <20090618141430.GA11942@comcast.net> <20090623214928.GG4324@me> Message-ID: Hi Sasha, On Tue, Jun 23, 2009 at 5:49 PM, Sasha Khapyorsky wrote: > Hi Hal, > > On 10:14 Thu 18 Jun     , Hal Rosenstock wrote: >> >> rather than starting from VL 0 >> >> Signed-off-by: Robert Pearson >> Signed-off-by: Hal Rosenstock > > See the comments below. > > > >> diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c >> index 12b5e34..540214c 100644 >> --- a/opensm/opensm/osm_ucast_lash.c >> +++ b/opensm/opensm/osm_ucast_lash.c >> @@ -478,7 +478,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) >>       cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix; >>       int *num_mst_in_lane = p_lash->num_mst_in_lane; >>       int ***virtual_location = p_lash->virtual_location; >> -     int min_filled_lane, max_filled_lane, trials; >> +     int min_filled_lane, max_filled_lane, trials, max_vl; >>       int old_min_filled_lane, old_max_filled_lane, new_num_min_lane, >>           new_num_max_lane; >>       unsigned int i, j; >> @@ -486,9 +486,13 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) >>       int next_switch2, output_link2; >>       int stop = 0, cycle_found; >>       int cycle_found2; >> +     osm_subn_opt_t *opt = &p_lash->p_osm->subn.opt; > > In this function 'opt' is used only to get 'lash_start_vl', so wouldn't > it be better to do something like: > >        unsigned start_vl = &p_lash->p_osm->subn.opt.lash_start_vl; > ? > > This is relevant for all other similar places in the patch. > >> >> -     max_filled_lane = 0; >> -     min_filled_lane = lanes_needed - 1; >> +     max_filled_lane = opt->lash_start_vl; >> +     max_vl = lanes_needed + opt->lash_start_vl; >> +     if (max_vl > IB_MAX_NUM_VLS) >> +             max_vl = IB_MAX_NUM_VLS; > > Isn't this case where LASH requires more VLs than available in IB? > >> +     min_filled_lane = max_vl - 1; >> >>       trials = num_mst_in_lane[max_filled_lane]; >>       if (lanes_needed == 1) >> @@ -590,7 +594,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) >>               new_num_min_lane = MAX_INT; >>               new_num_max_lane = 0; >> >> -             for (i = 0; i < lanes_needed; i++) { >> +             for (i = opt->lash_start_vl; i < max_vl; i++) { >> >>                       if (num_mst_in_lane[i] < new_num_min_lane) { >>                               new_num_min_lane = num_mst_in_lane[i]; >> @@ -673,12 +677,18 @@ static void free_lash_structures(lash_t * p_lash) >>  { >>       unsigned int i, j, k; >>       unsigned num_switches = p_lash->num_switches; >> +     int max_vl; >>       osm_log_t *p_log = &p_lash->p_osm->log; >> +     osm_subn_opt_t *opt = &p_lash->p_osm->subn.opt; >> >>       OSM_LOG_ENTER(p_log); >> >> +     max_vl = opt->lash_start_vl + p_lash->vl_min; >> +     if (max_vl > IB_MAX_NUM_VLS) >> +             max_vl = IB_MAX_NUM_VLS; >> + >>       /* free cdg_vertex_matrix */ >> -     for (i = 0; i < p_lash->vl_min; i++) { >> +     for (i = opt->lash_start_vl; i < max_vl; i++) { >>               for (j = 0; j < num_switches; j++) { >>                       for (k = 0; k < num_switches; k++) >>                               if (p_lash->cdg_vertex_matrix[i][j][k]) >> @@ -715,13 +725,19 @@ static int init_lash_structures(lash_t * p_lash) >>       osm_log_t *p_log = &p_lash->p_osm->log; >>       int status = 0; >>       unsigned int i, j, k; >> +     int max_vl; >> +     osm_subn_opt_t *opt = &p_lash->p_osm->subn.opt; >> >>       OSM_LOG_ENTER(p_log); >> >> +     max_vl = vl_min + opt->lash_start_vl; > > I'm not following... > > vl_min is total number of VLs available in a fabric, lash_start_vl is > VL number where LASH may start to use VLs, so VLs available for LASH are > lash_start_vl..vl_min. Assuming so what does 'max_vl = vl_min + > lash_start_vl' represent? It is always overflow in terms of available > VLs, no? > >> +     if (max_vl > IB_MAX_NUM_VLS) >> +             max_vl = IB_MAX_NUM_VLS; >> + >>       /* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */ >>       p_lash->cdg_vertex_matrix = >> -         (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****)); >> -     for (i = 0; i < vl_min; i++) { >> +         (cdg_vertex_t ****) calloc(max_vl, sizeof(cdg_vertex_t ****)); >> +     for (i = opt->lash_start_vl; i < max_vl; i++) { >>               p_lash->cdg_vertex_matrix[i] = >>                   (cdg_vertex_t ***) malloc(num_switches * >>                                             sizeof(cdg_vertex_t ***)); >> @@ -730,7 +746,7 @@ static int init_lash_structures(lash_t * p_lash) >>                       goto Exit_Mem_Error; >>       } >> >> -     for (i = 0; i < vl_min; i++) { >> +     for (i = opt->lash_start_vl; i < max_vl; i++) { >>               for (j = 0; j < num_switches; j++) { >>                       p_lash->cdg_vertex_matrix[i][j] = >>                           (cdg_vertex_t **) malloc(num_switches * >> @@ -763,7 +779,7 @@ static int init_lash_structures(lash_t * p_lash) >>       for (i = 0; i < num_switches; i++) { >>               for (j = 0; j < num_switches; j++) { >>                       p_lash->virtual_location[i][j] = >> -                         (int *)malloc(vl_min * sizeof(int *)); >> +                         (int *)calloc(max_vl, sizeof(int *)); >>                       if (p_lash->virtual_location[i][j] == NULL) >>                               goto Exit_Mem_Error; >>                       for (k = 0; k < vl_min; k++) { >> @@ -804,10 +820,12 @@ static int lash_core(lash_t * p_lash) >>       int cycle_found2 = 0; >>       int status = 0; >>       int *switch_bitmap = NULL;      /* Bitmap to check if we have processed this pair */ >> +     int max_vl; >> +     osm_subn_opt_t *opt = &p_lash->p_osm->subn.opt; >> >>       OSM_LOG_ENTER(p_log); >> >> -     if (p_lash->p_osm->subn.opt.do_mesh_analysis && osm_do_mesh_analysis(p_lash)) { >> +     if (opt->do_mesh_analysis && osm_do_mesh_analysis(p_lash)) { >>               OSM_LOG(p_log, OSM_LOG_ERROR, "Mesh analysis failed\n"); >>               goto Exit; >>       } >> @@ -838,11 +856,14 @@ static int lash_core(lash_t * p_lash) >>       } >> >>       for (i = 0; i < num_switches; i++) { >> -             for (dest_switch = 0; dest_switch < num_switches; dest_switch++) >> +             for (dest_switch = 0; dest_switch < num_switches; dest_switch++) { >> +                     max_vl = lanes_needed + opt->lash_start_vl; >> +                     if (max_vl > IB_MAX_NUM_VLS) >> +                             max_vl = IB_MAX_NUM_VLS; > > Shouldn't comparison be done against lsh->vl_min (number VLs available > in a fabric)? And return an error in case if it is exceeded (which means > LASH requires more VLs than available)? > >>                       if (dest_switch != i && switch_bitmap[i * num_switches + dest_switch] == 0) { >> -                             v_lane = 0; >> +                             v_lane = opt->lash_start_vl; >>                               stop = 0; >> -                             while (v_lane < lanes_needed && stop == 0) { >> +                             while (v_lane < max_vl && stop == 0) { >>                                       generate_cdg_for_sp(p_lash, i, dest_switch, v_lane); >>                                       generate_cdg_for_sp(p_lash, dest_switch, i, v_lane); >> >> @@ -906,7 +927,9 @@ static int lash_core(lash_t * p_lash) >>                               switches[dest_switch]->routing_table[i].lane = v_lane; >> >>                               if (cycle_found == 1 || cycle_found2 == 1) { >> -                                     if (++lanes_needed > p_lash->vl_min) >> +                                     lanes_needed++; >> +                                     if (lanes_needed > p_lash->vl_min || >> +                                         opt->lash_start_vl + lanes_needed - 1 >= IB_DROP_VL) > > Shouldn't be just > >        if (start_vl + lanes_needed > p_lash->vl_min) > > ? > >>                                               goto Error_Not_Enough_Lanes; >> >>                                       generate_cdg_for_sp(p_lash, i, dest_switch, v_lane); >> @@ -926,19 +949,24 @@ static int lash_core(lash_t * p_lash) >>                               switch_bitmap[i * num_switches + dest_switch] = 1; >>                               switch_bitmap[dest_switch * num_switches + i] = 1; >>                       } >> +             } >>       } >> >>       OSM_LOG(p_log, OSM_LOG_INFO, >>               "Lanes needed: %d, Balancing\n", lanes_needed); >> >> -     for (i = 0; i < lanes_needed; i++) { >> +     max_vl = lanes_needed + opt->lash_start_vl; >> +     if (max_vl > IB_MAX_NUM_VLS) > > Shouldn't this be compared with p_lash->vl_min (which is maximum number > of VLs available in a fabric) and not with IB_MAX_NUM_VLS? > >> +             max_vl = IB_MAX_NUM_VLS; > > When max_vl is larger than number of available VLs (IOW lash needs more > VLs than fabric has) shouldn't it fallback to another routing algorithm? > >> + >> +     for (i = opt->lash_start_vl; i < max_vl; i++) { >>               OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", >>                       i, p_lash->num_mst_in_lane[i]); >>       } >> >>       balance_virtual_lanes(p_lash, lanes_needed); >> >> -     for (i = 0; i < lanes_needed; i++) { >> +     for (i = opt->lash_start_vl; i < max_vl; i++) { >>               OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", >>                       i, p_lash->num_mst_in_lane[i]); >>       } >> @@ -1271,6 +1299,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port, >>       unsigned dst_id; >>       unsigned src_id; >>       osm_switch_t *p_sw; >> +     osm_subn_opt_t *opt = &p_osm->subn.opt; >> >>       if (p_osm->routing_engine_used != OSM_ROUTING_ENGINE_TYPE_LASH) >>               return OSM_DEFAULT_SL; >> @@ -1286,7 +1315,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port, >> >>       src_id = get_lash_id(p_sw); >>       if (src_id == dst_id) >> -             return OSM_DEFAULT_SL; >> +             return opt->lash_start_vl; > > Is this correct? As far as I understand this is SL for paths between > CA ports connected to the same switch (which should be safe in sense of > credit loops). Should lash be involved here? Is there a reason not to use the LASH SL for this ? I think it's more in the spirit of LASH to do this. All other comments addressed in v2 of patch to come shortly. -- Hal > Sasha > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From hnrose at comcast.net Mon Jul 6 08:46:43 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Mon, 6 Jul 2009 11:46:43 -0400 Subject: [ofa-general] [PATCHv2] opensm/lash: Set minimum VL for LASH to use Message-ID: <20090706154643.GA17356@comcast.net> rather than starting from VL 0 Signed-off-by: Robert Pearson Signed-off-by: Hal Rosenstock --- Changes since v1: Fixed comparisons with maximum VL Better lash_start_vl option handling Both as pointed out by Sasha diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 59a32ad..da8cc5e 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -221,6 +222,7 @@ typedef struct osm_subn_opt { char *prefix_routes_file; boolean_t consolidate_ipv6_snm_req; struct osm_subn_opt *file_opts; /* used for update */ + uint8_t lash_start_vl; /* starting vl to use in lash */ } osm_subn_opt_t; /* * FIELDS diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in index 66d2fe6..e8801c9 100644 --- a/opensm/man/opensm.8.in +++ b/opensm/man/opensm.8.in @@ -1,4 +1,4 @@ -.TH OPENSM 8 "April 22, 2009" "OpenIB" "OpenIB Management" +.TH OPENSM 8 "May 28, 2009" "OpenIB" "OpenIB Management" .SH NAME opensm \- InfiniBand subnet manager and administration (SM/SA) @@ -15,6 +15,7 @@ opensm \- InfiniBand subnet manager and administration (SM/SA) [\-r(eassign_lids)] [\-R | \-\-routing_engine ] [\-\-do_mesh_analysis] +[\-\-lash_start_vl ] [\-A | \-\-ucast_cache] [\-z | \-\-connect_roots] [\-M | \-\-lid_matrix_file ] @@ -147,6 +148,10 @@ This option enables additional analysis for the lash routing engine to precondition switch port assignments in regular cartesian meshes which may reduce the number of SLs required to give a deadlock free routing. .TP +\fB\-\-lash_start_vl\fR +This option sets the starting VL to use for the lash routing algorithm. +Defaults to 0. +.TP \fB\-A\fR, \fB\-\-ucast_cache\fR This option enables unicast routing cache and prevents routing recalculation (which is a heavy task in a large cluster) when diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c index 296d5d5..d682ff5 100644 --- a/opensm/opensm/main.c +++ b/opensm/opensm/main.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009 HNR Consulting. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -181,6 +182,9 @@ static void show_usage(void) " routing engine to precondition switch port assignments\n" " in regular cartesian meshes which may reduce the number\n" " of SLs required to give a deadlock free routing\n\n"); + printf("--lash_start_vl \n" + " Sets the starting VL to use for the lash routing algorithm.\n" + " Defaults to 0.\n"); printf("--connect_roots, -z\n" " This option enforces a routing engine (currently\n" " up/down only) to make connectivity between root switches\n" @@ -601,6 +605,7 @@ int main(int argc, char *argv[]) {"prefix_routes_file", 1, NULL, 3}, {"consolidate_ipv6_snm_req", 0, NULL, 4}, {"do_mesh_analysis", 0, NULL, 5}, + {"lash_start_vl", 1, NULL, 6}, {NULL, 0, NULL, 0} /* Required at the end of the array */ }; @@ -951,6 +956,15 @@ int main(int argc, char *argv[]) case 5: opt.do_mesh_analysis = TRUE; break; + case 6: + temp = strtol(optarg, NULL, 0); + if (temp < 0 || temp >= IB_MAX_NUM_VLS) { + fprintf(stderr, + "ERROR: starting lash vl must be between 0 and 15\n"); + return (-1); + } + opt.lash_start_vl = (uint8_t) temp; + break; case 'h': case '?': case ':': diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index ec15f8a..fda2eb0 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -389,6 +390,7 @@ static const opt_rec_t opt_tbl[] = { { "no_clients_rereg", OPT_OFFSET(no_clients_rereg), opts_parse_boolean, NULL, 1 }, { "prefix_routes_file", OPT_OFFSET(prefix_routes_file), opts_parse_charp, NULL, 0 }, { "consolidate_ipv6_snm_req", OPT_OFFSET(consolidate_ipv6_snm_req), opts_parse_boolean, NULL, 1 }, + { "lash_start_vl", OPT_OFFSET(lash_start_vl), opts_parse_uint8, NULL, 1 }, {0} }; @@ -749,6 +751,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt) p_opt->no_clients_rereg = FALSE; p_opt->prefix_routes_file = strdup(OSM_DEFAULT_PREFIX_ROUTES_FILE); p_opt->consolidate_ipv6_snm_req = FALSE; + p_opt->lash_start_vl = 0; subn_init_qos_options(&p_opt->qos_options, NULL); subn_init_qos_options(&p_opt->qos_ca_options, NULL); subn_init_qos_options(&p_opt->qos_sw0_options, NULL); @@ -1432,6 +1435,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) p_opts->do_mesh_analysis ? "TRUE" : "FALSE"); fprintf(out, + "# Starting VL for LASH algorithm\n" + "lash_start_vl %d\n\n", + p_opts->lash_start_vl); + + fprintf(out, "# SA database file name\nsa_db_file %s\n\n", p_opts->sa_db_file ? p_opts->sa_db_file : null_str); diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 12b5e34..3db75a0 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -478,7 +478,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix; int *num_mst_in_lane = p_lash->num_mst_in_lane; int ***virtual_location = p_lash->virtual_location; - int min_filled_lane, max_filled_lane, trials; + int min_filled_lane, max_filled_lane, trials, max_vl; int old_min_filled_lane, old_max_filled_lane, new_num_min_lane, new_num_max_lane; unsigned int i, j; @@ -486,9 +486,11 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) int next_switch2, output_link2; int stop = 0, cycle_found; int cycle_found2; + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; - max_filled_lane = 0; - min_filled_lane = lanes_needed - 1; + max_filled_lane = start_vl; + max_vl = lanes_needed + start_vl; + min_filled_lane = max_vl - 1; trials = num_mst_in_lane[max_filled_lane]; if (lanes_needed == 1) @@ -590,7 +592,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) new_num_min_lane = MAX_INT; new_num_max_lane = 0; - for (i = 0; i < lanes_needed; i++) { + for (i = start_vl; i < max_vl; i++) { if (num_mst_in_lane[i] < new_num_min_lane) { new_num_min_lane = num_mst_in_lane[i]; @@ -674,11 +676,12 @@ static void free_lash_structures(lash_t * p_lash) unsigned int i, j, k; unsigned num_switches = p_lash->num_switches; osm_log_t *p_log = &p_lash->p_osm->log; + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; OSM_LOG_ENTER(p_log); /* free cdg_vertex_matrix */ - for (i = 0; i < p_lash->vl_min; i++) { + for (i = start_vl; i < p_lash->vl_min; i++) { for (j = 0; j < num_switches; j++) { for (k = 0; k < num_switches; k++) if (p_lash->cdg_vertex_matrix[i][j][k]) @@ -715,13 +718,14 @@ static int init_lash_structures(lash_t * p_lash) osm_log_t *p_log = &p_lash->p_osm->log; int status = 0; unsigned int i, j, k; + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; OSM_LOG_ENTER(p_log); /* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */ p_lash->cdg_vertex_matrix = - (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****)); - for (i = 0; i < vl_min; i++) { + (cdg_vertex_t ****) calloc(vl_min, sizeof(cdg_vertex_t ****)); + for (i = start_vl; i < vl_min; i++) { p_lash->cdg_vertex_matrix[i] = (cdg_vertex_t ***) malloc(num_switches * sizeof(cdg_vertex_t ***)); @@ -730,7 +734,7 @@ static int init_lash_structures(lash_t * p_lash) goto Exit_Mem_Error; } - for (i = 0; i < vl_min; i++) { + for (i = start_vl; i < vl_min; i++) { for (j = 0; j < num_switches; j++) { p_lash->cdg_vertex_matrix[i][j] = (cdg_vertex_t **) malloc(num_switches * @@ -763,7 +767,7 @@ static int init_lash_structures(lash_t * p_lash) for (i = 0; i < num_switches; i++) { for (j = 0; j < num_switches; j++) { p_lash->virtual_location[i][j] = - (int *)malloc(vl_min * sizeof(int *)); + (int *)calloc(vl_min, sizeof(int *)); if (p_lash->virtual_location[i][j] == NULL) goto Exit_Mem_Error; for (k = 0; k < vl_min; k++) { @@ -804,6 +808,8 @@ static int lash_core(lash_t * p_lash) int cycle_found2 = 0; int status = 0; int *switch_bitmap = NULL; /* Bitmap to check if we have processed this pair */ + int max_vl; + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; OSM_LOG_ENTER(p_log); @@ -838,11 +844,14 @@ static int lash_core(lash_t * p_lash) } for (i = 0; i < num_switches; i++) { - for (dest_switch = 0; dest_switch < num_switches; dest_switch++) + for (dest_switch = 0; dest_switch < num_switches; dest_switch++) { + max_vl = lanes_needed + start_vl; + if (max_vl > p_lash->vl_min) + goto Error_Not_Enough_Lanes; if (dest_switch != i && switch_bitmap[i * num_switches + dest_switch] == 0) { - v_lane = 0; + v_lane = start_vl; stop = 0; - while (v_lane < lanes_needed && stop == 0) { + while (v_lane < max_vl && stop == 0) { generate_cdg_for_sp(p_lash, i, dest_switch, v_lane); generate_cdg_for_sp(p_lash, dest_switch, i, v_lane); @@ -906,7 +915,8 @@ static int lash_core(lash_t * p_lash) switches[dest_switch]->routing_table[i].lane = v_lane; if (cycle_found == 1 || cycle_found2 == 1) { - if (++lanes_needed > p_lash->vl_min) + lanes_needed++; + if (start_vl + lanes_needed > p_lash->vl_min) goto Error_Not_Enough_Lanes; generate_cdg_for_sp(p_lash, i, dest_switch, v_lane); @@ -926,19 +936,24 @@ static int lash_core(lash_t * p_lash) switch_bitmap[i * num_switches + dest_switch] = 1; switch_bitmap[dest_switch * num_switches + i] = 1; } + } } - OSM_LOG(p_log, OSM_LOG_INFO, - "Lanes needed: %d, Balancing\n", lanes_needed); + max_vl = lanes_needed + start_vl; + if (max_vl > p_lash->vl_min) + goto Error_Not_Enough_Lanes; - for (i = 0; i < lanes_needed; i++) { + for (i = start_vl; i < max_vl; i++) { OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", i, p_lash->num_mst_in_lane[i]); } + OSM_LOG(p_log, OSM_LOG_INFO, + "Lanes needed: %d, Balancing\n", lanes_needed); + balance_virtual_lanes(p_lash, lanes_needed); - for (i = 0; i < lanes_needed; i++) { + for (i = start_vl; i < max_vl; i++) { OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", i, p_lash->num_mst_in_lane[i]); } @@ -948,8 +963,9 @@ static int lash_core(lash_t * p_lash) Error_Not_Enough_Lanes: status = -1; OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4D02: " - "Lane requirements (%d) exceed available lanes (%d)\n", - lanes_needed, p_lash->vl_min); + "Lane requirements (%d) exceed available lanes (%d)" + " with starting lane (%d)\n", + lanes_needed, p_lash->vl_min, start_vl); Exit: if (switch_bitmap) free(switch_bitmap); @@ -1286,7 +1302,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port, src_id = get_lash_id(p_sw); if (src_id == dst_id) - return OSM_DEFAULT_SL; + return p_osm->subn.opt.lash_start_vl; return (uint8_t) ((switch_t *) p_sw->priv)->routing_table[dst_id].lane; } From rdreier at cisco.com Mon Jul 6 09:24:10 2009 From: rdreier at cisco.com (Roland Dreier) Date: Mon, 06 Jul 2009 09:24:10 -0700 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A521784.5090304@Voltaire.COM> (Moni Shoua's message of "Mon, 06 Jul 2009 18:25:56 +0300") References: <4A521784.5090304@Voltaire.COM> Message-ID: > Whenever an illegal multicast address is passed to IPoIB for it to join it stops all > subsequent requests from being joined. That happens because IPoIB joins to multicast > addresses in the order they arrived and doesn't handle the next group's join until the > current join finishes with success. This phenomena happens a lot when a bonding interface > enslaves IPoIB devices. Before enslaving IPoIB interfaces the bonding device acts like an > Ethernet device, including the way it translates muticast IP addresses to HW addresses. When > it comes up without slaves it translates the group 224.0.0.1 (all hosts) as if it were an > Ethernet device and when it enslaves IPoIB devices this is the address that they get for > joining (which is a garbage for them) Wouldn't it make more sense to fix the bonding device or core networking (wherever the problem is) so that it recomputes the multicast list when the bonding hardware type changes? After this patch, do we end up with an IPoIB interface that's not a member of the all hosts multicast group? (That seems like it would lead to confusing breakage later) - R. From akepner at sgi.com Mon Jul 6 10:26:39 2009 From: akepner at sgi.com (akepner at sgi.com) Date: Mon, 6 Jul 2009 10:26:39 -0700 Subject: [ofa-general] [PATCH RESEND] ipoib: fix racing uses of ipoib_neigh in CM with RCU Message-ID: <20090706172639.GC5256@sgi.com> No comments since this was originally posted. Too beautiful for words? It does seem an awfully complex solution to a rarely encountered bug, but I don't see a better solution. If you do, please let me know. ----------------------------------------------------------------- There is a race between the neighbour cleanup code, and some uses of the closely related ipoib_neigh structure when using IPoIB-CM. For example, a IPoIB-CM connection may time out, invoking ipoib_neigh_free(), which cleans up and frees the struct ipoib_neigh, while at the same time a neighbour entry times out, invoking ipoib_neigh_cleanup(), cleaning up and freeing the ipoib_neigh structure a second time. The root cause is that the struct ipoib_neigh pointer that's stashed away in struct neighbour is read (and subsequently used) in ipoib_neigh_cleanup() without using locking that's consistent with other reads/writes of this pointer. (The pointer must be read before it can be known which lock to use, so it's difficult to avoid.) To fix this, use RCU methods to read/write the struct ipoib_neigh pointer that's stashed away in struct neighbour, cleanup (most of) the struct ipoib_neigh's internal state, and free the structure in an RCU callback. Updates of the struct ipoib_neigh pointer are protected by struct ipoib_dev_priv's lock. The pointer can be read, and most of the internal state of the ipoib_neigh can be used, without locking, inside a RCU read-side critical section. The internal state of the struct ipoib_neigh which requires mutual exclusion (the list_head element) is protected by a new spinlock. Signed-off-by: Arthur Kepner --- ipoib.h | 38 ++++++++++++++-- ipoib_cm.c | 23 +-------- ipoib_main.c | 127 +++++++++++++++++++++++++++++------------------------- ipoib_multicast.c | 56 ++++++++++++----------- 4 files changed, 138 insertions(+), 106 deletions(-) diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h index 753a983..c84ab84 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib.h +++ b/drivers/infiniband/ulp/ipoib/ipoib.h @@ -105,6 +105,7 @@ enum { MAX_SEND_CQE = 16, IPOIB_CM_COPYBREAK = 256, + IPOIB_FLAG_CM_CLEAR = 1, }; #define IPOIB_OP_RECV (1ul << 31) @@ -382,6 +383,10 @@ struct ipoib_path { }; struct ipoib_neigh { + spinlock_t lock; + int dead; + int rcu_flags; /* flags used during rcu callback */ + struct rcu_head rcu; struct ipoib_ah *ah; #ifdef CONFIG_INFINIBAND_IPOIB_CM struct ipoib_cm_tx *cm; @@ -409,15 +414,42 @@ static inline int ipoib_ud_need_sg(unsigned int ib_mtu) * sure that this pointer is stored aligned so that an unaligned * load is not needed to dereference it. */ -static inline struct ipoib_neigh **to_ipoib_neigh(struct neighbour *neigh) +static inline struct ipoib_neigh* ipoib_neigh_retrieve(struct neighbour *n) +{ + struct ipoib_neigh **np; + + np = (void*) n + ALIGN(offsetof(struct neighbour, ha) + + INFINIBAND_ALEN, sizeof(void *)); + + return rcu_dereference(*np); +} + +static inline void ipoib_neigh_assign(struct neighbour *n, + struct ipoib_neigh *in) +{ + struct ipoib_neigh **np; + + np = (void*) n + ALIGN(offsetof(struct neighbour, ha) + + INFINIBAND_ALEN, sizeof(void *)); + + rcu_assign_pointer(*np, in); +} + +static inline int ipoib_cm_get_rcu_flag(struct ipoib_neigh *neigh, int flag) +{ + return !!(neigh->rcu_flags & flag); +} + +static inline void ipoib_cm_set_rcu_flag(struct ipoib_neigh *neigh, int flag) { - return (void*) neigh + ALIGN(offsetof(struct neighbour, ha) + - INFINIBAND_ALEN, sizeof(void *)); + neigh->rcu_flags |= flag; } struct ipoib_neigh *ipoib_neigh_alloc(struct neighbour *neigh, struct net_device *dev); void ipoib_neigh_free(struct net_device *dev, struct ipoib_neigh *neigh); +void ipoib_neigh_tidy(struct net_device *dev, struct ipoib_neigh *neigh, + int cm); extern struct workqueue_struct *ipoib_workqueue; diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c index 181b1f3..f3eead1 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c @@ -806,12 +806,7 @@ void ipoib_cm_handle_tx_wc(struct net_device *dev, struct ib_wc *wc) neigh = tx->neigh; if (neigh) { - neigh->cm = NULL; - list_del(&neigh->list); - if (neigh->ah) - ipoib_put_ah(neigh->ah); - ipoib_neigh_free(dev, neigh); - + ipoib_neigh_tidy(dev, neigh, 1); tx->neigh = NULL; } @@ -1226,12 +1221,7 @@ static int ipoib_cm_tx_handler(struct ib_cm_id *cm_id, neigh = tx->neigh; if (neigh) { - neigh->cm = NULL; - list_del(&neigh->list); - if (neigh->ah) - ipoib_put_ah(neigh->ah); - ipoib_neigh_free(dev, neigh); - + ipoib_neigh_tidy(dev, neigh, 1); tx->neigh = NULL; } @@ -1315,13 +1305,8 @@ static void ipoib_cm_tx_start(struct work_struct *work) if (ret) { neigh = p->neigh; - if (neigh) { - neigh->cm = NULL; - list_del(&neigh->list); - if (neigh->ah) - ipoib_put_ah(neigh->ah); - ipoib_neigh_free(dev, neigh); - } + if (neigh) + ipoib_neigh_tidy(dev, neigh, 1); list_del(&p->list); kfree(p); } diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index e319d91..409453a 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -273,18 +273,8 @@ static void path_free(struct net_device *dev, struct ipoib_path *path) spin_lock_irqsave(&priv->lock, flags); - list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) { - /* - * It's safe to call ipoib_put_ah() inside priv->lock - * here, because we know that path->ah will always - * hold one more reference, so ipoib_put_ah() will - * never do more than decrement the ref count. - */ - if (neigh->ah) - ipoib_put_ah(neigh->ah); - + list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) ipoib_neigh_free(dev, neigh); - } spin_unlock_irqrestore(&priv->lock, flags); @@ -466,10 +456,7 @@ static void path_rec_completion(int status, path, neigh)); if (!ipoib_cm_get(neigh)) { - list_del(&neigh->list); - if (neigh->ah) - ipoib_put_ah(neigh->ah); - ipoib_neigh_free(dev, neigh); + ipoib_neigh_tidy(dev, neigh, 0); continue; } } @@ -561,15 +548,16 @@ static void neigh_add_path(struct sk_buff *skb, struct net_device *dev) struct ipoib_neigh *neigh; unsigned long flags; + spin_lock_irqsave(&priv->lock, flags); + neigh = ipoib_neigh_alloc(skb_dst(skb)->neighbour, skb->dev); if (!neigh) { + spin_unlock_irqrestore(&priv->lock, flags); ++dev->stats.tx_dropped; dev_kfree_skb_any(skb); return; } - spin_lock_irqsave(&priv->lock, flags); - path = __path_find(dev, skb_dst(skb)->neighbour->ha + 4); if (!path) { path = path_rec_create(dev, skb_dst(skb)->neighbour->ha + 4); @@ -591,10 +579,7 @@ static void neigh_add_path(struct sk_buff *skb, struct net_device *dev) if (!ipoib_cm_get(neigh)) ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh)); if (!ipoib_cm_get(neigh)) { - list_del(&neigh->list); - if (neigh->ah) - ipoib_put_ah(neigh->ah); - ipoib_neigh_free(dev, neigh); + ipoib_neigh_tidy(dev, neigh, 0); goto err_drop; } if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) @@ -709,30 +694,22 @@ static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev) unsigned long flags; if (likely(skb_dst(skb) && skb_dst(skb)->neighbour)) { - if (unlikely(!*to_ipoib_neigh(skb_dst(skb)->neighbour))) { + struct neighbour *n = skb_dst(skb)->neighbour; + rcu_read_lock(); + neigh = ipoib_neigh_retrieve(n); + if (unlikely(!neigh)) { + rcu_read_unlock(); ipoib_path_lookup(skb, dev); return NETDEV_TX_OK; } - neigh = *to_ipoib_neigh(skb_dst(skb)->neighbour); - - if (unlikely((memcmp(&neigh->dgid.raw, - skb_dst(skb)->neighbour->ha + 4, + if (unlikely((memcmp(&neigh->dgid.raw, n->ha +4, sizeof(union ib_gid))) || (neigh->dev != dev))) { spin_lock_irqsave(&priv->lock, flags); - /* - * It's safe to call ipoib_put_ah() inside - * priv->lock here, because we know that - * path->ah will always hold one more reference, - * so ipoib_put_ah() will never do more than - * decrement the ref count. - */ - if (neigh->ah) - ipoib_put_ah(neigh->ah); - list_del(&neigh->list); - ipoib_neigh_free(dev, neigh); + ipoib_neigh_tidy(dev, neigh, 0); spin_unlock_irqrestore(&priv->lock, flags); + rcu_read_unlock(); ipoib_path_lookup(skb, dev); return NETDEV_TX_OK; } @@ -740,10 +717,12 @@ static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev) if (ipoib_cm_get(neigh)) { if (ipoib_cm_up(neigh)) { ipoib_cm_send(dev, skb, ipoib_cm_get(neigh)); + rcu_read_unlock(); return NETDEV_TX_OK; } } else if (neigh->ah) { - ipoib_send(dev, skb, neigh->ah, IPOIB_QPN(skb_dst(skb)->neighbour->ha)); + ipoib_send(dev, skb, neigh->ah, IPOIB_QPN(n->ha)); + rcu_read_unlock(); return NETDEV_TX_OK; } @@ -755,6 +734,7 @@ static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev) ++dev->stats.tx_dropped; dev_kfree_skb_any(skb); } + rcu_read_unlock(); } else { struct ipoib_pseudoheader *phdr = (struct ipoib_pseudoheader *) skb->data; @@ -843,62 +823,91 @@ static void ipoib_neigh_cleanup(struct neighbour *n) struct ipoib_neigh *neigh; struct ipoib_dev_priv *priv = netdev_priv(n->dev); unsigned long flags; - struct ipoib_ah *ah = NULL; - neigh = *to_ipoib_neigh(n); + rcu_read_lock(); + + neigh = ipoib_neigh_retrieve(n); if (neigh) priv = netdev_priv(neigh->dev); - else + else { + rcu_read_unlock(); return; + } ipoib_dbg(priv, "neigh_cleanup for %06x %pI6\n", IPOIB_QPN(n->ha), n->ha + 4); spin_lock_irqsave(&priv->lock, flags); - - if (neigh->ah) - ah = neigh->ah; - list_del(&neigh->list); - ipoib_neigh_free(n->dev, neigh); - + ipoib_neigh_tidy(n->dev, neigh, 0); spin_unlock_irqrestore(&priv->lock, flags); - - if (ah) - ipoib_put_ah(ah); + rcu_read_unlock(); } -struct ipoib_neigh *ipoib_neigh_alloc(struct neighbour *neighbour, +struct ipoib_neigh *ipoib_neigh_alloc(struct neighbour *n, struct net_device *dev) { struct ipoib_neigh *neigh; - neigh = kmalloc(sizeof *neigh, GFP_ATOMIC); + neigh = kzalloc(sizeof *neigh, GFP_ATOMIC); if (!neigh) return NULL; - neigh->neighbour = neighbour; + spin_lock_init(&neigh->lock); + neigh->neighbour = n; neigh->dev = dev; - *to_ipoib_neigh(neighbour) = neigh; skb_queue_head_init(&neigh->queue); - ipoib_cm_set(neigh, NULL); - + ipoib_neigh_assign(n, neigh); return neigh; } -void ipoib_neigh_free(struct net_device *dev, struct ipoib_neigh *neigh) +static void ipoib_neigh_free_rcu(struct rcu_head *head) { + struct ipoib_neigh *neigh = container_of(head, struct ipoib_neigh, rcu); + struct net_device *dev = neigh->dev; struct sk_buff *skb; - *to_ipoib_neigh(neigh->neighbour) = NULL; + while ((skb = __skb_dequeue(&neigh->queue))) { ++dev->stats.tx_dropped; dev_kfree_skb_any(skb); } + + if (ipoib_cm_get_rcu_flag(neigh, IPOIB_FLAG_CM_CLEAR)) + ipoib_cm_set(neigh, NULL); + if (ipoib_cm_get(neigh)) ipoib_cm_destroy_tx(ipoib_cm_get(neigh)); + + if (neigh->ah) + ipoib_put_ah(neigh->ah); + kfree(neigh); } +void ipoib_neigh_free(struct net_device *dev, struct ipoib_neigh *neigh) +{ + struct neighbour *n = neigh->neighbour; + ipoib_neigh_assign(n, NULL); + + call_rcu(&neigh->rcu, ipoib_neigh_free_rcu); +} + +void ipoib_neigh_tidy(struct net_device *dev, struct ipoib_neigh *neigh, int cm) +{ + spin_lock(&neigh->lock); + if (neigh->dead) { + /* lost the race */ + spin_unlock(&neigh->lock); + return; + } + neigh->dead = 1; + list_del(&neigh->list); + spin_unlock(&neigh->lock); + if (cm) + ipoib_cm_set_rcu_flag(neigh, IPOIB_FLAG_CM_CLEAR); + ipoib_neigh_free(dev, neigh); +} + static int ipoib_neigh_setup_dev(struct net_device *dev, struct neigh_parms *parms) { parms->neigh_cleanup = ipoib_neigh_cleanup; @@ -1376,6 +1385,8 @@ static void ipoib_remove_one(struct ib_device *device) dev_list = ib_get_client_data(device, &ipoib_client); + rcu_barrier(); + list_for_each_entry_safe(priv, tmp, dev_list, list) { ib_unregister_event_handler(&priv->event_handler); diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c index a0e9753..ec96f82 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c @@ -76,17 +76,8 @@ static void ipoib_mcast_free(struct ipoib_mcast *mcast) spin_lock_irq(&priv->lock); - list_for_each_entry_safe(neigh, tmp, &mcast->neigh_list, list) { - /* - * It's safe to call ipoib_put_ah() inside priv->lock - * here, because we know that mcast->ah will always - * hold one more reference, so ipoib_put_ah() will - * never do more than decrement the ref count. - */ - if (neigh->ah) - ipoib_put_ah(neigh->ah); + list_for_each_entry_safe(neigh, tmp, &mcast->neigh_list, list) ipoib_neigh_free(dev, neigh); - } spin_unlock_irq(&priv->lock); @@ -658,9 +649,7 @@ void ipoib_mcast_send(struct net_device *dev, void *mgid, struct sk_buff *skb) if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags) || !priv->broadcast || !test_bit(IPOIB_MCAST_FLAG_ATTACHED, &priv->broadcast->flags)) { - ++dev->stats.tx_dropped; - dev_kfree_skb_any(skb); - goto unlock; + goto drop; } mcast = __ipoib_mcast_find(dev, mgid); @@ -673,9 +662,7 @@ void ipoib_mcast_send(struct net_device *dev, void *mgid, struct sk_buff *skb) if (!mcast) { ipoib_warn(priv, "unable to allocate memory for " "multicast structure\n"); - ++dev->stats.tx_dropped; - dev_kfree_skb_any(skb); - goto out; + goto drop; } set_bit(IPOIB_MCAST_FLAG_SENDONLY, &mcast->flags); @@ -705,25 +692,42 @@ void ipoib_mcast_send(struct net_device *dev, void *mgid, struct sk_buff *skb) mcast = NULL; } -out: if (mcast && mcast->ah) { - if (skb_dst(skb) && - skb_dst(skb)->neighbour && - !*to_ipoib_neigh(skb_dst(skb)->neighbour)) { - struct ipoib_neigh *neigh = ipoib_neigh_alloc(skb_dst(skb)->neighbour, - skb->dev); + if (skb_dst(skb) && skb_dst(skb)->neighbour) { + struct neighbour *n = skb_dst(skb)->neighbour; + struct ipoib_neigh *neigh; + int drop = 0; + + rcu_read_lock(); + neigh = ipoib_neigh_retrieve(n); + + if (!neigh) + neigh = ipoib_neigh_alloc(n, skb->dev); if (neigh) { - kref_get(&mcast->ah->ref); - neigh->ah = mcast->ah; - list_add_tail(&neigh->list, &mcast->neigh_list); + spin_lock(&neigh->lock); + if (!neigh->dead) { + kref_get(&mcast->ah->ref); + neigh->ah = mcast->ah; + list_add_tail(&neigh->list, &mcast->neigh_list); + } else + drop = 1; + spin_unlock(&neigh->lock); } + rcu_read_unlock(); + if (drop) + goto drop; } ipoib_send(dev, skb, mcast->ah, IB_MULTICAST_QPN); } -unlock: + spin_unlock_irqrestore(&priv->lock, flags); + return; + +drop: + ++dev->stats.tx_dropped; + dev_kfree_skb_any(skb); spin_unlock_irqrestore(&priv->lock, flags); } From ranjit.pandit.ib at gmail.com Mon Jul 6 11:39:02 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Mon, 6 Jul 2009 11:39:02 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> Message-ID: <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> peer2peer setting made a difference. I'm not getting the connect errors any more. However, there is still some difference in behavior between running on IB vs iWarp. In IB's case the test completes but over iWarp's both sides hang (no connect errors this time though) on iWarp: [root at lv4 examples]# ./ucmatose 'cmatose: starting server initiating data transfers completing sends --> Hangs here [root at lv5 examples]# ./ucmatose -s 192.168.10.33 cmatose: starting client cmatose: connecting receiving data transfers sending replies data transfers complete --> Hangs here On Fri, Jul 3, 2009 at 12:03 PM, Davis, Arlin R wrote: > >>-----Original Message----- >>From: Hefty, Sean >>Sent: Thursday, July 02, 2009 11:44 PM >>To: Davis, Arlin R; 'pandit ib'; general at lists.openfabrics.org >>Subject: RE: [ofa-general] cmatose fails whereas rping passes on iWarp >> >>>If this test sends data from server side first you could >>>be running into the iWARP requirement of sending from >>>client first. >> >>This was my thought as well.  I think Chelsio supports sending >>from the server >>side first, but I'm not sure, or if it's enabled by default. > > I don't believe it is enabled by default. To enable: > > echo 1 > /sys/module/iw_cxgb3/parameters/peer2peer > >  or > > add following /etc/modprobe.conf driver option: > > options iw_cxgb3 peer2peer=1 From swise at opengridcomputing.com Mon Jul 6 11:45:19 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Mon, 06 Jul 2009 13:45:19 -0500 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> Message-ID: <4A52463F.9060903@opengridcomputing.com> pandit ib wrote: > peer2peer setting made a difference. > I'm not getting the connect errors any more. > > However, there is still some difference in behavior between running on > IB vs iWarp. > In IB's case the test completes but over iWarp's both sides hang (no > connect errors this time though) > > on iWarp: > > [root at lv4 examples]# ./ucmatose > 'cmatose: starting server > initiating data transfers > completing sends > --> Hangs here > > [root at lv5 examples]# ./ucmatose -s 192.168.10.33 > cmatose: starting client > cmatose: connecting > receiving data transfers > sending replies > data transfers complete > --> Hangs here > > Does dmesg show anything interesting? From ranjit.pandit.ib at gmail.com Mon Jul 6 11:56:17 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Mon, 6 Jul 2009 11:56:17 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <4A52463F.9060903@opengridcomputing.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <4A52463F.9060903@opengridcomputing.com> Message-ID: <96f8e60e0907061156t71f51a9la42a8ffdf1f5db7b@mail.gmail.com> No, nothing in /var/log/messages. On Mon, Jul 6, 2009 at 11:45 AM, Steve Wise wrote: > pandit ib wrote: >> >> peer2peer setting made a difference. >> I'm not getting the connect errors any more. >> >> However, there is still some difference in behavior between running on >> IB vs iWarp. >> In IB's case the test completes but over iWarp's both sides hang (no >> connect errors this time though) >> >> on iWarp: >> >> [root at lv4 examples]# ./ucmatose >> 'cmatose: starting server >> initiating data transfers >> completing sends >> --> Hangs here >> >> [root at lv5 examples]# ./ucmatose -s 192.168.10.33 >> cmatose: starting client >> cmatose: connecting >> receiving data transfers >> sending replies >> data transfers complete >> --> Hangs here >> >> > > Does dmesg show anything interesting? > > > From swise at opengridcomputing.com Mon Jul 6 12:25:37 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Mon, 06 Jul 2009 14:25:37 -0500 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <96f8e60e0907061156t71f51a9la42a8ffdf1f5db7b@mail.gmail.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <4A52463F.9060903@opengridcomputing.com> <96f8e60e0907061156t71f51a9la42a8ffdf1f5db7b@mail.gmail.com> Message-ID: <4A524FB1.4020704@opengridcomputing.com> Is the cq properly sized in cmatose? A CQ overflow shows up as an empty cq with Chelsio's rnic. pandit ib wrote: > No, nothing in /var/log/messages. > > On Mon, Jul 6, 2009 at 11:45 AM, Steve Wise wrote: > >> pandit ib wrote: >> >>> peer2peer setting made a difference. >>> I'm not getting the connect errors any more. >>> >>> However, there is still some difference in behavior between running on >>> IB vs iWarp. >>> In IB's case the test completes but over iWarp's both sides hang (no >>> connect errors this time though) >>> >>> on iWarp: >>> >>> [root at lv4 examples]# ./ucmatose >>> 'cmatose: starting server >>> initiating data transfers >>> completing sends >>> --> Hangs here >>> >>> [root at lv5 examples]# ./ucmatose -s 192.168.10.33 >>> cmatose: starting client >>> cmatose: connecting >>> receiving data transfers >>> sending replies >>> data transfers complete >>> --> Hangs here >>> >>> >>> >> Does dmesg show anything interesting? >> >> >> >> From arlin.r.davis at intel.com Mon Jul 6 12:51:57 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Mon, 6 Jul 2009 12:51:57 -0700 Subject: [ofa-general] [PATCH 03/11] uDAPL windows: all dlist functions have been moved to the header file. Message-ID: >From b8a14ff1cc257defa2f74373d143600f5f471823 Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Mon, 29 Jun 2009 12:34:54 -0700 Subject: remove references to dlist.c Signed-off-by: Sean Hefty --- dapl/openib_cma/device.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/dapl/openib_cma/device.c b/dapl/openib_cma/device.c index 0e974f6..22e5045 100644 --- a/dapl/openib_cma/device.c +++ b/dapl/openib_cma/device.c @@ -55,7 +55,6 @@ struct dapl_llist_entry *g_hca_list; #if defined(_WIN64) || defined(_WIN32) #include "..\..\..\..\..\etc\user\comp_channel.cpp" -#include "..\..\..\..\..\etc\user\dlist.c" #include struct ibvw_windata windata; -- 1.5.2.5 From arlin.r.davis at intel.com Mon Jul 6 12:51:50 2009 From: arlin.r.davis at intel.com (Arlin Davis) Date: Mon, 6 Jul 2009 12:51:50 -0700 Subject: [ofa-general] [PATCH 01/11] uDAPL openib_common: reorganize provider code base to share common mem, cq, qp, dto functions Message-ID: <786C4F7F03564A429E091D36C9B85F45@amr.corp.intel.com> Patch set to build framework for new uCM provider. Reorganize code base to make better use of common code for the ofa providers. add new openib_common directory with cq, qp, util, dto, mem function calls and definitions. This basically leaves the unique CM and Device definitions and functions to the individual providers directory of openib_scm and openib_cma. modifications to dapl_cr_accept required. ep->cm_handle is allocated and managed entirely in provider so dapl common code should not update ep_handle->cm_handle from the cr->cm_handle automatically. The provider should determine which cm_handle is required for the accept. Most changes are simply code relocation. openib_cma defines _OPENIB_CMA_ and openib_scm defines _OPENIB_SCM_ for provider specific build needs in common code. --- Makefile.am | 32 +- dapl/common/dapl_cr_accept.c | 1 - dapl/dirs | 2 +- dapl/include/dapl_debug.h | 3 +- dapl/openib_cma/SOURCES | 21 +- dapl/openib_cma/cm.c | 1338 +++++++++++++++++++++++++ dapl/openib_cma/dapl_ib_cm.c | 1297 ------------------------ dapl/openib_cma/dapl_ib_cq.c | 559 ----------- dapl/openib_cma/dapl_ib_dto.h | 472 --------- dapl/openib_cma/dapl_ib_extensions.c | 329 ------ dapl/openib_cma/dapl_ib_mem.c | 380 ------- dapl/openib_cma/dapl_ib_qp.c | 331 ------ dapl/openib_cma/dapl_ib_util.c | 1134 --------------------- dapl/openib_cma/dapl_ib_util.h | 252 +---- dapl/openib_cma/device.c | 847 ++++++++++++++++ dapl/openib_common/cq.c | 491 +++++++++ dapl/openib_common/dapl_ib_common.h | 299 ++++++ dapl/openib_common/dapl_ib_dto.h | 504 ++++++++++ dapl/openib_common/ib_extensions.c | 360 +++++++ dapl/openib_common/mem.c | 370 +++++++ dapl/openib_common/qp.c | 515 ++++++++++ dapl/openib_common/util.c | 375 +++++++ dapl/openib_scm/SOURCES | 21 +- dapl/openib_scm/cm.c | 1839 ++++++++++++++++++++++++++++++++++ dapl/openib_scm/dapl_ib_cm.c | 1786 --------------------------------- dapl/openib_scm/dapl_ib_cq.c | 705 ------------- dapl/openib_scm/dapl_ib_dto.h | 527 ---------- dapl/openib_scm/dapl_ib_extensions.c | 371 ------- dapl/openib_scm/dapl_ib_mem.c | 382 ------- dapl/openib_scm/dapl_ib_qp.c | 513 ---------- dapl/openib_scm/dapl_ib_util.c | 743 -------------- dapl/openib_scm/dapl_ib_util.h | 300 +------ dapl/openib_scm/device.c | 412 ++++++++ 33 files changed, 7433 insertions(+), 10078 deletions(-) create mode 100644 dapl/openib_cma/cm.c delete mode 100755 dapl/openib_cma/dapl_ib_cm.c delete mode 100755 dapl/openib_cma/dapl_ib_cq.c delete mode 100644 dapl/openib_cma/dapl_ib_dto.h delete mode 100755 dapl/openib_cma/dapl_ib_extensions.c delete mode 100755 dapl/openib_cma/dapl_ib_mem.c delete mode 100755 dapl/openib_cma/dapl_ib_qp.c delete mode 100755 dapl/openib_cma/dapl_ib_util.c create mode 100644 dapl/openib_cma/device.c create mode 100644 dapl/openib_common/cq.c create mode 100644 dapl/openib_common/dapl_ib_common.h create mode 100644 dapl/openib_common/dapl_ib_dto.h create mode 100644 dapl/openib_common/ib_extensions.c create mode 100644 dapl/openib_common/mem.c create mode 100644 dapl/openib_common/qp.c create mode 100644 dapl/openib_common/util.c create mode 100644 dapl/openib_scm/cm.c delete mode 100644 dapl/openib_scm/dapl_ib_cm.c delete mode 100644 dapl/openib_scm/dapl_ib_cq.c delete mode 100644 dapl/openib_scm/dapl_ib_dto.h delete mode 100755 dapl/openib_scm/dapl_ib_extensions.c delete mode 100644 dapl/openib_scm/dapl_ib_mem.c delete mode 100644 dapl/openib_scm/dapl_ib_qp.c delete mode 100644 dapl/openib_scm/dapl_ib_util.c create mode 100644 dapl/openib_scm/device.c diff --git a/Makefile.am b/Makefile.am index fa47165..cf75a88 100755 --- a/Makefile.am +++ b/Makefile.am @@ -17,8 +17,8 @@ endif if EXT_TYPE_IB XFLAGS = -DDAT_EXTENSIONS -XPROGRAMS_CMA = dapl/openib_cma/dapl_ib_extensions.c -XPROGRAMS_SCM = dapl/openib_scm/dapl_ib_extensions.c +XPROGRAMS_CMA = dapl/openib_common/ib_extensions.c +XPROGRAMS_SCM = dapl/openib_common/ib_extensions.c else XFLAGS = XPROGRAMS_CMA = @@ -47,6 +47,7 @@ dapl_udapl_libdaplofa_la_CFLAGS = $(AM_CFLAGS) -D_GNU_SOURCE $(OSFLAGS) $(XFLAGS -DOPENIB -DCQ_WAIT_OBJECT \ -I$(srcdir)/dat/include/ -I$(srcdir)/dapl/include/ \ -I$(srcdir)/dapl/common -I$(srcdir)/dapl/udapl/linux \ + -I$(srcdir)/dapl/openib_common \ -I$(srcdir)/dapl/openib_cma \ -I$(srcdir)/dapl/openib_cma/linux @@ -54,6 +55,7 @@ dapl_udapl_libdaploscm_la_CFLAGS = $(AM_CFLAGS) -D_GNU_SOURCE $(OSFLAGS) $(XFLAG -DOPENIB -DCQ_WAIT_OBJECT \ -I$(srcdir)/dat/include/ -I$(srcdir)/dapl/include/ \ -I$(srcdir)/dapl/common -I$(srcdir)/dapl/udapl/linux \ + -I$(srcdir)/dapl/openib_common \ -I$(srcdir)/dapl/openib_scm \ -I$(srcdir)/dapl/openib_scm/linux @@ -185,11 +187,12 @@ dapl_udapl_libdaplofa_la_SOURCES = dapl/udapl/dapl_init.c \ dapl/common/dapl_csp.c \ dapl/common/dapl_ep_post_send_invalidate.c \ dapl/common/dapl_ep_post_rdma_read_to_rmr.c \ - dapl/openib_cma/dapl_ib_util.c \ - dapl/openib_cma/dapl_ib_cq.c \ - dapl/openib_cma/dapl_ib_qp.c \ - dapl/openib_cma/dapl_ib_cm.c \ - dapl/openib_cma/dapl_ib_mem.c $(XPROGRAMS_CMA) + dapl/openib_common/mem.c \ + dapl/openib_common/cq.c \ + dapl/openib_common/qp.c \ + dapl/openib_common/util.c \ + dapl/openib_cma/cm.c \ + dapl/openib_cma/device.c $(XPROGRAMS_CMA) dapl_udapl_libdaplofa_la_LDFLAGS = -version-info 2:0:0 $(daplofa_version_script) \ -Wl,-init,dapl_init -Wl,-fini,dapl_fini \ @@ -298,11 +301,12 @@ dapl_udapl_libdaploscm_la_SOURCES = dapl/udapl/dapl_init.c \ dapl/common/dapl_csp.c \ dapl/common/dapl_ep_post_send_invalidate.c \ dapl/common/dapl_ep_post_rdma_read_to_rmr.c \ - dapl/openib_scm/dapl_ib_util.c \ - dapl/openib_scm/dapl_ib_cq.c \ - dapl/openib_scm/dapl_ib_qp.c \ - dapl/openib_scm/dapl_ib_cm.c \ - dapl/openib_scm/dapl_ib_mem.c $(XPROGRAMS_SCM) + dapl/openib_common/mem.c \ + dapl/openib_common/cq.c \ + dapl/openib_common/qp.c \ + dapl/openib_common/util.c \ + dapl/openib_scm/cm.c \ + dapl/openib_scm/device.c $(XPROGRAMS_SCM) dapl_udapl_libdaploscm_la_LDFLAGS = -version-info 2:0:0 $(daploscm_version_script) \ -Wl,-init,dapl_init -Wl,-fini,dapl_fini \ @@ -365,10 +369,10 @@ EXTRA_DIST = dat/common/dat_dictionary.h \ dapl/include/dapl_debug.h \ dapl/include/dapl_ipoib_names.h \ dapl/include/dapl_vendor.h \ - dapl/openib_cma/dapl_ib_dto.h \ + dapl/openib_common/dapl_ib_dto.h \ + dapl/openib_common/dapl_ib_common.h \ dapl/openib_cma/dapl_ib_util.h \ dapl/openib_cma/linux/openib_osd.h \ - dapl/openib_scm/dapl_ib_dto.h \ dapl/openib_scm/dapl_ib_util.h \ dapl/openib_scm/linux/openib_osd.h \ dat/udat/libdat2.map \ diff --git a/dapl/common/dapl_cr_accept.c b/dapl/common/dapl_cr_accept.c index 76a841e..5df9458 100644 --- a/dapl/common/dapl_cr_accept.c +++ b/dapl/common/dapl_cr_accept.c @@ -180,7 +180,6 @@ dapl_cr_accept(IN DAT_CR_HANDLE cr_handle, entry_ep_state = ep_ptr->param.ep_state; entry_ep_handle = cr_ptr->param.local_ep_handle; ep_ptr->param.ep_state = DAT_EP_STATE_COMPLETION_PENDING; - ep_ptr->cm_handle = cr_ptr->ib_cm_handle; ep_ptr->cr_ptr = cr_ptr; ep_ptr->param.remote_ia_address_ptr = cr_ptr->param.remote_ia_address_ptr; diff --git a/dapl/dirs b/dapl/dirs index e865dfb..e721ef5 100644 --- a/dapl/dirs +++ b/dapl/dirs @@ -1 +1 @@ -DIRS = ibal openib_scm openib_cma +DIRS = ibal openib_common openib_scm openib_cma diff --git a/dapl/include/dapl_debug.h b/dapl/include/dapl_debug.h index 92e3d3b..37edf90 100644 --- a/dapl/include/dapl_debug.h +++ b/dapl/include/dapl_debug.h @@ -66,7 +66,8 @@ typedef enum DAPL_DBG_TYPE_EXCEPTION = 0x0400, DAPL_DBG_TYPE_SRQ = 0x0800, DAPL_DBG_TYPE_CNTR = 0x1000, - DAPL_DBG_TYPE_CM_LIST = 0x2000 + DAPL_DBG_TYPE_CM_LIST = 0x2000, + DAPL_DBG_TYPE_THREAD = 0x4000 } DAPL_DBG_TYPE; diff --git a/dapl/openib_cma/SOURCES b/dapl/openib_cma/SOURCES index fd67d07..f1c5002 100644 --- a/dapl/openib_cma/SOURCES +++ b/dapl/openib_cma/SOURCES @@ -18,16 +18,17 @@ USE_MSVCRT = 1 SOURCES = \ udapl.rc \ - ..\dapl_common_src.c \ - ..\dapl_udapl_src.c \ - dapl_ib_cq.c \ - dapl_ib_extensions.c \ - dapl_ib_mem.c \ - dapl_ib_qp.c \ - dapl_ib_util.c \ - dapl_ib_cm.c - -INCLUDES = ..\include;..\common;windows;..\..\dat\include;\ + ..\dapl_common_src.c \ + ..\dapl_udapl_src.c \ + ..\openib_common\mem.c \ + ..\openib_common\util.c \ + ..\openib_common\cq.c \ + ..\openib_common\qp.c \ + ..\openib_common\ib_extensions.c \ + device.c \ + cm.c + +INCLUDES = ..\include;..\openib_common;..\common;windows;..\..\dat\include;\ ..\..\dat\udat\windows;..\udapl\windows;\ ..\..\..\..\inc;..\..\..\..\inc\user;..\..\..\libibverbs\include;\ ..\..\..\librdmacm\include diff --git a/dapl/openib_cma/cm.c b/dapl/openib_cma/cm.c new file mode 100644 index 0000000..497f78a --- /dev/null +++ b/dapl/openib_cma/cm.c @@ -0,0 +1,1338 @@ +/* + * Copyright (c) 2005 Voltaire Inc. All rights reserved. + * Copyright (c) 2005-2007 Intel Corporation. All rights reserved. + * Copyright (c) 2004-2005, Mellanox Technologies, Inc. All rights reserved. + * Copyright (c) 2003 Topspin Corporation. All rights reserved. + * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ib_cm.c + * + * PURPOSE: The OFED provider - uCMA, name and route resolution + * + * $Id: $ + * + **********************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" +#include "dapl_cr_util.h" +#include "dapl_name_service.h" +#include "dapl_ib_util.h" +#include "dapl_vendor.h" +#include "dapl_osd.h" + +extern struct rdma_event_channel *g_cm_events; + +/* local prototypes */ +static struct dapl_cm_id *dapli_req_recv(struct dapl_cm_id *conn, + struct rdma_cm_event *event); +static void dapli_cm_active_cb(struct dapl_cm_id *conn, + struct rdma_cm_event *event); +static void dapli_cm_passive_cb(struct dapl_cm_id *conn, + struct rdma_cm_event *event); +static void dapli_addr_resolve(struct dapl_cm_id *conn); +static void dapli_route_resolve(struct dapl_cm_id *conn); + +/* cma requires 16 bit SID, in network order */ +#define IB_PORT_MOD 32001 +#define IB_PORT_BASE (65535 - IB_PORT_MOD) +#define SID_TO_PORT(SID) \ + (SID > 0xffff ? \ + htons((unsigned short)((SID % IB_PORT_MOD) + IB_PORT_BASE)) :\ + htons((unsigned short)SID)) + +#define PORT_TO_SID(p) ntohs(p) + +/* private data header to validate consumer rejects versus abnormal events */ +struct dapl_pdata_hdr { + DAT_UINT32 version; +}; + +static void dapli_addr_resolve(struct dapl_cm_id *conn) +{ + int ret; +#ifdef DAPL_DBG + struct rdma_addr *ipaddr = &conn->cm_id->route.addr; +#endif + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " addr_resolve: cm_id %p SRC %x DST %x\n", + conn->cm_id, ntohl(((struct sockaddr_in *) + &ipaddr->src_addr)->sin_addr.s_addr), + ntohl(((struct sockaddr_in *) + &ipaddr->dst_addr)->sin_addr.s_addr)); + + ret = rdma_resolve_route(conn->cm_id, conn->route_timeout); + if (ret) { + dapl_log(DAPL_DBG_TYPE_ERR, + " dapl_cma_connect: rdma_resolve_route ERR 0x%x %s\n", + ret, strerror(errno)); + dapl_evd_connection_callback(conn, + IB_CME_LOCAL_FAILURE, + NULL, conn->ep); + } +} + +static void dapli_route_resolve(struct dapl_cm_id *conn) +{ + int ret; +#ifdef DAPL_DBG + struct rdma_addr *ipaddr = &conn->cm_id->route.addr; + struct ib_addr *ibaddr = &conn->cm_id->route.addr.addr.ibaddr; +#endif + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " route_resolve: cm_id %p SRC %x DST %x PORT %d\n", + conn->cm_id, ntohl(((struct sockaddr_in *) + &ipaddr->src_addr)->sin_addr.s_addr), + ntohl(((struct sockaddr_in *) + &ipaddr->dst_addr)->sin_addr.s_addr), + ntohs(((struct sockaddr_in *) + &ipaddr->dst_addr)->sin_port)); + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " route_resolve: SRC GID subnet %016llx id %016llx\n", + (unsigned long long) + ntohll(ibaddr->sgid.global.subnet_prefix), + (unsigned long long) + ntohll(ibaddr->sgid.global.interface_id)); + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " route_resolve: DST GID subnet %016llx id %016llx\n", + (unsigned long long) + ntohll(ibaddr->dgid.global.subnet_prefix), + (unsigned long long) + ntohll(ibaddr->dgid.global.interface_id)); + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " route_resolve: cm_id %p pdata %p plen %d rr %d ind %d\n", + conn->cm_id, + conn->params.private_data, + conn->params.private_data_len, + conn->params.responder_resources, + conn->params.initiator_depth); + + ret = rdma_connect(conn->cm_id, &conn->params); + if (ret) { + dapl_log(DAPL_DBG_TYPE_ERR, + " dapl_cma_connect: rdma_connect ERR %d %s\n", + ret, strerror(errno)); + goto bail; + } + return; + + bail: + dapl_evd_connection_callback(conn, + IB_CME_LOCAL_FAILURE, NULL, conn->ep); +} + +dp_ib_cm_handle_t dapls_ib_cm_create(DAPL_EP *ep) +{ + dp_ib_cm_handle_t conn; + struct rdma_cm_id *cm_id; + + /* Allocate CM and initialize lock */ + if ((conn = dapl_os_alloc(sizeof(*conn))) == NULL) + return NULL; + + dapl_os_memzero(conn, sizeof(*conn)); + dapl_os_lock_init(&conn->lock); + + /* create CM_ID, bind to local device, create QP */ + if (rdma_create_id(g_cm_events, &cm_id, (void *)conn, RDMA_PS_TCP)) { + dapl_os_free(conn, sizeof(*conn)); + return NULL; + } + conn->cm_id = cm_id; + + /* setup timers for address and route resolution */ + conn->arp_timeout = dapl_os_get_env_val("DAPL_CM_ARP_TIMEOUT_MS", + IB_ARP_TIMEOUT); + conn->arp_retries = dapl_os_get_env_val("DAPL_CM_ARP_RETRY_COUNT", + IB_ARP_RETRY_COUNT); + conn->route_timeout = dapl_os_get_env_val("DAPL_CM_ROUTE_TIMEOUT_MS", + IB_ROUTE_TIMEOUT); + conn->route_retries = dapl_os_get_env_val("DAPL_CM_ROUTE_RETRY_COUNT", + IB_ROUTE_RETRY_COUNT); + if (ep != NULL) { + conn->ep = ep; + conn->hca = ((DAPL_IA *)ep->param.ia_handle)->hca_ptr; + } + + return conn; +} + +/* + * Called from consumer thread via dat_ep_free(). + * CANNOT be called from the async event processing thread + * dapli_cma_event_cb() since a cm_id reference is held and + * a deadlock will occur. + */ + +void dapls_ib_cm_free(dp_ib_cm_handle_t conn, DAPL_EP *ep) +{ + struct rdma_cm_id *cm_id; + + if (conn == NULL) + return; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " destroy_conn: conn %p id %d\n", conn, conn->cm_id); + + dapl_os_lock(&conn->lock); + conn->destroy = 1; + + if (ep != NULL) { + ep->cm_handle = NULL; + ep->qp_handle = NULL; + ep->qp_state = IB_QP_STATE_ERROR; + } + + cm_id = conn->cm_id; + conn->cm_id = NULL; + dapl_os_unlock(&conn->lock); + + /* + * rdma_destroy_id will force synchronization with async CM event + * thread since it blocks until the in-process event reference + * is cleared during our event processing call exit. + */ + if (cm_id) { + if (cm_id->qp) + rdma_destroy_qp(cm_id); + + rdma_destroy_id(cm_id); + } + dapl_os_free(conn, sizeof(*conn)); +} + +static struct dapl_cm_id *dapli_req_recv(struct dapl_cm_id *conn, + struct rdma_cm_event *event) +{ + struct dapl_cm_id *new_conn; +#ifdef DAPL_DBG + struct rdma_addr *ipaddr = &event->id->route.addr; +#endif + + if (conn->sp == NULL) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " dapli_rep_recv: on invalid listen " "handle\n"); + return NULL; + } + + /* allocate new cm_id and merge listen parameters */ + new_conn = dapl_os_alloc(sizeof(*new_conn)); + if (new_conn) { + (void)dapl_os_memzero(new_conn, sizeof(*new_conn)); + dapl_os_lock_init(&new_conn->lock); + new_conn->cm_id = event->id; /* provided by uCMA */ + event->id->context = new_conn; /* update CM_ID context */ + new_conn->sp = conn->sp; + new_conn->hca = conn->hca; + + /* Get requesters connect data, setup for accept */ + new_conn->params.responder_resources = + DAPL_MIN(event->param.conn.responder_resources, + conn->hca->ib_trans.rd_atom_in); + new_conn->params.initiator_depth = + DAPL_MIN(event->param.conn.initiator_depth, + conn->hca->ib_trans.rd_atom_out); + + new_conn->params.flow_control = event->param.conn.flow_control; + new_conn->params.rnr_retry_count = + event->param.conn.rnr_retry_count; + new_conn->params.retry_count = event->param.conn.retry_count; + + /* save private data */ + if (event->param.conn.private_data_len) { + dapl_os_memcpy(new_conn->p_data, + event->param.conn.private_data, + event->param.conn.private_data_len); + new_conn->params.private_data = new_conn->p_data; + new_conn->params.private_data_len = + event->param.conn.private_data_len; + } + + dapl_dbg_log(DAPL_DBG_TYPE_CM, " passive_cb: " + "REQ: SP %p PORT %d LID %d " + "NEW CONN %p ID %p pdata %p,%d\n", + new_conn->sp, ntohs(((struct sockaddr_in *) + &ipaddr->src_addr)->sin_port), + event->listen_id, new_conn, event->id, + event->param.conn.private_data, + event->param.conn.private_data_len); + + dapl_dbg_log(DAPL_DBG_TYPE_CM, " passive_cb: " + "REQ: IP SRC %x PORT %d DST %x PORT %d " + "rr %d init %d\n", ntohl(((struct sockaddr_in *) + &ipaddr->src_addr)-> + sin_addr.s_addr), + ntohs(((struct sockaddr_in *) + &ipaddr->src_addr)->sin_port), + ntohl(((struct sockaddr_in *) + &ipaddr->dst_addr)->sin_addr.s_addr), + ntohs(((struct sockaddr_in *) + &ipaddr->dst_addr)->sin_port), + new_conn->params.responder_resources, + new_conn->params.initiator_depth); + } + return new_conn; +} + +static void dapli_cm_active_cb(struct dapl_cm_id *conn, + struct rdma_cm_event *event) +{ + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " active_cb: conn %p id %d event %d\n", + conn, conn->cm_id, event->event); + + dapl_os_lock(&conn->lock); + if (conn->destroy) { + dapl_os_unlock(&conn->lock); + return; + } + dapl_os_unlock(&conn->lock); + + /* There is a chance that we can get events after + * the consumer calls disconnect in a pending state + * since the IB CM and uDAPL states are not shared. + * In some cases, IB CM could generate either a DCONN + * or CONN_ERR after the consumer returned from + * dapl_ep_disconnect with a DISCONNECTED event + * already queued. Check state here and bail to + * avoid any events after a disconnect. + */ + if (DAPL_BAD_HANDLE(conn->ep, DAPL_MAGIC_EP)) + return; + + dapl_os_lock(&conn->ep->header.lock); + if (conn->ep->param.ep_state == DAT_EP_STATE_DISCONNECTED) { + dapl_os_unlock(&conn->ep->header.lock); + return; + } + if (event->event == RDMA_CM_EVENT_DISCONNECTED) + conn->ep->param.ep_state = DAT_EP_STATE_DISCONNECTED; + + dapl_os_unlock(&conn->ep->header.lock); + + switch (event->event) { + case RDMA_CM_EVENT_UNREACHABLE: + case RDMA_CM_EVENT_CONNECT_ERROR: + { + dapl_log(DAPL_DBG_TYPE_WARN, + "dapl_cma_active: CONN_ERR event=0x%x" + " status=%d %s DST %s, %d\n", + event->event, event->status, + (event->status == -ETIMEDOUT) ? "TIMEOUT" : "", + inet_ntoa(((struct sockaddr_in *) + &conn->cm_id->route.addr.dst_addr)-> + sin_addr), + ntohs(((struct sockaddr_in *) + &conn->cm_id->route.addr.dst_addr)-> + sin_port)); + + /* per DAT SPEC provider always returns UNREACHABLE */ + dapl_evd_connection_callback(conn, + IB_CME_DESTINATION_UNREACHABLE, + NULL, conn->ep); + break; + } + case RDMA_CM_EVENT_REJECTED: + { + ib_cm_events_t cm_event; + unsigned char *pdata = NULL; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " dapli_cm_active_handler: REJECTED reason=%d\n", + event->status); + + /* valid REJ from consumer will always contain private data */ + if (event->status == 28 && + event->param.conn.private_data_len) { + cm_event = + IB_CME_DESTINATION_REJECT_PRIVATE_DATA; + pdata = + (unsigned char *)event->param.conn. + private_data + + sizeof(struct dapl_pdata_hdr); + } else { + cm_event = IB_CME_DESTINATION_REJECT; + dapl_log(DAPL_DBG_TYPE_WARN, + "dapl_cma_active: non-consumer REJ," + " reason=%d, DST %s, %d\n", + event->status, + inet_ntoa(((struct sockaddr_in *) + &conn->cm_id->route.addr. + dst_addr)->sin_addr), + ntohs(((struct sockaddr_in *) + &conn->cm_id->route.addr. + dst_addr)->sin_port)); + } + dapl_evd_connection_callback(conn, cm_event, pdata, + conn->ep); + break; + } + case RDMA_CM_EVENT_ESTABLISHED: + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " active_cb: cm_id %d PORT %d CONNECTED to %s!\n", + conn->cm_id, ntohs(((struct sockaddr_in *) + &conn->cm_id->route.addr. + dst_addr)->sin_port), + inet_ntoa(((struct sockaddr_in *) + &conn->cm_id->route.addr.dst_addr)-> + sin_addr)); + + /* setup local and remote ports for ep query */ + conn->ep->param.remote_port_qual = + PORT_TO_SID(rdma_get_dst_port(conn->cm_id)); + conn->ep->param.local_port_qual = + PORT_TO_SID(rdma_get_src_port(conn->cm_id)); + + dapl_evd_connection_callback(conn, IB_CME_CONNECTED, + event->param.conn.private_data, + conn->ep); + break; + + case RDMA_CM_EVENT_DISCONNECTED: + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " active_cb: DISC EVENT - EP %p\n",conn->ep); + rdma_disconnect(conn->cm_id); /* required for DREP */ + /* validate EP handle */ + if (!DAPL_BAD_HANDLE(conn->ep, DAPL_MAGIC_EP)) + dapl_evd_connection_callback(conn, + IB_CME_DISCONNECTED, + NULL, conn->ep); + break; + default: + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " dapli_cm_active_cb_handler: Unexpected CM " + "event %d on ID 0x%p\n", event->event, + conn->cm_id); + break; + } + + return; +} + +static void dapli_cm_passive_cb(struct dapl_cm_id *conn, + struct rdma_cm_event *event) +{ + struct dapl_cm_id *new_conn; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " passive_cb: conn %p id %d event %d\n", + conn, event->id, event->event); + + dapl_os_lock(&conn->lock); + if (conn->destroy) { + dapl_os_unlock(&conn->lock); + return; + } + dapl_os_unlock(&conn->lock); + + switch (event->event) { + case RDMA_CM_EVENT_CONNECT_REQUEST: + /* create new conn object with new conn_id from event */ + new_conn = dapli_req_recv(conn, event); + + if (new_conn) + dapls_cr_callback(new_conn, + IB_CME_CONNECTION_REQUEST_PENDING, + event->param.conn.private_data, + new_conn->sp); + break; + case RDMA_CM_EVENT_UNREACHABLE: + case RDMA_CM_EVENT_CONNECT_ERROR: + dapl_log(DAPL_DBG_TYPE_WARN, + "dapl_cm_passive: CONN_ERR event=0x%x status=%d %s," + " DST %s,%d\n", + event->event, event->status, + (event->status == -ETIMEDOUT) ? "TIMEOUT" : "", + inet_ntoa(((struct sockaddr_in *) + &conn->cm_id->route.addr.dst_addr)-> + sin_addr), ntohs(((struct sockaddr_in *) + &conn->cm_id->route.addr. + dst_addr)->sin_port)); + + dapls_cr_callback(conn, IB_CME_DESTINATION_UNREACHABLE, + NULL, conn->sp); + break; + + case RDMA_CM_EVENT_REJECTED: + { + /* will alwasys be abnormal NON-consumer from active side */ + dapl_log(DAPL_DBG_TYPE_WARN, + "dapl_cm_passive: non-consumer REJ, reason=%d," + " DST %s, %d\n", + event->status, + inet_ntoa(((struct sockaddr_in *) + &conn->cm_id->route.addr.dst_addr)-> + sin_addr), + ntohs(((struct sockaddr_in *) + &conn->cm_id->route.addr.dst_addr)-> + sin_port)); + + dapls_cr_callback(conn, IB_CME_DESTINATION_REJECT, + NULL, conn->sp); + break; + } + case RDMA_CM_EVENT_ESTABLISHED: + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " passive_cb: cm_id %p PORT %d CONNECTED from 0x%x!\n", + conn->cm_id, ntohs(((struct sockaddr_in *) + &conn->cm_id->route.addr. + src_addr)->sin_port), + ntohl(((struct sockaddr_in *) + &conn->cm_id->route.addr.dst_addr)-> + sin_addr.s_addr)); + + dapls_cr_callback(conn, IB_CME_CONNECTED, NULL, conn->sp); + + break; + case RDMA_CM_EVENT_DISCONNECTED: + rdma_disconnect(conn->cm_id); /* required for DREP */ + /* validate SP handle context */ + if (!DAPL_BAD_HANDLE(conn->sp, DAPL_MAGIC_PSP) || + !DAPL_BAD_HANDLE(conn->sp, DAPL_MAGIC_RSP)) + dapls_cr_callback(conn, + IB_CME_DISCONNECTED, NULL, conn->sp); + break; + default: + dapl_dbg_log(DAPL_DBG_TYPE_ERR, " passive_cb: " + "Unexpected CM event %d on ID 0x%p\n", + event->event, conn->cm_id); + break; + } + + return; +} + +/************************ DAPL provider entry points **********************/ + +/* + * dapls_ib_connect + * + * Initiate a connection with the passive listener on another node + * + * Input: + * ep_handle, + * remote_ia_address, + * remote_conn_qual, + * prd_size size of private data and structure + * prd_prt pointer to private data structure + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle, + IN DAT_IA_ADDRESS_PTR r_addr, + IN DAT_CONN_QUAL r_qual, + IN DAT_COUNT p_size, IN void *p_data) +{ + struct dapl_ep *ep_ptr = ep_handle; + struct dapl_cm_id *conn = ep_ptr->cm_handle; + int ret; + + /* Sanity check */ + if (NULL == ep_ptr) + return DAT_SUCCESS; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " connect: rSID 0x%llx rPort %d, pdata %p, ln %d\n", + r_qual, ntohs(SID_TO_PORT(r_qual)), p_data, p_size); + + /* rdma conn and cm_id pre-bound; reference via ep_ptr->cm_handle */ + + /* Setup QP/CM parameters and private data in cm_id */ + (void)dapl_os_memzero(&conn->params, sizeof(conn->params)); + conn->params.responder_resources = + ep_ptr->param.ep_attr.max_rdma_read_in; + conn->params.initiator_depth = ep_ptr->param.ep_attr.max_rdma_read_out; + conn->params.flow_control = 1; + conn->params.rnr_retry_count = IB_RNR_RETRY_COUNT; + conn->params.retry_count = IB_RC_RETRY_COUNT; + if (p_size) { + dapl_os_memcpy(conn->p_data, p_data, p_size); + conn->params.private_data = conn->p_data; + conn->params.private_data_len = p_size; + } + + /* copy in remote address, need a copy for retry attempts */ + dapl_os_memcpy(&conn->r_addr, r_addr, sizeof(*r_addr)); + + /* Resolve remote address, src already bound during QP create */ + ((struct sockaddr_in *)&conn->r_addr)->sin_port = SID_TO_PORT(r_qual); + ((struct sockaddr_in *)&conn->r_addr)->sin_family = AF_INET; + + ret = rdma_resolve_addr(conn->cm_id, NULL, + (struct sockaddr *)&conn->r_addr, + conn->arp_timeout); + if (ret) { + dapl_log(DAPL_DBG_TYPE_ERR, + " dapl_cma_connect: rdma_resolve_addr ERR 0x%x %s\n", + ret, strerror(errno)); + return dapl_convert_errno(errno, "ib_connect"); + } + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " connect: resolve_addr: cm_id %p -> %s port %d\n", + conn->cm_id, + inet_ntoa(((struct sockaddr_in *)&conn->r_addr)->sin_addr), + ((struct sockaddr_in *)&conn->r_addr)->sin_port); + + return DAT_SUCCESS; +} + +/* + * dapls_ib_disconnect + * + * Disconnect an EP + * + * Input: + * ep_handle, + * disconnect_flags + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * + */ +DAT_RETURN +dapls_ib_disconnect(IN DAPL_EP * ep_ptr, IN DAT_CLOSE_FLAGS close_flags) +{ + dp_ib_cm_handle_t conn = ep_ptr->cm_handle; + int ret; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " disconnect(ep %p, conn %p, id %d flags %x)\n", + ep_ptr, conn, (conn ? conn->cm_id : 0), close_flags); + + if ((conn == IB_INVALID_HANDLE) || (conn->cm_id == NULL)) + return DAT_SUCCESS; + + /* no graceful half-pipe disconnect option */ + ret = rdma_disconnect(conn->cm_id); + if (ret) + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " disconnect: ID %p ret 0x%x\n", + ep_ptr->cm_handle, ret); + + /* + * DAT event notification occurs from the callback + * Note: will fire even if DREQ goes unanswered on timeout + */ + return DAT_SUCCESS; +} + +/* + * dapls_ib_disconnect_clean + * + * Clean up outstanding connection data. This routine is invoked + * after the final disconnect callback has occurred. Only on the + * ACTIVE side of a connection. + * + * Input: + * ep_ptr DAPL_EP + * active Indicates active side of connection + * + * Output: + * none + * + * Returns: + * void + * + */ +void +dapls_ib_disconnect_clean(IN DAPL_EP * ep_ptr, + IN DAT_BOOLEAN active, + IN const ib_cm_events_t ib_cm_event) +{ + /* nothing to do */ + return; +} + +/* + * dapl_ib_setup_conn_listener + * + * Have the CM set up a connection listener. + * + * Input: + * ibm_hca_handle HCA handle + * qp_handle QP handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INTERNAL_ERROR + * DAT_CONN_QUAL_UNAVAILBLE + * DAT_CONN_QUAL_IN_USE + * + */ +DAT_RETURN +dapls_ib_setup_conn_listener(IN DAPL_IA * ia_ptr, + IN DAT_UINT64 ServiceID, IN DAPL_SP * sp_ptr) +{ + DAT_RETURN dat_status = DAT_SUCCESS; + ib_cm_srvc_handle_t conn; + DAT_SOCK_ADDR6 addr; /* local binding address */ + + /* Allocate CM and initialize lock */ + if ((conn = dapl_os_alloc(sizeof(*conn))) == NULL) + return DAT_INSUFFICIENT_RESOURCES; + + dapl_os_memzero(conn, sizeof(*conn)); + dapl_os_lock_init(&conn->lock); + + /* create CM_ID, bind to local device, create QP */ + if (rdma_create_id + (g_cm_events, &conn->cm_id, (void *)conn, RDMA_PS_TCP)) { + dapl_os_free(conn, sizeof(*conn)); + return (dapl_convert_errno(errno, "setup_listener")); + } + + /* open identifies the local device; per DAT specification */ + /* Get family and address then set port to consumer's ServiceID */ + dapl_os_memcpy(&addr, &ia_ptr->hca_ptr->hca_address, sizeof(addr)); + ((struct sockaddr_in *)&addr)->sin_port = SID_TO_PORT(ServiceID); + + if (rdma_bind_addr(conn->cm_id, (struct sockaddr *)&addr)) { + if ((errno == EBUSY) || (errno == EADDRINUSE)) + dat_status = DAT_CONN_QUAL_IN_USE; + else + dat_status = + dapl_convert_errno(errno, "setup_listener"); + goto bail; + } + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " listen(ia_ptr %p SID 0x%llx Port %d sp %p conn %p id %d)\n", + ia_ptr, ServiceID, ntohs(SID_TO_PORT(ServiceID)), + sp_ptr, conn, conn->cm_id); + + sp_ptr->cm_srvc_handle = conn; + conn->sp = sp_ptr; + conn->hca = ia_ptr->hca_ptr; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " listen(conn=%p cm_id=%d)\n", + sp_ptr->cm_srvc_handle, conn->cm_id); + + if (rdma_listen(conn->cm_id, 0)) { /* max cma backlog */ + + if ((errno == EBUSY) || (errno == EADDRINUSE)) + dat_status = DAT_CONN_QUAL_IN_USE; + else + dat_status = + dapl_convert_errno(errno, "setup_listener"); + goto bail; + } + + /* success */ + return DAT_SUCCESS; + + bail: + rdma_destroy_id(conn->cm_id); + dapl_os_free(conn, sizeof(*conn)); + return dat_status; +} + +/* + * dapl_ib_remove_conn_listener + * + * Have the CM remove a connection listener. + * + * Input: + * ia_handle IA handle + * ServiceID IB Channel Service ID + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_STATE + * + */ +DAT_RETURN +dapls_ib_remove_conn_listener(IN DAPL_IA * ia_ptr, IN DAPL_SP * sp_ptr) +{ + ib_cm_srvc_handle_t conn = sp_ptr->cm_srvc_handle; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " remove_listen(ia_ptr %p sp_ptr %p cm_ptr %p)\n", + ia_ptr, sp_ptr, conn); + + if (conn != IB_INVALID_HANDLE) { + sp_ptr->cm_srvc_handle = NULL; + dapls_ib_cm_free(conn, NULL); + } + return DAT_SUCCESS; +} + +/* + * dapls_ib_accept_connection + * + * Perform necessary steps to accept a connection + * + * Input: + * cr_handle + * ep_handle + * private_data_size + * private_data + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INTERNAL_ERROR + * + */ +DAT_RETURN +dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle, + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT p_size, IN const DAT_PVOID p_data) +{ + DAPL_CR *cr_ptr = (DAPL_CR *) cr_handle; + DAPL_EP *ep_ptr = (DAPL_EP *) ep_handle; + DAPL_IA *ia_ptr = ep_ptr->header.owner_ia; + struct dapl_cm_id *cr_conn = cr_ptr->ib_cm_handle; + int ret; + DAT_RETURN dat_status; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " accept(cr %p conn %p, id %p, p_data %p, p_sz=%d)\n", + cr_ptr, cr_conn, cr_conn->cm_id, p_data, p_size); + + /* Obtain size of private data structure & contents */ + if (p_size > IB_MAX_REP_PDATA_SIZE) { + dat_status = DAT_ERROR(DAT_LENGTH_ERROR, DAT_NO_SUBTYPE); + goto bail; + } + + if (ep_ptr->qp_state == DAPL_QP_STATE_UNATTACHED) { + /* + * If we are lazy attaching the QP then we may need to + * hook it up here. Typically, we run this code only for + * DAT_PSP_PROVIDER_FLAG + */ + dat_status = dapls_ib_qp_alloc(ia_ptr, ep_ptr, NULL); + if (dat_status != DAT_SUCCESS) { + dapl_log(DAPL_DBG_TYPE_ERR, + " dapl_cma_accept: qp_alloc ERR %d\n", + dat_status); + goto bail; + } + } + + /* + * Validate device and port in EP cm_id against inbound + * CR cm_id. The pre-allocated EP cm_id is already bound to + * a local device (cm_id and QP) when created. Move the QP + * to the new cm_id only if device and port numbers match. + */ + if (ep_ptr->cm_handle->cm_id->verbs == cr_conn->cm_id->verbs && + ep_ptr->cm_handle->cm_id->port_num == cr_conn->cm_id->port_num) { + /* move QP to new cr_conn, remove QP ref in EP cm_id */ + cr_conn->cm_id->qp = ep_ptr->cm_handle->cm_id->qp; + ep_ptr->cm_handle->cm_id->qp = NULL; + dapls_ib_cm_free(ep_ptr->cm_handle, NULL); + } else { + dapl_log(DAPL_DBG_TYPE_ERR, + " dapl_cma_accept: ERR dev(%p!=%p) or" + " port mismatch(%d!=%d)\n", + ep_ptr->cm_handle->cm_id->verbs, cr_conn->cm_id->verbs, + ntohs(ep_ptr->cm_handle->cm_id->port_num), + ntohs(cr_conn->cm_id->port_num)); + dat_status = DAT_INTERNAL_ERROR; + goto bail; + } + + cr_ptr->param.local_ep_handle = ep_handle; + cr_conn->params.private_data = p_data; + cr_conn->params.private_data_len = p_size; + + ret = rdma_accept(cr_conn->cm_id, &cr_conn->params); + if (ret) { + dapl_log(DAPL_DBG_TYPE_ERR, " dapl_cma_accept: ERR %d %s\n", + ret, strerror(errno)); + dat_status = dapl_convert_errno(ret, "accept"); + goto bail; + } + + /* save accepted conn and EP reference, qp_handle unchanged */ + ep_ptr->cm_handle = cr_conn; + cr_conn->ep = ep_ptr; + + /* setup local and remote ports for ep query */ + /* Note: port qual in network order */ + ep_ptr->param.remote_port_qual = + PORT_TO_SID(rdma_get_dst_port(cr_conn->cm_id)); + ep_ptr->param.local_port_qual = + PORT_TO_SID(rdma_get_src_port(cr_conn->cm_id)); + + return DAT_SUCCESS; + bail: + rdma_reject(cr_conn->cm_id, NULL, 0); + dapls_ib_cm_free(cr_conn, NULL); + return dat_status; +} + +/* + * dapls_ib_reject_connection + * + * Reject a connection + * + * Input: + * cr_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INTERNAL_ERROR + * + */ +DAT_RETURN +dapls_ib_reject_connection(IN dp_ib_cm_handle_t cm_handle, + IN int reason, + IN DAT_COUNT private_data_size, + IN const DAT_PVOID private_data) +{ + int ret; + int offset = sizeof(struct dapl_pdata_hdr); + struct dapl_pdata_hdr pdata_hdr; + + memset(&pdata_hdr, 0, sizeof pdata_hdr); + pdata_hdr.version = htonl((DAT_VERSION_MAJOR << 24) | + (DAT_VERSION_MINOR << 16) | + (VN_PROVIDER_MAJOR << 8) | + (VN_PROVIDER_MINOR)); + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " reject: handle %p reason %x, ver=%x, data %p, sz=%d\n", + cm_handle, reason, ntohl(pdata_hdr.version), + private_data, private_data_size); + + if (cm_handle == IB_INVALID_HANDLE) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " reject: invalid handle: reason %d\n", reason); + return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CR); + } + + if (private_data_size > + dapls_ib_private_data_size(NULL, DAPL_PDATA_CONN_REJ, + cm_handle->hca)) + return DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); + + /* setup pdata_hdr and users data, in CR pdata buffer */ + dapl_os_memcpy(cm_handle->p_data, &pdata_hdr, offset); + if (private_data_size) + dapl_os_memcpy(cm_handle->p_data + offset, + private_data, private_data_size); + + /* + * Always some private data with reject so active peer can + * determine real application reject from an abnormal + * application termination + */ + ret = rdma_reject(cm_handle->cm_id, + cm_handle->p_data, offset + private_data_size); + + dapls_ib_cm_free(cm_handle, NULL); + return dapl_convert_errno(ret, "reject"); +} + +/* + * dapls_ib_cm_remote_addr + * + * Obtain the remote IP address given a connection + * + * Input: + * cr_handle + * + * Output: + * remote_ia_address: where to place the remote address + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * + */ +DAT_RETURN +dapls_ib_cm_remote_addr(IN DAT_HANDLE dat_handle, OUT DAT_SOCK_ADDR6 * raddr) +{ + DAPL_HEADER *header; + dp_ib_cm_handle_t ib_cm_handle; + struct rdma_addr *ipaddr; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " remote_addr(cm_handle=%p, r_addr=%p)\n", + dat_handle, raddr); + + header = (DAPL_HEADER *) dat_handle; + + if (header->magic == DAPL_MAGIC_EP) + ib_cm_handle = ((DAPL_EP *) dat_handle)->cm_handle; + else if (header->magic == DAPL_MAGIC_CR) + ib_cm_handle = ((DAPL_CR *) dat_handle)->ib_cm_handle; + else + return DAT_INVALID_HANDLE; + + /* get remote IP address from cm_id route */ + ipaddr = &ib_cm_handle->cm_id->route.addr; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " remote_addr: conn %p id %p SRC %x DST %x PORT %d\n", + ib_cm_handle, ib_cm_handle->cm_id, + ntohl(((struct sockaddr_in *) + &ipaddr->src_addr)->sin_addr.s_addr), + ntohl(((struct sockaddr_in *) + &ipaddr->dst_addr)->sin_addr.s_addr), + ntohs(((struct sockaddr_in *) + &ipaddr->dst_addr)->sin_port)); + + dapl_os_memcpy(raddr, &ipaddr->dst_addr, sizeof(DAT_SOCK_ADDR)); + return DAT_SUCCESS; +} + +/* + * dapls_ib_private_data_size + * + * Return the size of private data given a connection op type + * + * Input: + * prd_ptr private data pointer + * conn_op connection operation type + * hca_ptr hca pointer, needed for transport type + * + * If prd_ptr is NULL, this is a query for the max size supported by + * the provider, otherwise it is the actual size of the private data + * contained in prd_ptr. + * + * + * Output: + * None + * + * Returns: + * length of private data + * + */ +int dapls_ib_private_data_size(IN DAPL_PRIVATE * prd_ptr, + IN DAPL_PDATA_OP conn_op, IN DAPL_HCA * hca_ptr) +{ + int size; + + if (hca_ptr->ib_hca_handle->device->transport_type + == IBV_TRANSPORT_IWARP) + return (IWARP_MAX_PDATA_SIZE - sizeof(struct dapl_pdata_hdr)); + + switch (conn_op) { + + case DAPL_PDATA_CONN_REQ: + size = IB_MAX_REQ_PDATA_SIZE; + break; + case DAPL_PDATA_CONN_REP: + size = IB_MAX_REP_PDATA_SIZE; + break; + case DAPL_PDATA_CONN_REJ: + size = IB_MAX_REJ_PDATA_SIZE - sizeof(struct dapl_pdata_hdr); + break; + case DAPL_PDATA_CONN_DREQ: + size = IB_MAX_DREQ_PDATA_SIZE; + break; + case DAPL_PDATA_CONN_DREP: + size = IB_MAX_DREP_PDATA_SIZE; + break; + default: + size = 0; + + } /* end case */ + + return size; +} + +/* + * Map all CMA event codes to the DAT equivelent. + */ +#define DAPL_IB_EVENT_CNT 13 + +static struct ib_cm_event_map { + const ib_cm_events_t ib_cm_event; + DAT_EVENT_NUMBER dat_event_num; +} ib_cm_event_map[DAPL_IB_EVENT_CNT] = { + /* 00 */ { + IB_CME_CONNECTED, DAT_CONNECTION_EVENT_ESTABLISHED}, + /* 01 */ { + IB_CME_DISCONNECTED, DAT_CONNECTION_EVENT_DISCONNECTED}, + /* 02 */ { + IB_CME_DISCONNECTED_ON_LINK_DOWN, + DAT_CONNECTION_EVENT_DISCONNECTED}, + /* 03 */ { + IB_CME_CONNECTION_REQUEST_PENDING, DAT_CONNECTION_REQUEST_EVENT}, + /* 04 */ { + IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA, + DAT_CONNECTION_REQUEST_EVENT}, + /* 05 */ { + IB_CME_CONNECTION_REQUEST_ACKED, DAT_CONNECTION_REQUEST_EVENT}, + /* 06 */ { + IB_CME_DESTINATION_REJECT, + DAT_CONNECTION_EVENT_NON_PEER_REJECTED}, + /* 07 */ { + IB_CME_DESTINATION_REJECT_PRIVATE_DATA, + DAT_CONNECTION_EVENT_PEER_REJECTED}, + /* 08 */ { + IB_CME_DESTINATION_UNREACHABLE, DAT_CONNECTION_EVENT_UNREACHABLE}, + /* 09 */ { + IB_CME_TOO_MANY_CONNECTION_REQUESTS, + DAT_CONNECTION_EVENT_NON_PEER_REJECTED}, + /* 10 */ { + IB_CME_LOCAL_FAILURE, DAT_CONNECTION_EVENT_BROKEN}, + /* 11 */ { + IB_CME_BROKEN, DAT_CONNECTION_EVENT_BROKEN}, + /* 12 */ { +IB_CME_TIMEOUT, DAT_CONNECTION_EVENT_TIMED_OUT},}; + +/* + * dapls_ib_get_cm_event + * + * Return a DAT connection event given a provider CM event. + * + * Input: + * dat_event_num DAT event we need an equivelent CM event for + * + * Output: + * none + * + * Returns: + * ib_cm_event of translated DAPL value + */ +DAT_EVENT_NUMBER +dapls_ib_get_dat_event(IN const ib_cm_events_t ib_cm_event, + IN DAT_BOOLEAN active) +{ + DAT_EVENT_NUMBER dat_event_num; + int i; + + active = active; + + dat_event_num = 0; + for (i = 0; i < DAPL_IB_EVENT_CNT; i++) { + if (ib_cm_event == ib_cm_event_map[i].ib_cm_event) { + dat_event_num = ib_cm_event_map[i].dat_event_num; + break; + } + } + dapl_dbg_log(DAPL_DBG_TYPE_CALLBACK, + "dapls_ib_get_dat_event: event(%s) ib=0x%x dat=0x%x\n", + active ? "active" : "passive", ib_cm_event, dat_event_num); + + return dat_event_num; +} + +/* + * dapls_ib_get_dat_event + * + * Return a DAT connection event given a provider CM event. + * + * Input: + * ib_cm_event event provided to the dapl callback routine + * active switch indicating active or passive connection + * + * Output: + * none + * + * Returns: + * DAT_EVENT_NUMBER of translated provider value + */ +ib_cm_events_t dapls_ib_get_cm_event(IN DAT_EVENT_NUMBER dat_event_num) +{ + ib_cm_events_t ib_cm_event; + int i; + + ib_cm_event = 0; + for (i = 0; i < DAPL_IB_EVENT_CNT; i++) { + if (dat_event_num == ib_cm_event_map[i].dat_event_num) { + ib_cm_event = ib_cm_event_map[i].ib_cm_event; + break; + } + } + return ib_cm_event; +} + +void dapli_cma_event_cb(void) +{ + struct rdma_cm_event *event; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cm_event()\n"); + + /* process one CM event, fairness */ + if (!rdma_get_cm_event(g_cm_events, &event)) { + struct dapl_cm_id *conn; + + /* set proper conn from cm_id context */ + if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) + conn = (struct dapl_cm_id *)event->listen_id->context; + else + conn = (struct dapl_cm_id *)event->id->context; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " cm_event: EVENT=%d ID=%p LID=%p CTX=%p\n", + event->event, event->id, event->listen_id, conn); + + switch (event->event) { + case RDMA_CM_EVENT_ADDR_RESOLVED: + dapli_addr_resolve(conn); + break; + + case RDMA_CM_EVENT_ROUTE_RESOLVED: + dapli_route_resolve(conn); + break; + + case RDMA_CM_EVENT_ADDR_ERROR: + dapl_log(DAPL_DBG_TYPE_WARN, + "dapl_cma_active: CM ADDR ERROR: ->" + " DST %s retry (%d)..\n", + inet_ntoa(((struct sockaddr_in *) + &conn->r_addr)->sin_addr), + conn->arp_retries); + + /* retry address resolution */ + if ((--conn->arp_retries) && + (event->status == -ETIMEDOUT)) { + int ret; + ret = rdma_resolve_addr(conn->cm_id, NULL, + (struct sockaddr *) + &conn->r_addr, + conn->arp_timeout); + if (!ret) + break; + else { + dapl_dbg_log(DAPL_DBG_TYPE_WARN, + " ERROR: rdma_resolve_addr = " + "%d %s\n", + ret, strerror(errno)); + } + } + /* retries exhausted or resolve_addr failed */ + dapl_log(DAPL_DBG_TYPE_ERR, + "dapl_cma_active: ARP_ERR, retries(%d)" + " exhausted -> DST %s,%d\n", + IB_ARP_RETRY_COUNT, + inet_ntoa(((struct sockaddr_in *) + &conn->cm_id->route.addr.dst_addr)-> + sin_addr), + ntohs(((struct sockaddr_in *) + &conn->cm_id->route.addr.dst_addr)-> + sin_port)); + + dapl_evd_connection_callback(conn, + IB_CME_DESTINATION_UNREACHABLE, + NULL, conn->ep); + break; + + case RDMA_CM_EVENT_ROUTE_ERROR: + dapl_log(DAPL_DBG_TYPE_WARN, + "dapl_cma_active: CM ROUTE ERROR: ->" + " DST %s retry (%d)..\n", + inet_ntoa(((struct sockaddr_in *) + &conn->r_addr)->sin_addr), + conn->route_retries); + + /* retry route resolution */ + if ((--conn->route_retries) && + (event->status == -ETIMEDOUT)) + dapli_addr_resolve(conn); + else { + dapl_log(DAPL_DBG_TYPE_ERR, + "dapl_cma_active: PATH_RECORD_ERR," + " retries(%d) exhausted, DST %s,%d\n", + IB_ROUTE_RETRY_COUNT, + inet_ntoa(((struct sockaddr_in *) + &conn->cm_id->route.addr. + dst_addr)->sin_addr), + ntohs(((struct sockaddr_in *) + &conn->cm_id->route.addr. + dst_addr)->sin_port)); + + dapl_evd_connection_callback(conn, + IB_CME_DESTINATION_UNREACHABLE, + NULL, conn->ep); + } + break; + + case RDMA_CM_EVENT_DEVICE_REMOVAL: + dapl_evd_connection_callback(conn, + IB_CME_LOCAL_FAILURE, + NULL, conn->ep); + break; + case RDMA_CM_EVENT_CONNECT_REQUEST: + case RDMA_CM_EVENT_CONNECT_ERROR: + case RDMA_CM_EVENT_UNREACHABLE: + case RDMA_CM_EVENT_REJECTED: + case RDMA_CM_EVENT_ESTABLISHED: + case RDMA_CM_EVENT_DISCONNECTED: + /* passive or active */ + if (conn->sp) + dapli_cm_passive_cb(conn, event); + else + dapli_cm_active_cb(conn, event); + break; + case RDMA_CM_EVENT_CONNECT_RESPONSE: + default: + dapl_dbg_log(DAPL_DBG_TYPE_WARN, + " cm_event: UNEXPECTED EVENT=%p ID=%p CTX=%p\n", + event->event, event->id, + event->id->context); + break; + } + /* ack event, unblocks destroy_cm_id in consumer threads */ + rdma_ack_cm_event(event); + } +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/dapl/openib_cma/dapl_ib_cm.c b/dapl/openib_cma/dapl_ib_cm.c deleted file mode 100755 index 946cfbd..0000000 --- a/dapl/openib_cma/dapl_ib_cm.c +++ /dev/null @@ -1,1297 +0,0 @@ -/* - * Copyright (c) 2005 Voltaire Inc. All rights reserved. - * Copyright (c) 2005-2007 Intel Corporation. All rights reserved. - * Copyright (c) 2004-2005, Mellanox Technologies, Inc. All rights reserved. - * Copyright (c) 2003 Topspin Corporation. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/********************************************************************** - * - * MODULE: dapl_ib_cm.c - * - * PURPOSE: The OFED provider - uCMA, name and route resolution - * - * $Id: $ - * - **********************************************************************/ - -#include "dapl.h" -#include "dapl_adapter_util.h" -#include "dapl_evd_util.h" -#include "dapl_cr_util.h" -#include "dapl_name_service.h" -#include "dapl_ib_util.h" -#include "dapl_vendor.h" -#include "dapl_osd.h" - -extern struct rdma_event_channel *g_cm_events; - -/* local prototypes */ -static struct dapl_cm_id *dapli_req_recv(struct dapl_cm_id *conn, - struct rdma_cm_event *event); -static void dapli_cm_active_cb(struct dapl_cm_id *conn, - struct rdma_cm_event *event); -static void dapli_cm_passive_cb(struct dapl_cm_id *conn, - struct rdma_cm_event *event); -static void dapli_addr_resolve(struct dapl_cm_id *conn); -static void dapli_route_resolve(struct dapl_cm_id *conn); - -/* cma requires 16 bit SID, in network order */ -#define IB_PORT_MOD 32001 -#define IB_PORT_BASE (65535 - IB_PORT_MOD) -#define SID_TO_PORT(SID) \ - (SID > 0xffff ? \ - htons((unsigned short)((SID % IB_PORT_MOD) + IB_PORT_BASE)) :\ - htons((unsigned short)SID)) - -#define PORT_TO_SID(p) ntohs(p) - -/* private data header to validate consumer rejects versus abnormal events */ -struct dapl_pdata_hdr { - DAT_UINT32 version; -}; - -static void dapli_addr_resolve(struct dapl_cm_id *conn) -{ - int ret; -#ifdef DAPL_DBG - struct rdma_addr *ipaddr = &conn->cm_id->route.addr; -#endif - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " addr_resolve: cm_id %p SRC %x DST %x\n", - conn->cm_id, ntohl(((struct sockaddr_in *) - &ipaddr->src_addr)->sin_addr.s_addr), - ntohl(((struct sockaddr_in *) - &ipaddr->dst_addr)->sin_addr.s_addr)); - - ret = rdma_resolve_route(conn->cm_id, conn->route_timeout); - if (ret) { - dapl_log(DAPL_DBG_TYPE_ERR, - " dapl_cma_connect: rdma_resolve_route ERR 0x%x %s\n", - ret, strerror(errno)); - dapl_evd_connection_callback(conn, - IB_CME_LOCAL_FAILURE, - NULL, conn->ep); - } -} - -static void dapli_route_resolve(struct dapl_cm_id *conn) -{ - int ret; -#ifdef DAPL_DBG - struct rdma_addr *ipaddr = &conn->cm_id->route.addr; - struct ib_addr *ibaddr = &conn->cm_id->route.addr.addr.ibaddr; -#endif - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " route_resolve: cm_id %p SRC %x DST %x PORT %d\n", - conn->cm_id, ntohl(((struct sockaddr_in *) - &ipaddr->src_addr)->sin_addr.s_addr), - ntohl(((struct sockaddr_in *) - &ipaddr->dst_addr)->sin_addr.s_addr), - ntohs(((struct sockaddr_in *) - &ipaddr->dst_addr)->sin_port)); - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " route_resolve: SRC GID subnet %016llx id %016llx\n", - (unsigned long long) - ntohll(ibaddr->sgid.global.subnet_prefix), - (unsigned long long) - ntohll(ibaddr->sgid.global.interface_id)); - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " route_resolve: DST GID subnet %016llx id %016llx\n", - (unsigned long long) - ntohll(ibaddr->dgid.global.subnet_prefix), - (unsigned long long) - ntohll(ibaddr->dgid.global.interface_id)); - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " route_resolve: cm_id %p pdata %p plen %d rr %d ind %d\n", - conn->cm_id, - conn->params.private_data, - conn->params.private_data_len, - conn->params.responder_resources, - conn->params.initiator_depth); - - ret = rdma_connect(conn->cm_id, &conn->params); - if (ret) { - dapl_log(DAPL_DBG_TYPE_ERR, - " dapl_cma_connect: rdma_connect ERR %d %s\n", - ret, strerror(errno)); - goto bail; - } - return; - - bail: - dapl_evd_connection_callback(conn, - IB_CME_LOCAL_FAILURE, NULL, conn->ep); -} - -/* - * Called from consumer thread via dat_ep_free(). - * CANNOT be called from the async event processing thread - * dapli_cma_event_cb() since a cm_id reference is held and - * a deadlock will occur. - */ -void dapli_destroy_conn(struct dapl_cm_id *conn) -{ - struct rdma_cm_id *cm_id; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " destroy_conn: conn %p id %d\n", conn, conn->cm_id); - - dapl_os_lock(&conn->lock); - conn->destroy = 1; - - if (conn->ep) { - conn->ep->cm_handle = IB_INVALID_HANDLE; - conn->ep->qp_handle = IB_INVALID_HANDLE; - } - - cm_id = conn->cm_id; - conn->cm_id = NULL; - dapl_os_unlock(&conn->lock); - - /* - * rdma_destroy_id will force synchronization with async CM event - * thread since it blocks until the in-process event reference - * is cleared during our event processing call exit. - */ - if (cm_id) { - if (cm_id->qp) - rdma_destroy_qp(cm_id); - - rdma_destroy_id(cm_id); - } - dapl_os_free(conn, sizeof(*conn)); -} - -static struct dapl_cm_id *dapli_req_recv(struct dapl_cm_id *conn, - struct rdma_cm_event *event) -{ - struct dapl_cm_id *new_conn; -#ifdef DAPL_DBG - struct rdma_addr *ipaddr = &event->id->route.addr; -#endif - - if (conn->sp == NULL) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " dapli_rep_recv: on invalid listen " "handle\n"); - return NULL; - } - - /* allocate new cm_id and merge listen parameters */ - new_conn = dapl_os_alloc(sizeof(*new_conn)); - if (new_conn) { - (void)dapl_os_memzero(new_conn, sizeof(*new_conn)); - dapl_os_lock_init(&new_conn->lock); - new_conn->cm_id = event->id; /* provided by uCMA */ - event->id->context = new_conn; /* update CM_ID context */ - new_conn->sp = conn->sp; - new_conn->hca = conn->hca; - - /* Get requesters connect data, setup for accept */ - new_conn->params.responder_resources = - DAPL_MIN(event->param.conn.responder_resources, - conn->hca->ib_trans.max_rdma_rd_in); - new_conn->params.initiator_depth = - DAPL_MIN(event->param.conn.initiator_depth, - conn->hca->ib_trans.max_rdma_rd_out); - - new_conn->params.flow_control = event->param.conn.flow_control; - new_conn->params.rnr_retry_count = - event->param.conn.rnr_retry_count; - new_conn->params.retry_count = event->param.conn.retry_count; - - /* save private data */ - if (event->param.conn.private_data_len) { - dapl_os_memcpy(new_conn->p_data, - event->param.conn.private_data, - event->param.conn.private_data_len); - new_conn->params.private_data = new_conn->p_data; - new_conn->params.private_data_len = - event->param.conn.private_data_len; - } - - dapl_dbg_log(DAPL_DBG_TYPE_CM, " passive_cb: " - "REQ: SP %p PORT %d LID %d " - "NEW CONN %p ID %p pD %p,%d\n", - new_conn->sp, ntohs(((struct sockaddr_in *) - &ipaddr->src_addr)->sin_port), - event->listen_id, new_conn, event->id, - event->param.conn.private_data, - event->param.conn.private_data_len); - - dapl_dbg_log(DAPL_DBG_TYPE_CM, " passive_cb: " - "REQ: IP SRC %x PORT %d DST %x PORT %d " - "rr %d init %d\n", ntohl(((struct sockaddr_in *) - &ipaddr->src_addr)-> - sin_addr.s_addr), - ntohs(((struct sockaddr_in *) - &ipaddr->src_addr)->sin_port), - ntohl(((struct sockaddr_in *) - &ipaddr->dst_addr)->sin_addr.s_addr), - ntohs(((struct sockaddr_in *) - &ipaddr->dst_addr)->sin_port), - new_conn->params.responder_resources, - new_conn->params.initiator_depth); - } - return new_conn; -} - -static void dapli_cm_active_cb(struct dapl_cm_id *conn, - struct rdma_cm_event *event) -{ - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " active_cb: conn %p id %d event %d\n", - conn, conn->cm_id, event->event); - - dapl_os_lock(&conn->lock); - if (conn->destroy) { - dapl_os_unlock(&conn->lock); - return; - } - dapl_os_unlock(&conn->lock); - - /* There is a chance that we can get events after - * the consumer calls disconnect in a pending state - * since the IB CM and uDAPL states are not shared. - * In some cases, IB CM could generate either a DCONN - * or CONN_ERR after the consumer returned from - * dapl_ep_disconnect with a DISCONNECTED event - * already queued. Check state here and bail to - * avoid any events after a disconnect. - */ - if (DAPL_BAD_HANDLE(conn->ep, DAPL_MAGIC_EP)) - return; - - dapl_os_lock(&conn->ep->header.lock); - if (conn->ep->param.ep_state == DAT_EP_STATE_DISCONNECTED) { - dapl_os_unlock(&conn->ep->header.lock); - return; - } - if (event->event == RDMA_CM_EVENT_DISCONNECTED) - conn->ep->param.ep_state = DAT_EP_STATE_DISCONNECTED; - - dapl_os_unlock(&conn->ep->header.lock); - - switch (event->event) { - case RDMA_CM_EVENT_UNREACHABLE: - case RDMA_CM_EVENT_CONNECT_ERROR: - { - dapl_log(DAPL_DBG_TYPE_WARN, - "dapl_cma_active: CONN_ERR event=0x%x" - " status=%d %s DST %s, %d\n", - event->event, event->status, - (event->status == -ETIMEDOUT) ? "TIMEOUT" : "", - inet_ntoa(((struct sockaddr_in *) - &conn->cm_id->route.addr.dst_addr)-> - sin_addr), - ntohs(((struct sockaddr_in *) - &conn->cm_id->route.addr.dst_addr)-> - sin_port)); - - /* per DAT SPEC provider always returns UNREACHABLE */ - dapl_evd_connection_callback(conn, - IB_CME_DESTINATION_UNREACHABLE, - NULL, conn->ep); - break; - } - case RDMA_CM_EVENT_REJECTED: - { - ib_cm_events_t cm_event; - unsigned char *pdata = NULL; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " dapli_cm_active_handler: REJECTED reason=%d\n", - event->status); - - /* valid REJ from consumer will always contain private data */ - if (event->status == 28 && - event->param.conn.private_data_len) { - cm_event = - IB_CME_DESTINATION_REJECT_PRIVATE_DATA; - pdata = - (unsigned char *)event->param.conn. - private_data + - sizeof(struct dapl_pdata_hdr); - } else { - cm_event = IB_CME_DESTINATION_REJECT; - dapl_log(DAPL_DBG_TYPE_WARN, - "dapl_cma_active: non-consumer REJ," - " reason=%d, DST %s, %d\n", - event->status, - inet_ntoa(((struct sockaddr_in *) - &conn->cm_id->route.addr. - dst_addr)->sin_addr), - ntohs(((struct sockaddr_in *) - &conn->cm_id->route.addr. - dst_addr)->sin_port)); - } - dapl_evd_connection_callback(conn, cm_event, pdata, - conn->ep); - break; - } - case RDMA_CM_EVENT_ESTABLISHED: - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " active_cb: cm_id %d PORT %d CONNECTED to %s!\n", - conn->cm_id, ntohs(((struct sockaddr_in *) - &conn->cm_id->route.addr. - dst_addr)->sin_port), - inet_ntoa(((struct sockaddr_in *) - &conn->cm_id->route.addr.dst_addr)-> - sin_addr)); - - /* setup local and remote ports for ep query */ - conn->ep->param.remote_port_qual = - PORT_TO_SID(rdma_get_dst_port(conn->cm_id)); - conn->ep->param.local_port_qual = - PORT_TO_SID(rdma_get_src_port(conn->cm_id)); - - dapl_evd_connection_callback(conn, IB_CME_CONNECTED, - event->param.conn.private_data, - conn->ep); - break; - - case RDMA_CM_EVENT_DISCONNECTED: - rdma_disconnect(conn->cm_id); /* required for DREP */ - /* validate EP handle */ - if (!DAPL_BAD_HANDLE(conn->ep, DAPL_MAGIC_EP)) - dapl_evd_connection_callback(conn, - IB_CME_DISCONNECTED, - NULL, conn->ep); - break; - default: - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " dapli_cm_active_cb_handler: Unexpected CM " - "event %d on ID 0x%p\n", event->event, - conn->cm_id); - break; - } - - return; -} - -static void dapli_cm_passive_cb(struct dapl_cm_id *conn, - struct rdma_cm_event *event) -{ - struct dapl_cm_id *new_conn; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " passive_cb: conn %p id %d event %d\n", - conn, event->id, event->event); - - dapl_os_lock(&conn->lock); - if (conn->destroy) { - dapl_os_unlock(&conn->lock); - return; - } - dapl_os_unlock(&conn->lock); - - switch (event->event) { - case RDMA_CM_EVENT_CONNECT_REQUEST: - /* create new conn object with new conn_id from event */ - new_conn = dapli_req_recv(conn, event); - - if (new_conn) - dapls_cr_callback(new_conn, - IB_CME_CONNECTION_REQUEST_PENDING, - event->param.conn.private_data, - new_conn->sp); - break; - case RDMA_CM_EVENT_UNREACHABLE: - case RDMA_CM_EVENT_CONNECT_ERROR: - dapl_log(DAPL_DBG_TYPE_WARN, - "dapl_cm_passive: CONN_ERR event=0x%x status=%d %s," - " DST %s,%d\n", - event->event, event->status, - (event->status == -ETIMEDOUT) ? "TIMEOUT" : "", - inet_ntoa(((struct sockaddr_in *) - &conn->cm_id->route.addr.dst_addr)-> - sin_addr), ntohs(((struct sockaddr_in *) - &conn->cm_id->route.addr. - dst_addr)->sin_port)); - - dapls_cr_callback(conn, IB_CME_DESTINATION_UNREACHABLE, - NULL, conn->sp); - break; - - case RDMA_CM_EVENT_REJECTED: - { - /* will alwasys be abnormal NON-consumer from active side */ - dapl_log(DAPL_DBG_TYPE_WARN, - "dapl_cm_passive: non-consumer REJ, reason=%d," - " DST %s, %d\n", - event->status, - inet_ntoa(((struct sockaddr_in *) - &conn->cm_id->route.addr.dst_addr)-> - sin_addr), - ntohs(((struct sockaddr_in *) - &conn->cm_id->route.addr.dst_addr)-> - sin_port)); - - dapls_cr_callback(conn, IB_CME_DESTINATION_REJECT, - NULL, conn->sp); - break; - } - case RDMA_CM_EVENT_ESTABLISHED: - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " passive_cb: cm_id %p PORT %d CONNECTED from 0x%x!\n", - conn->cm_id, ntohs(((struct sockaddr_in *) - &conn->cm_id->route.addr. - src_addr)->sin_port), - ntohl(((struct sockaddr_in *) - &conn->cm_id->route.addr.dst_addr)-> - sin_addr.s_addr)); - - dapls_cr_callback(conn, IB_CME_CONNECTED, NULL, conn->sp); - - break; - case RDMA_CM_EVENT_DISCONNECTED: - rdma_disconnect(conn->cm_id); /* required for DREP */ - /* validate SP handle context */ - if (!DAPL_BAD_HANDLE(conn->sp, DAPL_MAGIC_PSP) || - !DAPL_BAD_HANDLE(conn->sp, DAPL_MAGIC_RSP)) - dapls_cr_callback(conn, - IB_CME_DISCONNECTED, NULL, conn->sp); - break; - default: - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " passive_cb: " - "Unexpected CM event %d on ID 0x%p\n", - event->event, conn->cm_id); - break; - } - - return; -} - -/************************ DAPL provider entry points **********************/ - -/* - * dapls_ib_connect - * - * Initiate a connection with the passive listener on another node - * - * Input: - * ep_handle, - * remote_ia_address, - * remote_conn_qual, - * prd_size size of private data and structure - * prd_prt pointer to private data structure - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INVALID_PARAMETER - * - */ -DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle, - IN DAT_IA_ADDRESS_PTR r_addr, - IN DAT_CONN_QUAL r_qual, - IN DAT_COUNT p_size, IN void *p_data) -{ - struct dapl_ep *ep_ptr = ep_handle; - struct dapl_cm_id *conn; - int ret; - - /* Sanity check */ - if (NULL == ep_ptr) - return DAT_SUCCESS; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " connect: rSID 0x%llx rPort %d, pdata %p, ln %d\n", - r_qual, ntohs(SID_TO_PORT(r_qual)), p_data, p_size); - - /* rdma conn and cm_id pre-bound; reference via qp_handle */ - conn = ep_ptr->cm_handle = ep_ptr->qp_handle; - - /* Setup QP/CM parameters and private data in cm_id */ - (void)dapl_os_memzero(&conn->params, sizeof(conn->params)); - conn->params.responder_resources = - ep_ptr->param.ep_attr.max_rdma_read_in; - conn->params.initiator_depth = ep_ptr->param.ep_attr.max_rdma_read_out; - conn->params.flow_control = 1; - conn->params.rnr_retry_count = IB_RNR_RETRY_COUNT; - conn->params.retry_count = IB_RC_RETRY_COUNT; - if (p_size) { - dapl_os_memcpy(conn->p_data, p_data, p_size); - conn->params.private_data = conn->p_data; - conn->params.private_data_len = p_size; - } - - /* copy in remote address, need a copy for retry attempts */ - dapl_os_memcpy(&conn->r_addr, r_addr, sizeof(*r_addr)); - - /* Resolve remote address, src already bound during QP create */ - ((struct sockaddr_in *)&conn->r_addr)->sin_port = SID_TO_PORT(r_qual); - ((struct sockaddr_in *)&conn->r_addr)->sin_family = AF_INET; - - ret = rdma_resolve_addr(conn->cm_id, NULL, - (struct sockaddr *)&conn->r_addr, - conn->arp_timeout); - if (ret) { - dapl_log(DAPL_DBG_TYPE_ERR, - " dapl_cma_connect: rdma_resolve_addr ERR 0x%x %s\n", - ret, strerror(errno)); - return dapl_convert_errno(errno, "ib_connect"); - } - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " connect: resolve_addr: cm_id %p -> %s port %d\n", - conn->cm_id, - inet_ntoa(((struct sockaddr_in *)&conn->r_addr)->sin_addr), - ((struct sockaddr_in *)&conn->r_addr)->sin_port); - - return DAT_SUCCESS; -} - -/* - * dapls_ib_disconnect - * - * Disconnect an EP - * - * Input: - * ep_handle, - * disconnect_flags - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * - */ -DAT_RETURN -dapls_ib_disconnect(IN DAPL_EP * ep_ptr, IN DAT_CLOSE_FLAGS close_flags) -{ - dp_ib_cm_handle_t conn = ep_ptr->cm_handle; - int ret; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " disconnect(ep %p, conn %p, id %d flags %x)\n", - ep_ptr, conn, (conn ? conn->cm_id : 0), close_flags); - - if ((conn == IB_INVALID_HANDLE) || (conn->cm_id == NULL)) - return DAT_SUCCESS; - - /* no graceful half-pipe disconnect option */ - ret = rdma_disconnect(conn->cm_id); - if (ret) - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " disconnect: ID %p ret 0x%x\n", - ep_ptr->cm_handle, ret); - - /* - * DAT event notification occurs from the callback - * Note: will fire even if DREQ goes unanswered on timeout - */ - return DAT_SUCCESS; -} - -/* - * dapls_ib_disconnect_clean - * - * Clean up outstanding connection data. This routine is invoked - * after the final disconnect callback has occurred. Only on the - * ACTIVE side of a connection. - * - * Input: - * ep_ptr DAPL_EP - * active Indicates active side of connection - * - * Output: - * none - * - * Returns: - * void - * - */ -void -dapls_ib_disconnect_clean(IN DAPL_EP * ep_ptr, - IN DAT_BOOLEAN active, - IN const ib_cm_events_t ib_cm_event) -{ - /* nothing to do */ - return; -} - -/* - * dapl_ib_setup_conn_listener - * - * Have the CM set up a connection listener. - * - * Input: - * ibm_hca_handle HCA handle - * qp_handle QP handle - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INTERNAL_ERROR - * DAT_CONN_QUAL_UNAVAILBLE - * DAT_CONN_QUAL_IN_USE - * - */ -DAT_RETURN -dapls_ib_setup_conn_listener(IN DAPL_IA * ia_ptr, - IN DAT_UINT64 ServiceID, IN DAPL_SP * sp_ptr) -{ - DAT_RETURN dat_status = DAT_SUCCESS; - ib_cm_srvc_handle_t conn; - DAT_SOCK_ADDR6 addr; /* local binding address */ - - /* Allocate CM and initialize lock */ - if ((conn = dapl_os_alloc(sizeof(*conn))) == NULL) - return DAT_INSUFFICIENT_RESOURCES; - - dapl_os_memzero(conn, sizeof(*conn)); - dapl_os_lock_init(&conn->lock); - - /* create CM_ID, bind to local device, create QP */ - if (rdma_create_id - (g_cm_events, &conn->cm_id, (void *)conn, RDMA_PS_TCP)) { - dapl_os_free(conn, sizeof(*conn)); - return (dapl_convert_errno(errno, "setup_listener")); - } - - /* open identifies the local device; per DAT specification */ - /* Get family and address then set port to consumer's ServiceID */ - dapl_os_memcpy(&addr, &ia_ptr->hca_ptr->hca_address, sizeof(addr)); - ((struct sockaddr_in *)&addr)->sin_port = SID_TO_PORT(ServiceID); - - if (rdma_bind_addr(conn->cm_id, (struct sockaddr *)&addr)) { - if ((errno == EBUSY) || (errno == EADDRINUSE)) - dat_status = DAT_CONN_QUAL_IN_USE; - else - dat_status = - dapl_convert_errno(errno, "setup_listener"); - goto bail; - } - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " listen(ia_ptr %p SID 0x%llx Port %d sp %p conn %p id %d)\n", - ia_ptr, ServiceID, ntohs(SID_TO_PORT(ServiceID)), - sp_ptr, conn, conn->cm_id); - - sp_ptr->cm_srvc_handle = conn; - conn->sp = sp_ptr; - conn->hca = ia_ptr->hca_ptr; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " listen(conn=%p cm_id=%d)\n", - sp_ptr->cm_srvc_handle, conn->cm_id); - - if (rdma_listen(conn->cm_id, 0)) { /* max cma backlog */ - - if ((errno == EBUSY) || (errno == EADDRINUSE)) - dat_status = DAT_CONN_QUAL_IN_USE; - else - dat_status = - dapl_convert_errno(errno, "setup_listener"); - goto bail; - } - - /* success */ - return DAT_SUCCESS; - - bail: - rdma_destroy_id(conn->cm_id); - dapl_os_free(conn, sizeof(*conn)); - return dat_status; -} - -/* - * dapl_ib_remove_conn_listener - * - * Have the CM remove a connection listener. - * - * Input: - * ia_handle IA handle - * ServiceID IB Channel Service ID - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_STATE - * - */ -DAT_RETURN -dapls_ib_remove_conn_listener(IN DAPL_IA * ia_ptr, IN DAPL_SP * sp_ptr) -{ - ib_cm_srvc_handle_t conn = sp_ptr->cm_srvc_handle; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " remove_listen(ia_ptr %p sp_ptr %p cm_ptr %p)\n", - ia_ptr, sp_ptr, conn); - - if (conn != IB_INVALID_HANDLE) { - sp_ptr->cm_srvc_handle = NULL; - dapli_destroy_conn(conn); - } - return DAT_SUCCESS; -} - -/* - * dapls_ib_accept_connection - * - * Perform necessary steps to accept a connection - * - * Input: - * cr_handle - * ep_handle - * private_data_size - * private_data - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INTERNAL_ERROR - * - */ -DAT_RETURN -dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle, - IN DAT_EP_HANDLE ep_handle, - IN DAT_COUNT p_size, IN const DAT_PVOID p_data) -{ - DAPL_CR *cr_ptr = (DAPL_CR *) cr_handle; - DAPL_EP *ep_ptr = (DAPL_EP *) ep_handle; - DAPL_IA *ia_ptr = ep_ptr->header.owner_ia; - struct dapl_cm_id *cr_conn = cr_ptr->ib_cm_handle; - int ret; - DAT_RETURN dat_status; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " accept(cr %p conn %p, id %p, p_data %p, p_sz=%d)\n", - cr_ptr, cr_conn, cr_conn->cm_id, p_data, p_size); - - /* Obtain size of private data structure & contents */ - if (p_size > IB_MAX_REP_PDATA_SIZE) { - dat_status = DAT_ERROR(DAT_LENGTH_ERROR, DAT_NO_SUBTYPE); - goto bail; - } - - if (ep_ptr->qp_state == DAPL_QP_STATE_UNATTACHED) { - /* - * If we are lazy attaching the QP then we may need to - * hook it up here. Typically, we run this code only for - * DAT_PSP_PROVIDER_FLAG - */ - dat_status = dapls_ib_qp_alloc(ia_ptr, ep_ptr, NULL); - if (dat_status != DAT_SUCCESS) { - dapl_log(DAPL_DBG_TYPE_ERR, - " dapl_cma_accept: qp_alloc ERR %d\n", - dat_status); - goto bail; - } - } - - /* - * Validate device and port in EP cm_id against inbound - * CR cm_id. The pre-allocated EP cm_id is already bound to - * a local device (cm_id and QP) when created. Move the QP - * to the new cm_id only if device and port numbers match. - */ - if (ep_ptr->qp_handle->cm_id->verbs == cr_conn->cm_id->verbs && - ep_ptr->qp_handle->cm_id->port_num == cr_conn->cm_id->port_num) { - /* move QP to new cr_conn, remove QP ref in EP cm_id */ - cr_conn->cm_id->qp = ep_ptr->qp_handle->cm_id->qp; - ep_ptr->qp_handle->cm_id->qp = NULL; - dapli_destroy_conn(ep_ptr->qp_handle); - } else { - dapl_log(DAPL_DBG_TYPE_ERR, - " dapl_cma_accept: ERR dev(%p!=%p) or" - " port mismatch(%d!=%d)\n", - ep_ptr->qp_handle->cm_id->verbs, cr_conn->cm_id->verbs, - ntohs(ep_ptr->qp_handle->cm_id->port_num), - ntohs(cr_conn->cm_id->port_num)); - dat_status = DAT_INTERNAL_ERROR; - goto bail; - } - - cr_ptr->param.local_ep_handle = ep_handle; - cr_conn->params.private_data = p_data; - cr_conn->params.private_data_len = p_size; - - ret = rdma_accept(cr_conn->cm_id, &cr_conn->params); - if (ret) { - dapl_log(DAPL_DBG_TYPE_ERR, " dapl_cma_accept: ERR %d %s\n", - ret, strerror(errno)); - dat_status = dapl_convert_errno(ret, "accept"); - goto bail; - } - - /* save accepted conn and EP reference */ - ep_ptr->qp_handle = cr_conn; - ep_ptr->cm_handle = cr_conn; - cr_conn->ep = ep_ptr; - - /* setup local and remote ports for ep query */ - /* Note: port qual in network order */ - ep_ptr->param.remote_port_qual = - PORT_TO_SID(rdma_get_dst_port(cr_conn->cm_id)); - ep_ptr->param.local_port_qual = - PORT_TO_SID(rdma_get_src_port(cr_conn->cm_id)); - - return DAT_SUCCESS; - bail: - rdma_reject(cr_conn->cm_id, NULL, 0); - dapli_destroy_conn(cr_conn); - return dat_status; -} - -/* - * dapls_ib_reject_connection - * - * Reject a connection - * - * Input: - * cr_handle - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INTERNAL_ERROR - * - */ -DAT_RETURN -dapls_ib_reject_connection(IN dp_ib_cm_handle_t cm_handle, - IN int reason, - IN DAT_COUNT private_data_size, - IN const DAT_PVOID private_data) -{ - int ret; - int offset = sizeof(struct dapl_pdata_hdr); - struct dapl_pdata_hdr pdata_hdr; - - memset(&pdata_hdr, 0, sizeof pdata_hdr); - pdata_hdr.version = htonl((DAT_VERSION_MAJOR << 24) | - (DAT_VERSION_MINOR << 16) | - (VN_PROVIDER_MAJOR << 8) | - (VN_PROVIDER_MINOR)); - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " reject: handle %p reason %x, ver=%x, data %p, sz=%d\n", - cm_handle, reason, ntohl(pdata_hdr.version), - private_data, private_data_size); - - if (cm_handle == IB_INVALID_HANDLE) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " reject: invalid handle: reason %d\n", reason); - return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_CR); - } - - if (private_data_size > - dapls_ib_private_data_size(NULL, DAPL_PDATA_CONN_REJ, - cm_handle->hca)) - return DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG3); - - /* setup pdata_hdr and users data, in CR pdata buffer */ - dapl_os_memcpy(cm_handle->p_data, &pdata_hdr, offset); - if (private_data_size) - dapl_os_memcpy(cm_handle->p_data + offset, - private_data, private_data_size); - - /* - * Always some private data with reject so active peer can - * determine real application reject from an abnormal - * application termination - */ - ret = rdma_reject(cm_handle->cm_id, - cm_handle->p_data, offset + private_data_size); - - dapli_destroy_conn(cm_handle); - return dapl_convert_errno(ret, "reject"); -} - -/* - * dapls_ib_cm_remote_addr - * - * Obtain the remote IP address given a connection - * - * Input: - * cr_handle - * - * Output: - * remote_ia_address: where to place the remote address - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_HANDLE - * - */ -DAT_RETURN -dapls_ib_cm_remote_addr(IN DAT_HANDLE dat_handle, OUT DAT_SOCK_ADDR6 * raddr) -{ - DAPL_HEADER *header; - dp_ib_cm_handle_t ib_cm_handle; - struct rdma_addr *ipaddr; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " remote_addr(cm_handle=%p, r_addr=%p)\n", - dat_handle, raddr); - - header = (DAPL_HEADER *) dat_handle; - - if (header->magic == DAPL_MAGIC_EP) - ib_cm_handle = ((DAPL_EP *) dat_handle)->cm_handle; - else if (header->magic == DAPL_MAGIC_CR) - ib_cm_handle = ((DAPL_CR *) dat_handle)->ib_cm_handle; - else - return DAT_INVALID_HANDLE; - - /* get remote IP address from cm_id route */ - ipaddr = &ib_cm_handle->cm_id->route.addr; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " remote_addr: conn %p id %p SRC %x DST %x PORT %d\n", - ib_cm_handle, ib_cm_handle->cm_id, - ntohl(((struct sockaddr_in *) - &ipaddr->src_addr)->sin_addr.s_addr), - ntohl(((struct sockaddr_in *) - &ipaddr->dst_addr)->sin_addr.s_addr), - ntohs(((struct sockaddr_in *) - &ipaddr->dst_addr)->sin_port)); - - dapl_os_memcpy(raddr, &ipaddr->dst_addr, sizeof(DAT_SOCK_ADDR)); - return DAT_SUCCESS; -} - -/* - * dapls_ib_private_data_size - * - * Return the size of private data given a connection op type - * - * Input: - * prd_ptr private data pointer - * conn_op connection operation type - * hca_ptr hca pointer, needed for transport type - * - * If prd_ptr is NULL, this is a query for the max size supported by - * the provider, otherwise it is the actual size of the private data - * contained in prd_ptr. - * - * - * Output: - * None - * - * Returns: - * length of private data - * - */ -int dapls_ib_private_data_size(IN DAPL_PRIVATE * prd_ptr, - IN DAPL_PDATA_OP conn_op, IN DAPL_HCA * hca_ptr) -{ - int size; - - if (hca_ptr->ib_hca_handle->device->transport_type - == IBV_TRANSPORT_IWARP) - return (IWARP_MAX_PDATA_SIZE - sizeof(struct dapl_pdata_hdr)); - - switch (conn_op) { - - case DAPL_PDATA_CONN_REQ: - size = IB_MAX_REQ_PDATA_SIZE; - break; - case DAPL_PDATA_CONN_REP: - size = IB_MAX_REP_PDATA_SIZE; - break; - case DAPL_PDATA_CONN_REJ: - size = IB_MAX_REJ_PDATA_SIZE - sizeof(struct dapl_pdata_hdr); - break; - case DAPL_PDATA_CONN_DREQ: - size = IB_MAX_DREQ_PDATA_SIZE; - break; - case DAPL_PDATA_CONN_DREP: - size = IB_MAX_DREP_PDATA_SIZE; - break; - default: - size = 0; - - } /* end case */ - - return size; -} - -/* - * Map all CMA event codes to the DAT equivelent. - */ -#define DAPL_IB_EVENT_CNT 13 - -static struct ib_cm_event_map { - const ib_cm_events_t ib_cm_event; - DAT_EVENT_NUMBER dat_event_num; -} ib_cm_event_map[DAPL_IB_EVENT_CNT] = { - /* 00 */ { - IB_CME_CONNECTED, DAT_CONNECTION_EVENT_ESTABLISHED}, - /* 01 */ { - IB_CME_DISCONNECTED, DAT_CONNECTION_EVENT_DISCONNECTED}, - /* 02 */ { - IB_CME_DISCONNECTED_ON_LINK_DOWN, - DAT_CONNECTION_EVENT_DISCONNECTED}, - /* 03 */ { - IB_CME_CONNECTION_REQUEST_PENDING, DAT_CONNECTION_REQUEST_EVENT}, - /* 04 */ { - IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA, - DAT_CONNECTION_REQUEST_EVENT}, - /* 05 */ { - IB_CME_CONNECTION_REQUEST_ACKED, DAT_CONNECTION_REQUEST_EVENT}, - /* 06 */ { - IB_CME_DESTINATION_REJECT, - DAT_CONNECTION_EVENT_NON_PEER_REJECTED}, - /* 07 */ { - IB_CME_DESTINATION_REJECT_PRIVATE_DATA, - DAT_CONNECTION_EVENT_PEER_REJECTED}, - /* 08 */ { - IB_CME_DESTINATION_UNREACHABLE, DAT_CONNECTION_EVENT_UNREACHABLE}, - /* 09 */ { - IB_CME_TOO_MANY_CONNECTION_REQUESTS, - DAT_CONNECTION_EVENT_NON_PEER_REJECTED}, - /* 10 */ { - IB_CME_LOCAL_FAILURE, DAT_CONNECTION_EVENT_BROKEN}, - /* 11 */ { - IB_CME_BROKEN, DAT_CONNECTION_EVENT_BROKEN}, - /* 12 */ { -IB_CME_TIMEOUT, DAT_CONNECTION_EVENT_TIMED_OUT},}; - -/* - * dapls_ib_get_cm_event - * - * Return a DAT connection event given a provider CM event. - * - * Input: - * dat_event_num DAT event we need an equivelent CM event for - * - * Output: - * none - * - * Returns: - * ib_cm_event of translated DAPL value - */ -DAT_EVENT_NUMBER -dapls_ib_get_dat_event(IN const ib_cm_events_t ib_cm_event, - IN DAT_BOOLEAN active) -{ - DAT_EVENT_NUMBER dat_event_num; - int i; - - active = active; - - dat_event_num = 0; - for (i = 0; i < DAPL_IB_EVENT_CNT; i++) { - if (ib_cm_event == ib_cm_event_map[i].ib_cm_event) { - dat_event_num = ib_cm_event_map[i].dat_event_num; - break; - } - } - dapl_dbg_log(DAPL_DBG_TYPE_CALLBACK, - "dapls_ib_get_dat_event: event(%s) ib=0x%x dat=0x%x\n", - active ? "active" : "passive", ib_cm_event, dat_event_num); - - return dat_event_num; -} - -/* - * dapls_ib_get_dat_event - * - * Return a DAT connection event given a provider CM event. - * - * Input: - * ib_cm_event event provided to the dapl callback routine - * active switch indicating active or passive connection - * - * Output: - * none - * - * Returns: - * DAT_EVENT_NUMBER of translated provider value - */ -ib_cm_events_t dapls_ib_get_cm_event(IN DAT_EVENT_NUMBER dat_event_num) -{ - ib_cm_events_t ib_cm_event; - int i; - - ib_cm_event = 0; - for (i = 0; i < DAPL_IB_EVENT_CNT; i++) { - if (dat_event_num == ib_cm_event_map[i].dat_event_num) { - ib_cm_event = ib_cm_event_map[i].ib_cm_event; - break; - } - } - return ib_cm_event; -} - -void dapli_cma_event_cb(void) -{ - struct rdma_cm_event *event; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cm_event()\n"); - - /* process one CM event, fairness */ - if (!rdma_get_cm_event(g_cm_events, &event)) { - struct dapl_cm_id *conn; - - /* set proper conn from cm_id context */ - if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) - conn = (struct dapl_cm_id *)event->listen_id->context; - else - conn = (struct dapl_cm_id *)event->id->context; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " cm_event: EVENT=%d ID=%p LID=%p CTX=%p\n", - event->event, event->id, event->listen_id, conn); - - switch (event->event) { - case RDMA_CM_EVENT_ADDR_RESOLVED: - dapli_addr_resolve(conn); - break; - - case RDMA_CM_EVENT_ROUTE_RESOLVED: - dapli_route_resolve(conn); - break; - - case RDMA_CM_EVENT_ADDR_ERROR: - dapl_log(DAPL_DBG_TYPE_WARN, - "dapl_cma_active: CM ADDR ERROR: ->" - " DST %s retry (%d)..\n", - inet_ntoa(((struct sockaddr_in *) - &conn->r_addr)->sin_addr), - conn->arp_retries); - - /* retry address resolution */ - if ((--conn->arp_retries) && - (event->status == -ETIMEDOUT)) { - int ret; - ret = rdma_resolve_addr(conn->cm_id, NULL, - (struct sockaddr *) - &conn->r_addr, - conn->arp_timeout); - if (!ret) - break; - else { - dapl_dbg_log(DAPL_DBG_TYPE_WARN, - " ERROR: rdma_resolve_addr = " - "%d %s\n", - ret, strerror(errno)); - } - } - /* retries exhausted or resolve_addr failed */ - dapl_log(DAPL_DBG_TYPE_ERR, - "dapl_cma_active: ARP_ERR, retries(%d)" - " exhausted -> DST %s,%d\n", - IB_ARP_RETRY_COUNT, - inet_ntoa(((struct sockaddr_in *) - &conn->cm_id->route.addr.dst_addr)-> - sin_addr), - ntohs(((struct sockaddr_in *) - &conn->cm_id->route.addr.dst_addr)-> - sin_port)); - - dapl_evd_connection_callback(conn, - IB_CME_DESTINATION_UNREACHABLE, - NULL, conn->ep); - break; - - case RDMA_CM_EVENT_ROUTE_ERROR: - dapl_log(DAPL_DBG_TYPE_WARN, - "dapl_cma_active: CM ROUTE ERROR: ->" - " DST %s retry (%d)..\n", - inet_ntoa(((struct sockaddr_in *) - &conn->r_addr)->sin_addr), - conn->route_retries); - - /* retry route resolution */ - if ((--conn->route_retries) && - (event->status == -ETIMEDOUT)) - dapli_addr_resolve(conn); - else { - dapl_log(DAPL_DBG_TYPE_ERR, - "dapl_cma_active: PATH_RECORD_ERR," - " retries(%d) exhausted, DST %s,%d\n", - IB_ROUTE_RETRY_COUNT, - inet_ntoa(((struct sockaddr_in *) - &conn->cm_id->route.addr. - dst_addr)->sin_addr), - ntohs(((struct sockaddr_in *) - &conn->cm_id->route.addr. - dst_addr)->sin_port)); - - dapl_evd_connection_callback(conn, - IB_CME_DESTINATION_UNREACHABLE, - NULL, conn->ep); - } - break; - - case RDMA_CM_EVENT_DEVICE_REMOVAL: - dapl_evd_connection_callback(conn, - IB_CME_LOCAL_FAILURE, - NULL, conn->ep); - break; - case RDMA_CM_EVENT_CONNECT_REQUEST: - case RDMA_CM_EVENT_CONNECT_ERROR: - case RDMA_CM_EVENT_UNREACHABLE: - case RDMA_CM_EVENT_REJECTED: - case RDMA_CM_EVENT_ESTABLISHED: - case RDMA_CM_EVENT_DISCONNECTED: - /* passive or active */ - if (conn->sp) - dapli_cm_passive_cb(conn, event); - else - dapli_cm_active_cb(conn, event); - break; - case RDMA_CM_EVENT_CONNECT_RESPONSE: - default: - dapl_dbg_log(DAPL_DBG_TYPE_WARN, - " cm_event: UNEXPECTED EVENT=%p ID=%p CTX=%p\n", - event->event, event->id, - event->id->context); - break; - } - /* ack event, unblocks destroy_cm_id in consumer threads */ - rdma_ack_cm_event(event); - } -} - -/* - * Local variables: - * c-indent-level: 4 - * c-basic-offset: 4 - * tab-width: 8 - * End: - */ diff --git a/dapl/openib_cma/dapl_ib_cq.c b/dapl/openib_cma/dapl_ib_cq.c deleted file mode 100755 index 7f67982..0000000 --- a/dapl/openib_cma/dapl_ib_cq.c +++ /dev/null @@ -1,559 +0,0 @@ -/* - * Copyright (c) 2005-2007 Intel Corporation. All rights reserved. - * - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/********************************************************************** - * - * MODULE: dapl_ib_cq.c - * - * PURPOSE: completion queues for OFED IB Verbs - * - * $Id: $ - * - **********************************************************************/ - -#include "openib_osd.h" -#include "dapl.h" -#include "dapl_adapter_util.h" -#include "dapl_lmr_util.h" -#include "dapl_evd_util.h" -#include "dapl_ring_buffer_util.h" - -/* One CQ event channel per HCA */ -void dapli_cq_event_cb(struct _ib_hca_transport *hca) -{ - /* check all comp events on this device */ - struct dapl_evd *evd_ptr = NULL; - struct ibv_cq *ibv_cq = NULL; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapli_cq_event_cb(%p)\n", hca); - - if (!ibv_get_cq_event(hca->ib_cq, &ibv_cq, (void *)&evd_ptr)) { - - if (DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD)) { - ibv_ack_cq_events(ibv_cq, 1); - return; - } - - /* process DTO event via callback */ - dapl_evd_dto_callback(hca->cm_id->verbs, - evd_ptr->ib_cq_handle, (void *)evd_ptr); - - ibv_ack_cq_events(ibv_cq, 1); - } -} - -/* - * Map all verbs DTO completion codes to the DAT equivelent. - * - * Not returned by verbs: DAT_DTO_ERR_PARTIAL_PACKET - */ -static struct ib_status_map { - int ib_status; - DAT_DTO_COMPLETION_STATUS dat_status; -} ib_status_map[] = { -/* 00 */ { - IBV_WC_SUCCESS, DAT_DTO_SUCCESS}, -/* 01 */ { - IBV_WC_LOC_LEN_ERR, DAT_DTO_ERR_LOCAL_LENGTH}, -/* 02 */ { - IBV_WC_LOC_QP_OP_ERR, DAT_DTO_ERR_LOCAL_EP}, -/* 03 */ { - IBV_WC_LOC_EEC_OP_ERR, DAT_DTO_ERR_TRANSPORT}, -/* 04 */ { - IBV_WC_LOC_PROT_ERR, DAT_DTO_ERR_LOCAL_PROTECTION}, -/* 05 */ { - IBV_WC_WR_FLUSH_ERR, DAT_DTO_ERR_FLUSHED}, -/* 06 */ { - IBV_WC_MW_BIND_ERR, DAT_RMR_OPERATION_FAILED}, -/* 07 */ { - IBV_WC_BAD_RESP_ERR, DAT_DTO_ERR_BAD_RESPONSE}, -/* 08 */ { - IBV_WC_LOC_ACCESS_ERR, DAT_DTO_ERR_LOCAL_PROTECTION}, -/* 09 */ { - IBV_WC_REM_INV_REQ_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, -/* 10 */ { - IBV_WC_REM_ACCESS_ERR, DAT_DTO_ERR_REMOTE_ACCESS}, -/* 11 */ { - IBV_WC_REM_OP_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, -/* 12 */ { - IBV_WC_RETRY_EXC_ERR, DAT_DTO_ERR_TRANSPORT}, -/* 13 */ { - IBV_WC_RNR_RETRY_EXC_ERR, DAT_DTO_ERR_RECEIVER_NOT_READY}, -/* 14 */ { - IBV_WC_LOC_RDD_VIOL_ERR, DAT_DTO_ERR_LOCAL_PROTECTION}, -/* 15 */ { - IBV_WC_REM_INV_RD_REQ_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, -/* 16 */ { - IBV_WC_REM_ABORT_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, -/* 17 */ { - IBV_WC_INV_EECN_ERR, DAT_DTO_ERR_TRANSPORT}, -/* 18 */ { - IBV_WC_INV_EEC_STATE_ERR, DAT_DTO_ERR_TRANSPORT}, -/* 19 */ { - IBV_WC_FATAL_ERR, DAT_DTO_ERR_TRANSPORT}, -/* 20 */ { - IBV_WC_RESP_TIMEOUT_ERR, DAT_DTO_ERR_RECEIVER_NOT_READY}, -/* 21 */ { -IBV_WC_GENERAL_ERR, DAT_DTO_ERR_TRANSPORT},}; - -/* - * dapls_ib_get_dto_status - * - * Return the DAT status of a DTO operation - * - * Input: - * cqe_ptr pointer to completion queue entry - * - * Output: - * none - * - * Returns: - * Value from ib_status_map table above - */ - -DAT_DTO_COMPLETION_STATUS -dapls_ib_get_dto_status(IN ib_work_completion_t * cqe_ptr) -{ - uint32_t ib_status; - int i; - - ib_status = DAPL_GET_CQE_STATUS(cqe_ptr); - - /* - * Due to the implementation of verbs completion code, we need to - * search the table for the correct value rather than assuming - * linear distribution. - */ - for (i = 0; i <= IBV_WC_GENERAL_ERR; i++) { - if (ib_status == ib_status_map[i].ib_status) { - if (ib_status != IBV_WC_SUCCESS) { - dapl_dbg_log(DAPL_DBG_TYPE_DTO_COMP_ERR, - " DTO completion ERROR: %d: op %#x\n", - ib_status, - DAPL_GET_CQE_OPTYPE(cqe_ptr)); - } - return ib_status_map[i].dat_status; - } - } - - dapl_dbg_log(DAPL_DBG_TYPE_DTO_COMP_ERR, - " DTO completion ERROR: %d: op %#x\n", - ib_status, DAPL_GET_CQE_OPTYPE(cqe_ptr)); - - return DAT_DTO_FAILURE; -} - -DAT_RETURN dapls_ib_get_async_event(IN ib_error_record_t * err_record, - OUT DAT_EVENT_NUMBER * async_event) -{ - DAT_RETURN dat_status = DAT_SUCCESS; - int err_code = err_record->event_type; - - switch (err_code) { - /* OVERFLOW error */ - case IBV_EVENT_CQ_ERR: - *async_event = DAT_ASYNC_ERROR_EVD_OVERFLOW; - break; - /* INTERNAL errors */ - case IBV_EVENT_DEVICE_FATAL: - *async_event = DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR; - break; - /* CATASTROPHIC errors */ - case IBV_EVENT_PORT_ERR: - *async_event = DAT_ASYNC_ERROR_IA_CATASTROPHIC; - break; - /* BROKEN QP error */ - case IBV_EVENT_SQ_DRAINED: - case IBV_EVENT_QP_FATAL: - case IBV_EVENT_QP_REQ_ERR: - case IBV_EVENT_QP_ACCESS_ERR: - *async_event = DAT_ASYNC_ERROR_EP_BROKEN; - break; - /* connection completion */ - case IBV_EVENT_COMM_EST: - *async_event = DAT_CONNECTION_EVENT_ESTABLISHED; - break; - /* TODO: process HW state changes */ - case IBV_EVENT_PATH_MIG: - case IBV_EVENT_PATH_MIG_ERR: - case IBV_EVENT_PORT_ACTIVE: - case IBV_EVENT_LID_CHANGE: - case IBV_EVENT_PKEY_CHANGE: - case IBV_EVENT_SM_CHANGE: - default: - dat_status = DAT_ERROR(DAT_NOT_IMPLEMENTED, 0); - } - return dat_status; -} - -/* - * dapl_ib_cq_alloc - * - * Alloc a CQ - * - * Input: - * ia_handle IA handle - * evd_ptr pointer to EVD struct - * cqlen minimum QLen - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN -dapls_ib_cq_alloc(IN DAPL_IA * ia_ptr, - IN DAPL_EVD * evd_ptr, IN DAT_COUNT * cqlen) -{ - struct ibv_comp_channel *channel = ia_ptr->hca_ptr->ib_trans.ib_cq; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - "dapls_ib_cq_alloc: evd %p cqlen=%d \n", evd_ptr, *cqlen); - -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) - channel = evd_ptr->cq_wait_obj_handle->events; -#endif - - /* Call IB verbs to create CQ */ - evd_ptr->ib_cq_handle = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, - *cqlen, evd_ptr, channel, 0); - - if (evd_ptr->ib_cq_handle == IB_INVALID_HANDLE) - return (dapl_convert_errno(errno, "create_cq")); - - /* arm cq for events */ - dapls_set_cq_notify(ia_ptr, evd_ptr); - - /* update with returned cq entry size */ - *cqlen = evd_ptr->ib_cq_handle->cqe; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - "dapls_ib_cq_alloc: new_cq %p cqlen=%d \n", - evd_ptr->ib_cq_handle, *cqlen); - - return DAT_SUCCESS; -} - -/* - * dapl_ib_cq_resize - * - * Alloc a CQ - * - * Input: - * ia_handle IA handle - * evd_ptr pointer to EVD struct - * cqlen minimum QLen - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_PARAMETER - * - */ -DAT_RETURN -dapls_ib_cq_resize(IN DAPL_IA * ia_ptr, - IN DAPL_EVD * evd_ptr, IN DAT_COUNT * cqlen) -{ - ib_cq_handle_t new_cq; - struct ibv_comp_channel *channel = ia_ptr->hca_ptr->ib_trans.ib_cq; - - /* IB verbs doe not support resize. Try to re-create CQ - * with new size. Can only be done if QP is not attached. - * destroy EBUSY == QP still attached. - */ - -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) - channel = evd_ptr->cq_wait_obj_handle->events; -#endif - - /* Call IB verbs to create CQ */ - new_cq = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, *cqlen, - evd_ptr, channel, 0); - - if (new_cq == IB_INVALID_HANDLE) - return DAT_INSUFFICIENT_RESOURCES; - - /* destroy the original and replace if successful */ - if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) { - ibv_destroy_cq(new_cq); - return (dapl_convert_errno(errno, "resize_cq")); - } - - /* update EVD with new cq handle and size */ - evd_ptr->ib_cq_handle = new_cq; - *cqlen = new_cq->cqe; - - /* arm cq for events */ - dapls_set_cq_notify(ia_ptr, evd_ptr); - - return DAT_SUCCESS; -} - -/* - * dapls_ib_cq_free - * - * destroy a CQ - * - * Input: - * ia_handle IA handle - * evd_ptr pointer to EVD struct - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_PARAMETER - * - */ -DAT_RETURN dapls_ib_cq_free(IN DAPL_IA * ia_ptr, IN DAPL_EVD * evd_ptr) -{ - if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) { - /* copy all entries on CQ to EVD before destroying */ - dapls_evd_copy_cq(evd_ptr); - if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) - return (dapl_convert_errno(errno, "destroy_cq")); - evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; - } - return DAT_SUCCESS; -} - -/* - * dapls_set_cq_notify - * - * Set the CQ notification for next - * - * Input: - * hca_handl hca handle - * DAPL_EVD evd handle - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * dapl_convert_errno - */ -DAT_RETURN dapls_set_cq_notify(IN DAPL_IA * ia_ptr, IN DAPL_EVD * evd_ptr) -{ - if (ibv_req_notify_cq(evd_ptr->ib_cq_handle, 0)) - return (dapl_convert_errno(errno, "notify_cq")); - else - return DAT_SUCCESS; -} - -/* - * dapls_ib_completion_notify - * - * Set the CQ notification type - * - * Input: - * hca_handl hca handle - * evd_ptr evd handle - * type notification type - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * dapl_convert_errno - */ -DAT_RETURN dapls_ib_completion_notify(IN ib_hca_handle_t hca_handle, - IN DAPL_EVD * evd_ptr, - IN ib_notification_type_t type) -{ - if (ibv_req_notify_cq(evd_ptr->ib_cq_handle, type)) - return (dapl_convert_errno(errno, "notify_cq_type")); - else - return DAT_SUCCESS; -} - -/* - * dapls_ib_completion_poll - * - * CQ poll for completions - * - * Input: - * hca_handl hca handle - * evd_ptr evd handle - * wc_ptr work completion - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_QUEUE_EMPTY - * - */ -DAT_RETURN dapls_ib_completion_poll(IN DAPL_HCA * hca_ptr, - IN DAPL_EVD * evd_ptr, - IN ib_work_completion_t * wc_ptr) -{ - if (ibv_poll_cq(evd_ptr->ib_cq_handle, 1, wc_ptr) == 1) - return DAT_SUCCESS; - - return DAT_QUEUE_EMPTY; -} - -#ifdef CQ_WAIT_OBJECT - -/* NEW common wait objects for providers with direct CQ wait objects */ -DAT_RETURN -dapls_ib_wait_object_create(IN DAPL_EVD * evd_ptr, - IN ib_wait_obj_handle_t * p_cq_wait_obj_handle) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_create: (%p,%p)\n", - evd_ptr, p_cq_wait_obj_handle); - - *p_cq_wait_obj_handle = - dapl_os_alloc(sizeof(struct _ib_wait_obj_handle)); - - if (*p_cq_wait_obj_handle == NULL) - return (dapl_convert_errno(ENOMEM, " wait_object_create")); - - dapl_os_memzero(*p_cq_wait_obj_handle, - sizeof(struct _ib_wait_obj_handle)); - - /* create pipe for waking up work thread */ - if (pipe((*p_cq_wait_obj_handle)->pipe)) - goto bail; - - /* set cq_wait object to evd_ptr */ - (*p_cq_wait_obj_handle)->events = - ibv_create_comp_channel(evd_ptr->header.owner_ia->hca_ptr-> - ib_hca_handle); - - if ((*p_cq_wait_obj_handle)->events == NULL) - goto bail; - - return DAT_SUCCESS; - bail: - dapl_os_free(*p_cq_wait_obj_handle, sizeof(struct _ib_wait_obj_handle)); - *p_cq_wait_obj_handle = NULL; - return (dapl_convert_errno(errno, " wait_object_create")); -} - -DAT_RETURN -dapls_ib_wait_object_destroy(IN ib_wait_obj_handle_t p_cq_wait_obj_handle) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_destroy: wait_obj=%p\n", p_cq_wait_obj_handle); - - ibv_destroy_comp_channel(p_cq_wait_obj_handle->events); - - dapl_os_free(p_cq_wait_obj_handle, sizeof(struct _ib_wait_obj_handle)); - - return DAT_SUCCESS; -} - -DAT_RETURN -dapls_ib_wait_object_wakeup(IN ib_wait_obj_handle_t p_cq_wait_obj_handle) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wakeup: wait_obj=%p\n", p_cq_wait_obj_handle); - - /* write to pipe for wake up */ - if (write(p_cq_wait_obj_handle->pipe[1], "w", sizeof "w") == -1) - dapl_log(DAPL_DBG_TYPE_UTIL, - " wait object wakeup write error = %s\n", - strerror(errno)); - return DAT_SUCCESS; -} - -DAT_RETURN -dapls_ib_wait_object_wait(IN ib_wait_obj_handle_t p_cq_wait_obj_handle, - IN u_int32_t timeout) -{ - struct dapl_evd *evd_ptr; - struct ibv_cq *ibv_cq = NULL; - int status = 0; - int timeout_ms = -1; - struct pollfd ufds[2]; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wait: CQ channel %p time %d\n", - p_cq_wait_obj_handle, timeout); - - /* setup cq event channel and pipe fd for consumer wakeup */ - ufds[0].fd = p_cq_wait_obj_handle->events->fd; - ufds[0].events = POLLIN; - ufds[0].revents = 0; - ufds[1].fd = p_cq_wait_obj_handle->pipe[0]; - ufds[1].events = POLLIN; - ufds[1].revents = 0; - - /* uDAPL timeout values in usecs */ - if (timeout != DAT_TIMEOUT_INFINITE) - timeout_ms = timeout / 1000; - - /* restart syscall */ - while ((status = poll(ufds, 2, timeout_ms)) == -1) - if (errno == EINTR) - continue; - - /* returned event */ - if (status > 0) { - if (ufds[0].revents == POLLIN) { - if (!ibv_get_cq_event(p_cq_wait_obj_handle->events, - &ibv_cq, (void *)&evd_ptr)) { - ibv_ack_cq_events(ibv_cq, 1); - } - } - status = 0; - - /* timeout */ - } else if (status == 0) - status = ETIMEDOUT; - else - status = errno; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wait: RET evd %p ibv_cq %p %s\n", - evd_ptr, ibv_cq, strerror(errno)); - - return (dapl_convert_errno(status, "cq_wait_object_wait")); - -} -#endif - -/* - * Local variables: - * c-indent-level: 4 - * c-basic-offset: 4 - * tab-width: 8 - * End: - */ diff --git a/dapl/openib_cma/dapl_ib_dto.h b/dapl/openib_cma/dapl_ib_dto.h deleted file mode 100644 index d97c26b..0000000 --- a/dapl/openib_cma/dapl_ib_dto.h +++ /dev/null @@ -1,472 +0,0 @@ -/* - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/*************************************************************************** - * - * Module: uDAPL - * - * Filename: dapl_ib_dto.h - * - * Author: Arlin Davis - * - * Created: 3/10/2005 - * - * Description: - * - * The OpenIB uCMA provider - DTO operations and CQE macros - * - **************************************************************************** - * Source Control System Information - * - * $Id: $ - * - * Copyright (c) 2005 Intel Corporation. All rights reserved. - * - **************************************************************************/ -#ifndef _DAPL_IB_DTO_H_ -#define _DAPL_IB_DTO_H_ - -#include "dapl_ib_util.h" - -#ifdef DAT_EXTENSIONS -#include -#endif - -STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p); - -/* - * dapls_ib_post_recv - * - * Provider specific Post RECV function - */ -STATIC _INLINE_ DAT_RETURN -dapls_ib_post_recv ( - IN DAPL_EP *ep_ptr, - IN DAPL_COOKIE *cookie, - IN DAT_COUNT segments, - IN DAT_LMR_TRIPLET *local_iov ) -{ - struct ibv_recv_wr wr; - struct ibv_recv_wr *bad_wr; - ib_data_segment_t *ds = (ib_data_segment_t *)local_iov; - DAT_COUNT i, total_len; - int ret; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_rcv: ep %p cookie %p segs %d l_iov %p\n", - ep_ptr, cookie, segments, local_iov); - - /* setup work request */ - total_len = 0; - wr.next = 0; - wr.num_sge = segments; - wr.wr_id = (uint64_t)(uintptr_t)cookie; - wr.sg_list = ds; - - if (cookie != NULL) { - for (i = 0; i < segments; i++) { - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_rcv: l_key 0x%x va %p len %d\n", - ds->lkey, ds->addr, ds->length); - total_len += ds->length; - ds++; - } - cookie->val.dto.size = total_len; - } - - ret = ibv_post_recv(ep_ptr->qp_handle->cm_id->qp, &wr, &bad_wr); - - if (ret) - return( dapl_convert_errno(errno,"ibv_recv") ); - - DAPL_CNTR(ep_ptr, DCNT_EP_POST_RECV); - DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_RECV_DATA, total_len); - - return DAT_SUCCESS; -} - -/* - * dapls_ib_post_send - * - * Provider specific Post SEND function - */ -STATIC _INLINE_ DAT_RETURN -dapls_ib_post_send ( - IN DAPL_EP *ep_ptr, - IN ib_send_op_type_t op_type, - IN DAPL_COOKIE *cookie, - IN DAT_COUNT segments, - IN DAT_LMR_TRIPLET *local_iov, - IN const DAT_RMR_TRIPLET *remote_iov, - IN DAT_COMPLETION_FLAGS completion_flags) -{ - struct ibv_send_wr wr; - struct ibv_send_wr *bad_wr; - ib_data_segment_t *ds = (ib_data_segment_t *)local_iov; - ib_hca_transport_t *ibt_ptr = - &ep_ptr->header.owner_ia->hca_ptr->ib_trans; - DAT_COUNT i, total_len; - int ret; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd: ep %p op %d ck %p sgs", - "%d l_iov %p r_iov %p f %d\n", - ep_ptr, op_type, cookie, segments, local_iov, - remote_iov, completion_flags); - - /* setup the work request */ - wr.next = 0; - wr.opcode = op_type; - wr.num_sge = segments; - wr.send_flags = 0; - wr.wr_id = (uint64_t)(uintptr_t)cookie; - wr.sg_list = ds; - total_len = 0; - - if (cookie != NULL) { - for (i = 0; i < segments; i++ ) { - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd: lkey 0x%x va %p len %d\n", - ds->lkey, ds->addr, ds->length ); - total_len += ds->length; - ds++; - } - cookie->val.dto.size = total_len; - } - - if (wr.num_sge && - (op_type == OP_RDMA_WRITE || op_type == OP_RDMA_READ)) { - wr.wr.rdma.remote_addr = remote_iov->virtual_address; - wr.wr.rdma.rkey = remote_iov->rmr_context; - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd_rdma: rkey 0x%x va %#016Lx\n", - wr.wr.rdma.rkey, wr.wr.rdma.remote_addr); - } - - /* inline data for send or write ops */ - if ((total_len <= ibt_ptr->max_inline_send) && - ((op_type == OP_SEND) || (op_type == OP_RDMA_WRITE))) - wr.send_flags |= IBV_SEND_INLINE; - - /* set completion flags in work request */ - wr.send_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & - completion_flags) ? 0 : IBV_SEND_SIGNALED; - wr.send_flags |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & - completion_flags) ? IBV_SEND_FENCE : 0; - wr.send_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & - completion_flags) ? IBV_SEND_SOLICITED : 0; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd: op 0x%x flags 0x%x sglist %p, %d\n", - wr.opcode, wr.send_flags, wr.sg_list, wr.num_sge); - - ret = ibv_post_send(ep_ptr->qp_handle->cm_id->qp, &wr, &bad_wr); - - if (ret) - return( dapl_convert_errno(errno,"ibv_send") ); - -#ifdef DAPL_COUNTERS - switch (op_type) { - case OP_SEND: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND); - DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_DATA,total_len); - break; - case OP_RDMA_WRITE: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE); - DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_WRITE_DATA,total_len); - break; - case OP_RDMA_READ: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_READ); - DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_READ_DATA,total_len); - break; - default: - break; - } -#endif /* DAPL_COUNTERS */ - - dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n"); - return DAT_SUCCESS; -} - -/* map Work Completions to DAPL WR operations */ -STATIC _INLINE_ DAT_DTOS dapls_cqe_dtos_opcode(ib_work_completion_t *cqe_p) -{ - switch (cqe_p->opcode) { - - case IBV_WC_SEND: - return (DAT_DTO_SEND); - case IBV_WC_RDMA_READ: - return (DAT_DTO_RDMA_READ); - case IBV_WC_BIND_MW: - return (DAT_DTO_BIND_MW); -#ifdef DAT_EXTENSIONS - case IBV_WC_RDMA_WRITE: - if (cqe_p->wc_flags & IBV_WC_WITH_IMM) - return (DAT_IB_DTO_RDMA_WRITE_IMMED); - else - return (DAT_DTO_RDMA_WRITE); - case IBV_WC_COMP_SWAP: - return (DAT_IB_DTO_CMP_SWAP); - case IBV_WC_FETCH_ADD: - return (DAT_IB_DTO_FETCH_ADD); - case IBV_WC_RECV_RDMA_WITH_IMM: - return (DAT_IB_DTO_RECV_IMMED); -#else - case IBV_WC_RDMA_WRITE: - return (DAT_DTO_RDMA_WRITE); -#endif - case IBV_WC_RECV: - return (DAT_DTO_RECEIVE); - default: - return (0xff); - } -} -#define DAPL_GET_CQE_DTOS_OPTYPE(cqe_p) dapls_cqe_dtos_opcode(cqe_p) - - -#ifdef DAT_EXTENSIONS -/* - * dapls_ib_post_ext_send - * - * Provider specific extended Post SEND function for atomics - * OP_COMP_AND_SWAP and OP_FETCH_AND_ADD - */ -STATIC _INLINE_ DAT_RETURN -dapls_ib_post_ext_send ( - IN DAPL_EP *ep_ptr, - IN ib_send_op_type_t op_type, - IN DAPL_COOKIE *cookie, - IN DAT_COUNT segments, - IN DAT_LMR_TRIPLET *local_iov, - IN const DAT_RMR_TRIPLET *remote_iov, - IN DAT_UINT32 immed_data, - IN DAT_UINT64 compare_add, - IN DAT_UINT64 swap, - IN DAT_COMPLETION_FLAGS completion_flags) -{ - struct ibv_send_wr wr; - struct ibv_send_wr *bad_wr; - ib_data_segment_t *ds = (ib_data_segment_t *)local_iov; - DAT_COUNT i, total_len; - int ret; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_ext_snd: ep %p op %d ck %p sgs", - "%d l_iov %p r_iov %p f %d\n", - ep_ptr, op_type, cookie, segments, local_iov, - remote_iov, completion_flags); - - /* setup the work request */ - wr.next = 0; - wr.opcode = op_type; - wr.num_sge = segments; - wr.send_flags = 0; - wr.wr_id = (uint64_t)(uintptr_t)cookie; - wr.sg_list = ds; - total_len = 0; - - if (cookie != NULL) { - for (i = 0; i < segments; i++ ) { - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_ext_snd: lkey 0x%x va %p ln %d\n", - ds->lkey, ds->addr, ds->length); - total_len += ds->length; - ds++; - } - cookie->val.dto.size = total_len; - } - - switch (op_type) { - case OP_RDMA_WRITE_IMM: - /* OP_RDMA_WRITE)IMMED has direct IB wr_type mapping */ - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_ext: rkey 0x%x va %#016Lx immed=0x%x\n", - remote_iov?remote_iov->rmr_context:0, - remote_iov?remote_iov->virtual_address:0, - immed_data); - - wr.imm_data = immed_data; - if (wr.num_sge) { - wr.wr.rdma.remote_addr = remote_iov->virtual_address; - wr.wr.rdma.rkey = remote_iov->rmr_context; - } - break; - case OP_COMP_AND_SWAP: - /* OP_COMP_AND_SWAP has direct IB wr_type mapping */ - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_ext: OP_COMP_AND_SWAP=%lx," - "%lx rkey 0x%x va %#016Lx\n", - compare_add, swap, remote_iov->rmr_context, - remote_iov->virtual_address); - - wr.wr.atomic.compare_add = compare_add; - wr.wr.atomic.swap = swap; - wr.wr.atomic.remote_addr = remote_iov->virtual_address; - wr.wr.atomic.rkey = remote_iov->rmr_context; - break; - case OP_FETCH_AND_ADD: - /* OP_FETCH_AND_ADD has direct IB wr_type mapping */ - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_ext: OP_FETCH_AND_ADD=%lx," - "%lx rkey 0x%x va %#016Lx\n", - compare_add, remote_iov->rmr_context, - remote_iov->virtual_address); - - wr.wr.atomic.compare_add = compare_add; - wr.wr.atomic.remote_addr = remote_iov->virtual_address; - wr.wr.atomic.rkey = remote_iov->rmr_context; - break; - default: - break; - } - - /* set completion flags in work request */ - wr.send_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & - completion_flags) ? 0 : IBV_SEND_SIGNALED; - wr.send_flags |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & - completion_flags) ? IBV_SEND_FENCE : 0; - wr.send_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & - completion_flags) ? IBV_SEND_SOLICITED : 0; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd: op 0x%x flags 0x%x sglist %p, %d\n", - wr.opcode, wr.send_flags, wr.sg_list, wr.num_sge); - - ret = ibv_post_send(ep_ptr->qp_handle->cm_id->qp, &wr, &bad_wr); - - if (ret) - return( dapl_convert_errno(errno,"ibv_send") ); - -#ifdef DAPL_COUNTERS - switch (op_type) { - case OP_RDMA_WRITE_IMM: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE_IMM); - DAPL_CNTR_DATA(ep_ptr, - DCNT_EP_POST_WRITE_IMM_DATA, total_len); - break; - case OP_COMP_AND_SWAP: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_CMP_SWAP); - break; - case OP_FETCH_AND_ADD: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_FETCH_ADD); - break; - default: - break; - } -#endif /* DAPL_COUNTERS */ - - dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n"); - return DAT_SUCCESS; -} -#endif - -STATIC _INLINE_ DAT_RETURN -dapls_ib_optional_prv_dat( - IN DAPL_CR *cr_ptr, - IN const void *event_data, - OUT DAPL_CR **cr_pp) -{ - return DAT_SUCCESS; -} - -/* map Work Completions to DAPL WR operations */ -STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p) -{ -#ifdef DAPL_COUNTERS - DAPL_COOKIE *cookie = (DAPL_COOKIE *)(uintptr_t)cqe_p->wr_id; -#endif /* DAPL_COUNTERS */ - - switch (cqe_p->opcode) { - case IBV_WC_SEND: - return (OP_SEND); - case IBV_WC_RDMA_WRITE: - if (cqe_p->wc_flags & IBV_WC_WITH_IMM) - return (OP_RDMA_WRITE_IMM); - else - return (OP_RDMA_WRITE); - case IBV_WC_RDMA_READ: - return (OP_RDMA_READ); - case IBV_WC_COMP_SWAP: - return (OP_COMP_AND_SWAP); - case IBV_WC_FETCH_ADD: - return (OP_FETCH_AND_ADD); - case IBV_WC_BIND_MW: - return (OP_BIND_MW); - case IBV_WC_RECV: - if (cqe_p->wc_flags & IBV_WC_WITH_IMM) { - DAPL_CNTR(cookie->ep, DCNT_EP_RECV_IMM); - DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_IMM_DATA, - cqe_p->byte_len); - return (OP_RECEIVE_IMM); - } else { - DAPL_CNTR(cookie->ep, DCNT_EP_RECV); - DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_DATA, - cqe_p->byte_len); - return (OP_RECEIVE); - } - case IBV_WC_RECV_RDMA_WITH_IMM: - DAPL_CNTR(cookie->ep, DCNT_EP_RECV_RDMA_IMM); - DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_RDMA_IMM_DATA, - cqe_p->byte_len); - return (OP_RECEIVE_IMM); - default: - return (OP_INVALID); - } -} - -#define DAPL_GET_CQE_OPTYPE(cqe_p) dapls_cqe_opcode(cqe_p) -#define DAPL_GET_CQE_WRID(cqe_p) ((ib_work_completion_t*)cqe_p)->wr_id -#define DAPL_GET_CQE_STATUS(cqe_p) ((ib_work_completion_t*)cqe_p)->status -#define DAPL_GET_CQE_VENDOR_ERR(cqe_p) ((ib_work_completion_t*)cqe_p)->vendor_err -#define DAPL_GET_CQE_BYTESNUM(cqe_p) ((ib_work_completion_t*)cqe_p)->byte_len -#define DAPL_GET_CQE_IMMED_DATA(cqe_p) ((ib_work_completion_t*)cqe_p)->imm_data - -STATIC _INLINE_ char * dapls_dto_op_str(int op) -{ - static char *optable[] = - { - "OP_RDMA_WRITE", - "OP_RDMA_WRITE_IMM", - "OP_SEND", - "OP_SEND_IMM", - "OP_RDMA_READ", - "OP_COMP_AND_SWAP", - "OP_FETCH_AND_ADD", - "OP_RECEIVE", - "OP_RECEIVE_IMM", - "OP_BIND_MW" - }; - return ((op < 0 || op > 9) ? "Invalid CQE OP?" : optable[op]); -} - -static _INLINE_ char * -dapls_cqe_op_str(IN ib_work_completion_t *cqe_ptr) -{ - return dapls_dto_op_str(DAPL_GET_CQE_OPTYPE(cqe_ptr)); -} - -#define DAPL_GET_CQE_OP_STR(cqe) dapls_cqe_op_str(cqe) - -#endif /* _DAPL_IB_DTO_H_ */ diff --git a/dapl/openib_cma/dapl_ib_extensions.c b/dapl/openib_cma/dapl_ib_extensions.c deleted file mode 100755 index 3bcde58..0000000 --- a/dapl/openib_cma/dapl_ib_extensions.c +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Copyright (c) 2007 Intel Corporation. All rights reserved. - * - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/********************************************************************** - * - * MODULE: dapl_ib_extensions.c - * - * PURPOSE: Extensions routines for OpenIB uCMA provider - * - * $Id: $ - * - **********************************************************************/ - -#include "dapl.h" -#include "dapl_adapter_util.h" -#include "dapl_evd_util.h" -#include "dapl_ib_util.h" -#include "dapl_ep_util.h" -#include "dapl_cookie.h" -#include - -DAT_RETURN -dapli_post_ext(IN DAT_EP_HANDLE ep_handle, - IN DAT_UINT64 cmp_add, - IN DAT_UINT64 swap, - IN DAT_UINT32 immed_data, - IN DAT_COUNT segments, - IN DAT_LMR_TRIPLET * local_iov, - IN DAT_DTO_COOKIE user_cookie, - IN const DAT_RMR_TRIPLET * remote_iov, - IN int op_type, IN DAT_COMPLETION_FLAGS flags); - -/* - * dapl_extensions - * - * Process extension requests - * - * Input: - * ext_type, - * ... - * - * Output: - * Depends.... - * - * Returns: - * DAT_SUCCESS - * DAT_NOT_IMPLEMENTED - * ..... - * - */ -DAT_RETURN -dapl_extensions(IN DAT_HANDLE dat_handle, - IN DAT_EXTENDED_OP ext_op, IN va_list args) -{ - DAT_EP_HANDLE ep; - DAT_LMR_TRIPLET *lmr_p; - DAT_DTO_COOKIE cookie; - const DAT_RMR_TRIPLET *rmr_p; - DAT_UINT64 dat_uint64a, dat_uint64b; - DAT_UINT32 dat_uint32; - DAT_COUNT segments = 1; - DAT_COMPLETION_FLAGS comp_flags; - DAT_RETURN status = DAT_NOT_IMPLEMENTED; - - dapl_dbg_log(DAPL_DBG_TYPE_API, - "dapl_extensions(hdl %p operation %d, ...)\n", - dat_handle, ext_op); - - switch ((int)ext_op) { - - case DAT_IB_RDMA_WRITE_IMMED_OP: - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " WRITE_IMMED_DATA extension call\n"); - - ep = dat_handle; /* ep_handle */ - segments = va_arg(args, DAT_COUNT); /* num segments */ - lmr_p = va_arg(args, DAT_LMR_TRIPLET *); - cookie = va_arg(args, DAT_DTO_COOKIE); - rmr_p = va_arg(args, const DAT_RMR_TRIPLET *); - dat_uint32 = va_arg(args, DAT_UINT32); /* immed data */ - comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); - - status = dapli_post_ext(ep, 0, 0, dat_uint32, segments, lmr_p, - cookie, rmr_p, OP_RDMA_WRITE_IMM, - comp_flags); - break; - - case DAT_IB_CMP_AND_SWAP_OP: - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " CMP_AND_SWAP extension call\n"); - - ep = dat_handle; /* ep_handle */ - dat_uint64a = va_arg(args, DAT_UINT64); /* cmp_value */ - dat_uint64b = va_arg(args, DAT_UINT64); /* swap_value */ - lmr_p = va_arg(args, DAT_LMR_TRIPLET *); - cookie = va_arg(args, DAT_DTO_COOKIE); - rmr_p = va_arg(args, const DAT_RMR_TRIPLET *); - comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); - - status = dapli_post_ext(ep, dat_uint64a, dat_uint64b, - 0, segments, lmr_p, cookie, rmr_p, - OP_COMP_AND_SWAP, comp_flags); - break; - - case DAT_IB_FETCH_AND_ADD_OP: - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " FETCH_AND_ADD extension call\n"); - - ep = dat_handle; /* ep_handle */ - dat_uint64a = va_arg(args, DAT_UINT64); /* add value */ - lmr_p = va_arg(args, DAT_LMR_TRIPLET *); - cookie = va_arg(args, DAT_DTO_COOKIE); - rmr_p = va_arg(args, const DAT_RMR_TRIPLET *); - comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); - - status = dapli_post_ext(ep, dat_uint64a, 0, 0, segments, - lmr_p, cookie, rmr_p, - OP_FETCH_AND_ADD, comp_flags); - - break; - -#ifdef DAPL_COUNTERS - case DAT_QUERY_COUNTERS_OP: - { - int cntr, reset; - DAT_UINT64 *p_cntr_out; - - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " Query counter extension call\n"); - - cntr = va_arg(args, int); - p_cntr_out = va_arg(args, DAT_UINT64 *); - reset = va_arg(args, int); - - status = dapl_query_counter(dat_handle, cntr, - p_cntr_out, reset); - break; - } - case DAT_PRINT_COUNTERS_OP: - { - int cntr, reset; - - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " Print counter extension call\n"); - - cntr = va_arg(args, int); - reset = va_arg(args, int); - - dapl_print_counter(dat_handle, cntr, reset); - status = DAT_SUCCESS; - break; - } -#endif /* DAPL_COUNTERS */ - - default: - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - "unsupported extension(%d)\n", (int)ext_op); - } - - return (status); -} - -DAT_RETURN -dapli_post_ext(IN DAT_EP_HANDLE ep_handle, - IN DAT_UINT64 cmp_add, - IN DAT_UINT64 swap, - IN DAT_UINT32 immed_data, - IN DAT_COUNT segments, - IN DAT_LMR_TRIPLET * local_iov, - IN DAT_DTO_COOKIE user_cookie, - IN const DAT_RMR_TRIPLET * remote_iov, - IN int op_type, IN DAT_COMPLETION_FLAGS flags) -{ - DAPL_EP *ep_ptr; - ib_qp_handle_t qp_ptr; - DAPL_COOKIE *cookie = NULL; - DAT_RETURN dat_status = DAT_SUCCESS; - - dapl_dbg_log(DAPL_DBG_TYPE_API, - " post_ext_op: ep %p cmp_val %d " - "swap_val %d cookie 0x%x, r_iov %p, flags 0x%x\n", - ep_handle, (unsigned)cmp_add, (unsigned)swap, - (unsigned)user_cookie.as_64, remote_iov, flags); - - if (DAPL_BAD_HANDLE(ep_handle, DAPL_MAGIC_EP)) - return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP)); - - ep_ptr = (DAPL_EP *) ep_handle; - qp_ptr = ep_ptr->qp_handle; - - /* - * Synchronization ok since this buffer is only used for send - * requests, which aren't allowed to race with each other. - */ - dat_status = dapls_dto_cookie_alloc(&ep_ptr->req_buffer, - DAPL_DTO_TYPE_EXTENSION, - user_cookie, &cookie); - if (dat_status != DAT_SUCCESS) - goto bail; - - /* - * Take reference before posting to avoid race conditions with - * completions - */ - dapl_os_atomic_inc(&ep_ptr->req_count); - - /* - * Invoke provider specific routine to post DTO - */ - dat_status = dapls_ib_post_ext_send(ep_ptr, op_type, cookie, segments, /* data segments */ - local_iov, remote_iov, immed_data, /* immed data */ - cmp_add, /* compare or add */ - swap, /* swap */ - flags); - - if (dat_status != DAT_SUCCESS) { - dapl_os_atomic_dec(&ep_ptr->req_count); - dapls_cookie_dealloc(&ep_ptr->req_buffer, cookie); - } - - bail: - return dat_status; - -} - -/* - * New provider routine to process extended DTO events - */ -void -dapls_cqe_to_event_extension(IN DAPL_EP * ep_ptr, - IN DAPL_COOKIE * cookie, - IN ib_work_completion_t * cqe_ptr, - IN DAT_EVENT * event_ptr) -{ - uint32_t ibtype; - DAT_DTO_COMPLETION_EVENT_DATA *dto = - &event_ptr->event_data.dto_completion_event_data; - DAT_IB_EXTENSION_EVENT_DATA *ext_data = (DAT_IB_EXTENSION_EVENT_DATA *) - & event_ptr->event_extension_data[0]; - DAT_DTO_COMPLETION_STATUS dto_status; - - /* Get status from cqe */ - dto_status = dapls_ib_get_dto_status(cqe_ptr); - - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: dto_ptr %p ext_ptr %p status %d\n", - dto, ext_data, dto_status); - - event_ptr->event_number = DAT_IB_DTO_EVENT; - dto->ep_handle = cookie->ep; - dto->user_cookie = cookie->val.dto.cookie; - dto->operation = DAPL_GET_CQE_DTOS_OPTYPE(cqe_ptr); /* new for 2.0 */ - dto->status = ext_data->status = dto_status; - - if (dto_status != DAT_DTO_SUCCESS) - return; - - /* - * Get operation type from CQ work completion entry and - * if extented operation then set extended event data - */ - ibtype = DAPL_GET_CQE_OPTYPE(cqe_ptr); - - switch (ibtype) { - - case OP_RDMA_WRITE_IMM: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: OP_RDMA_WRITE_IMMED\n"); - - /* type and outbound rdma write transfer size */ - dto->transfered_length = cookie->val.dto.size; - ext_data->type = DAT_IB_RDMA_WRITE_IMMED; - break; - case OP_RECEIVE_IMM: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: OP_RECEIVE_RDMA_IMMED\n"); - - /* immed recvd, type and inbound rdma write transfer size */ - dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); - ext_data->type = DAT_IB_RDMA_WRITE_IMMED_DATA; - ext_data->val.immed.data = DAPL_GET_CQE_IMMED_DATA(cqe_ptr); - break; - case OP_COMP_AND_SWAP: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: COMP_AND_SWAP_RESP\n"); - - /* original data is returned in LMR provided with post */ - ext_data->type = DAT_IB_CMP_AND_SWAP; - dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); - break; - case OP_FETCH_AND_ADD: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: FETCH_AND_ADD_RESP\n"); - - /* original data is returned in LMR provided with post */ - ext_data->type = DAT_IB_FETCH_AND_ADD; - dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); - break; - default: - /* not extended operation */ - ext_data->status = DAT_IB_OP_ERR; - dto->status = DAT_DTO_ERR_TRANSPORT; - break; - } -} diff --git a/dapl/openib_cma/dapl_ib_mem.c b/dapl/openib_cma/dapl_ib_mem.c deleted file mode 100755 index 7e73044..0000000 --- a/dapl/openib_cma/dapl_ib_mem.c +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Copyright (c) 2005-2007 Intel Corporation. All rights reserved. - * - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/********************************************************************** - * - * MODULE: dapl_ib_mem.c - * - * PURPOSE: Memory windows, registration, and protection domain - * - * $Id:$ - * - **********************************************************************/ - -#include "dapl.h" -#include "dapl_adapter_util.h" -#include "dapl_lmr_util.h" - -/* - * dapls_convert_privileges - * - * Convert LMR privileges to provider - * - * Input: - * DAT_MEM_PRIV_FLAGS - * - * Output: - * none - * - * Returns: - * ibv_access_flags - * - */ -STATIC _INLINE_ int dapls_convert_privileges(IN DAT_MEM_PRIV_FLAGS privileges) -{ - int access = 0; - - /* - * if (DAT_MEM_PRIV_LOCAL_READ_FLAG & privileges) do nothing - */ - if (DAT_MEM_PRIV_LOCAL_WRITE_FLAG & privileges) - access |= IBV_ACCESS_LOCAL_WRITE; - if (DAT_MEM_PRIV_REMOTE_WRITE_FLAG & privileges) - access |= IBV_ACCESS_REMOTE_WRITE; - if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) - access |= IBV_ACCESS_REMOTE_READ; - if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) - access |= IBV_ACCESS_REMOTE_READ; - if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) - access |= IBV_ACCESS_REMOTE_READ; -#ifdef DAT_EXTENSIONS - if (DAT_IB_MEM_PRIV_REMOTE_ATOMIC & privileges) - access |= IBV_ACCESS_REMOTE_ATOMIC; -#endif - - return access; -} - -/* - * dapl_ib_pd_alloc - * - * Alloc a PD - * - * Input: - * ia_handle IA handle - * pz pointer to PZ struct - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN dapls_ib_pd_alloc(IN DAPL_IA * ia_ptr, IN DAPL_PZ * pz) -{ - /* get a protection domain */ - pz->pd_handle = ibv_alloc_pd(ia_ptr->hca_ptr->ib_hca_handle); - if (!pz->pd_handle) - return (dapl_convert_errno(ENOMEM, "alloc_pd")); - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " pd_alloc: pd_handle=%p\n", pz->pd_handle); - - return DAT_SUCCESS; -} - -/* - * dapl_ib_pd_free - * - * Free a PD - * - * Input: - * ia_handle IA handle - * PZ_ptr pointer to PZ struct - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_STATE - * - */ -DAT_RETURN dapls_ib_pd_free(IN DAPL_PZ * pz) -{ - if (pz->pd_handle != IB_INVALID_HANDLE) { - if (ibv_dealloc_pd(pz->pd_handle)) - return (dapl_convert_errno(errno, "dealloc_pd")); - pz->pd_handle = IB_INVALID_HANDLE; - } - return DAT_SUCCESS; -} - -/* - * dapl_ib_mr_register - * - * Register a virtual memory region - * - * Input: - * ia_handle IA handle - * lmr pointer to dapl_lmr struct - * virt_addr virtual address of beginning of mem region - * length length of memory region - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN -dapls_ib_mr_register(IN DAPL_IA * ia_ptr, - IN DAPL_LMR * lmr, - IN DAT_PVOID virt_addr, - IN DAT_VLEN length, - IN DAT_MEM_PRIV_FLAGS privileges, IN DAT_VA_TYPE va_type) -{ - ib_pd_handle_t ib_pd_handle; - - ib_pd_handle = ((DAPL_PZ *) lmr->param.pz_handle)->pd_handle; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " mr_register: ia=%p, lmr=%p va=%p ln=%d pv=0x%x\n", - ia_ptr, lmr, virt_addr, length, privileges); - - /* TODO: shared memory */ - if (lmr->param.mem_type == DAT_MEM_TYPE_SHARED_VIRTUAL) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " mr_register_shared: NOT IMPLEMENTED\n"); - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); - } - - /* IB verbs does not support */ - if (va_type == DAT_VA_TYPE_ZB) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " va_type == DAT_VA_TYPE_ZB: NOT SUPPORTED\n"); - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); - } - - /* local read is default on IB */ - lmr->mr_handle = - ibv_reg_mr(((DAPL_PZ *) lmr->param.pz_handle)->pd_handle, - virt_addr, length, dapls_convert_privileges(privileges)); - - if (!lmr->mr_handle) - return (dapl_convert_errno(ENOMEM, "reg_mr")); - - lmr->param.lmr_context = lmr->mr_handle->lkey; - lmr->param.rmr_context = lmr->mr_handle->rkey; - lmr->param.registered_size = length; - lmr->param.registered_address = (DAT_VADDR) (uintptr_t) virt_addr; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " mr_register: mr=%p addr=%p pd %p ctx %p " - "lkey=0x%x rkey=0x%x priv=%x\n", - lmr->mr_handle, lmr->mr_handle->addr, - lmr->mr_handle->pd, lmr->mr_handle->context, - lmr->mr_handle->lkey, lmr->mr_handle->rkey, - length, dapls_convert_privileges(privileges)); - - return DAT_SUCCESS; -} - -/* - * dapl_ib_mr_deregister - * - * Free a memory region - * - * Input: - * lmr pointer to dapl_lmr struct - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_STATE - * - */ -DAT_RETURN dapls_ib_mr_deregister(IN DAPL_LMR * lmr) -{ - if (lmr->mr_handle != IB_INVALID_HANDLE) { - if (ibv_dereg_mr(lmr->mr_handle)) - return (dapl_convert_errno(errno, "dereg_pd")); - lmr->mr_handle = IB_INVALID_HANDLE; - } - return DAT_SUCCESS; -} - -/* - * dapl_ib_mr_register_shared - * - * Register a virtual memory region - * - * Input: - * ia_ptr IA handle - * lmr pointer to dapl_lmr struct - * virt_addr virtual address of beginning of mem region - * length length of memory region - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN -dapls_ib_mr_register_shared(IN DAPL_IA * ia_ptr, - IN DAPL_LMR * lmr, - IN DAT_MEM_PRIV_FLAGS privileges, - IN DAT_VA_TYPE va_type) -{ - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " mr_register_shared: NOT IMPLEMENTED\n"); - - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); -} - -/* - * dapls_ib_mw_alloc - * - * Bind a protection domain to a memory window - * - * Input: - * rmr Initialized rmr to hold binding handles - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN dapls_ib_mw_alloc(IN DAPL_RMR * rmr) -{ - - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_alloc: NOT IMPLEMENTED\n"); - - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); -} - -/* - * dapls_ib_mw_free - * - * Release bindings of a protection domain to a memory window - * - * Input: - * rmr Initialized rmr to hold binding handles - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_STATE - * - */ -DAT_RETURN dapls_ib_mw_free(IN DAPL_RMR * rmr) -{ - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_free: NOT IMPLEMENTED\n"); - - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); -} - -/* - * dapls_ib_mw_bind - * - * Bind a protection domain to a memory window - * - * Input: - * rmr Initialized rmr to hold binding handles - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_PARAMETER; - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN -dapls_ib_mw_bind(IN DAPL_RMR * rmr, - IN DAPL_LMR * lmr, - IN DAPL_EP * ep, - IN DAPL_COOKIE * cookie, - IN DAT_VADDR virtual_address, - IN DAT_VLEN length, - IN DAT_MEM_PRIV_FLAGS mem_priv, IN DAT_BOOLEAN is_signaled) -{ - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_bind: NOT IMPLEMENTED\n"); - - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); -} - -/* - * dapls_ib_mw_unbind - * - * Unbind a protection domain from a memory window - * - * Input: - * rmr Initialized rmr to hold binding handles - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_PARAMETER; - * DAT_INVALID_STATE; - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN -dapls_ib_mw_unbind(IN DAPL_RMR * rmr, - IN DAPL_EP * ep, - IN DAPL_COOKIE * cookie, IN DAT_BOOLEAN is_signaled) -{ - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_unbind: NOT IMPLEMENTED\n"); - - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); -} - -/* - * Local variables: - * c-indent-level: 4 - * c-basic-offset: 4 - * tab-width: 8 - * End: - */ diff --git a/dapl/openib_cma/dapl_ib_qp.c b/dapl/openib_cma/dapl_ib_qp.c deleted file mode 100755 index c9a61c3..0000000 --- a/dapl/openib_cma/dapl_ib_qp.c +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Copyright (c) 2005-2007 Intel Corporation. All rights reserved. - * - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/********************************************************************** - * - * MODULE: dapl_ib_qp.c - * - * PURPOSE: QP routines for access to OFED IB Verbs - * - * $Id: $ - * - **********************************************************************/ - -#include "dapl.h" -#include "dapl_adapter_util.h" - -extern struct rdma_event_channel *g_cm_events; - -/* - * dapl_ib_qp_alloc - * - * Alloc a QP - * - * Input: - * *ep_ptr pointer to EP INFO - * ib_hca_handle provider HCA handle - * ib_pd_handle provider protection domain handle - * cq_recv provider recv CQ handle - * cq_send provider send CQ handle - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INTERNAL_ERROR - * - */ -DAT_RETURN dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr, - IN DAPL_EP * ep_ptr, IN DAPL_EP * ep_ctx_ptr) -{ - DAT_EP_ATTR *attr; - DAPL_EVD *rcv_evd, *req_evd; - ib_cq_handle_t rcv_cq, req_cq; - ib_pd_handle_t ib_pd_handle; - struct ibv_qp_init_attr qp_create; - dp_ib_cm_handle_t conn; - struct rdma_cm_id *cm_id; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " qp_alloc: ia_ptr %p ep_ptr %p ep_ctx_ptr %p\n", - ia_ptr, ep_ptr, ep_ctx_ptr); - - attr = &ep_ptr->param.ep_attr; - ib_pd_handle = ((DAPL_PZ *) ep_ptr->param.pz_handle)->pd_handle; - rcv_evd = (DAPL_EVD *) ep_ptr->param.recv_evd_handle; - req_evd = (DAPL_EVD *) ep_ptr->param.request_evd_handle; - - /* - * DAT allows usage model of EP's with no EVD's but IB does not. - * Create a CQ with zero entries under the covers to support and - * catch any invalid posting. - */ - if (rcv_evd != DAT_HANDLE_NULL) - rcv_cq = rcv_evd->ib_cq_handle; - else if (!ia_ptr->hca_ptr->ib_trans.ib_cq_empty) - rcv_cq = ia_ptr->hca_ptr->ib_trans.ib_cq_empty; - else { - struct ibv_comp_channel *channel = - ia_ptr->hca_ptr->ib_trans.ib_cq; -#ifdef CQ_WAIT_OBJECT - if (rcv_evd->cq_wait_obj_handle) - channel = rcv_evd->cq_wait_obj_handle->events; -#endif - /* Call IB verbs to create CQ */ - rcv_cq = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, - 0, NULL, channel, 0); - - if (rcv_cq == IB_INVALID_HANDLE) - return (dapl_convert_errno(ENOMEM, "create_cq")); - - ia_ptr->hca_ptr->ib_trans.ib_cq_empty = rcv_cq; - } - if (req_evd != DAT_HANDLE_NULL) - req_cq = req_evd->ib_cq_handle; - else - req_cq = ia_ptr->hca_ptr->ib_trans.ib_cq_empty; - - /* - * IMPLEMENTATION NOTE: - * uDAPL allows consumers to post buffers on the EP after creation - * and before a connect request (outbound and inbound). This forces - * a binding to a device during the hca_open call and requires the - * consumer to predetermine which device to listen on or connect from. - * This restriction eliminates any option of listening or connecting - * over multiple devices. uDAPL should add API's to resolve addresses - * and bind to the device at the approriate time (before connect - * and after CR arrives). Discovery should happen at connection time - * based on addressing and not on static configuration during open. - */ - - /* Allocate CM and initialize lock */ - if ((conn = dapl_os_alloc(sizeof(*conn))) == NULL) - return (dapl_convert_errno(ENOMEM, "create_cq")); - - dapl_os_memzero(conn, sizeof(*conn)); - dapl_os_lock_init(&conn->lock); - - /* create CM_ID, bind to local device, create QP */ - if (rdma_create_id(g_cm_events, &cm_id, (void *)conn, RDMA_PS_TCP)) { - dapl_os_free(conn, sizeof(*conn)); - return (dapl_convert_errno(errno, "create_qp")); - } - - /* open identifies the local device; per DAT specification */ - if (rdma_bind_addr(cm_id, - (struct sockaddr *)&ia_ptr->hca_ptr->hca_address)) - goto bail; - - /* Setup attributes and create qp */ - dapl_os_memzero((void *)&qp_create, sizeof(qp_create)); - qp_create.cap.max_send_wr = attr->max_request_dtos; - qp_create.cap.max_send_sge = attr->max_request_iov; - qp_create.cap.max_inline_data = - ia_ptr->hca_ptr->ib_trans.max_inline_send; - qp_create.send_cq = req_cq; - - /* ibv assumes rcv_cq is never NULL, set to req_cq */ - if (rcv_cq == NULL) { - qp_create.recv_cq = req_cq; - qp_create.cap.max_recv_wr = 0; - qp_create.cap.max_recv_sge = 0; - } else { - qp_create.recv_cq = rcv_cq; - qp_create.cap.max_recv_wr = attr->max_recv_dtos; - qp_create.cap.max_recv_sge = attr->max_recv_iov; - } - qp_create.qp_type = IBV_QPT_RC; - qp_create.qp_context = (void *)ep_ptr; - - /* Let uCMA transition QP states */ - if (rdma_create_qp(cm_id, ib_pd_handle, &qp_create)) - goto bail; - - conn->cm_id = cm_id; - conn->ep = ep_ptr; - conn->hca = ia_ptr->hca_ptr; - - /* setup timers for address and route resolution */ - conn->arp_timeout = dapl_os_get_env_val("DAPL_CM_ARP_TIMEOUT_MS", - IB_ARP_TIMEOUT); - conn->arp_retries = dapl_os_get_env_val("DAPL_CM_ARP_RETRY_COUNT", - IB_ARP_RETRY_COUNT); - conn->route_timeout = dapl_os_get_env_val("DAPL_CM_ROUTE_TIMEOUT_MS", - IB_ROUTE_TIMEOUT); - conn->route_retries = dapl_os_get_env_val("DAPL_CM_ROUTE_RETRY_COUNT", - IB_ROUTE_RETRY_COUNT); - - /* setup up ep->param to reference the bound local address and port */ - ep_ptr->param.local_ia_address_ptr = &cm_id->route.addr.src_addr; - ep_ptr->param.local_port_qual = rdma_get_src_port(cm_id); - - ep_ptr->qp_handle = conn; - ep_ptr->qp_state = IB_QP_STATE_INIT; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " qp_alloc: qpn %p sq %d,%d rq %d,%d port=%d\n", - ep_ptr->qp_handle->cm_id->qp->qp_num, - qp_create.cap.max_send_wr, qp_create.cap.max_send_sge, - qp_create.cap.max_recv_wr, qp_create.cap.max_recv_sge, - ep_ptr->param.local_port_qual); - - return DAT_SUCCESS; - bail: - rdma_destroy_id(cm_id); - dapl_os_free(conn, sizeof(*conn)); - return (dapl_convert_errno(errno, "create_qp")); -} - -/* - * dapl_ib_qp_free - * - * Free a QP - * - * Input: - * ia_handle IA handle - * *ep_ptr pointer to EP INFO - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * dapl_convert_errno - * - */ -DAT_RETURN dapls_ib_qp_free(IN DAPL_IA * ia_ptr, IN DAPL_EP * ep_ptr) -{ - dapl_dbg_log(DAPL_DBG_TYPE_EP, " qp_free: ep_ptr %p qp %p\n", - ep_ptr, ep_ptr->qp_handle); - - if (ep_ptr->qp_handle != IB_INVALID_HANDLE) { - /* qp_handle is conn object with reference to cm_id and qp */ - dapli_destroy_conn(ep_ptr->qp_handle); - ep_ptr->qp_handle = IB_INVALID_HANDLE; - ep_ptr->qp_state = IB_QP_STATE_ERROR; - } - return DAT_SUCCESS; -} - -/* - * dapl_ib_qp_modify - * - * Set the QP to the parameters specified in an EP_PARAM - * - * The EP_PARAM structure that is provided has been - * sanitized such that only non-zero values are valid. - * - * Input: - * ib_hca_handle HCA handle - * qp_handle QP handle - * ep_attr Sanitized EP Params - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INVALID_PARAMETER - * - */ -DAT_RETURN dapls_ib_qp_modify(IN DAPL_IA * ia_ptr, - IN DAPL_EP * ep_ptr, IN DAT_EP_ATTR * attr) -{ - struct ibv_qp_attr qp_attr; - - if (ep_ptr->qp_handle == IB_INVALID_HANDLE) - return DAT_INVALID_PARAMETER; - - /* - * Check if we have the right qp_state to modify attributes - */ - if ((ep_ptr->qp_handle->cm_id->qp->state != IBV_QPS_RTR) && - (ep_ptr->qp_handle->cm_id->qp->state != IBV_QPS_RTS)) - return DAT_INVALID_STATE; - - /* Adjust to current EP attributes */ - dapl_os_memzero((void *)&qp_attr, sizeof(qp_attr)); - qp_attr.cap.max_send_wr = attr->max_request_dtos; - qp_attr.cap.max_recv_wr = attr->max_recv_dtos; - qp_attr.cap.max_send_sge = attr->max_request_iov; - qp_attr.cap.max_recv_sge = attr->max_recv_iov; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - "modify_qp: qp %p sq %d,%d, rq %d,%d\n", - ep_ptr->qp_handle->cm_id->qp, - qp_attr.cap.max_send_wr, qp_attr.cap.max_send_sge, - qp_attr.cap.max_recv_wr, qp_attr.cap.max_recv_sge); - - if (ibv_modify_qp(ep_ptr->qp_handle->cm_id->qp, &qp_attr, IBV_QP_CAP)) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - "modify_qp: modify ep %p qp %p failed\n", - ep_ptr, ep_ptr->qp_handle->cm_id->qp); - return (dapl_convert_errno(errno, "modify_qp_state")); - } - - return DAT_SUCCESS; -} - -/* - * dapls_ib_reinit_ep - * - * Move the QP to INIT state again. - * - * Input: - * ep_ptr DAPL_EP - * - * Output: - * none - * - * Returns: - * void - * - */ -void dapls_ib_reinit_ep(IN DAPL_EP * ep_ptr) -{ - /* uCMA does not allow reuse of CM_ID, destroy and create new one */ - if (ep_ptr->qp_handle != IB_INVALID_HANDLE) { - - /* destroy */ - dapli_destroy_conn(ep_ptr->qp_handle); - - /* create new CM_ID and QP */ - ep_ptr->qp_handle = IB_INVALID_HANDLE; - dapls_ib_qp_alloc(ep_ptr->header.owner_ia, ep_ptr, ep_ptr); - } -} - -/* - * Local variables: - * c-indent-level: 4 - * c-basic-offset: 4 - * tab-width: 8 - * End: - */ diff --git a/dapl/openib_cma/dapl_ib_util.c b/dapl/openib_cma/dapl_ib_util.c deleted file mode 100755 index bf23d43..0000000 --- a/dapl/openib_cma/dapl_ib_util.c +++ /dev/null @@ -1,1134 +0,0 @@ -/* - * Copyright (c) 2005-2008 Intel Corporation. All rights reserved. - * - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/********************************************************************** - * - * MODULE: dapl_ib_util.c - * - * PURPOSE: OFED provider - init, open, close, utilities, work thread - * - * $Id:$ - * - **********************************************************************/ - -#ifdef RCSID -static const char rcsid[] = "$Id: $"; -#endif - -#include "openib_osd.h" -#include "dapl.h" -#include "dapl_adapter_util.h" -#include "dapl_ib_util.h" -#include "dapl_osd.h" - -#include - -int g_dapl_loopback_connection = 0; -struct rdma_event_channel *g_cm_events = NULL; -ib_thread_state_t g_ib_thread_state = 0; -DAPL_OS_THREAD g_ib_thread; -DAPL_OS_LOCK g_hca_lock; -struct dapl_llist_entry *g_hca_list; - -#if defined(_WIN64) || defined(_WIN32) -#include "..\..\..\..\..\etc\user\comp_channel.cpp" -#include "..\..\..\..\..\etc\user\dlist.c" -#include - -struct ibvw_windata windata; - -static int getipaddr_netdev(char *name, char *addr, int addr_len) -{ - IWVProvider *prov; - WV_DEVICE_ADDRESS devaddr; - struct addrinfo *res, *ai; - HRESULT hr; - int index; - - if (strncmp(name, "rdma_dev", 8)) { - return EINVAL; - } - - index = atoi(name + 8); - - hr = WvGetObject(&IID_IWVProvider, (LPVOID *) &prov); - if (FAILED(hr)) { - return hr; - } - - hr = getaddrinfo("..localmachine", NULL, NULL, &res); - if (hr) { - goto release; - } - - for (ai = res; ai; ai = ai->ai_next) { - hr = prov->lpVtbl->TranslateAddress(prov, ai->ai_addr, &devaddr); - if (SUCCEEDED(hr) && (ai->ai_addrlen <= addr_len) && (index-- == 0)) { - memcpy(addr, ai->ai_addr, ai->ai_addrlen); - goto free; - } - } - hr = ENODEV; - -free: - freeaddrinfo(res); -release: - prov->lpVtbl->Release(prov); - return hr; -} - -static int dapls_os_init(void) -{ - return ibvw_get_windata(&windata, IBVW_WINDATA_VERSION); -} - -static void dapls_os_release(void) -{ - if (windata.comp_mgr) - ibvw_release_windata(&windata, IBVW_WINDATA_VERSION); - windata.comp_mgr = NULL; -} - -static int dapls_config_comp_channel(struct ibv_comp_channel *channel) -{ - channel->comp_channel.Milliseconds = 0; - return 0; -} - -static int dapls_config_cm_channel(struct rdma_event_channel *channel) -{ - channel->channel.Milliseconds = 0; - return 0; -} - -static int dapls_config_verbs(struct ibv_context *verbs) -{ - verbs->channel.Milliseconds = 0; - return 0; -} - -static int dapls_thread_signal(void) -{ - CompManagerCancel(windata.comp_mgr); - return 0; -} -#else // _WIN64 || WIN32 -int g_ib_pipe[2]; - -static int dapls_os_init(void) -{ - /* create pipe for waking up work thread */ - return pipe(g_ib_pipe); -} - -static void dapls_os_release(void) -{ - /* close pipe? */ -} - -/* Get IP address using network device name */ -static int getipaddr_netdev(char *name, char *addr, int addr_len) -{ - struct ifreq ifr; - int skfd, ret, len; - - /* Fill in the structure */ - snprintf(ifr.ifr_name, IFNAMSIZ, "%s", name); - ifr.ifr_hwaddr.sa_family = ARPHRD_INFINIBAND; - - /* Create a socket fd */ - skfd = socket(PF_INET, SOCK_STREAM, 0); - ret = ioctl(skfd, SIOCGIFADDR, &ifr); - if (ret) - goto bail; - - switch (ifr.ifr_addr.sa_family) { -#ifdef AF_INET6 - case AF_INET6: - len = sizeof(struct sockaddr_in6); - break; -#endif - case AF_INET: - default: - len = sizeof(struct sockaddr); - break; - } - - if (len <= addr_len) - memcpy(addr, &ifr.ifr_addr, len); - else - ret = EINVAL; - - bail: - close(skfd); - return ret; -} - -static int dapls_config_fd(int fd) -{ - int opts; - - opts = fcntl(fd, F_GETFL); - if (opts < 0 || fcntl(fd, F_SETFL, opts | O_NONBLOCK) < 0) { - dapl_log(DAPL_DBG_TYPE_ERR, - " dapls_config_fd: fcntl on fd %d ERR %d %s\n", - fd, opts, strerror(errno)); - return errno; - } - - return 0; -} - -static int dapls_config_comp_channel(struct ibv_comp_channel *channel) -{ - return dapls_config_fd(channel->fd); -} - -static int dapls_config_cm_channel(struct rdma_event_channel *channel) -{ - return dapls_config_fd(channel->fd); -} - -static int dapls_config_verbs(struct ibv_context *verbs) -{ - return dapls_config_fd(verbs->async_fd); -} - -static int dapls_thread_signal(void) -{ - return write(g_ib_pipe[1], "w", sizeof "w"); -} -#endif - -/* Get IP address using network name, address, or device name */ -static int getipaddr(char *name, char *addr, int len) -{ - struct addrinfo *res; - - /* assume netdev for first attempt, then network and address type */ - if (getipaddr_netdev(name, addr, len)) { - if (getaddrinfo(name, NULL, NULL, &res)) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: getaddr_netdev ERROR:" - " %s. Is %s configured?\n", - strerror(errno), name); - return 1; - } else { - if (len >= res->ai_addrlen) - memcpy(addr, res->ai_addr, res->ai_addrlen); - else { - freeaddrinfo(res); - return 1; - } - freeaddrinfo(res); - } - } - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " getipaddr: family %d port %d addr %d.%d.%d.%d\n", - ((struct sockaddr_in *)addr)->sin_family, - ((struct sockaddr_in *)addr)->sin_port, - ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 0 & 0xff, - ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 8 & 0xff, - ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 16 & 0xff, - ((struct sockaddr_in *)addr)->sin_addr. - s_addr >> 24 & 0xff); - - return 0; -} - -/* - * dapls_ib_init, dapls_ib_release - * - * Initialize Verb related items for device open - * - * Input: - * none - * - * Output: - * none - * - * Returns: - * 0 success, -1 error - * - */ -int32_t dapls_ib_init(void) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_init: \n"); - - /* initialize hca_list lock */ - dapl_os_lock_init(&g_hca_lock); - - /* initialize hca list for CQ events */ - dapl_llist_init_head(&g_hca_list); - - if (dapls_os_init()) - return 1; - - return 0; -} - -int32_t dapls_ib_release(void) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_release: \n"); - dapli_ib_thread_destroy(); - if (g_cm_events != NULL) - rdma_destroy_event_channel(g_cm_events); - dapls_os_release(); - return 0; -} - -/* - * dapls_ib_open_hca - * - * Open HCA - * - * Input: - * *hca_name pointer to provider device name - * *ib_hca_handle_p pointer to provide HCA handle - * - * Output: - * none - * - * Return: - * DAT_SUCCESS - * dapl_convert_errno - * - */ -DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr) -{ - struct rdma_cm_id *cm_id = NULL; - union ibv_gid *gid; - int ret; - DAT_RETURN dat_status; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " open_hca: %s - %p\n", hca_name, hca_ptr); - - /* Setup the global cm event channel */ - dapl_os_lock(&g_hca_lock); - if (g_cm_events == NULL) { - g_cm_events = rdma_create_event_channel(); - if (g_cm_events == NULL) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " open_hca: ERR - RDMA channel %s\n", - strerror(errno)); - return DAT_INTERNAL_ERROR; - } - } - dapl_os_unlock(&g_hca_lock); - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " open_hca: RDMA channel created (%p)\n", g_cm_events); - - dat_status = dapli_ib_thread_init(); - if (dat_status != DAT_SUCCESS) - return dat_status; - - /* HCA name will be hostname or IP address */ - if (getipaddr((char *)hca_name, - (char *)&hca_ptr->hca_address, sizeof(DAT_SOCK_ADDR6))) - return DAT_INVALID_ADDRESS; - - /* cm_id will bind local device/GID based on IP address */ - if (rdma_create_id(g_cm_events, &cm_id, (void *)hca_ptr, RDMA_PS_TCP)) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: rdma_create_id ERR %s\n", strerror(errno)); - return DAT_INTERNAL_ERROR; - } - ret = rdma_bind_addr(cm_id, (struct sockaddr *)&hca_ptr->hca_address); - if ((ret) || (cm_id->verbs == NULL)) { - rdma_destroy_id(cm_id); - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: rdma_bind ERR %s." - " Is %s configured?\n", strerror(errno), hca_name); - return DAT_INVALID_ADDRESS; - } - - /* keep reference to IB device and cm_id */ - hca_ptr->ib_trans.cm_id = cm_id; - hca_ptr->ib_hca_handle = cm_id->verbs; - dapls_config_verbs(cm_id->verbs); - hca_ptr->port_num = cm_id->port_num; - gid = &cm_id->route.addr.addr.ibaddr.sgid; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " open_hca: ctx=%p port=%d GID subnet %016llx id %016llx\n", - cm_id->verbs, cm_id->port_num, - (unsigned long long)ntohll(gid->global.subnet_prefix), - (unsigned long long)ntohll(gid->global.interface_id)); - - /* set inline max with env or default, get local lid and gid 0 */ - if (hca_ptr->ib_hca_handle->device->transport_type - == IBV_TRANSPORT_IWARP) - hca_ptr->ib_trans.max_inline_send = - dapl_os_get_env_val("DAPL_MAX_INLINE", - INLINE_SEND_IWARP_DEFAULT); - else - hca_ptr->ib_trans.max_inline_send = - dapl_os_get_env_val("DAPL_MAX_INLINE", - INLINE_SEND_IB_DEFAULT); - - /* set CM timer defaults */ - hca_ptr->ib_trans.max_cm_timeout = - dapl_os_get_env_val("DAPL_MAX_CM_RESPONSE_TIME", - IB_CM_RESPONSE_TIMEOUT); - hca_ptr->ib_trans.max_cm_retries = - dapl_os_get_env_val("DAPL_MAX_CM_RETRIES", IB_CM_RETRIES); - - /* EVD events without direct CQ channels, non-blocking */ - hca_ptr->ib_trans.ib_cq = - ibv_create_comp_channel(hca_ptr->ib_hca_handle); - if (hca_ptr->ib_trans.ib_cq == NULL) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: ibv_create_comp_channel ERR %s\n", - strerror(errno)); - goto bail; - } - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " open_hca: CQ channel created\n"); - - if (dapls_config_comp_channel(hca_ptr->ib_trans.ib_cq)) { - goto bail; - } - - /* - * Put new hca_transport on list for async and CQ event processing - * Wakeup work thread to add to polling list - */ - dapl_llist_init_entry((DAPL_LLIST_ENTRY *) & hca_ptr->ib_trans.entry); - dapl_os_lock(&g_hca_lock); - dapl_llist_add_tail(&g_hca_list, - (DAPL_LLIST_ENTRY *) & hca_ptr->ib_trans.entry, - &hca_ptr->ib_trans.entry); - if (dapls_thread_signal() == -1) - dapl_log(DAPL_DBG_TYPE_UTIL, - " open_hca: thread wakeup error = %s\n", - strerror(errno)); - dapl_os_unlock(&g_hca_lock); - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " open_hca: %s, %s %d.%d.%d.%d INLINE_MAX=%d\n", hca_name, - ((struct sockaddr_in *) - &hca_ptr->hca_address)->sin_family == AF_INET ? - "AF_INET" : "AF_INET6", ((struct sockaddr_in *) - &hca_ptr->hca_address)->sin_addr. - s_addr >> 0 & 0xff, ((struct sockaddr_in *) - &hca_ptr->hca_address)->sin_addr. - s_addr >> 8 & 0xff, ((struct sockaddr_in *) - &hca_ptr->hca_address)->sin_addr. - s_addr >> 16 & 0xff, ((struct sockaddr_in *) - &hca_ptr->hca_address)->sin_addr. - s_addr >> 24 & 0xff, hca_ptr->ib_trans.max_inline_send); - - hca_ptr->ib_trans.d_hca = hca_ptr; - return DAT_SUCCESS; - bail: - rdma_destroy_id(hca_ptr->ib_trans.cm_id); - hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; - return DAT_INTERNAL_ERROR; -} - -/* - * dapls_ib_close_hca - * - * Open HCA - * - * Input: - * DAPL_HCA provide CA handle - * - * Output: - * none - * - * Return: - * DAT_SUCCESS - * dapl_convert_errno - * - */ -DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p->%p\n", - hca_ptr, hca_ptr->ib_hca_handle); - - if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) { - if (rdma_destroy_id(hca_ptr->ib_trans.cm_id)) - return (dapl_convert_errno(errno, "ib_close_device")); - hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; - } - - dapl_os_lock(&g_hca_lock); - if (g_ib_thread_state != IB_THREAD_RUN) { - dapl_os_unlock(&g_hca_lock); - goto bail; - } - dapl_os_unlock(&g_hca_lock); - - /* - * Remove hca from async and CQ event processing list - * Wakeup work thread to remove from polling list - */ - hca_ptr->ib_trans.destroy = 1; - if (dapls_thread_signal() == -1) - dapl_log(DAPL_DBG_TYPE_UTIL, - " destroy: thread wakeup error = %s\n", - strerror(errno)); - - /* wait for thread to remove HCA references */ - while (hca_ptr->ib_trans.destroy != 2) { - if (dapls_thread_signal() == -1) - dapl_log(DAPL_DBG_TYPE_UTIL, - " destroy: thread wakeup error = %s\n", - strerror(errno)); - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread_destroy: wait on hca %p destroy\n"); - dapl_os_sleep_usec(10000); - } - bail: - return (DAT_SUCCESS); -} - -/* - * dapls_ib_query_hca - * - * Query the hca attribute - * - * Input: - * hca_handl hca handle - * ia_attr attribute of the ia - * ep_attr attribute of the ep - * ip_addr ip address of DET NIC - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_HANDLE - */ - -DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr, - OUT DAT_IA_ATTR * ia_attr, - OUT DAT_EP_ATTR * ep_attr, - OUT DAT_SOCK_ADDR6 * ip_addr) -{ - struct ibv_device_attr dev_attr; - struct ibv_port_attr port_attr; - - if (hca_ptr->ib_hca_handle == NULL) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " query_hca: BAD handle\n"); - return (DAT_INVALID_HANDLE); - } - - /* local IP address of device, set during ia_open */ - if (ip_addr != NULL) - memcpy(ip_addr, &hca_ptr->hca_address, sizeof(DAT_SOCK_ADDR6)); - - if (ia_attr == NULL && ep_attr == NULL) - return DAT_SUCCESS; - - /* query verbs for this device and port attributes */ - if (ibv_query_device(hca_ptr->ib_hca_handle, &dev_attr) || - ibv_query_port(hca_ptr->ib_hca_handle, - hca_ptr->port_num, &port_attr)) - return (dapl_convert_errno(errno, "ib_query_hca")); - - /* - * There is no query for inline data so there is no way to - * calculate the impact on sge nor the max inline send. Most - * implementions consume 1 or none so just reduce by 1 until - * we are provided with a query mechanism from verbs. - */ - if (hca_ptr->ib_trans.max_inline_send) - dev_attr.max_sge--; - - if (ia_attr != NULL) { - (void)dapl_os_memzero(ia_attr, sizeof(*ia_attr)); - ia_attr->adapter_name[DAT_NAME_MAX_LENGTH - 1] = '\0'; - ia_attr->vendor_name[DAT_NAME_MAX_LENGTH - 1] = '\0'; - ia_attr->ia_address_ptr = - (DAT_IA_ADDRESS_PTR) & hca_ptr->hca_address; - - dapl_log(DAPL_DBG_TYPE_UTIL, - "dapl_query_hca: %s %s %s\n", hca_ptr->name, - ((struct sockaddr_in *) - ia_attr->ia_address_ptr)->sin_family == AF_INET ? - "AF_INET" : "AF_INET6", - inet_ntoa(((struct sockaddr_in *) - ia_attr->ia_address_ptr)->sin_addr)); - - ia_attr->hardware_version_major = dev_attr.hw_ver; - ia_attr->max_eps = dev_attr.max_qp; - ia_attr->max_dto_per_ep = dev_attr.max_qp_wr; - ia_attr->max_rdma_read_in = dev_attr.max_res_rd_atom; - ia_attr->max_rdma_read_out = dev_attr.max_qp_init_rd_atom; - ia_attr->max_rdma_read_per_ep_in = dev_attr.max_qp_rd_atom; - ia_attr->max_rdma_read_per_ep_out = - dev_attr.max_qp_init_rd_atom; - ia_attr->max_rdma_read_per_ep_in_guaranteed = DAT_TRUE; - ia_attr->max_rdma_read_per_ep_out_guaranteed = DAT_TRUE; - ia_attr->max_evds = dev_attr.max_cq; - ia_attr->max_evd_qlen = dev_attr.max_cqe; - ia_attr->max_iov_segments_per_dto = dev_attr.max_sge; - ia_attr->max_lmrs = dev_attr.max_mr; - /* 32bit attribute from 64bit, 4G-1 limit, DAT v2 needs fix */ - ia_attr->max_lmr_block_size = - (dev_attr.max_mr_size >> 32) ? ~0 : dev_attr.max_mr_size; - ia_attr->max_rmrs = dev_attr.max_mw; - ia_attr->max_lmr_virtual_address = dev_attr.max_mr_size; - ia_attr->max_rmr_target_address = dev_attr.max_mr_size; - ia_attr->max_pzs = dev_attr.max_pd; - ia_attr->max_mtu_size = port_attr.max_msg_sz; - ia_attr->max_rdma_size = port_attr.max_msg_sz; - ia_attr->num_transport_attr = 0; - ia_attr->transport_attr = NULL; - ia_attr->num_vendor_attr = 0; - ia_attr->vendor_attr = NULL; - /* iWARP spec. - 1 sge for RDMA reads */ - if (hca_ptr->ib_hca_handle->device->transport_type - == IBV_TRANSPORT_IWARP) - ia_attr->max_iov_segments_per_rdma_read = 1; - else - ia_attr->max_iov_segments_per_rdma_read = - dev_attr.max_sge; - - ia_attr->max_iov_segments_per_rdma_write = dev_attr.max_sge; - /* save rd_atom for peer validation during connect requests */ - hca_ptr->ib_trans.max_rdma_rd_in = dev_attr.max_qp_rd_atom; - hca_ptr->ib_trans.max_rdma_rd_out = - dev_attr.max_qp_init_rd_atom; -#ifdef DAT_EXTENSIONS - ia_attr->extension_supported = DAT_EXTENSION_IB; - ia_attr->extension_version = DAT_IB_EXTENSION_VERSION; -#endif - dapl_log(DAPL_DBG_TYPE_UTIL, - "dapl_query_hca: (ver=%x) ep's %d ep_q %d" - " evd's %d evd_q %d mr %u\n", - ia_attr->hardware_version_major, - ia_attr->max_eps, ia_attr->max_dto_per_ep, - ia_attr->max_evds, ia_attr->max_evd_qlen, - ia_attr->max_lmr_block_size); - dapl_log(DAPL_DBG_TYPE_UTIL, - "dapl_query_hca: msg %llu rdma %llu iov's %d" - " lmr %d rmr %d rd_in,out %d,%d inline=%d\n", - ia_attr->max_mtu_size, ia_attr->max_rdma_size, - ia_attr->max_iov_segments_per_dto, ia_attr->max_lmrs, - ia_attr->max_rmrs, ia_attr->max_rdma_read_per_ep_in, - ia_attr->max_rdma_read_per_ep_out, - hca_ptr->ib_trans.max_inline_send); - } - - if (ep_attr != NULL) { - (void)dapl_os_memzero(ep_attr, sizeof(*ep_attr)); - ep_attr->max_mtu_size = port_attr.max_msg_sz; - ep_attr->max_rdma_size = port_attr.max_msg_sz; - ep_attr->max_recv_dtos = dev_attr.max_qp_wr; - ep_attr->max_request_dtos = dev_attr.max_qp_wr; - ep_attr->max_recv_iov = dev_attr.max_sge; - ep_attr->max_request_iov = dev_attr.max_sge; - ep_attr->max_rdma_read_in = dev_attr.max_qp_rd_atom; - ep_attr->max_rdma_read_out = dev_attr.max_qp_init_rd_atom; - /* iWARP spec. - 1 sge for RDMA reads */ - if (hca_ptr->ib_hca_handle->device->transport_type - == IBV_TRANSPORT_IWARP) - ep_attr->max_rdma_read_iov = 1; - else - ep_attr->max_rdma_read_iov = dev_attr.max_sge; - - ep_attr->max_rdma_write_iov = dev_attr.max_sge; - dapl_log(DAPL_DBG_TYPE_UTIL, - "dapl_query_hca: MAX msg %llu dto %d iov %d" - " rdma i%d,o%d\n", - ep_attr->max_mtu_size, - ep_attr->max_recv_dtos, ep_attr->max_recv_iov, - ep_attr->max_rdma_read_in, ep_attr->max_rdma_read_out); - } - return DAT_SUCCESS; -} - -/* - * dapls_ib_setup_async_callback - * - * Set up an asynchronous callbacks of various kinds - * - * Input: - * ia_handle IA handle - * handler_type type of handler to set up - * callback_handle handle param for completion callbacks - * callback callback routine pointer - * context argument for callback routine - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INVALID_PARAMETER - * - */ -DAT_RETURN dapls_ib_setup_async_callback(IN DAPL_IA * ia_ptr, - IN DAPL_ASYNC_HANDLER_TYPE type, - IN DAPL_EVD * evd_ptr, - IN ib_async_handler_t callback, - IN void *context) -{ - ib_hca_transport_t *hca_ptr; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " setup_async_cb: ia %p type %d hdl %p cb %p ctx %p\n", - ia_ptr, type, evd_ptr, callback, context); - - hca_ptr = &ia_ptr->hca_ptr->ib_trans; - switch (type) { - case DAPL_ASYNC_UNAFILIATED: - hca_ptr->async_unafiliated = (ib_async_handler_t) callback; - hca_ptr->async_un_ctx = context; - break; - case DAPL_ASYNC_CQ_ERROR: - hca_ptr->async_cq_error = (ib_async_cq_handler_t) callback; - break; - case DAPL_ASYNC_CQ_COMPLETION: - hca_ptr->async_cq = (ib_async_dto_handler_t) callback; - break; - case DAPL_ASYNC_QP_ERROR: - hca_ptr->async_qp_error = (ib_async_qp_handler_t) callback; - break; - default: - break; - } - return DAT_SUCCESS; -} - -DAT_RETURN dapli_ib_thread_init(void) -{ - DAT_RETURN dat_status; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread_init(%d)\n", dapl_os_getpid()); - - dapl_os_lock(&g_hca_lock); - if (g_ib_thread_state != IB_THREAD_INIT) { - dapl_os_unlock(&g_hca_lock); - return DAT_SUCCESS; - } - - /* uCMA events non-blocking */ - if (dapls_config_cm_channel(g_cm_events)) { - dapl_os_unlock(&g_hca_lock); - return (dapl_convert_errno(errno, "create_thread ERR: cm_fd")); - } - - g_ib_thread_state = IB_THREAD_CREATE; - dapl_os_unlock(&g_hca_lock); - - /* create thread to process inbound connect request */ - dat_status = dapl_os_thread_create(dapli_thread, NULL, &g_ib_thread); - if (dat_status != DAT_SUCCESS) - return (dapl_convert_errno(errno, - "create_thread ERR:" - " check resource limits")); - - /* wait for thread to start */ - dapl_os_lock(&g_hca_lock); - while (g_ib_thread_state != IB_THREAD_RUN) { - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread_init: waiting for ib_thread\n"); - dapl_os_unlock(&g_hca_lock); - dapl_os_sleep_usec(2000); - dapl_os_lock(&g_hca_lock); - } - dapl_os_unlock(&g_hca_lock); - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread_init(%d) exit\n", dapl_os_getpid()); - - return DAT_SUCCESS; -} - -void dapli_ib_thread_destroy(void) -{ - int retries = 10; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread_destroy(%d)\n", dapl_os_getpid()); - /* - * wait for async thread to terminate. - * pthread_join would be the correct method - * but some applications have some issues - */ - - /* destroy ib_thread, wait for termination, if not already */ - dapl_os_lock(&g_hca_lock); - if (g_ib_thread_state != IB_THREAD_RUN) - goto bail; - - g_ib_thread_state = IB_THREAD_CANCEL; - if (dapls_thread_signal() == -1) - dapl_log(DAPL_DBG_TYPE_UTIL, - " destroy: thread wakeup error = %s\n", - strerror(errno)); - while ((g_ib_thread_state != IB_THREAD_EXIT) && (retries--)) { - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread_destroy: waiting for ib_thread\n"); - if (dapls_thread_signal() == -1) - dapl_log(DAPL_DBG_TYPE_UTIL, - " destroy: thread wakeup error = %s\n", - strerror(errno)); - dapl_os_unlock(&g_hca_lock); - dapl_os_sleep_usec(2000); - dapl_os_lock(&g_hca_lock); - } - - bail: - dapl_os_unlock(&g_hca_lock); - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread_destroy(%d) exit\n", dapl_os_getpid()); -} - -void dapli_async_event_cb(struct _ib_hca_transport *hca) -{ - struct ibv_async_event event; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " async_event(%p)\n", hca); - - if (hca->destroy) - return; - - if (!ibv_get_async_event(hca->cm_id->verbs, &event)) { - - switch (event.event_type) { - case IBV_EVENT_CQ_ERR: - { - struct dapl_ep *evd_ptr = - event.element.cq->cq_context; - - dapl_log(DAPL_DBG_TYPE_ERR, - "dapl async_event CQ (%p) ERR %d\n", - evd_ptr, event.event_type); - - /* report up if async callback still setup */ - if (hca->async_cq_error) - hca->async_cq_error(hca->cm_id->verbs, - event.element.cq, - &event, - (void *)evd_ptr); - break; - } - case IBV_EVENT_COMM_EST: - { - /* Received msgs on connected QP before RTU */ - dapl_log(DAPL_DBG_TYPE_UTIL, - " async_event COMM_EST(%p) rdata beat RTU\n", - event.element.qp); - - break; - } - case IBV_EVENT_QP_FATAL: - case IBV_EVENT_QP_REQ_ERR: - case IBV_EVENT_QP_ACCESS_ERR: - case IBV_EVENT_QP_LAST_WQE_REACHED: - case IBV_EVENT_SRQ_ERR: - case IBV_EVENT_SRQ_LIMIT_REACHED: - case IBV_EVENT_SQ_DRAINED: - { - struct dapl_ep *ep_ptr = - event.element.qp->qp_context; - - dapl_log(DAPL_DBG_TYPE_ERR, - "dapl async_event QP (%p) ERR %d\n", - ep_ptr, event.event_type); - - /* report up if async callback still setup */ - if (hca->async_qp_error) - hca->async_qp_error(hca->cm_id->verbs, - ep_ptr->qp_handle, - &event, - (void *)ep_ptr); - break; - } - case IBV_EVENT_PATH_MIG: - case IBV_EVENT_PATH_MIG_ERR: - case IBV_EVENT_DEVICE_FATAL: - case IBV_EVENT_PORT_ACTIVE: - case IBV_EVENT_PORT_ERR: - case IBV_EVENT_LID_CHANGE: - case IBV_EVENT_PKEY_CHANGE: - case IBV_EVENT_SM_CHANGE: - { - dapl_log(DAPL_DBG_TYPE_WARN, - "dapl async_event: DEV ERR %d\n", - event.event_type); - - /* report up if async callback still setup */ - if (hca->async_unafiliated) - hca->async_unafiliated(hca->cm_id-> - verbs, &event, - hca-> - async_un_ctx); - break; - } - case IBV_EVENT_CLIENT_REREGISTER: - /* no need to report this event this time */ - dapl_log(DAPL_DBG_TYPE_UTIL, - " async_event: IBV_EVENT_CLIENT_REREGISTER\n"); - break; - - default: - dapl_log(DAPL_DBG_TYPE_WARN, - "dapl async_event: %d UNKNOWN\n", - event.event_type); - break; - - } - ibv_ack_async_event(&event); - } -} - -#if defined(_WIN64) || defined(_WIN32) -/* work thread for uAT, uCM, CQ, and async events */ -void dapli_thread(void *arg) -{ - struct _ib_hca_transport *hca; - struct _ib_hca_transport *uhca[8]; - COMP_CHANNEL *channel; - int ret, idx, cnt; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " ib_thread(%d,0x%x): ENTER: \n", - dapl_os_getpid(), g_ib_thread); - - dapl_os_lock(&g_hca_lock); - for (g_ib_thread_state = IB_THREAD_RUN; - g_ib_thread_state == IB_THREAD_RUN; dapl_os_lock(&g_hca_lock)) { - - idx = 0; - hca = dapl_llist_is_empty(&g_hca_list) ? NULL : - dapl_llist_peek_head(&g_hca_list); - - while (hca) { - uhca[idx++] = hca; - hca = dapl_llist_next_entry(&g_hca_list, - (DAPL_LLIST_ENTRY *) & hca-> - entry); - } - cnt = idx; - - dapl_os_unlock(&g_hca_lock); - ret = CompManagerPoll(windata.comp_mgr, INFINITE, &channel); - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread(%d) poll_event 0x%x\n", - dapl_os_getpid(), ret); - - dapli_cma_event_cb(); - - /* check and process CQ and ASYNC events, per device */ - for (idx = 0; idx < cnt; idx++) { - if (uhca[idx]->destroy == 1) { - dapl_os_lock(&g_hca_lock); - dapl_llist_remove_entry(&g_hca_list, - (DAPL_LLIST_ENTRY *) & - uhca[idx]->entry); - dapl_os_unlock(&g_hca_lock); - uhca[idx]->destroy = 2; - } else { - dapli_cq_event_cb(uhca[idx]); - dapli_async_event_cb(uhca[idx]); - } - } - } - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " ib_thread(%d) EXIT\n", - dapl_os_getpid()); - g_ib_thread_state = IB_THREAD_EXIT; - dapl_os_unlock(&g_hca_lock); -} -#else // _WIN64 || WIN32 -/* work thread for uAT, uCM, CQ, and async events */ -void dapli_thread(void *arg) -{ - struct pollfd ufds[__FD_SETSIZE]; - struct _ib_hca_transport *uhca[__FD_SETSIZE] = { NULL }; - struct _ib_hca_transport *hca; - int ret, idx, fds; - char rbuf[2]; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread(%d,0x%x): ENTER: pipe %d ucma %d\n", - dapl_os_getpid(), g_ib_thread, g_ib_pipe[0], - g_cm_events->fd); - - /* Poll across pipe, CM, AT never changes */ - dapl_os_lock(&g_hca_lock); - g_ib_thread_state = IB_THREAD_RUN; - - ufds[0].fd = g_ib_pipe[0]; /* pipe */ - ufds[0].events = POLLIN; - ufds[1].fd = g_cm_events->fd; /* uCMA */ - ufds[1].events = POLLIN; - - while (g_ib_thread_state == IB_THREAD_RUN) { - - /* build ufds after pipe and uCMA events */ - ufds[0].revents = 0; - ufds[1].revents = 0; - idx = 1; - - /* Walk HCA list and setup async and CQ events */ - if (!dapl_llist_is_empty(&g_hca_list)) - hca = dapl_llist_peek_head(&g_hca_list); - else - hca = NULL; - - while (hca) { - - /* uASYNC events */ - ufds[++idx].fd = hca->cm_id->verbs->async_fd; - ufds[idx].events = POLLIN; - ufds[idx].revents = 0; - uhca[idx] = hca; - - /* uCQ, non-direct events */ - ufds[++idx].fd = hca->ib_cq->fd; - ufds[idx].events = POLLIN; - ufds[idx].revents = 0; - uhca[idx] = hca; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread(%d) poll_fd: hca[%d]=%p, async=%d" - " pipe=%d cm=%d cq=d\n", - dapl_os_getpid(), hca, ufds[idx - 1].fd, - ufds[0].fd, ufds[1].fd, ufds[idx].fd); - - hca = dapl_llist_next_entry(&g_hca_list, - (DAPL_LLIST_ENTRY *) & hca-> - entry); - } - - /* unlock, and setup poll */ - fds = idx + 1; - dapl_os_unlock(&g_hca_lock); - ret = poll(ufds, fds, -1); - if (ret <= 0) { - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread(%d): ERR %s poll\n", - dapl_os_getpid(), strerror(errno)); - dapl_os_lock(&g_hca_lock); - continue; - } - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " ib_thread(%d) poll_event: " - " async=0x%x pipe=0x%x cm=0x%x cq=0x%x\n", - dapl_os_getpid(), ufds[idx - 1].revents, - ufds[0].revents, ufds[1].revents, - ufds[idx].revents); - - /* uCMA events */ - if (ufds[1].revents == POLLIN) - dapli_cma_event_cb(); - - /* check and process CQ and ASYNC events, per device */ - for (idx = 2; idx < fds; idx++) { - if (ufds[idx].revents == POLLIN) { - dapli_cq_event_cb(uhca[idx]); - dapli_async_event_cb(uhca[idx]); - } - } - - /* check and process user events, PIPE */ - if (ufds[0].revents == POLLIN) { - if (read(g_ib_pipe[0], rbuf, 2) == -1) - dapl_log(DAPL_DBG_TYPE_UTIL, - " cr_thread: pipe rd err= %s\n", - strerror(errno)); - - /* cleanup any device on list marked for destroy */ - for (idx = 3; idx < fds; idx++) { - if (uhca[idx] && uhca[idx]->destroy == 1) { - dapl_os_lock(&g_hca_lock); - dapl_llist_remove_entry(&g_hca_list, - (DAPL_LLIST_ENTRY - *) - & uhca[idx]-> - entry); - dapl_os_unlock(&g_hca_lock); - uhca[idx]->destroy = 2; - } - } - } - dapl_os_lock(&g_hca_lock); - } - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " ib_thread(%d) EXIT\n", - dapl_os_getpid()); - g_ib_thread_state = IB_THREAD_EXIT; - dapl_os_unlock(&g_hca_lock); -} -#endif - -/* - * dapls_set_provider_specific_attr - * - * Input: - * attr_ptr Pointer provider specific attributes - * - * Output: - * none - * - * Returns: - * void - */ -DAT_NAMED_ATTR ib_attrs[] = { -#ifdef DAT_EXTENSIONS - { - "DAT_EXTENSION_INTERFACE", "TRUE"} - , - { - DAT_IB_ATTR_FETCH_AND_ADD, "TRUE"} - , - { - DAT_IB_ATTR_CMP_AND_SWAP, "TRUE"} - , - { - DAT_IB_ATTR_IMMED_DATA, "TRUE"} - , -#ifdef DAPL_COUNTERS - { - DAT_ATTR_COUNTERS, "TRUE"} - , -#endif /* DAPL_COUNTERS */ -#endif -}; - -#define SPEC_ATTR_SIZE( x ) (sizeof( x ) / sizeof( DAT_NAMED_ATTR)) - -void dapls_query_provider_specific_attr(IN DAPL_IA * ia_ptr, - IN DAT_PROVIDER_ATTR * attr_ptr) -{ - attr_ptr->num_provider_specific_attr = SPEC_ATTR_SIZE(ib_attrs); - attr_ptr->provider_specific_attr = ib_attrs; -} diff --git a/dapl/openib_cma/dapl_ib_util.h b/dapl/openib_cma/dapl_ib_util.h index dde5fac..f466c06 100755 --- a/dapl/openib_cma/dapl_ib_util.h +++ b/dapl/openib_cma/dapl_ib_util.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005-2008 Intel Corporation. All rights reserved. + * Copyright (c) 2005-2009 Intel Corporation. All rights reserved. * * This Software is licensed under one of the following licenses: * @@ -24,35 +24,18 @@ * notice, one of the license notices in the documentation * and/or other materials provided with the distribution. */ - -/********************************************************************** - * - * MODULE: dapl_ib_util.h - * - * PURPOSE: The OFED provider - definitions, prototypes, - * - * $Id: $ - * - **********************************************************************/ - +/* + * Definitions specific to OpenIB CMA provider. + * Connection manager - rdma_cma, provided in separate library. + */ #ifndef _DAPL_IB_UTIL_H_ #define _DAPL_IB_UTIL_H_ +#define _OPENIB_CMA_ -#include "openib_osd.h" #include #include - -/* Typedefs to map common DAPL provider types to IB verbs */ -typedef struct dapl_cm_id *ib_qp_handle_t; -typedef struct ibv_cq *ib_cq_handle_t; -typedef struct ibv_pd *ib_pd_handle_t; -typedef struct ibv_mr *ib_mr_handle_t; -typedef struct ibv_mw *ib_mw_handle_t; -typedef struct ibv_wc ib_work_completion_t; - -/* HCA context type maps to IB verbs */ -typedef struct ibv_context *ib_hca_handle_t; -typedef ib_hca_handle_t dapl_ibal_ca_t; +#include "openib_osd.h" +#include "dapl_ib_common.h" #define IB_RC_RETRY_COUNT 7 #define IB_RNR_RETRY_COUNT 7 @@ -64,56 +47,6 @@ typedef ib_hca_handle_t dapl_ibal_ca_t; #define IB_ROUTE_RETRY_COUNT 15 /* 60 sec total */ #define IB_MAX_AT_RETRY 3 -typedef enum { - IB_CME_CONNECTED, - IB_CME_DISCONNECTED, - IB_CME_DISCONNECTED_ON_LINK_DOWN, - IB_CME_CONNECTION_REQUEST_PENDING, - IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA, - IB_CME_CONNECTION_REQUEST_ACKED, - IB_CME_DESTINATION_REJECT, - IB_CME_DESTINATION_REJECT_PRIVATE_DATA, - IB_CME_DESTINATION_UNREACHABLE, - IB_CME_TOO_MANY_CONNECTION_REQUESTS, - IB_CME_LOCAL_FAILURE, - IB_CME_BROKEN, - IB_CME_TIMEOUT -} ib_cm_events_t; - -/* CQ notifications */ -typedef enum -{ - IB_NOTIFY_ON_NEXT_COMP, - IB_NOTIFY_ON_SOLIC_COMP - -} ib_notification_type_t; - -/* other mappings */ -typedef int ib_bool_t; -typedef union ibv_gid GID; -typedef char *IB_HCA_NAME; -typedef uint16_t ib_hca_port_t; -typedef uint32_t ib_comp_handle_t; - -#ifdef CQ_WAIT_OBJECT - -/* CQ event channel, plus pipe to enable consumer wakeup */ -typedef struct _ib_wait_obj_handle -{ - struct ibv_comp_channel *events; - int pipe[2]; - -} *ib_wait_obj_handle_t; - -#endif - -/* Definitions */ -#define IB_INVALID_HANDLE NULL - -/* inline send rdma threshold */ -#define INLINE_SEND_IWARP_DEFAULT 64 -#define INLINE_SEND_IB_DEFAULT 200 - /* CMA private data areas */ #define CMA_PDATA_HDR 36 #define IB_MAX_REQ_PDATA_SIZE (92-CMA_PDATA_HDR) @@ -123,38 +56,6 @@ typedef struct _ib_wait_obj_handle #define IB_MAX_DREP_PDATA_SIZE (224-CMA_PDATA_HDR) #define IWARP_MAX_PDATA_SIZE (512-CMA_PDATA_HDR) -/* DTO OPs, ordered for DAPL ENUM definitions */ -#define OP_RDMA_WRITE IBV_WR_RDMA_WRITE -#define OP_RDMA_WRITE_IMM IBV_WR_RDMA_WRITE_WITH_IMM -#define OP_SEND IBV_WR_SEND -#define OP_SEND_IMM IBV_WR_SEND_WITH_IMM -#define OP_RDMA_READ IBV_WR_RDMA_READ -#define OP_COMP_AND_SWAP IBV_WR_ATOMIC_CMP_AND_SWP -#define OP_FETCH_AND_ADD IBV_WR_ATOMIC_FETCH_AND_ADD -#define OP_RECEIVE 7 /* internal op */ -#define OP_RECEIVE_IMM 8 /* internel op */ -#define OP_BIND_MW 9 /* internal op */ -#define OP_INVALID 0xff - -/* Definitions to map QP state */ -#define IB_QP_STATE_RESET IBV_QPS_RESET -#define IB_QP_STATE_INIT IBV_QPS_INIT -#define IB_QP_STATE_RTR IBV_QPS_RTR -#define IB_QP_STATE_RTS IBV_QPS_RTS -#define IB_QP_STATE_SQD IBV_QPS_SQD -#define IB_QP_STATE_SQE IBV_QPS_SQE -#define IB_QP_STATE_ERROR IBV_QPS_ERR - -typedef enum -{ - IB_THREAD_INIT, - IB_THREAD_CREATE, - IB_THREAD_RUN, - IB_THREAD_CANCEL, - IB_THREAD_EXIT - -} ib_thread_state_t; - struct dapl_cm_id { DAPL_OS_LOCK lock; int destroy; @@ -171,67 +72,13 @@ struct dapl_cm_id { DAT_SOCK_ADDR6 r_addr; int p_len; unsigned char p_data[256]; /* dapl max private data size */ + ib_qp_cm_t dst; /* dapls_modify_qp_state */ + struct ibv_ah *ah; /* dapls_modify_qp_state */ }; typedef struct dapl_cm_id *dp_ib_cm_handle_t; typedef struct dapl_cm_id *ib_cm_srvc_handle_t; -/* Operation and state mappings */ -typedef int ib_send_op_type_t; -typedef struct ibv_sge ib_data_segment_t; -typedef enum ibv_qp_state ib_qp_state_t; -typedef enum ibv_event_type ib_async_event_type; -typedef struct ibv_async_event ib_error_record_t; - -/* Definitions for ibverbs/mthca return codes, should be defined in verbs.h */ -/* some are errno and some are -n values */ - -/** - * ibv_get_device_name - Return kernel device name - * ibv_get_device_guid - Return device's node GUID - * ibv_open_device - Return ibv_context or NULL - * ibv_close_device - Return 0, (errno?) - * ibv_get_async_event - Return 0, -1 - * ibv_alloc_pd - Return ibv_pd, NULL - * ibv_dealloc_pd - Return 0, errno - * ibv_reg_mr - Return ibv_mr, NULL - * ibv_dereg_mr - Return 0, errno - * ibv_create_cq - Return ibv_cq, NULL - * ibv_destroy_cq - Return 0, errno - * ibv_get_cq_event - Return 0 & ibv_cq/context, int - * ibv_poll_cq - Return n & ibv_wc, 0 ok, -1 empty, -2 error - * ibv_req_notify_cq - Return 0 (void?) - * ibv_create_qp - Return ibv_qp, NULL - * ibv_modify_qp - Return 0, errno - * ibv_destroy_qp - Return 0, errno - * ibv_post_send - Return 0, -1 & bad_wr - * ibv_post_recv - Return 0, -1 & bad_wr - */ - -/* async handlers for DTO, CQ, QP, and unafiliated */ -typedef void (*ib_async_dto_handler_t)( - IN ib_hca_handle_t ib_hca_handle, - IN ib_error_record_t *err_code, - IN void *context); - -typedef void (*ib_async_cq_handler_t)( - IN ib_hca_handle_t ib_hca_handle, - IN ib_cq_handle_t ib_cq_handle, - IN ib_error_record_t *err_code, - IN void *context); - -typedef void (*ib_async_qp_handler_t)( - IN ib_hca_handle_t ib_hca_handle, - IN ib_qp_handle_t ib_qp_handle, - IN ib_error_record_t *err_code, - IN void *context); - -typedef void (*ib_async_handler_t)( - IN ib_hca_handle_t ib_hca_handle, - IN ib_error_record_t *err_code, - IN void *context); - - /* ib_hca_transport_t, specific to this implementation */ typedef struct _ib_hca_transport { @@ -250,79 +97,38 @@ typedef struct _ib_hca_transport uint8_t max_cm_timeout; uint8_t max_cm_retries; /* device attributes */ - int max_rdma_rd_in; - int max_rdma_rd_out; + int rd_atom_in; + int rd_atom_out; + struct ibv_device *ib_dev; + /* dapls_modify_qp_state */ + uint16_t lid; + uint8_t ack_timer; + uint8_t ack_retry; + uint8_t rnr_timer; + uint8_t rnr_retry; + uint8_t global; + uint8_t hop_limit; + uint8_t tclass; + uint8_t mtu; + DAT_NAMED_ATTR named_attr; } ib_hca_transport_t; -/* provider specfic fields for shared memory support */ -typedef uint32_t ib_shm_transport_t; - /* prototypes */ -int32_t dapls_ib_init (void); -int32_t dapls_ib_release (void); void dapli_thread(void *arg); DAT_RETURN dapli_ib_thread_init(void); void dapli_ib_thread_destroy(void); void dapli_cma_event_cb(void); -void dapli_cq_event_cb(struct _ib_hca_transport *hca); void dapli_async_event_cb(struct _ib_hca_transport *hca); -void dapli_destroy_conn(struct dapl_cm_id *conn); - -DAT_RETURN -dapls_modify_qp_state ( IN ib_qp_handle_t qp_handle, - IN ib_qp_state_t qp_state, - IN struct dapl_cm_id *conn ); - -/* inline functions */ -STATIC _INLINE_ IB_HCA_NAME dapl_ib_convert_name (IN char *name) -{ - /* use ascii; name of local device */ - return dapl_os_strdup(name); -} - -STATIC _INLINE_ void dapl_ib_release_name (IN IB_HCA_NAME name) -{ - return; -} +dp_ib_cm_handle_t dapls_ib_cm_create(DAPL_EP *ep); +void dapls_ib_cm_free(dp_ib_cm_handle_t cm, DAPL_EP *ep); +DAT_RETURN dapls_modify_qp_state(IN ib_qp_handle_t qp_handle, + IN ib_qp_state_t qp_state, + IN dp_ib_cm_handle_t cm); STATIC _INLINE_ void dapls_print_cm_list(IN DAPL_IA * ia_ptr) { return; } -/* - * Convert errno to DAT_RETURN values - */ -STATIC _INLINE_ DAT_RETURN -dapl_convert_errno( IN int err, IN const char *str ) -{ - if (!err) return DAT_SUCCESS; - -#if DAPL_DBG - if ((err != EAGAIN) && (err != ETIMEDOUT)) - dapl_dbg_log (DAPL_DBG_TYPE_ERR," %s %s\n", str, strerror(err)); -#endif - - switch( err ) - { - case EOVERFLOW : return DAT_LENGTH_ERROR; - case EACCES : return DAT_PRIVILEGES_VIOLATION; - case EPERM : return DAT_PROTECTION_VIOLATION; - case EINVAL : return DAT_INVALID_HANDLE; - case EISCONN : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_CONNECTED; - case ECONNREFUSED : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_NOTREADY; - case ETIMEDOUT : return DAT_TIMEOUT_EXPIRED; - case ENETUNREACH: return DAT_INVALID_ADDRESS | DAT_INVALID_ADDRESS_UNREACHABLE; - case EADDRINUSE : return DAT_CONN_QUAL_IN_USE; - case EALREADY : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_ACTCONNPENDING; - case ENOMEM : return DAT_INSUFFICIENT_RESOURCES; - case EAGAIN : return DAT_QUEUE_EMPTY; - case EINTR : return DAT_INTERRUPTED_CALL; - case EAFNOSUPPORT : return DAT_INVALID_ADDRESS | DAT_INVALID_ADDRESS_MALFORMED; - case EFAULT : - default : return DAT_INTERNAL_ERROR; - } - } - #endif /* _DAPL_IB_UTIL_H_ */ diff --git a/dapl/openib_cma/device.c b/dapl/openib_cma/device.c new file mode 100644 index 0000000..0e974f6 --- /dev/null +++ b/dapl/openib_cma/device.c @@ -0,0 +1,847 @@ +/* + * Copyright (c) 2005-2008 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/********************************************************************** + * + * MODULE: dapl_ib_util.c + * + * PURPOSE: OFED provider - init, open, close, utilities, work thread + * + * $Id:$ + * + **********************************************************************/ + +#ifdef RCSID +static const char rcsid[] = "$Id: $"; +#endif + +#include "openib_osd.h" +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_ib_util.h" +#include "dapl_osd.h" + +#include + +struct rdma_event_channel *g_cm_events = NULL; +ib_thread_state_t g_ib_thread_state = 0; +DAPL_OS_THREAD g_ib_thread; +DAPL_OS_LOCK g_hca_lock; +struct dapl_llist_entry *g_hca_list; + +#if defined(_WIN64) || defined(_WIN32) +#include "..\..\..\..\..\etc\user\comp_channel.cpp" +#include "..\..\..\..\..\etc\user\dlist.c" +#include + +struct ibvw_windata windata; + +static int getipaddr_netdev(char *name, char *addr, int addr_len) +{ + IWVProvider *prov; + WV_DEVICE_ADDRESS devaddr; + struct addrinfo *res, *ai; + HRESULT hr; + int index; + + if (strncmp(name, "rdma_dev", 8)) { + return EINVAL; + } + + index = atoi(name + 8); + + hr = WvGetObject(&IID_IWVProvider, (LPVOID *) &prov); + if (FAILED(hr)) { + return hr; + } + + hr = getaddrinfo("..localmachine", NULL, NULL, &res); + if (hr) { + goto release; + } + + for (ai = res; ai; ai = ai->ai_next) { + hr = prov->lpVtbl->TranslateAddress(prov, ai->ai_addr, &devaddr); + if (SUCCEEDED(hr) && (ai->ai_addrlen <= addr_len) && (index-- == 0)) { + memcpy(addr, ai->ai_addr, ai->ai_addrlen); + goto free; + } + } + hr = ENODEV; + +free: + freeaddrinfo(res); +release: + prov->lpVtbl->Release(prov); + return hr; +} + +static int dapls_os_init(void) +{ + return ibvw_get_windata(&windata, IBVW_WINDATA_VERSION); +} + +static void dapls_os_release(void) +{ + if (windata.comp_mgr) + ibvw_release_windata(&windata, IBVW_WINDATA_VERSION); + windata.comp_mgr = NULL; +} + +static int dapls_config_cm_channel(struct rdma_event_channel *channel) +{ + channel->channel.Milliseconds = 0; + return 0; +} + +static int dapls_config_verbs(struct ibv_context *verbs) +{ + verbs->channel.Milliseconds = 0; + return 0; +} + +static int dapls_thread_signal(void) +{ + CompManagerCancel(windata.comp_mgr); + return 0; +} +#else // _WIN64 || WIN32 +int g_ib_pipe[2]; + +static int dapls_os_init(void) +{ + /* create pipe for waking up work thread */ + return pipe(g_ib_pipe); +} + +static void dapls_os_release(void) +{ + /* close pipe? */ +} + +/* Get IP address using network device name */ +static int getipaddr_netdev(char *name, char *addr, int addr_len) +{ + struct ifreq ifr; + int skfd, ret, len; + + /* Fill in the structure */ + snprintf(ifr.ifr_name, IFNAMSIZ, "%s", name); + ifr.ifr_hwaddr.sa_family = ARPHRD_INFINIBAND; + + /* Create a socket fd */ + skfd = socket(PF_INET, SOCK_STREAM, 0); + ret = ioctl(skfd, SIOCGIFADDR, &ifr); + if (ret) + goto bail; + + switch (ifr.ifr_addr.sa_family) { +#ifdef AF_INET6 + case AF_INET6: + len = sizeof(struct sockaddr_in6); + break; +#endif + case AF_INET: + default: + len = sizeof(struct sockaddr); + break; + } + + if (len <= addr_len) + memcpy(addr, &ifr.ifr_addr, len); + else + ret = EINVAL; + + bail: + close(skfd); + return ret; +} + +static int dapls_config_fd(int fd) +{ + int opts; + + opts = fcntl(fd, F_GETFL); + if (opts < 0 || fcntl(fd, F_SETFL, opts | O_NONBLOCK) < 0) { + dapl_log(DAPL_DBG_TYPE_ERR, + " dapls_config_fd: fcntl on fd %d ERR %d %s\n", + fd, opts, strerror(errno)); + return errno; + } + + return 0; +} + +static int dapls_config_cm_channel(struct rdma_event_channel *channel) +{ + return dapls_config_fd(channel->fd); +} + +static int dapls_config_verbs(struct ibv_context *verbs) +{ + return dapls_config_fd(verbs->async_fd); +} + +static int dapls_thread_signal(void) +{ + return write(g_ib_pipe[1], "w", sizeof "w"); +} +#endif + +/* Get IP address using network name, address, or device name */ +static int getipaddr(char *name, char *addr, int len) +{ + struct addrinfo *res; + + /* assume netdev for first attempt, then network and address type */ + if (getipaddr_netdev(name, addr, len)) { + if (getaddrinfo(name, NULL, NULL, &res)) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: getaddr_netdev ERROR:" + " %s. Is %s configured?\n", + strerror(errno), name); + return 1; + } else { + if (len >= res->ai_addrlen) + memcpy(addr, res->ai_addr, res->ai_addrlen); + else { + freeaddrinfo(res); + return 1; + } + freeaddrinfo(res); + } + } + + dapl_dbg_log( + DAPL_DBG_TYPE_UTIL, + " getipaddr: family %d port %d addr %d.%d.%d.%d\n", + ((struct sockaddr_in *)addr)->sin_family, + ((struct sockaddr_in *)addr)->sin_port, + ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 0 & 0xff, + ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 8 & 0xff, + ((struct sockaddr_in *)addr)->sin_addr.s_addr >> 16 & 0xff, + ((struct sockaddr_in *)addr)->sin_addr. + s_addr >> 24 & 0xff); + + return 0; +} + +/* + * dapls_ib_init, dapls_ib_release + * + * Initialize Verb related items for device open + * + * Input: + * none + * + * Output: + * none + * + * Returns: + * 0 success, -1 error + * + */ +int32_t dapls_ib_init(void) +{ + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_init: \n"); + + /* initialize hca_list lock */ + dapl_os_lock_init(&g_hca_lock); + + /* initialize hca list for CQ events */ + dapl_llist_init_head(&g_hca_list); + + if (dapls_os_init()) + return 1; + + return 0; +} + +int32_t dapls_ib_release(void) +{ + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_release: \n"); + dapli_ib_thread_destroy(); + if (g_cm_events != NULL) + rdma_destroy_event_channel(g_cm_events); + dapls_os_release(); + return 0; +} + +/* + * dapls_ib_open_hca + * + * Open HCA + * + * Input: + * *hca_name pointer to provider device name + * *ib_hca_handle_p pointer to provide HCA handle + * + * Output: + * none + * + * Return: + * DAT_SUCCESS + * dapl_convert_errno + * + */ +DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr) +{ + struct rdma_cm_id *cm_id = NULL; + union ibv_gid *gid; + int ret; + DAT_RETURN dat_status; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " open_hca: %s - %p\n", hca_name, hca_ptr); + + /* Setup the global cm event channel */ + dapl_os_lock(&g_hca_lock); + if (g_cm_events == NULL) { + g_cm_events = rdma_create_event_channel(); + if (g_cm_events == NULL) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " open_hca: ERR - RDMA channel %s\n", + strerror(errno)); + return DAT_INTERNAL_ERROR; + } + } + dapl_os_unlock(&g_hca_lock); + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " open_hca: RDMA channel created (%p)\n", g_cm_events); + + dat_status = dapli_ib_thread_init(); + if (dat_status != DAT_SUCCESS) + return dat_status; + + /* HCA name will be hostname or IP address */ + if (getipaddr((char *)hca_name, + (char *)&hca_ptr->hca_address, + sizeof(DAT_SOCK_ADDR6))) + return DAT_INVALID_ADDRESS; + + /* cm_id will bind local device/GID based on IP address */ + if (rdma_create_id(g_cm_events, &cm_id, + (void *)hca_ptr, RDMA_PS_TCP)) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: rdma_create ERR %s\n", strerror(errno)); + return DAT_INTERNAL_ERROR; + } + ret = rdma_bind_addr(cm_id, (struct sockaddr *)&hca_ptr->hca_address); + if ((ret) || (cm_id->verbs == NULL)) { + rdma_destroy_id(cm_id); + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: rdma_bind ERR %s." + " Is %s configured?\n", strerror(errno), hca_name); + return DAT_INVALID_ADDRESS; + } + + /* keep reference to IB device and cm_id */ + hca_ptr->ib_trans.cm_id = cm_id; + hca_ptr->ib_hca_handle = cm_id->verbs; + dapls_config_verbs(cm_id->verbs); + hca_ptr->port_num = cm_id->port_num; + hca_ptr->ib_trans.ib_dev = cm_id->verbs->device; + gid = &cm_id->route.addr.addr.ibaddr.sgid; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " open_hca: ctx=%p port=%d GID subnet %016llx" + " id %016llx\n", cm_id->verbs, cm_id->port_num, + (unsigned long long)ntohll(gid->global.subnet_prefix), + (unsigned long long)ntohll(gid->global.interface_id)); + + /* set inline max with env or default, get local lid and gid 0 */ + if (hca_ptr->ib_hca_handle->device->transport_type + == IBV_TRANSPORT_IWARP) + hca_ptr->ib_trans.max_inline_send = + dapl_os_get_env_val("DAPL_MAX_INLINE", + INLINE_SEND_IWARP_DEFAULT); + else + hca_ptr->ib_trans.max_inline_send = + dapl_os_get_env_val("DAPL_MAX_INLINE", + INLINE_SEND_IB_DEFAULT); + + /* set CM timer defaults */ + hca_ptr->ib_trans.max_cm_timeout = + dapl_os_get_env_val("DAPL_MAX_CM_RESPONSE_TIME", + IB_CM_RESPONSE_TIMEOUT); + hca_ptr->ib_trans.max_cm_retries = + dapl_os_get_env_val("DAPL_MAX_CM_RETRIES", IB_CM_RETRIES); + + /* set default IB MTU */ + hca_ptr->ib_trans.mtu = dapl_ib_mtu(2048); + + /* + * Put new hca_transport on list for async and CQ event processing + * Wakeup work thread to add to polling list + */ + dapl_llist_init_entry((DAPL_LLIST_ENTRY *) & hca_ptr->ib_trans.entry); + dapl_os_lock(&g_hca_lock); + dapl_llist_add_tail(&g_hca_list, + (DAPL_LLIST_ENTRY *) & hca_ptr->ib_trans.entry, + &hca_ptr->ib_trans.entry); + if (dapls_thread_signal() == -1) + dapl_log(DAPL_DBG_TYPE_UTIL, + " open_hca: thread wakeup error = %s\n", + strerror(errno)); + dapl_os_unlock(&g_hca_lock); + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " open_hca: %s, %s %d.%d.%d.%d INLINE_MAX=%d\n", hca_name, + ((struct sockaddr_in *) + &hca_ptr->hca_address)->sin_family == AF_INET ? + "AF_INET" : "AF_INET6", + ((struct sockaddr_in *) + &hca_ptr->hca_address)->sin_addr.s_addr >> 0 & 0xff, + ((struct sockaddr_in *) + &hca_ptr->hca_address)->sin_addr.s_addr >> 8 & 0xff, + ((struct sockaddr_in *) + &hca_ptr->hca_address)->sin_addr.s_addr >> 16 & 0xff, + ((struct sockaddr_in *) + &hca_ptr->hca_address)->sin_addr.s_addr >> 24 & 0xff, + hca_ptr->ib_trans.max_inline_send); + + hca_ptr->ib_trans.d_hca = hca_ptr; + return DAT_SUCCESS; +} + +/* + * dapls_ib_close_hca + * + * Open HCA + * + * Input: + * DAPL_HCA provide CA handle + * + * Output: + * none + * + * Return: + * DAT_SUCCESS + * dapl_convert_errno + * + */ +DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr) +{ + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p->%p\n", + hca_ptr, hca_ptr->ib_hca_handle); + + if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) { + if (rdma_destroy_id(hca_ptr->ib_trans.cm_id)) + return (dapl_convert_errno(errno, "ib_close_device")); + hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; + } + + dapl_os_lock(&g_hca_lock); + if (g_ib_thread_state != IB_THREAD_RUN) { + dapl_os_unlock(&g_hca_lock); + goto bail; + } + dapl_os_unlock(&g_hca_lock); + + /* + * Remove hca from async event processing list + * Wakeup work thread to remove from polling list + */ + hca_ptr->ib_trans.destroy = 1; + if (dapls_thread_signal() == -1) + dapl_log(DAPL_DBG_TYPE_UTIL, + " destroy: thread wakeup error = %s\n", + strerror(errno)); + + /* wait for thread to remove HCA references */ + while (hca_ptr->ib_trans.destroy != 2) { + if (dapls_thread_signal() == -1) + dapl_log(DAPL_DBG_TYPE_UTIL, + " destroy: thread wakeup error = %s\n", + strerror(errno)); + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " ib_thread_destroy: wait on hca %p destroy\n"); + dapl_os_sleep_usec(1000); + } +bail: + return (DAT_SUCCESS); +} + + +DAT_RETURN dapli_ib_thread_init(void) +{ + DAT_RETURN dat_status; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " ib_thread_init(%d)\n", dapl_os_getpid()); + + dapl_os_lock(&g_hca_lock); + if (g_ib_thread_state != IB_THREAD_INIT) { + dapl_os_unlock(&g_hca_lock); + return DAT_SUCCESS; + } + + /* uCMA events non-blocking */ + if (dapls_config_cm_channel(g_cm_events)) { + dapl_os_unlock(&g_hca_lock); + return (dapl_convert_errno(errno, "create_thread ERR: cm_fd")); + } + + g_ib_thread_state = IB_THREAD_CREATE; + dapl_os_unlock(&g_hca_lock); + + /* create thread to process inbound connect request */ + dat_status = dapl_os_thread_create(dapli_thread, NULL, &g_ib_thread); + if (dat_status != DAT_SUCCESS) + return (dapl_convert_errno(errno, + "create_thread ERR:" + " check resource limits")); + + /* wait for thread to start */ + dapl_os_lock(&g_hca_lock); + while (g_ib_thread_state != IB_THREAD_RUN) { + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " ib_thread_init: waiting for ib_thread\n"); + dapl_os_unlock(&g_hca_lock); + dapl_os_sleep_usec(1000); + dapl_os_lock(&g_hca_lock); + } + dapl_os_unlock(&g_hca_lock); + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " ib_thread_init(%d) exit\n", dapl_os_getpid()); + + return DAT_SUCCESS; +} + +void dapli_ib_thread_destroy(void) +{ + int retries = 10; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " ib_thread_destroy(%d)\n", dapl_os_getpid()); + /* + * wait for async thread to terminate. + * pthread_join would be the correct method + * but some applications have some issues + */ + + /* destroy ib_thread, wait for termination, if not already */ + dapl_os_lock(&g_hca_lock); + if (g_ib_thread_state != IB_THREAD_RUN) + goto bail; + + g_ib_thread_state = IB_THREAD_CANCEL; + if (dapls_thread_signal() == -1) + dapl_log(DAPL_DBG_TYPE_UTIL, + " destroy: thread wakeup error = %s\n", + strerror(errno)); + while ((g_ib_thread_state != IB_THREAD_EXIT) && (retries--)) { + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " ib_thread_destroy: waiting for ib_thread\n"); + if (dapls_thread_signal() == -1) + dapl_log(DAPL_DBG_TYPE_UTIL, + " destroy: thread wakeup error = %s\n", + strerror(errno)); + dapl_os_unlock(&g_hca_lock); + dapl_os_sleep_usec(2000); + dapl_os_lock(&g_hca_lock); + } +bail: + dapl_os_unlock(&g_hca_lock); + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " ib_thread_destroy(%d) exit\n", dapl_os_getpid()); +} + +void dapli_async_event_cb(struct _ib_hca_transport *hca) +{ + struct ibv_async_event event; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " async_event(%p)\n", hca); + + if (hca->destroy) + return; + + if (!ibv_get_async_event(hca->cm_id->verbs, &event)) { + + switch (event.event_type) { + case IBV_EVENT_CQ_ERR: + { + struct dapl_ep *evd_ptr = + event.element.cq->cq_context; + + dapl_log(DAPL_DBG_TYPE_ERR, + "dapl async_event CQ (%p) ERR %d\n", + evd_ptr, event.event_type); + + /* report up if async callback still setup */ + if (hca->async_cq_error) + hca->async_cq_error(hca->cm_id->verbs, + event.element.cq, + &event, + (void *)evd_ptr); + break; + } + case IBV_EVENT_COMM_EST: + { + /* Received msgs on connected QP before RTU */ + dapl_log(DAPL_DBG_TYPE_UTIL, + " async_event COMM_EST(%p) rdata beat RTU\n", + event.element.qp); + + break; + } + case IBV_EVENT_QP_FATAL: + case IBV_EVENT_QP_REQ_ERR: + case IBV_EVENT_QP_ACCESS_ERR: + case IBV_EVENT_QP_LAST_WQE_REACHED: + case IBV_EVENT_SRQ_ERR: + case IBV_EVENT_SRQ_LIMIT_REACHED: + case IBV_EVENT_SQ_DRAINED: + { + struct dapl_ep *ep_ptr = + event.element.qp->qp_context; + + dapl_log(DAPL_DBG_TYPE_ERR, + "dapl async_event QP (%p) ERR %d\n", + ep_ptr, event.event_type); + + /* report up if async callback still setup */ + if (hca->async_qp_error) + hca->async_qp_error(hca->cm_id->verbs, + ep_ptr->qp_handle, + &event, + (void *)ep_ptr); + break; + } + case IBV_EVENT_PATH_MIG: + case IBV_EVENT_PATH_MIG_ERR: + case IBV_EVENT_DEVICE_FATAL: + case IBV_EVENT_PORT_ACTIVE: + case IBV_EVENT_PORT_ERR: + case IBV_EVENT_LID_CHANGE: + case IBV_EVENT_PKEY_CHANGE: + case IBV_EVENT_SM_CHANGE: + { + dapl_log(DAPL_DBG_TYPE_WARN, + "dapl async_event: DEV ERR %d\n", + event.event_type); + + /* report up if async callback still setup */ + if (hca->async_unafiliated) + hca->async_unafiliated(hca->cm_id-> + verbs, &event, + hca-> + async_un_ctx); + break; + } + case IBV_EVENT_CLIENT_REREGISTER: + /* no need to report this event this time */ + dapl_log(DAPL_DBG_TYPE_UTIL, + " async_event: IBV_CLIENT_REREGISTER\n"); + break; + + default: + dapl_log(DAPL_DBG_TYPE_WARN, + "dapl async_event: %d UNKNOWN\n", + event.event_type); + break; + + } + ibv_ack_async_event(&event); + } +} + +#if defined(_WIN64) || defined(_WIN32) +/* work thread for uAT, uCM, CQ, and async events */ +void dapli_thread(void *arg) +{ + struct _ib_hca_transport *hca; + struct _ib_hca_transport *uhca[8]; + COMP_CHANNEL *channel; + int ret, idx, cnt; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " ib_thread(%d,0x%x): ENTER: \n", + dapl_os_getpid(), g_ib_thread); + + dapl_os_lock(&g_hca_lock); + for (g_ib_thread_state = IB_THREAD_RUN; + g_ib_thread_state == IB_THREAD_RUN; + dapl_os_lock(&g_hca_lock)) { + + idx = 0; + hca = dapl_llist_is_empty(&g_hca_list) ? NULL : + dapl_llist_peek_head(&g_hca_list); + + while (hca) { + uhca[idx++] = hca; + hca = dapl_llist_next_entry(&g_hca_list, + (DAPL_LLIST_ENTRY *) + &hca->entry); + } + cnt = idx; + + dapl_os_unlock(&g_hca_lock); + ret = CompManagerPoll(windata.comp_mgr, INFINITE, &channel); + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " ib_thread(%d) poll_event 0x%x\n", + dapl_os_getpid(), ret); + + dapli_cma_event_cb(); + + /* check and process ASYNC events, per device */ + for (idx = 0; idx < cnt; idx++) { + if (uhca[idx]->destroy == 1) { + dapl_os_lock(&g_hca_lock); + dapl_llist_remove_entry(&g_hca_list, + (DAPL_LLIST_ENTRY *) + &uhca[idx]->entry); + dapl_os_unlock(&g_hca_lock); + uhca[idx]->destroy = 2; + } else { + dapli_async_event_cb(uhca[idx]); + } + } + } + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " ib_thread(%d) EXIT\n", + dapl_os_getpid()); + g_ib_thread_state = IB_THREAD_EXIT; + dapl_os_unlock(&g_hca_lock); +} +#else // _WIN64 || WIN32 +/* work thread for uAT, uCM, CQ, and async events */ +void dapli_thread(void *arg) +{ + struct pollfd ufds[__FD_SETSIZE]; + struct _ib_hca_transport *uhca[__FD_SETSIZE] = { NULL }; + struct _ib_hca_transport *hca; + int ret, idx, fds; + char rbuf[2]; + + dapl_dbg_log(DAPL_DBG_TYPE_THREAD, + " ib_thread(%d,0x%x): ENTER: pipe %d ucma %d\n", + dapl_os_getpid(), g_ib_thread, g_ib_pipe[0], + g_cm_events->fd); + + /* Poll across pipe, CM, AT never changes */ + dapl_os_lock(&g_hca_lock); + g_ib_thread_state = IB_THREAD_RUN; + + ufds[0].fd = g_ib_pipe[0]; /* pipe */ + ufds[0].events = POLLIN; + ufds[1].fd = g_cm_events->fd; /* uCMA */ + ufds[1].events = POLLIN; + + while (g_ib_thread_state == IB_THREAD_RUN) { + + /* build ufds after pipe and uCMA events */ + ufds[0].revents = 0; + ufds[1].revents = 0; + idx = 1; + + /* Walk HCA list and setup async and CQ events */ + if (!dapl_llist_is_empty(&g_hca_list)) + hca = dapl_llist_peek_head(&g_hca_list); + else + hca = NULL; + + while (hca) { + + /* uASYNC events */ + ufds[++idx].fd = hca->cm_id->verbs->async_fd; + ufds[idx].events = POLLIN; + ufds[idx].revents = 0; + uhca[idx] = hca; + + dapl_dbg_log(DAPL_DBG_TYPE_THREAD, + " ib_thread(%d) poll_fd: hca[%d]=%p," + " async=%d pipe=%d cm=%d \n", + dapl_os_getpid(), hca, ufds[idx - 1].fd, + ufds[0].fd, ufds[1].fd); + + hca = dapl_llist_next_entry(&g_hca_list, + (DAPL_LLIST_ENTRY *) + &hca->entry); + } + + /* unlock, and setup poll */ + fds = idx + 1; + dapl_os_unlock(&g_hca_lock); + ret = poll(ufds, fds, -1); + if (ret <= 0) { + dapl_dbg_log(DAPL_DBG_TYPE_THREAD, + " ib_thread(%d): ERR %s poll\n", + dapl_os_getpid(), strerror(errno)); + dapl_os_lock(&g_hca_lock); + continue; + } + + dapl_dbg_log(DAPL_DBG_TYPE_THREAD, + " ib_thread(%d) poll_event: " + " async=0x%x pipe=0x%x cm=0x%x \n", + dapl_os_getpid(), ufds[idx].revents, + ufds[0].revents, ufds[1].revents); + + /* uCMA events */ + if (ufds[1].revents == POLLIN) + dapli_cma_event_cb(); + + /* check and process ASYNC events, per device */ + for (idx = 2; idx < fds; idx++) { + if (ufds[idx].revents == POLLIN) { + dapli_async_event_cb(uhca[idx]); + } + } + + /* check and process user events, PIPE */ + if (ufds[0].revents == POLLIN) { + if (read(g_ib_pipe[0], rbuf, 2) == -1) + dapl_log(DAPL_DBG_TYPE_THREAD, + " cr_thread: pipe rd err= %s\n", + strerror(errno)); + + /* cleanup any device on list marked for destroy */ + for (idx = 2; idx < fds; idx++) { + if (uhca[idx] && uhca[idx]->destroy == 1) { + dapl_os_lock(&g_hca_lock); + dapl_llist_remove_entry( + &g_hca_list, + (DAPL_LLIST_ENTRY*) + &uhca[idx]->entry); + dapl_os_unlock(&g_hca_lock); + uhca[idx]->destroy = 2; + } + } + } + dapl_os_lock(&g_hca_lock); + } + + dapl_dbg_log(DAPL_DBG_TYPE_THREAD, " ib_thread(%d) EXIT\n", + dapl_os_getpid()); + g_ib_thread_state = IB_THREAD_EXIT; + dapl_os_unlock(&g_hca_lock); +} +#endif diff --git a/dapl/openib_common/cq.c b/dapl/openib_common/cq.c new file mode 100644 index 0000000..74a5940 --- /dev/null +++ b/dapl/openib_common/cq.c @@ -0,0 +1,491 @@ +/* + * Copyright (c) 2009 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ +#include "openib_osd.h" +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_lmr_util.h" +#include "dapl_evd_util.h" +#include "dapl_ring_buffer_util.h" + +/* + * Map all verbs DTO completion codes to the DAT equivelent. + * + * Not returned by verbs: DAT_DTO_ERR_PARTIAL_PACKET + */ +static struct ib_status_map { + int ib_status; + DAT_DTO_COMPLETION_STATUS dat_status; +} ib_status_map[] = { +/* 00 */ {IBV_WC_SUCCESS, DAT_DTO_SUCCESS}, +/* 01 */ {IBV_WC_LOC_LEN_ERR, DAT_DTO_ERR_LOCAL_LENGTH}, +/* 02 */ {IBV_WC_LOC_QP_OP_ERR, DAT_DTO_ERR_LOCAL_EP}, +/* 03 */ {IBV_WC_LOC_EEC_OP_ERR, DAT_DTO_ERR_TRANSPORT}, +/* 04 */ {IBV_WC_LOC_PROT_ERR, DAT_DTO_ERR_LOCAL_PROTECTION}, +/* 05 */ {IBV_WC_WR_FLUSH_ERR, DAT_DTO_ERR_FLUSHED}, +/* 06 */ {IBV_WC_MW_BIND_ERR, DAT_RMR_OPERATION_FAILED}, +/* 07 */ {IBV_WC_BAD_RESP_ERR, DAT_DTO_ERR_BAD_RESPONSE}, +/* 08 */ {IBV_WC_LOC_ACCESS_ERR, DAT_DTO_ERR_LOCAL_PROTECTION}, +/* 09 */ {IBV_WC_REM_INV_REQ_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, +/* 10 */ {IBV_WC_REM_ACCESS_ERR, DAT_DTO_ERR_REMOTE_ACCESS}, +/* 11 */ {IBV_WC_REM_OP_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, +/* 12 */ {IBV_WC_RETRY_EXC_ERR, DAT_DTO_ERR_TRANSPORT}, +/* 13 */ {IBV_WC_RNR_RETRY_EXC_ERR, DAT_DTO_ERR_RECEIVER_NOT_READY}, +/* 14 */ {IBV_WC_LOC_RDD_VIOL_ERR, DAT_DTO_ERR_LOCAL_PROTECTION}, +/* 15 */ {IBV_WC_REM_INV_RD_REQ_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, +/* 16 */ {IBV_WC_REM_ABORT_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, +/* 17 */ {IBV_WC_INV_EECN_ERR, DAT_DTO_ERR_TRANSPORT}, +/* 18 */ {IBV_WC_INV_EEC_STATE_ERR, DAT_DTO_ERR_TRANSPORT}, +/* 19 */ {IBV_WC_FATAL_ERR, DAT_DTO_ERR_TRANSPORT}, +/* 20 */ {IBV_WC_RESP_TIMEOUT_ERR, DAT_DTO_ERR_RECEIVER_NOT_READY}, +/* 21 */ {IBV_WC_GENERAL_ERR, DAT_DTO_ERR_TRANSPORT}, +}; + +/* + * dapls_ib_get_dto_status + * + * Return the DAT status of a DTO operation + * + * Input: + * cqe_ptr pointer to completion queue entry + * + * Output: + * none + * + * Returns: + * Value from ib_status_map table above + */ + +DAT_DTO_COMPLETION_STATUS +dapls_ib_get_dto_status(IN ib_work_completion_t * cqe_ptr) +{ + uint32_t ib_status; + int i; + + ib_status = DAPL_GET_CQE_STATUS(cqe_ptr); + + /* + * Due to the implementation of verbs completion code, we need to + * search the table for the correct value rather than assuming + * linear distribution. + */ + for (i = 0; i <= IBV_WC_GENERAL_ERR; i++) { + if (ib_status == ib_status_map[i].ib_status) { + if (ib_status != IBV_WC_SUCCESS) { + dapl_log(DAPL_DBG_TYPE_DTO_COMP_ERR, + " DTO completion ERROR: %d:" + " op %#x\n", + ib_status, + DAPL_GET_CQE_OPTYPE(cqe_ptr)); + } + return ib_status_map[i].dat_status; + } + } + + return DAT_DTO_FAILURE; +} + +DAT_RETURN dapls_ib_get_async_event(IN ib_error_record_t * err_record, + OUT DAT_EVENT_NUMBER * async_event) +{ + DAT_RETURN dat_status = DAT_SUCCESS; + int err_code = err_record->event_type; + + switch (err_code) { + /* OVERFLOW error */ + case IBV_EVENT_CQ_ERR: + *async_event = DAT_ASYNC_ERROR_EVD_OVERFLOW; + break; + /* INTERNAL errors */ + case IBV_EVENT_DEVICE_FATAL: + *async_event = DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR; + break; + /* CATASTROPHIC errors */ + case IBV_EVENT_PORT_ERR: + *async_event = DAT_ASYNC_ERROR_IA_CATASTROPHIC; + break; + /* BROKEN QP error */ + case IBV_EVENT_SQ_DRAINED: + case IBV_EVENT_QP_FATAL: + case IBV_EVENT_QP_REQ_ERR: + case IBV_EVENT_QP_ACCESS_ERR: + *async_event = DAT_ASYNC_ERROR_EP_BROKEN; + break; + + /* connection completion */ + case IBV_EVENT_COMM_EST: + *async_event = DAT_CONNECTION_EVENT_ESTABLISHED; + break; + + /* TODO: process HW state changes */ + case IBV_EVENT_PATH_MIG: + case IBV_EVENT_PATH_MIG_ERR: + case IBV_EVENT_PORT_ACTIVE: + case IBV_EVENT_LID_CHANGE: + case IBV_EVENT_PKEY_CHANGE: + case IBV_EVENT_SM_CHANGE: + default: + dat_status = DAT_ERROR(DAT_NOT_IMPLEMENTED, 0); + } + return dat_status; +} + +/* + * dapl_ib_cq_alloc + * + * Alloc a CQ + * + * Input: + * ia_handle IA handle + * evd_ptr pointer to EVD struct + * cqlen minimum QLen + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_cq_alloc(IN DAPL_IA * ia_ptr, + IN DAPL_EVD * evd_ptr, IN DAT_COUNT * cqlen) +{ + struct ibv_comp_channel *channel = evd_ptr->cq_wait_obj_handle; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + "dapls_ib_cq_alloc: evd %p cqlen=%d \n", evd_ptr, *cqlen); + + /* Call IB verbs to create CQ */ + evd_ptr->ib_cq_handle = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, + *cqlen, evd_ptr, channel, 0); + + if (evd_ptr->ib_cq_handle == IB_INVALID_HANDLE) + return DAT_INSUFFICIENT_RESOURCES; + + /* arm cq for events */ + dapls_set_cq_notify(ia_ptr, evd_ptr); + + /* update with returned cq entry size */ + *cqlen = evd_ptr->ib_cq_handle->cqe; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + "dapls_ib_cq_alloc: new_cq %p cqlen=%d \n", + evd_ptr->ib_cq_handle, *cqlen); + + return DAT_SUCCESS; +} + +/* + * dapl_ib_cq_resize + * + * Alloc a CQ + * + * Input: + * ia_handle IA handle + * evd_ptr pointer to EVD struct + * cqlen minimum QLen + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_cq_resize(IN DAPL_IA * ia_ptr, + IN DAPL_EVD * evd_ptr, IN DAT_COUNT * cqlen) +{ + ib_cq_handle_t new_cq; + struct ibv_comp_channel *channel = evd_ptr->cq_wait_obj_handle; + + /* IB verbs DOES support resize. REDO THIS. + * Try to re-create CQ + * with new size. Can only be done if QP is not attached. + * destroy EBUSY == QP still attached. + */ + + /* Call IB verbs to create CQ */ + new_cq = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, *cqlen, + evd_ptr, channel, 0); + + if (new_cq == IB_INVALID_HANDLE) + return DAT_INSUFFICIENT_RESOURCES; + + /* destroy the original and replace if successful */ + if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) { + ibv_destroy_cq(new_cq); + return (dapl_convert_errno(errno, "resize_cq")); + } + + /* update EVD with new cq handle and size */ + evd_ptr->ib_cq_handle = new_cq; + *cqlen = new_cq->cqe; + + /* arm cq for events */ + dapls_set_cq_notify(ia_ptr, evd_ptr); + + return DAT_SUCCESS; +} + +/* + * dapls_ib_cq_free + * + * destroy a CQ + * + * Input: + * ia_handle IA handle + * evd_ptr pointer to EVD struct + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN dapls_ib_cq_free(IN DAPL_IA * ia_ptr, IN DAPL_EVD * evd_ptr) +{ + DAT_EVENT event; + ib_work_completion_t wc; + + if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) { + /* pull off CQ and EVD entries and toss */ + while (ibv_poll_cq(evd_ptr->ib_cq_handle, 1, &wc) == 1) ; + while (dapl_evd_dequeue(evd_ptr, &event) == DAT_SUCCESS) ; + if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) + return (dapl_convert_errno(errno, "ibv_destroy_cq")); + evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; + } + return DAT_SUCCESS; +} + +/* + * dapls_set_cq_notify + * + * Set the CQ notification for next + * + * Input: + * hca_handl hca handle + * DAPL_EVD evd handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * dapl_convert_errno + */ +DAT_RETURN dapls_set_cq_notify(IN DAPL_IA * ia_ptr, IN DAPL_EVD * evd_ptr) +{ + if (ibv_req_notify_cq(evd_ptr->ib_cq_handle, 0)) + return (dapl_convert_errno(errno, "notify_cq")); + else + return DAT_SUCCESS; +} + +/* + * dapls_ib_completion_notify + * + * Set the CQ notification type + * + * Input: + * hca_handl hca handle + * evd_ptr evd handle + * type notification type + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * dapl_convert_errno + */ +DAT_RETURN dapls_ib_completion_notify(IN ib_hca_handle_t hca_handle, + IN DAPL_EVD * evd_ptr, + IN ib_notification_type_t type) +{ + if (ibv_req_notify_cq(evd_ptr->ib_cq_handle, type)) + return (dapl_convert_errno(errno, "notify_cq_type")); + else + return DAT_SUCCESS; +} + +/* + * dapls_ib_completion_poll + * + * CQ poll for completions + * + * Input: + * hca_handl hca handle + * evd_ptr evd handle + * wc_ptr work completion + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_QUEUE_EMPTY + * + */ +DAT_RETURN dapls_ib_completion_poll(IN DAPL_HCA * hca_ptr, + IN DAPL_EVD * evd_ptr, + IN ib_work_completion_t * wc_ptr) +{ + int ret; + + ret = ibv_poll_cq(evd_ptr->ib_cq_handle, 1, wc_ptr); + if (ret == 1) + return DAT_SUCCESS; + + return DAT_QUEUE_EMPTY; +} + +/* NEW common wait objects for providers with direct CQ wait objects */ +DAT_RETURN +dapls_ib_wait_object_create(IN DAPL_EVD * evd_ptr, + IN ib_wait_obj_handle_t * p_cq_wait_obj_handle) +{ + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_create: (%p,%p)\n", + evd_ptr, p_cq_wait_obj_handle); + + /* set cq_wait object to evd_ptr */ + *p_cq_wait_obj_handle = + ibv_create_comp_channel(evd_ptr->header.owner_ia->hca_ptr-> + ib_hca_handle); + + return DAT_SUCCESS; +} + +DAT_RETURN +dapls_ib_wait_object_destroy(IN ib_wait_obj_handle_t p_cq_wait_obj_handle) +{ + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_destroy: wait_obj=%p\n", p_cq_wait_obj_handle); + + ibv_destroy_comp_channel(p_cq_wait_obj_handle); + + return DAT_SUCCESS; +} + +DAT_RETURN +dapls_ib_wait_object_wakeup(IN ib_wait_obj_handle_t p_cq_wait_obj_handle) +{ + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wakeup: wait_obj=%p\n", p_cq_wait_obj_handle); + + /* no wake up mechanism */ + return DAT_SUCCESS; +} + +#if defined(_WIN32) || defined(_WIN64) +DAT_RETURN +dapls_ib_wait_object_wait(IN ib_wait_obj_handle_t p_cq_wait_obj_handle, + IN uint32_t timeout) +{ + struct dapl_evd *evd_ptr; + struct ibv_cq *ibv_cq = NULL; + int status = 0; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wait: CQ channel %p time %d\n", + p_cq_wait_obj_handle, timeout); + + /* uDAPL timeout values in usecs */ + p_cq_wait_obj_handle->comp_channel.Milliseconds = timeout / 1000; + + /* returned event */ + status = ibv_get_cq_event(p_cq_wait_obj_handle, &ibv_cq, + (void *)&evd_ptr); + if (status == 0) { + ibv_ack_cq_events(ibv_cq, 1); + } + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wait: RET evd %p ibv_cq %p %s\n", + evd_ptr, ibv_cq, strerror(errno)); + + return (dapl_convert_errno(status, "cq_wait_object_wait")); +} +#else //_WIN32 || _WIN64 +DAT_RETURN +dapls_ib_wait_object_wait(IN ib_wait_obj_handle_t p_cq_wait_obj_handle, + IN uint32_t timeout) +{ + struct dapl_evd *evd_ptr; + struct ibv_cq *ibv_cq = NULL; + int status = 0; + int timeout_ms = -1; + struct pollfd cq_fd = { + .fd = p_cq_wait_obj_handle->fd, + .events = POLLIN, + .revents = 0 + }; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wait: CQ channel %p time %d\n", + p_cq_wait_obj_handle, timeout); + + /* uDAPL timeout values in usecs */ + if (timeout != DAT_TIMEOUT_INFINITE) + timeout_ms = timeout / 1000; + + status = poll(&cq_fd, 1, timeout_ms); + + /* returned event */ + if (status > 0) { + if (!ibv_get_cq_event(p_cq_wait_obj_handle, + &ibv_cq, (void *)&evd_ptr)) { + ibv_ack_cq_events(ibv_cq, 1); + } + status = 0; + + /* timeout */ + } else if (status == 0) + status = ETIMEDOUT; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wait: RET evd %p ibv_cq %p %s\n", + evd_ptr, ibv_cq, strerror(errno)); + + return (dapl_convert_errno(status, "cq_wait_object_wait")); + +} +#endif //_WIN32 || _WIN64 + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h new file mode 100644 index 0000000..b61e50e --- /dev/null +++ b/dapl/openib_common/dapl_ib_common.h @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2009 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/* + * Definitions common to all OpenIB providers, cma, scm, ucm + */ + +#ifndef _DAPL_IB_COMMON_H_ +#define _DAPL_IB_COMMON_H_ + +#include + +#ifdef DAT_EXTENSIONS +#include +#endif + +#ifndef __cplusplus +#define false 0 +#define true 1 +#endif /*__cplusplus */ + +/* Typedefs to map common DAPL provider types to IB verbs */ +typedef struct ibv_qp *ib_qp_handle_t; +typedef struct ibv_cq *ib_cq_handle_t; +typedef struct ibv_pd *ib_pd_handle_t; +typedef struct ibv_mr *ib_mr_handle_t; +typedef struct ibv_mw *ib_mw_handle_t; +typedef struct ibv_wc ib_work_completion_t; + +/* HCA context type maps to IB verbs */ +typedef struct ibv_context *ib_hca_handle_t; +typedef ib_hca_handle_t dapl_ibal_ca_t; + +/* QP info to exchange, wire protocol version for these CM's */ +#define DCM_VER 4 +typedef struct _ib_qp_cm +{ + uint16_t ver; + uint16_t rej; + uint16_t lid; + uint16_t port; + uint32_t qpn; + uint32_t p_size; + union ibv_gid gid; + DAT_SOCK_ADDR6 ia_address; + uint16_t qp_type; +} ib_qp_cm_t; + +/* CM events */ +typedef enum { + IB_CME_CONNECTED, + IB_CME_DISCONNECTED, + IB_CME_DISCONNECTED_ON_LINK_DOWN, + IB_CME_CONNECTION_REQUEST_PENDING, + IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA, + IB_CME_CONNECTION_REQUEST_ACKED, + IB_CME_DESTINATION_REJECT, + IB_CME_DESTINATION_REJECT_PRIVATE_DATA, + IB_CME_DESTINATION_UNREACHABLE, + IB_CME_TOO_MANY_CONNECTION_REQUESTS, + IB_CME_LOCAL_FAILURE, + IB_CME_BROKEN, + IB_CME_TIMEOUT +} ib_cm_events_t; + +/* Operation and state mappings */ +typedef int ib_send_op_type_t; +typedef struct ibv_sge ib_data_segment_t; +typedef enum ibv_qp_state ib_qp_state_t; +typedef enum ibv_event_type ib_async_event_type; +typedef struct ibv_async_event ib_error_record_t; + +/* CQ notifications */ +typedef enum +{ + IB_NOTIFY_ON_NEXT_COMP, + IB_NOTIFY_ON_SOLIC_COMP + +} ib_notification_type_t; + +/* other mappings */ +typedef int ib_bool_t; +typedef union ibv_gid GID; +typedef char *IB_HCA_NAME; +typedef uint16_t ib_hca_port_t; +typedef uint32_t ib_comp_handle_t; + +typedef struct ibv_comp_channel *ib_wait_obj_handle_t; + +/* Definitions */ +#define IB_INVALID_HANDLE NULL + +/* inline send rdma threshold */ +#define INLINE_SEND_IWARP_DEFAULT 64 +#define INLINE_SEND_IB_DEFAULT 200 + +/* qkey for UD QP's */ +#define DAT_UD_QKEY 0x78654321 + +/* DTO OPs, ordered for DAPL ENUM definitions */ +#define OP_RDMA_WRITE IBV_WR_RDMA_WRITE +#define OP_RDMA_WRITE_IMM IBV_WR_RDMA_WRITE_WITH_IMM +#define OP_SEND IBV_WR_SEND +#define OP_SEND_IMM IBV_WR_SEND_WITH_IMM +#define OP_RDMA_READ IBV_WR_RDMA_READ +#define OP_COMP_AND_SWAP IBV_WR_ATOMIC_CMP_AND_SWP +#define OP_FETCH_AND_ADD IBV_WR_ATOMIC_FETCH_AND_ADD +#define OP_RECEIVE 7 /* internal op */ +#define OP_RECEIVE_IMM 8 /* rdma write with immed, internel op */ +#define OP_RECEIVE_MSG_IMM 9 /* recv msg with immed, internel op */ +#define OP_BIND_MW 10 /* internal op */ +#define OP_SEND_UD 11 /* internal op */ +#define OP_RECV_UD 12 /* internal op */ +#define OP_INVALID 0xff + +/* Definitions to map QP state */ +#define IB_QP_STATE_RESET IBV_QPS_RESET +#define IB_QP_STATE_INIT IBV_QPS_INIT +#define IB_QP_STATE_RTR IBV_QPS_RTR +#define IB_QP_STATE_RTS IBV_QPS_RTS +#define IB_QP_STATE_SQD IBV_QPS_SQD +#define IB_QP_STATE_SQE IBV_QPS_SQE +#define IB_QP_STATE_ERROR IBV_QPS_ERR + +/* Definitions for ibverbs/mthca return codes, should be defined in verbs.h */ +/* some are errno and some are -n values */ + +/** + * ibv_get_device_name - Return kernel device name + * ibv_get_device_guid - Return device's node GUID + * ibv_open_device - Return ibv_context or NULL + * ibv_close_device - Return 0, (errno?) + * ibv_get_async_event - Return 0, -1 + * ibv_alloc_pd - Return ibv_pd, NULL + * ibv_dealloc_pd - Return 0, errno + * ibv_reg_mr - Return ibv_mr, NULL + * ibv_dereg_mr - Return 0, errno + * ibv_create_cq - Return ibv_cq, NULL + * ibv_destroy_cq - Return 0, errno + * ibv_get_cq_event - Return 0 & ibv_cq/context, int + * ibv_poll_cq - Return n & ibv_wc, 0 ok, -1 empty, -2 error + * ibv_req_notify_cq - Return 0 (void?) + * ibv_create_qp - Return ibv_qp, NULL + * ibv_modify_qp - Return 0, errno + * ibv_destroy_qp - Return 0, errno + * ibv_post_send - Return 0, -1 & bad_wr + * ibv_post_recv - Return 0, -1 & bad_wr + */ + +/* async handler for DTO, CQ, QP, and unafiliated */ +typedef void (*ib_async_dto_handler_t)( + IN ib_hca_handle_t ib_hca_handle, + IN ib_error_record_t *err_code, + IN void *context); + +typedef void (*ib_async_cq_handler_t)( + IN ib_hca_handle_t ib_hca_handle, + IN ib_cq_handle_t ib_cq_handle, + IN ib_error_record_t *err_code, + IN void *context); + +typedef void (*ib_async_qp_handler_t)( + IN ib_hca_handle_t ib_hca_handle, + IN ib_qp_handle_t ib_qp_handle, + IN ib_error_record_t *err_code, + IN void *context); + +typedef void (*ib_async_handler_t)( + IN ib_hca_handle_t ib_hca_handle, + IN ib_error_record_t *err_code, + IN void *context); + +typedef enum +{ + IB_THREAD_INIT, + IB_THREAD_CREATE, + IB_THREAD_RUN, + IB_THREAD_CANCEL, + IB_THREAD_EXIT + +} ib_thread_state_t; + + +/* provider specfic fields for shared memory support */ +typedef uint32_t ib_shm_transport_t; + +/* prototypes */ +int32_t dapls_ib_init(void); +int32_t dapls_ib_release(void); +enum ibv_mtu dapl_ib_mtu(int mtu); +char *dapl_ib_mtu_str(enum ibv_mtu mtu); +DAT_RETURN getlocalipaddr(DAT_SOCK_ADDR *addr, int addr_len); + +/* inline functions */ +STATIC _INLINE_ IB_HCA_NAME dapl_ib_convert_name (IN char *name) +{ + /* use ascii; name of local device */ + return dapl_os_strdup(name); +} + +STATIC _INLINE_ void dapl_ib_release_name (IN IB_HCA_NAME name) +{ + return; +} + +/* + * Convert errno to DAT_RETURN values + */ +STATIC _INLINE_ DAT_RETURN +dapl_convert_errno( IN int err, IN const char *str ) +{ + if (!err) return DAT_SUCCESS; + +#if DAPL_DBG + if ((err != EAGAIN) && (err != ETIMEDOUT)) + dapl_dbg_log (DAPL_DBG_TYPE_ERR," %s %s\n", str, strerror(err)); +#endif + + switch( err ) + { + case EOVERFLOW : return DAT_LENGTH_ERROR; + case EACCES : return DAT_PRIVILEGES_VIOLATION; + case EPERM : return DAT_PROTECTION_VIOLATION; + case EINVAL : return DAT_INVALID_HANDLE; + case EISCONN : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_CONNECTED; + case ECONNREFUSED : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_NOTREADY; + case ETIMEDOUT : return DAT_TIMEOUT_EXPIRED; + case ENETUNREACH: return DAT_INVALID_ADDRESS | DAT_INVALID_ADDRESS_UNREACHABLE; + case EADDRINUSE : return DAT_CONN_QUAL_IN_USE; + case EALREADY : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_ACTCONNPENDING; + case ENOMEM : return DAT_INSUFFICIENT_RESOURCES; + case EAGAIN : return DAT_QUEUE_EMPTY; + case EINTR : return DAT_INTERRUPTED_CALL; + case EAFNOSUPPORT : return DAT_INVALID_ADDRESS | DAT_INVALID_ADDRESS_MALFORMED; + case EFAULT : + default : return DAT_INTERNAL_ERROR; + } + } + +typedef enum dapl_cm_state +{ + DCM_INIT, + DCM_LISTEN, + DCM_CONN_PENDING, + DCM_RTU_PENDING, + DCM_ACCEPTING, + DCM_ACCEPTING_DATA, + DCM_ACCEPTED, + DCM_REJECTED, + DCM_CONNECTED, + DCM_RELEASED, + DCM_DISCONNECTED, + DCM_DESTROY +} DAPL_CM_STATE; + +STATIC _INLINE_ char * dapl_cm_state_str(IN int st) +{ + static char *state[] = { + "CM_INIT", + "CM_LISTEN", + "CM_CONN_PENDING", + "CM_RTU_PENDING", + "CM_ACCEPTING", + "CM_ACCEPTING_DATA", + "CM_ACCEPTED", + "CM_REJECTED", + "CM_CONNECTED", + "CM_RELEASED", + "CM_DISCONNECTED", + "CM_DESTROY" + }; + return ((st < 0 || st > 11) ? "Invalid CM state?" : state[st]); +} + +#endif /* _DAPL_IB_COMMON_H_ */ diff --git a/dapl/openib_common/dapl_ib_dto.h b/dapl/openib_common/dapl_ib_dto.h new file mode 100644 index 0000000..e6c03b2 --- /dev/null +++ b/dapl/openib_common/dapl_ib_dto.h @@ -0,0 +1,504 @@ +/* + * Copyright (c) 2009 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ +#ifndef _DAPL_IB_DTO_H_ +#define _DAPL_IB_DTO_H_ + +#include "dapl_ib_util.h" + +#ifdef DAT_EXTENSIONS +#include +#endif + +STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p); + +#define CQE_WR_TYPE_UD(id) \ + (((DAPL_COOKIE *)(uintptr_t)id)->ep->qp_handle->qp_type == IBV_QPT_UD) + +/* + * dapls_ib_post_recv + * + * Provider specific Post RECV function + */ +STATIC _INLINE_ DAT_RETURN +dapls_ib_post_recv ( + IN DAPL_EP *ep_ptr, + IN DAPL_COOKIE *cookie, + IN DAT_COUNT segments, + IN DAT_LMR_TRIPLET *local_iov ) +{ + struct ibv_recv_wr wr; + struct ibv_recv_wr *bad_wr; + ib_data_segment_t *ds = (ib_data_segment_t *)local_iov; + DAT_COUNT i, total_len; + int ret; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_rcv: ep %p cookie %p segs %d l_iov %p\n", + ep_ptr, cookie, segments, local_iov); + + /* setup work request */ + total_len = 0; + wr.next = 0; + wr.num_sge = segments; + wr.wr_id = (uint64_t)(uintptr_t)cookie; + wr.sg_list = ds; + + if (cookie != NULL) { + for (i = 0; i < segments; i++) { + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_rcv: l_key 0x%x va %p len %d\n", + ds->lkey, ds->addr, ds->length ); + total_len += ds->length; + ds++; + } + cookie->val.dto.size = total_len; + } + + ret = ibv_post_recv(ep_ptr->qp_handle, &wr, &bad_wr); + + if (ret) + return(dapl_convert_errno(errno,"ibv_recv")); + + DAPL_CNTR(ep_ptr, DCNT_EP_POST_RECV); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_RECV_DATA, total_len); + + return DAT_SUCCESS; +} + +/* + * dapls_ib_post_send + * + * Provider specific Post SEND function + */ +STATIC _INLINE_ DAT_RETURN +dapls_ib_post_send ( + IN DAPL_EP *ep_ptr, + IN ib_send_op_type_t op_type, + IN DAPL_COOKIE *cookie, + IN DAT_COUNT segments, + IN DAT_LMR_TRIPLET *local_iov, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_COMPLETION_FLAGS completion_flags) +{ + struct ibv_send_wr wr; + struct ibv_send_wr *bad_wr; + ib_data_segment_t *ds = (ib_data_segment_t *)local_iov; + ib_hca_transport_t *ibt_ptr = + &ep_ptr->header.owner_ia->hca_ptr->ib_trans; + DAT_COUNT i, total_len; + int ret; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_snd: ep %p op %d ck %p sgs", + "%d l_iov %p r_iov %p f %d\n", + ep_ptr, op_type, cookie, segments, local_iov, + remote_iov, completion_flags); + +#ifdef DAT_EXTENSIONS + if (ep_ptr->qp_handle->qp_type != IBV_QPT_RC) + return(DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP)); +#endif + /* setup the work request */ + wr.next = 0; + wr.opcode = op_type; + wr.num_sge = segments; + wr.send_flags = 0; + wr.wr_id = (uint64_t)(uintptr_t)cookie; + wr.sg_list = ds; + total_len = 0; + + if (cookie != NULL) { + for (i = 0; i < segments; i++ ) { + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_snd: lkey 0x%x va %p len %d\n", + ds->lkey, ds->addr, ds->length ); + total_len += ds->length; + ds++; + } + cookie->val.dto.size = total_len; + } + + if (wr.num_sge && + (op_type == OP_RDMA_WRITE || op_type == OP_RDMA_READ)) { + wr.wr.rdma.remote_addr = remote_iov->virtual_address; + wr.wr.rdma.rkey = remote_iov->rmr_context; + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_snd_rdma: rkey 0x%x va %#016Lx\n", + wr.wr.rdma.rkey, wr.wr.rdma.remote_addr); + } + + + /* inline data for send or write ops */ + if ((total_len <= ibt_ptr->max_inline_send) && + ((op_type == OP_SEND) || (op_type == OP_RDMA_WRITE))) + wr.send_flags |= IBV_SEND_INLINE; + + /* set completion flags in work request */ + wr.send_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & + completion_flags) ? 0 : IBV_SEND_SIGNALED; + wr.send_flags |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & + completion_flags) ? IBV_SEND_FENCE : 0; + wr.send_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & + completion_flags) ? IBV_SEND_SOLICITED : 0; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_snd: op 0x%x flags 0x%x sglist %p, %d\n", + wr.opcode, wr.send_flags, wr.sg_list, wr.num_sge); + + ret = ibv_post_send(ep_ptr->qp_handle, &wr, &bad_wr); + + if (ret) + return(dapl_convert_errno(errno,"ibv_send")); + +#ifdef DAPL_COUNTERS + switch (op_type) { + case OP_SEND: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_DATA,total_len); + break; + case OP_RDMA_WRITE: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_WRITE_DATA,total_len); + break; + case OP_RDMA_READ: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_READ); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_READ_DATA,total_len); + break; + default: + break; + } +#endif /* DAPL_COUNTERS */ + + dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n"); + return DAT_SUCCESS; +} + +/* map Work Completions to DAPL WR operations */ +STATIC _INLINE_ DAT_DTOS dapls_cqe_dtos_opcode(ib_work_completion_t *cqe_p) +{ + switch (cqe_p->opcode) { + + case IBV_WC_SEND: +#ifdef DAT_EXTENSIONS + if (CQE_WR_TYPE_UD(cqe_p->wr_id)) + return (DAT_IB_DTO_SEND_UD); + else +#endif + return (DAT_DTO_SEND); + case IBV_WC_RDMA_READ: + return (DAT_DTO_RDMA_READ); + case IBV_WC_BIND_MW: + return (DAT_DTO_BIND_MW); +#ifdef DAT_EXTENSIONS + case IBV_WC_RDMA_WRITE: + if (cqe_p->wc_flags & IBV_WC_WITH_IMM) + return (DAT_IB_DTO_RDMA_WRITE_IMMED); + else + return (DAT_DTO_RDMA_WRITE); + case IBV_WC_COMP_SWAP: + return (DAT_IB_DTO_CMP_SWAP); + case IBV_WC_FETCH_ADD: + return (DAT_IB_DTO_FETCH_ADD); + case IBV_WC_RECV_RDMA_WITH_IMM: + return (DAT_IB_DTO_RECV_IMMED); +#else + case IBV_WC_RDMA_WRITE: + return (DAT_DTO_RDMA_WRITE); +#endif + case IBV_WC_RECV: +#ifdef DAT_EXTENSIONS + if (CQE_WR_TYPE_UD(cqe_p->wr_id)) + return (DAT_IB_DTO_RECV_UD); + else if (cqe_p->wc_flags & IBV_WC_WITH_IMM) + return (DAT_IB_DTO_RECV_MSG_IMMED); + else +#endif + return (DAT_DTO_RECEIVE); + default: + return (0xff); + } +} +#define DAPL_GET_CQE_DTOS_OPTYPE(cqe_p) dapls_cqe_dtos_opcode(cqe_p) + + +#ifdef DAT_EXTENSIONS +/* + * dapls_ib_post_ext_send + * + * Provider specific extended Post SEND function for atomics + * OP_COMP_AND_SWAP and OP_FETCH_AND_ADD + */ +STATIC _INLINE_ DAT_RETURN +dapls_ib_post_ext_send ( + IN DAPL_EP *ep_ptr, + IN ib_send_op_type_t op_type, + IN DAPL_COOKIE *cookie, + IN DAT_COUNT segments, + IN DAT_LMR_TRIPLET *local_iov, + IN const DAT_RMR_TRIPLET *remote_iov, + IN DAT_UINT32 immed_data, + IN DAT_UINT64 compare_add, + IN DAT_UINT64 swap, + IN DAT_COMPLETION_FLAGS completion_flags, + IN DAT_IB_ADDR_HANDLE *remote_ah) +{ + struct ibv_send_wr wr; + struct ibv_send_wr *bad_wr; + ib_data_segment_t *ds = (ib_data_segment_t *)local_iov; + DAT_COUNT i, total_len; + int ret; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_ext_snd: ep %p op %d ck %p sgs", + "%d l_iov %p r_iov %p f %d\n", + ep_ptr, op_type, cookie, segments, local_iov, + remote_iov, completion_flags, remote_ah); + + /* setup the work request */ + wr.next = 0; + wr.opcode = op_type; + wr.num_sge = segments; + wr.send_flags = 0; + wr.wr_id = (uint64_t)(uintptr_t)cookie; + wr.sg_list = ds; + total_len = 0; + + if (cookie != NULL) { + for (i = 0; i < segments; i++ ) { + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_snd: lkey 0x%x va %p len %d\n", + ds->lkey, ds->addr, ds->length ); + total_len += ds->length; + ds++; + } + cookie->val.dto.size = total_len; + } + + switch (op_type) { + case OP_RDMA_WRITE_IMM: + /* OP_RDMA_WRITE)IMMED has direct IB wr_type mapping */ + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_ext: rkey 0x%x va %#016Lx immed=0x%x\n", + remote_iov?remote_iov->rmr_context:0, + remote_iov?remote_iov->virtual_address:0, + immed_data); + + wr.imm_data = immed_data; + if (wr.num_sge) { + wr.wr.rdma.remote_addr = remote_iov->virtual_address; + wr.wr.rdma.rkey = remote_iov->rmr_context; + } + break; + case OP_COMP_AND_SWAP: + /* OP_COMP_AND_SWAP has direct IB wr_type mapping */ + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_ext: OP_COMP_AND_SWAP=%lx," + "%lx rkey 0x%x va %#016Lx\n", + compare_add, swap, remote_iov->rmr_context, + remote_iov->virtual_address); + + wr.wr.atomic.compare_add = compare_add; + wr.wr.atomic.swap = swap; + wr.wr.atomic.remote_addr = remote_iov->virtual_address; + wr.wr.atomic.rkey = remote_iov->rmr_context; + break; + case OP_FETCH_AND_ADD: + /* OP_FETCH_AND_ADD has direct IB wr_type mapping */ + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_ext: OP_FETCH_AND_ADD=%lx," + "%lx rkey 0x%x va %#016Lx\n", + compare_add, remote_iov->rmr_context, + remote_iov->virtual_address); + + wr.wr.atomic.compare_add = compare_add; + wr.wr.atomic.remote_addr = remote_iov->virtual_address; + wr.wr.atomic.rkey = remote_iov->rmr_context; + break; + case OP_SEND_UD: + /* post must be on EP with service_type of UD */ + if (ep_ptr->qp_handle->qp_type != IBV_QPT_UD) + return(DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP)); + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_ext: OP_SEND_UD ah=%p" + " qp_num=0x%x\n", + remote_ah, remote_ah->qpn); + + wr.opcode = OP_SEND; + wr.wr.ud.ah = remote_ah->ah; + wr.wr.ud.remote_qpn = remote_ah->qpn; + wr.wr.ud.remote_qkey = DAT_UD_QKEY; + break; + default: + break; + } + + /* set completion flags in work request */ + wr.send_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & + completion_flags) ? 0 : IBV_SEND_SIGNALED; + wr.send_flags |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & + completion_flags) ? IBV_SEND_FENCE : 0; + wr.send_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & + completion_flags) ? IBV_SEND_SOLICITED : 0; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " post_snd: op 0x%x flags 0x%x sglist %p, %d\n", + wr.opcode, wr.send_flags, wr.sg_list, wr.num_sge); + + ret = ibv_post_send(ep_ptr->qp_handle, &wr, &bad_wr); + + if (ret) + return( dapl_convert_errno(errno,"ibv_send") ); + +#ifdef DAPL_COUNTERS + switch (op_type) { + case OP_RDMA_WRITE_IMM: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE_IMM); + DAPL_CNTR_DATA(ep_ptr, + DCNT_EP_POST_WRITE_IMM_DATA, total_len); + break; + case OP_COMP_AND_SWAP: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_CMP_SWAP); + break; + case OP_FETCH_AND_ADD: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_FETCH_ADD); + break; + case OP_SEND_UD: + DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND_UD); + DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_UD_DATA, total_len); + break; + default: + break; + } +#endif /* DAPL_COUNTERS */ + + dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n"); + return DAT_SUCCESS; +} +#endif + +STATIC _INLINE_ DAT_RETURN +dapls_ib_optional_prv_dat( + IN DAPL_CR *cr_ptr, + IN const void *event_data, + OUT DAPL_CR **cr_pp) +{ + return DAT_SUCCESS; +} + + +/* map Work Completions to DAPL WR operations */ +STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p) +{ +#ifdef DAPL_COUNTERS + DAPL_COOKIE *cookie = (DAPL_COOKIE *)(uintptr_t)cqe_p->wr_id; +#endif /* DAPL_COUNTERS */ + + switch (cqe_p->opcode) { + case IBV_WC_SEND: + if (CQE_WR_TYPE_UD(cqe_p->wr_id)) + return(OP_SEND_UD); + else + return (OP_SEND); + case IBV_WC_RDMA_WRITE: + if (cqe_p->wc_flags & IBV_WC_WITH_IMM) + return (OP_RDMA_WRITE_IMM); + else + return (OP_RDMA_WRITE); + case IBV_WC_RDMA_READ: + return (OP_RDMA_READ); + case IBV_WC_COMP_SWAP: + return (OP_COMP_AND_SWAP); + case IBV_WC_FETCH_ADD: + return (OP_FETCH_AND_ADD); + case IBV_WC_BIND_MW: + return (OP_BIND_MW); + case IBV_WC_RECV: + if (CQE_WR_TYPE_UD(cqe_p->wr_id)) { + DAPL_CNTR(cookie->ep, DCNT_EP_RECV_UD); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_UD_DATA, + cqe_p->byte_len); + return (OP_RECV_UD); + } + else if (cqe_p->wc_flags & IBV_WC_WITH_IMM) { + DAPL_CNTR(cookie->ep, DCNT_EP_RECV_IMM); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_IMM_DATA, + cqe_p->byte_len); + return (OP_RECEIVE_IMM); + } else { + DAPL_CNTR(cookie->ep, DCNT_EP_RECV); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_DATA, + cqe_p->byte_len); + return (OP_RECEIVE); + } + case IBV_WC_RECV_RDMA_WITH_IMM: + DAPL_CNTR(cookie->ep, DCNT_EP_RECV_RDMA_IMM); + DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_RDMA_IMM_DATA, + cqe_p->byte_len); + return (OP_RECEIVE_IMM); + default: + return (OP_INVALID); + } +} + +#define DAPL_GET_CQE_OPTYPE(cqe_p) dapls_cqe_opcode(cqe_p) +#define DAPL_GET_CQE_WRID(cqe_p) ((ib_work_completion_t*)cqe_p)->wr_id +#define DAPL_GET_CQE_STATUS(cqe_p) ((ib_work_completion_t*)cqe_p)->status +#define DAPL_GET_CQE_VENDOR_ERR(cqe_p) ((ib_work_completion_t*)cqe_p)->vendor_err +#define DAPL_GET_CQE_BYTESNUM(cqe_p) ((ib_work_completion_t*)cqe_p)->byte_len +#define DAPL_GET_CQE_IMMED_DATA(cqe_p) ((ib_work_completion_t*)cqe_p)->imm_data + +STATIC _INLINE_ char * dapls_dto_op_str(int op) +{ + static char *optable[] = + { + "OP_RDMA_WRITE", + "OP_RDMA_WRITE_IMM", + "OP_SEND", + "OP_SEND_IMM", + "OP_RDMA_READ", + "OP_COMP_AND_SWAP", + "OP_FETCH_AND_ADD", + "OP_RECEIVE", + "OP_RECEIVE_MSG_IMM", + "OP_RECEIVE_RDMA_IMM", + "OP_BIND_MW" + "OP_SEND_UD" + "OP_RECV_UD" + }; + return ((op < 0 || op > 12) ? "Invalid CQE OP?" : optable[op]); +} + +static _INLINE_ char * +dapls_cqe_op_str(IN ib_work_completion_t *cqe_ptr) +{ + return dapls_dto_op_str(DAPL_GET_CQE_OPTYPE(cqe_ptr)); +} + +#define DAPL_GET_CQE_OP_STR(cqe) dapls_cqe_op_str(cqe) + +#endif /* _DAPL_IB_DTO_H_ */ diff --git a/dapl/openib_common/ib_extensions.c b/dapl/openib_common/ib_extensions.c new file mode 100644 index 0000000..3c418e1 --- /dev/null +++ b/dapl/openib_common/ib_extensions.c @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2007-2009 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" +#include "dapl_ib_util.h" +#include "dapl_ep_util.h" +#include "dapl_cookie.h" +#include + +DAT_RETURN +dapli_post_ext(IN DAT_EP_HANDLE ep_handle, + IN DAT_UINT64 cmp_add, + IN DAT_UINT64 swap, + IN DAT_UINT32 immed_data, + IN DAT_COUNT segments, + IN DAT_LMR_TRIPLET * local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET * remote_iov, + IN int op_type, + IN DAT_COMPLETION_FLAGS flags, IN DAT_IB_ADDR_HANDLE * ah); + +/* + * dapl_extensions + * + * Process extension requests + * + * Input: + * ext_type, + * ... + * + * Output: + * Depends.... + * + * Returns: + * DAT_SUCCESS + * DAT_NOT_IMPLEMENTED + * ..... + * + */ +DAT_RETURN +dapl_extensions(IN DAT_HANDLE dat_handle, + IN DAT_EXTENDED_OP ext_op, IN va_list args) +{ + DAT_EP_HANDLE ep; + DAT_IB_ADDR_HANDLE *ah = NULL; + DAT_LMR_TRIPLET *lmr_p; + DAT_DTO_COOKIE cookie; + const DAT_RMR_TRIPLET *rmr_p; + DAT_UINT64 dat_uint64a, dat_uint64b; + DAT_UINT32 dat_uint32; + DAT_COUNT segments = 1; + DAT_COMPLETION_FLAGS comp_flags; + DAT_RETURN status = DAT_NOT_IMPLEMENTED; + + dapl_dbg_log(DAPL_DBG_TYPE_API, + "dapl_extensions(hdl %p operation %d, ...)\n", + dat_handle, ext_op); + + switch ((int)ext_op) { + + case DAT_IB_RDMA_WRITE_IMMED_OP: + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " WRITE_IMMED_DATA extension call\n"); + + ep = dat_handle; /* ep_handle */ + segments = va_arg(args, DAT_COUNT); /* num segments */ + lmr_p = va_arg(args, DAT_LMR_TRIPLET *); + cookie = va_arg(args, DAT_DTO_COOKIE); + rmr_p = va_arg(args, const DAT_RMR_TRIPLET *); + dat_uint32 = va_arg(args, DAT_UINT32); /* immed data */ + comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); + + status = dapli_post_ext(ep, 0, 0, dat_uint32, segments, lmr_p, + cookie, rmr_p, OP_RDMA_WRITE_IMM, + comp_flags, ah); + break; + + case DAT_IB_CMP_AND_SWAP_OP: + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " CMP_AND_SWAP extension call\n"); + + ep = dat_handle; /* ep_handle */ + dat_uint64a = va_arg(args, DAT_UINT64); /* cmp_value */ + dat_uint64b = va_arg(args, DAT_UINT64); /* swap_value */ + lmr_p = va_arg(args, DAT_LMR_TRIPLET *); + cookie = va_arg(args, DAT_DTO_COOKIE); + rmr_p = va_arg(args, const DAT_RMR_TRIPLET *); + comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); + + status = dapli_post_ext(ep, dat_uint64a, dat_uint64b, + 0, segments, lmr_p, cookie, rmr_p, + OP_COMP_AND_SWAP, comp_flags, ah); + break; + + case DAT_IB_FETCH_AND_ADD_OP: + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " FETCH_AND_ADD extension call\n"); + + ep = dat_handle; /* ep_handle */ + dat_uint64a = va_arg(args, DAT_UINT64); /* add value */ + lmr_p = va_arg(args, DAT_LMR_TRIPLET *); + cookie = va_arg(args, DAT_DTO_COOKIE); + rmr_p = va_arg(args, const DAT_RMR_TRIPLET *); + comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); + + status = dapli_post_ext(ep, dat_uint64a, 0, 0, segments, + lmr_p, cookie, rmr_p, + OP_FETCH_AND_ADD, comp_flags, ah); + break; + + case DAT_IB_UD_SEND_OP: + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " UD post_send extension call\n"); + + ep = dat_handle; /* ep_handle */ + segments = va_arg(args, DAT_COUNT); /* segments */ + lmr_p = va_arg(args, DAT_LMR_TRIPLET *); + ah = va_arg(args, DAT_IB_ADDR_HANDLE *); + cookie = va_arg(args, DAT_DTO_COOKIE); + comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); + + status = dapli_post_ext(ep, 0, 0, 0, segments, + lmr_p, cookie, NULL, + OP_SEND_UD, comp_flags, ah); + break; + +#ifdef DAPL_COUNTERS + case DAT_QUERY_COUNTERS_OP: + { + int cntr, reset; + DAT_UINT64 *p_cntr_out; + + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " Query counter extension call\n"); + + cntr = va_arg(args, int); + p_cntr_out = va_arg(args, DAT_UINT64 *); + reset = va_arg(args, int); + + status = dapl_query_counter(dat_handle, cntr, + p_cntr_out, reset); + break; + } + case DAT_PRINT_COUNTERS_OP: + { + int cntr, reset; + + dapl_dbg_log(DAPL_DBG_TYPE_RTN, + " Print counter extension call\n"); + + cntr = va_arg(args, int); + reset = va_arg(args, int); + + dapl_print_counter(dat_handle, cntr, reset); + status = DAT_SUCCESS; + break; + } +#endif /* DAPL_COUNTERS */ + + default: + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + "unsupported extension(%d)\n", (int)ext_op); + } + + return (status); +} + +DAT_RETURN +dapli_post_ext(IN DAT_EP_HANDLE ep_handle, + IN DAT_UINT64 cmp_add, + IN DAT_UINT64 swap, + IN DAT_UINT32 immed_data, + IN DAT_COUNT segments, + IN DAT_LMR_TRIPLET * local_iov, + IN DAT_DTO_COOKIE user_cookie, + IN const DAT_RMR_TRIPLET * remote_iov, + IN int op_type, + IN DAT_COMPLETION_FLAGS flags, IN DAT_IB_ADDR_HANDLE * ah) +{ + DAPL_EP *ep_ptr; + ib_qp_handle_t qp_ptr; + DAPL_COOKIE *cookie = NULL; + DAT_RETURN dat_status = DAT_SUCCESS; + + dapl_dbg_log(DAPL_DBG_TYPE_API, + " post_ext_op: ep %p cmp_val %d " + "swap_val %d cookie 0x%x, r_iov %p, flags 0x%x, ah %p\n", + ep_handle, (unsigned)cmp_add, (unsigned)swap, + (unsigned)user_cookie.as_64, remote_iov, flags, ah); + + if (DAPL_BAD_HANDLE(ep_handle, DAPL_MAGIC_EP)) + return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP)); + + ep_ptr = (DAPL_EP *) ep_handle; + qp_ptr = ep_ptr->qp_handle; + + /* + * Synchronization ok since this buffer is only used for send + * requests, which aren't allowed to race with each other. + */ + dat_status = dapls_dto_cookie_alloc(&ep_ptr->req_buffer, + DAPL_DTO_TYPE_EXTENSION, + user_cookie, &cookie); + if (dat_status != DAT_SUCCESS) + goto bail; + + /* + * Take reference before posting to avoid race conditions with + * completions + */ + dapl_os_atomic_inc(&ep_ptr->req_count); + + /* + * Invoke provider specific routine to post DTO + */ + dat_status = dapls_ib_post_ext_send(ep_ptr, op_type, cookie, segments, /* data segments */ + local_iov, remote_iov, immed_data, /* immed data */ + cmp_add, /* compare or add */ + swap, /* swap */ + flags, ah); + + if (dat_status != DAT_SUCCESS) { + dapl_os_atomic_dec(&ep_ptr->req_count); + dapls_cookie_dealloc(&ep_ptr->req_buffer, cookie); + } + + bail: + return dat_status; + +} + +/* + * New provider routine to process extended DTO events + */ +void +dapls_cqe_to_event_extension(IN DAPL_EP * ep_ptr, + IN DAPL_COOKIE * cookie, + IN ib_work_completion_t * cqe_ptr, + IN DAT_EVENT * event_ptr) +{ + uint32_t ibtype; + DAT_DTO_COMPLETION_EVENT_DATA *dto = + &event_ptr->event_data.dto_completion_event_data; + DAT_IB_EXTENSION_EVENT_DATA *ext_data = (DAT_IB_EXTENSION_EVENT_DATA *) + & event_ptr->event_extension_data[0]; + DAT_DTO_COMPLETION_STATUS dto_status; + + /* Get status from cqe */ + dto_status = dapls_ib_get_dto_status(cqe_ptr); + + dapl_dbg_log(DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: dto_ptr %p ext_ptr %p status %d\n", + dto, ext_data, dto_status); + + event_ptr->event_number = DAT_IB_DTO_EVENT; + dto->ep_handle = cookie->ep; + dto->user_cookie = cookie->val.dto.cookie; + dto->operation = DAPL_GET_CQE_DTOS_OPTYPE(cqe_ptr); /* new for 2.0 */ + dto->status = ext_data->status = dto_status; + + if (dto_status != DAT_DTO_SUCCESS) + return; + + /* + * Get operation type from CQ work completion entry and + * if extented operation then set extended event data + */ + ibtype = DAPL_GET_CQE_OPTYPE(cqe_ptr); + + switch (ibtype) { + + case OP_RDMA_WRITE_IMM: + dapl_dbg_log(DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: OP_RDMA_WRITE_IMMED\n"); + + /* type and outbound rdma write transfer size */ + dto->transfered_length = cookie->val.dto.size; + ext_data->type = DAT_IB_RDMA_WRITE_IMMED; + break; + case OP_RECEIVE_IMM: + dapl_dbg_log(DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: OP_RECEIVE_RDMA_IMMED\n"); + + /* immed recvd, type and inbound rdma write transfer size */ + dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); + ext_data->type = DAT_IB_RDMA_WRITE_IMMED_DATA; + ext_data->val.immed.data = DAPL_GET_CQE_IMMED_DATA(cqe_ptr); + break; + case OP_RECEIVE_MSG_IMM: + dapl_dbg_log(DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: OP_RECEIVE_MSG_IMMED\n"); + + /* immed recvd, type and inbound recv message transfer size */ + dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); + ext_data->type = DAT_IB_RECV_IMMED_DATA; + ext_data->val.immed.data = DAPL_GET_CQE_IMMED_DATA(cqe_ptr); + break; + case OP_COMP_AND_SWAP: + dapl_dbg_log(DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: COMP_AND_SWAP_RESP\n"); + + /* original data is returned in LMR provided with post */ + ext_data->type = DAT_IB_CMP_AND_SWAP; + dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); + break; + case OP_FETCH_AND_ADD: + dapl_dbg_log(DAPL_DBG_TYPE_EVD, + " cqe_to_event_ext: FETCH_AND_ADD_RESP\n"); + + /* original data is returned in LMR provided with post */ + ext_data->type = DAT_IB_FETCH_AND_ADD; + dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); + break; + case OP_SEND_UD: + dapl_dbg_log(DAPL_DBG_TYPE_EVD, " cqe_to_event_ext: UD_SEND\n"); + + /* type and outbound send transfer size */ + ext_data->type = DAT_IB_UD_SEND; + dto->transfered_length = cookie->val.dto.size; + break; + case OP_RECV_UD: + dapl_dbg_log(DAPL_DBG_TYPE_EVD, " cqe_to_event_ext: UD_RECV\n"); + + /* type and inbound recv message transfer size */ + ext_data->type = DAT_IB_UD_RECV; + dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); + break; + + default: + /* not extended operation */ + ext_data->status = DAT_IB_OP_ERR; + dto->status = DAT_DTO_ERR_TRANSPORT; + break; + } +} diff --git a/dapl/openib_common/mem.c b/dapl/openib_common/mem.c new file mode 100644 index 0000000..8a3e152 --- /dev/null +++ b/dapl/openib_common/mem.c @@ -0,0 +1,370 @@ +/* + * Copyright (c) 2005-2007 Intel Corporation. All rights reserved. + * + * This Software is licensed under one of the following licenses: + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_lmr_util.h" + +/* + * dapls_convert_privileges + * + * Convert LMR privileges to provider + * + * Input: + * DAT_MEM_PRIV_FLAGS + * + * Output: + * none + * + * Returns: + * ibv_access_flags + * + */ +STATIC _INLINE_ int dapls_convert_privileges(IN DAT_MEM_PRIV_FLAGS privileges) +{ + int access = 0; + + /* + * if (DAT_MEM_PRIV_LOCAL_READ_FLAG & privileges) do nothing + */ + if (DAT_MEM_PRIV_LOCAL_WRITE_FLAG & privileges) + access |= IBV_ACCESS_LOCAL_WRITE; + if (DAT_MEM_PRIV_REMOTE_WRITE_FLAG & privileges) + access |= IBV_ACCESS_REMOTE_WRITE; + if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) + access |= IBV_ACCESS_REMOTE_READ; + if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) + access |= IBV_ACCESS_REMOTE_READ; + if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) + access |= IBV_ACCESS_REMOTE_READ; +#ifdef DAT_EXTENSIONS + if (DAT_IB_MEM_PRIV_REMOTE_ATOMIC & privileges) + access |= IBV_ACCESS_REMOTE_ATOMIC; +#endif + + return access; +} + +/* + * dapl_ib_pd_alloc + * + * Alloc a PD + * + * Input: + * ia_handle IA handle + * pz pointer to PZ struct + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN dapls_ib_pd_alloc(IN DAPL_IA * ia_ptr, IN DAPL_PZ * pz) +{ + /* get a protection domain */ + pz->pd_handle = ibv_alloc_pd(ia_ptr->hca_ptr->ib_hca_handle); + if (!pz->pd_handle) + return (dapl_convert_errno(ENOMEM, "alloc_pd")); + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " pd_alloc: pd_handle=%p\n", pz->pd_handle); + + return DAT_SUCCESS; +} + +/* + * dapl_ib_pd_free + * + * Free a PD + * + * Input: + * ia_handle IA handle + * PZ_ptr pointer to PZ struct + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_STATE + * + */ +DAT_RETURN dapls_ib_pd_free(IN DAPL_PZ * pz) +{ + if (pz->pd_handle != IB_INVALID_HANDLE) { + if (ibv_dealloc_pd(pz->pd_handle)) + return (dapl_convert_errno(errno, "ibv_dealloc_pd")); + pz->pd_handle = IB_INVALID_HANDLE; + } + return DAT_SUCCESS; +} + +/* + * dapl_ib_mr_register + * + * Register a virtual memory region + * + * Input: + * ia_handle IA handle + * lmr pointer to dapl_lmr struct + * virt_addr virtual address of beginning of mem region + * length length of memory region + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mr_register(IN DAPL_IA * ia_ptr, + IN DAPL_LMR * lmr, + IN DAT_PVOID virt_addr, + IN DAT_VLEN length, + IN DAT_MEM_PRIV_FLAGS privileges, IN DAT_VA_TYPE va_type) +{ + ib_pd_handle_t ib_pd_handle; + struct ibv_device *ibv_dev = ia_ptr->hca_ptr->ib_hca_handle->device; + + ib_pd_handle = ((DAPL_PZ *) lmr->param.pz_handle)->pd_handle; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " mr_register: ia=%p, lmr=%p va=%p ln=%d pv=0x%x\n", + ia_ptr, lmr, virt_addr, length, privileges); + + /* TODO: shared memory */ + if (lmr->param.mem_type == DAT_MEM_TYPE_SHARED_VIRTUAL) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " mr_register_shared: NOT IMPLEMENTED\n"); + return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); + } + + /* iWARP only support */ + if ((va_type == DAT_VA_TYPE_ZB) && + (ibv_dev->transport_type != IBV_TRANSPORT_IWARP)) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " va_type == DAT_VA_TYPE_ZB: NOT SUPPORTED\n"); + return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); + } + + /* local read is default on IB */ + lmr->mr_handle = + ibv_reg_mr(((DAPL_PZ *) lmr->param.pz_handle)->pd_handle, + virt_addr, length, dapls_convert_privileges(privileges)); + + if (!lmr->mr_handle) + return (dapl_convert_errno(ENOMEM, "reg_mr")); + + lmr->param.lmr_context = lmr->mr_handle->lkey; + lmr->param.rmr_context = lmr->mr_handle->rkey; + lmr->param.registered_size = length; + lmr->param.registered_address = (DAT_VADDR) (uintptr_t) virt_addr; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " mr_register: mr=%p addr=%p pd %p ctx %p " + "lkey=0x%x rkey=0x%x priv=%x\n", + lmr->mr_handle, lmr->mr_handle->addr, + lmr->mr_handle->pd, lmr->mr_handle->context, + lmr->mr_handle->lkey, lmr->mr_handle->rkey, + length, dapls_convert_privileges(privileges)); + + return DAT_SUCCESS; +} + +/* + * dapl_ib_mr_deregister + * + * Free a memory region + * + * Input: + * lmr pointer to dapl_lmr struct + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_STATE + * + */ +DAT_RETURN dapls_ib_mr_deregister(IN DAPL_LMR * lmr) +{ + if (lmr->mr_handle != IB_INVALID_HANDLE) { + if (ibv_dereg_mr(lmr->mr_handle)) + return (dapl_convert_errno(errno, "dereg_pd")); + lmr->mr_handle = IB_INVALID_HANDLE; + } + return DAT_SUCCESS; +} + +/* + * dapl_ib_mr_register_shared + * + * Register a virtual memory region + * + * Input: + * ia_ptr IA handle + * lmr pointer to dapl_lmr struct + * privileges + * va_type + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mr_register_shared(IN DAPL_IA * ia_ptr, + IN DAPL_LMR * lmr, + IN DAT_MEM_PRIV_FLAGS privileges, + IN DAT_VA_TYPE va_type) +{ + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " mr_register_shared: NOT IMPLEMENTED\n"); + + return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); +} + +/* + * dapls_ib_mw_alloc + * + * Bind a protection domain to a memory window + * + * Input: + * rmr Initialized rmr to hold binding handles + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN dapls_ib_mw_alloc(IN DAPL_RMR * rmr) +{ + + dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_alloc: NOT IMPLEMENTED\n"); + + return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); +} + +/* + * dapls_ib_mw_free + * + * Release bindings of a protection domain to a memory window + * + * Input: + * rmr Initialized rmr to hold binding handles + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_STATE + * + */ +DAT_RETURN dapls_ib_mw_free(IN DAPL_RMR * rmr) +{ + dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_free: NOT IMPLEMENTED\n"); + + return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); +} + +/* + * dapls_ib_mw_bind + * + * Bind a protection domain to a memory window + * + * Input: + * rmr Initialized rmr to hold binding handles + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER; + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mw_bind(IN DAPL_RMR * rmr, + IN DAPL_LMR * lmr, + IN DAPL_EP * ep, + IN DAPL_COOKIE * cookie, + IN DAT_VADDR virtual_address, + IN DAT_VLEN length, + IN DAT_MEM_PRIV_FLAGS mem_priv, IN DAT_BOOLEAN is_signaled) +{ + dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_bind: NOT IMPLEMENTED\n"); + + return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); +} + +/* + * dapls_ib_mw_unbind + * + * Unbind a protection domain from a memory window + * + * Input: + * rmr Initialized rmr to hold binding handles + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_PARAMETER; + * DAT_INVALID_STATE; + * DAT_INSUFFICIENT_RESOURCES + * + */ +DAT_RETURN +dapls_ib_mw_unbind(IN DAPL_RMR * rmr, + IN DAPL_EP * ep, + IN DAPL_COOKIE * cookie, IN DAT_BOOLEAN is_signaled) +{ + dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_unbind: NOT IMPLEMENTED\n"); + + return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/dapl/openib_common/qp.c b/dapl/openib_common/qp.c new file mode 100644 index 0000000..9fb7c96 --- /dev/null +++ b/dapl/openib_common/qp.c @@ -0,0 +1,515 @@ +/* + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ +#include "dapl.h" +#include "dapl_adapter_util.h" + +/* + * dapl_ib_qp_alloc + * + * Alloc a QP + * + * Input: + * *ep_ptr pointer to EP INFO + * ib_hca_handle provider HCA handle + * ib_pd_handle provider protection domain handle + * cq_recv provider recv CQ handle + * cq_send provider send CQ handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INTERNAL_ERROR + * + */ +DAT_RETURN +dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr, + IN DAPL_EP * ep_ptr, IN DAPL_EP * ep_ctx_ptr) +{ + DAT_EP_ATTR *attr; + DAPL_EVD *rcv_evd, *req_evd; + ib_cq_handle_t rcv_cq, req_cq; + ib_pd_handle_t ib_pd_handle; + struct ibv_qp_init_attr qp_create; +#ifdef _OPENIB_CMA_ + dp_ib_cm_handle_t conn; +#endif + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " qp_alloc: ia_ptr %p ep_ptr %p ep_ctx_ptr %p\n", + ia_ptr, ep_ptr, ep_ctx_ptr); + + attr = &ep_ptr->param.ep_attr; + ib_pd_handle = ((DAPL_PZ *) ep_ptr->param.pz_handle)->pd_handle; + rcv_evd = (DAPL_EVD *) ep_ptr->param.recv_evd_handle; + req_evd = (DAPL_EVD *) ep_ptr->param.request_evd_handle; + + /* + * DAT allows usage model of EP's with no EVD's but IB does not. + * Create a CQ with zero entries under the covers to support and + * catch any invalid posting. + */ + if (rcv_evd != DAT_HANDLE_NULL) + rcv_cq = rcv_evd->ib_cq_handle; + else if (!ia_ptr->hca_ptr->ib_trans.ib_cq_empty) + rcv_cq = ia_ptr->hca_ptr->ib_trans.ib_cq_empty; + else { + struct ibv_comp_channel *channel = + rcv_evd->cq_wait_obj_handle; + + /* Call IB verbs to create CQ */ + rcv_cq = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, + 0, NULL, channel, 0); + + if (rcv_cq == IB_INVALID_HANDLE) + return (dapl_convert_errno(ENOMEM, "create_cq")); + + ia_ptr->hca_ptr->ib_trans.ib_cq_empty = rcv_cq; + } + if (req_evd != DAT_HANDLE_NULL) + req_cq = req_evd->ib_cq_handle; + else + req_cq = ia_ptr->hca_ptr->ib_trans.ib_cq_empty; + + /* + * IMPLEMENTATION NOTE: + * uDAPL allows consumers to post buffers on the EP after creation + * and before a connect request (outbound and inbound). This forces + * a binding to a device during the hca_open call and requires the + * consumer to predetermine which device to listen on or connect from. + * This restriction eliminates any option of listening or connecting + * over multiple devices. uDAPL should add API's to resolve addresses + * and bind to the device at the approriate time (before connect + * and after CR arrives). Discovery should happen at connection time + * based on addressing and not on static configuration during open. + */ + +#ifdef _OPENIB_CMA_ + /* Allocate CM and initialize lock */ + if ((conn = dapls_ib_cm_create(ep_ptr)) == NULL) + return (dapl_convert_errno(ENOMEM, "create_cq")); + + /* open identifies the local device; per DAT specification */ + if (rdma_bind_addr(conn->cm_id, + (struct sockaddr *)&ia_ptr->hca_ptr->hca_address)) + return (dapl_convert_errno(EAFNOSUPPORT, "create_cq")); +#endif + /* Setup attributes and create qp */ + dapl_os_memzero((void *)&qp_create, sizeof(qp_create)); + qp_create.send_cq = req_cq; + qp_create.cap.max_send_wr = attr->max_request_dtos; + qp_create.cap.max_send_sge = attr->max_request_iov; + qp_create.cap.max_inline_data = + ia_ptr->hca_ptr->ib_trans.max_inline_send; + qp_create.qp_type = IBV_QPT_RC; + qp_create.qp_context = (void *)ep_ptr; + +#ifdef DAT_EXTENSIONS + if (attr->service_type == DAT_IB_SERVICE_TYPE_UD) { +#ifdef _OPENIB_CMA_ + return (DAT_NOT_IMPLEMENTED); +#endif + qp_create.qp_type = IBV_QPT_UD; + if (attr->max_message_size > + (128 << ia_ptr->hca_ptr->ib_trans.mtu)) { + return (DAT_INVALID_PARAMETER | DAT_INVALID_ARG6); + } + } +#endif + + /* ibv assumes rcv_cq is never NULL, set to req_cq */ + if (rcv_cq == NULL) { + qp_create.recv_cq = req_cq; + qp_create.cap.max_recv_wr = 0; + qp_create.cap.max_recv_sge = 0; + } else { + qp_create.recv_cq = rcv_cq; + qp_create.cap.max_recv_wr = attr->max_recv_dtos; + qp_create.cap.max_recv_sge = attr->max_recv_iov; + } + +#ifdef _OPENIB_CMA_ + if (rdma_create_qp(conn->cm_id, ib_pd_handle, &qp_create)) { + dapls_ib_cm_free(conn, ep_ptr); + return (dapl_convert_errno(errno, "create_qp")); + } + ep_ptr->qp_handle = conn->cm_id->qp; + ep_ptr->cm_handle = conn; + ep_ptr->qp_state = IBV_QPS_INIT; + + /* setup up ep->param to reference the bound local address and port */ + ep_ptr->param.local_ia_address_ptr = + &conn->cm_id->route.addr.src_addr; + ep_ptr->param.local_port_qual = rdma_get_src_port(conn->cm_id); +#else + ep_ptr->qp_handle = ibv_create_qp(ib_pd_handle, &qp_create); + if (!ep_ptr->qp_handle) + return (dapl_convert_errno(ENOMEM, "create_qp")); + + /* Setup QP attributes for INIT state on the way out */ + if (dapls_modify_qp_state(ep_ptr->qp_handle, + IBV_QPS_INIT, NULL) != DAT_SUCCESS) { + ibv_destroy_qp(ep_ptr->qp_handle); + ep_ptr->qp_handle = IB_INVALID_HANDLE; + return DAT_INTERNAL_ERROR; + } +#endif + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " qp_alloc: qpn %p sq %d,%d rq %d,%d\n", + ep_ptr->qp_handle->qp_num, + qp_create.cap.max_send_wr, qp_create.cap.max_send_sge, + qp_create.cap.max_recv_wr, qp_create.cap.max_recv_sge); + + return DAT_SUCCESS; +} + +/* + * dapl_ib_qp_free + * + * Free a QP + * + * Input: + * ia_handle IA handle + * *ep_ptr pointer to EP INFO + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * dapl_convert_errno + * + */ +DAT_RETURN dapls_ib_qp_free(IN DAPL_IA * ia_ptr, IN DAPL_EP * ep_ptr) +{ + dapl_dbg_log(DAPL_DBG_TYPE_EP, " qp_free: ep_ptr %p qp %p\n", + ep_ptr, ep_ptr->qp_handle); + + if (ep_ptr->cm_handle != NULL) { + dapls_ib_cm_free(ep_ptr->cm_handle, ep_ptr); + } + + if (ep_ptr->qp_handle != NULL) { + /* force error state to flush queue, then destroy */ + dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_ERR, NULL); + + if (ibv_destroy_qp(ep_ptr->qp_handle)) + return (dapl_convert_errno(errno, "destroy_qp")); + + ep_ptr->qp_handle = NULL; + } + +#ifdef DAT_EXTENSIONS + /* UD endpoints can have many CR associations and will not + * set ep->cm_handle. Call provider with cm_ptr null to incidate + * UD type multi CR's for this EP. It will parse internal list + * and cleanup all associations. + */ + if (ep_ptr->param.ep_attr.service_type == DAT_IB_SERVICE_TYPE_UD) + dapls_ib_cm_free(NULL, ep_ptr); +#endif + + return DAT_SUCCESS; +} + +/* + * dapl_ib_qp_modify + * + * Set the QP to the parameters specified in an EP_PARAM + * + * The EP_PARAM structure that is provided has been + * sanitized such that only non-zero values are valid. + * + * Input: + * ib_hca_handle HCA handle + * qp_handle QP handle + * ep_attr Sanitized EP Params + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_qp_modify(IN DAPL_IA * ia_ptr, + IN DAPL_EP * ep_ptr, IN DAT_EP_ATTR * attr) +{ + struct ibv_qp_attr qp_attr; + + if (ep_ptr->qp_handle == IB_INVALID_HANDLE) + return DAT_INVALID_PARAMETER; + + /* + * EP state, qp_handle state should be an indication + * of current state but the only way to be sure is with + * a user mode ibv_query_qp call which is NOT available + */ + + /* move to error state if necessary */ + if ((ep_ptr->qp_state == IB_QP_STATE_ERROR) && + (ep_ptr->qp_handle->state != IBV_QPS_ERR)) { + return (dapls_modify_qp_state(ep_ptr->qp_handle, + IBV_QPS_ERR, NULL)); + } + + /* + * Check if we have the right qp_state to modify attributes + */ + if ((ep_ptr->qp_handle->state != IBV_QPS_RTR) && + (ep_ptr->qp_handle->state != IBV_QPS_RTS)) + return DAT_INVALID_STATE; + + /* Adjust to current EP attributes */ + dapl_os_memzero((void *)&qp_attr, sizeof(qp_attr)); + qp_attr.cap.max_send_wr = attr->max_request_dtos; + qp_attr.cap.max_recv_wr = attr->max_recv_dtos; + qp_attr.cap.max_send_sge = attr->max_request_iov; + qp_attr.cap.max_recv_sge = attr->max_recv_iov; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + "modify_qp: qp %p sq %d,%d, rq %d,%d\n", + ep_ptr->qp_handle, + qp_attr.cap.max_send_wr, qp_attr.cap.max_send_sge, + qp_attr.cap.max_recv_wr, qp_attr.cap.max_recv_sge); + + if (ibv_modify_qp(ep_ptr->qp_handle, &qp_attr, IBV_QP_CAP)) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + "modify_qp: modify ep %p qp %p failed\n", + ep_ptr, ep_ptr->qp_handle); + return (dapl_convert_errno(errno, "modify_qp_state")); + } + + return DAT_SUCCESS; +} + +/* + * dapls_ib_reinit_ep + * + * Move the QP to INIT state again. + * + * Input: + * ep_ptr DAPL_EP + * + * Output: + * none + * + * Returns: + * void + * + */ +#if defined(_WIN32) || defined(_WIN64) +void dapls_ib_reinit_ep(IN DAPL_EP * ep_ptr) +{ + /* work around bug in low level driver - 3/24/09 */ + /* RTS -> RESET -> INIT -> ERROR QP transition crashes system */ + if (ep_ptr->qp_handle != IB_INVALID_HANDLE) { + dapls_ib_qp_free(ep_ptr->header.owner_ia, ep_ptr); + dapls_ib_qp_alloc(ep_ptr->header.owner_ia, ep_ptr, ep_ptr); + } +} +#else // _WIN32 || _WIN64 +void dapls_ib_reinit_ep(IN DAPL_EP * ep_ptr) +{ + if (ep_ptr->qp_handle != IB_INVALID_HANDLE && + ep_ptr->qp_handle->qp_type != IBV_QPT_UD) { + /* move to RESET state and then to INIT */ + dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_RESET, 0); + dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_INIT, 0); + } +} +#endif // _WIN32 || _WIN64 + +/* + * Generic QP modify for init, reset, error, RTS, RTR + * For UD, create_ah on RTR, qkey on INIT + */ +DAT_RETURN +dapls_modify_qp_state(IN ib_qp_handle_t qp_handle, + IN ib_qp_state_t qp_state, + IN dp_ib_cm_handle_t cm_ptr) +{ + struct ibv_qp_attr qp_attr; + enum ibv_qp_attr_mask mask = IBV_QP_STATE; + DAPL_EP *ep_ptr = (DAPL_EP *) qp_handle->qp_context; + DAPL_IA *ia_ptr = ep_ptr->header.owner_ia; + ib_qp_cm_t *qp_cm = &cm_ptr->dst; + int ret; + + dapl_os_memzero((void *)&qp_attr, sizeof(qp_attr)); + qp_attr.qp_state = qp_state; + switch (qp_state) { + /* additional attributes with RTR and RTS */ + case IBV_QPS_RTR: + { + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " QPS_RTR: type %d state %d qpn %x lid %x" + " port %x ep %p qp_state %d\n", + qp_handle->qp_type, qp_handle->qp_type, + qp_cm->qpn, qp_cm->lid, qp_cm->port, + ep_ptr, ep_ptr->qp_state); + + mask |= IBV_QP_AV | + IBV_QP_PATH_MTU | + IBV_QP_DEST_QPN | + IBV_QP_RQ_PSN | + IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER; + + qp_attr.dest_qp_num = qp_cm->qpn; + qp_attr.rq_psn = 1; + qp_attr.path_mtu = ia_ptr->hca_ptr->ib_trans.mtu; + qp_attr.max_dest_rd_atomic = + ep_ptr->param.ep_attr.max_rdma_read_out; + qp_attr.min_rnr_timer = + ia_ptr->hca_ptr->ib_trans.rnr_timer; + + /* address handle. RC and UD */ + qp_attr.ah_attr.dlid = qp_cm->lid; + if (ia_ptr->hca_ptr->ib_trans.global) { + qp_attr.ah_attr.is_global = 1; + qp_attr.ah_attr.grh.dgid = qp_cm->gid; + qp_attr.ah_attr.grh.hop_limit = + ia_ptr->hca_ptr->ib_trans.hop_limit; + qp_attr.ah_attr.grh.traffic_class = + ia_ptr->hca_ptr->ib_trans.tclass; + } + qp_attr.ah_attr.sl = 0; + qp_attr.ah_attr.src_path_bits = 0; + qp_attr.ah_attr.port_num = ia_ptr->hca_ptr->port_num; +#ifdef DAT_EXTENSIONS + /* UD: create AH for remote side */ + if (qp_handle->qp_type == IBV_QPT_UD) { + ib_pd_handle_t pz; + pz = ((DAPL_PZ *) + ep_ptr->param.pz_handle)->pd_handle; + mask = IBV_QP_STATE; + cm_ptr->ah = ibv_create_ah(pz, + &qp_attr.ah_attr); + if (!cm_ptr->ah) + return (dapl_convert_errno(errno, + "ibv_ah")); + + /* already RTR, multi remote AH's on QP */ + if (ep_ptr->qp_state == IBV_QPS_RTR || + ep_ptr->qp_state == IBV_QPS_RTS) + return DAT_SUCCESS; + } +#endif + break; + } + case IBV_QPS_RTS: + { + /* RC only */ + if (qp_handle->qp_type == IBV_QPT_RC) { + mask |= IBV_QP_SQ_PSN | + IBV_QP_TIMEOUT | + IBV_QP_RETRY_CNT | + IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC; + qp_attr.timeout = + ia_ptr->hca_ptr->ib_trans.ack_timer; + qp_attr.retry_cnt = + ia_ptr->hca_ptr->ib_trans.ack_retry; + qp_attr.rnr_retry = + ia_ptr->hca_ptr->ib_trans.rnr_retry; + qp_attr.max_rd_atomic = + ep_ptr->param.ep_attr.max_rdma_read_out; + } + /* RC and UD */ + qp_attr.qp_state = IBV_QPS_RTS; + qp_attr.sq_psn = 1; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " QPS_RTS: psn %x rd_atomic %d ack %d " + " retry %d rnr_retry %d ep %p qp_state %d\n", + qp_attr.sq_psn, qp_attr.max_rd_atomic, + qp_attr.timeout, qp_attr.retry_cnt, + qp_attr.rnr_retry, ep_ptr, + ep_ptr->qp_state); +#ifdef DAT_EXTENSIONS + if (qp_handle->qp_type == IBV_QPT_UD) { + /* already RTS, multi remote AH's on QP */ + if (ep_ptr->qp_state == IBV_QPS_RTS) + return DAT_SUCCESS; + else + mask = IBV_QP_STATE | IBV_QP_SQ_PSN; + } +#endif + break; + } + case IBV_QPS_INIT: + { + mask |= IBV_QP_PKEY_INDEX | IBV_QP_PORT; + if (qp_handle->qp_type == IBV_QPT_RC) { + mask |= IBV_QP_ACCESS_FLAGS; + qp_attr.qp_access_flags = + IBV_ACCESS_LOCAL_WRITE | + IBV_ACCESS_REMOTE_WRITE | + IBV_ACCESS_REMOTE_READ | + IBV_ACCESS_REMOTE_ATOMIC | + IBV_ACCESS_MW_BIND; + } +#ifdef DAT_EXTENSIONS + if (qp_handle->qp_type == IBV_QPT_UD) { + /* already INIT, multi remote AH's on QP */ + if (ep_ptr->qp_state == IBV_QPS_INIT) + return DAT_SUCCESS; + mask |= IBV_QP_QKEY; + qp_attr.qkey = DAT_UD_QKEY; + } +#endif + qp_attr.pkey_index = 0; + qp_attr.port_num = ia_ptr->hca_ptr->port_num; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " QPS_INIT: pi %x port %x acc %x qkey 0x%x\n", + qp_attr.pkey_index, qp_attr.port_num, + qp_attr.qp_access_flags, qp_attr.qkey); + break; + } + default: + break; + + } + + ret = ibv_modify_qp(qp_handle, &qp_attr, mask); + if (ret == 0) { + ep_ptr->qp_state = qp_state; + return DAT_SUCCESS; + } else { + return (dapl_convert_errno(errno, "modify_qp_state")); + } +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 8 + * End: + */ diff --git a/dapl/openib_common/util.c b/dapl/openib_common/util.c new file mode 100644 index 0000000..da913c5 --- /dev/null +++ b/dapl/openib_common/util.c @@ -0,0 +1,375 @@ +/* + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_ib_util.h" +#include "dapl_osd.h" + +#include + +int g_dapl_loopback_connection = 0; + +enum ibv_mtu dapl_ib_mtu(int mtu) +{ + switch (mtu) { + case 256: + return IBV_MTU_256; + case 512: + return IBV_MTU_512; + case 1024: + return IBV_MTU_1024; + case 2048: + return IBV_MTU_2048; + case 4096: + return IBV_MTU_4096; + default: + return IBV_MTU_1024; + } +} + +char *dapl_ib_mtu_str(enum ibv_mtu mtu) +{ + switch (mtu) { + case IBV_MTU_256: + return "256"; + case IBV_MTU_512: + return "512"; + case IBV_MTU_1024: + return "1024"; + case IBV_MTU_2048: + return "2048"; + case IBV_MTU_4096: + return "4096"; + default: + return "1024"; + } +} + +DAT_RETURN getlocalipaddr(DAT_SOCK_ADDR * addr, int addr_len) +{ + struct sockaddr_in *sin; + struct addrinfo *res, hint, *ai; + int ret; + char hostname[256]; + + if (addr_len < sizeof(*sin)) { + return DAT_INTERNAL_ERROR; + } + + ret = gethostname(hostname, 256); + if (ret) + return dapl_convert_errno(ret, "gethostname"); + + memset(&hint, 0, sizeof hint); + hint.ai_flags = AI_PASSIVE; + hint.ai_family = AF_INET; + hint.ai_socktype = SOCK_STREAM; + hint.ai_protocol = IPPROTO_TCP; + + ret = getaddrinfo(hostname, NULL, &hint, &res); + if (ret) { + dapl_log(DAPL_DBG_TYPE_ERR, + " getaddrinfo ERR: %d %s\n", ret, gai_strerror(ret)); + return DAT_INVALID_ADDRESS; + } + + ret = DAT_INVALID_ADDRESS; + for (ai = res; ai; ai = ai->ai_next) { + sin = (struct sockaddr_in *)ai->ai_addr; + if (*((uint32_t *) & sin->sin_addr) != htonl(0x7f000001)) { + *((struct sockaddr_in *)addr) = *sin; + ret = DAT_SUCCESS; + break; + } + } + + freeaddrinfo(res); + return ret; +} + +/* + * dapls_ib_query_hca + * + * Query the hca attribute + * + * Input: + * hca_handl hca handle + * ia_attr attribute of the ia + * ep_attr attribute of the ep + * ip_addr ip address of DET NIC + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + */ + +DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr, + OUT DAT_IA_ATTR * ia_attr, + OUT DAT_EP_ATTR * ep_attr, + OUT DAT_SOCK_ADDR6 * ip_addr) +{ + struct ibv_device_attr dev_attr; + struct ibv_port_attr port_attr; + + if (hca_ptr->ib_hca_handle == NULL) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, " query_hca: BAD handle\n"); + return (DAT_INVALID_HANDLE); + } + + /* local IP address of device, set during ia_open */ + if (ip_addr != NULL) + memcpy(ip_addr, &hca_ptr->hca_address, sizeof(DAT_SOCK_ADDR6)); + + if (ia_attr == NULL && ep_attr == NULL) + return DAT_SUCCESS; + + /* query verbs for this device and port attributes */ + if (ibv_query_device(hca_ptr->ib_hca_handle, &dev_attr) || + ibv_query_port(hca_ptr->ib_hca_handle, + hca_ptr->port_num, &port_attr)) + return (dapl_convert_errno(errno, "ib_query_hca")); + + if (ia_attr != NULL) { + (void)dapl_os_memzero(ia_attr, sizeof(*ia_attr)); + ia_attr->adapter_name[DAT_NAME_MAX_LENGTH - 1] = '\0'; + ia_attr->vendor_name[DAT_NAME_MAX_LENGTH - 1] = '\0'; + ia_attr->ia_address_ptr = + (DAT_IA_ADDRESS_PTR) & hca_ptr->hca_address; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " query_hca: %s %s \n", + ibv_get_device_name(hca_ptr->ib_trans.ib_dev), + inet_ntoa(((struct sockaddr_in *) + &hca_ptr->hca_address)->sin_addr)); + + ia_attr->hardware_version_major = dev_attr.hw_ver; + /* ia_attr->hardware_version_minor = dev_attr.fw_ver; */ + ia_attr->max_eps = dev_attr.max_qp; + ia_attr->max_dto_per_ep = dev_attr.max_qp_wr; + ia_attr->max_rdma_read_in = dev_attr.max_qp_rd_atom; + ia_attr->max_rdma_read_out = dev_attr.max_qp_init_rd_atom; + ia_attr->max_rdma_read_per_ep_in = dev_attr.max_qp_rd_atom; + ia_attr->max_rdma_read_per_ep_out = + dev_attr.max_qp_init_rd_atom; + ia_attr->max_rdma_read_per_ep_in_guaranteed = DAT_TRUE; + ia_attr->max_rdma_read_per_ep_out_guaranteed = DAT_TRUE; + ia_attr->max_evds = dev_attr.max_cq; + ia_attr->max_evd_qlen = dev_attr.max_cqe; + ia_attr->max_iov_segments_per_dto = dev_attr.max_sge; + ia_attr->max_lmrs = dev_attr.max_mr; + /* 32bit attribute from 64bit, 4G-1 limit, DAT v2 needs fix */ + ia_attr->max_lmr_block_size = + (dev_attr.max_mr_size >> 32) ? ~0 : dev_attr.max_mr_size; + ia_attr->max_rmrs = dev_attr.max_mw; + ia_attr->max_lmr_virtual_address = dev_attr.max_mr_size; + ia_attr->max_rmr_target_address = dev_attr.max_mr_size; + ia_attr->max_pzs = dev_attr.max_pd; + ia_attr->max_message_size = port_attr.max_msg_sz; + ia_attr->max_rdma_size = port_attr.max_msg_sz; + /* iWARP spec. - 1 sge for RDMA reads */ + if (hca_ptr->ib_hca_handle->device->transport_type + == IBV_TRANSPORT_IWARP) + ia_attr->max_iov_segments_per_rdma_read = 1; + else + ia_attr->max_iov_segments_per_rdma_read = + dev_attr.max_sge; + ia_attr->max_iov_segments_per_rdma_write = dev_attr.max_sge; + ia_attr->num_transport_attr = 0; + ia_attr->transport_attr = NULL; + ia_attr->num_vendor_attr = 0; + ia_attr->vendor_attr = NULL; +#ifdef DAT_EXTENSIONS + ia_attr->extension_supported = DAT_EXTENSION_IB; + ia_attr->extension_version = DAT_IB_EXTENSION_VERSION; +#endif + /* save key device attributes for CM exchange */ + hca_ptr->ib_trans.rd_atom_in = dev_attr.max_qp_rd_atom; + hca_ptr->ib_trans.rd_atom_out = dev_attr.max_qp_init_rd_atom; + + hca_ptr->ib_trans.mtu = DAPL_MIN(port_attr.active_mtu, + hca_ptr->ib_trans.mtu); + hca_ptr->ib_trans.ack_timer = + DAPL_MAX(dev_attr.local_ca_ack_delay, + hca_ptr->ib_trans.ack_timer); + + /* set MTU in transport specific named attribute */ + hca_ptr->ib_trans.named_attr.name = "DAT_IB_TRANSPORT_MTU"; + hca_ptr->ib_trans.named_attr.value = + dapl_ib_mtu_str(hca_ptr->ib_trans.mtu); + + dapl_log(DAPL_DBG_TYPE_UTIL, + " query_hca: (%x.%x) eps %d, sz %d evds %d," + " sz %d mtu %d\n", + ia_attr->hardware_version_major, + ia_attr->hardware_version_minor, + ia_attr->max_eps, ia_attr->max_dto_per_ep, + ia_attr->max_evds, ia_attr->max_evd_qlen, + 128 << hca_ptr->ib_trans.mtu); + + dapl_log(DAPL_DBG_TYPE_UTIL, + " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d" + " ack_time %d mr %u\n", + ia_attr->max_message_size, ia_attr->max_rdma_size, + ia_attr->max_iov_segments_per_dto, + ia_attr->max_lmrs, ia_attr->max_rmrs, + hca_ptr->ib_trans.ack_timer, + ia_attr->max_lmr_block_size); + } + + if (ep_attr != NULL) { + (void)dapl_os_memzero(ep_attr, sizeof(*ep_attr)); + ep_attr->max_message_size = port_attr.max_msg_sz; + ep_attr->max_rdma_size = port_attr.max_msg_sz; + ep_attr->max_recv_dtos = dev_attr.max_qp_wr; + ep_attr->max_request_dtos = dev_attr.max_qp_wr; + ep_attr->max_recv_iov = dev_attr.max_sge; + ep_attr->max_request_iov = dev_attr.max_sge; + ep_attr->max_rdma_read_in = dev_attr.max_qp_rd_atom; + ep_attr->max_rdma_read_out = dev_attr.max_qp_init_rd_atom; + ep_attr->max_rdma_read_iov = dev_attr.max_sge; + ep_attr->max_rdma_write_iov = dev_attr.max_sge; + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " query_hca: MAX msg %llu mtu %d qsz %d iov %d" + " rdma i%d,o%d\n", + ep_attr->max_message_size, + 128 << hca_ptr->ib_trans.mtu, + ep_attr->max_recv_dtos, + ep_attr->max_recv_iov, + ep_attr->max_rdma_read_in, + ep_attr->max_rdma_read_out); + } + return DAT_SUCCESS; +} + +/* + * dapls_ib_setup_async_callback + * + * Set up an asynchronous callbacks of various kinds + * + * Input: + * ia_handle IA handle + * handler_type type of handler to set up + * callback_handle handle param for completion callbacks + * callback callback routine pointer + * context argument for callback routine + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN dapls_ib_setup_async_callback(IN DAPL_IA * ia_ptr, + IN DAPL_ASYNC_HANDLER_TYPE + handler_type, IN DAPL_EVD * evd_ptr, + IN ib_async_handler_t callback, + IN void *context) +{ + ib_hca_transport_t *hca_ptr; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " setup_async_cb: ia %p type %d handle %p cb %p ctx %p\n", + ia_ptr, handler_type, evd_ptr, callback, context); + + hca_ptr = &ia_ptr->hca_ptr->ib_trans; + switch (handler_type) { + case DAPL_ASYNC_UNAFILIATED: + hca_ptr->async_unafiliated = (ib_async_handler_t) callback; + hca_ptr->async_un_ctx = context; + break; + case DAPL_ASYNC_CQ_ERROR: + hca_ptr->async_cq_error = (ib_async_cq_handler_t) callback; + break; + case DAPL_ASYNC_CQ_COMPLETION: + hca_ptr->async_cq = (ib_async_dto_handler_t) callback; + break; + case DAPL_ASYNC_QP_ERROR: + hca_ptr->async_qp_error = (ib_async_qp_handler_t) callback; + break; + default: + break; + } + return DAT_SUCCESS; +} + +/* + * dapls_set_provider_specific_attr + * + * Input: + * attr_ptr Pointer provider specific attributes + * + * Output: + * none + * + * Returns: + * void + */ +DAT_NAMED_ATTR ib_attrs[] = { + { + "DAT_IB_TRANSPORT_MTU", "2048"} + , +#ifdef DAT_EXTENSIONS + { + "DAT_EXTENSION_INTERFACE", "TRUE"} + , + { + DAT_IB_ATTR_FETCH_AND_ADD, "TRUE"} + , + { + DAT_IB_ATTR_CMP_AND_SWAP, "TRUE"} + , + { + DAT_IB_ATTR_IMMED_DATA, "TRUE"} + , +#ifndef _OPENIB_CMA_ + { + DAT_IB_ATTR_UD, "TRUE"} + , +#endif +#ifdef DAPL_COUNTERS + { + DAT_ATTR_COUNTERS, "TRUE"} + , +#endif /* DAPL_COUNTERS */ +#endif +}; + +#define SPEC_ATTR_SIZE( x ) (sizeof( x ) / sizeof( DAT_NAMED_ATTR)) + +void dapls_query_provider_specific_attr(IN DAPL_IA * ia_ptr, + IN DAT_PROVIDER_ATTR * attr_ptr) +{ + attr_ptr->num_provider_specific_attr = SPEC_ATTR_SIZE(ib_attrs); + attr_ptr->provider_specific_attr = ib_attrs; + + /* set MTU to actual settings */ + ib_attrs[0].value = ia_ptr->hca_ptr->ib_trans.named_attr.value; +} diff --git a/dapl/openib_scm/SOURCES b/dapl/openib_scm/SOURCES index f9204d9..5714aa3 100644 --- a/dapl/openib_scm/SOURCES +++ b/dapl/openib_scm/SOURCES @@ -18,16 +18,17 @@ USE_MSVCRT = 1 SOURCES = \ udapl.rc \ - ..\dapl_common_src.c \ - ..\dapl_udapl_src.c \ - dapl_ib_cq.c \ - dapl_ib_extensions.c \ - dapl_ib_mem.c \ - dapl_ib_qp.c \ - dapl_ib_util.c \ - dapl_ib_cm.c - -INCLUDES = ..\include;..\common;windows;..\..\dat\include;\ + ..\dapl_common_src.c \ + ..\dapl_udapl_src.c \ + ..\openib_common\mem.c \ + ..\openib_common\util.c \ + ..\openib_common\cq.c \ + ..\openib_common\qp.c \ + ..\openib_common\ib_extensions.c \ + device.c \ + cm.c + +INCLUDES = ..\include;..\openib_common\;..\common;windows;..\..\dat\include;\ ..\..\dat\udat\windows;..\udapl\windows;\ ..\..\..\..\inc;..\..\..\..\inc\user;..\..\..\libibverbs\include diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c new file mode 100644 index 0000000..5708214 --- /dev/null +++ b/dapl/openib_scm/cm.c @@ -0,0 +1,1839 @@ +/* + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/*************************************************************************** + * + * Module: uDAPL + * + * Filename: dapl_ib_cm.c + * + * Author: Arlin Davis + * + * Created: 3/10/2005 + * + * Description: + * + * The uDAPL openib provider - connection management + * + **************************************************************************** + * Source Control System Information + * + * $Id: $ + * + * Copyright (c) 2005 Intel Corporation. All rights reserved. + * + **************************************************************************/ + +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_evd_util.h" +#include "dapl_cr_util.h" +#include "dapl_name_service.h" +#include "dapl_ib_util.h" +#include "dapl_osd.h" + +#if defined(_WIN32) || defined(_WIN64) +enum DAPL_FD_EVENTS { + DAPL_FD_READ = 0x1, + DAPL_FD_WRITE = 0x2, + DAPL_FD_ERROR = 0x4 +}; + +static int dapl_config_socket(DAPL_SOCKET s) +{ + unsigned long nonblocking = 1; + return ioctlsocket(s, FIONBIO, &nonblocking); +} + +static int dapl_connect_socket(DAPL_SOCKET s, struct sockaddr *addr, + int addrlen) +{ + int err; + + err = connect(s, addr, addrlen); + if (err == SOCKET_ERROR) + err = WSAGetLastError(); + return (err == WSAEWOULDBLOCK) ? EAGAIN : err; +} + +struct dapl_fd_set { + struct fd_set set[3]; +}; + +static struct dapl_fd_set *dapl_alloc_fd_set(void) +{ + return dapl_os_alloc(sizeof(struct dapl_fd_set)); +} + +static void dapl_fd_zero(struct dapl_fd_set *set) +{ + FD_ZERO(&set->set[0]); + FD_ZERO(&set->set[1]); + FD_ZERO(&set->set[2]); +} + +static int dapl_fd_set(DAPL_SOCKET s, struct dapl_fd_set *set, + enum DAPL_FD_EVENTS event) +{ + FD_SET(s, &set->set[(event == DAPL_FD_READ) ? 0 : 1]); + FD_SET(s, &set->set[2]); + return 0; +} + +static enum DAPL_FD_EVENTS dapl_poll(DAPL_SOCKET s, enum DAPL_FD_EVENTS event) +{ + struct fd_set rw_fds; + struct fd_set err_fds; + struct timeval tv; + int ret; + + FD_ZERO(&rw_fds); + FD_ZERO(&err_fds); + FD_SET(s, &rw_fds); + FD_SET(s, &err_fds); + + tv.tv_sec = 0; + tv.tv_usec = 0; + + if (event == DAPL_FD_READ) + ret = select(1, &rw_fds, NULL, &err_fds, &tv); + else + ret = select(1, NULL, &rw_fds, &err_fds, &tv); + + if (ret == 0) + return 0; + else if (ret == SOCKET_ERROR) + return WSAGetLastError(); + else if (FD_ISSET(s, &rw_fds)) + return event; + else + return DAPL_FD_ERROR; +} + +static int dapl_select(struct dapl_fd_set *set) +{ + int ret; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, " dapl_select: sleep\n"); + ret = select(0, &set->set[0], &set->set[1], &set->set[2], NULL); + dapl_dbg_log(DAPL_DBG_TYPE_CM, " dapl_select: wakeup\n"); + + if (ret == SOCKET_ERROR) + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " dapl_select: error 0x%x\n", WSAGetLastError()); + + return ret; +} +#else // _WIN32 || _WIN64 +enum DAPL_FD_EVENTS { + DAPL_FD_READ = POLLIN, + DAPL_FD_WRITE = POLLOUT, + DAPL_FD_ERROR = POLLERR +}; + +static int dapl_config_socket(DAPL_SOCKET s) +{ + int ret; + + ret = fcntl(s, F_GETFL); + if (ret >= 0) + ret = fcntl(s, F_SETFL, ret | O_NONBLOCK); + return ret; +} + +static int dapl_connect_socket(DAPL_SOCKET s, struct sockaddr *addr, + int addrlen) +{ + int ret; + + ret = connect(s, addr, addrlen); + + return (errno == EINPROGRESS) ? EAGAIN : ret; +} + +struct dapl_fd_set { + int index; + struct pollfd set[DAPL_FD_SETSIZE]; +}; + +static struct dapl_fd_set *dapl_alloc_fd_set(void) +{ + return dapl_os_alloc(sizeof(struct dapl_fd_set)); +} + +static void dapl_fd_zero(struct dapl_fd_set *set) +{ + set->index = 0; +} + +static int dapl_fd_set(DAPL_SOCKET s, struct dapl_fd_set *set, + enum DAPL_FD_EVENTS event) +{ + if (set->index == DAPL_FD_SETSIZE - 1) { + dapl_log(DAPL_DBG_TYPE_ERR, + "SCM ERR: cm_thread exceeded FD_SETSIZE %d\n", + set->index + 1); + return -1; + } + + set->set[set->index].fd = s; + set->set[set->index].revents = 0; + set->set[set->index++].events = event; + return 0; +} + +static enum DAPL_FD_EVENTS dapl_poll(DAPL_SOCKET s, enum DAPL_FD_EVENTS event) +{ + struct pollfd fds; + int ret; + + fds.fd = s; + fds.events = event; + fds.revents = 0; + ret = poll(&fds, 1, 0); + dapl_log(DAPL_DBG_TYPE_CM, " dapl_poll: fd=%d ret=%d, evnts=0x%x\n", + s, ret, fds.revents); + if (ret == 0) + return 0; + else if (fds.revents & (POLLERR | POLLHUP | POLLNVAL)) + return DAPL_FD_ERROR; + else + return fds.revents; +} + +static int dapl_select(struct dapl_fd_set *set) +{ + int ret; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, " dapl_select: sleep, fds=%d\n", + set->index); + ret = poll(set->set, set->index, -1); + dapl_dbg_log(DAPL_DBG_TYPE_CM, " dapl_select: wakeup, ret=0x%x\n", ret); + return ret; +} +#endif + +dp_ib_cm_handle_t dapls_ib_cm_create(DAPL_EP *ep) +{ + dp_ib_cm_handle_t cm_ptr; + + /* Allocate CM, init lock, and initialize */ + if ((cm_ptr = dapl_os_alloc(sizeof(*cm_ptr))) == NULL) + return NULL; + + (void)dapl_os_memzero(cm_ptr, sizeof(*cm_ptr)); + if (dapl_os_lock_init(&cm_ptr->lock)) + goto bail; + + cm_ptr->dst.ver = htons(DCM_VER); + cm_ptr->socket = DAPL_INVALID_SOCKET; + cm_ptr->ep = ep; + return cm_ptr; +bail: + dapl_os_free(cm_ptr, sizeof(*cm_ptr)); + return NULL; +} + +/* mark for destroy, remove all references, schedule cleanup */ +/* cm_ptr == NULL (UD), then multi CR's, kill all associated with EP */ +void dapls_ib_cm_free(dp_ib_cm_handle_t cm_ptr, DAPL_EP *ep) +{ + DAPL_IA *ia_ptr; + DAPL_HCA *hca_ptr = NULL; + dp_ib_cm_handle_t cr, next_cr; + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " cm_destroy: cm %p ep %p\n", cm_ptr, ep); + + if (cm_ptr == NULL) + goto multi_cleanup; + + /* to notify cleanup thread */ + hca_ptr = cm_ptr->hca; + + /* cleanup, never made it to work queue */ + if (cm_ptr->state == DCM_INIT) { + if (cm_ptr->socket != DAPL_INVALID_SOCKET) { + shutdown(cm_ptr->socket, SHUT_RDWR); + closesocket(cm_ptr->socket); + } + dapl_os_free(cm_ptr, sizeof(*cm_ptr)); + return; + } + + dapl_os_lock(&cm_ptr->lock); + cm_ptr->state = DCM_DESTROY; + if ((cm_ptr->ep) && (cm_ptr->ep->cm_handle == cm_ptr)) { + cm_ptr->ep->cm_handle = IB_INVALID_HANDLE; + cm_ptr->ep = NULL; + } + + /* close socket if still active */ + if (cm_ptr->socket != DAPL_INVALID_SOCKET) { + shutdown(cm_ptr->socket, SHUT_RDWR); + closesocket(cm_ptr->socket); + cm_ptr->socket = DAPL_INVALID_SOCKET; + } + dapl_os_unlock(&cm_ptr->lock); + goto notify_thread; + +multi_cleanup: + + /* + * UD CR objects are kept active because of direct private data references + * from CONN events. The cr->socket is closed and marked inactive but the + * object remains allocated and queued on the CR resource list. There can + * be multiple CR's associated with a given EP. There is no way to determine + * when consumer is finished with event until the dat_ep_free. + * + * Schedule destruction for all CR's associated with this EP, cr_thread will + * complete the cleanup with state == DCM_DESTROY. + */ + ia_ptr = ep->header.owner_ia; + dapl_os_lock(&ia_ptr->hca_ptr->ib_trans.lock); + if (!dapl_llist_is_empty((DAPL_LLIST_HEAD*) + &ia_ptr->hca_ptr->ib_trans.list)) + next_cr = dapl_llist_peek_head((DAPL_LLIST_HEAD*) + &ia_ptr->hca_ptr->ib_trans.list); + else + next_cr = NULL; + + while (next_cr) { + cr = next_cr; + next_cr = dapl_llist_next_entry((DAPL_LLIST_HEAD*) + &ia_ptr->hca_ptr->ib_trans.list, + (DAPL_LLIST_ENTRY*)&cr->entry); + if (cr->ep == ep) { + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " qp_free CR: ep %p cr %p\n", ep, cr); + dapli_socket_disconnect(cr); + dapl_os_lock(&cr->lock); + hca_ptr = cr->hca; + cr->ep = NULL; + cr->state = DCM_DESTROY; + dapl_os_unlock(&cr->lock); + } + } + dapl_os_unlock(&ia_ptr->hca_ptr->ib_trans.lock); + +notify_thread: + + /* wakeup work thread, if something destroyed */ + if (hca_ptr != NULL) { + if (send(hca_ptr->ib_trans.scm[1], + "w", sizeof "w", 0) == -1) + dapl_log(DAPL_DBG_TYPE_CM, + " cm_destroy: thread wakeup error = %s\n", + strerror(errno)); + } +} + +/* queue socket for processing CM work */ +static void dapli_cm_queue(struct ib_cm_handle *cm_ptr) +{ + /* add to work queue for cr thread processing */ + dapl_llist_init_entry((DAPL_LLIST_ENTRY *) & cm_ptr->entry); + dapl_os_lock(&cm_ptr->hca->ib_trans.lock); + dapl_llist_add_tail(&cm_ptr->hca->ib_trans.list, + (DAPL_LLIST_ENTRY *) & cm_ptr->entry, cm_ptr); + dapl_os_unlock(&cm_ptr->hca->ib_trans.lock); + + /* wakeup CM work thread */ + if (send(cm_ptr->hca->ib_trans.scm[1], "w", sizeof "w", 0) == -1) + dapl_log(DAPL_DBG_TYPE_CM, + " cm_queue: thread wakeup error = %s\n", + strerror(errno)); +} + +/* + * ACTIVE/PASSIVE: called from CR thread or consumer via ep_disconnect + * or from ep_free + */ +DAT_RETURN dapli_socket_disconnect(dp_ib_cm_handle_t cm_ptr) +{ + DAPL_EP *ep_ptr = cm_ptr->ep; + DAT_UINT32 disc_data = htonl(0xdead); + + if (ep_ptr == NULL) + return DAT_SUCCESS; + + dapl_os_lock(&cm_ptr->lock); + if ((cm_ptr->state == DCM_INIT) || + (cm_ptr->state == DCM_DISCONNECTED) || + (cm_ptr->state == DCM_DESTROY)) { + dapl_os_unlock(&cm_ptr->lock); + return DAT_SUCCESS; + } else { + /* send disc date, close socket, schedule destroy */ + if (cm_ptr->socket != DAPL_INVALID_SOCKET) { + if (send(cm_ptr->socket, (char *)&disc_data, + sizeof(disc_data), 0) == -1) + dapl_log(DAPL_DBG_TYPE_WARN, + " cm_disc: write error = %s\n", + strerror(errno)); + shutdown(cm_ptr->socket, SHUT_RDWR); + closesocket(cm_ptr->socket); + cm_ptr->socket = DAPL_INVALID_SOCKET; + } + cm_ptr->state = DCM_DISCONNECTED; + } + dapl_os_unlock(&cm_ptr->lock); + + /* disconnect events for RC's only */ + if (ep_ptr->param.ep_attr.service_type == DAT_SERVICE_TYPE_RC) { + if (ep_ptr->cr_ptr) { + dapls_cr_callback(cm_ptr, + IB_CME_DISCONNECTED, + NULL, + ((DAPL_CR *) ep_ptr->cr_ptr)->sp_ptr); + } else { + dapl_evd_connection_callback(ep_ptr->cm_handle, + IB_CME_DISCONNECTED, + NULL, ep_ptr); + } + } + + /* scheduled destroy via disconnect clean in callback */ + return DAT_SUCCESS; +} + +/* + * ACTIVE: socket connected, send QP information to peer + */ +static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err) +{ + int len, opt = 1; + struct iovec iov[2]; + struct dapl_ep *ep_ptr = cm_ptr->ep; + + if (err) { + dapl_log(DAPL_DBG_TYPE_ERR, + " CONN_PENDING: %s ERR %s -> %s %d\n", + err == -1 ? "POLL" : "SOCKOPT", + err == -1 ? strerror(errno) : strerror(err), + inet_ntoa(((struct sockaddr_in *) + ep_ptr->param. + remote_ia_address_ptr)->sin_addr), + ntohs(((struct sockaddr_in *) + &cm_ptr->dst.ia_address)->sin_port)); + goto bail; + } + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " socket connected, write QP and private data\n"); + + /* no delay for small packets */ + setsockopt(cm_ptr->socket, IPPROTO_TCP, TCP_NODELAY, + (char *)&opt, sizeof(opt)); + + /* send qp info and pdata to remote peer */ + iov[0].iov_base = (void *)&cm_ptr->dst; + iov[0].iov_len = sizeof(ib_qp_cm_t); + if (cm_ptr->dst.p_size) { + iov[1].iov_base = cm_ptr->p_data; + iov[1].iov_len = ntohl(cm_ptr->dst.p_size); + len = writev(cm_ptr->socket, iov, 2); + } else { + len = writev(cm_ptr->socket, iov, 1); + } + + if (len != (ntohl(cm_ptr->dst.p_size) + sizeof(ib_qp_cm_t))) { + dapl_log(DAPL_DBG_TYPE_ERR, + " CONN_PENDING write: ERR %s, wcnt=%d -> %s\n", + strerror(errno), len, inet_ntoa(((struct sockaddr_in *) + ep_ptr->param. + remote_ia_address_ptr)-> + sin_addr)); + goto bail; + } + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " connected: sending SRC port=0x%x lid=0x%x," + " qpn=0x%x, psize=%d\n", + ntohs(cm_ptr->dst.port), ntohs(cm_ptr->dst.lid), + ntohl(cm_ptr->dst.qpn), ntohl(cm_ptr->dst.p_size)); + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " connected: sending SRC GID subnet %016llx id %016llx\n", + (unsigned long long) + htonll(cm_ptr->dst.gid.global.subnet_prefix), + (unsigned long long) + htonll(cm_ptr->dst.gid.global.interface_id)); + + /* queue up to work thread to avoid blocking consumer */ + cm_ptr->state = DCM_RTU_PENDING; + return; + bail: + /* close socket, free cm structure and post error event */ + dapls_ib_cm_free(cm_ptr, cm_ptr->ep); + dapl_evd_connection_callback(NULL, IB_CME_LOCAL_FAILURE, NULL, ep_ptr); +} + +/* + * ACTIVE: Create socket, connect, defer exchange QP information to CR thread + * to avoid blocking. + */ +DAT_RETURN +dapli_socket_connect(DAPL_EP * ep_ptr, + DAT_IA_ADDRESS_PTR r_addr, + DAT_CONN_QUAL r_qual, DAT_COUNT p_size, DAT_PVOID p_data) +{ + dp_ib_cm_handle_t cm_ptr; + int ret; + DAPL_IA *ia_ptr = ep_ptr->header.owner_ia; + struct sockaddr_in addr; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect: r_qual %d p_size=%d\n", + r_qual, p_size); + + cm_ptr = dapls_ib_cm_create(ep_ptr); + if (cm_ptr == NULL) + return DAT_INSUFFICIENT_RESOURCES; + + /* create, connect, sockopt, and exchange QP information */ + if ((cm_ptr->socket = + socket(AF_INET, SOCK_STREAM, 0)) == DAPL_INVALID_SOCKET) { + dapl_os_free(cm_ptr, sizeof(*cm_ptr)); + return DAT_INSUFFICIENT_RESOURCES; + } + + ret = dapl_config_socket(cm_ptr->socket); + if (ret < 0) { + dapl_log(DAPL_DBG_TYPE_ERR, + " socket connect: config socket %d ERR %d %s\n", + cm_ptr->socket, ret, strerror(errno)); + goto bail; + } + + dapl_os_memcpy(&addr, r_addr, sizeof(addr)); + addr.sin_port = htons(r_qual); + ret = dapl_connect_socket(cm_ptr->socket, (struct sockaddr *)&addr, + sizeof(addr)); + if (ret && ret != EAGAIN) { + dapl_log(DAPL_DBG_TYPE_ERR, + " socket connect ERROR: %s -> %s r_qual %d\n", + strerror(errno), + inet_ntoa(addr.sin_addr), (unsigned int)r_qual); + dapls_ib_cm_free(cm_ptr, cm_ptr->ep); + return DAT_INVALID_ADDRESS; + } + + /* Send QP info, IA address, and private data */ + cm_ptr->dst.qpn = htonl(ep_ptr->qp_handle->qp_num); +#ifdef DAT_EXTENSIONS + cm_ptr->dst.qp_type = htons(ep_ptr->qp_handle->qp_type); +#endif + cm_ptr->dst.port = htons(ia_ptr->hca_ptr->port_num); + cm_ptr->dst.lid = ia_ptr->hca_ptr->ib_trans.lid; + cm_ptr->dst.gid = ia_ptr->hca_ptr->ib_trans.gid; + + /* save references */ + cm_ptr->hca = ia_ptr->hca_ptr; + cm_ptr->ep = ep_ptr; + cm_ptr->dst.ia_address = ia_ptr->hca_ptr->hca_address; + ((struct sockaddr_in *) + &cm_ptr->dst.ia_address)->sin_port = ntohs(r_qual); + + if (p_size) { + cm_ptr->dst.p_size = htonl(p_size); + dapl_os_memcpy(cm_ptr->p_data, p_data, p_size); + } + + /* connected or pending, either way results via async event */ + if (ret == 0) + dapli_socket_connected(cm_ptr, 0); + else + cm_ptr->state = DCM_CONN_PENDING; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " connect: socket %d to %s r_qual %d pending\n", + cm_ptr->socket, + inet_ntoa(addr.sin_addr), (unsigned int)r_qual); + + dapli_cm_queue(cm_ptr); + return DAT_SUCCESS; + bail: + dapl_log(DAPL_DBG_TYPE_ERR, + " socket connect ERROR: %s query lid(0x%x)/gid" + " -> %s r_qual %d\n", + strerror(errno), ntohs(cm_ptr->dst.lid), + inet_ntoa(((struct sockaddr_in *)r_addr)->sin_addr), + (unsigned int)r_qual); + + /* close socket, free cm structure */ + dapls_ib_cm_free(cm_ptr, cm_ptr->ep); + return DAT_INTERNAL_ERROR; +} + +/* + * ACTIVE: exchange QP information, called from CR thread + */ +static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr) +{ + DAPL_EP *ep_ptr = cm_ptr->ep; + int len; + short rtu_data = htons(0x0E0F); + ib_cm_events_t event = IB_CME_DESTINATION_REJECT; + + /* read DST information into cm_ptr, overwrite SRC info */ + dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect_rtu: recv peer QP data\n"); + + len = recv(cm_ptr->socket, (char *)&cm_ptr->dst, sizeof(ib_qp_cm_t), 0); + if (len != sizeof(ib_qp_cm_t) || ntohs(cm_ptr->dst.ver) != DCM_VER) { + dapl_log(DAPL_DBG_TYPE_ERR, + " CONN_RTU read: ERR %s, rcnt=%d, ver=%d -> %s\n", + strerror(errno), len, cm_ptr->dst.ver, + inet_ntoa(((struct sockaddr_in *) + ep_ptr->param.remote_ia_address_ptr)-> + sin_addr)); + goto bail; + } + + /* convert peer response values to host order */ + cm_ptr->dst.port = ntohs(cm_ptr->dst.port); + cm_ptr->dst.lid = ntohs(cm_ptr->dst.lid); + cm_ptr->dst.qpn = ntohl(cm_ptr->dst.qpn); +#ifdef DAT_EXTENSIONS + cm_ptr->dst.qp_type = ntohs(cm_ptr->dst.qp_type); +#endif + cm_ptr->dst.p_size = ntohl(cm_ptr->dst.p_size); + + /* save remote address information */ + dapl_os_memcpy(&ep_ptr->remote_ia_address, + &cm_ptr->dst.ia_address, + sizeof(ep_ptr->remote_ia_address)); + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " CONN_RTU: DST %s port=0x%x lid=0x%x," + " qpn=0x%x, qp_type=%d, psize=%d\n", + inet_ntoa(((struct sockaddr_in *) + &cm_ptr->dst.ia_address)->sin_addr), + cm_ptr->dst.port, cm_ptr->dst.lid, + cm_ptr->dst.qpn, cm_ptr->dst.qp_type, cm_ptr->dst.p_size); + + /* validate private data size before reading */ + if (cm_ptr->dst.p_size > IB_MAX_REP_PDATA_SIZE) { + dapl_log(DAPL_DBG_TYPE_ERR, + " CONN_RTU read: psize (%d) wrong -> %s\n", + cm_ptr->dst.p_size, inet_ntoa(((struct sockaddr_in *) + ep_ptr->param. + remote_ia_address_ptr)-> + sin_addr)); + goto bail; + } + + /* read private data into cm_handle if any present */ + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " socket connected, read private data\n"); + if (cm_ptr->dst.p_size) { + len = + recv(cm_ptr->socket, cm_ptr->p_data, cm_ptr->dst.p_size, 0); + if (len != cm_ptr->dst.p_size) { + dapl_log(DAPL_DBG_TYPE_ERR, + " CONN_RTU read pdata: ERR %s, rcnt=%d -> %s\n", + strerror(errno), len, + inet_ntoa(((struct sockaddr_in *) + ep_ptr->param. + remote_ia_address_ptr)->sin_addr)); + goto bail; + } + } + + /* check for consumer reject */ + if (cm_ptr->dst.rej) { + dapl_log(DAPL_DBG_TYPE_CM, + " CONN_RTU read: PEER REJ reason=0x%x -> %s\n", + ntohs(cm_ptr->dst.rej), + inet_ntoa(((struct sockaddr_in *) + ep_ptr->param.remote_ia_address_ptr)-> + sin_addr)); + event = IB_CME_DESTINATION_REJECT_PRIVATE_DATA; +#ifdef DAT_EXTENSIONS + if (cm_ptr->dst.qp_type == IBV_QPT_UD) + goto ud_bail; + else +#endif + goto bail; + } + + /* modify QP to RTR and then to RTS with remote info */ + dapl_os_lock(&ep_ptr->header.lock); + if (dapls_modify_qp_state(ep_ptr->qp_handle, + IBV_QPS_RTR, cm_ptr) != DAT_SUCCESS) { + dapl_log(DAPL_DBG_TYPE_ERR, + " CONN_RTU: QPS_RTR ERR %s -> %s\n", + strerror(errno), inet_ntoa(((struct sockaddr_in *) + ep_ptr->param. + remote_ia_address_ptr)-> + sin_addr)); + dapl_os_unlock(&ep_ptr->header.lock); + goto bail; + } + if (dapls_modify_qp_state(ep_ptr->qp_handle, + IBV_QPS_RTS, cm_ptr) != DAT_SUCCESS) { + dapl_log(DAPL_DBG_TYPE_ERR, + " CONN_RTU: QPS_RTS ERR %s -> %s\n", + strerror(errno), inet_ntoa(((struct sockaddr_in *) + ep_ptr->param. + remote_ia_address_ptr)-> + sin_addr)); + dapl_os_unlock(&ep_ptr->header.lock); + goto bail; + } + dapl_os_unlock(&ep_ptr->header.lock); + dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect_rtu: send RTU\n"); + + /* complete handshake after final QP state change */ + if (send(cm_ptr->socket, (char *)&rtu_data, sizeof(rtu_data), 0) == -1) { + dapl_log(DAPL_DBG_TYPE_ERR, + " CONN_RTU: write error = %s\n", strerror(errno)); + goto bail; + } + /* init cm_handle and post the event with private data */ + cm_ptr->state = DCM_CONNECTED; + event = IB_CME_CONNECTED; + dapl_dbg_log(DAPL_DBG_TYPE_EP, " ACTIVE: connected!\n"); + +#ifdef DAT_EXTENSIONS +ud_bail: + if (cm_ptr->dst.qp_type == IBV_QPT_UD) { + DAT_IB_EXTENSION_EVENT_DATA xevent; + + /* post EVENT, modify_qp created ah */ + xevent.status = 0; + xevent.type = DAT_IB_UD_REMOTE_AH; + xevent.remote_ah.ah = cm_ptr->ah; + xevent.remote_ah.qpn = cm_ptr->dst.qpn; + dapl_os_memcpy(&xevent.remote_ah.ia_addr, + &cm_ptr->dst.ia_address, + sizeof(cm_ptr->dst.ia_address)); + + if (event == IB_CME_CONNECTED) + event = DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED; + else + event = DAT_IB_UD_CONNECTION_REJECT_EVENT; + + dapls_evd_post_connection_event_ext((DAPL_EVD *) ep_ptr->param. + connect_evd_handle, + event, + (DAT_EP_HANDLE) ep_ptr, + (DAT_COUNT) cm_ptr->dst.p_size, + (DAT_PVOID *) cm_ptr->p_data, + (DAT_PVOID *) &xevent); + + /* done with socket, don't destroy cm_ptr, need pdata */ + closesocket(cm_ptr->socket); + cm_ptr->socket = DAPL_INVALID_SOCKET; + cm_ptr->state = DCM_RELEASED; + } else +#endif + { + ep_ptr->cm_handle = cm_ptr; /* only RC, multi CR's on UD */ + dapl_evd_connection_callback(cm_ptr, + IB_CME_CONNECTED, + cm_ptr->p_data, ep_ptr); + } + return; + +bail: + /* close socket, and post error event */ + dapls_ib_reinit_ep(ep_ptr); /* reset QP state */ + closesocket(cm_ptr->socket); + cm_ptr->socket = DAPL_INVALID_SOCKET; + dapl_evd_connection_callback(NULL, event, cm_ptr->p_data, ep_ptr); +} + +/* + * PASSIVE: Create socket, listen, accept, exchange QP information + */ +DAT_RETURN +dapli_socket_listen(DAPL_IA * ia_ptr, DAT_CONN_QUAL serviceID, DAPL_SP * sp_ptr) +{ + struct sockaddr_in addr; + ib_cm_srvc_handle_t cm_ptr = NULL; + int opt = 1; + DAT_RETURN dat_status = DAT_SUCCESS; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " listen(ia_ptr %p ServiceID %d sp_ptr %p)\n", + ia_ptr, serviceID, sp_ptr); + + cm_ptr = dapls_ib_cm_create(NULL); + if (cm_ptr == NULL) + return DAT_INSUFFICIENT_RESOURCES; + + cm_ptr->sp = sp_ptr; + cm_ptr->hca = ia_ptr->hca_ptr; + + /* bind, listen, set sockopt, accept, exchange data */ + if ((cm_ptr->socket = + socket(AF_INET, SOCK_STREAM, 0)) == DAPL_INVALID_SOCKET) { + dapl_log(DAPL_DBG_TYPE_ERR, " ERR: listen socket create: %s\n", + strerror(errno)); + dat_status = DAT_INSUFFICIENT_RESOURCES; + goto bail; + } + + setsockopt(cm_ptr->socket, SOL_SOCKET, SO_REUSEADDR, + (char *)&opt, sizeof(opt)); + addr.sin_port = htons(serviceID); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = INADDR_ANY; + + if ((bind(cm_ptr->socket, (struct sockaddr *)&addr, sizeof(addr)) < 0) + || (listen(cm_ptr->socket, 128) < 0)) { + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " listen: ERROR %s on conn_qual 0x%x\n", + strerror(errno), serviceID); + if (errno == EADDRINUSE) + dat_status = DAT_CONN_QUAL_IN_USE; + else + dat_status = DAT_CONN_QUAL_UNAVAILABLE; + goto bail; + } + + /* set cm_handle for this service point, save listen socket */ + sp_ptr->cm_srvc_handle = cm_ptr; + + /* queue up listen socket to process inbound CR's */ + cm_ptr->state = DCM_LISTEN; + dapli_cm_queue(cm_ptr); + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " listen: qual 0x%x cr %p s_fd %d\n", + ntohs(serviceID), cm_ptr, cm_ptr->socket); + + return dat_status; + bail: + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " listen: ERROR on conn_qual 0x%x\n", serviceID); + dapls_ib_cm_free(cm_ptr, cm_ptr->ep); + return dat_status; +} + +/* + * PASSIVE: accept socket + */ +static void dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr) +{ + dp_ib_cm_handle_t acm_ptr; + int len; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, " socket_accept\n"); + + /* + * Accept all CR's on this port to avoid half-connection (SYN_RCV) + * stalls with many to one connection storms + */ + do { + /* Allocate accept CM and initialize */ + if ((acm_ptr = dapls_ib_cm_create(NULL)) == NULL) + return; + + acm_ptr->sp = cm_ptr->sp; + acm_ptr->hca = cm_ptr->hca; + + len = sizeof(acm_ptr->dst.ia_address); + acm_ptr->socket = accept(cm_ptr->socket, + (struct sockaddr *) + &acm_ptr->dst.ia_address, + (socklen_t *) & len); + if (acm_ptr->socket == DAPL_INVALID_SOCKET) { + dapl_log(DAPL_DBG_TYPE_ERR, + " accept: ERR %s on FD %d l_cr %p\n", + strerror(errno), cm_ptr->socket, cm_ptr); + dapls_ib_cm_free(acm_ptr, acm_ptr->ep); + return; + } + + acm_ptr->state = DCM_ACCEPTING; + dapli_cm_queue(acm_ptr); + + } while (dapl_poll(cm_ptr->socket, DAPL_FD_READ) == DAPL_FD_READ); +} + +/* + * PASSIVE: receive peer QP information, private data, post cr_event + */ +static void dapli_socket_accept_data(ib_cm_srvc_handle_t acm_ptr) +{ + int len; + void *p_data = NULL; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, " socket accepted, read QP data\n"); + + /* read in DST QP info, IA address. check for private data */ + len = + recv(acm_ptr->socket, (char *)&acm_ptr->dst, sizeof(ib_qp_cm_t), 0); + if (len != sizeof(ib_qp_cm_t) || ntohs(acm_ptr->dst.ver) != DCM_VER) { + dapl_log(DAPL_DBG_TYPE_ERR, + " accept read: ERR %s, rcnt=%d, ver=%d\n", + strerror(errno), len, ntohs(acm_ptr->dst.ver)); + goto bail; + } + + /* convert accepted values to host order */ + acm_ptr->dst.port = ntohs(acm_ptr->dst.port); + acm_ptr->dst.lid = ntohs(acm_ptr->dst.lid); + acm_ptr->dst.qpn = ntohl(acm_ptr->dst.qpn); +#ifdef DAT_EXTENSIONS + acm_ptr->dst.qp_type = ntohs(acm_ptr->dst.qp_type); +#endif + acm_ptr->dst.p_size = ntohl(acm_ptr->dst.p_size); + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " accept: DST %s port=0x%x lid=0x%x, qpn=0x%x, psize=%d\n", + inet_ntoa(((struct sockaddr_in *)&acm_ptr->dst. + ia_address)->sin_addr), acm_ptr->dst.port, + acm_ptr->dst.lid, acm_ptr->dst.qpn, acm_ptr->dst.p_size); + + /* validate private data size before reading */ + if (acm_ptr->dst.p_size > IB_MAX_REQ_PDATA_SIZE) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " accept read: psize (%d) wrong\n", + acm_ptr->dst.p_size); + goto bail; + } + + dapl_dbg_log(DAPL_DBG_TYPE_EP, " socket accepted, read private data\n"); + + /* read private data into cm_handle if any present */ + if (acm_ptr->dst.p_size) { + len = + recv(acm_ptr->socket, acm_ptr->p_data, acm_ptr->dst.p_size, + 0); + if (len != acm_ptr->dst.p_size) { + dapl_log(DAPL_DBG_TYPE_ERR, + " accept read pdata: ERR %s, rcnt=%d\n", + strerror(errno), len); + goto bail; + } + dapl_dbg_log(DAPL_DBG_TYPE_EP, " accept: psize=%d read\n", len); + p_data = acm_ptr->p_data; + } + + acm_ptr->state = DCM_ACCEPTING_DATA; + +#ifdef DAT_EXTENSIONS + if (acm_ptr->dst.qp_type == IBV_QPT_UD) { + DAT_IB_EXTENSION_EVENT_DATA xevent; + + /* post EVENT, modify_qp created ah */ + xevent.status = 0; + xevent.type = DAT_IB_UD_CONNECT_REQUEST; + + dapls_evd_post_cr_event_ext(acm_ptr->sp, + DAT_IB_UD_CONNECTION_REQUEST_EVENT, + acm_ptr, + (DAT_COUNT) acm_ptr->dst.p_size, + (DAT_PVOID *) acm_ptr->p_data, + (DAT_PVOID *) & xevent); + } else +#endif + /* trigger CR event and return SUCCESS */ + dapls_cr_callback(acm_ptr, + IB_CME_CONNECTION_REQUEST_PENDING, + p_data, acm_ptr->sp); + return; + bail: + /* close socket, free cm structure, active will see socket close as reject */ + dapls_ib_cm_free(acm_ptr, acm_ptr->ep); + return; +} + +/* + * PASSIVE: consumer accept, send local QP information, private data, + * queue on work thread to receive RTU information to avoid blocking + * user thread. + */ +DAT_RETURN +dapli_socket_accept_usr(DAPL_EP * ep_ptr, + DAPL_CR * cr_ptr, DAT_COUNT p_size, DAT_PVOID p_data) +{ + DAPL_IA *ia_ptr = ep_ptr->header.owner_ia; + dp_ib_cm_handle_t cm_ptr = cr_ptr->ib_cm_handle; + ib_qp_cm_t local; + struct iovec iov[2]; + int len; + + if (p_size > IB_MAX_REP_PDATA_SIZE) + return DAT_LENGTH_ERROR; + + /* must have a accepted socket */ + if (cm_ptr->socket == DAPL_INVALID_SOCKET) + return DAT_INTERNAL_ERROR; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " ACCEPT_USR: remote port=0x%x lid=0x%x" + " qpn=0x%x qp_type %d, psize=%d\n", + cm_ptr->dst.port, cm_ptr->dst.lid, + cm_ptr->dst.qpn, cm_ptr->dst.qp_type, cm_ptr->dst.p_size); + +#ifdef DAT_EXTENSIONS + if (cm_ptr->dst.qp_type == IBV_QPT_UD && + ep_ptr->qp_handle->qp_type != IBV_QPT_UD) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " ACCEPT_USR: ERR remote QP is UD," + ", but local QP is not\n"); + return (DAT_INVALID_HANDLE | DAT_INVALID_HANDLE_EP); + } +#endif + + /* modify QP to RTR and then to RTS with remote info already read */ + dapl_os_lock(&ep_ptr->header.lock); + if (dapls_modify_qp_state(ep_ptr->qp_handle, + IBV_QPS_RTR, cm_ptr) != DAT_SUCCESS) { + dapl_log(DAPL_DBG_TYPE_ERR, + " ACCEPT_USR: QPS_RTR ERR %s -> %s\n", + strerror(errno), inet_ntoa(((struct sockaddr_in *) + &cm_ptr->dst.ia_address)-> + sin_addr)); + dapl_os_unlock(&ep_ptr->header.lock); + goto bail; + } + if (dapls_modify_qp_state(ep_ptr->qp_handle, + IBV_QPS_RTS, cm_ptr) != DAT_SUCCESS) { + dapl_log(DAPL_DBG_TYPE_ERR, + " ACCEPT_USR: QPS_RTS ERR %s -> %s\n", + strerror(errno), inet_ntoa(((struct sockaddr_in *) + &cm_ptr->dst.ia_address)-> + sin_addr)); + dapl_os_unlock(&ep_ptr->header.lock); + goto bail; + } + dapl_os_unlock(&ep_ptr->header.lock); + + /* save remote address information */ + dapl_os_memcpy(&ep_ptr->remote_ia_address, + &cm_ptr->dst.ia_address, + sizeof(ep_ptr->remote_ia_address)); + + /* send our QP info, IA address, pdata. Don't overwrite dst data */ + local.ver = htons(DCM_VER); + local.rej = 0; + local.qpn = htonl(ep_ptr->qp_handle->qp_num); + local.qp_type = htons(ep_ptr->qp_handle->qp_type); + local.port = htons(ia_ptr->hca_ptr->port_num); + local.lid = ia_ptr->hca_ptr->ib_trans.lid; + local.gid = ia_ptr->hca_ptr->ib_trans.gid; + local.ia_address = ia_ptr->hca_ptr->hca_address; + ((struct sockaddr_in *)&local.ia_address)->sin_port = + ntohs(cm_ptr->sp->conn_qual); + + local.p_size = htonl(p_size); + iov[0].iov_base = (void *)&local; + iov[0].iov_len = sizeof(ib_qp_cm_t); + if (p_size) { + iov[1].iov_base = p_data; + iov[1].iov_len = p_size; + len = writev(cm_ptr->socket, iov, 2); + } else { + len = writev(cm_ptr->socket, iov, 1); + } + + if (len != (p_size + sizeof(ib_qp_cm_t))) { + dapl_log(DAPL_DBG_TYPE_ERR, + " ACCEPT_USR: ERR %s, wcnt=%d -> %s\n", + strerror(errno), len, inet_ntoa(((struct sockaddr_in *) + &cm_ptr->dst. + ia_address)-> + sin_addr)); + goto bail; + } + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " ACCEPT_USR: local port=0x%x lid=0x%x" + " qpn=0x%x psize=%d\n", + ntohs(local.port), ntohs(local.lid), + ntohl(local.qpn), ntohl(local.p_size)); + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " ACCEPT_USR SRC GID subnet %016llx id %016llx\n", + (unsigned long long) + htonll(local.gid.global.subnet_prefix), + (unsigned long long) + htonll(local.gid.global.interface_id)); + + /* save state and reference to EP, queue for RTU data */ + cm_ptr->ep = ep_ptr; + cm_ptr->hca = ia_ptr->hca_ptr; + cm_ptr->state = DCM_ACCEPTED; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, " PASSIVE: accepted!\n"); + return DAT_SUCCESS; + bail: + dapls_ib_cm_free(cm_ptr, cm_ptr->ep); + dapls_ib_reinit_ep(ep_ptr); /* reset QP state */ + return DAT_INTERNAL_ERROR; +} + +/* + * PASSIVE: read RTU from active peer, post CONN event + */ +void dapli_socket_accept_rtu(dp_ib_cm_handle_t cm_ptr) +{ + int len; + short rtu_data = 0; + + /* complete handshake after final QP state change */ + len = recv(cm_ptr->socket, (char *)&rtu_data, sizeof(rtu_data), 0); + if (len != sizeof(rtu_data) || ntohs(rtu_data) != 0x0e0f) { + dapl_log(DAPL_DBG_TYPE_ERR, + " ACCEPT_RTU: ERR %s, rcnt=%d rdata=%x\n", + strerror(errno), len, ntohs(rtu_data), + inet_ntoa(((struct sockaddr_in *) + &cm_ptr->dst.ia_address)->sin_addr)); + goto bail; + } + + /* save state and reference to EP, queue for disc event */ + cm_ptr->state = DCM_CONNECTED; + + /* final data exchange if remote QP state is good to go */ + dapl_dbg_log(DAPL_DBG_TYPE_EP, " PASSIVE: connected!\n"); + +#ifdef DAT_EXTENSIONS + if (cm_ptr->dst.qp_type == IBV_QPT_UD) { + DAT_IB_EXTENSION_EVENT_DATA xevent; + + /* post EVENT, modify_qp created ah */ + xevent.status = 0; + xevent.type = DAT_IB_UD_PASSIVE_REMOTE_AH; + xevent.remote_ah.ah = cm_ptr->ah; + xevent.remote_ah.qpn = cm_ptr->dst.qpn; + dapl_os_memcpy(&xevent.remote_ah.ia_addr, + &cm_ptr->dst.ia_address, + sizeof(cm_ptr->dst.ia_address)); + + dapls_evd_post_connection_event_ext((DAPL_EVD *) cm_ptr->ep-> + param.connect_evd_handle, + DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED, + (DAT_EP_HANDLE) cm_ptr->ep, + (DAT_COUNT) cm_ptr->dst.p_size, + (DAT_PVOID *) cm_ptr->p_data, + (DAT_PVOID *) &xevent); + + /* done with socket, don't destroy cm_ptr, need pdata */ + closesocket(cm_ptr->socket); + cm_ptr->socket = DAPL_INVALID_SOCKET; + cm_ptr->state = DCM_RELEASED; + } else { +#endif + cm_ptr->ep->cm_handle = cm_ptr; /* only RC, multi CR's on UD */ + dapls_cr_callback(cm_ptr, IB_CME_CONNECTED, NULL, cm_ptr->sp); + } + return; + +bail: + dapls_ib_reinit_ep(cm_ptr->ep); /* reset QP state */ + dapls_ib_cm_free(cm_ptr, cm_ptr->ep); + dapls_cr_callback(cm_ptr, IB_CME_DESTINATION_REJECT, NULL, cm_ptr->sp); +} + +/* + * dapls_ib_connect + * + * Initiate a connection with the passive listener on another node + * + * Input: + * ep_handle, + * remote_ia_address, + * remote_conn_qual, + * prd_size size of private data and structure + * prd_prt pointer to private data structure + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INVALID_PARAMETER + * + */ +DAT_RETURN +dapls_ib_connect(IN DAT_EP_HANDLE ep_handle, + IN DAT_IA_ADDRESS_PTR remote_ia_address, + IN DAT_CONN_QUAL remote_conn_qual, + IN DAT_COUNT private_data_size, IN void *private_data) +{ + DAPL_EP *ep_ptr; + ib_qp_handle_t qp_ptr; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " connect(ep_handle %p ....)\n", ep_handle); + + ep_ptr = (DAPL_EP *) ep_handle; + qp_ptr = ep_ptr->qp_handle; + + return (dapli_socket_connect(ep_ptr, remote_ia_address, + remote_conn_qual, + private_data_size, private_data)); +} + +/* + * dapls_ib_disconnect + * + * Disconnect an EP + * + * Input: + * ep_handle, + * disconnect_flags + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + */ +DAT_RETURN +dapls_ib_disconnect(IN DAPL_EP * ep_ptr, IN DAT_CLOSE_FLAGS close_flags) +{ + dapl_dbg_log(DAPL_DBG_TYPE_EP, + "dapls_ib_disconnect(ep_handle %p ....)\n", ep_ptr); + + /* reinit to modify QP state */ + dapls_ib_reinit_ep(ep_ptr); + + if (ep_ptr->cm_handle == NULL || + ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED) + return DAT_SUCCESS; + else + return (dapli_socket_disconnect(ep_ptr->cm_handle)); +} + +/* + * dapls_ib_disconnect_clean + * + * Clean up outstanding connection data. This routine is invoked + * after the final disconnect callback has occurred. Only on the + * ACTIVE side of a connection. It is also called if dat_ep_connect + * times out using the consumer supplied timeout value. + * + * Input: + * ep_ptr DAPL_EP + * active Indicates active side of connection + * + * Output: + * none + * + * Returns: + * void + * + */ +void +dapls_ib_disconnect_clean(IN DAPL_EP * ep_ptr, + IN DAT_BOOLEAN active, + IN const ib_cm_events_t ib_cm_event) +{ + /* NOTE: SCM will only initialize cm_handle with RC type + * + * For UD there can many in-flight CR's so you + * cannot cleanup timed out CR's with EP reference + * alone since they share the same EP. The common + * code that handles connection timeout logic needs + * updated for UD support. + */ + if (ep_ptr->cm_handle) + dapls_ib_cm_free(ep_ptr->cm_handle, ep_ptr); + + return; +} + +/* + * dapl_ib_setup_conn_listener + * + * Have the CM set up a connection listener. + * + * Input: + * ibm_hca_handle HCA handle + * qp_handle QP handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INTERNAL_ERROR + * DAT_CONN_QUAL_UNAVAILBLE + * DAT_CONN_QUAL_IN_USE + * + */ +DAT_RETURN +dapls_ib_setup_conn_listener(IN DAPL_IA * ia_ptr, + IN DAT_UINT64 ServiceID, IN DAPL_SP * sp_ptr) +{ + return (dapli_socket_listen(ia_ptr, ServiceID, sp_ptr)); +} + +/* + * dapl_ib_remove_conn_listener + * + * Have the CM remove a connection listener. + * + * Input: + * ia_handle IA handle + * ServiceID IB Channel Service ID + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_STATE + * + */ +DAT_RETURN +dapls_ib_remove_conn_listener(IN DAPL_IA * ia_ptr, IN DAPL_SP * sp_ptr) +{ + ib_cm_srvc_handle_t cm_ptr = sp_ptr->cm_srvc_handle; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + "dapls_ib_remove_conn_listener(ia_ptr %p sp_ptr %p cm_ptr %p)\n", + ia_ptr, sp_ptr, cm_ptr); + + /* close accepted socket, free cm_srvc_handle and return */ + if (cm_ptr != NULL) { + if (cm_ptr->socket != DAPL_INVALID_SOCKET) { + shutdown(cm_ptr->socket, SHUT_RDWR); + closesocket(cm_ptr->socket); + cm_ptr->socket = DAPL_INVALID_SOCKET; + } + /* cr_thread will free */ + cm_ptr->state = DCM_DESTROY; + sp_ptr->cm_srvc_handle = NULL; + if (send(cm_ptr->hca->ib_trans.scm[1], + "w", sizeof "w", 0) == -1) + dapl_log(DAPL_DBG_TYPE_CM, + " cm_destroy: thread wakeup error = %s\n", + strerror(errno)); + } + return DAT_SUCCESS; +} + +/* + * dapls_ib_accept_connection + * + * Perform necessary steps to accept a connection + * + * Input: + * cr_handle + * ep_handle + * private_data_size + * private_data + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INSUFFICIENT_RESOURCES + * DAT_INTERNAL_ERROR + * + */ +DAT_RETURN +dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle, + IN DAT_EP_HANDLE ep_handle, + IN DAT_COUNT p_size, IN const DAT_PVOID p_data) +{ + DAPL_CR *cr_ptr; + DAPL_EP *ep_ptr; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + "dapls_ib_accept_connection(cr %p ep %p prd %p,%d)\n", + cr_handle, ep_handle, p_data, p_size); + + cr_ptr = (DAPL_CR *) cr_handle; + ep_ptr = (DAPL_EP *) ep_handle; + + /* allocate and attach a QP if necessary */ + if (ep_ptr->qp_state == DAPL_QP_STATE_UNATTACHED) { + DAT_RETURN status; + status = dapls_ib_qp_alloc(ep_ptr->header.owner_ia, + ep_ptr, ep_ptr); + if (status != DAT_SUCCESS) + return status; + } + return (dapli_socket_accept_usr(ep_ptr, cr_ptr, p_size, p_data)); +} + +/* + * dapls_ib_reject_connection + * + * Reject a connection + * + * Input: + * cr_handle + * + * Output: + * none + * + * Returns: + * DAT_SUCCESS + * DAT_INTERNAL_ERROR + * + */ +DAT_RETURN +dapls_ib_reject_connection(IN dp_ib_cm_handle_t cm_ptr, + IN int reason, + IN DAT_COUNT psize, IN const DAT_PVOID pdata) +{ + struct iovec iov[2]; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + " reject(cm %p reason %x, pdata %p, psize %d)\n", + cm_ptr, reason, pdata, psize); + + if (psize > IB_MAX_REJ_PDATA_SIZE) + return DAT_LENGTH_ERROR; + + /* write reject data to indicate reject */ + if (cm_ptr->socket != DAPL_INVALID_SOCKET) { + cm_ptr->dst.rej = (uint16_t) reason; + cm_ptr->dst.rej = htons(cm_ptr->dst.rej); + cm_ptr->dst.p_size = htonl(psize); + /* get qp_type from request */ + cm_ptr->dst.qp_type = ntohs(cm_ptr->dst.qp_type); + + iov[0].iov_base = (void *)&cm_ptr->dst; + iov[0].iov_len = sizeof(ib_qp_cm_t); + if (psize) { + iov[1].iov_base = pdata; + iov[1].iov_len = psize; + writev(cm_ptr->socket, iov, 2); + } else { + writev(cm_ptr->socket, iov, 1); + } + + shutdown(cm_ptr->socket, SHUT_RDWR); + closesocket(cm_ptr->socket); + cm_ptr->socket = DAPL_INVALID_SOCKET; + } + + /* cr_thread will destroy CR */ + cm_ptr->state = DCM_DESTROY; + if (send(cm_ptr->hca->ib_trans.scm[1], "w", sizeof "w", 0) == -1) + dapl_log(DAPL_DBG_TYPE_CM, + " cm_destroy: thread wakeup error = %s\n", + strerror(errno)); + return DAT_SUCCESS; +} + +/* + * dapls_ib_cm_remote_addr + * + * Obtain the remote IP address given a connection + * + * Input: + * cr_handle + * + * Output: + * remote_ia_address: where to place the remote address + * + * Returns: + * DAT_SUCCESS + * DAT_INVALID_HANDLE + * + */ +DAT_RETURN +dapls_ib_cm_remote_addr(IN DAT_HANDLE dat_handle, + OUT DAT_SOCK_ADDR6 * remote_ia_address) +{ + DAPL_HEADER *header; + dp_ib_cm_handle_t ib_cm_handle; + + dapl_dbg_log(DAPL_DBG_TYPE_EP, + "dapls_ib_cm_remote_addr(dat_handle %p, ....)\n", + dat_handle); + + header = (DAPL_HEADER *) dat_handle; + + if (header->magic == DAPL_MAGIC_EP) + ib_cm_handle = ((DAPL_EP *) dat_handle)->cm_handle; + else if (header->magic == DAPL_MAGIC_CR) + ib_cm_handle = ((DAPL_CR *) dat_handle)->ib_cm_handle; + else + return DAT_INVALID_HANDLE; + + dapl_os_memcpy(remote_ia_address, + &ib_cm_handle->dst.ia_address, sizeof(DAT_SOCK_ADDR6)); + + return DAT_SUCCESS; +} + +/* + * dapls_ib_private_data_size + * + * Return the size of private data given a connection op type + * + * Input: + * prd_ptr private data pointer + * conn_op connection operation type + * + * If prd_ptr is NULL, this is a query for the max size supported by + * the provider, otherwise it is the actual size of the private data + * contained in prd_ptr. + * + * + * Output: + * None + * + * Returns: + * length of private data + * + */ +int dapls_ib_private_data_size(IN DAPL_PRIVATE * prd_ptr, + IN DAPL_PDATA_OP conn_op, IN DAPL_HCA * hca_ptr) +{ + int size; + + switch (conn_op) { + case DAPL_PDATA_CONN_REQ: + { + size = IB_MAX_REQ_PDATA_SIZE; + break; + } + case DAPL_PDATA_CONN_REP: + { + size = IB_MAX_REP_PDATA_SIZE; + break; + } + case DAPL_PDATA_CONN_REJ: + { + size = IB_MAX_REJ_PDATA_SIZE; + break; + } + case DAPL_PDATA_CONN_DREQ: + { + size = IB_MAX_DREQ_PDATA_SIZE; + break; + } + case DAPL_PDATA_CONN_DREP: + { + size = IB_MAX_DREP_PDATA_SIZE; + break; + } + default: + { + size = 0; + } + + } /* end case */ + + return size; +} + +/* + * Map all socket CM event codes to the DAT equivelent. + */ +#define DAPL_IB_EVENT_CNT 10 + +static struct ib_cm_event_map { + const ib_cm_events_t ib_cm_event; + DAT_EVENT_NUMBER dat_event_num; +} ib_cm_event_map[DAPL_IB_EVENT_CNT] = { +/* 00 */ {IB_CME_CONNECTED, + DAT_CONNECTION_EVENT_ESTABLISHED}, +/* 01 */ {IB_CME_DISCONNECTED, + DAT_CONNECTION_EVENT_DISCONNECTED}, +/* 02 */ {IB_CME_DISCONNECTED_ON_LINK_DOWN, + DAT_CONNECTION_EVENT_DISCONNECTED}, +/* 03 */ {IB_CME_CONNECTION_REQUEST_PENDING, + DAT_CONNECTION_REQUEST_EVENT}, +/* 04 */ {IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA, + DAT_CONNECTION_REQUEST_EVENT}, +/* 05 */ {IB_CME_DESTINATION_REJECT, + DAT_CONNECTION_EVENT_NON_PEER_REJECTED}, +/* 06 */ {IB_CME_DESTINATION_REJECT_PRIVATE_DATA, + DAT_CONNECTION_EVENT_PEER_REJECTED}, +/* 07 */ {IB_CME_DESTINATION_UNREACHABLE, + DAT_CONNECTION_EVENT_UNREACHABLE}, +/* 08 */ {IB_CME_TOO_MANY_CONNECTION_REQUESTS, + DAT_CONNECTION_EVENT_NON_PEER_REJECTED}, +/* 09 */ {IB_CME_LOCAL_FAILURE, + DAT_CONNECTION_EVENT_BROKEN} +}; + +/* + * dapls_ib_get_cm_event + * + * Return a DAT connection event given a provider CM event. + * + * Input: + * dat_event_num DAT event we need an equivelent CM event for + * + * Output: + * none + * + * Returns: + * ib_cm_event of translated DAPL value + */ +DAT_EVENT_NUMBER +dapls_ib_get_dat_event(IN const ib_cm_events_t ib_cm_event, + IN DAT_BOOLEAN active) +{ + DAT_EVENT_NUMBER dat_event_num; + int i; + + active = active; + + if (ib_cm_event > IB_CME_LOCAL_FAILURE) + return (DAT_EVENT_NUMBER) 0; + + dat_event_num = 0; + for (i = 0; i < DAPL_IB_EVENT_CNT; i++) { + if (ib_cm_event == ib_cm_event_map[i].ib_cm_event) { + dat_event_num = ib_cm_event_map[i].dat_event_num; + break; + } + } + dapl_dbg_log(DAPL_DBG_TYPE_CALLBACK, + "dapls_ib_get_dat_event: event translate(%s) ib=0x%x dat=0x%x\n", + active ? "active" : "passive", ib_cm_event, dat_event_num); + + return dat_event_num; +} + +/* + * dapls_ib_get_dat_event + * + * Return a DAT connection event given a provider CM event. + * + * Input: + * ib_cm_event event provided to the dapl callback routine + * active switch indicating active or passive connection + * + * Output: + * none + * + * Returns: + * DAT_EVENT_NUMBER of translated provider value + */ +ib_cm_events_t dapls_ib_get_cm_event(IN DAT_EVENT_NUMBER dat_event_num) +{ + ib_cm_events_t ib_cm_event; + int i; + + ib_cm_event = 0; + for (i = 0; i < DAPL_IB_EVENT_CNT; i++) { + if (dat_event_num == ib_cm_event_map[i].dat_event_num) { + ib_cm_event = ib_cm_event_map[i].ib_cm_event; + break; + } + } + return ib_cm_event; +} + +/* outbound/inbound CR processing thread to avoid blocking applications */ +void cr_thread(void *arg) +{ + struct dapl_hca *hca_ptr = arg; + dp_ib_cm_handle_t cr, next_cr; + int opt, ret; + socklen_t opt_len; + char rbuf[2]; + struct dapl_fd_set *set; + enum DAPL_FD_EVENTS event; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cr_thread: ENTER hca %p\n", hca_ptr); + set = dapl_alloc_fd_set(); + if (!set) + goto out; + + dapl_os_lock(&hca_ptr->ib_trans.lock); + hca_ptr->ib_trans.cr_state = IB_THREAD_RUN; + + while (1) { + dapl_fd_zero(set); + dapl_fd_set(hca_ptr->ib_trans.scm[0], set, DAPL_FD_READ); + + if (!dapl_llist_is_empty(&hca_ptr->ib_trans.list)) + next_cr = dapl_llist_peek_head(&hca_ptr->ib_trans.list); + else + next_cr = NULL; + + while (next_cr) { + cr = next_cr; + next_cr = dapl_llist_next_entry(&hca_ptr->ib_trans.list, + (DAPL_LLIST_ENTRY *) & + cr->entry); + if (cr->state == DCM_DESTROY + || hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) { + dapl_llist_remove_entry(&hca_ptr->ib_trans.list, + (DAPL_LLIST_ENTRY *) & + cr->entry); + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " CR FREE: %p ep=%p st=%d sock=%d\n", + cr, cr->ep, cr->state, cr->socket); + dapl_os_free(cr, sizeof(*cr)); + continue; + } + if (cr->socket == DAPL_INVALID_SOCKET) + continue; + + event = (cr->state == DCM_CONN_PENDING) ? + DAPL_FD_WRITE : DAPL_FD_READ; + if (dapl_fd_set(cr->socket, set, event)) { + dapl_log(DAPL_DBG_TYPE_ERR, + " cr_thread: DESTROY CR st=%d fd %d" + " -> %s\n", cr->state, cr->socket, + inet_ntoa(((struct sockaddr_in *) + &cr->dst.ia_address)-> + sin_addr)); + dapls_ib_cm_free(cr, cr->ep); + continue; + } + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " poll cr=%p, socket=%d\n", cr, + cr->socket); + dapl_os_unlock(&hca_ptr->ib_trans.lock); + + ret = dapl_poll(cr->socket, event); + + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " poll ret=0x%x cr->state=%d socket=%d\n", + ret, cr->state, cr->socket); + + /* data on listen, qp exchange, and on disc req */ + if (ret == DAPL_FD_READ) { + if (cr->socket != DAPL_INVALID_SOCKET) { + switch (cr->state) { + case DCM_LISTEN: + dapli_socket_accept(cr); + break; + case DCM_ACCEPTING: + dapli_socket_accept_data(cr); + break; + case DCM_ACCEPTED: + dapli_socket_accept_rtu(cr); + break; + case DCM_RTU_PENDING: + dapli_socket_connect_rtu(cr); + break; + case DCM_CONNECTED: + dapli_socket_disconnect(cr); + break; + default: + break; + } + } + /* connect socket is writable, check status */ + } else if (ret == DAPL_FD_WRITE || + (cr->state == DCM_CONN_PENDING && + ret == DAPL_FD_ERROR)) { + opt = 0; + opt_len = sizeof(opt); + ret = getsockopt(cr->socket, SOL_SOCKET, + SO_ERROR, (char *)&opt, + &opt_len); + if (!ret) + dapli_socket_connected(cr, opt); + else + dapli_socket_connected(cr, errno); + + /* POLLUP, ERR, NVAL, or poll error - DISC */ + } else if (ret < 0 || ret == DAPL_FD_ERROR) { + dapl_log(DAPL_DBG_TYPE_CM, + " poll=%d cr->st=%s sk=%d ep %p, %d\n", + ret, dapl_cm_state_str(cr->state), + cr->socket, cr->ep, + cr->ep ? cr->ep->param.ep_state:0); + dapli_socket_disconnect(cr); + } + dapl_os_lock(&hca_ptr->ib_trans.lock); + } + + /* set to exit and all resources destroyed */ + if ((hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) && + (dapl_llist_is_empty(&hca_ptr->ib_trans.list))) + break; + + dapl_os_unlock(&hca_ptr->ib_trans.lock); + dapl_select(set); + + /* if pipe used to wakeup, consume */ + while (dapl_poll(hca_ptr->ib_trans.scm[0], + DAPL_FD_READ) == DAPL_FD_READ) { + if (recv(hca_ptr->ib_trans.scm[0], rbuf, 2, 0) == -1) + dapl_log(DAPL_DBG_TYPE_CM, + " cr_thread: read pipe error = %s\n", + strerror(errno)); + } + dapl_os_lock(&hca_ptr->ib_trans.lock); + + /* set to exit and all resources destroyed */ + if ((hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) && + (dapl_llist_is_empty(&hca_ptr->ib_trans.list))) + break; + } + + dapl_os_unlock(&hca_ptr->ib_trans.lock); + free(set); + out: + hca_ptr->ib_trans.cr_state = IB_THREAD_EXIT; + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cr_thread(hca %p) exit\n", hca_ptr); +} + + +#ifdef DAPL_COUNTERS +/* Debug aid: List all Connections in process and state */ +void dapls_print_cm_list(IN DAPL_IA *ia_ptr) +{ + /* Print in process CR's for this IA, if debug type set */ + int i = 0; + dp_ib_cm_handle_t cr, next_cr; + + dapl_os_lock(&ia_ptr->hca_ptr->ib_trans.lock); + if (!dapl_llist_is_empty((DAPL_LLIST_HEAD*) + &ia_ptr->hca_ptr->ib_trans.list)) + next_cr = dapl_llist_peek_head((DAPL_LLIST_HEAD*) + &ia_ptr->hca_ptr->ib_trans.list); + else + next_cr = NULL; + + printf("\n DAPL IA CONNECTIONS IN PROCESS:\n"); + while (next_cr) { + cr = next_cr; + next_cr = dapl_llist_next_entry((DAPL_LLIST_HEAD*) + &ia_ptr->hca_ptr->ib_trans.list, + (DAPL_LLIST_ENTRY*)&cr->entry); + + printf( " CONN[%d]: sp %p ep %p sock %d %s %s %s %s %d\n", + i, cr->sp, cr->ep, cr->socket, + cr->dst.qp_type == IBV_QPT_RC ? "RC" : "UD", + dapl_cm_state_str(cr->state), + cr->sp ? "<-" : "->", + cr->state == DCM_LISTEN ? + inet_ntoa(((struct sockaddr_in *) + &ia_ptr->hca_ptr->hca_address)->sin_addr) : + inet_ntoa(((struct sockaddr_in *) + &cr->dst.ia_address)->sin_addr), + cr->sp ? (int)cr->sp->conn_qual : + ntohs(((struct sockaddr_in *) + &cr->dst.ia_address)->sin_port)); + i++; + } + printf("\n"); + dapl_os_unlock(&ia_ptr->hca_ptr->ib_trans.lock); +} +#endif diff --git a/dapl/openib_scm/dapl_ib_cm.c b/dapl/openib_scm/dapl_ib_cm.c deleted file mode 100644 index 90d6d27..0000000 --- a/dapl/openib_scm/dapl_ib_cm.c +++ /dev/null @@ -1,1786 +0,0 @@ -/* - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/*************************************************************************** - * - * Module: uDAPL - * - * Filename: dapl_ib_cm.c - * - * Author: Arlin Davis - * - * Created: 3/10/2005 - * - * Description: - * - * The uDAPL openib provider - connection management - * - **************************************************************************** - * Source Control System Information - * - * $Id: $ - * - * Copyright (c) 2005 Intel Corporation. All rights reserved. - * - **************************************************************************/ - -#include "dapl.h" -#include "dapl_adapter_util.h" -#include "dapl_evd_util.h" -#include "dapl_cr_util.h" -#include "dapl_name_service.h" -#include "dapl_ib_util.h" -#include "dapl_osd.h" - -#if defined(_WIN32) || defined(_WIN64) -enum DAPL_FD_EVENTS { - DAPL_FD_READ = 0x1, - DAPL_FD_WRITE = 0x2, - DAPL_FD_ERROR = 0x4 -}; - -static int dapl_config_socket(DAPL_SOCKET s) -{ - unsigned long nonblocking = 1; - return ioctlsocket(s, FIONBIO, &nonblocking); -} - -static int dapl_connect_socket(DAPL_SOCKET s, struct sockaddr *addr, - int addrlen) -{ - int err; - - err = connect(s, addr, addrlen); - if (err == SOCKET_ERROR) - err = WSAGetLastError(); - return (err == WSAEWOULDBLOCK) ? EAGAIN : err; -} - -struct dapl_fd_set { - struct fd_set set[3]; -}; - -static struct dapl_fd_set *dapl_alloc_fd_set(void) -{ - return dapl_os_alloc(sizeof(struct dapl_fd_set)); -} - -static void dapl_fd_zero(struct dapl_fd_set *set) -{ - FD_ZERO(&set->set[0]); - FD_ZERO(&set->set[1]); - FD_ZERO(&set->set[2]); -} - -static int dapl_fd_set(DAPL_SOCKET s, struct dapl_fd_set *set, - enum DAPL_FD_EVENTS event) -{ - FD_SET(s, &set->set[(event == DAPL_FD_READ) ? 0 : 1]); - FD_SET(s, &set->set[2]); - return 0; -} - -static enum DAPL_FD_EVENTS dapl_poll(DAPL_SOCKET s, enum DAPL_FD_EVENTS event) -{ - struct fd_set rw_fds; - struct fd_set err_fds; - struct timeval tv; - int ret; - - FD_ZERO(&rw_fds); - FD_ZERO(&err_fds); - FD_SET(s, &rw_fds); - FD_SET(s, &err_fds); - - tv.tv_sec = 0; - tv.tv_usec = 0; - - if (event == DAPL_FD_READ) - ret = select(1, &rw_fds, NULL, &err_fds, &tv); - else - ret = select(1, NULL, &rw_fds, &err_fds, &tv); - - if (ret == 0) - return 0; - else if (ret == SOCKET_ERROR) - return WSAGetLastError(); - else if (FD_ISSET(s, &rw_fds)) - return event; - else - return DAPL_FD_ERROR; -} - -static int dapl_select(struct dapl_fd_set *set) -{ - int ret; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, " dapl_select: sleep\n"); - ret = select(0, &set->set[0], &set->set[1], &set->set[2], NULL); - dapl_dbg_log(DAPL_DBG_TYPE_CM, " dapl_select: wakeup\n"); - - if (ret == SOCKET_ERROR) - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " dapl_select: error 0x%x\n", WSAGetLastError()); - - return ret; -} -#else // _WIN32 || _WIN64 -enum DAPL_FD_EVENTS { - DAPL_FD_READ = POLLIN, - DAPL_FD_WRITE = POLLOUT, - DAPL_FD_ERROR = POLLERR -}; - -static int dapl_config_socket(DAPL_SOCKET s) -{ - int ret; - - ret = fcntl(s, F_GETFL); - if (ret >= 0) - ret = fcntl(s, F_SETFL, ret | O_NONBLOCK); - return ret; -} - -static int dapl_connect_socket(DAPL_SOCKET s, struct sockaddr *addr, - int addrlen) -{ - int ret; - - ret = connect(s, addr, addrlen); - - return (errno == EINPROGRESS) ? EAGAIN : ret; -} - -struct dapl_fd_set { - int index; - struct pollfd set[DAPL_FD_SETSIZE]; -}; - -static struct dapl_fd_set *dapl_alloc_fd_set(void) -{ - return dapl_os_alloc(sizeof(struct dapl_fd_set)); -} - -static void dapl_fd_zero(struct dapl_fd_set *set) -{ - set->index = 0; -} - -static int dapl_fd_set(DAPL_SOCKET s, struct dapl_fd_set *set, - enum DAPL_FD_EVENTS event) -{ - if (set->index == DAPL_FD_SETSIZE - 1) { - dapl_log(DAPL_DBG_TYPE_ERR, - "SCM ERR: cm_thread exceeded FD_SETSIZE %d\n", - set->index + 1); - return -1; - } - - set->set[set->index].fd = s; - set->set[set->index].revents = 0; - set->set[set->index++].events = event; - return 0; -} - -static enum DAPL_FD_EVENTS dapl_poll(DAPL_SOCKET s, enum DAPL_FD_EVENTS event) -{ - struct pollfd fds; - int ret; - - fds.fd = s; - fds.events = event; - fds.revents = 0; - ret = poll(&fds, 1, 0); - dapl_log(DAPL_DBG_TYPE_CM, " dapl_poll: fd=%d ret=%d, evnts=0x%x\n", - s, ret, fds.revents); - if (ret == 0) - return 0; - else if (fds.revents & (POLLERR | POLLHUP | POLLNVAL)) - return DAPL_FD_ERROR; - else - return fds.revents; -} - -static int dapl_select(struct dapl_fd_set *set) -{ - int ret; - - dapl_dbg_log(DAPL_DBG_TYPE_CM, " dapl_select: sleep, fds=%d\n", - set->index); - ret = poll(set->set, set->index, -1); - dapl_dbg_log(DAPL_DBG_TYPE_CM, " dapl_select: wakeup, ret=0x%x\n", ret); - return ret; -} -#endif - -static struct ib_cm_handle *dapli_cm_create(void) -{ - struct ib_cm_handle *cm_ptr; - - /* Allocate CM, init lock, and initialize */ - if ((cm_ptr = dapl_os_alloc(sizeof(*cm_ptr))) == NULL) - return NULL; - - (void)dapl_os_memzero(cm_ptr, sizeof(*cm_ptr)); - if (dapl_os_lock_init(&cm_ptr->lock)) - goto bail; - - cm_ptr->dst.ver = htons(DSCM_VER); - cm_ptr->socket = DAPL_INVALID_SOCKET; - return cm_ptr; - bail: - dapl_os_free(cm_ptr, sizeof(*cm_ptr)); - return NULL; -} - -/* mark for destroy, remove all references, schedule cleanup */ -static void dapli_cm_destroy(struct ib_cm_handle *cm_ptr) -{ - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " cm_destroy: cm %p ep %p\n", cm_ptr, cm_ptr->ep); - - /* cleanup, never made it to work queue */ - if (cm_ptr->state == SCM_INIT) { - if (cm_ptr->socket != DAPL_INVALID_SOCKET) { - shutdown(cm_ptr->socket, SHUT_RDWR); - closesocket(cm_ptr->socket); - } - dapl_os_free(cm_ptr, sizeof(*cm_ptr)); - return; - } - - dapl_os_lock(&cm_ptr->lock); - cm_ptr->state = SCM_DESTROY; - if ((cm_ptr->ep) && (cm_ptr->ep->cm_handle == cm_ptr)) { - cm_ptr->ep->cm_handle = IB_INVALID_HANDLE; - cm_ptr->ep = NULL; - } - - /* close socket if still active */ - if (cm_ptr->socket != DAPL_INVALID_SOCKET) { - shutdown(cm_ptr->socket, SHUT_RDWR); - closesocket(cm_ptr->socket); - cm_ptr->socket = DAPL_INVALID_SOCKET; - } - dapl_os_unlock(&cm_ptr->lock); - - /* wakeup work thread */ - if (send(cm_ptr->hca->ib_trans.scm[1], "w", sizeof "w", 0) == -1) - dapl_log(DAPL_DBG_TYPE_CM, - " cm_destroy: thread wakeup error = %s\n", - strerror(errno)); -} - -/* queue socket for processing CM work */ -static void dapli_cm_queue(struct ib_cm_handle *cm_ptr) -{ - /* add to work queue for cr thread processing */ - dapl_llist_init_entry((DAPL_LLIST_ENTRY *) & cm_ptr->entry); - dapl_os_lock(&cm_ptr->hca->ib_trans.lock); - dapl_llist_add_tail(&cm_ptr->hca->ib_trans.list, - (DAPL_LLIST_ENTRY *) & cm_ptr->entry, cm_ptr); - dapl_os_unlock(&cm_ptr->hca->ib_trans.lock); - - /* wakeup CM work thread */ - if (send(cm_ptr->hca->ib_trans.scm[1], "w", sizeof "w", 0) == -1) - dapl_log(DAPL_DBG_TYPE_CM, - " cm_queue: thread wakeup error = %s\n", - strerror(errno)); -} - -/* - * ACTIVE/PASSIVE: called from CR thread or consumer via ep_disconnect - * or from ep_free - */ -DAT_RETURN dapli_socket_disconnect(dp_ib_cm_handle_t cm_ptr) -{ - DAPL_EP *ep_ptr = cm_ptr->ep; - DAT_UINT32 disc_data = htonl(0xdead); - - if (ep_ptr == NULL) - return DAT_SUCCESS; - - dapl_os_lock(&cm_ptr->lock); - if ((cm_ptr->state == SCM_INIT) || - (cm_ptr->state == SCM_DISCONNECTED) || - (cm_ptr->state == SCM_DESTROY)) { - dapl_os_unlock(&cm_ptr->lock); - return DAT_SUCCESS; - } else { - /* send disc date, close socket, schedule destroy */ - if (cm_ptr->socket != DAPL_INVALID_SOCKET) { - if (send(cm_ptr->socket, (char *)&disc_data, - sizeof(disc_data), 0) == -1) - dapl_log(DAPL_DBG_TYPE_WARN, - " cm_disc: write error = %s\n", - strerror(errno)); - shutdown(cm_ptr->socket, SHUT_RDWR); - closesocket(cm_ptr->socket); - cm_ptr->socket = DAPL_INVALID_SOCKET; - } - cm_ptr->state = SCM_DISCONNECTED; - } - dapl_os_unlock(&cm_ptr->lock); - - /* disconnect events for RC's only */ - if (ep_ptr->param.ep_attr.service_type == DAT_SERVICE_TYPE_RC) { - if (ep_ptr->cr_ptr) { - dapls_cr_callback(cm_ptr, - IB_CME_DISCONNECTED, - NULL, - ((DAPL_CR *) ep_ptr->cr_ptr)->sp_ptr); - } else { - dapl_evd_connection_callback(ep_ptr->cm_handle, - IB_CME_DISCONNECTED, - NULL, ep_ptr); - } - } - - /* scheduled destroy via disconnect clean in callback */ - return DAT_SUCCESS; -} - -/* - * ACTIVE: socket connected, send QP information to peer - */ -static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err) -{ - int len, opt = 1; - struct iovec iov[2]; - struct dapl_ep *ep_ptr = cm_ptr->ep; - - if (err) { - dapl_log(DAPL_DBG_TYPE_ERR, - " CONN_PENDING: %s ERR %s -> %s %d\n", - err == -1 ? "POLL" : "SOCKOPT", - err == -1 ? strerror(errno) : strerror(err), - inet_ntoa(((struct sockaddr_in *) - ep_ptr->param. - remote_ia_address_ptr)->sin_addr), - ntohs(((struct sockaddr_in *) - &cm_ptr->dst.ia_address)->sin_port)); - goto bail; - } - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " socket connected, write QP and private data\n"); - - /* no delay for small packets */ - setsockopt(cm_ptr->socket, IPPROTO_TCP, TCP_NODELAY, - (char *)&opt, sizeof(opt)); - - /* send qp info and pdata to remote peer */ - iov[0].iov_base = (void *)&cm_ptr->dst; - iov[0].iov_len = sizeof(ib_qp_cm_t); - if (cm_ptr->dst.p_size) { - iov[1].iov_base = cm_ptr->p_data; - iov[1].iov_len = ntohl(cm_ptr->dst.p_size); - len = writev(cm_ptr->socket, iov, 2); - } else { - len = writev(cm_ptr->socket, iov, 1); - } - - if (len != (ntohl(cm_ptr->dst.p_size) + sizeof(ib_qp_cm_t))) { - dapl_log(DAPL_DBG_TYPE_ERR, - " CONN_PENDING write: ERR %s, wcnt=%d -> %s\n", - strerror(errno), len, inet_ntoa(((struct sockaddr_in *) - ep_ptr->param. - remote_ia_address_ptr)-> - sin_addr)); - goto bail; - } - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " connected: sending SRC port=0x%x lid=0x%x," - " qpn=0x%x, psize=%d\n", - ntohs(cm_ptr->dst.port), ntohs(cm_ptr->dst.lid), - ntohl(cm_ptr->dst.qpn), ntohl(cm_ptr->dst.p_size)); - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " connected: sending SRC GID subnet %016llx id %016llx\n", - (unsigned long long) - htonll(cm_ptr->dst.gid.global.subnet_prefix), - (unsigned long long) - htonll(cm_ptr->dst.gid.global.interface_id)); - - /* queue up to work thread to avoid blocking consumer */ - cm_ptr->state = SCM_RTU_PENDING; - return; - bail: - /* close socket, free cm structure and post error event */ - dapli_cm_destroy(cm_ptr); - dapl_evd_connection_callback(NULL, IB_CME_LOCAL_FAILURE, NULL, ep_ptr); -} - -/* - * ACTIVE: Create socket, connect, defer exchange QP information to CR thread - * to avoid blocking. - */ -DAT_RETURN -dapli_socket_connect(DAPL_EP * ep_ptr, - DAT_IA_ADDRESS_PTR r_addr, - DAT_CONN_QUAL r_qual, DAT_COUNT p_size, DAT_PVOID p_data) -{ - dp_ib_cm_handle_t cm_ptr; - int ret; - DAPL_IA *ia_ptr = ep_ptr->header.owner_ia; - struct sockaddr_in addr; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect: r_qual %d p_size=%d\n", - r_qual, p_size); - - cm_ptr = dapli_cm_create(); - if (cm_ptr == NULL) - return DAT_INSUFFICIENT_RESOURCES; - - /* create, connect, sockopt, and exchange QP information */ - if ((cm_ptr->socket = - socket(AF_INET, SOCK_STREAM, 0)) == DAPL_INVALID_SOCKET) { - dapl_os_free(cm_ptr, sizeof(*cm_ptr)); - return DAT_INSUFFICIENT_RESOURCES; - } - - ret = dapl_config_socket(cm_ptr->socket); - if (ret < 0) { - dapl_log(DAPL_DBG_TYPE_ERR, - " socket connect: config socket %d ERR %d %s\n", - cm_ptr->socket, ret, strerror(errno)); - goto bail; - } - - dapl_os_memcpy(&addr, r_addr, sizeof(addr)); - addr.sin_port = htons(r_qual); - ret = dapl_connect_socket(cm_ptr->socket, (struct sockaddr *)&addr, - sizeof(addr)); - if (ret && ret != EAGAIN) { - dapl_log(DAPL_DBG_TYPE_ERR, - " socket connect ERROR: %s -> %s r_qual %d\n", - strerror(errno), - inet_ntoa(addr.sin_addr), (unsigned int)r_qual); - dapli_cm_destroy(cm_ptr); - return DAT_INVALID_ADDRESS; - } - - /* Send QP info, IA address, and private data */ - cm_ptr->dst.qpn = htonl(ep_ptr->qp_handle->qp_num); -#ifdef DAT_EXTENSIONS - cm_ptr->dst.qp_type = htons(ep_ptr->qp_handle->qp_type); -#endif - cm_ptr->dst.port = htons(ia_ptr->hca_ptr->port_num); - cm_ptr->dst.lid = ia_ptr->hca_ptr->ib_trans.lid; - cm_ptr->dst.gid = ia_ptr->hca_ptr->ib_trans.gid; - - /* save references */ - cm_ptr->hca = ia_ptr->hca_ptr; - cm_ptr->ep = ep_ptr; - cm_ptr->dst.ia_address = ia_ptr->hca_ptr->hca_address; - ((struct sockaddr_in *) - &cm_ptr->dst.ia_address)->sin_port = ntohs(r_qual); - - if (p_size) { - cm_ptr->dst.p_size = htonl(p_size); - dapl_os_memcpy(cm_ptr->p_data, p_data, p_size); - } - - /* connected or pending, either way results via async event */ - if (ret == 0) - dapli_socket_connected(cm_ptr, 0); - else - cm_ptr->state = SCM_CONN_PENDING; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " connect: socket %d to %s r_qual %d pending\n", - cm_ptr->socket, - inet_ntoa(addr.sin_addr), (unsigned int)r_qual); - - dapli_cm_queue(cm_ptr); - return DAT_SUCCESS; - bail: - dapl_log(DAPL_DBG_TYPE_ERR, - " socket connect ERROR: %s query lid(0x%x)/gid" - " -> %s r_qual %d\n", - strerror(errno), ntohs(cm_ptr->dst.lid), - inet_ntoa(((struct sockaddr_in *)r_addr)->sin_addr), - (unsigned int)r_qual); - - /* close socket, free cm structure */ - dapli_cm_destroy(cm_ptr); - return DAT_INTERNAL_ERROR; -} - -/* - * ACTIVE: exchange QP information, called from CR thread - */ -static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr) -{ - DAPL_EP *ep_ptr = cm_ptr->ep; - int len; - short rtu_data = htons(0x0E0F); - ib_cm_events_t event = IB_CME_DESTINATION_REJECT; - - /* read DST information into cm_ptr, overwrite SRC info */ - dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect_rtu: recv peer QP data\n"); - - len = recv(cm_ptr->socket, (char *)&cm_ptr->dst, sizeof(ib_qp_cm_t), 0); - if (len != sizeof(ib_qp_cm_t) || ntohs(cm_ptr->dst.ver) != DSCM_VER) { - dapl_log(DAPL_DBG_TYPE_ERR, - " CONN_RTU read: ERR %s, rcnt=%d, ver=%d -> %s\n", - strerror(errno), len, cm_ptr->dst.ver, - inet_ntoa(((struct sockaddr_in *) - ep_ptr->param.remote_ia_address_ptr)-> - sin_addr)); - goto bail; - } - - /* convert peer response values to host order */ - cm_ptr->dst.port = ntohs(cm_ptr->dst.port); - cm_ptr->dst.lid = ntohs(cm_ptr->dst.lid); - cm_ptr->dst.qpn = ntohl(cm_ptr->dst.qpn); -#ifdef DAT_EXTENSIONS - cm_ptr->dst.qp_type = ntohs(cm_ptr->dst.qp_type); -#endif - cm_ptr->dst.p_size = ntohl(cm_ptr->dst.p_size); - - /* save remote address information */ - dapl_os_memcpy(&ep_ptr->remote_ia_address, - &cm_ptr->dst.ia_address, - sizeof(ep_ptr->remote_ia_address)); - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " CONN_RTU: DST %s port=0x%x lid=0x%x," - " qpn=0x%x, qp_type=%d, psize=%d\n", - inet_ntoa(((struct sockaddr_in *) - &cm_ptr->dst.ia_address)->sin_addr), - cm_ptr->dst.port, cm_ptr->dst.lid, - cm_ptr->dst.qpn, cm_ptr->dst.qp_type, cm_ptr->dst.p_size); - - /* validate private data size before reading */ - if (cm_ptr->dst.p_size > IB_MAX_REP_PDATA_SIZE) { - dapl_log(DAPL_DBG_TYPE_ERR, - " CONN_RTU read: psize (%d) wrong -> %s\n", - cm_ptr->dst.p_size, inet_ntoa(((struct sockaddr_in *) - ep_ptr->param. - remote_ia_address_ptr)-> - sin_addr)); - goto bail; - } - - /* read private data into cm_handle if any present */ - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " socket connected, read private data\n"); - if (cm_ptr->dst.p_size) { - len = - recv(cm_ptr->socket, cm_ptr->p_data, cm_ptr->dst.p_size, 0); - if (len != cm_ptr->dst.p_size) { - dapl_log(DAPL_DBG_TYPE_ERR, - " CONN_RTU read pdata: ERR %s, rcnt=%d -> %s\n", - strerror(errno), len, - inet_ntoa(((struct sockaddr_in *) - ep_ptr->param. - remote_ia_address_ptr)->sin_addr)); - goto bail; - } - } - - /* check for consumer reject */ - if (cm_ptr->dst.rej) { - dapl_log(DAPL_DBG_TYPE_CM, - " CONN_RTU read: PEER REJ reason=0x%x -> %s\n", - ntohs(cm_ptr->dst.rej), - inet_ntoa(((struct sockaddr_in *) - ep_ptr->param.remote_ia_address_ptr)-> - sin_addr)); - event = IB_CME_DESTINATION_REJECT_PRIVATE_DATA; -#ifdef DAT_EXTENSIONS - if (cm_ptr->dst.qp_type == IBV_QPT_UD) - goto ud_bail; - else -#endif - goto bail; - } - - /* modify QP to RTR and then to RTS with remote info */ - dapl_os_lock(&ep_ptr->header.lock); - if (dapls_modify_qp_state(ep_ptr->qp_handle, - IBV_QPS_RTR, cm_ptr) != DAT_SUCCESS) { - dapl_log(DAPL_DBG_TYPE_ERR, - " CONN_RTU: QPS_RTR ERR %s -> %s\n", - strerror(errno), inet_ntoa(((struct sockaddr_in *) - ep_ptr->param. - remote_ia_address_ptr)-> - sin_addr)); - dapl_os_unlock(&ep_ptr->header.lock); - goto bail; - } - if (dapls_modify_qp_state(ep_ptr->qp_handle, - IBV_QPS_RTS, cm_ptr) != DAT_SUCCESS) { - dapl_log(DAPL_DBG_TYPE_ERR, - " CONN_RTU: QPS_RTS ERR %s -> %s\n", - strerror(errno), inet_ntoa(((struct sockaddr_in *) - ep_ptr->param. - remote_ia_address_ptr)-> - sin_addr)); - dapl_os_unlock(&ep_ptr->header.lock); - goto bail; - } - dapl_os_unlock(&ep_ptr->header.lock); - dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect_rtu: send RTU\n"); - - /* complete handshake after final QP state change */ - if (send(cm_ptr->socket, (char *)&rtu_data, sizeof(rtu_data), 0) == -1) { - dapl_log(DAPL_DBG_TYPE_ERR, - " CONN_RTU: write error = %s\n", strerror(errno)); - goto bail; - } - /* init cm_handle and post the event with private data */ - cm_ptr->state = SCM_CONNECTED; - event = IB_CME_CONNECTED; - dapl_dbg_log(DAPL_DBG_TYPE_EP, " ACTIVE: connected!\n"); - -#ifdef DAT_EXTENSIONS -ud_bail: - if (cm_ptr->dst.qp_type == IBV_QPT_UD) { - DAT_IB_EXTENSION_EVENT_DATA xevent; - - /* post EVENT, modify_qp created ah */ - xevent.status = 0; - xevent.type = DAT_IB_UD_REMOTE_AH; - xevent.remote_ah.ah = cm_ptr->ah; - xevent.remote_ah.qpn = cm_ptr->dst.qpn; - dapl_os_memcpy(&xevent.remote_ah.ia_addr, - &cm_ptr->dst.ia_address, - sizeof(cm_ptr->dst.ia_address)); - - if (event == IB_CME_CONNECTED) - event = DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED; - else - event = DAT_IB_UD_CONNECTION_REJECT_EVENT; - - dapls_evd_post_connection_event_ext((DAPL_EVD *) ep_ptr->param. - connect_evd_handle, - event, - (DAT_EP_HANDLE) ep_ptr, - (DAT_COUNT) cm_ptr->dst.p_size, - (DAT_PVOID *) cm_ptr->p_data, - (DAT_PVOID *) &xevent); - - /* done with socket, don't destroy cm_ptr, need pdata */ - closesocket(cm_ptr->socket); - cm_ptr->socket = DAPL_INVALID_SOCKET; - cm_ptr->state = SCM_RELEASED; - } else -#endif - { - ep_ptr->cm_handle = cm_ptr; /* only RC, multi CR's on UD */ - dapl_evd_connection_callback(cm_ptr, - IB_CME_CONNECTED, - cm_ptr->p_data, ep_ptr); - } - return; - -bail: - /* close socket, and post error event */ - dapls_ib_reinit_ep(ep_ptr); /* reset QP state */ - closesocket(cm_ptr->socket); - cm_ptr->socket = DAPL_INVALID_SOCKET; - dapl_evd_connection_callback(NULL, event, cm_ptr->p_data, ep_ptr); -} - -/* - * PASSIVE: Create socket, listen, accept, exchange QP information - */ -DAT_RETURN -dapli_socket_listen(DAPL_IA * ia_ptr, DAT_CONN_QUAL serviceID, DAPL_SP * sp_ptr) -{ - struct sockaddr_in addr; - ib_cm_srvc_handle_t cm_ptr = NULL; - int opt = 1; - DAT_RETURN dat_status = DAT_SUCCESS; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " listen(ia_ptr %p ServiceID %d sp_ptr %p)\n", - ia_ptr, serviceID, sp_ptr); - - cm_ptr = dapli_cm_create(); - if (cm_ptr == NULL) - return DAT_INSUFFICIENT_RESOURCES; - - cm_ptr->sp = sp_ptr; - cm_ptr->hca = ia_ptr->hca_ptr; - - /* bind, listen, set sockopt, accept, exchange data */ - if ((cm_ptr->socket = - socket(AF_INET, SOCK_STREAM, 0)) == DAPL_INVALID_SOCKET) { - dapl_log(DAPL_DBG_TYPE_ERR, " ERR: listen socket create: %s\n", - strerror(errno)); - dat_status = DAT_INSUFFICIENT_RESOURCES; - goto bail; - } - - setsockopt(cm_ptr->socket, SOL_SOCKET, SO_REUSEADDR, - (char *)&opt, sizeof(opt)); - addr.sin_port = htons(serviceID); - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - - if ((bind(cm_ptr->socket, (struct sockaddr *)&addr, sizeof(addr)) < 0) - || (listen(cm_ptr->socket, 128) < 0)) { - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " listen: ERROR %s on conn_qual 0x%x\n", - strerror(errno), serviceID); - if (errno == EADDRINUSE) - dat_status = DAT_CONN_QUAL_IN_USE; - else - dat_status = DAT_CONN_QUAL_UNAVAILABLE; - goto bail; - } - - /* set cm_handle for this service point, save listen socket */ - sp_ptr->cm_srvc_handle = cm_ptr; - - /* queue up listen socket to process inbound CR's */ - cm_ptr->state = SCM_LISTEN; - dapli_cm_queue(cm_ptr); - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " listen: qual 0x%x cr %p s_fd %d\n", - ntohs(serviceID), cm_ptr, cm_ptr->socket); - - return dat_status; - bail: - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " listen: ERROR on conn_qual 0x%x\n", serviceID); - dapli_cm_destroy(cm_ptr); - return dat_status; -} - -/* - * PASSIVE: accept socket - */ -static void dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr) -{ - dp_ib_cm_handle_t acm_ptr; - int len; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, " socket_accept\n"); - - /* - * Accept all CR's on this port to avoid half-connection (SYN_RCV) - * stalls with many to one connection storms - */ - do { - /* Allocate accept CM and initialize */ - if ((acm_ptr = dapli_cm_create()) == NULL) - return; - - acm_ptr->sp = cm_ptr->sp; - acm_ptr->hca = cm_ptr->hca; - - len = sizeof(acm_ptr->dst.ia_address); - acm_ptr->socket = accept(cm_ptr->socket, - (struct sockaddr *) - &acm_ptr->dst.ia_address, - (socklen_t *) & len); - if (acm_ptr->socket == DAPL_INVALID_SOCKET) { - dapl_log(DAPL_DBG_TYPE_ERR, - " accept: ERR %s on FD %d l_cr %p\n", - strerror(errno), cm_ptr->socket, cm_ptr); - dapli_cm_destroy(acm_ptr); - return; - } - - acm_ptr->state = SCM_ACCEPTING; - dapli_cm_queue(acm_ptr); - - } while (dapl_poll(cm_ptr->socket, DAPL_FD_READ) == DAPL_FD_READ); -} - -/* - * PASSIVE: receive peer QP information, private data, post cr_event - */ -static void dapli_socket_accept_data(ib_cm_srvc_handle_t acm_ptr) -{ - int len; - void *p_data = NULL; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, " socket accepted, read QP data\n"); - - /* read in DST QP info, IA address. check for private data */ - len = - recv(acm_ptr->socket, (char *)&acm_ptr->dst, sizeof(ib_qp_cm_t), 0); - if (len != sizeof(ib_qp_cm_t) || ntohs(acm_ptr->dst.ver) != DSCM_VER) { - dapl_log(DAPL_DBG_TYPE_ERR, - " accept read: ERR %s, rcnt=%d, ver=%d\n", - strerror(errno), len, ntohs(acm_ptr->dst.ver)); - goto bail; - } - - /* convert accepted values to host order */ - acm_ptr->dst.port = ntohs(acm_ptr->dst.port); - acm_ptr->dst.lid = ntohs(acm_ptr->dst.lid); - acm_ptr->dst.qpn = ntohl(acm_ptr->dst.qpn); -#ifdef DAT_EXTENSIONS - acm_ptr->dst.qp_type = ntohs(acm_ptr->dst.qp_type); -#endif - acm_ptr->dst.p_size = ntohl(acm_ptr->dst.p_size); - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " accept: DST %s port=0x%x lid=0x%x, qpn=0x%x, psize=%d\n", - inet_ntoa(((struct sockaddr_in *)&acm_ptr->dst. - ia_address)->sin_addr), acm_ptr->dst.port, - acm_ptr->dst.lid, acm_ptr->dst.qpn, acm_ptr->dst.p_size); - - /* validate private data size before reading */ - if (acm_ptr->dst.p_size > IB_MAX_REQ_PDATA_SIZE) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " accept read: psize (%d) wrong\n", - acm_ptr->dst.p_size); - goto bail; - } - - dapl_dbg_log(DAPL_DBG_TYPE_EP, " socket accepted, read private data\n"); - - /* read private data into cm_handle if any present */ - if (acm_ptr->dst.p_size) { - len = - recv(acm_ptr->socket, acm_ptr->p_data, acm_ptr->dst.p_size, - 0); - if (len != acm_ptr->dst.p_size) { - dapl_log(DAPL_DBG_TYPE_ERR, - " accept read pdata: ERR %s, rcnt=%d\n", - strerror(errno), len); - goto bail; - } - dapl_dbg_log(DAPL_DBG_TYPE_EP, " accept: psize=%d read\n", len); - p_data = acm_ptr->p_data; - } - - acm_ptr->state = SCM_ACCEPTING_DATA; - -#ifdef DAT_EXTENSIONS - if (acm_ptr->dst.qp_type == IBV_QPT_UD) { - DAT_IB_EXTENSION_EVENT_DATA xevent; - - /* post EVENT, modify_qp created ah */ - xevent.status = 0; - xevent.type = DAT_IB_UD_CONNECT_REQUEST; - - dapls_evd_post_cr_event_ext(acm_ptr->sp, - DAT_IB_UD_CONNECTION_REQUEST_EVENT, - acm_ptr, - (DAT_COUNT) acm_ptr->dst.p_size, - (DAT_PVOID *) acm_ptr->p_data, - (DAT_PVOID *) & xevent); - } else -#endif - /* trigger CR event and return SUCCESS */ - dapls_cr_callback(acm_ptr, - IB_CME_CONNECTION_REQUEST_PENDING, - p_data, acm_ptr->sp); - return; - bail: - /* close socket, free cm structure, active will see socket close as reject */ - dapli_cm_destroy(acm_ptr); - return; -} - -/* - * PASSIVE: consumer accept, send local QP information, private data, - * queue on work thread to receive RTU information to avoid blocking - * user thread. - */ -DAT_RETURN -dapli_socket_accept_usr(DAPL_EP * ep_ptr, - DAPL_CR * cr_ptr, DAT_COUNT p_size, DAT_PVOID p_data) -{ - DAPL_IA *ia_ptr = ep_ptr->header.owner_ia; - dp_ib_cm_handle_t cm_ptr = cr_ptr->ib_cm_handle; - ib_qp_cm_t local; - struct iovec iov[2]; - int len; - - if (p_size > IB_MAX_REP_PDATA_SIZE) - return DAT_LENGTH_ERROR; - - /* must have a accepted socket */ - if (cm_ptr->socket == DAPL_INVALID_SOCKET) - return DAT_INTERNAL_ERROR; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " ACCEPT_USR: remote port=0x%x lid=0x%x" - " qpn=0x%x qp_type %d, psize=%d\n", - cm_ptr->dst.port, cm_ptr->dst.lid, - cm_ptr->dst.qpn, cm_ptr->dst.qp_type, cm_ptr->dst.p_size); - -#ifdef DAT_EXTENSIONS - if (cm_ptr->dst.qp_type == IBV_QPT_UD && - ep_ptr->qp_handle->qp_type != IBV_QPT_UD) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " ACCEPT_USR: ERR remote QP is UD," - ", but local QP is not\n"); - return (DAT_INVALID_HANDLE | DAT_INVALID_HANDLE_EP); - } -#endif - - /* modify QP to RTR and then to RTS with remote info already read */ - dapl_os_lock(&ep_ptr->header.lock); - if (dapls_modify_qp_state(ep_ptr->qp_handle, - IBV_QPS_RTR, cm_ptr) != DAT_SUCCESS) { - dapl_log(DAPL_DBG_TYPE_ERR, - " ACCEPT_USR: QPS_RTR ERR %s -> %s\n", - strerror(errno), inet_ntoa(((struct sockaddr_in *) - &cm_ptr->dst.ia_address)-> - sin_addr)); - dapl_os_unlock(&ep_ptr->header.lock); - goto bail; - } - if (dapls_modify_qp_state(ep_ptr->qp_handle, - IBV_QPS_RTS, cm_ptr) != DAT_SUCCESS) { - dapl_log(DAPL_DBG_TYPE_ERR, - " ACCEPT_USR: QPS_RTS ERR %s -> %s\n", - strerror(errno), inet_ntoa(((struct sockaddr_in *) - &cm_ptr->dst.ia_address)-> - sin_addr)); - dapl_os_unlock(&ep_ptr->header.lock); - goto bail; - } - dapl_os_unlock(&ep_ptr->header.lock); - - /* save remote address information */ - dapl_os_memcpy(&ep_ptr->remote_ia_address, - &cm_ptr->dst.ia_address, - sizeof(ep_ptr->remote_ia_address)); - - /* send our QP info, IA address, pdata. Don't overwrite dst data */ - local.ver = htons(DSCM_VER); - local.rej = 0; - local.qpn = htonl(ep_ptr->qp_handle->qp_num); - local.qp_type = htons(ep_ptr->qp_handle->qp_type); - local.port = htons(ia_ptr->hca_ptr->port_num); - local.lid = ia_ptr->hca_ptr->ib_trans.lid; - local.gid = ia_ptr->hca_ptr->ib_trans.gid; - local.ia_address = ia_ptr->hca_ptr->hca_address; - ((struct sockaddr_in *)&local.ia_address)->sin_port = - ntohs(cm_ptr->sp->conn_qual); - - local.p_size = htonl(p_size); - iov[0].iov_base = (void *)&local; - iov[0].iov_len = sizeof(ib_qp_cm_t); - if (p_size) { - iov[1].iov_base = p_data; - iov[1].iov_len = p_size; - len = writev(cm_ptr->socket, iov, 2); - } else { - len = writev(cm_ptr->socket, iov, 1); - } - - if (len != (p_size + sizeof(ib_qp_cm_t))) { - dapl_log(DAPL_DBG_TYPE_ERR, - " ACCEPT_USR: ERR %s, wcnt=%d -> %s\n", - strerror(errno), len, inet_ntoa(((struct sockaddr_in *) - &cm_ptr->dst. - ia_address)-> - sin_addr)); - goto bail; - } - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " ACCEPT_USR: local port=0x%x lid=0x%x" - " qpn=0x%x psize=%d\n", - ntohs(local.port), ntohs(local.lid), - ntohl(local.qpn), ntohl(local.p_size)); - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " ACCEPT_USR SRC GID subnet %016llx id %016llx\n", - (unsigned long long) - htonll(local.gid.global.subnet_prefix), - (unsigned long long) - htonll(local.gid.global.interface_id)); - - /* save state and reference to EP, queue for RTU data */ - cm_ptr->ep = ep_ptr; - cm_ptr->hca = ia_ptr->hca_ptr; - cm_ptr->state = SCM_ACCEPTED; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, " PASSIVE: accepted!\n"); - return DAT_SUCCESS; - bail: - dapli_cm_destroy(cm_ptr); - dapls_ib_reinit_ep(ep_ptr); /* reset QP state */ - return DAT_INTERNAL_ERROR; -} - -/* - * PASSIVE: read RTU from active peer, post CONN event - */ -void dapli_socket_accept_rtu(dp_ib_cm_handle_t cm_ptr) -{ - int len; - short rtu_data = 0; - - /* complete handshake after final QP state change */ - len = recv(cm_ptr->socket, (char *)&rtu_data, sizeof(rtu_data), 0); - if (len != sizeof(rtu_data) || ntohs(rtu_data) != 0x0e0f) { - dapl_log(DAPL_DBG_TYPE_ERR, - " ACCEPT_RTU: ERR %s, rcnt=%d rdata=%x\n", - strerror(errno), len, ntohs(rtu_data), - inet_ntoa(((struct sockaddr_in *) - &cm_ptr->dst.ia_address)->sin_addr)); - goto bail; - } - - /* save state and reference to EP, queue for disc event */ - cm_ptr->state = SCM_CONNECTED; - - /* final data exchange if remote QP state is good to go */ - dapl_dbg_log(DAPL_DBG_TYPE_EP, " PASSIVE: connected!\n"); - -#ifdef DAT_EXTENSIONS - if (cm_ptr->dst.qp_type == IBV_QPT_UD) { - DAT_IB_EXTENSION_EVENT_DATA xevent; - - /* post EVENT, modify_qp created ah */ - xevent.status = 0; - xevent.type = DAT_IB_UD_PASSIVE_REMOTE_AH; - xevent.remote_ah.ah = cm_ptr->ah; - xevent.remote_ah.qpn = cm_ptr->dst.qpn; - dapl_os_memcpy(&xevent.remote_ah.ia_addr, - &cm_ptr->dst.ia_address, - sizeof(cm_ptr->dst.ia_address)); - - dapls_evd_post_connection_event_ext((DAPL_EVD *) cm_ptr->ep-> - param.connect_evd_handle, - DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED, - (DAT_EP_HANDLE) cm_ptr->ep, - (DAT_COUNT) cm_ptr->dst.p_size, - (DAT_PVOID *) cm_ptr->p_data, - (DAT_PVOID *) &xevent); - - /* done with socket, don't destroy cm_ptr, need pdata */ - closesocket(cm_ptr->socket); - cm_ptr->socket = DAPL_INVALID_SOCKET; - cm_ptr->state = SCM_RELEASED; - } else -#endif - dapls_cr_callback(cm_ptr, IB_CME_CONNECTED, NULL, cm_ptr->sp); - return; - bail: - dapls_ib_reinit_ep(cm_ptr->ep); /* reset QP state */ - dapli_cm_destroy(cm_ptr); - dapls_cr_callback(cm_ptr, IB_CME_DESTINATION_REJECT, NULL, cm_ptr->sp); -} - -/* - * dapls_ib_connect - * - * Initiate a connection with the passive listener on another node - * - * Input: - * ep_handle, - * remote_ia_address, - * remote_conn_qual, - * prd_size size of private data and structure - * prd_prt pointer to private data structure - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INVALID_PARAMETER - * - */ -DAT_RETURN -dapls_ib_connect(IN DAT_EP_HANDLE ep_handle, - IN DAT_IA_ADDRESS_PTR remote_ia_address, - IN DAT_CONN_QUAL remote_conn_qual, - IN DAT_COUNT private_data_size, IN void *private_data) -{ - DAPL_EP *ep_ptr; - ib_qp_handle_t qp_ptr; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " connect(ep_handle %p ....)\n", ep_handle); - - ep_ptr = (DAPL_EP *) ep_handle; - qp_ptr = ep_ptr->qp_handle; - - return (dapli_socket_connect(ep_ptr, remote_ia_address, - remote_conn_qual, - private_data_size, private_data)); -} - -/* - * dapls_ib_disconnect - * - * Disconnect an EP - * - * Input: - * ep_handle, - * disconnect_flags - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - */ -DAT_RETURN -dapls_ib_disconnect(IN DAPL_EP * ep_ptr, IN DAT_CLOSE_FLAGS close_flags) -{ - dapl_dbg_log(DAPL_DBG_TYPE_EP, - "dapls_ib_disconnect(ep_handle %p ....)\n", ep_ptr); - - /* reinit to modify QP state */ - dapls_ib_reinit_ep(ep_ptr); - - if (ep_ptr->cm_handle == NULL || - ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED) - return DAT_SUCCESS; - else - return (dapli_socket_disconnect(ep_ptr->cm_handle)); -} - -/* - * dapls_ib_disconnect_clean - * - * Clean up outstanding connection data. This routine is invoked - * after the final disconnect callback has occurred. Only on the - * ACTIVE side of a connection. It is also called if dat_ep_connect - * times out using the consumer supplied timeout value. - * - * Input: - * ep_ptr DAPL_EP - * active Indicates active side of connection - * - * Output: - * none - * - * Returns: - * void - * - */ -void -dapls_ib_disconnect_clean(IN DAPL_EP * ep_ptr, - IN DAT_BOOLEAN active, - IN const ib_cm_events_t ib_cm_event) -{ - /* NOTE: SCM will only initialize cm_handle with RC type - * - * For UD there can many in-flight CR's so you - * cannot cleanup timed out CR's with EP reference - * alone since they share the same EP. The common - * code that handles connection timeout logic needs - * updated for UD support. - */ - if (ep_ptr->cm_handle) - dapli_cm_destroy(ep_ptr->cm_handle); - - return; -} - -/* - * dapl_ib_setup_conn_listener - * - * Have the CM set up a connection listener. - * - * Input: - * ibm_hca_handle HCA handle - * qp_handle QP handle - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INTERNAL_ERROR - * DAT_CONN_QUAL_UNAVAILBLE - * DAT_CONN_QUAL_IN_USE - * - */ -DAT_RETURN -dapls_ib_setup_conn_listener(IN DAPL_IA * ia_ptr, - IN DAT_UINT64 ServiceID, IN DAPL_SP * sp_ptr) -{ - return (dapli_socket_listen(ia_ptr, ServiceID, sp_ptr)); -} - -/* - * dapl_ib_remove_conn_listener - * - * Have the CM remove a connection listener. - * - * Input: - * ia_handle IA handle - * ServiceID IB Channel Service ID - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_STATE - * - */ -DAT_RETURN -dapls_ib_remove_conn_listener(IN DAPL_IA * ia_ptr, IN DAPL_SP * sp_ptr) -{ - ib_cm_srvc_handle_t cm_ptr = sp_ptr->cm_srvc_handle; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - "dapls_ib_remove_conn_listener(ia_ptr %p sp_ptr %p cm_ptr %p)\n", - ia_ptr, sp_ptr, cm_ptr); - - /* close accepted socket, free cm_srvc_handle and return */ - if (cm_ptr != NULL) { - if (cm_ptr->socket != DAPL_INVALID_SOCKET) { - shutdown(cm_ptr->socket, SHUT_RDWR); - closesocket(cm_ptr->socket); - cm_ptr->socket = DAPL_INVALID_SOCKET; - } - /* cr_thread will free */ - cm_ptr->state = SCM_DESTROY; - sp_ptr->cm_srvc_handle = NULL; - if (send(cm_ptr->hca->ib_trans.scm[1], - "w", sizeof "w", 0) == -1) - dapl_log(DAPL_DBG_TYPE_CM, - " cm_destroy: thread wakeup error = %s\n", - strerror(errno)); - } - return DAT_SUCCESS; -} - -/* - * dapls_ib_accept_connection - * - * Perform necessary steps to accept a connection - * - * Input: - * cr_handle - * ep_handle - * private_data_size - * private_data - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INTERNAL_ERROR - * - */ -DAT_RETURN -dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle, - IN DAT_EP_HANDLE ep_handle, - IN DAT_COUNT p_size, IN const DAT_PVOID p_data) -{ - DAPL_CR *cr_ptr; - DAPL_EP *ep_ptr; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - "dapls_ib_accept_connection(cr %p ep %p prd %p,%d)\n", - cr_handle, ep_handle, p_data, p_size); - - cr_ptr = (DAPL_CR *) cr_handle; - ep_ptr = (DAPL_EP *) ep_handle; - - /* allocate and attach a QP if necessary */ - if (ep_ptr->qp_state == DAPL_QP_STATE_UNATTACHED) { - DAT_RETURN status; - status = dapls_ib_qp_alloc(ep_ptr->header.owner_ia, - ep_ptr, ep_ptr); - if (status != DAT_SUCCESS) - return status; - } - return (dapli_socket_accept_usr(ep_ptr, cr_ptr, p_size, p_data)); -} - -/* - * dapls_ib_reject_connection - * - * Reject a connection - * - * Input: - * cr_handle - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INTERNAL_ERROR - * - */ -DAT_RETURN -dapls_ib_reject_connection(IN dp_ib_cm_handle_t cm_ptr, - IN int reason, - IN DAT_COUNT psize, IN const DAT_PVOID pdata) -{ - struct iovec iov[2]; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " reject(cm %p reason %x, pdata %p, psize %d)\n", - cm_ptr, reason, pdata, psize); - - if (psize > IB_MAX_REJ_PDATA_SIZE) - return DAT_LENGTH_ERROR; - - /* write reject data to indicate reject */ - if (cm_ptr->socket != DAPL_INVALID_SOCKET) { - cm_ptr->dst.rej = (uint16_t) reason; - cm_ptr->dst.rej = htons(cm_ptr->dst.rej); - cm_ptr->dst.p_size = htonl(psize); - /* get qp_type from request */ - cm_ptr->dst.qp_type = ntohs(cm_ptr->dst.qp_type); - - iov[0].iov_base = (void *)&cm_ptr->dst; - iov[0].iov_len = sizeof(ib_qp_cm_t); - if (psize) { - iov[1].iov_base = pdata; - iov[1].iov_len = psize; - writev(cm_ptr->socket, iov, 2); - } else { - writev(cm_ptr->socket, iov, 1); - } - - shutdown(cm_ptr->socket, SHUT_RDWR); - closesocket(cm_ptr->socket); - cm_ptr->socket = DAPL_INVALID_SOCKET; - } - - /* cr_thread will destroy CR */ - cm_ptr->state = SCM_DESTROY; - if (send(cm_ptr->hca->ib_trans.scm[1], "w", sizeof "w", 0) == -1) - dapl_log(DAPL_DBG_TYPE_CM, - " cm_destroy: thread wakeup error = %s\n", - strerror(errno)); - return DAT_SUCCESS; -} - -/* - * dapls_ib_cm_remote_addr - * - * Obtain the remote IP address given a connection - * - * Input: - * cr_handle - * - * Output: - * remote_ia_address: where to place the remote address - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_HANDLE - * - */ -DAT_RETURN -dapls_ib_cm_remote_addr(IN DAT_HANDLE dat_handle, - OUT DAT_SOCK_ADDR6 * remote_ia_address) -{ - DAPL_HEADER *header; - dp_ib_cm_handle_t ib_cm_handle; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - "dapls_ib_cm_remote_addr(dat_handle %p, ....)\n", - dat_handle); - - header = (DAPL_HEADER *) dat_handle; - - if (header->magic == DAPL_MAGIC_EP) - ib_cm_handle = ((DAPL_EP *) dat_handle)->cm_handle; - else if (header->magic == DAPL_MAGIC_CR) - ib_cm_handle = ((DAPL_CR *) dat_handle)->ib_cm_handle; - else - return DAT_INVALID_HANDLE; - - dapl_os_memcpy(remote_ia_address, - &ib_cm_handle->dst.ia_address, sizeof(DAT_SOCK_ADDR6)); - - return DAT_SUCCESS; -} - -/* - * dapls_ib_private_data_size - * - * Return the size of private data given a connection op type - * - * Input: - * prd_ptr private data pointer - * conn_op connection operation type - * - * If prd_ptr is NULL, this is a query for the max size supported by - * the provider, otherwise it is the actual size of the private data - * contained in prd_ptr. - * - * - * Output: - * None - * - * Returns: - * length of private data - * - */ -int dapls_ib_private_data_size(IN DAPL_PRIVATE * prd_ptr, - IN DAPL_PDATA_OP conn_op, IN DAPL_HCA * hca_ptr) -{ - int size; - - switch (conn_op) { - case DAPL_PDATA_CONN_REQ: - { - size = IB_MAX_REQ_PDATA_SIZE; - break; - } - case DAPL_PDATA_CONN_REP: - { - size = IB_MAX_REP_PDATA_SIZE; - break; - } - case DAPL_PDATA_CONN_REJ: - { - size = IB_MAX_REJ_PDATA_SIZE; - break; - } - case DAPL_PDATA_CONN_DREQ: - { - size = IB_MAX_DREQ_PDATA_SIZE; - break; - } - case DAPL_PDATA_CONN_DREP: - { - size = IB_MAX_DREP_PDATA_SIZE; - break; - } - default: - { - size = 0; - } - - } /* end case */ - - return size; -} - -/* - * Map all socket CM event codes to the DAT equivelent. - */ -#define DAPL_IB_EVENT_CNT 11 - -static struct ib_cm_event_map { - const ib_cm_events_t ib_cm_event; - DAT_EVENT_NUMBER dat_event_num; -} ib_cm_event_map[DAPL_IB_EVENT_CNT] = { - /* 00 */ { - IB_CME_CONNECTED, DAT_CONNECTION_EVENT_ESTABLISHED}, - /* 01 */ { - IB_CME_DISCONNECTED, DAT_CONNECTION_EVENT_DISCONNECTED}, - /* 02 */ { - IB_CME_DISCONNECTED_ON_LINK_DOWN, - DAT_CONNECTION_EVENT_DISCONNECTED}, - /* 03 */ { - IB_CME_CONNECTION_REQUEST_PENDING, DAT_CONNECTION_REQUEST_EVENT}, - /* 04 */ { - IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA, - DAT_CONNECTION_REQUEST_EVENT}, - /* 05 */ { - IB_CME_DESTINATION_REJECT, - DAT_CONNECTION_EVENT_NON_PEER_REJECTED}, - /* 06 */ { - IB_CME_DESTINATION_REJECT_PRIVATE_DATA, - DAT_CONNECTION_EVENT_PEER_REJECTED}, - /* 07 */ { - IB_CME_DESTINATION_UNREACHABLE, DAT_CONNECTION_EVENT_UNREACHABLE}, - /* 08 */ { - IB_CME_TOO_MANY_CONNECTION_REQUESTS, - DAT_CONNECTION_EVENT_NON_PEER_REJECTED}, - /* 09 */ { - IB_CME_LOCAL_FAILURE, DAT_CONNECTION_EVENT_BROKEN}, - /* 10 */ { - IB_CM_LOCAL_FAILURE, DAT_CONNECTION_EVENT_BROKEN} -}; - -/* - * dapls_ib_get_cm_event - * - * Return a DAT connection event given a provider CM event. - * - * Input: - * dat_event_num DAT event we need an equivelent CM event for - * - * Output: - * none - * - * Returns: - * ib_cm_event of translated DAPL value - */ -DAT_EVENT_NUMBER -dapls_ib_get_dat_event(IN const ib_cm_events_t ib_cm_event, - IN DAT_BOOLEAN active) -{ - DAT_EVENT_NUMBER dat_event_num; - int i; - - active = active; - - if (ib_cm_event > IB_CM_LOCAL_FAILURE) - return (DAT_EVENT_NUMBER) 0; - - dat_event_num = 0; - for (i = 0; i < DAPL_IB_EVENT_CNT; i++) { - if (ib_cm_event == ib_cm_event_map[i].ib_cm_event) { - dat_event_num = ib_cm_event_map[i].dat_event_num; - break; - } - } - dapl_dbg_log(DAPL_DBG_TYPE_CALLBACK, - "dapls_ib_get_dat_event: event translate(%s) ib=0x%x dat=0x%x\n", - active ? "active" : "passive", ib_cm_event, dat_event_num); - - return dat_event_num; -} - -/* - * dapls_ib_get_dat_event - * - * Return a DAT connection event given a provider CM event. - * - * Input: - * ib_cm_event event provided to the dapl callback routine - * active switch indicating active or passive connection - * - * Output: - * none - * - * Returns: - * DAT_EVENT_NUMBER of translated provider value - */ -ib_cm_events_t dapls_ib_get_cm_event(IN DAT_EVENT_NUMBER dat_event_num) -{ - ib_cm_events_t ib_cm_event; - int i; - - ib_cm_event = 0; - for (i = 0; i < DAPL_IB_EVENT_CNT; i++) { - if (dat_event_num == ib_cm_event_map[i].dat_event_num) { - ib_cm_event = ib_cm_event_map[i].ib_cm_event; - break; - } - } - return ib_cm_event; -} - -/* outbound/inbound CR processing thread to avoid blocking applications */ -void cr_thread(void *arg) -{ - struct dapl_hca *hca_ptr = arg; - dp_ib_cm_handle_t cr, next_cr; - int opt, ret; - socklen_t opt_len; - char rbuf[2]; - struct dapl_fd_set *set; - enum DAPL_FD_EVENTS event; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cr_thread: ENTER hca %p\n", hca_ptr); - set = dapl_alloc_fd_set(); - if (!set) - goto out; - - dapl_os_lock(&hca_ptr->ib_trans.lock); - hca_ptr->ib_trans.cr_state = IB_THREAD_RUN; - - while (1) { - dapl_fd_zero(set); - dapl_fd_set(hca_ptr->ib_trans.scm[0], set, DAPL_FD_READ); - - if (!dapl_llist_is_empty(&hca_ptr->ib_trans.list)) - next_cr = dapl_llist_peek_head(&hca_ptr->ib_trans.list); - else - next_cr = NULL; - - while (next_cr) { - cr = next_cr; - next_cr = dapl_llist_next_entry(&hca_ptr->ib_trans.list, - (DAPL_LLIST_ENTRY *) & - cr->entry); - if (cr->state == SCM_DESTROY - || hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) { - dapl_llist_remove_entry(&hca_ptr->ib_trans.list, - (DAPL_LLIST_ENTRY *) & - cr->entry); - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " CR FREE: %p ep=%p st=%d sock=%d\n", - cr, cr->ep, cr->state, cr->socket); - dapl_os_free(cr, sizeof(*cr)); - continue; - } - if (cr->socket == DAPL_INVALID_SOCKET) - continue; - - event = (cr->state == SCM_CONN_PENDING) ? - DAPL_FD_WRITE : DAPL_FD_READ; - if (dapl_fd_set(cr->socket, set, event)) { - dapl_log(DAPL_DBG_TYPE_ERR, - " cr_thread: DESTROY CR st=%d fd %d" - " -> %s\n", cr->state, cr->socket, - inet_ntoa(((struct sockaddr_in *) - &cr->dst.ia_address)-> - sin_addr)); - dapli_cm_destroy(cr); - continue; - } - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " poll cr=%p, socket=%d\n", cr, - cr->socket); - dapl_os_unlock(&hca_ptr->ib_trans.lock); - - ret = dapl_poll(cr->socket, event); - - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " poll ret=0x%x cr->state=%d socket=%d\n", - ret, cr->state, cr->socket); - - /* data on listen, qp exchange, and on disc req */ - if (ret == DAPL_FD_READ) { - if (cr->socket != DAPL_INVALID_SOCKET) { - switch (cr->state) { - case SCM_LISTEN: - dapli_socket_accept(cr); - break; - case SCM_ACCEPTING: - dapli_socket_accept_data(cr); - break; - case SCM_ACCEPTED: - dapli_socket_accept_rtu(cr); - break; - case SCM_RTU_PENDING: - dapli_socket_connect_rtu(cr); - break; - case SCM_CONNECTED: - dapli_socket_disconnect(cr); - break; - default: - break; - } - } - /* connect socket is writable, check status */ - } else if (ret == DAPL_FD_WRITE || - (cr->state == SCM_CONN_PENDING && - ret == DAPL_FD_ERROR)) { - opt = 0; - opt_len = sizeof(opt); - ret = getsockopt(cr->socket, SOL_SOCKET, - SO_ERROR, (char *)&opt, - &opt_len); - if (!ret) - dapli_socket_connected(cr, opt); - else - dapli_socket_connected(cr, errno); - - /* POLLUP, ERR, NVAL, or poll error - DISC */ - } else if (ret < 0 || ret == DAPL_FD_ERROR) { - dapl_log(DAPL_DBG_TYPE_WARN, - " poll=%d cr->st=%s sk=%d ep %p, %d\n", - ret, dapl_cm_state_str(cr->state), - cr->socket, cr->ep, - cr->ep ? cr->ep->param.ep_state:0); - dapli_socket_disconnect(cr); - } - dapl_os_lock(&hca_ptr->ib_trans.lock); - } - - /* set to exit and all resources destroyed */ - if ((hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) && - (dapl_llist_is_empty(&hca_ptr->ib_trans.list))) - break; - - dapl_os_unlock(&hca_ptr->ib_trans.lock); - dapl_select(set); - - /* if pipe used to wakeup, consume */ - while (dapl_poll(hca_ptr->ib_trans.scm[0], - DAPL_FD_READ) == DAPL_FD_READ) { - if (recv(hca_ptr->ib_trans.scm[0], rbuf, 2, 0) == -1) - dapl_log(DAPL_DBG_TYPE_CM, - " cr_thread: read pipe error = %s\n", - strerror(errno)); - } - dapl_os_lock(&hca_ptr->ib_trans.lock); - - /* set to exit and all resources destroyed */ - if ((hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) && - (dapl_llist_is_empty(&hca_ptr->ib_trans.list))) - break; - } - - dapl_os_unlock(&hca_ptr->ib_trans.lock); - free(set); - out: - hca_ptr->ib_trans.cr_state = IB_THREAD_EXIT; - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cr_thread(hca %p) exit\n", hca_ptr); -} - - -#ifdef DAPL_COUNTERS -/* Debug aid: List all Connections in process and state */ -void dapls_print_cm_list(IN DAPL_IA *ia_ptr) -{ - /* Print in process CR's for this IA, if debug type set */ - int i = 0; - dp_ib_cm_handle_t cr, next_cr; - - dapl_os_lock(&ia_ptr->hca_ptr->ib_trans.lock); - if (!dapl_llist_is_empty((DAPL_LLIST_HEAD*) - &ia_ptr->hca_ptr->ib_trans.list)) - next_cr = dapl_llist_peek_head((DAPL_LLIST_HEAD*) - &ia_ptr->hca_ptr->ib_trans.list); - else - next_cr = NULL; - - printf("\n DAPL IA CONNECTIONS IN PROCESS:\n"); - while (next_cr) { - cr = next_cr; - next_cr = dapl_llist_next_entry((DAPL_LLIST_HEAD*) - &ia_ptr->hca_ptr->ib_trans.list, - (DAPL_LLIST_ENTRY*)&cr->entry); - - printf( " CONN[%d]: sp %p ep %p sock %d %s %s %s %s %d\n", - i, cr->sp, cr->ep, cr->socket, - cr->dst.qp_type == IBV_QPT_RC ? "RC" : "UD", - dapl_cm_state_str(cr->state), - cr->sp ? "<-" : "->", - cr->state == SCM_LISTEN ? - inet_ntoa(((struct sockaddr_in *) - &ia_ptr->hca_ptr->hca_address)->sin_addr) : - inet_ntoa(((struct sockaddr_in *) - &cr->dst.ia_address)->sin_addr), - cr->sp ? (int)cr->sp->conn_qual : - ntohs(((struct sockaddr_in *) - &cr->dst.ia_address)->sin_port)); - i++; - } - printf("\n"); - dapl_os_unlock(&ia_ptr->hca_ptr->ib_trans.lock); -} -#endif diff --git a/dapl/openib_scm/dapl_ib_cq.c b/dapl/openib_scm/dapl_ib_cq.c deleted file mode 100644 index 2af1889..0000000 --- a/dapl/openib_scm/dapl_ib_cq.c +++ /dev/null @@ -1,705 +0,0 @@ -/* - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/*************************************************************************** - * - * Module: uDAPL - * - * Filename: dapl_ib_cq.c - * - * Author: Arlin Davis - * - * Created: 3/10/2005 - * - * Description: - * - * The uDAPL openib provider - completion queue - * - **************************************************************************** - * Source Control System Information - * - * $Id: $ - * - * Copyright (c) 2005 Intel Corporation. All rights reserved. - * - **************************************************************************/ - -#include "openib_osd.h" -#include "dapl.h" -#include "dapl_adapter_util.h" -#include "dapl_lmr_util.h" -#include "dapl_evd_util.h" -#include "dapl_ring_buffer_util.h" - -#if defined(_WIN64) || defined(_WIN32) -#include "..\..\..\..\..\etc\user\comp_channel.cpp" -#include "..\..\..\..\..\etc\user\dlist.c" - -void dapli_cq_thread_destroy(struct dapl_hca *hca_ptr) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cq_thread_destroy(%p)\n", hca_ptr); - - if (hca_ptr->ib_trans.cq_state != IB_THREAD_RUN) - return; - - /* destroy cr_thread and lock */ - hca_ptr->ib_trans.cq_state = IB_THREAD_CANCEL; - CompChannelCancel(&hca_ptr->ib_trans.ib_cq->comp_channel); - dapl_dbg_log(DAPL_DBG_TYPE_CM, " cq_thread_destroy(%p) cancel\n", - hca_ptr); - while (hca_ptr->ib_trans.cq_state != IB_THREAD_EXIT) { - dapl_os_sleep_usec(20000); - } - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cq_thread_destroy(%d) exit\n", - dapl_os_getpid()); -} - -static void cq_thread(void *arg) -{ - struct dapl_hca *hca_ptr = arg; - struct dapl_evd *evd_ptr; - struct ibv_cq *ibv_cq = NULL; - - hca_ptr->ib_trans.cq_state = IB_THREAD_RUN; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cq_thread: ENTER hca %p\n", hca_ptr); - - /* wait on DTO event, or signal to abort */ - while (hca_ptr->ib_trans.cq_state == IB_THREAD_RUN) { - if (!ibv_get_cq_event - (hca_ptr->ib_trans.ib_cq, &ibv_cq, (void *)&evd_ptr)) { - - if (DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD)) { - ibv_ack_cq_events(ibv_cq, 1); - return; - } - - /* process DTO event via callback */ - dapl_evd_dto_callback(hca_ptr->ib_hca_handle, - evd_ptr->ib_cq_handle, - (void *)evd_ptr); - - ibv_ack_cq_events(ibv_cq, 1); - } - } - hca_ptr->ib_trans.cq_state = IB_THREAD_EXIT; - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cq_thread: EXIT: hca %p \n", - hca_ptr); -} - -#else // _WIN32 || _WIN64 - -void dapli_cq_thread_destroy(struct dapl_hca *hca_ptr) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cq_thread_destroy(%p)\n", hca_ptr); - - if (hca_ptr->ib_trans.cq_state != IB_THREAD_RUN) - return; - - /* destroy cr_thread and lock */ - hca_ptr->ib_trans.cq_state = IB_THREAD_CANCEL; - pthread_kill(hca_ptr->ib_trans.cq_thread, SIGUSR1); - dapl_dbg_log(DAPL_DBG_TYPE_CM, " cq_thread_destroy(%p) cancel\n", - hca_ptr); - while (hca_ptr->ib_trans.cq_state != IB_THREAD_EXIT) { - dapl_os_sleep_usec(20000); - } - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cq_thread_destroy(%d) exit\n", - dapl_os_getpid()); -} - -/* catch the signal */ -static void ib_cq_handler(int signum) -{ - return; -} - -static void cq_thread(void *arg) -{ - struct dapl_hca *hca_ptr = arg; - struct dapl_evd *evd_ptr; - struct ibv_cq *ibv_cq = NULL; - sigset_t sigset; - - sigemptyset(&sigset); - sigaddset(&sigset, SIGUSR1); - pthread_sigmask(SIG_UNBLOCK, &sigset, NULL); - signal(SIGUSR1, ib_cq_handler); - - hca_ptr->ib_trans.cq_state = IB_THREAD_RUN; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cq_thread: ENTER hca %p\n", hca_ptr); - - /* wait on DTO event, or signal to abort */ - while (hca_ptr->ib_trans.cq_state == IB_THREAD_RUN) { - struct pollfd cq_fd = { - .fd = hca_ptr->ib_trans.ib_cq->fd, - .events = POLLIN, - .revents = 0 - }; - if ((poll(&cq_fd, 1, -1) == 1) && - (!ibv_get_cq_event - (hca_ptr->ib_trans.ib_cq, &ibv_cq, (void *)&evd_ptr))) { - - if (DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD)) { - ibv_ack_cq_events(ibv_cq, 1); - return; - } - - /* process DTO event via callback */ - dapl_evd_dto_callback(hca_ptr->ib_hca_handle, - evd_ptr->ib_cq_handle, - (void *)evd_ptr); - - ibv_ack_cq_events(ibv_cq, 1); - } - } - hca_ptr->ib_trans.cq_state = IB_THREAD_EXIT; - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cq_thread: EXIT: hca %p \n", - hca_ptr); -} - -#endif // _WIN32 || _WIN64 - -int dapli_cq_thread_init(struct dapl_hca *hca_ptr) -{ - DAT_RETURN dat_status; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cq_thread_init(%p)\n", hca_ptr); - - /* create thread to process inbound connect request */ - hca_ptr->ib_trans.cq_state = IB_THREAD_INIT; - dat_status = - dapl_os_thread_create(cq_thread, (void *)hca_ptr, - &hca_ptr->ib_trans.cq_thread); - if (dat_status != DAT_SUCCESS) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " cq_thread_init: failed to create thread\n"); - return 1; - } - - /* wait for thread to start */ - while (hca_ptr->ib_trans.cq_state != IB_THREAD_RUN) { - dapl_os_sleep_usec(20000); - } - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cq_thread_init(%d) exit\n", - dapl_os_getpid()); - return 0; -} - -/* - * Map all verbs DTO completion codes to the DAT equivelent. - * - * Not returned by verbs: DAT_DTO_ERR_PARTIAL_PACKET - */ -static struct ib_status_map { - int ib_status; - DAT_DTO_COMPLETION_STATUS dat_status; -} ib_status_map[] = { - /* 00 */ { - IBV_WC_SUCCESS, DAT_DTO_SUCCESS}, - /* 01 */ { - IBV_WC_LOC_LEN_ERR, DAT_DTO_ERR_LOCAL_LENGTH}, - /* 02 */ { - IBV_WC_LOC_QP_OP_ERR, DAT_DTO_ERR_LOCAL_EP}, - /* 03 */ { - IBV_WC_LOC_EEC_OP_ERR, DAT_DTO_ERR_TRANSPORT}, - /* 04 */ { - IBV_WC_LOC_PROT_ERR, DAT_DTO_ERR_LOCAL_PROTECTION}, - /* 05 */ { - IBV_WC_WR_FLUSH_ERR, DAT_DTO_ERR_FLUSHED}, - /* 06 */ { - IBV_WC_MW_BIND_ERR, DAT_RMR_OPERATION_FAILED}, - /* 07 */ { - IBV_WC_BAD_RESP_ERR, DAT_DTO_ERR_BAD_RESPONSE}, - /* 08 */ { - IBV_WC_LOC_ACCESS_ERR, DAT_DTO_ERR_LOCAL_PROTECTION}, - /* 09 */ { - IBV_WC_REM_INV_REQ_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, - /* 10 */ { - IBV_WC_REM_ACCESS_ERR, DAT_DTO_ERR_REMOTE_ACCESS}, - /* 11 */ { - IBV_WC_REM_OP_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, - /* 12 */ { - IBV_WC_RETRY_EXC_ERR, DAT_DTO_ERR_TRANSPORT}, - /* 13 */ { - IBV_WC_RNR_RETRY_EXC_ERR, DAT_DTO_ERR_RECEIVER_NOT_READY}, - /* 14 */ { - IBV_WC_LOC_RDD_VIOL_ERR, DAT_DTO_ERR_LOCAL_PROTECTION}, - /* 15 */ { - IBV_WC_REM_INV_RD_REQ_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, - /* 16 */ { - IBV_WC_REM_ABORT_ERR, DAT_DTO_ERR_REMOTE_RESPONDER}, - /* 17 */ { - IBV_WC_INV_EECN_ERR, DAT_DTO_ERR_TRANSPORT}, - /* 18 */ { - IBV_WC_INV_EEC_STATE_ERR, DAT_DTO_ERR_TRANSPORT}, - /* 19 */ { - IBV_WC_FATAL_ERR, DAT_DTO_ERR_TRANSPORT}, - /* 20 */ { - IBV_WC_RESP_TIMEOUT_ERR, DAT_DTO_ERR_RECEIVER_NOT_READY}, - /* 21 */ { -IBV_WC_GENERAL_ERR, DAT_DTO_ERR_TRANSPORT},}; - -/* - * dapls_ib_get_dto_status - * - * Return the DAT status of a DTO operation - * - * Input: - * cqe_ptr pointer to completion queue entry - * - * Output: - * none - * - * Returns: - * Value from ib_status_map table above - */ - -DAT_DTO_COMPLETION_STATUS -dapls_ib_get_dto_status(IN ib_work_completion_t * cqe_ptr) -{ - uint32_t ib_status; - int i; - - ib_status = DAPL_GET_CQE_STATUS(cqe_ptr); - - /* - * Due to the implementation of verbs completion code, we need to - * search the table for the correct value rather than assuming - * linear distribution. - */ - for (i = 0; i <= IBV_WC_GENERAL_ERR; i++) { - if (ib_status == ib_status_map[i].ib_status) { - if (ib_status != IBV_WC_SUCCESS) { - dapl_dbg_log(DAPL_DBG_TYPE_DTO_COMP_ERR, - " DTO completion ERROR: %d: op %#x\n", - ib_status, - DAPL_GET_CQE_OPTYPE(cqe_ptr)); - } - return ib_status_map[i].dat_status; - } - } - - dapl_dbg_log(DAPL_DBG_TYPE_DTO_COMP_ERR, - " DTO completion ERROR: %d: op %#x\n", - ib_status, DAPL_GET_CQE_OPTYPE(cqe_ptr)); - - return DAT_DTO_FAILURE; -} - -DAT_RETURN dapls_ib_get_async_event(IN ib_error_record_t * err_record, - OUT DAT_EVENT_NUMBER * async_event) -{ - DAT_RETURN dat_status = DAT_SUCCESS; - int err_code = err_record->event_type; - - switch (err_code) { - /* OVERFLOW error */ - case IBV_EVENT_CQ_ERR: - *async_event = DAT_ASYNC_ERROR_EVD_OVERFLOW; - break; - /* INTERNAL errors */ - case IBV_EVENT_DEVICE_FATAL: - *async_event = DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR; - break; - /* CATASTROPHIC errors */ - case IBV_EVENT_PORT_ERR: - *async_event = DAT_ASYNC_ERROR_IA_CATASTROPHIC; - break; - /* BROKEN QP error */ - case IBV_EVENT_SQ_DRAINED: - case IBV_EVENT_QP_FATAL: - case IBV_EVENT_QP_REQ_ERR: - case IBV_EVENT_QP_ACCESS_ERR: - *async_event = DAT_ASYNC_ERROR_EP_BROKEN; - break; - - /* connection completion */ - case IBV_EVENT_COMM_EST: - *async_event = DAT_CONNECTION_EVENT_ESTABLISHED; - break; - - /* TODO: process HW state changes */ - case IBV_EVENT_PATH_MIG: - case IBV_EVENT_PATH_MIG_ERR: - case IBV_EVENT_PORT_ACTIVE: - case IBV_EVENT_LID_CHANGE: - case IBV_EVENT_PKEY_CHANGE: - case IBV_EVENT_SM_CHANGE: - default: - dat_status = DAT_ERROR(DAT_NOT_IMPLEMENTED, 0); - } - return dat_status; -} - -/* - * dapl_ib_cq_alloc - * - * Alloc a CQ - * - * Input: - * ia_handle IA handle - * evd_ptr pointer to EVD struct - * cqlen minimum QLen - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN -dapls_ib_cq_alloc(IN DAPL_IA * ia_ptr, - IN DAPL_EVD * evd_ptr, IN DAT_COUNT * cqlen) -{ - struct ibv_comp_channel *channel = ia_ptr->hca_ptr->ib_trans.ib_cq; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - "dapls_ib_cq_alloc: evd %p cqlen=%d \n", evd_ptr, *cqlen); - -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) - channel = evd_ptr->cq_wait_obj_handle; -#endif - - /* Call IB verbs to create CQ */ - evd_ptr->ib_cq_handle = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, - *cqlen, evd_ptr, channel, 0); - - if (evd_ptr->ib_cq_handle == IB_INVALID_HANDLE) - return DAT_INSUFFICIENT_RESOURCES; - - /* arm cq for events */ - dapls_set_cq_notify(ia_ptr, evd_ptr); - - /* update with returned cq entry size */ - *cqlen = evd_ptr->ib_cq_handle->cqe; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - "dapls_ib_cq_alloc: new_cq %p cqlen=%d \n", - evd_ptr->ib_cq_handle, *cqlen); - - return DAT_SUCCESS; -} - -/* - * dapl_ib_cq_resize - * - * Alloc a CQ - * - * Input: - * ia_handle IA handle - * evd_ptr pointer to EVD struct - * cqlen minimum QLen - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_PARAMETER - * - */ -DAT_RETURN -dapls_ib_cq_resize(IN DAPL_IA * ia_ptr, - IN DAPL_EVD * evd_ptr, IN DAT_COUNT * cqlen) -{ - ib_cq_handle_t new_cq; - struct ibv_comp_channel *channel = ia_ptr->hca_ptr->ib_trans.ib_cq; - - /* IB verbs doe not support resize. Try to re-create CQ - * with new size. Can only be done if QP is not attached. - * destroy EBUSY == QP still attached. - */ - -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) - channel = evd_ptr->cq_wait_obj_handle; -#endif - - /* Call IB verbs to create CQ */ - new_cq = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, *cqlen, - evd_ptr, channel, 0); - - if (new_cq == IB_INVALID_HANDLE) - return DAT_INSUFFICIENT_RESOURCES; - - /* destroy the original and replace if successful */ - if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) { - ibv_destroy_cq(new_cq); - return (dapl_convert_errno(errno, "resize_cq")); - } - - /* update EVD with new cq handle and size */ - evd_ptr->ib_cq_handle = new_cq; - *cqlen = new_cq->cqe; - - /* arm cq for events */ - dapls_set_cq_notify(ia_ptr, evd_ptr); - - return DAT_SUCCESS; -} - -/* - * dapls_ib_cq_free - * - * destroy a CQ - * - * Input: - * ia_handle IA handle - * evd_ptr pointer to EVD struct - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_PARAMETER - * - */ -DAT_RETURN dapls_ib_cq_free(IN DAPL_IA * ia_ptr, IN DAPL_EVD * evd_ptr) -{ - DAT_EVENT event; - ib_work_completion_t wc; - - if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) { - /* pull off CQ and EVD entries and toss */ - while (ibv_poll_cq(evd_ptr->ib_cq_handle, 1, &wc) == 1) ; - while (dapl_evd_dequeue(evd_ptr, &event) == DAT_SUCCESS) ; - if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) - return (dapl_convert_errno(errno, "ibv_destroy_cq")); - evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; - } - return DAT_SUCCESS; -} - -/* - * dapls_set_cq_notify - * - * Set the CQ notification for next - * - * Input: - * hca_handl hca handle - * DAPL_EVD evd handle - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * dapl_convert_errno - */ -DAT_RETURN dapls_set_cq_notify(IN DAPL_IA * ia_ptr, IN DAPL_EVD * evd_ptr) -{ - if (ibv_req_notify_cq(evd_ptr->ib_cq_handle, 0)) - return (dapl_convert_errno(errno, "notify_cq")); - else - return DAT_SUCCESS; -} - -/* - * dapls_ib_completion_notify - * - * Set the CQ notification type - * - * Input: - * hca_handl hca handle - * evd_ptr evd handle - * type notification type - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * dapl_convert_errno - */ -DAT_RETURN dapls_ib_completion_notify(IN ib_hca_handle_t hca_handle, - IN DAPL_EVD * evd_ptr, - IN ib_notification_type_t type) -{ - if (ibv_req_notify_cq(evd_ptr->ib_cq_handle, type)) - return (dapl_convert_errno(errno, "notify_cq_type")); - else - return DAT_SUCCESS; -} - -/* - * dapls_ib_completion_poll - * - * CQ poll for completions - * - * Input: - * hca_handl hca handle - * evd_ptr evd handle - * wc_ptr work completion - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_QUEUE_EMPTY - * - */ -DAT_RETURN dapls_ib_completion_poll(IN DAPL_HCA * hca_ptr, - IN DAPL_EVD * evd_ptr, - IN ib_work_completion_t * wc_ptr) -{ - int ret; - - ret = ibv_poll_cq(evd_ptr->ib_cq_handle, 1, wc_ptr); - if (ret == 1) - return DAT_SUCCESS; - - return DAT_QUEUE_EMPTY; -} - -#ifdef CQ_WAIT_OBJECT - -/* NEW common wait objects for providers with direct CQ wait objects */ -DAT_RETURN -dapls_ib_wait_object_create(IN DAPL_EVD * evd_ptr, - IN ib_wait_obj_handle_t * p_cq_wait_obj_handle) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_create: (%p,%p)\n", - evd_ptr, p_cq_wait_obj_handle); - - /* set cq_wait object to evd_ptr */ - *p_cq_wait_obj_handle = - ibv_create_comp_channel(evd_ptr->header.owner_ia->hca_ptr-> - ib_hca_handle); - - return DAT_SUCCESS; -} - -DAT_RETURN -dapls_ib_wait_object_destroy(IN ib_wait_obj_handle_t p_cq_wait_obj_handle) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_destroy: wait_obj=%p\n", p_cq_wait_obj_handle); - - ibv_destroy_comp_channel(p_cq_wait_obj_handle); - - return DAT_SUCCESS; -} - -DAT_RETURN -dapls_ib_wait_object_wakeup(IN ib_wait_obj_handle_t p_cq_wait_obj_handle) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wakeup: wait_obj=%p\n", p_cq_wait_obj_handle); - - /* no wake up mechanism */ - return DAT_SUCCESS; -} - -#if defined(_WIN32) || defined(_WIN64) -DAT_RETURN -dapls_ib_wait_object_wait(IN ib_wait_obj_handle_t p_cq_wait_obj_handle, - IN uint32_t timeout) -{ - struct dapl_evd *evd_ptr; - struct ibv_cq *ibv_cq = NULL; - int status = 0; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wait: CQ channel %p time %d\n", - p_cq_wait_obj_handle, timeout); - - /* uDAPL timeout values in usecs */ - p_cq_wait_obj_handle->comp_channel.Milliseconds = timeout / 1000; - - /* returned event */ - status = ibv_get_cq_event(p_cq_wait_obj_handle, &ibv_cq, - (void *)&evd_ptr); - if (status == 0) { - ibv_ack_cq_events(ibv_cq, 1); - } - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wait: RET evd %p ibv_cq %p %s\n", - evd_ptr, ibv_cq, strerror(errno)); - - return (dapl_convert_errno(status, "cq_wait_object_wait")); -} -#else //_WIN32 || _WIN64 -DAT_RETURN -dapls_ib_wait_object_wait(IN ib_wait_obj_handle_t p_cq_wait_obj_handle, - IN uint32_t timeout) -{ - struct dapl_evd *evd_ptr; - struct ibv_cq *ibv_cq = NULL; - int status = 0; - int timeout_ms = -1; - struct pollfd cq_fd = { - .fd = p_cq_wait_obj_handle->fd, - .events = POLLIN, - .revents = 0 - }; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wait: CQ channel %p time %d\n", - p_cq_wait_obj_handle, timeout); - - /* uDAPL timeout values in usecs */ - if (timeout != DAT_TIMEOUT_INFINITE) - timeout_ms = timeout / 1000; - - status = poll(&cq_fd, 1, timeout_ms); - - /* returned event */ - if (status > 0) { - if (!ibv_get_cq_event(p_cq_wait_obj_handle, - &ibv_cq, (void *)&evd_ptr)) { - ibv_ack_cq_events(ibv_cq, 1); - } - status = 0; - - /* timeout */ - } else if (status == 0) - status = ETIMEDOUT; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wait: RET evd %p ibv_cq %p %s\n", - evd_ptr, ibv_cq, strerror(errno)); - - return (dapl_convert_errno(status, "cq_wait_object_wait")); - -} -#endif //_WIN32 || _WIN64 -#endif // CQ_WAIT_OBJECT - -/* - * Local variables: - * c-indent-level: 4 - * c-basic-offset: 4 - * tab-width: 8 - * End: - */ diff --git a/dapl/openib_scm/dapl_ib_dto.h b/dapl/openib_scm/dapl_ib_dto.h deleted file mode 100644 index 9118b2e..0000000 --- a/dapl/openib_scm/dapl_ib_dto.h +++ /dev/null @@ -1,527 +0,0 @@ -/* - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/*************************************************************************** - * - * Module: uDAPL - * - * Filename: dapl_ib_dto.h - * - * Author: Arlin Davis - * - * Created: 3/10/2005 - * - * Description: - * - * The OpenIB uCMA provider - DTO operations and CQE macros - * - **************************************************************************** - * Source Control System Information - * - * $Id: $ - * - * Copyright (c) 2005 Intel Corporation. All rights reserved. - * - **************************************************************************/ -#ifndef _DAPL_IB_DTO_H_ -#define _DAPL_IB_DTO_H_ - -#include "dapl_ib_util.h" - -#ifdef DAT_EXTENSIONS -#include -#endif - -#define DEFAULT_DS_ENTRIES 8 - -STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p); - -#define CQE_WR_TYPE_UD(id) \ - (((DAPL_COOKIE *)(uintptr_t)id)->ep->qp_handle->qp_type == IBV_QPT_UD) - -/* - * dapls_ib_post_recv - * - * Provider specific Post RECV function - */ -STATIC _INLINE_ DAT_RETURN -dapls_ib_post_recv ( - IN DAPL_EP *ep_ptr, - IN DAPL_COOKIE *cookie, - IN DAT_COUNT segments, - IN DAT_LMR_TRIPLET *local_iov ) -{ - struct ibv_recv_wr wr; - struct ibv_recv_wr *bad_wr; - ib_data_segment_t *ds = (ib_data_segment_t *)local_iov; - DAT_COUNT i, total_len; - int ret; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_rcv: ep %p cookie %p segs %d l_iov %p\n", - ep_ptr, cookie, segments, local_iov); - - /* setup work request */ - total_len = 0; - wr.next = 0; - wr.num_sge = segments; - wr.wr_id = (uint64_t)(uintptr_t)cookie; - wr.sg_list = ds; - - if (cookie != NULL) { - for (i = 0; i < segments; i++) { - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_rcv: l_key 0x%x va %p len %d\n", - ds->lkey, ds->addr, ds->length ); - total_len += ds->length; - ds++; - } - cookie->val.dto.size = total_len; - } - - ret = ibv_post_recv(ep_ptr->qp_handle, &wr, &bad_wr); - - if (ret) - return(dapl_convert_errno(errno,"ibv_recv")); - - DAPL_CNTR(ep_ptr, DCNT_EP_POST_RECV); - DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_RECV_DATA, total_len); - - return DAT_SUCCESS; -} - -/* - * dapls_ib_post_send - * - * Provider specific Post SEND function - */ -STATIC _INLINE_ DAT_RETURN -dapls_ib_post_send ( - IN DAPL_EP *ep_ptr, - IN ib_send_op_type_t op_type, - IN DAPL_COOKIE *cookie, - IN DAT_COUNT segments, - IN DAT_LMR_TRIPLET *local_iov, - IN const DAT_RMR_TRIPLET *remote_iov, - IN DAT_COMPLETION_FLAGS completion_flags) -{ - struct ibv_send_wr wr; - struct ibv_send_wr *bad_wr; - ib_data_segment_t *ds = (ib_data_segment_t *)local_iov; - ib_hca_transport_t *ibt_ptr = - &ep_ptr->header.owner_ia->hca_ptr->ib_trans; - DAT_COUNT i, total_len; - int ret; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd: ep %p op %d ck %p sgs", - "%d l_iov %p r_iov %p f %d\n", - ep_ptr, op_type, cookie, segments, local_iov, - remote_iov, completion_flags); - -#ifdef DAT_EXTENSIONS - if (ep_ptr->qp_handle->qp_type != IBV_QPT_RC) - return(DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP)); -#endif - /* setup the work request */ - wr.next = 0; - wr.opcode = op_type; - wr.num_sge = segments; - wr.send_flags = 0; - wr.wr_id = (uint64_t)(uintptr_t)cookie; - wr.sg_list = ds; - total_len = 0; - - if (cookie != NULL) { - for (i = 0; i < segments; i++ ) { - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd: lkey 0x%x va %p len %d\n", - ds->lkey, ds->addr, ds->length ); - total_len += ds->length; - ds++; - } - cookie->val.dto.size = total_len; - } - - if (wr.num_sge && - (op_type == OP_RDMA_WRITE || op_type == OP_RDMA_READ)) { - wr.wr.rdma.remote_addr = remote_iov->virtual_address; - wr.wr.rdma.rkey = remote_iov->rmr_context; - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd_rdma: rkey 0x%x va %#016Lx\n", - wr.wr.rdma.rkey, wr.wr.rdma.remote_addr); - } - - - /* inline data for send or write ops */ - if ((total_len <= ibt_ptr->max_inline_send) && - ((op_type == OP_SEND) || (op_type == OP_RDMA_WRITE))) - wr.send_flags |= IBV_SEND_INLINE; - - /* set completion flags in work request */ - wr.send_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & - completion_flags) ? 0 : IBV_SEND_SIGNALED; - wr.send_flags |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & - completion_flags) ? IBV_SEND_FENCE : 0; - wr.send_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & - completion_flags) ? IBV_SEND_SOLICITED : 0; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd: op 0x%x flags 0x%x sglist %p, %d\n", - wr.opcode, wr.send_flags, wr.sg_list, wr.num_sge); - - ret = ibv_post_send(ep_ptr->qp_handle, &wr, &bad_wr); - - if (ret) - return(dapl_convert_errno(errno,"ibv_send")); - -#ifdef DAPL_COUNTERS - switch (op_type) { - case OP_SEND: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND); - DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_DATA,total_len); - break; - case OP_RDMA_WRITE: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE); - DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_WRITE_DATA,total_len); - break; - case OP_RDMA_READ: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_READ); - DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_READ_DATA,total_len); - break; - default: - break; - } -#endif /* DAPL_COUNTERS */ - - dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n"); - return DAT_SUCCESS; -} - -/* map Work Completions to DAPL WR operations */ -STATIC _INLINE_ DAT_DTOS dapls_cqe_dtos_opcode(ib_work_completion_t *cqe_p) -{ - switch (cqe_p->opcode) { - - case IBV_WC_SEND: -#ifdef DAT_EXTENSIONS - if (CQE_WR_TYPE_UD(cqe_p->wr_id)) - return (DAT_IB_DTO_SEND_UD); - else -#endif - return (DAT_DTO_SEND); - case IBV_WC_RDMA_READ: - return (DAT_DTO_RDMA_READ); - case IBV_WC_BIND_MW: - return (DAT_DTO_BIND_MW); -#ifdef DAT_EXTENSIONS - case IBV_WC_RDMA_WRITE: - if (cqe_p->wc_flags & IBV_WC_WITH_IMM) - return (DAT_IB_DTO_RDMA_WRITE_IMMED); - else - return (DAT_DTO_RDMA_WRITE); - case IBV_WC_COMP_SWAP: - return (DAT_IB_DTO_CMP_SWAP); - case IBV_WC_FETCH_ADD: - return (DAT_IB_DTO_FETCH_ADD); - case IBV_WC_RECV_RDMA_WITH_IMM: - return (DAT_IB_DTO_RECV_IMMED); -#else - case IBV_WC_RDMA_WRITE: - return (DAT_DTO_RDMA_WRITE); -#endif - case IBV_WC_RECV: -#ifdef DAT_EXTENSIONS - if (CQE_WR_TYPE_UD(cqe_p->wr_id)) - return (DAT_IB_DTO_RECV_UD); - else if (cqe_p->wc_flags & IBV_WC_WITH_IMM) - return (DAT_IB_DTO_RECV_MSG_IMMED); - else -#endif - return (DAT_DTO_RECEIVE); - default: - return (0xff); - } -} -#define DAPL_GET_CQE_DTOS_OPTYPE(cqe_p) dapls_cqe_dtos_opcode(cqe_p) - - -#ifdef DAT_EXTENSIONS -/* - * dapls_ib_post_ext_send - * - * Provider specific extended Post SEND function for atomics - * OP_COMP_AND_SWAP and OP_FETCH_AND_ADD - */ -STATIC _INLINE_ DAT_RETURN -dapls_ib_post_ext_send ( - IN DAPL_EP *ep_ptr, - IN ib_send_op_type_t op_type, - IN DAPL_COOKIE *cookie, - IN DAT_COUNT segments, - IN DAT_LMR_TRIPLET *local_iov, - IN const DAT_RMR_TRIPLET *remote_iov, - IN DAT_UINT32 immed_data, - IN DAT_UINT64 compare_add, - IN DAT_UINT64 swap, - IN DAT_COMPLETION_FLAGS completion_flags, - IN DAT_IB_ADDR_HANDLE *remote_ah) -{ - struct ibv_send_wr wr; - struct ibv_send_wr *bad_wr; - ib_data_segment_t *ds = (ib_data_segment_t *)local_iov; - DAT_COUNT i, total_len; - int ret; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_ext_snd: ep %p op %d ck %p sgs", - "%d l_iov %p r_iov %p f %d\n", - ep_ptr, op_type, cookie, segments, local_iov, - remote_iov, completion_flags, remote_ah); - - /* setup the work request */ - wr.next = 0; - wr.opcode = op_type; - wr.num_sge = segments; - wr.send_flags = 0; - wr.wr_id = (uint64_t)(uintptr_t)cookie; - wr.sg_list = ds; - total_len = 0; - - if (cookie != NULL) { - for (i = 0; i < segments; i++ ) { - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd: lkey 0x%x va %p len %d\n", - ds->lkey, ds->addr, ds->length ); - total_len += ds->length; - ds++; - } - cookie->val.dto.size = total_len; - } - - switch (op_type) { - case OP_RDMA_WRITE_IMM: - /* OP_RDMA_WRITE)IMMED has direct IB wr_type mapping */ - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_ext: rkey 0x%x va %#016Lx immed=0x%x\n", - remote_iov?remote_iov->rmr_context:0, - remote_iov?remote_iov->virtual_address:0, - immed_data); - - wr.imm_data = immed_data; - if (wr.num_sge) { - wr.wr.rdma.remote_addr = remote_iov->virtual_address; - wr.wr.rdma.rkey = remote_iov->rmr_context; - } - break; - case OP_COMP_AND_SWAP: - /* OP_COMP_AND_SWAP has direct IB wr_type mapping */ - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_ext: OP_COMP_AND_SWAP=%lx," - "%lx rkey 0x%x va %#016Lx\n", - compare_add, swap, remote_iov->rmr_context, - remote_iov->virtual_address); - - wr.wr.atomic.compare_add = compare_add; - wr.wr.atomic.swap = swap; - wr.wr.atomic.remote_addr = remote_iov->virtual_address; - wr.wr.atomic.rkey = remote_iov->rmr_context; - break; - case OP_FETCH_AND_ADD: - /* OP_FETCH_AND_ADD has direct IB wr_type mapping */ - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_ext: OP_FETCH_AND_ADD=%lx," - "%lx rkey 0x%x va %#016Lx\n", - compare_add, remote_iov->rmr_context, - remote_iov->virtual_address); - - wr.wr.atomic.compare_add = compare_add; - wr.wr.atomic.remote_addr = remote_iov->virtual_address; - wr.wr.atomic.rkey = remote_iov->rmr_context; - break; - case OP_SEND_UD: - /* post must be on EP with service_type of UD */ - if (ep_ptr->qp_handle->qp_type != IBV_QPT_UD) - return(DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP)); - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_ext: OP_SEND_UD ah=%p" - " qp_num=0x%x\n", - remote_ah, remote_ah->qpn); - - wr.opcode = OP_SEND; - wr.wr.ud.ah = remote_ah->ah; - wr.wr.ud.remote_qpn = remote_ah->qpn; - wr.wr.ud.remote_qkey = SCM_UD_QKEY; - break; - default: - break; - } - - /* set completion flags in work request */ - wr.send_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & - completion_flags) ? 0 : IBV_SEND_SIGNALED; - wr.send_flags |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & - completion_flags) ? IBV_SEND_FENCE : 0; - wr.send_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & - completion_flags) ? IBV_SEND_SOLICITED : 0; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " post_snd: op 0x%x flags 0x%x sglist %p, %d\n", - wr.opcode, wr.send_flags, wr.sg_list, wr.num_sge); - - ret = ibv_post_send(ep_ptr->qp_handle, &wr, &bad_wr); - - if (ret) - return( dapl_convert_errno(errno,"ibv_send") ); - -#ifdef DAPL_COUNTERS - switch (op_type) { - case OP_RDMA_WRITE_IMM: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE_IMM); - DAPL_CNTR_DATA(ep_ptr, - DCNT_EP_POST_WRITE_IMM_DATA, total_len); - break; - case OP_COMP_AND_SWAP: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_CMP_SWAP); - break; - case OP_FETCH_AND_ADD: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_FETCH_ADD); - break; - case OP_SEND_UD: - DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND_UD); - DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_UD_DATA, total_len); - break; - default: - break; - } -#endif /* DAPL_COUNTERS */ - - dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n"); - return DAT_SUCCESS; -} -#endif - -STATIC _INLINE_ DAT_RETURN -dapls_ib_optional_prv_dat( - IN DAPL_CR *cr_ptr, - IN const void *event_data, - OUT DAPL_CR **cr_pp) -{ - return DAT_SUCCESS; -} - - -/* map Work Completions to DAPL WR operations */ -STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p) -{ -#ifdef DAPL_COUNTERS - DAPL_COOKIE *cookie = (DAPL_COOKIE *)(uintptr_t)cqe_p->wr_id; -#endif /* DAPL_COUNTERS */ - - switch (cqe_p->opcode) { - case IBV_WC_SEND: - if (CQE_WR_TYPE_UD(cqe_p->wr_id)) - return(OP_SEND_UD); - else - return (OP_SEND); - case IBV_WC_RDMA_WRITE: - if (cqe_p->wc_flags & IBV_WC_WITH_IMM) - return (OP_RDMA_WRITE_IMM); - else - return (OP_RDMA_WRITE); - case IBV_WC_RDMA_READ: - return (OP_RDMA_READ); - case IBV_WC_COMP_SWAP: - return (OP_COMP_AND_SWAP); - case IBV_WC_FETCH_ADD: - return (OP_FETCH_AND_ADD); - case IBV_WC_BIND_MW: - return (OP_BIND_MW); - case IBV_WC_RECV: - if (CQE_WR_TYPE_UD(cqe_p->wr_id)) { - DAPL_CNTR(cookie->ep, DCNT_EP_RECV_UD); - DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_UD_DATA, - cqe_p->byte_len); - return (OP_RECV_UD); - } - else if (cqe_p->wc_flags & IBV_WC_WITH_IMM) { - DAPL_CNTR(cookie->ep, DCNT_EP_RECV_IMM); - DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_IMM_DATA, - cqe_p->byte_len); - return (OP_RECEIVE_IMM); - } else { - DAPL_CNTR(cookie->ep, DCNT_EP_RECV); - DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_DATA, - cqe_p->byte_len); - return (OP_RECEIVE); - } - case IBV_WC_RECV_RDMA_WITH_IMM: - DAPL_CNTR(cookie->ep, DCNT_EP_RECV_RDMA_IMM); - DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_RDMA_IMM_DATA, - cqe_p->byte_len); - return (OP_RECEIVE_IMM); - default: - return (OP_INVALID); - } -} - -#define DAPL_GET_CQE_OPTYPE(cqe_p) dapls_cqe_opcode(cqe_p) -#define DAPL_GET_CQE_WRID(cqe_p) ((ib_work_completion_t*)cqe_p)->wr_id -#define DAPL_GET_CQE_STATUS(cqe_p) ((ib_work_completion_t*)cqe_p)->status -#define DAPL_GET_CQE_VENDOR_ERR(cqe_p) ((ib_work_completion_t*)cqe_p)->vendor_err -#define DAPL_GET_CQE_BYTESNUM(cqe_p) ((ib_work_completion_t*)cqe_p)->byte_len -#define DAPL_GET_CQE_IMMED_DATA(cqe_p) ((ib_work_completion_t*)cqe_p)->imm_data - -STATIC _INLINE_ char * dapls_dto_op_str(int op) -{ - static char *optable[] = - { - "OP_RDMA_WRITE", - "OP_RDMA_WRITE_IMM", - "OP_SEND", - "OP_SEND_IMM", - "OP_RDMA_READ", - "OP_COMP_AND_SWAP", - "OP_FETCH_AND_ADD", - "OP_RECEIVE", - "OP_RECEIVE_MSG_IMM", - "OP_RECEIVE_RDMA_IMM", - "OP_BIND_MW" - "OP_SEND_UD" - "OP_RECV_UD" - }; - return ((op < 0 || op > 12) ? "Invalid CQE OP?" : optable[op]); -} - -static _INLINE_ char * -dapls_cqe_op_str(IN ib_work_completion_t *cqe_ptr) -{ - return dapls_dto_op_str(DAPL_GET_CQE_OPTYPE(cqe_ptr)); -} - -#define DAPL_GET_CQE_OP_STR(cqe) dapls_cqe_op_str(cqe) - -#endif /* _DAPL_IB_DTO_H_ */ diff --git a/dapl/openib_scm/dapl_ib_extensions.c b/dapl/openib_scm/dapl_ib_extensions.c deleted file mode 100755 index 98a07ec..0000000 --- a/dapl/openib_scm/dapl_ib_extensions.c +++ /dev/null @@ -1,371 +0,0 @@ -/* - * Copyright (c) 2007 Intel Corporation. All rights reserved. - * - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/********************************************************************** - * - * MODULE: dapl_ib_extensions.c - * - * PURPOSE: Extensions routines for OpenIB uCMA provider - * - * $Id: $ - * - **********************************************************************/ - -#include "dapl.h" -#include "dapl_adapter_util.h" -#include "dapl_evd_util.h" -#include "dapl_ib_util.h" -#include "dapl_ep_util.h" -#include "dapl_cookie.h" -#include - -DAT_RETURN -dapli_post_ext(IN DAT_EP_HANDLE ep_handle, - IN DAT_UINT64 cmp_add, - IN DAT_UINT64 swap, - IN DAT_UINT32 immed_data, - IN DAT_COUNT segments, - IN DAT_LMR_TRIPLET * local_iov, - IN DAT_DTO_COOKIE user_cookie, - IN const DAT_RMR_TRIPLET * remote_iov, - IN int op_type, - IN DAT_COMPLETION_FLAGS flags, IN DAT_IB_ADDR_HANDLE * ah); - -/* - * dapl_extensions - * - * Process extension requests - * - * Input: - * ext_type, - * ... - * - * Output: - * Depends.... - * - * Returns: - * DAT_SUCCESS - * DAT_NOT_IMPLEMENTED - * ..... - * - */ -DAT_RETURN -dapl_extensions(IN DAT_HANDLE dat_handle, - IN DAT_EXTENDED_OP ext_op, IN va_list args) -{ - DAT_EP_HANDLE ep; - DAT_IB_ADDR_HANDLE *ah = NULL; - DAT_LMR_TRIPLET *lmr_p; - DAT_DTO_COOKIE cookie; - const DAT_RMR_TRIPLET *rmr_p; - DAT_UINT64 dat_uint64a, dat_uint64b; - DAT_UINT32 dat_uint32; - DAT_COUNT segments = 1; - DAT_COMPLETION_FLAGS comp_flags; - DAT_RETURN status = DAT_NOT_IMPLEMENTED; - - dapl_dbg_log(DAPL_DBG_TYPE_API, - "dapl_extensions(hdl %p operation %d, ...)\n", - dat_handle, ext_op); - - switch ((int)ext_op) { - - case DAT_IB_RDMA_WRITE_IMMED_OP: - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " WRITE_IMMED_DATA extension call\n"); - - ep = dat_handle; /* ep_handle */ - segments = va_arg(args, DAT_COUNT); /* num segments */ - lmr_p = va_arg(args, DAT_LMR_TRIPLET *); - cookie = va_arg(args, DAT_DTO_COOKIE); - rmr_p = va_arg(args, const DAT_RMR_TRIPLET *); - dat_uint32 = va_arg(args, DAT_UINT32); /* immed data */ - comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); - - status = dapli_post_ext(ep, 0, 0, dat_uint32, segments, lmr_p, - cookie, rmr_p, OP_RDMA_WRITE_IMM, - comp_flags, ah); - break; - - case DAT_IB_CMP_AND_SWAP_OP: - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " CMP_AND_SWAP extension call\n"); - - ep = dat_handle; /* ep_handle */ - dat_uint64a = va_arg(args, DAT_UINT64); /* cmp_value */ - dat_uint64b = va_arg(args, DAT_UINT64); /* swap_value */ - lmr_p = va_arg(args, DAT_LMR_TRIPLET *); - cookie = va_arg(args, DAT_DTO_COOKIE); - rmr_p = va_arg(args, const DAT_RMR_TRIPLET *); - comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); - - status = dapli_post_ext(ep, dat_uint64a, dat_uint64b, - 0, segments, lmr_p, cookie, rmr_p, - OP_COMP_AND_SWAP, comp_flags, ah); - break; - - case DAT_IB_FETCH_AND_ADD_OP: - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " FETCH_AND_ADD extension call\n"); - - ep = dat_handle; /* ep_handle */ - dat_uint64a = va_arg(args, DAT_UINT64); /* add value */ - lmr_p = va_arg(args, DAT_LMR_TRIPLET *); - cookie = va_arg(args, DAT_DTO_COOKIE); - rmr_p = va_arg(args, const DAT_RMR_TRIPLET *); - comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); - - status = dapli_post_ext(ep, dat_uint64a, 0, 0, segments, - lmr_p, cookie, rmr_p, - OP_FETCH_AND_ADD, comp_flags, ah); - break; - - case DAT_IB_UD_SEND_OP: - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " UD post_send extension call\n"); - - ep = dat_handle; /* ep_handle */ - segments = va_arg(args, DAT_COUNT); /* segments */ - lmr_p = va_arg(args, DAT_LMR_TRIPLET *); - ah = va_arg(args, DAT_IB_ADDR_HANDLE *); - cookie = va_arg(args, DAT_DTO_COOKIE); - comp_flags = va_arg(args, DAT_COMPLETION_FLAGS); - - status = dapli_post_ext(ep, 0, 0, 0, segments, - lmr_p, cookie, NULL, - OP_SEND_UD, comp_flags, ah); - break; - -#ifdef DAPL_COUNTERS - case DAT_QUERY_COUNTERS_OP: - { - int cntr, reset; - DAT_UINT64 *p_cntr_out; - - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " Query counter extension call\n"); - - cntr = va_arg(args, int); - p_cntr_out = va_arg(args, DAT_UINT64 *); - reset = va_arg(args, int); - - status = dapl_query_counter(dat_handle, cntr, - p_cntr_out, reset); - break; - } - case DAT_PRINT_COUNTERS_OP: - { - int cntr, reset; - - dapl_dbg_log(DAPL_DBG_TYPE_RTN, - " Print counter extension call\n"); - - cntr = va_arg(args, int); - reset = va_arg(args, int); - - dapl_print_counter(dat_handle, cntr, reset); - status = DAT_SUCCESS; - break; - } -#endif /* DAPL_COUNTERS */ - - default: - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - "unsupported extension(%d)\n", (int)ext_op); - } - - return (status); -} - -DAT_RETURN -dapli_post_ext(IN DAT_EP_HANDLE ep_handle, - IN DAT_UINT64 cmp_add, - IN DAT_UINT64 swap, - IN DAT_UINT32 immed_data, - IN DAT_COUNT segments, - IN DAT_LMR_TRIPLET * local_iov, - IN DAT_DTO_COOKIE user_cookie, - IN const DAT_RMR_TRIPLET * remote_iov, - IN int op_type, - IN DAT_COMPLETION_FLAGS flags, IN DAT_IB_ADDR_HANDLE * ah) -{ - DAPL_EP *ep_ptr; - ib_qp_handle_t qp_ptr; - DAPL_COOKIE *cookie = NULL; - DAT_RETURN dat_status = DAT_SUCCESS; - - dapl_dbg_log(DAPL_DBG_TYPE_API, - " post_ext_op: ep %p cmp_val %d " - "swap_val %d cookie 0x%x, r_iov %p, flags 0x%x, ah %p\n", - ep_handle, (unsigned)cmp_add, (unsigned)swap, - (unsigned)user_cookie.as_64, remote_iov, flags, ah); - - if (DAPL_BAD_HANDLE(ep_handle, DAPL_MAGIC_EP)) - return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP)); - - ep_ptr = (DAPL_EP *) ep_handle; - qp_ptr = ep_ptr->qp_handle; - - /* - * Synchronization ok since this buffer is only used for send - * requests, which aren't allowed to race with each other. - */ - dat_status = dapls_dto_cookie_alloc(&ep_ptr->req_buffer, - DAPL_DTO_TYPE_EXTENSION, - user_cookie, &cookie); - if (dat_status != DAT_SUCCESS) - goto bail; - - /* - * Take reference before posting to avoid race conditions with - * completions - */ - dapl_os_atomic_inc(&ep_ptr->req_count); - - /* - * Invoke provider specific routine to post DTO - */ - dat_status = dapls_ib_post_ext_send(ep_ptr, op_type, cookie, segments, /* data segments */ - local_iov, remote_iov, immed_data, /* immed data */ - cmp_add, /* compare or add */ - swap, /* swap */ - flags, ah); - - if (dat_status != DAT_SUCCESS) { - dapl_os_atomic_dec(&ep_ptr->req_count); - dapls_cookie_dealloc(&ep_ptr->req_buffer, cookie); - } - - bail: - return dat_status; - -} - -/* - * New provider routine to process extended DTO events - */ -void -dapls_cqe_to_event_extension(IN DAPL_EP * ep_ptr, - IN DAPL_COOKIE * cookie, - IN ib_work_completion_t * cqe_ptr, - IN DAT_EVENT * event_ptr) -{ - uint32_t ibtype; - DAT_DTO_COMPLETION_EVENT_DATA *dto = - &event_ptr->event_data.dto_completion_event_data; - DAT_IB_EXTENSION_EVENT_DATA *ext_data = (DAT_IB_EXTENSION_EVENT_DATA *) - & event_ptr->event_extension_data[0]; - DAT_DTO_COMPLETION_STATUS dto_status; - - /* Get status from cqe */ - dto_status = dapls_ib_get_dto_status(cqe_ptr); - - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: dto_ptr %p ext_ptr %p status %d\n", - dto, ext_data, dto_status); - - event_ptr->event_number = DAT_IB_DTO_EVENT; - dto->ep_handle = cookie->ep; - dto->user_cookie = cookie->val.dto.cookie; - dto->operation = DAPL_GET_CQE_DTOS_OPTYPE(cqe_ptr); /* new for 2.0 */ - dto->status = ext_data->status = dto_status; - - if (dto_status != DAT_DTO_SUCCESS) - return; - - /* - * Get operation type from CQ work completion entry and - * if extented operation then set extended event data - */ - ibtype = DAPL_GET_CQE_OPTYPE(cqe_ptr); - - switch (ibtype) { - - case OP_RDMA_WRITE_IMM: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: OP_RDMA_WRITE_IMMED\n"); - - /* type and outbound rdma write transfer size */ - dto->transfered_length = cookie->val.dto.size; - ext_data->type = DAT_IB_RDMA_WRITE_IMMED; - break; - case OP_RECEIVE_IMM: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: OP_RECEIVE_RDMA_IMMED\n"); - - /* immed recvd, type and inbound rdma write transfer size */ - dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); - ext_data->type = DAT_IB_RDMA_WRITE_IMMED_DATA; - ext_data->val.immed.data = DAPL_GET_CQE_IMMED_DATA(cqe_ptr); - break; - case OP_RECEIVE_MSG_IMM: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: OP_RECEIVE_MSG_IMMED\n"); - - /* immed recvd, type and inbound recv message transfer size */ - dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); - ext_data->type = DAT_IB_RECV_IMMED_DATA; - ext_data->val.immed.data = DAPL_GET_CQE_IMMED_DATA(cqe_ptr); - break; - case OP_COMP_AND_SWAP: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: COMP_AND_SWAP_RESP\n"); - - /* original data is returned in LMR provided with post */ - ext_data->type = DAT_IB_CMP_AND_SWAP; - dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); - break; - case OP_FETCH_AND_ADD: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, - " cqe_to_event_ext: FETCH_AND_ADD_RESP\n"); - - /* original data is returned in LMR provided with post */ - ext_data->type = DAT_IB_FETCH_AND_ADD; - dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); - break; - case OP_SEND_UD: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, " cqe_to_event_ext: UD_SEND\n"); - - /* type and outbound send transfer size */ - ext_data->type = DAT_IB_UD_SEND; - dto->transfered_length = cookie->val.dto.size; - break; - case OP_RECV_UD: - dapl_dbg_log(DAPL_DBG_TYPE_EVD, " cqe_to_event_ext: UD_RECV\n"); - - /* type and inbound recv message transfer size */ - ext_data->type = DAT_IB_UD_RECV; - dto->transfered_length = DAPL_GET_CQE_BYTESNUM(cqe_ptr); - break; - - default: - /* not extended operation */ - ext_data->status = DAT_IB_OP_ERR; - dto->status = DAT_DTO_ERR_TRANSPORT; - break; - } -} diff --git a/dapl/openib_scm/dapl_ib_mem.c b/dapl/openib_scm/dapl_ib_mem.c deleted file mode 100644 index e45a2b3..0000000 --- a/dapl/openib_scm/dapl_ib_mem.c +++ /dev/null @@ -1,382 +0,0 @@ - /* - * Copyright (c) 2005-2007 Intel Corporation. All rights reserved. - * - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/********************************************************************** - * - * MODULE: dapl_ib_mem.c - * - * PURPOSE: Memory windows, registration, and protection domain - * - * $Id:$ - * - **********************************************************************/ - -#include "dapl.h" -#include "dapl_adapter_util.h" -#include "dapl_lmr_util.h" - -/* - * dapls_convert_privileges - * - * Convert LMR privileges to provider - * - * Input: - * DAT_MEM_PRIV_FLAGS - * - * Output: - * none - * - * Returns: - * ibv_access_flags - * - */ -STATIC _INLINE_ int dapls_convert_privileges(IN DAT_MEM_PRIV_FLAGS privileges) -{ - int access = 0; - - /* - * if (DAT_MEM_PRIV_LOCAL_READ_FLAG & privileges) do nothing - */ - if (DAT_MEM_PRIV_LOCAL_WRITE_FLAG & privileges) - access |= IBV_ACCESS_LOCAL_WRITE; - if (DAT_MEM_PRIV_REMOTE_WRITE_FLAG & privileges) - access |= IBV_ACCESS_REMOTE_WRITE; - if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) - access |= IBV_ACCESS_REMOTE_READ; - if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) - access |= IBV_ACCESS_REMOTE_READ; - if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges) - access |= IBV_ACCESS_REMOTE_READ; -#ifdef DAT_EXTENSIONS - if (DAT_IB_MEM_PRIV_REMOTE_ATOMIC & privileges) - access |= IBV_ACCESS_REMOTE_ATOMIC; -#endif - - return access; -} - -/* - * dapl_ib_pd_alloc - * - * Alloc a PD - * - * Input: - * ia_handle IA handle - * pz pointer to PZ struct - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN dapls_ib_pd_alloc(IN DAPL_IA * ia_ptr, IN DAPL_PZ * pz) -{ - /* get a protection domain */ - pz->pd_handle = ibv_alloc_pd(ia_ptr->hca_ptr->ib_hca_handle); - if (!pz->pd_handle) - return (dapl_convert_errno(ENOMEM, "alloc_pd")); - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " pd_alloc: pd_handle=%p\n", pz->pd_handle); - - return DAT_SUCCESS; -} - -/* - * dapl_ib_pd_free - * - * Free a PD - * - * Input: - * ia_handle IA handle - * PZ_ptr pointer to PZ struct - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_STATE - * - */ -DAT_RETURN dapls_ib_pd_free(IN DAPL_PZ * pz) -{ - if (pz->pd_handle != IB_INVALID_HANDLE) { - if (ibv_dealloc_pd(pz->pd_handle)) - return (dapl_convert_errno(errno, "ibv_dealloc_pd")); - pz->pd_handle = IB_INVALID_HANDLE; - } - return DAT_SUCCESS; -} - -/* - * dapl_ib_mr_register - * - * Register a virtual memory region - * - * Input: - * ia_handle IA handle - * lmr pointer to dapl_lmr struct - * virt_addr virtual address of beginning of mem region - * length length of memory region - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN -dapls_ib_mr_register(IN DAPL_IA * ia_ptr, - IN DAPL_LMR * lmr, - IN DAT_PVOID virt_addr, - IN DAT_VLEN length, - IN DAT_MEM_PRIV_FLAGS privileges, IN DAT_VA_TYPE va_type) -{ - ib_pd_handle_t ib_pd_handle; - struct ibv_device *ibv_dev = ia_ptr->hca_ptr->ib_hca_handle->device; - - ib_pd_handle = ((DAPL_PZ *) lmr->param.pz_handle)->pd_handle; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " mr_register: ia=%p, lmr=%p va=%p ln=%d pv=0x%x\n", - ia_ptr, lmr, virt_addr, length, privileges); - - /* TODO: shared memory */ - if (lmr->param.mem_type == DAT_MEM_TYPE_SHARED_VIRTUAL) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " mr_register_shared: NOT IMPLEMENTED\n"); - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); - } - - /* iWARP only support */ - if ((va_type == DAT_VA_TYPE_ZB) && - (ibv_dev->transport_type != IBV_TRANSPORT_IWARP)) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " va_type == DAT_VA_TYPE_ZB: NOT SUPPORTED\n"); - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); - } - - /* local read is default on IB */ - lmr->mr_handle = - ibv_reg_mr(((DAPL_PZ *) lmr->param.pz_handle)->pd_handle, - virt_addr, length, dapls_convert_privileges(privileges)); - - if (!lmr->mr_handle) - return (dapl_convert_errno(ENOMEM, "reg_mr")); - - lmr->param.lmr_context = lmr->mr_handle->lkey; - lmr->param.rmr_context = lmr->mr_handle->rkey; - lmr->param.registered_size = length; - lmr->param.registered_address = (DAT_VADDR) (uintptr_t) virt_addr; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " mr_register: mr=%p addr=%p pd %p ctx %p " - "lkey=0x%x rkey=0x%x priv=%x\n", - lmr->mr_handle, lmr->mr_handle->addr, - lmr->mr_handle->pd, lmr->mr_handle->context, - lmr->mr_handle->lkey, lmr->mr_handle->rkey, - length, dapls_convert_privileges(privileges)); - - return DAT_SUCCESS; -} - -/* - * dapl_ib_mr_deregister - * - * Free a memory region - * - * Input: - * lmr pointer to dapl_lmr struct - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_STATE - * - */ -DAT_RETURN dapls_ib_mr_deregister(IN DAPL_LMR * lmr) -{ - if (lmr->mr_handle != IB_INVALID_HANDLE) { - if (ibv_dereg_mr(lmr->mr_handle)) - return (dapl_convert_errno(errno, "dereg_pd")); - lmr->mr_handle = IB_INVALID_HANDLE; - } - return DAT_SUCCESS; -} - -/* - * dapl_ib_mr_register_shared - * - * Register a virtual memory region - * - * Input: - * ia_ptr IA handle - * lmr pointer to dapl_lmr struct - * privileges - * va_type - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN -dapls_ib_mr_register_shared(IN DAPL_IA * ia_ptr, - IN DAPL_LMR * lmr, - IN DAT_MEM_PRIV_FLAGS privileges, - IN DAT_VA_TYPE va_type) -{ - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " mr_register_shared: NOT IMPLEMENTED\n"); - - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); -} - -/* - * dapls_ib_mw_alloc - * - * Bind a protection domain to a memory window - * - * Input: - * rmr Initialized rmr to hold binding handles - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN dapls_ib_mw_alloc(IN DAPL_RMR * rmr) -{ - - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_alloc: NOT IMPLEMENTED\n"); - - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); -} - -/* - * dapls_ib_mw_free - * - * Release bindings of a protection domain to a memory window - * - * Input: - * rmr Initialized rmr to hold binding handles - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_STATE - * - */ -DAT_RETURN dapls_ib_mw_free(IN DAPL_RMR * rmr) -{ - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_free: NOT IMPLEMENTED\n"); - - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); -} - -/* - * dapls_ib_mw_bind - * - * Bind a protection domain to a memory window - * - * Input: - * rmr Initialized rmr to hold binding handles - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_PARAMETER; - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN -dapls_ib_mw_bind(IN DAPL_RMR * rmr, - IN DAPL_LMR * lmr, - IN DAPL_EP * ep, - IN DAPL_COOKIE * cookie, - IN DAT_VADDR virtual_address, - IN DAT_VLEN length, - IN DAT_MEM_PRIV_FLAGS mem_priv, IN DAT_BOOLEAN is_signaled) -{ - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_bind: NOT IMPLEMENTED\n"); - - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); -} - -/* - * dapls_ib_mw_unbind - * - * Unbind a protection domain from a memory window - * - * Input: - * rmr Initialized rmr to hold binding handles - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_PARAMETER; - * DAT_INVALID_STATE; - * DAT_INSUFFICIENT_RESOURCES - * - */ -DAT_RETURN -dapls_ib_mw_unbind(IN DAPL_RMR * rmr, - IN DAPL_EP * ep, - IN DAPL_COOKIE * cookie, IN DAT_BOOLEAN is_signaled) -{ - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " mw_unbind: NOT IMPLEMENTED\n"); - - return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE); -} - -/* - * Local variables: - * c-indent-level: 4 - * c-basic-offset: 4 - * tab-width: 8 - * End: - */ diff --git a/dapl/openib_scm/dapl_ib_qp.c b/dapl/openib_scm/dapl_ib_qp.c deleted file mode 100644 index f943ff8..0000000 --- a/dapl/openib_scm/dapl_ib_qp.c +++ /dev/null @@ -1,513 +0,0 @@ -/* - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/********************************************************************** - * - * MODULE: dapl_ib_qp.c - * - * PURPOSE: QP routines for access to ofa rdma verbs - * - * $Id: $ - **********************************************************************/ - -#include "dapl.h" -#include "dapl_adapter_util.h" - -/* - * dapl_ib_qp_alloc - * - * Alloc a QP - * - * Input: - * *ep_ptr pointer to EP INFO - * ib_hca_handle provider HCA handle - * ib_pd_handle provider protection domain handle - * cq_recv provider recv CQ handle - * cq_send provider send CQ handle - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INTERNAL_ERROR - * - */ -DAT_RETURN -dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr, - IN DAPL_EP * ep_ptr, IN DAPL_EP * ep_ctx_ptr) -{ - DAT_EP_ATTR *attr; - DAPL_EVD *rcv_evd, *req_evd; - ib_cq_handle_t rcv_cq, req_cq; - ib_pd_handle_t ib_pd_handle; - struct ibv_qp_init_attr qp_create; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " qp_alloc: ia_ptr %p ep_ptr %p ep_ctx_ptr %p\n", - ia_ptr, ep_ptr, ep_ctx_ptr); - - attr = &ep_ptr->param.ep_attr; - ib_pd_handle = ((DAPL_PZ *) ep_ptr->param.pz_handle)->pd_handle; - rcv_evd = (DAPL_EVD *) ep_ptr->param.recv_evd_handle; - req_evd = (DAPL_EVD *) ep_ptr->param.request_evd_handle; - - /* - * DAT allows usage model of EP's with no EVD's but IB does not. - * Create a CQ with zero entries under the covers to support and - * catch any invalid posting. - */ - if (rcv_evd != DAT_HANDLE_NULL) - rcv_cq = rcv_evd->ib_cq_handle; - else if (!ia_ptr->hca_ptr->ib_trans.ib_cq_empty) - rcv_cq = ia_ptr->hca_ptr->ib_trans.ib_cq_empty; - else { - struct ibv_comp_channel *channel = - ia_ptr->hca_ptr->ib_trans.ib_cq; -#ifdef CQ_WAIT_OBJECT - if (rcv_evd->cq_wait_obj_handle) - channel = rcv_evd->cq_wait_obj_handle; -#endif - /* Call IB verbs to create CQ */ - rcv_cq = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, - 0, NULL, channel, 0); - - if (rcv_cq == IB_INVALID_HANDLE) - return (dapl_convert_errno(ENOMEM, "create_cq")); - - ia_ptr->hca_ptr->ib_trans.ib_cq_empty = rcv_cq; - } - if (req_evd != DAT_HANDLE_NULL) - req_cq = req_evd->ib_cq_handle; - else - req_cq = ia_ptr->hca_ptr->ib_trans.ib_cq_empty; - - /* Setup attributes and create qp */ - dapl_os_memzero((void *)&qp_create, sizeof(qp_create)); - qp_create.send_cq = req_cq; - qp_create.cap.max_send_wr = attr->max_request_dtos; - qp_create.cap.max_send_sge = attr->max_request_iov; - qp_create.cap.max_inline_data = - ia_ptr->hca_ptr->ib_trans.max_inline_send; - qp_create.qp_type = IBV_QPT_RC; - -#ifdef DAT_EXTENSIONS - if (attr->service_type == DAT_IB_SERVICE_TYPE_UD) { - qp_create.qp_type = IBV_QPT_UD; - if (attr->max_message_size > - (128 << ia_ptr->hca_ptr->ib_trans.mtu)) { - return (DAT_INVALID_PARAMETER | DAT_INVALID_ARG6); - } - } -#endif - qp_create.qp_context = (void *)ep_ptr; - - /* ibv assumes rcv_cq is never NULL, set to req_cq */ - if (rcv_cq == NULL) { - qp_create.recv_cq = req_cq; - qp_create.cap.max_recv_wr = 0; - qp_create.cap.max_recv_sge = 0; - } else { - qp_create.recv_cq = rcv_cq; - qp_create.cap.max_recv_wr = attr->max_recv_dtos; - qp_create.cap.max_recv_sge = attr->max_recv_iov; - } - - ep_ptr->qp_handle = ibv_create_qp(ib_pd_handle, &qp_create); - if (!ep_ptr->qp_handle) - return (dapl_convert_errno(ENOMEM, "create_qp")); - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " qp_alloc: qpn %p sq %d,%d rq %d,%d\n", - ep_ptr->qp_handle->qp_num, - qp_create.cap.max_send_wr, qp_create.cap.max_send_sge, - qp_create.cap.max_recv_wr, qp_create.cap.max_recv_sge); - - /* Setup QP attributes for INIT state on the way out */ - if (dapls_modify_qp_state(ep_ptr->qp_handle, - IBV_QPS_INIT, NULL) != DAT_SUCCESS) { - ibv_destroy_qp(ep_ptr->qp_handle); - ep_ptr->qp_handle = IB_INVALID_HANDLE; - return DAT_INTERNAL_ERROR; - } - - return DAT_SUCCESS; -} - -/* - * dapl_ib_qp_free - * - * Free a QP - * - * Input: - * ia_handle IA handle - * *ep_ptr pointer to EP INFO - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * dapl_convert_errno - * - */ -DAT_RETURN dapls_ib_qp_free(IN DAPL_IA * ia_ptr, IN DAPL_EP * ep_ptr) -{ - dapl_dbg_log(DAPL_DBG_TYPE_EP, " qp_free: ep_ptr %p qp %p\n", - ep_ptr, ep_ptr->qp_handle); - - if (ep_ptr->qp_handle != IB_INVALID_HANDLE) { - /* force error state to flush queue, then destroy */ - dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_ERR, NULL); - - if (ibv_destroy_qp(ep_ptr->qp_handle)) - return (dapl_convert_errno(errno, "destroy_qp")); - - ep_ptr->qp_handle = IB_INVALID_HANDLE; - } - -#ifdef DAT_EXTENSIONS -{ - dp_ib_cm_handle_t cr, next_cr; - - /* - * UD CR objects are kept active because of direct private data references - * from CONN events. The cr->socket is closed and marked inactive but the - * object remains allocated and queued on the CR resource list. There can - * be multiple CR's associated with a given EP. There is no way to determine - * when consumer is finished with event until the dat_ep_free. - * - * Schedule destruction for all CR's associated with this EP, cr_thread will - * complete the cleanup with state == SCM_DESTROY. - */ - dapl_os_lock(&ia_ptr->hca_ptr->ib_trans.lock); - if (!dapl_llist_is_empty((DAPL_LLIST_HEAD*) - &ia_ptr->hca_ptr->ib_trans.list)) - next_cr = dapl_llist_peek_head((DAPL_LLIST_HEAD*) - &ia_ptr->hca_ptr->ib_trans.list); - else - next_cr = NULL; - - while (next_cr) { - cr = next_cr; - next_cr = dapl_llist_next_entry((DAPL_LLIST_HEAD*) - &ia_ptr->hca_ptr->ib_trans.list, - (DAPL_LLIST_ENTRY*)&cr->entry); - if (cr->ep == ep_ptr) { - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " qp_free CR: ep %p cr %p\n", ep_ptr, cr); - dapli_socket_disconnect(cr); - dapl_os_lock(&cr->lock); - cr->ep = NULL; - cr->state = SCM_DESTROY; - dapl_os_unlock(&cr->lock); - } - } - dapl_os_unlock(&ia_ptr->hca_ptr->ib_trans.lock); - send(ia_ptr->hca_ptr->ib_trans.scm[1], "w", sizeof "w", 0); -} -#endif - - return DAT_SUCCESS; -} - -/* - * dapl_ib_qp_modify - * - * Set the QP to the parameters specified in an EP_PARAM - * - * The EP_PARAM structure that is provided has been - * sanitized such that only non-zero values are valid. - * - * Input: - * ib_hca_handle HCA handle - * qp_handle QP handle - * ep_attr Sanitized EP Params - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INVALID_PARAMETER - * - */ -DAT_RETURN -dapls_ib_qp_modify(IN DAPL_IA * ia_ptr, - IN DAPL_EP * ep_ptr, IN DAT_EP_ATTR * attr) -{ - struct ibv_qp_attr qp_attr; - - if (ep_ptr->qp_handle == IB_INVALID_HANDLE) - return DAT_INVALID_PARAMETER; - - /* - * EP state, qp_handle state should be an indication - * of current state but the only way to be sure is with - * a user mode ibv_query_qp call which is NOT available - */ - - /* move to error state if necessary */ - if ((ep_ptr->qp_state == IB_QP_STATE_ERROR) && - (ep_ptr->qp_handle->state != IBV_QPS_ERR)) { - return (dapls_modify_qp_state(ep_ptr->qp_handle, - IBV_QPS_ERR, NULL)); - } - - /* - * Check if we have the right qp_state to modify attributes - */ - if ((ep_ptr->qp_handle->state != IBV_QPS_RTR) && - (ep_ptr->qp_handle->state != IBV_QPS_RTS)) - return DAT_INVALID_STATE; - - /* Adjust to current EP attributes */ - dapl_os_memzero((void *)&qp_attr, sizeof(qp_attr)); - qp_attr.cap.max_send_wr = attr->max_request_dtos; - qp_attr.cap.max_recv_wr = attr->max_recv_dtos; - qp_attr.cap.max_send_sge = attr->max_request_iov; - qp_attr.cap.max_recv_sge = attr->max_recv_iov; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - "modify_qp: qp %p sq %d,%d, rq %d,%d\n", - ep_ptr->qp_handle, - qp_attr.cap.max_send_wr, qp_attr.cap.max_send_sge, - qp_attr.cap.max_recv_wr, qp_attr.cap.max_recv_sge); - - if (ibv_modify_qp(ep_ptr->qp_handle, &qp_attr, IBV_QP_CAP)) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - "modify_qp: modify ep %p qp %p failed\n", - ep_ptr, ep_ptr->qp_handle); - return (dapl_convert_errno(errno, "modify_qp_state")); - } - - return DAT_SUCCESS; -} - -/* - * dapls_ib_reinit_ep - * - * Move the QP to INIT state again. - * - * Input: - * ep_ptr DAPL_EP - * - * Output: - * none - * - * Returns: - * void - * - */ -#if defined(_WIN32) || defined(_WIN64) -void dapls_ib_reinit_ep(IN DAPL_EP * ep_ptr) -{ - /* work around bug in low level driver - 3/24/09 */ - /* RTS -> RESET -> INIT -> ERROR QP transition crashes system */ - if (ep_ptr->qp_handle != IB_INVALID_HANDLE) { - dapls_ib_qp_free(ep_ptr->header.owner_ia, ep_ptr); - dapls_ib_qp_alloc(ep_ptr->header.owner_ia, ep_ptr, ep_ptr); - } -} -#else // _WIN32 || _WIN64 -void dapls_ib_reinit_ep(IN DAPL_EP * ep_ptr) -{ - if (ep_ptr->qp_handle != IB_INVALID_HANDLE && - ep_ptr->qp_handle->qp_type != IBV_QPT_UD) { - /* move to RESET state and then to INIT */ - dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_RESET, 0); - dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_INIT, 0); - } -} -#endif // _WIN32 || _WIN64 - -/* - * Generic QP modify for init, reset, error, RTS, RTR - * For UD, create_ah on RTR, qkey on INIT - */ -DAT_RETURN -dapls_modify_qp_state(IN ib_qp_handle_t qp_handle, - IN ib_qp_state_t qp_state, IN struct ib_cm_handle *cm_ptr) -{ - struct ibv_qp_attr qp_attr; - enum ibv_qp_attr_mask mask = IBV_QP_STATE; - DAPL_EP *ep_ptr = (DAPL_EP *) qp_handle->qp_context; - DAPL_IA *ia_ptr = ep_ptr->header.owner_ia; - ib_qp_cm_t *qp_cm = &cm_ptr->dst; - int ret; - - dapl_os_memzero((void *)&qp_attr, sizeof(qp_attr)); - qp_attr.qp_state = qp_state; - switch (qp_state) { - /* additional attributes with RTR and RTS */ - case IBV_QPS_RTR: - { - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " QPS_RTR: type %d state %d qpn %x lid %x" - " port %x ep %p qp_state %d\n", - qp_handle->qp_type, qp_handle->qp_type, - qp_cm->qpn, qp_cm->lid, qp_cm->port, - ep_ptr, ep_ptr->qp_state); - - mask |= IBV_QP_AV | - IBV_QP_PATH_MTU | - IBV_QP_DEST_QPN | - IBV_QP_RQ_PSN | - IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER; - - qp_attr.dest_qp_num = qp_cm->qpn; - qp_attr.rq_psn = 1; - qp_attr.path_mtu = ia_ptr->hca_ptr->ib_trans.mtu; - qp_attr.max_dest_rd_atomic = - ep_ptr->param.ep_attr.max_rdma_read_out; - qp_attr.min_rnr_timer = - ia_ptr->hca_ptr->ib_trans.rnr_timer; - - /* address handle. RC and UD */ - qp_attr.ah_attr.dlid = qp_cm->lid; - if (ia_ptr->hca_ptr->ib_trans.global) { - qp_attr.ah_attr.is_global = 1; - qp_attr.ah_attr.grh.dgid = qp_cm->gid; - qp_attr.ah_attr.grh.hop_limit = - ia_ptr->hca_ptr->ib_trans.hop_limit; - qp_attr.ah_attr.grh.traffic_class = - ia_ptr->hca_ptr->ib_trans.tclass; - } - qp_attr.ah_attr.sl = 0; - qp_attr.ah_attr.src_path_bits = 0; - qp_attr.ah_attr.port_num = ia_ptr->hca_ptr->port_num; -#ifdef DAT_EXTENSIONS - /* UD: create AH for remote side */ - if (qp_handle->qp_type == IBV_QPT_UD) { - ib_pd_handle_t pz; - pz = ((DAPL_PZ *) - ep_ptr->param.pz_handle)->pd_handle; - mask = IBV_QP_STATE; - cm_ptr->ah = ibv_create_ah(pz, - &qp_attr.ah_attr); - if (!cm_ptr->ah) - return (dapl_convert_errno(errno, - "ibv_ah")); - - /* already RTR, multi remote AH's on QP */ - if (ep_ptr->qp_state == IBV_QPS_RTR || - ep_ptr->qp_state == IBV_QPS_RTS) - return DAT_SUCCESS; - } -#endif - break; - } - case IBV_QPS_RTS: - { - /* RC only */ - if (qp_handle->qp_type == IBV_QPT_RC) { - mask |= IBV_QP_SQ_PSN | - IBV_QP_TIMEOUT | - IBV_QP_RETRY_CNT | - IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC; - qp_attr.timeout = - ia_ptr->hca_ptr->ib_trans.ack_timer; - qp_attr.retry_cnt = - ia_ptr->hca_ptr->ib_trans.ack_retry; - qp_attr.rnr_retry = - ia_ptr->hca_ptr->ib_trans.rnr_retry; - qp_attr.max_rd_atomic = - ep_ptr->param.ep_attr.max_rdma_read_out; - } - /* RC and UD */ - qp_attr.qp_state = IBV_QPS_RTS; - qp_attr.sq_psn = 1; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " QPS_RTS: psn %x rd_atomic %d ack %d " - " retry %d rnr_retry %d ep %p qp_state %d\n", - qp_attr.sq_psn, qp_attr.max_rd_atomic, - qp_attr.timeout, qp_attr.retry_cnt, - qp_attr.rnr_retry, ep_ptr, - ep_ptr->qp_state); -#ifdef DAT_EXTENSIONS - if (qp_handle->qp_type == IBV_QPT_UD) { - /* already RTS, multi remote AH's on QP */ - if (ep_ptr->qp_state == IBV_QPS_RTS) - return DAT_SUCCESS; - else - mask = IBV_QP_STATE | IBV_QP_SQ_PSN; - } -#endif - break; - } - case IBV_QPS_INIT: - { - mask |= IBV_QP_PKEY_INDEX | IBV_QP_PORT; - if (qp_handle->qp_type == IBV_QPT_RC) { - mask |= IBV_QP_ACCESS_FLAGS; - qp_attr.qp_access_flags = - IBV_ACCESS_LOCAL_WRITE | - IBV_ACCESS_REMOTE_WRITE | - IBV_ACCESS_REMOTE_READ | - IBV_ACCESS_REMOTE_ATOMIC | - IBV_ACCESS_MW_BIND; - } -#ifdef DAT_EXTENSIONS - if (qp_handle->qp_type == IBV_QPT_UD) { - /* already INIT, multi remote AH's on QP */ - if (ep_ptr->qp_state == IBV_QPS_INIT) - return DAT_SUCCESS; - mask |= IBV_QP_QKEY; - qp_attr.qkey = SCM_UD_QKEY; - } -#endif - qp_attr.pkey_index = 0; - qp_attr.port_num = ia_ptr->hca_ptr->port_num; - - dapl_dbg_log(DAPL_DBG_TYPE_EP, - " QPS_INIT: pi %x port %x acc %x qkey 0x%x\n", - qp_attr.pkey_index, qp_attr.port_num, - qp_attr.qp_access_flags, qp_attr.qkey); - break; - } - default: - break; - - } - - ret = ibv_modify_qp(qp_handle, &qp_attr, mask); - if (ret == 0) { - ep_ptr->qp_state = qp_state; - return DAT_SUCCESS; - } else { - return (dapl_convert_errno(errno, "modify_qp_state")); - } -} - -/* - * Local variables: - * c-indent-level: 4 - * c-basic-offset: 4 - * tab-width: 8 - * End: - */ diff --git a/dapl/openib_scm/dapl_ib_util.c b/dapl/openib_scm/dapl_ib_util.c deleted file mode 100644 index ad30f73..0000000 --- a/dapl/openib_scm/dapl_ib_util.c +++ /dev/null @@ -1,743 +0,0 @@ -/* - * This Software is licensed under one of the following licenses: - * - * 1) under the terms of the "Common Public License 1.0" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/cpl.php. - * - * 2) under the terms of the "The BSD License" a copy of which is - * available from the Open Source Initiative, see - * http://www.opensource.org/licenses/bsd-license.php. - * - * 3) under the terms of the "GNU General Public License (GPL) Version 2" a - * copy of which is available from the Open Source Initiative, see - * http://www.opensource.org/licenses/gpl-license.php. - * - * Licensee has the right to choose one of the above licenses. - * - * Redistributions of source code must retain the above copyright - * notice and one of the license notices. - * - * Redistributions in binary form must reproduce both the above copyright - * notice, one of the license notices in the documentation - * and/or other materials provided with the distribution. - */ - -/*************************************************************************** - * - * Module: uDAPL - * - * Filename: dapl_ib_util.c - * - * Author: Arlin Davis - * - * Created: 3/10/2005 - * - * Description: - * - * The uDAPL openib provider - init, open, close, utilities - * - **************************************************************************** - * Source Control System Information - * - * $Id: $ - * - * Copyright (c) 2005 Intel Corporation. All rights reserved. - * - **************************************************************************/ -#ifdef RCSID -static const char rcsid[] = "$Id: $"; -#endif - -#include "openib_osd.h" -#include "dapl.h" -#include "dapl_adapter_util.h" -#include "dapl_ib_util.h" -#include "dapl_osd.h" - -#include - -int g_dapl_loopback_connection = 0; - -enum ibv_mtu dapl_ib_mtu(int mtu) -{ - switch (mtu) { - case 256: - return IBV_MTU_256; - case 512: - return IBV_MTU_512; - case 1024: - return IBV_MTU_1024; - case 2048: - return IBV_MTU_2048; - case 4096: - return IBV_MTU_4096; - default: - return IBV_MTU_1024; - } -} - -char *dapl_ib_mtu_str(enum ibv_mtu mtu) -{ - switch (mtu) { - case IBV_MTU_256: - return "256"; - case IBV_MTU_512: - return "512"; - case IBV_MTU_1024: - return "1024"; - case IBV_MTU_2048: - return "2048"; - case IBV_MTU_4096: - return "4096"; - default: - return "1024"; - } -} - -static DAT_RETURN getlocalipaddr(DAT_SOCK_ADDR * addr, int addr_len) -{ - struct sockaddr_in *sin; - struct addrinfo *res, hint, *ai; - int ret; - char hostname[256]; - - if (addr_len < sizeof(*sin)) { - return DAT_INTERNAL_ERROR; - } - - ret = gethostname(hostname, 256); - if (ret) - return dapl_convert_errno(ret, "gethostname"); - - memset(&hint, 0, sizeof hint); - hint.ai_flags = AI_PASSIVE; - hint.ai_family = AF_INET; - hint.ai_socktype = SOCK_STREAM; - hint.ai_protocol = IPPROTO_TCP; - - ret = getaddrinfo(hostname, NULL, &hint, &res); - if (ret) { - dapl_log(DAPL_DBG_TYPE_ERR, - " getaddrinfo ERR: %d %s\n", ret, gai_strerror(ret)); - return DAT_INVALID_ADDRESS; - } - - ret = DAT_INVALID_ADDRESS; - for (ai = res; ai; ai = ai->ai_next) { - sin = (struct sockaddr_in *)ai->ai_addr; - if (*((uint32_t *) & sin->sin_addr) != htonl(0x7f000001)) { - *((struct sockaddr_in *)addr) = *sin; - ret = DAT_SUCCESS; - break; - } - } - - freeaddrinfo(res); - return ret; -} - -static int32_t create_cr_pipe(IN DAPL_HCA * hca_ptr) -{ - DAPL_SOCKET listen_socket; - struct sockaddr_in addr; - socklen_t addrlen = sizeof(addr); - int ret; - - listen_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - if (listen_socket == DAPL_INVALID_SOCKET) - return 1; - - memset(&addr, 0, sizeof addr); - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = htonl(0x7f000001); - ret = bind(listen_socket, (struct sockaddr *)&addr, sizeof addr); - if (ret) - goto err1; - - ret = getsockname(listen_socket, (struct sockaddr *)&addr, &addrlen); - if (ret) - goto err1; - - ret = listen(listen_socket, 0); - if (ret) - goto err1; - - hca_ptr->ib_trans.scm[1] = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - if (hca_ptr->ib_trans.scm[1] == DAPL_INVALID_SOCKET) - goto err1; - - ret = connect(hca_ptr->ib_trans.scm[1], - (struct sockaddr *)&addr, sizeof(addr)); - if (ret) - goto err2; - - hca_ptr->ib_trans.scm[0] = accept(listen_socket, NULL, NULL); - if (hca_ptr->ib_trans.scm[0] == DAPL_INVALID_SOCKET) - goto err2; - - closesocket(listen_socket); - return 0; - - err2: - closesocket(hca_ptr->ib_trans.scm[1]); - err1: - closesocket(listen_socket); - return 1; -} - -static void destroy_cr_pipe(IN DAPL_HCA * hca_ptr) -{ - closesocket(hca_ptr->ib_trans.scm[0]); - closesocket(hca_ptr->ib_trans.scm[1]); -} - - -/* - * dapls_ib_init, dapls_ib_release - * - * Initialize Verb related items for device open - * - * Input: - * none - * - * Output: - * none - * - * Returns: - * 0 success, -1 error - * - */ -int32_t dapls_ib_init(void) -{ - return 0; -} - -int32_t dapls_ib_release(void) -{ - return 0; -} - -#if defined(_WIN64) || defined(_WIN32) -int dapls_config_comp_channel(struct ibv_comp_channel *channel) -{ - return 0; -} -#else // _WIN64 || WIN32 -int dapls_config_comp_channel(struct ibv_comp_channel *channel) -{ - int opts; - - opts = fcntl(channel->fd, F_GETFL); /* uCQ */ - if (opts < 0 || fcntl(channel->fd, F_SETFL, opts | O_NONBLOCK) < 0) { - dapl_log(DAPL_DBG_TYPE_ERR, - " dapls_create_comp_channel: fcntl on ib_cq->fd %d ERR %d %s\n", - channel->fd, opts, strerror(errno)); - return errno; - } - - return 0; -} -#endif - -/* - * dapls_ib_open_hca - * - * Open HCA - * - * Input: - * *hca_name pointer to provider device name - * *ib_hca_handle_p pointer to provide HCA handle - * - * Output: - * none - * - * Return: - * DAT_SUCCESS - * dapl_convert_errno - * - */ -DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr) -{ - struct ibv_device **dev_list; - struct ibv_port_attr port_attr; - int i; - DAT_RETURN dat_status; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " open_hca: %s - %p\n", hca_name, hca_ptr); - - /* get the IP address of the device */ - dat_status = getlocalipaddr((DAT_SOCK_ADDR *) & hca_ptr->hca_address, - sizeof(DAT_SOCK_ADDR6)); - if (dat_status != DAT_SUCCESS) - return dat_status; - - /* Get list of all IB devices, find match, open */ - dev_list = ibv_get_device_list(NULL); - if (!dev_list) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, - " open_hca: ibv_get_device_list() failed\n", - hca_name); - return DAT_INTERNAL_ERROR; - } - - for (i = 0; dev_list[i]; ++i) { - hca_ptr->ib_trans.ib_dev = dev_list[i]; - if (!strcmp(ibv_get_device_name(hca_ptr->ib_trans.ib_dev), - hca_name)) - goto found; - } - - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: device %s not found\n", hca_name); - goto err; - - found: - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " open_hca: Found dev %s %016llx\n", - ibv_get_device_name(hca_ptr->ib_trans.ib_dev), - (unsigned long long) - ntohll(ibv_get_device_guid(hca_ptr->ib_trans.ib_dev))); - - hca_ptr->ib_hca_handle = ibv_open_device(hca_ptr->ib_trans.ib_dev); - if (!hca_ptr->ib_hca_handle) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: dev open failed for %s, err=%s\n", - ibv_get_device_name(hca_ptr->ib_trans.ib_dev), - strerror(errno)); - goto err; - } - - /* get lid for this hca-port, network order */ - if (ibv_query_port(hca_ptr->ib_hca_handle, - (uint8_t) hca_ptr->port_num, &port_attr)) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: get lid ERR for %s, err=%s\n", - ibv_get_device_name(hca_ptr->ib_trans.ib_dev), - strerror(errno)); - goto err; - } else { - hca_ptr->ib_trans.lid = htons(port_attr.lid); - } - - /* get gid for this hca-port, network order */ - if (ibv_query_gid(hca_ptr->ib_hca_handle, - (uint8_t) hca_ptr->port_num, - 0, &hca_ptr->ib_trans.gid)) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: query GID ERR for %s, err=%s\n", - ibv_get_device_name(hca_ptr->ib_trans.ib_dev), - strerror(errno)); - goto err; - } - - /* set RC tunables via enviroment or default */ - hca_ptr->ib_trans.max_inline_send = - dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_DEFAULT); - hca_ptr->ib_trans.ack_retry = - dapl_os_get_env_val("DAPL_ACK_RETRY", SCM_ACK_RETRY); - hca_ptr->ib_trans.ack_timer = - dapl_os_get_env_val("DAPL_ACK_TIMER", SCM_ACK_TIMER); - hca_ptr->ib_trans.rnr_retry = - dapl_os_get_env_val("DAPL_RNR_RETRY", SCM_RNR_RETRY); - hca_ptr->ib_trans.rnr_timer = - dapl_os_get_env_val("DAPL_RNR_TIMER", SCM_RNR_TIMER); - hca_ptr->ib_trans.global = - dapl_os_get_env_val("DAPL_GLOBAL_ROUTING", SCM_GLOBAL); - hca_ptr->ib_trans.hop_limit = - dapl_os_get_env_val("DAPL_HOP_LIMIT", SCM_HOP_LIMIT); - hca_ptr->ib_trans.tclass = - dapl_os_get_env_val("DAPL_TCLASS", SCM_TCLASS); - hca_ptr->ib_trans.mtu = - dapl_ib_mtu(dapl_os_get_env_val("DAPL_IB_MTU", SCM_IB_MTU)); - -#ifndef CQ_WAIT_OBJECT - /* initialize cq_lock */ - dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.cq_lock); - if (dat_status != DAT_SUCCESS) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: failed to init cq_lock\n"); - goto bail; - } - /* EVD events without direct CQ channels, non-blocking */ - hca_ptr->ib_trans.ib_cq = - ibv_create_comp_channel(hca_ptr->ib_hca_handle); - if (hca_ptr->ib_trans.ib_cq == NULL) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: ibv_create_comp_channel ERR %s\n", - strerror(errno)); - goto bail; - } - - if (dapls_config_comp_channel(hca_ptr->ib_trans.ib_cq)) { - goto bail; - } - - if (dapli_cq_thread_init(hca_ptr)) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: cq_thread_init failed for %s\n", - ibv_get_device_name(hca_ptr->ib_trans.ib_dev)); - goto bail; - } -#endif /* CQ_WAIT_OBJECT */ - - /* initialize cr_list lock */ - dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.lock); - if (dat_status != DAT_SUCCESS) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: failed to init cr_list lock\n"); - goto bail; - } - - /* initialize CM list for listens on this HCA */ - dapl_llist_init_head(&hca_ptr->ib_trans.list); - - /* initialize pipe, user level wakeup on select */ - if (create_cr_pipe(hca_ptr)) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: failed to init cr pipe - %s\n", - strerror(errno)); - goto bail; - } - - /* create thread to process inbound connect request */ - hca_ptr->ib_trans.cr_state = IB_THREAD_INIT; - dat_status = dapl_os_thread_create(cr_thread, - (void *)hca_ptr, - &hca_ptr->ib_trans.thread); - if (dat_status != DAT_SUCCESS) { - dapl_log(DAPL_DBG_TYPE_ERR, - " open_hca: failed to create thread\n"); - goto bail; - } - - /* wait for thread */ - while (hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) { - dapl_os_sleep_usec(2000); - } - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " open_hca: devname %s, port %d, hostname_IP %s\n", - ibv_get_device_name(hca_ptr->ib_trans.ib_dev), - hca_ptr->port_num, inet_ntoa(((struct sockaddr_in *) - &hca_ptr->hca_address)-> - sin_addr)); - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " open_hca: LID 0x%x GID Subnet 0x" F64x " ID 0x" F64x - "\n", ntohs(hca_ptr->ib_trans.lid), (unsigned long long) - htonll(hca_ptr->ib_trans.gid.global.subnet_prefix), - (unsigned long long)htonll(hca_ptr->ib_trans.gid.global. - interface_id)); - - ibv_free_device_list(dev_list); - return dat_status; - - bail: - ibv_close_device(hca_ptr->ib_hca_handle); - hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; - err: - ibv_free_device_list(dev_list); - return DAT_INTERNAL_ERROR; -} - -/* - * dapls_ib_close_hca - * - * Open HCA - * - * Input: - * DAPL_HCA provide CA handle - * - * Output: - * none - * - * Return: - * DAT_SUCCESS - * dapl_convert_errno - * - */ -DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p\n", hca_ptr); - -#ifndef CQ_WAIT_OBJECT - dapli_cq_thread_destroy(hca_ptr); - dapl_os_lock_destroy(&hca_ptr->ib_trans.cq_lock); -#endif /* CQ_WAIT_OBJECT */ - - if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) { - if (ibv_close_device(hca_ptr->ib_hca_handle)) - return (dapl_convert_errno(errno, "ib_close_device")); - hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; - } - - /* destroy cr_thread and lock */ - hca_ptr->ib_trans.cr_state = IB_THREAD_CANCEL; - if (send(hca_ptr->ib_trans.scm[1], "w", sizeof "w", 0) == -1) - dapl_log(DAPL_DBG_TYPE_UTIL, - " thread_destroy: thread wakeup err = %s\n", - strerror(errno)); - while (hca_ptr->ib_trans.cr_state != IB_THREAD_EXIT) { - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " close_hca: waiting for cr_thread\n"); - if (send(hca_ptr->ib_trans.scm[1], "w", sizeof "w", 0) == -1) - dapl_log(DAPL_DBG_TYPE_UTIL, - " thread_destroy: thread wakeup err = %s\n", - strerror(errno)); - dapl_os_sleep_usec(2000); - } - dapl_os_lock_destroy(&hca_ptr->ib_trans.lock); - destroy_cr_pipe(hca_ptr); /* no longer need pipe */ - return (DAT_SUCCESS); -} - -/* - * dapls_ib_query_hca - * - * Query the hca attribute - * - * Input: - * hca_handl hca handle - * ia_attr attribute of the ia - * ep_attr attribute of the ep - * ip_addr ip address of DET NIC - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INVALID_HANDLE - */ - -DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr, - OUT DAT_IA_ATTR * ia_attr, - OUT DAT_EP_ATTR * ep_attr, - OUT DAT_SOCK_ADDR6 * ip_addr) -{ - struct ibv_device_attr dev_attr; - struct ibv_port_attr port_attr; - - if (hca_ptr->ib_hca_handle == NULL) { - dapl_dbg_log(DAPL_DBG_TYPE_ERR, " query_hca: BAD handle\n"); - return (DAT_INVALID_HANDLE); - } - - /* local IP address of device, set during ia_open */ - if (ip_addr != NULL) - memcpy(ip_addr, &hca_ptr->hca_address, sizeof(DAT_SOCK_ADDR6)); - - if (ia_attr == NULL && ep_attr == NULL) - return DAT_SUCCESS; - - /* query verbs for this device and port attributes */ - if (ibv_query_device(hca_ptr->ib_hca_handle, &dev_attr) || - ibv_query_port(hca_ptr->ib_hca_handle, - hca_ptr->port_num, &port_attr)) - return (dapl_convert_errno(errno, "ib_query_hca")); - - if (ia_attr != NULL) { - (void)dapl_os_memzero(ia_attr, sizeof(*ia_attr)); - ia_attr->adapter_name[DAT_NAME_MAX_LENGTH - 1] = '\0'; - ia_attr->vendor_name[DAT_NAME_MAX_LENGTH - 1] = '\0'; - ia_attr->ia_address_ptr = - (DAT_IA_ADDRESS_PTR) & hca_ptr->hca_address; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " query_hca: %s %s \n", - ibv_get_device_name(hca_ptr->ib_trans.ib_dev), - inet_ntoa(((struct sockaddr_in *) - &hca_ptr->hca_address)->sin_addr)); - - ia_attr->hardware_version_major = dev_attr.hw_ver; - /* ia_attr->hardware_version_minor = dev_attr.fw_ver; */ - ia_attr->max_eps = dev_attr.max_qp; - ia_attr->max_dto_per_ep = dev_attr.max_qp_wr; - ia_attr->max_rdma_read_in = dev_attr.max_qp_rd_atom; - ia_attr->max_rdma_read_out = dev_attr.max_qp_init_rd_atom; - ia_attr->max_rdma_read_per_ep_in = dev_attr.max_qp_rd_atom; - ia_attr->max_rdma_read_per_ep_out = - dev_attr.max_qp_init_rd_atom; - ia_attr->max_rdma_read_per_ep_in_guaranteed = DAT_TRUE; - ia_attr->max_rdma_read_per_ep_out_guaranteed = DAT_TRUE; - ia_attr->max_evds = dev_attr.max_cq; - ia_attr->max_evd_qlen = dev_attr.max_cqe; - ia_attr->max_iov_segments_per_dto = dev_attr.max_sge; - ia_attr->max_lmrs = dev_attr.max_mr; - /* 32bit attribute from 64bit, 4G-1 limit, DAT v2 needs fix */ - ia_attr->max_lmr_block_size = - (dev_attr.max_mr_size >> 32) ? ~0 : dev_attr.max_mr_size; - ia_attr->max_rmrs = dev_attr.max_mw; - ia_attr->max_lmr_virtual_address = dev_attr.max_mr_size; - ia_attr->max_rmr_target_address = dev_attr.max_mr_size; - ia_attr->max_pzs = dev_attr.max_pd; - ia_attr->max_message_size = port_attr.max_msg_sz; - ia_attr->max_rdma_size = port_attr.max_msg_sz; - ia_attr->max_iov_segments_per_rdma_read = dev_attr.max_sge; - ia_attr->max_iov_segments_per_rdma_write = dev_attr.max_sge; - ia_attr->num_transport_attr = 0; - ia_attr->transport_attr = NULL; - ia_attr->num_vendor_attr = 0; - ia_attr->vendor_attr = NULL; -#ifdef DAT_EXTENSIONS - ia_attr->extension_supported = DAT_EXTENSION_IB; - ia_attr->extension_version = DAT_IB_EXTENSION_VERSION; -#endif - hca_ptr->ib_trans.mtu = DAPL_MIN(port_attr.active_mtu, - hca_ptr->ib_trans.mtu); - hca_ptr->ib_trans.ack_timer = - DAPL_MAX(dev_attr.local_ca_ack_delay, - hca_ptr->ib_trans.ack_timer); - - /* set MTU in transport specific named attribute */ - hca_ptr->ib_trans.named_attr.name = "DAT_IB_TRANSPORT_MTU"; - hca_ptr->ib_trans.named_attr.value = - dapl_ib_mtu_str(hca_ptr->ib_trans.mtu); - - dapl_log(DAPL_DBG_TYPE_UTIL, - " query_hca: (%x.%x) ep %d ep_q %d evd %d" - " evd_q %d mtu %d\n", - ia_attr->hardware_version_major, - ia_attr->hardware_version_minor, - ia_attr->max_eps, ia_attr->max_dto_per_ep, - ia_attr->max_evds, ia_attr->max_evd_qlen, - 128 << hca_ptr->ib_trans.mtu); - - dapl_log(DAPL_DBG_TYPE_UTIL, - " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d" - " ack_time %d mr %u\n", - ia_attr->max_message_size, ia_attr->max_rdma_size, - ia_attr->max_iov_segments_per_dto, - ia_attr->max_lmrs, ia_attr->max_rmrs, - hca_ptr->ib_trans.ack_timer, - ia_attr->max_lmr_block_size); - } - - if (ep_attr != NULL) { - (void)dapl_os_memzero(ep_attr, sizeof(*ep_attr)); - ep_attr->max_message_size = port_attr.max_msg_sz; - ep_attr->max_rdma_size = port_attr.max_msg_sz; - ep_attr->max_recv_dtos = dev_attr.max_qp_wr; - ep_attr->max_request_dtos = dev_attr.max_qp_wr; - ep_attr->max_recv_iov = dev_attr.max_sge; - ep_attr->max_request_iov = dev_attr.max_sge; - ep_attr->max_rdma_read_in = dev_attr.max_qp_rd_atom; - ep_attr->max_rdma_read_out = dev_attr.max_qp_init_rd_atom; - ep_attr->max_rdma_read_iov = dev_attr.max_sge; - ep_attr->max_rdma_write_iov = dev_attr.max_sge; - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " query_hca: MAX msg %llu mtu %d dto %d iov %d" - " rdma i%d,o%d\n", - ep_attr->max_message_size, - ep_attr->max_recv_dtos, ep_attr->max_recv_iov, - ep_attr->max_rdma_read_in, - ep_attr->max_rdma_read_out); - } - return DAT_SUCCESS; -} - -/* - * dapls_ib_setup_async_callback - * - * Set up an asynchronous callbacks of various kinds - * - * Input: - * ia_handle IA handle - * handler_type type of handler to set up - * callback_handle handle param for completion callbacks - * callback callback routine pointer - * context argument for callback routine - * - * Output: - * none - * - * Returns: - * DAT_SUCCESS - * DAT_INSUFFICIENT_RESOURCES - * DAT_INVALID_PARAMETER - * - */ -DAT_RETURN dapls_ib_setup_async_callback(IN DAPL_IA * ia_ptr, - IN DAPL_ASYNC_HANDLER_TYPE - handler_type, IN DAPL_EVD * evd_ptr, - IN ib_async_handler_t callback, - IN void *context) -{ - ib_hca_transport_t *hca_ptr; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " setup_async_cb: ia %p type %d handle %p cb %p ctx %p\n", - ia_ptr, handler_type, evd_ptr, callback, context); - - hca_ptr = &ia_ptr->hca_ptr->ib_trans; - switch (handler_type) { - case DAPL_ASYNC_UNAFILIATED: - hca_ptr->async_unafiliated = (ib_async_handler_t) callback; - hca_ptr->async_un_ctx = context; - break; - case DAPL_ASYNC_CQ_ERROR: - hca_ptr->async_cq_error = (ib_async_cq_handler_t) callback; - break; - case DAPL_ASYNC_CQ_COMPLETION: - hca_ptr->async_cq = (ib_async_dto_handler_t) callback; - break; - case DAPL_ASYNC_QP_ERROR: - hca_ptr->async_qp_error = (ib_async_qp_handler_t) callback; - break; - default: - break; - } - return DAT_SUCCESS; -} - -/* - * dapls_set_provider_specific_attr - * - * Input: - * attr_ptr Pointer provider specific attributes - * - * Output: - * none - * - * Returns: - * void - */ -DAT_NAMED_ATTR ib_attrs[] = { - { - "DAT_IB_TRANSPORT_MTU", "2048"} - , -#ifdef DAT_EXTENSIONS - { - "DAT_EXTENSION_INTERFACE", "TRUE"} - , - { - DAT_IB_ATTR_FETCH_AND_ADD, "TRUE"} - , - { - DAT_IB_ATTR_CMP_AND_SWAP, "TRUE"} - , - { - DAT_IB_ATTR_IMMED_DATA, "TRUE"} - , - { - DAT_IB_ATTR_UD, "TRUE"} - , -#ifdef DAPL_COUNTERS - { - DAT_ATTR_COUNTERS, "TRUE"} - , -#endif /* DAPL_COUNTERS */ -#endif -}; - -#define SPEC_ATTR_SIZE( x ) (sizeof( x ) / sizeof( DAT_NAMED_ATTR)) - -void dapls_query_provider_specific_attr(IN DAPL_IA * ia_ptr, - IN DAT_PROVIDER_ATTR * attr_ptr) -{ - attr_ptr->num_provider_specific_attr = SPEC_ATTR_SIZE(ib_attrs); - attr_ptr->provider_specific_attr = ib_attrs; - - /* set MTU to actual settings */ - ib_attrs[0].value = ia_ptr->hca_ptr->ib_trans.named_attr.value; -} diff --git a/dapl/openib_scm/dapl_ib_util.h b/dapl/openib_scm/dapl_ib_util.h index a668af7..a5e734e 100644 --- a/dapl/openib_scm/dapl_ib_util.h +++ b/dapl/openib_scm/dapl_ib_util.h @@ -23,92 +23,19 @@ * and/or other materials provided with the distribution. */ -/*************************************************************************** - * - * Module: uDAPL - * - * Filename: dapl_ib_util.h - * - * Author: Arlin Davis - * - * Created: 3/10/2005 - * - * Description: - * - * The uDAPL openib provider - definitions, prototypes, - * - **************************************************************************** - * Source Control System Information - * - * $Id: $ - * - * Copyright (c) 2005 Intel Corporation. All rights reserved. - * - **************************************************************************/ - #ifndef _DAPL_IB_UTIL_H_ #define _DAPL_IB_UTIL_H_ +#define _OPENIB_SCM_ -#include "openib_osd.h" #include - -#ifdef DAT_EXTENSIONS -#include -#endif - -#ifndef __cplusplus -#define false 0 -#define true 1 -#endif /*__cplusplus */ - -/* Typedefs to map common DAPL provider types to IB verbs */ -typedef struct ibv_qp *ib_qp_handle_t; -typedef struct ibv_cq *ib_cq_handle_t; -typedef struct ibv_pd *ib_pd_handle_t; -typedef struct ibv_mr *ib_mr_handle_t; -typedef struct ibv_mw *ib_mw_handle_t; -typedef struct ibv_wc ib_work_completion_t; - -/* HCA context type maps to IB verbs */ -typedef struct ibv_context *ib_hca_handle_t; -typedef ib_hca_handle_t dapl_ibal_ca_t; - -/* destination info to exchange, define wire protocol version */ -#define DSCM_VER 4 -typedef struct _ib_qp_cm -{ - uint16_t ver; - uint16_t rej; - uint16_t lid; - uint16_t port; - uint32_t qpn; - uint32_t p_size; - union ibv_gid gid; - DAT_SOCK_ADDR6 ia_address; - uint16_t qp_type; -} ib_qp_cm_t; - -typedef enum scm_state -{ - SCM_INIT, - SCM_LISTEN, - SCM_CONN_PENDING, - SCM_RTU_PENDING, - SCM_ACCEPTING, - SCM_ACCEPTING_DATA, - SCM_ACCEPTED, - SCM_REJECTED, - SCM_CONNECTED, - SCM_RELEASED, - SCM_DISCONNECTED, - SCM_DESTROY -} SCM_STATE; +#include "openib_osd.h" +#include "dapl_ib_common.h" struct ib_cm_handle { struct dapl_llist_entry entry; DAPL_OS_LOCK lock; - SCM_STATE state; + int state; DAPL_SOCKET socket; struct dapl_hca *hca; struct dapl_sp *sp; @@ -121,58 +48,12 @@ struct ib_cm_handle typedef struct ib_cm_handle *dp_ib_cm_handle_t; typedef dp_ib_cm_handle_t ib_cm_srvc_handle_t; -/* CM events */ -typedef enum -{ - IB_CME_CONNECTED, - IB_CME_DISCONNECTED, - IB_CME_DISCONNECTED_ON_LINK_DOWN, - IB_CME_CONNECTION_REQUEST_PENDING, - IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA, - IB_CME_DESTINATION_REJECT, - IB_CME_DESTINATION_REJECT_PRIVATE_DATA, - IB_CME_DESTINATION_UNREACHABLE, - IB_CME_TOO_MANY_CONNECTION_REQUESTS, - IB_CME_LOCAL_FAILURE, - IB_CM_LOCAL_FAILURE - -} ib_cm_events_t; - -/* Operation and state mappings */ -typedef int ib_send_op_type_t; -typedef struct ibv_sge ib_data_segment_t; -typedef enum ibv_qp_state ib_qp_state_t; -typedef enum ibv_event_type ib_async_event_type; -typedef struct ibv_async_event ib_error_record_t; - -/* CQ notifications */ -typedef enum -{ - IB_NOTIFY_ON_NEXT_COMP, - IB_NOTIFY_ON_SOLIC_COMP - -} ib_notification_type_t; - -/* other mappings */ -typedef int ib_bool_t; -typedef union ibv_gid GID; -typedef char *IB_HCA_NAME; -typedef uint16_t ib_hca_port_t; -typedef uint32_t ib_comp_handle_t; - -#ifdef CQ_WAIT_OBJECT -typedef struct ibv_comp_channel *ib_wait_obj_handle_t; -#endif - /* Definitions */ #define IB_INVALID_HANDLE NULL /* inline send rdma threshold */ #define INLINE_SEND_DEFAULT 200 -/* qkey for UD QP's */ -#define SCM_UD_QKEY 0x78654321 - /* RC timer - retry count defaults */ #define SCM_ACK_TIMER 16 /* 5 bits, 4.096us*2^ack_timer. 16== 268ms */ #define SCM_ACK_RETRY 7 /* 3 bits, 7 * 268ms = 1.8 seconds */ @@ -193,87 +74,6 @@ typedef struct ibv_comp_channel *ib_wait_obj_handle_t; #define IB_MAX_DREP_PDATA_SIZE 224 #define IB_MAX_RTU_PDATA_SIZE 224 -/* DTO OPs, ordered for DAPL ENUM definitions */ -#define OP_RDMA_WRITE IBV_WR_RDMA_WRITE -#define OP_RDMA_WRITE_IMM IBV_WR_RDMA_WRITE_WITH_IMM -#define OP_SEND IBV_WR_SEND -#define OP_SEND_IMM IBV_WR_SEND_WITH_IMM -#define OP_RDMA_READ IBV_WR_RDMA_READ -#define OP_COMP_AND_SWAP IBV_WR_ATOMIC_CMP_AND_SWP -#define OP_FETCH_AND_ADD IBV_WR_ATOMIC_FETCH_AND_ADD -#define OP_RECEIVE 7 /* internal op */ -#define OP_RECEIVE_IMM 8 /* rdma write with immed, internel op */ -#define OP_RECEIVE_MSG_IMM 9 /* recv msg with immed, internel op */ -#define OP_BIND_MW 10 /* internal op */ -#define OP_SEND_UD 11 /* internal op */ -#define OP_RECV_UD 12 /* internal op */ -#define OP_INVALID 0xff - -/* Definitions to map QP state */ -#define IB_QP_STATE_RESET IBV_QPS_RESET -#define IB_QP_STATE_INIT IBV_QPS_INIT -#define IB_QP_STATE_RTR IBV_QPS_RTR -#define IB_QP_STATE_RTS IBV_QPS_RTS -#define IB_QP_STATE_SQD IBV_QPS_SQD -#define IB_QP_STATE_SQE IBV_QPS_SQE -#define IB_QP_STATE_ERROR IBV_QPS_ERR - -/* Definitions for ibverbs/mthca return codes, should be defined in verbs.h */ -/* some are errno and some are -n values */ - -/** - * ibv_get_device_name - Return kernel device name - * ibv_get_device_guid - Return device's node GUID - * ibv_open_device - Return ibv_context or NULL - * ibv_close_device - Return 0, (errno?) - * ibv_get_async_event - Return 0, -1 - * ibv_alloc_pd - Return ibv_pd, NULL - * ibv_dealloc_pd - Return 0, errno - * ibv_reg_mr - Return ibv_mr, NULL - * ibv_dereg_mr - Return 0, errno - * ibv_create_cq - Return ibv_cq, NULL - * ibv_destroy_cq - Return 0, errno - * ibv_get_cq_event - Return 0 & ibv_cq/context, int - * ibv_poll_cq - Return n & ibv_wc, 0 ok, -1 empty, -2 error - * ibv_req_notify_cq - Return 0 (void?) - * ibv_create_qp - Return ibv_qp, NULL - * ibv_modify_qp - Return 0, errno - * ibv_destroy_qp - Return 0, errno - * ibv_post_send - Return 0, -1 & bad_wr - * ibv_post_recv - Return 0, -1 & bad_wr - */ - -/* async handler for DTO, CQ, QP, and unafiliated */ -typedef void (*ib_async_dto_handler_t)( - IN ib_hca_handle_t ib_hca_handle, - IN ib_error_record_t *err_code, - IN void *context); - -typedef void (*ib_async_cq_handler_t)( - IN ib_hca_handle_t ib_hca_handle, - IN ib_cq_handle_t ib_cq_handle, - IN ib_error_record_t *err_code, - IN void *context); - -typedef void (*ib_async_qp_handler_t)( - IN ib_hca_handle_t ib_hca_handle, - IN ib_qp_handle_t ib_qp_handle, - IN ib_error_record_t *err_code, - IN void *context); - -typedef void (*ib_async_handler_t)( - IN ib_hca_handle_t ib_hca_handle, - IN ib_error_record_t *err_code, - IN void *context); - -typedef enum -{ - IB_THREAD_INIT, - IB_THREAD_RUN, - IB_THREAD_CANCEL, - IB_THREAD_EXIT - -} ib_thread_state_t; /* ib_hca_transport_t, specific to this implementation */ typedef struct _ib_hca_transport @@ -295,6 +95,8 @@ typedef struct _ib_hca_transport ib_async_cq_handler_t async_cq_error; ib_async_dto_handler_t async_cq; ib_async_qp_handler_t async_qp_error; + int rd_atom_in; + int rd_atom_out; uint16_t lid; uint8_t ack_timer; uint8_t ack_retry; @@ -308,96 +110,16 @@ typedef struct _ib_hca_transport DAPL_SOCKET scm[2]; } ib_hca_transport_t; -/* provider specfic fields for shared memory support */ -typedef uint32_t ib_shm_transport_t; - /* prototypes */ -int32_t dapls_ib_init (void); -int32_t dapls_ib_release (void); void cr_thread(void *arg); int dapli_cq_thread_init(struct dapl_hca *hca_ptr); void dapli_cq_thread_destroy(struct dapl_hca *hca_ptr); DAT_RETURN dapli_socket_disconnect(dp_ib_cm_handle_t cm_ptr); void dapls_print_cm_list(IN DAPL_IA *ia_ptr); - -DAT_RETURN -dapls_modify_qp_state ( IN ib_qp_handle_t qp_handle, - IN ib_qp_state_t qp_state, - IN struct ib_cm_handle *cm_ptr ); - -/* inline functions */ -STATIC _INLINE_ IB_HCA_NAME dapl_ib_convert_name (IN char *name) -{ - /* use ascii; name of local device */ - return dapl_os_strdup(name); -} - -STATIC _INLINE_ void dapl_ib_release_name (IN IB_HCA_NAME name) -{ - return; -} - -/* - * Convert errno to DAT_RETURN values - */ -STATIC _INLINE_ DAT_RETURN -dapl_convert_errno( IN int err, IN const char *str ) -{ - if (!err) return DAT_SUCCESS; - -#if DAPL_DBG - if ((err != EAGAIN) && (err != ETIMEDOUT)) - dapl_dbg_log (DAPL_DBG_TYPE_ERR," %s %s\n", str, strerror(err)); -#endif - - switch( err ) - { - case EOVERFLOW : return DAT_LENGTH_ERROR; - case EACCES : return DAT_PRIVILEGES_VIOLATION; - case EPERM : return DAT_PROTECTION_VIOLATION; - case EINVAL : return DAT_INVALID_HANDLE; - case EISCONN : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_CONNECTED; - case ECONNREFUSED : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_NOTREADY; - case ETIMEDOUT : return DAT_TIMEOUT_EXPIRED; - case ENETUNREACH: return DAT_INVALID_ADDRESS | DAT_INVALID_ADDRESS_UNREACHABLE; - case EADDRINUSE : return DAT_CONN_QUAL_IN_USE; - case EALREADY : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_ACTCONNPENDING; - case ENOMEM : return DAT_INSUFFICIENT_RESOURCES; - case EAGAIN : return DAT_QUEUE_EMPTY; - case EINTR : return DAT_INTERRUPTED_CALL; - case EAFNOSUPPORT : return DAT_INVALID_ADDRESS | DAT_INVALID_ADDRESS_MALFORMED; - case EFAULT : - default : return DAT_INTERNAL_ERROR; - } - } - -STATIC _INLINE_ char * dapl_cm_state_str(IN int st) -{ - static char *cm_state[] = { - "SCM_INIT", - "SCM_LISTEN", - "SCM_CONN_PENDING", - "SCM_RTU_PENDING", - "SCM_ACCEPTING", - "SCM_ACCEPTING_DATA", - "SCM_ACCEPTED", - "SCM_REJECTED", - "SCM_CONNECTED", - "SCM_RELEASED", - "SCM_DISCONNECTED", - "SCM_DESTROY" - }; - return ((st < 0 || st > 11) ? "Invalid CM state?" : cm_state[st]); -} - -/* - * Definitions required only for DAT 1.1 builds - */ -#define IB_ACCESS_LOCAL_READ IBV_ACCESS_LOCAL_WRITE -#define IB_ACCESS_LOCAL_WRITE IBV_ACCESS_LOCAL_WRITE -#define IB_ACCESS_REMOTE_READ IBV_ACCESS_REMOTE_READ -#define IB_ACCESS_REMOTE_WRITE IBV_ACCESS_REMOTE_WRITE -#define IB_ACCESS_MW_BIND IBV_ACCESS_LOCAL_WRITE -#define IB_ACCESS_ATOMIC +dp_ib_cm_handle_t dapls_ib_cm_create(DAPL_EP *ep); +void dapls_ib_cm_free(dp_ib_cm_handle_t cm, DAPL_EP *ep); +DAT_RETURN dapls_modify_qp_state(IN ib_qp_handle_t qp_handle, + IN ib_qp_state_t qp_state, + IN dp_ib_cm_handle_t cm); #endif /* _DAPL_IB_UTIL_H_ */ diff --git a/dapl/openib_scm/device.c b/dapl/openib_scm/device.c new file mode 100644 index 0000000..d5089aa --- /dev/null +++ b/dapl/openib_scm/device.c @@ -0,0 +1,412 @@ +/* + * This Software is licensed under one of the following licenses: + * + * 1) under the terms of the "Common Public License 1.0" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/cpl.php. + * + * 2) under the terms of the "The BSD License" a copy of which is + * available from the Open Source Initiative, see + * http://www.opensource.org/licenses/bsd-license.php. + * + * 3) under the terms of the "GNU General Public License (GPL) Version 2" a + * copy of which is available from the Open Source Initiative, see + * http://www.opensource.org/licenses/gpl-license.php. + * + * Licensee has the right to choose one of the above licenses. + * + * Redistributions of source code must retain the above copyright + * notice and one of the license notices. + * + * Redistributions in binary form must reproduce both the above copyright + * notice, one of the license notices in the documentation + * and/or other materials provided with the distribution. + */ + +/*************************************************************************** + * + * Module: uDAPL + * + * Filename: dapl_ib_util.c + * + * Author: Arlin Davis + * + * Created: 3/10/2005 + * + * Description: + * + * The uDAPL openib provider - init, open, close, utilities + * + **************************************************************************** + * Source Control System Information + * + * $Id: $ + * + * Copyright (c) 2005 Intel Corporation. All rights reserved. + * + **************************************************************************/ +#ifdef RCSID +static const char rcsid[] = "$Id: $"; +#endif + +#include "openib_osd.h" +#include "dapl.h" +#include "dapl_adapter_util.h" +#include "dapl_ib_util.h" +#include "dapl_osd.h" + +#include + +static int32_t create_cr_pipe(IN DAPL_HCA * hca_ptr) +{ + DAPL_SOCKET listen_socket; + struct sockaddr_in addr; + socklen_t addrlen = sizeof(addr); + int ret; + + listen_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (listen_socket == DAPL_INVALID_SOCKET) + return 1; + + memset(&addr, 0, sizeof addr); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(0x7f000001); + ret = bind(listen_socket, (struct sockaddr *)&addr, sizeof addr); + if (ret) + goto err1; + + ret = getsockname(listen_socket, (struct sockaddr *)&addr, &addrlen); + if (ret) + goto err1; + + ret = listen(listen_socket, 0); + if (ret) + goto err1; + + hca_ptr->ib_trans.scm[1] = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (hca_ptr->ib_trans.scm[1] == DAPL_INVALID_SOCKET) + goto err1; + + ret = connect(hca_ptr->ib_trans.scm[1], + (struct sockaddr *)&addr, sizeof(addr)); + if (ret) + goto err2; + + hca_ptr->ib_trans.scm[0] = accept(listen_socket, NULL, NULL); + if (hca_ptr->ib_trans.scm[0] == DAPL_INVALID_SOCKET) + goto err2; + + closesocket(listen_socket); + return 0; + + err2: + closesocket(hca_ptr->ib_trans.scm[1]); + err1: + closesocket(listen_socket); + return 1; +} + +static void destroy_cr_pipe(IN DAPL_HCA * hca_ptr) +{ + closesocket(hca_ptr->ib_trans.scm[0]); + closesocket(hca_ptr->ib_trans.scm[1]); +} + + +/* + * dapls_ib_init, dapls_ib_release + * + * Initialize Verb related items for device open + * + * Input: + * none + * + * Output: + * none + * + * Returns: + * 0 success, -1 error + * + */ +int32_t dapls_ib_init(void) +{ + return 0; +} + +int32_t dapls_ib_release(void) +{ + return 0; +} + +#if defined(_WIN64) || defined(_WIN32) +int dapls_config_comp_channel(struct ibv_comp_channel *channel) +{ + return 0; +} +#else // _WIN64 || WIN32 +int dapls_config_comp_channel(struct ibv_comp_channel *channel) +{ + int opts; + + opts = fcntl(channel->fd, F_GETFL); /* uCQ */ + if (opts < 0 || fcntl(channel->fd, F_SETFL, opts | O_NONBLOCK) < 0) { + dapl_log(DAPL_DBG_TYPE_ERR, + " dapls_create_comp_channel: fcntl on ib_cq->fd %d ERR %d %s\n", + channel->fd, opts, strerror(errno)); + return errno; + } + + return 0; +} +#endif + +/* + * dapls_ib_open_hca + * + * Open HCA + * + * Input: + * *hca_name pointer to provider device name + * *ib_hca_handle_p pointer to provide HCA handle + * + * Output: + * none + * + * Return: + * DAT_SUCCESS + * dapl_convert_errno + * + */ +DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr) +{ + struct ibv_device **dev_list; + struct ibv_port_attr port_attr; + int i; + DAT_RETURN dat_status; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " open_hca: %s - %p\n", hca_name, hca_ptr); + + /* get the IP address of the device */ + dat_status = getlocalipaddr((DAT_SOCK_ADDR *) &hca_ptr->hca_address, + sizeof(DAT_SOCK_ADDR6)); + if (dat_status != DAT_SUCCESS) + return dat_status; + + /* Get list of all IB devices, find match, open */ + dev_list = ibv_get_device_list(NULL); + if (!dev_list) { + dapl_dbg_log(DAPL_DBG_TYPE_ERR, + " open_hca: ibv_get_device_list() failed\n", + hca_name); + return DAT_INTERNAL_ERROR; + } + + for (i = 0; dev_list[i]; ++i) { + hca_ptr->ib_trans.ib_dev = dev_list[i]; + if (!strcmp(ibv_get_device_name(hca_ptr->ib_trans.ib_dev), + hca_name)) + goto found; + } + + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: device %s not found\n", hca_name); + goto err; + + found: + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " open_hca: Found dev %s %016llx\n", + ibv_get_device_name(hca_ptr->ib_trans.ib_dev), + (unsigned long long) + ntohll(ibv_get_device_guid(hca_ptr->ib_trans.ib_dev))); + + hca_ptr->ib_hca_handle = ibv_open_device(hca_ptr->ib_trans.ib_dev); + if (!hca_ptr->ib_hca_handle) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: dev open failed for %s, err=%s\n", + ibv_get_device_name(hca_ptr->ib_trans.ib_dev), + strerror(errno)); + goto err; + } + + /* get lid for this hca-port, network order */ + if (ibv_query_port(hca_ptr->ib_hca_handle, + (uint8_t) hca_ptr->port_num, &port_attr)) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: get lid ERR for %s, err=%s\n", + ibv_get_device_name(hca_ptr->ib_trans.ib_dev), + strerror(errno)); + goto err; + } else { + hca_ptr->ib_trans.lid = htons(port_attr.lid); + } + + /* get gid for this hca-port, network order */ + if (ibv_query_gid(hca_ptr->ib_hca_handle, + (uint8_t) hca_ptr->port_num, + 0, &hca_ptr->ib_trans.gid)) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: query GID ERR for %s, err=%s\n", + ibv_get_device_name(hca_ptr->ib_trans.ib_dev), + strerror(errno)); + goto err; + } + + /* set RC tunables via enviroment or default */ + hca_ptr->ib_trans.max_inline_send = + dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_DEFAULT); + hca_ptr->ib_trans.ack_retry = + dapl_os_get_env_val("DAPL_ACK_RETRY", SCM_ACK_RETRY); + hca_ptr->ib_trans.ack_timer = + dapl_os_get_env_val("DAPL_ACK_TIMER", SCM_ACK_TIMER); + hca_ptr->ib_trans.rnr_retry = + dapl_os_get_env_val("DAPL_RNR_RETRY", SCM_RNR_RETRY); + hca_ptr->ib_trans.rnr_timer = + dapl_os_get_env_val("DAPL_RNR_TIMER", SCM_RNR_TIMER); + hca_ptr->ib_trans.global = + dapl_os_get_env_val("DAPL_GLOBAL_ROUTING", SCM_GLOBAL); + hca_ptr->ib_trans.hop_limit = + dapl_os_get_env_val("DAPL_HOP_LIMIT", SCM_HOP_LIMIT); + hca_ptr->ib_trans.tclass = + dapl_os_get_env_val("DAPL_TCLASS", SCM_TCLASS); + hca_ptr->ib_trans.mtu = + dapl_ib_mtu(dapl_os_get_env_val("DAPL_IB_MTU", SCM_IB_MTU)); + +#ifndef CQ_WAIT_OBJECT + /* initialize cq_lock */ + dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.cq_lock); + if (dat_status != DAT_SUCCESS) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: failed to init cq_lock\n"); + goto bail; + } + /* EVD events without direct CQ channels, non-blocking */ + hca_ptr->ib_trans.ib_cq = + ibv_create_comp_channel(hca_ptr->ib_hca_handle); + if (hca_ptr->ib_trans.ib_cq == NULL) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: ibv_create_comp_channel ERR %s\n", + strerror(errno)); + goto bail; + } + + if (dapls_config_comp_channel(hca_ptr->ib_trans.ib_cq)) { + goto bail; + } + + if (dapli_cq_thread_init(hca_ptr)) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: cq_thread_init failed for %s\n", + ibv_get_device_name(hca_ptr->ib_trans.ib_dev)); + goto bail; + } +#endif /* CQ_WAIT_OBJECT */ + + /* initialize cr_list lock */ + dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.lock); + if (dat_status != DAT_SUCCESS) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: failed to init cr_list lock\n"); + goto bail; + } + + /* initialize CM list for listens on this HCA */ + dapl_llist_init_head(&hca_ptr->ib_trans.list); + + /* initialize pipe, user level wakeup on select */ + if (create_cr_pipe(hca_ptr)) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: failed to init cr pipe - %s\n", + strerror(errno)); + goto bail; + } + + /* create thread to process inbound connect request */ + hca_ptr->ib_trans.cr_state = IB_THREAD_INIT; + dat_status = dapl_os_thread_create(cr_thread, + (void *)hca_ptr, + &hca_ptr->ib_trans.thread); + if (dat_status != DAT_SUCCESS) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: failed to create thread\n"); + goto bail; + } + + /* wait for thread */ + while (hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) { + dapl_os_sleep_usec(2000); + } + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " open_hca: devname %s, port %d, hostname_IP %s\n", + ibv_get_device_name(hca_ptr->ib_trans.ib_dev), + hca_ptr->port_num, inet_ntoa(((struct sockaddr_in *) + &hca_ptr->hca_address)-> + sin_addr)); + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " open_hca: LID 0x%x GID Subnet 0x" F64x " ID 0x" F64x + "\n", ntohs(hca_ptr->ib_trans.lid), (unsigned long long) + htonll(hca_ptr->ib_trans.gid.global.subnet_prefix), + (unsigned long long)htonll(hca_ptr->ib_trans.gid.global. + interface_id)); + + ibv_free_device_list(dev_list); + return dat_status; + + bail: + ibv_close_device(hca_ptr->ib_hca_handle); + hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; + err: + ibv_free_device_list(dev_list); + return DAT_INTERNAL_ERROR; +} + +/* + * dapls_ib_close_hca + * + * Open HCA + * + * Input: + * DAPL_HCA provide CA handle + * + * Output: + * none + * + * Return: + * DAT_SUCCESS + * dapl_convert_errno + * + */ +DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr) +{ + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p\n", hca_ptr); + +#ifndef CQ_WAIT_OBJECT + dapli_cq_thread_destroy(hca_ptr); + dapl_os_lock_destroy(&hca_ptr->ib_trans.cq_lock); +#endif /* CQ_WAIT_OBJECT */ + + if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) { + if (ibv_close_device(hca_ptr->ib_hca_handle)) + return (dapl_convert_errno(errno, "ib_close_device")); + hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; + } + + /* destroy cr_thread and lock */ + hca_ptr->ib_trans.cr_state = IB_THREAD_CANCEL; + if (send(hca_ptr->ib_trans.scm[1], "w", sizeof "w", 0) == -1) + dapl_log(DAPL_DBG_TYPE_UTIL, + " thread_destroy: thread wakeup err = %s\n", + strerror(errno)); + while (hca_ptr->ib_trans.cr_state != IB_THREAD_EXIT) { + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " close_hca: waiting for cr_thread\n"); + if (send(hca_ptr->ib_trans.scm[1], "w", sizeof "w", 0) == -1) + dapl_log(DAPL_DBG_TYPE_UTIL, + " thread_destroy: thread wakeup err = %s\n", + strerror(errno)); + dapl_os_sleep_usec(2000); + } + dapl_os_lock_destroy(&hca_ptr->ib_trans.lock); + destroy_cr_pipe(hca_ptr); /* no longer need pipe */ + return (DAT_SUCCESS); +} -- 1.5.2.5 From arlin.r.davis at intel.com Mon Jul 6 12:52:08 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Mon, 6 Jul 2009 12:52:08 -0700 Subject: [ofa-general] [PATCH 05/11] uDAPL winof: fix build issues after consolidating cma, scm code base. Message-ID: Signed-off-by: Arlin Davis --- dapl/common/dapl_debug.c | 1 - dapl/openib_cma/SOURCES | 8 +-- dapl/openib_common.c | 6 ++ dapl/openib_scm/SOURCES | 6 +-- dat/udat/udat_exports.src | 1 + test/dapltest/scripts/dt-regression.bat | 88 ------------------------------- 6 files changed, 10 insertions(+), 100 deletions(-) create mode 100644 dapl/openib_common.c delete mode 100644 test/dapltest/scripts/dt-regression.bat diff --git a/dapl/common/dapl_debug.c b/dapl/common/dapl_debug.c index 18b6e7e..960bc00 100644 --- a/dapl/common/dapl_debug.c +++ b/dapl/common/dapl_debug.c @@ -198,7 +198,6 @@ char *dapl_query_counter_name(DAT_HANDLE dh, int counter) return NULL; } -#include void dapl_print_counter(DAT_HANDLE dh, int counter, int reset) { int i, max; diff --git a/dapl/openib_cma/SOURCES b/dapl/openib_cma/SOURCES index f1c5002..d6b97a2 100644 --- a/dapl/openib_cma/SOURCES +++ b/dapl/openib_cma/SOURCES @@ -20,11 +20,7 @@ SOURCES = \ udapl.rc \ ..\dapl_common_src.c \ ..\dapl_udapl_src.c \ - ..\openib_common\mem.c \ - ..\openib_common\util.c \ - ..\openib_common\cq.c \ - ..\openib_common\qp.c \ - ..\openib_common\ib_extensions.c \ + ..\openib_common.c \ device.c \ cm.c @@ -33,7 +29,7 @@ INCLUDES = ..\include;..\openib_common;..\common;windows;..\..\dat\include;\ ..\..\..\..\inc;..\..\..\..\inc\user;..\..\..\libibverbs\include;\ ..\..\..\librdmacm\include -DAPL_OPTS = -DEXPORT_DAPL_SYMBOLS -DDAT_EXTENSIONS -DOPENIB +DAPL_OPTS = -DEXPORT_DAPL_SYMBOLS -DDAT_EXTENSIONS -DOPENIB -DCQ_WAIT_OBJECT #-DDAPL_COUNTERS USER_C_FLAGS = $(USER_C_FLAGS) $(DAPL_OPTS) diff --git a/dapl/openib_common.c b/dapl/openib_common.c new file mode 100644 index 0000000..336eff9 --- /dev/null +++ b/dapl/openib_common.c @@ -0,0 +1,6 @@ + +#include "openib_common\mem.c" +#include "openib_common\util.c" +#include "openib_common\cq.c" +#include "openib_common\qp.c" +#include "openib_common\ib_extensions.c" diff --git a/dapl/openib_scm/SOURCES b/dapl/openib_scm/SOURCES index 5714aa3..6e4ad30 100644 --- a/dapl/openib_scm/SOURCES +++ b/dapl/openib_scm/SOURCES @@ -20,11 +20,7 @@ SOURCES = \ udapl.rc \ ..\dapl_common_src.c \ ..\dapl_udapl_src.c \ - ..\openib_common\mem.c \ - ..\openib_common\util.c \ - ..\openib_common\cq.c \ - ..\openib_common\qp.c \ - ..\openib_common\ib_extensions.c \ + ..\openib_common.c \ device.c \ cm.c diff --git a/dat/udat/udat_exports.src b/dat/udat/udat_exports.src index 42e3773..32b29ff 100644 --- a/dat/udat/udat_exports.src +++ b/dat/udat/udat_exports.src @@ -51,6 +51,7 @@ dat_rsp_create dat_rsp_free dat_rsp_query dat_strerror +dat_get_handle_type dats_get_ia_handle #ifdef DAT_EXTENSIONS dat_extension_op diff --git a/test/dapltest/scripts/dt-regression.bat b/test/dapltest/scripts/dt-regression.bat deleted file mode 100644 index 6d7c65d..0000000 --- a/test/dapltest/scripts/dt-regression.bat +++ /dev/null @@ -1,88 +0,0 @@ - at echo off -rem -rem DAPLtest regression test client - requires a running DAPLtest server. -rem Usage: dt-regression.bat hostname [anything == no pause after test] -rem -rem run various Dapl tests via the dapltest server running somewhere network accesible -rem - -SETLOCAL - -set D=ibnic0 -set TO=3 - -if "%*" EQU "" ( - set /P S= [DAPLtest Server-Hostname] -) else ( - if "%1" EQU "" ( - echo "Usage: %0 dapltest-hostname " - exit /B - ) - set S=%1 - if "%2" NEQ "" ( - set TO=0 - ) -) - -rem 0xfff -set X= -if not "%X%" == "" ( - set DAT_OS_DBG_TYPE=%X% - set DAT_DBG_TYPE=%X% - set DAT_DBG_LEVEL=%X% - set DAPL_DBG_LEVEL=%X% - set DAPL_DBG_TYPE=%X% -) - -rem set DAT_OVERRIDE=D:\dapl2\dat.conf -set ITR=25000 - -rem dapltest debug - set DB= -d - -set DT=dapl2test.exe - -echo ----- Connectivity -rem Connectivity test - client sends one buffer with one 4KB segments, 50 times. -rem add '-d' for debug output. -%DT% -T T %DB% -s %S% -D %D% -i 50 -t 1 -w 1 client SR 4096 server SR 4096 -IF %ERRORLEVEL% NEQ 0 exit /B -echo Connectivity test Finished. -if %TO% GTR 0 timeout /T 5 - -echo ----- Transaction test: %ITR% iterations -%DT% -T T %DB% -s %S% -D %D% -i %ITR% -t 1 -w 1 client SR 4096 server SR 4096 -IF %ERRORLEVEL% NEQ 0 exit /B -echo Transaction test - 500 iterations, Finished -if %TO% GTR 0 timeout /T 5 - -echo ----- Performance test: %ITR% iterations -%DT% -T P %DB% -s %S% -D %D% -i %ITR% RW 4096 2 -IF %ERRORLEVEL% NEQ 0 exit /B -echo Perf test Finished. -if %TO% GTR 0 timeout /T 5 - -echo ----- RDMA-read test: %ITR% iterations -%DT% -T P %DB% -s %S% -D %D% -i %ITR% RR 4096 2 -IF %ERRORLEVEL% NEQ 0 exit /B -echo RDMA-read Finished. -if %TO% GTR 0 timeout /T 5 - -echo ----- RDMA-write test: %ITR% iterations -%DT% -T P %DB% -s %S% -D %D% -i %ITR% RW 4096 2 -IF %ERRORLEVEL% NEQ 0 exit /B -echo RDMA-write Finished. -if %TO% GTR 0 timeout /T 5 - -rem %DT% -T T -V -d -t 2 -w 2 -i 1000111 -s %S% -D %D% -rem client RW 4096 1 server RW 2048 4 -rem client RR 1024 2 server RR 2048 2 -rem client SR 1024 3 -f server SR 256 3 -f - -rem %DT% -T T -P -d -t 1 -w 1 -i 1024 -s %S% -D %D% -rem client RW 4096 1 server RW 2048 4 -rem server RR 1024 2 client RR 2048 2 -rem client SR 1024 3 -f server SR 256 3 -f - -ENDLOCAL - - at echo on -- 1.5.2.5 From arlin.r.davis at intel.com Mon Jul 6 12:52:01 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Mon, 6 Jul 2009 12:52:01 -0700 Subject: [ofa-general] [PATCH 04/11] uDAPL cma: lock held when exiting as a result of a rdma_create_event_channel failure. Message-ID: Signed-off-by: Arlin Davis --- dapl/openib_cma/device.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/dapl/openib_cma/device.c b/dapl/openib_cma/device.c index 22e5045..81203bf 100644 --- a/dapl/openib_cma/device.c +++ b/dapl/openib_cma/device.c @@ -325,6 +325,7 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr) dapl_dbg_log(DAPL_DBG_TYPE_ERR, " open_hca: ERR - RDMA channel %s\n", strerror(errno)); + dapl_os_unlock(&g_hca_lock); return DAT_INTERNAL_ERROR; } } -- 1.5.2.5 From arlin.r.davis at intel.com Mon Jul 6 12:51:50 2009 From: arlin.r.davis at intel.com (Arlin Davis) Date: Mon, 6 Jul 2009 12:51:50 -0700 Subject: [ofa-general] [PATCH 02/11] uDAPL dtestcm windows: add build infrastructure for new dtestcm test suite Message-ID: Signed-off-by: Arlin Davis --- test/dtest/dtestcm.c | 1 - test/dtest/windows/dirs | 2 +- test/dtest/windows/dtestcm/SOURCES | 33 ++++++++++++++++++++++ test/dtest/windows/dtestcm/dtestcm.c | 2 + test/dtest/windows/dtestcm/dtestcm.rc | 48 +++++++++++++++++++++++++++++++++ test/dtest/windows/dtestcm/makefile | 7 +++++ 6 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 test/dtest/windows/dtestcm/SOURCES create mode 100644 test/dtest/windows/dtestcm/dtestcm.c create mode 100644 test/dtest/windows/dtestcm/dtestcm.rc create mode 100644 test/dtest/windows/dtestcm/makefile diff --git a/test/dtest/dtestcm.c b/test/dtest/dtestcm.c index 7bfe342..62af6a1 100644 --- a/test/dtest/dtestcm.c +++ b/test/dtest/dtestcm.c @@ -49,7 +49,6 @@ #include #include "..\..\..\..\etc\user\getopt.c" -#define ((int)GetCurrentProcessId()) #define F64x "%I64x" #define F64d "%I64d" diff --git a/test/dtest/windows/dirs b/test/dtest/windows/dirs index cac5a54..50f0131 100644 --- a/test/dtest/windows/dirs +++ b/test/dtest/windows/dirs @@ -1 +1 @@ -dirs = dtest dtestx +dirs = dtest dtestx dtestcm diff --git a/test/dtest/windows/dtestcm/SOURCES b/test/dtest/windows/dtestcm/SOURCES new file mode 100644 index 0000000..7b4264e --- /dev/null +++ b/test/dtest/windows/dtestcm/SOURCES @@ -0,0 +1,33 @@ +!if $(FREEBUILD) +TARGETNAME = dtestcm +!else +TARGETNAME = dtestcmd +!endif + +TARGETPATH = ..\..\..\..\..\..\bin\user\obj$(BUILD_ALT_DIR) +TARGETTYPE = PROGRAM +UMTYPE = console +USE_MSVCRT = 1 + +SOURCES = \ + dtestcm.rc \ + dtestcm.c + +INCLUDES = ..\..\..\..\dat\include;..\..\..\..\..\..\inc;\ + ..\..\..\..\..\..\inc\user; + +RCOPTIONS=/I..\..\..\..\..\..\inc; + +# Set defines particular to the driver. +#USER_C_FLAGS = $(USER_C_FLAGS) /DDAT_EXTENSIONS + +!if $(FREEBUILD) +DATLIB = dat2.lib +!else +DATLIB = dat2d.lib +!endif + +TARGETLIBS = $(TARGETPATH)\*\$(DATLIB) $(SDK_LIB_PATH)\ws2_32.lib + +# XXX do this ASAP - MSC_WARNING_LEVEL= /W3 +MSC_WARNING_LEVEL = /W1 diff --git a/test/dtest/windows/dtestcm/dtestcm.c b/test/dtest/windows/dtestcm/dtestcm.c new file mode 100644 index 0000000..f9a6db0 --- /dev/null +++ b/test/dtest/windows/dtestcm/dtestcm.c @@ -0,0 +1,2 @@ +#include "..\..\dtestcm.c" + diff --git a/test/dtest/windows/dtestcm/dtestcm.rc b/test/dtest/windows/dtestcm/dtestcm.rc new file mode 100644 index 0000000..a167485 --- /dev/null +++ b/test/dtest/windows/dtestcm/dtestcm.rc @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2009 Intel Corporation. All rights reserved. + * + * This software is available to you under the OpenIB.org BSD license + * below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id$ + */ + + +#include + +#define VER_FILETYPE VFT_APP +#define VER_FILESUBTYPE VFT2_UNKNOWN + +#if DBG +#define VER_FILEDESCRIPTION_STR "Measure DAPL connection rate scaling (Debug)" +#define VER_INTERNALNAME_STR "dtestcmd.exe" +#define VER_ORIGINALFILENAME_STR "dtestcmd.exe" +#else +#define VER_FILEDESCRIPTION_STR "Measure DAPL connection rate scaling" +#define VER_INTERNALNAME_STR "dtestcm.exe" +#define VER_ORIGINALFILENAME_STR "dtestcm.exe" +#endif + +#include diff --git a/test/dtest/windows/dtestcm/makefile b/test/dtest/windows/dtestcm/makefile new file mode 100644 index 0000000..5fb2ee8 --- /dev/null +++ b/test/dtest/windows/dtestcm/makefile @@ -0,0 +1,7 @@ +# +# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source +# file to this component. This file merely indirects to the real make file +# that is shared by all the driver components of the OpenIB Windows project. +# + +!INCLUDE ..\..\..\..\..\..\inc\openib.def -- 1.5.2.5 From arlin.r.davis at intel.com Mon Jul 6 12:52:14 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Mon, 6 Jul 2009 12:52:14 -0700 Subject: [ofa-general] [PATCH 06/11] uDAPL scm: remove old udapl_scm code replaced by openib_scm. Message-ID: Signed-off-by: Arlin Davis --- dapl/udapl_scm/SOURCES | 45 ------------- dapl/udapl_scm/makefile | 7 -- dapl/udapl_scm/udapl.rc | 48 -------------- dapl/udapl_scm/udapl_scm_exports.src | 14 ---- dapl/udapl_scm/udapl_sources.c | 120 ---------------------------------- 5 files changed, 0 insertions(+), 234 deletions(-) delete mode 100644 dapl/udapl_scm/SOURCES delete mode 100644 dapl/udapl_scm/makefile delete mode 100644 dapl/udapl_scm/udapl.rc delete mode 100644 dapl/udapl_scm/udapl_scm_exports.src delete mode 100644 dapl/udapl_scm/udapl_sources.c diff --git a/dapl/udapl_scm/SOURCES b/dapl/udapl_scm/SOURCES deleted file mode 100644 index f382923..0000000 --- a/dapl/udapl_scm/SOURCES +++ /dev/null @@ -1,45 +0,0 @@ -!if $(FREEBUILD) -TARGETNAME=dapl2-scm -!else -TARGETNAME=dapl2-scmd -!endif -TARGETPATH=..\..\..\..\bin\user\obj$(BUILD_ALT_DIR) -TARGETTYPE=DYNLINK -DLLENTRY=_DllMainCRTStartup -DLLDEF=$O\udapl_scm_exports.def -USE_CRTDLL=1 - -# pickup local files, then via udapl_sources.c get common files - -SOURCES=udapl.rc \ - udapl_sources.c - -INCLUDES=..\include;..\common;..\udapl-scm;..\udapl;..\udapl\windows;\ - ..\ibal-scm;..\ibal;..\..\dat\include;..\..\..\..\inc;\ - ..\..\..\..\inc\user; - -DAPL_OPTS= -DEXPORT_DAPL_SYMBOLS -D_VENDOR_IBAL_ -DDAPL_MERGE_CM_DTO - -DAPL_OPTS= $(DAPL_OPTS) -DDAT_EXTENSIONS=1 -DSOCK_CM=1 - -USER_C_FLAGS=$(USER_C_FLAGS) $(DAPL_OPTS) -!if !$(FREEBUILD) -USER_C_FLAGS=$(USER_C_FLAGS) -DDAPL_DBG #-DDAPL_COUNTERS -!endif - -TARGETLIBS= \ - $(SDK_LIB_PATH)\kernel32.lib \ - $(SDK_LIB_PATH)\ws2_32.lib \ -!if $(FREEBUILD) - $(TARGETPATH)\*\dat2.lib \ - $(TARGETPATH)\*\complib.lib \ - $(TARGETPATH)\*\ibal.lib -!else - $(TARGETPATH)\*\dat2d.lib \ - $(TARGETPATH)\*\complibd.lib \ - $(TARGETPATH)\*\ibald.lib -!endif - -# FIX ME ASAP -#MSC_WARNING_LEVEL= /W3 -MSC_WARNING_LEVEL= /W1 /wd4113 /Wp64 diff --git a/dapl/udapl_scm/makefile b/dapl/udapl_scm/makefile deleted file mode 100644 index e26e1c0..0000000 --- a/dapl/udapl_scm/makefile +++ /dev/null @@ -1,7 +0,0 @@ -# -# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source -# file to this component. This file merely indirects to the real make file -# that is shared by all the driver components of the OpenIB Windows project. -# - -!INCLUDE ..\..\..\..\inc\openib.def diff --git a/dapl/udapl_scm/udapl.rc b/dapl/udapl_scm/udapl.rc deleted file mode 100644 index 622169a..0000000 --- a/dapl/udapl_scm/udapl.rc +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2007 Intel Corporation. All rights reserved. - * - * This software is available to you under the OpenIB.org BSD license - * below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * $Id$ - */ - - -#include - -#define VER_FILETYPE VFT_DLL -#define VER_FILESUBTYPE VFT2_UNKNOWN - -#if DBG -#define VER_FILEDESCRIPTION_STR "Direct Access Provider Library v2.0 (socket-cm) (Debug)" -#define VER_INTERNALNAME_STR "dapl2-scmd.dll" -#define VER_ORIGINALFILENAME_STR "dapl2-scmd.dll" -#else -#define VER_FILEDESCRIPTION_STR "Direct Access Provider Library v2.0 (socket-cm)" -#define VER_INTERNALNAME_STR "dapl2-scm.dll" -#define VER_ORIGINALFILENAME_STR "dapl2-scm.dll" -#endif - -#include diff --git a/dapl/udapl_scm/udapl_scm_exports.src b/dapl/udapl_scm/udapl_scm_exports.src deleted file mode 100644 index 6702c89..0000000 --- a/dapl/udapl_scm/udapl_scm_exports.src +++ /dev/null @@ -1,14 +0,0 @@ -#if DBG -LIBRARY dapl2-scmd.dll -#else -LIBRARY dapl2-scm.dll -#endif - - -EXPORTS -dat_provider_init -dat_provider_fini -#ifdef DAT_EXTENSIONS -dapl_extensions -#endif - diff --git a/dapl/udapl_scm/udapl_sources.c b/dapl/udapl_scm/udapl_sources.c deleted file mode 100644 index 109c484..0000000 --- a/dapl/udapl_scm/udapl_sources.c +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Include all files that are not in children directories. - */ -#include "dapl_cno_util.c" -#include "dapl_cookie.c" -#include "dapl_cr_accept.c" -#include "dapl_cr_callback.c" -#include "dapl_cr_handoff.c" -#include "dapl_cr_query.c" -#include "dapl_cr_reject.c" -#include "dapl_cr_util.c" -#include "dapl_csp.c" -#include "dapl_debug.c" -#include "dapl_ep_connect.c" -#include "dapl_ep_create.c" -#include "dapl_ep_disconnect.c" -#include "dapl_ep_dup_connect.c" -#include "dapl_ep_free.c" -#include "dapl_ep_get_status.c" -#include "dapl_ep_modify.c" -#include "dapl_ep_post_rdma_read.c" -#include "dapl_ep_post_rdma_read_to_rmr.c" -#include "dapl_ep_post_rdma_write.c" -#include "dapl_ep_post_recv.c" -#include "dapl_ep_post_send.c" -#include "dapl_ep_post_send_invalidate.c" -#include "dapl_ep_query.c" -#include "dapl_ep_recv_query.c" -#include "dapl_ep_reset.c" -#include "dapl_ep_set_watermark.c" -#include "dapl_ep_util.c" -#include "dapl_evd_connection_callb.c" -#include "dapl_evd_cq_async_error_callb.c" -#include "dapl_evd_dequeue.c" -#include "dapl_evd_dto_callb.c" -#include "dapl_evd_free.c" -#include "dapl_evd_post_se.c" -#include "dapl_evd_qp_async_error_callb.c" -#include "dapl_evd_resize.c" -#include "dapl_evd_un_async_error_callb.c" -#include "dapl_evd_util.c" -#include "dapl_get_consumer_context.c" -#include "dapl_get_handle_type.c" -#include "dapl_hash.c" -#include "dapl_hca_util.c" -#include "dapl_ia_close.c" -#include "dapl_ia_ha.c" -#include "dapl_ia_open.c" -#include "dapl_ia_query.c" -#include "dapl_ia_util.c" -#include "dapl_llist.c" -#include "dapl_lmr_free.c" -#include "dapl_lmr_query.c" -#include "dapl_lmr_sync_rdma_read.c" -#include "dapl_lmr_sync_rdma_write.c" -#include "dapl_lmr_util.c" -#include "dapl_mr_util.c" -#include "dapl_name_service.c" -#include "dapl_provider.c" -#include "dapl_psp_create.c" -#include "dapl_psp_create_any.c" -#include "dapl_psp_free.c" -#include "dapl_psp_query.c" -#include "dapl_pz_create.c" -#include "dapl_pz_free.c" -#include "dapl_pz_query.c" -#include "dapl_pz_util.c" -#include "dapl_ring_buffer_util.c" -#include "dapl_rmr_bind.c" -#include "dapl_rmr_create.c" -#include "dapl_rmr_free.c" -#include "dapl_rmr_query.c" -#include "dapl_rmr_util.c" -#include "dapl_rsp_create.c" -#include "dapl_rsp_free.c" -#include "dapl_rsp_query.c" -#include "dapl_set_consumer_context.c" -#include "dapl_sp_util.c" -#include "dapl_ep_create_with_srq.c" -#include "dapl_srq_create.c" -#include "dapl_srq_free.c" -#include "dapl_srq_post_recv.c" -#include "dapl_srq_query.c" -#include "dapl_srq_resize.c" -#include "dapl_srq_set_lw.c" -#include "dapl_srq_util.c" -#include "dapl_timer_util.c" - -#include "..\udapl\dapl_cno_create.c" -#include "..\udapl\dapl_cno_free.c" -#include "..\udapl\dapl_cno_modify_agent.c" -#include "..\udapl\dapl_cno_query.c" -#include "..\udapl\dapl_cno_wait.c" -#include "..\udapl\dapl_evd_clear_unwaitable.c" -#include "..\udapl\dapl_evd_create.c" -#include "..\udapl\dapl_evd_disable.c" -#include "..\udapl\dapl_evd_enable.c" -#include "..\udapl\dapl_evd_modify_cno.c" -#include "..\udapl\dapl_evd_query.c" -#include "..\udapl\dapl_evd_set_unwaitable.c" -#include "..\udapl\dapl_evd_wait.c" -#include "..\udapl\dapl_init.c" -#include "..\udapl\dapl_lmr_create.c" - -#include "..\ibal-scm\dapl_ibal-scm_cm.c" -#include "..\ibal-scm\dapl_ibal-scm_util.c" - -#include "..\ibal\dapl_ibal_qp.c" -#include "..\ibal\dapl_ibal_cq.c" -#include "..\ibal\dapl_ibal_util.c" -#include "..\ibal\dapl_ibal_name_service.c" - -//#include "..\ibal\dapl_ibal_mrdb.c" - -#ifdef DAT_EXTENSIONS -#include "..\ibal\dapl_ibal_extensions.c" -#endif - -#include "windows\dapl_osd.c" - -- 1.5.2.5 From arlin.r.davis at intel.com Mon Jul 6 12:52:21 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Mon, 6 Jul 2009 12:52:21 -0700 Subject: [ofa-general] [PATCH 08/11] uDAPL scm: destroy QP called before disconnect Message-ID: Handle the case where QP is destroyed before disconnect processing. Windows supports reinit_qp during a disconnect call by destroying the QP and recreating the QO instead of state change from reset to init. Call disconnect in destroy CM code to handle this unexpected state. Signed-off-by: Arlin Davis --- dapl/openib_scm/cm.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c index 5708214..9713641 100644 --- a/dapl/openib_scm/cm.c +++ b/dapl/openib_scm/cm.c @@ -283,6 +283,9 @@ void dapls_ib_cm_free(dp_ib_cm_handle_t cm_ptr, DAPL_EP *ep) return; } + /* free could be called before disconnect */ + dapli_socket_disconnect(cm_ptr); + dapl_os_lock(&cm_ptr->lock); cm_ptr->state = DCM_DESTROY; if ((cm_ptr->ep) && (cm_ptr->ep->cm_handle == cm_ptr)) { -- 1.5.2.5 From arlin.r.davis at intel.com Mon Jul 6 12:52:17 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Mon, 6 Jul 2009 12:52:17 -0700 Subject: [ofa-general] [PATCH 07/11] uDAPL cma: add support for rdma_cm TIME_WAIT event. Message-ID: Nothing to process, simply ack the event. Signed-off-by: Arlin Davis --- dapl/openib_cma/cm.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/dapl/openib_cma/cm.c b/dapl/openib_cma/cm.c index 497f78a..b4e9437 100644 --- a/dapl/openib_cma/cm.c +++ b/dapl/openib_cma/cm.c @@ -1317,6 +1317,8 @@ void dapli_cma_event_cb(void) dapli_cm_active_cb(conn, event); break; case RDMA_CM_EVENT_CONNECT_RESPONSE: + case RDMA_CM_EVENT_TIMEWAIT_EXIT: + break; default: dapl_dbg_log(DAPL_DBG_TYPE_WARN, " cm_event: UNEXPECTED EVENT=%p ID=%p CTX=%p\n", -- 1.5.2.5 From arlin.r.davis at intel.com Mon Jul 6 12:52:25 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Mon, 6 Jul 2009 12:52:25 -0700 Subject: [ofa-general] [PATCH 09/11] uDAPL dtestcm: add UD type QP option to test Message-ID: Add -u for UD type QP's during connection setup. Will setup UD QPs and provide remote AH in connect establishment event. Measures setup/exchange rates. Signed-off-by: Arlin Davis --- test/dtest/dtestcm.c | 83 ++++++++++++++++++++++++++++++++++++------------- 1 files changed, 61 insertions(+), 22 deletions(-) diff --git a/test/dtest/dtestcm.c b/test/dtest/dtestcm.c index 62af6a1..932beac 100644 --- a/test/dtest/dtestcm.c +++ b/test/dtest/dtestcm.c @@ -97,6 +97,7 @@ /* Header files needed for DAT/uDAPL */ #include "dat2/udat.h" +#include "dat2/dat_ib_extensions.h" /* definitions */ #define SERVER_CONN_QUAL 45248 @@ -149,6 +150,7 @@ struct dt_time time; /* defaults */ static int connected = 0; static int multi_listens = 0; +static int ud_test = 0; static int server = 1; static int waiting = 0; static int verbose = 0; @@ -191,7 +193,7 @@ int main(int argc, char **argv) DAT_RETURN ret; /* parse arguments */ - while ((c = getopt(argc, argv, "smwvb:c:d:h:P:p:")) != -1) { + while ((c = getopt(argc, argv, "smwvub:c:d:h:P:p:")) != -1) { switch (c) { case 's': server = 1; @@ -202,6 +204,9 @@ int main(int argc, char **argv) case 'w': waiting = 1; break; + case 'u': + ud_test = 1; + break; case 'c': connections = atoi(optarg); break; @@ -246,11 +251,11 @@ int main(int argc, char **argv) #endif if (!server) { - printf(" Running client on %s with %d connections\n", - provider, connections); + printf(" Running client on %s with %d %s connections\n", + provider, connections, ud_test ? "UD" : "RC"); } else { - printf(" Running server on %s with %d connections\n", - provider, connections); + printf(" Running server on %s with %d %s connections\n", + provider, connections, ud_test ? "UD" : "RC"); } fflush(stdout); @@ -318,8 +323,15 @@ int main(int argc, char **argv) /* create EP */ memset(&ep_attr, 0, sizeof(ep_attr)); - ep_attr.service_type = DAT_SERVICE_TYPE_RC; - ep_attr.max_rdma_size = 0x10000; + if (ud_test) { + ep_attr.service_type = DAT_IB_SERVICE_TYPE_UD; + ep_attr.max_message_size = 2048; + } else { + ep_attr.service_type = DAT_SERVICE_TYPE_RC; + ep_attr.max_rdma_size = 0x10000; + ep_attr.max_rdma_read_in = 4; + ep_attr.max_rdma_read_out = 4; + } ep_attr.max_recv_dtos = 1; ep_attr.max_request_dtos = 1; ep_attr.max_recv_iov = 1; @@ -515,10 +527,10 @@ DAT_RETURN conn_server() DAT_RETURN ret; DAT_EVENT event; DAT_COUNT nmore; - DAT_CR_ARRIVAL_EVENT_DATA *cr_event = - &event.event_data.cr_arrival_event_data; int i,bi; unsigned char *buf; + DAT_CR_ARRIVAL_EVENT_DATA *cr_event = + &event.event_data.cr_arrival_event_data; DAT_CR_PARAM cr_param = { 0 }; printf(" Accepting...\n"); @@ -541,24 +553,26 @@ DAT_RETURN conn_server() } } - if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) { - fprintf(stderr, " Error unexpected CR event : %s\n", - DT_EventToSTr(event.event_number)); - return (DAT_ABORT); + if ((event.event_number != DAT_CONNECTION_REQUEST_EVENT) && + (ud_test && event.event_number != + DAT_IB_UD_CONNECTION_REQUEST_EVENT)) { + fprintf(stderr, " Error unexpected CR event : %s\n", + DT_EventToSTr(event.event_number)); + return (DAT_ABORT); } - + /* use to test rdma_cma timeout logic */ #if defined(_WIN32) || defined(_WIN64) - if (delay) { + if (delay) { printf(" Accept delayed by %d seconds...\n", delay); Sleep(delay * 1000); - } + } #else - if (delay) { + if (delay) { printf(" Accept delayed by %d seconds...\n", delay); sleep(delay); - } + } #endif /* accept connect request from client */ h_cr = cr_event->cr_handle; @@ -622,8 +636,10 @@ DAT_RETURN conn_server() return ret; } } + if ((event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) && + (ud_test && event.event_number != + DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED)) { - if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) { fprintf(stderr, " Error unexpected CR EST " "event : 0x%x %s\n", event.event_number, @@ -748,8 +764,10 @@ DAT_RETURN conn_client() printf("\n Rej Test Done. PASSED\n\n"); exit(0); #endif - if (event.event_number != - DAT_CONNECTION_EVENT_ESTABLISHED) { + if ((event.event_number != + DAT_CONNECTION_EVENT_ESTABLISHED) && + (ud_test && event.event_number != + DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED)) { fprintf(stderr, " Error unexpected conn " "event : 0x%x %s\n", event.event_number, @@ -802,8 +820,28 @@ DAT_RETURN disconnect_eps(void) DAT_CONNECTION_EVENT_DATA *conn_event = &event.event_data.connect_event_data; - if (!connected) + if (!connected) return DAT_SUCCESS; + + /* UD, no connection to disconnect, just free EP's */ + if (ud_test) { + for (i = 0; i < connections; i++) { + ret = dat_ep_free(h_ep[i]); + if (ret != DAT_SUCCESS) { + fprintf(stderr, + " ERR free EP[%d] %p: %s\n", + i, h_ep[i], DT_RetToString(ret)); + } else { + LOGPRINTF(" Freed EP[%d] %p\n", + i, h_ep[i]); + h_ep[i] = DAT_HANDLE_NULL; + } + } + stop = get_time(); + time.epf += ((stop - start) * 1.0e6); + time.total += time.epf; + return DAT_SUCCESS; + } /* * Only the client needs to call disconnect. The server _should_ be able @@ -811,6 +849,7 @@ DAT_RETURN disconnect_eps(void) * disconnect request and then exit. */ if (!server) { + start = get_time(); for (i = 0; i < connections; i++) { LOGPRINTF(" dat_ep_disconnect\n"); ret = dat_ep_disconnect(h_ep[i], -- 1.5.2.5 From arlin.r.davis at intel.com Mon Jul 6 12:52:27 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Mon, 6 Jul 2009 12:52:27 -0700 Subject: [ofa-general] [PATCH 10/11] uDAPL windows: remove obsolete files in dapl/udapl source tree Message-ID: SOURCES,makefile,udapl.r,udapl_exports.src,udapl_sources.c Signed-off-by: Arlin Davis --- dapl/udapl/SOURCES | 59 ------------------------ dapl/udapl/makefile | 7 --- dapl/udapl/udapl.rc | 48 -------------------- dapl/udapl/udapl_exports.src | 14 ------ dapl/udapl/udapl_sources.c | 101 ------------------------------------------ 5 files changed, 0 insertions(+), 229 deletions(-) delete mode 100644 dapl/udapl/SOURCES delete mode 100644 dapl/udapl/makefile delete mode 100644 dapl/udapl/udapl.rc delete mode 100644 dapl/udapl/udapl_exports.src delete mode 100644 dapl/udapl/udapl_sources.c diff --git a/dapl/udapl/SOURCES b/dapl/udapl/SOURCES deleted file mode 100644 index 5548d22..0000000 --- a/dapl/udapl/SOURCES +++ /dev/null @@ -1,59 +0,0 @@ -!if $(FREEBUILD) -TARGETNAME=dapl2 -!else -TARGETNAME=dapl2d -!endif -TARGETPATH=..\..\..\..\bin\user\obj$(BUILD_ALT_DIR) -TARGETTYPE=DYNLINK -DLLENTRY=_DllMainCRTStartup -DLLDEF=$O\udapl_exports.def -USE_CRTDLL=1 - -# pickup local files, then via udapl_sources.c get common files - -SOURCES=udapl.rc \ - dapl_cno_create.c \ - dapl_cno_free.c \ - dapl_cno_modify_agent.c \ - dapl_cno_query.c \ - dapl_cno_wait.c \ - dapl_evd_clear_unwaitable.c \ - dapl_evd_create.c \ - dapl_evd_disable.c \ - dapl_evd_enable.c \ - dapl_evd_modify_cno.c \ - dapl_evd_query.c \ - dapl_evd_set_unwaitable.c \ - dapl_evd_wait.c \ - dapl_init.c \ - dapl_lmr_create.c \ - udapl_sources.c - -INCLUDES=..\include;..\common;windows;..\ibal;..\..\dat\include;\ - ..\..\..\..\inc;..\..\..\..\inc\user; - -DAPL_OPTS= -DEXPORT_DAPL_SYMBOLS -D_VENDOR_IBAL_ -DDAPL_MERGE_CM_DTO - -DAPL_OPTS= $(DAPL_OPTS) -DDAT_EXTENSIONS=1 - -USER_C_FLAGS=$(USER_C_FLAGS) $(DAPL_OPTS) -!if !$(FREEBUILD) -USER_C_FLAGS=$(USER_C_FLAGS) -DDAPL_DBG #-DDAPL_COUNTERS -!endif - -TARGETLIBS= \ - $(SDK_LIB_PATH)\kernel32.lib \ - $(SDK_LIB_PATH)\ws2_32.lib \ -!if $(FREEBUILD) - $(TARGETPATH)\*\dat2.lib \ - $(TARGETPATH)\*\complib.lib \ - $(TARGETPATH)\*\ibal.lib -!else - $(TARGETPATH)\*\dat2d.lib \ - $(TARGETPATH)\*\complibd.lib \ - $(TARGETPATH)\*\ibald.lib -!endif - -# FIX ME ASAP -#MSC_WARNING_LEVEL= /W3 -MSC_WARNING_LEVEL= /W1 /wd4113 diff --git a/dapl/udapl/makefile b/dapl/udapl/makefile deleted file mode 100644 index e26e1c0..0000000 --- a/dapl/udapl/makefile +++ /dev/null @@ -1,7 +0,0 @@ -# -# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source -# file to this component. This file merely indirects to the real make file -# that is shared by all the driver components of the OpenIB Windows project. -# - -!INCLUDE ..\..\..\..\inc\openib.def diff --git a/dapl/udapl/udapl.rc b/dapl/udapl/udapl.rc deleted file mode 100644 index 7c2505a..0000000 --- a/dapl/udapl/udapl.rc +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2007 Intel Corporation. All rights reserved. - * - * This software is available to you under the OpenIB.org BSD license - * below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - * $Id$ - */ - - -#include - -#define VER_FILETYPE VFT_DLL -#define VER_FILESUBTYPE VFT2_UNKNOWN - -#if DBG -#define VER_FILEDESCRIPTION_STR "Direct Access Provider Library v2.0 (Debug)" -#define VER_INTERNALNAME_STR "dapl2d.dll" -#define VER_ORIGINALFILENAME_STR "dapl2d.dll" -#else -#define VER_FILEDESCRIPTION_STR "Direct Access Provider Library v2.0" -#define VER_INTERNALNAME_STR "dapl2.dll" -#define VER_ORIGINALFILENAME_STR "dapl2.dll" -#endif - -#include diff --git a/dapl/udapl/udapl_exports.src b/dapl/udapl/udapl_exports.src deleted file mode 100644 index 54b403b..0000000 --- a/dapl/udapl/udapl_exports.src +++ /dev/null @@ -1,14 +0,0 @@ -#if DBG -LIBRARY dapl2d.dll -#else -LIBRARY dapl2.dll -#endif - - -EXPORTS -dat_provider_init -dat_provider_fini -#ifdef DAT_EXTENSIONS -dapl_extensions -#endif - diff --git a/dapl/udapl/udapl_sources.c b/dapl/udapl/udapl_sources.c deleted file mode 100644 index bbb77f2..0000000 --- a/dapl/udapl/udapl_sources.c +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Include all files that are not in children directories. - */ -#include "dapl_cno_util.c" -#include "dapl_cookie.c" -#include "dapl_cr_accept.c" -#include "dapl_cr_callback.c" -#include "dapl_cr_handoff.c" -#include "dapl_cr_query.c" -#include "dapl_cr_reject.c" -#include "dapl_cr_util.c" -#include "dapl_csp.c" -#include "dapl_debug.c" -#include "dapl_ep_connect.c" -#include "dapl_ep_create.c" -#include "dapl_ep_disconnect.c" -#include "dapl_ep_dup_connect.c" -#include "dapl_ep_free.c" -#include "dapl_ep_get_status.c" -#include "dapl_ep_modify.c" -#include "dapl_ep_post_rdma_read.c" -#include "dapl_ep_post_rdma_read_to_rmr.c" -#include "dapl_ep_post_rdma_write.c" -#include "dapl_ep_post_recv.c" -#include "dapl_ep_post_send.c" -#include "dapl_ep_post_send_invalidate.c" -#include "dapl_ep_query.c" -#include "dapl_ep_recv_query.c" -#include "dapl_ep_reset.c" -#include "dapl_ep_set_watermark.c" -#include "dapl_ep_util.c" -#include "dapl_evd_connection_callb.c" -#include "dapl_evd_cq_async_error_callb.c" -#include "dapl_evd_dequeue.c" -#include "dapl_evd_dto_callb.c" -#include "dapl_evd_free.c" -#include "dapl_evd_post_se.c" -#include "dapl_evd_qp_async_error_callb.c" -#include "dapl_evd_resize.c" -#include "dapl_evd_un_async_error_callb.c" -#include "dapl_evd_util.c" -#include "dapl_get_consumer_context.c" -#include "dapl_get_handle_type.c" -#include "dapl_hash.c" -#include "dapl_hca_util.c" -#include "dapl_ia_close.c" -#include "dapl_ia_ha.c" -#include "dapl_ia_open.c" -#include "dapl_ia_query.c" -#include "dapl_ia_util.c" -#include "dapl_llist.c" -#include "dapl_lmr_free.c" -#include "dapl_lmr_query.c" -#include "dapl_lmr_sync_rdma_read.c" -#include "dapl_lmr_sync_rdma_write.c" -#include "dapl_lmr_util.c" -#include "dapl_mr_util.c" -#include "dapl_name_service.c" -#include "dapl_provider.c" -#include "dapl_psp_create.c" -#include "dapl_psp_create_any.c" -#include "dapl_psp_free.c" -#include "dapl_psp_query.c" -#include "dapl_pz_create.c" -#include "dapl_pz_free.c" -#include "dapl_pz_query.c" -#include "dapl_pz_util.c" -#include "dapl_ring_buffer_util.c" -#include "dapl_rmr_bind.c" -#include "dapl_rmr_create.c" -#include "dapl_rmr_free.c" -#include "dapl_rmr_query.c" -#include "dapl_rmr_util.c" -#include "dapl_rsp_create.c" -#include "dapl_rsp_free.c" -#include "dapl_rsp_query.c" -#include "dapl_set_consumer_context.c" -#include "dapl_sp_util.c" -#include "dapl_ep_create_with_srq.c" -#include "dapl_srq_create.c" -#include "dapl_srq_free.c" -#include "dapl_srq_post_recv.c" -#include "dapl_srq_query.c" -#include "dapl_srq_resize.c" -#include "dapl_srq_set_lw.c" -#include "dapl_srq_util.c" -#include "dapl_timer_util.c" - -#include "..\ibal\dapl_ibal_cm.c" -#include "..\ibal\dapl_ibal_name_service.c" -#include "..\ibal\dapl_ibal_qp.c" -#include "..\ibal\dapl_ibal_cq.c" -#include "..\ibal\dapl_ibal_util.c" - -//#include "..\ibal\dapl_ibal_mrdb.c" - -#ifdef DAT_EXTENSIONS -#include "..\ibal\dapl_ibal_extensions.c" -#endif - -#include "windows\dapl_osd.c" -- 1.5.2.5 From arlin.r.davis at intel.com Mon Jul 6 12:52:32 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Mon, 6 Jul 2009 12:52:32 -0700 Subject: [ofa-general] [PATCH 11/11] uDAPL scm: set TCP_NODELAY sockopt on the server side for sends. Message-ID: scm provider sends small messages from both server and client sides. Set NODELAY on both sides to avoid send delays either way. Signed-off-by: Arlin Davis --- dapl/openib_scm/cm.c | 21 +++++++++++++++++---- 1 files changed, 17 insertions(+), 4 deletions(-) diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c index 9713641..32e50fc 100644 --- a/dapl/openib_scm/cm.c +++ b/dapl/openib_scm/cm.c @@ -427,7 +427,7 @@ DAT_RETURN dapli_socket_disconnect(dp_ib_cm_handle_t cm_ptr) */ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err) { - int len, opt = 1; + int ret, len, opt = 1; struct iovec iov[2]; struct dapl_ep *ep_ptr = cm_ptr->ep; @@ -447,8 +447,12 @@ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err) " socket connected, write QP and private data\n"); /* no delay for small packets */ - setsockopt(cm_ptr->socket, IPPROTO_TCP, TCP_NODELAY, - (char *)&opt, sizeof(opt)); + ret = setsockopt(cm_ptr->socket, IPPROTO_TCP, TCP_NODELAY, + (char *)&opt, sizeof(opt)); + if (ret) + dapl_log(DAPL_DBG_TYPE_WARN, + " connected: NODELAY setsockopt: %s\n", + strerror(errno)); /* send qp info and pdata to remote peer */ iov[0].iov_base = (void *)&cm_ptr->dst; @@ -798,6 +802,7 @@ dapli_socket_listen(DAPL_IA * ia_ptr, DAT_CONN_QUAL serviceID, DAPL_SP * sp_ptr) setsockopt(cm_ptr->socket, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt)); + addr.sin_port = htons(serviceID); addr.sin_family = AF_INET; addr.sin_addr.s_addr = INADDR_ANY; @@ -839,7 +844,7 @@ dapli_socket_listen(DAPL_IA * ia_ptr, DAT_CONN_QUAL serviceID, DAPL_SP * sp_ptr) static void dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr) { dp_ib_cm_handle_t acm_ptr; - int len; + int ret, len, opt = 1; dapl_dbg_log(DAPL_DBG_TYPE_EP, " socket_accept\n"); @@ -868,6 +873,14 @@ static void dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr) return; } + /* no delay for small packets */ + ret = setsockopt(acm_ptr->socket, IPPROTO_TCP, TCP_NODELAY, + (char *)&opt, sizeof(opt)); + if (ret) + dapl_log(DAPL_DBG_TYPE_WARN, + " accept: NODELAY setsockopt: %s\n", + strerror(errno)); + acm_ptr->state = DCM_ACCEPTING; dapli_cm_queue(acm_ptr); -- 1.5.2.5 From yosefe at voltaire.com Mon Jul 6 12:54:33 2009 From: yosefe at voltaire.com (Yossi Etigin) Date: Mon, 06 Jul 2009 22:54:33 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A521784.5090304@Voltaire.COM> References: <4A521784.5090304@Voltaire.COM> Message-ID: <4A525679.7090907@voltaire.com> Moni Shoua wrote: > diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c > index a0e9753..024fd18 100644 > --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c > +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c > @@ -379,6 +379,7 @@ static int ipoib_mcast_join_complete(int status, > struct ipoib_mcast *mcast = multicast->context; > struct net_device *dev = mcast->dev; > struct ipoib_dev_priv *priv = netdev_priv(dev); > + struct ipoib_mcast *next_mcast; > > ipoib_dbg_mcast(priv, "join completion for %pI6 (status %d)\n", > mcast->mcmember.mgid.raw, status); > @@ -427,9 +428,16 @@ static int ipoib_mcast_join_complete(int status, > > mutex_lock(&mcast_mutex); > spin_lock_irq(&priv->lock); > - if (test_bit(IPOIB_MCAST_RUN, &priv->flags)) > - queue_delayed_work(ipoib_workqueue, &priv->mcast_task, > - mcast->backoff * HZ); > + if (test_bit(IPOIB_MCAST_RUN, &priv->flags)) { > + list_for_each_entry(next_mcast, &priv->multicast_list, list) { > + if (!test_bit(IPOIB_MCAST_FLAG_SENDONLY, &next_mcast->flags) > + && !test_bit(IPOIB_MCAST_FLAG_BUSY, &next_mcast->flags) > + && !test_bit(IPOIB_MCAST_FLAG_ATTACHED, &next_mcast->flags)) > + break; > + } > + queue_delayed_work(ipoib_workqueue, &priv->mcast_join_task, > + next_mcast->backoff * HZ); > + } > spin_unlock_irq(&priv->lock); > mutex_unlock(&mcast_mutex); > The scan is only done to get the backoff to use, right? You should also check (&next_mcast->list != &priv->multicast_list) to make sure something is really found. For example the device might be flushed and priv->multicast_list become empty (emptying the multicast list does not wait for the query to finish). > @@ -577,6 +585,9 @@ void ipoib_mcast_join_task(struct work_struct *work) > break; > } > > + if (!list_is_last(&mcast->list, &priv->multicast_list)) > + list_move_tail(&mcast->list, &priv->multicast_list); > + > ipoib_mcast_join(dev, mcast, 1); > return; > } I don't think the list_is_last() check is really needed here. And isn't priv->lock required for priv->multicast_list modifications? --Yossi From donald.e.wood at intel.com Mon Jul 6 13:39:16 2009 From: donald.e.wood at intel.com (Wood, Donald E) Date: Mon, 6 Jul 2009 13:39:16 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> Message-ID: <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> >[root at lv4 examples]# ./ucmatose >'cmatose: starting server >initiating data transfers >completing sends >--> Hangs here > > [root at lv5 examples]# ./ucmatose -s 192.168.10.33 >cmatose: starting client >cmatose: connecting >receiving data transfers >sending replies >data transfers complete >--> Hangs here I debugged this to see why it hangs. The problem, as I see it, is in the server side. The problem is shown in the diagram below (connection set up and tear down are omitted). Both sides post 10 receive buffers. 1. Server sends ten messages (not signaled) 2. Client polls recv cq and receives 10 messages 3. Client sends 10 messages 4. Server polls the send cq for completions 5. Server polls the recv cq for completions The server hangs in step 4 where it is looking for send completions but there will not be any because the signaled flag was not set in step 1. ucmatose completes when I change the following line: send_wr.send_flags = 0; to send_wr.send_flags = IBV_SEND_SIGNALED; Don Wood From rdreier at cisco.com Mon Jul 6 14:05:39 2009 From: rdreier at cisco.com (Roland Dreier) Date: Mon, 06 Jul 2009 14:05:39 -0700 Subject: [ofa-general] Re: A question about tx lock in ipoib_flush_paths In-Reply-To: <4A4E190D.7010004@voltaire.com> (Yossi Etigin's message of "Fri, 03 Jul 2009 17:43:25 +0300") References: <4A4E190D.7010004@voltaire.com> Message-ID: > In ipoib_flush_paths(), we take the netif_tx_lock to remove a path > My question is - what data does this lock protect? > It isn't path->list and path->rb_node, because priv->lock is enough to protect them. > > It might be neigh and neigh->ah, to avoid freeing the neighbour and its address > handle while ipoib_start_xmit() is using it, but this particular part is done *outside* > the tx lock. You probably already saw this from the history of the file, but this locking was added in 9217b27b: IB/ipoib: Fix flush/start xmit race (from code review) Prevent flush task from freeing the ipoib_neigh pointer, while ipoib_start_xmit() is accessing the ipoib_neigh through the pointer it has loaded from the skb's hardware address. > Unless I'm missing something - shouldn't the code: > > spin_unlock_irqrestore(&priv->lock, flags); > netif_tx_unlock_bh(dev); > wait_for_completion(&path->done); > release >> path_free(dev, path); > lock >> netif_tx_lock_bh(dev); > spin_lock_irqsave(&priv->lock, flags); > > Be like this: > > spin_unlock_irqrestore(&priv->lock, flags); > netif_tx_unlock_bh(dev); > wait_for_completion(&path->done); > lock >> netif_tx_lock_bh(dev); > release >> path_free(dev, path); > spin_lock_irqsave(&priv->lock, flags); I don't think it matters -- the path has been removed from every externally visible list/rbtree already so there's no way anyone could grab it after we drop the tx lock. - R. From yossi.openib at gmail.com Mon Jul 6 17:13:03 2009 From: yossi.openib at gmail.com (Yossi Etigin) Date: Tue, 07 Jul 2009 03:13:03 +0300 Subject: [ofa-general] Re: A question about tx lock in ipoib_flush_paths In-Reply-To: References: <4A4E190D.7010004@voltaire.com> Message-ID: <4A52930F.9050408@gmail.com> Roland Dreier wrote: > > In ipoib_flush_paths(), we take the netif_tx_lock to remove a path > > My question is - what data does this lock protect? > > It isn't path->list and path->rb_node, because priv->lock is enough to protect them. > > > > It might be neigh and neigh->ah, to avoid freeing the neighbour and its address > > handle while ipoib_start_xmit() is using it, but this particular part is done *outside* > > the tx lock. > > You probably already saw this from the history of the file, but this > locking was added in 9217b27b: > > IB/ipoib: Fix flush/start xmit race (from code review) > > Prevent flush task from freeing the ipoib_neigh pointer, while > ipoib_start_xmit() is accessing the ipoib_neigh through the pointer it > has loaded from the skb's hardware address. > > > Unless I'm missing something - shouldn't the code: > > > > spin_unlock_irqrestore(&priv->lock, flags); > > netif_tx_unlock_bh(dev); > > wait_for_completion(&path->done); > > release >> path_free(dev, path); > > lock >> netif_tx_lock_bh(dev); > > spin_lock_irqsave(&priv->lock, flags); > > > > Be like this: > > > > spin_unlock_irqrestore(&priv->lock, flags); > > netif_tx_unlock_bh(dev); > > wait_for_completion(&path->done); > > lock >> netif_tx_lock_bh(dev); > > release >> path_free(dev, path); > > spin_lock_irqsave(&priv->lock, flags); > > I don't think it matters -- the path has been removed from every > externally visible list/rbtree already so there's no way anyone could > grab it after we drop the tx lock. > > - R. Couldn't ipoib_start_xmit() grab ipoib_neigh, and cause the same thing the commit above was intended to fix? I saw in the mails that dropping the lock is the v2 of the patch, the original one did not drop it. --Yossi From ranjit.pandit.ib at gmail.com Mon Jul 6 19:32:14 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Mon, 6 Jul 2009 19:32:14 -0700 Subject: [ofa-general] 4GB physical memory limit on Chelsio Message-ID: <96f8e60e0907061932n19a8f91amb253a28a039b4d90@mail.gmail.com> In iwch_provider.c, line 718, iwch_get_dma_mr() there is a comment which seems to indicate that the Chelsio card supports registering only 4GB of physical memory. Is that correct? Most of our systems have way greater memory. In fact, the HPC7000 blades we are testing on has 64GB of memory. The problem we are running into is that our driver calls ib_get_dma_mr() to register all physical memory. When the driver attempts to post recv buffers it's getting -EINVAL error. I traced it down to line 253 in iwch_qp.c. if (sg_list[i].addr + ((u64) sg_list[i].length) > mhp->attr.va_fbo + ((u64) mhp->attr.len)) { PDBG("%s %d\n", __func__, __LINE__); return -EINVAL; } when i printed out those values I got the following: Jul 6 19:13:17 lv2 kernel: sg_list[i].addr faca80000, sg_list[i].length 7360 mhp->attr.va_fbo 0 mhp->attr.len 4294967295 The length of the registration indeed seems to be 4GB in size. Is there a way to make it work on systems with > 4GB of physical memory. Is there a configuration parameter? Thanks, Ranjit From ranjit.pandit.ib at gmail.com Mon Jul 6 20:07:19 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Mon, 6 Jul 2009 20:07:19 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> Message-ID: <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> I'm still seeing the hang. > I debugged this to see why it hangs.  The problem, as I see it, is in the server side.  The problem is shown in the diagram below (connection set up and tear down are omitted).  Both sides post 10 receive buffers. > > 1.                                              Server sends ten messages (not signaled) > 2.      Client polls recv cq and >          receives 10 messages > 3.      Client sends 10 messages > 4.                                              Server polls the send cq for completions > 5.                                              Server polls the recv cq for completions > > The server hangs in step 4 where it is looking for send completions but there will not be any because the signaled flag was not set in step 1. > > ucmatose completes when I change the following line: >        send_wr.send_flags = 0; > to >        send_wr.send_flags = IBV_SEND_SIGNALED; > > Don Wood > > From ranjit.pandit.ib at gmail.com Mon Jul 6 20:36:10 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Mon, 6 Jul 2009 20:36:10 -0700 Subject: [ofa-general] Loopback connection not working on Chelsio Message-ID: <96f8e60e0907062036p4e4d8850yd700c90f0bf70069@mail.gmail.com> Hi, I'm having trouble getting loopback rdma connections working on Chelsio. We were told that loopback connections are not supported. Is this an iWarp or Chelsio specific issue? What are the issues that is preventing it from working? Are there plans for supporting this capability? In our platform, clients and servers could be running on the same node and they talk to each other via rdma channels. Btw, here is the error with rping in loopback mode. Our application is getting the same error. [root at lv1 ~]# rping -c -v -d -a 192.168.10.30 -p 9999 port 9999 created cm_id 0x12782810 cma_event type RDMA_CM_EVENT_ADDR_RESOLVED cma_id 0x12782810 (parent) cma_event type RDMA_CM_EVENT_ROUTE_RESOLVED cma_id 0x12782810 (parent) rdma_resolve_addr - rdma_resolve_route successful created pd 0x127815d0 created channel 0x12781500 created cq 0x12782c80 created qp 0x12782f50 rping_setup_buffers called on cb 0x12780010 allocated & registered buffers... rdma_connect error -1 connect error -1 rping_free_buffers called on cb 0x12780010 cq_thread started. destroy cm_id 0x12782810 From sashak at voltaire.com Mon Jul 6 20:51:09 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Tue, 7 Jul 2009 06:51:09 +0300 Subject: [ofa-general] [PATCH] libibnetdisc/Makefile.am: add internal.h to SOURCES Message-ID: <20090707035109.GA15530@me> list [was: Re: [ewg] OFED-1.5-20090706-0600 build break] Reply-To: In-Reply-To: <4A525409.6000005 at mellanox.co.il> Without this 'make dist' (used in daily in release builds) generates broken infiniand-diags tarball. Signed-off-by: Sasha Khapyorsky --- On 22:44 Mon 06 Jul , Tziporet Koren wrote: > Adding Sasha - maybe he knows the reason Yes, I have an idea. Applying shortly. > > ... > > src/ibnetdisc.c:57:22: error: internal.h: No such file or directory > > In file included from src/ibnetdisc.c:58: infiniband-diags/libibnetdisc/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/infiniband-diags/libibnetdisc/Makefile.am b/infiniband-diags/libibnetdisc/Makefile.am index e6e3620..ddcf053 100644 --- a/infiniband-diags/libibnetdisc/Makefile.am +++ b/infiniband-diags/libibnetdisc/Makefile.am @@ -18,7 +18,7 @@ else libibnetdisc_version_script = endif -libibnetdisc_la_SOURCES = src/ibnetdisc.c src/chassis.c src/chassis.h +libibnetdisc_la_SOURCES = src/ibnetdisc.c src/chassis.c src/chassis.h src/internal.h libibnetdisc_la_CFLAGS = -Wall $(DBGFLAGS) libibnetdisc_la_LDFLAGS = -version-info $(ibnetdisc_api_version) \ -export-dynamic $(libibnetdisc_version_script) \ -- 1.6.3.3 From devesh28 at gmail.com Mon Jul 6 22:43:34 2009 From: devesh28 at gmail.com (Devesh Sharma) Date: Tue, 7 Jul 2009 11:13:34 +0530 Subject: Fwd: [ofa-general] Performance evaluation of Opensm In-Reply-To: <309a667c0907062242o1991e7f8lc64f15347cf39eda@mail.gmail.com> References: <309a667c0907060439u1c90cc85h3bc8aa43d74d9e31@mail.gmail.com> <200907061703.17711.cap@nsc.liu.se> <309a667c0907062242o1991e7f8lc64f15347cf39eda@mail.gmail.com> Message-ID: <309a667c0907062243v3d6b8cdbhca9f374f1553a38a@mail.gmail.com> On Mon, Jul 6, 2009 at 8:33 PM, Peter Kjellstrom wrote: > On Monday 06 July 2009, Devesh Sharma wrote: >> Hello list. >> >> Is there any tool or method available to get some performance numbers >> related to Opensm? What are the major performance parameters of OpenSM >> which can >> be used for performance measurement analaysis? > > Do you refer to the performance of the fabric as setup by OpenSM or the > performance of OpenSM itself? Performance of OpenSM itself. There are many tool available to measure performance of Fabric. > > /Peter > >> Thanks in advance >> regards >> Devesh Sharma > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From tdhanu_2000 at yahoo.com Mon Jul 6 22:56:28 2009 From: tdhanu_2000 at yahoo.com (dhananjay tembe) Date: Tue, 7 Jul 2009 11:26:28 +0530 (IST) Subject: Fwd: [ofa-general] Performance evaluation of Opensm Message-ID: <617255.53094.qm@web94203.mail.in2.yahoo.com> Hi Devesh, OFED installer comes with a tool called as "osmtest" which is a test program for opensm and for subnet administrator that comes within the opensm itself. The command is /usr/sbin/osmtest. Another way to test it might be to try it in a large fabric. If you do not have a large fabric, try using the simulator that comes with ofed. The command for the simulator is /usr/bin/ibsim I have not used the simulator yet. So I do not know much about it. Thanks and regards, ---Dhananjay. --- On Tue, 7/7/09, Devesh Sharma wrote: > From: Devesh Sharma > Subject: Fwd: [ofa-general] Performance evaluation of Opensm > To: general at lists.openfabrics.org > Date: Tuesday, 7 July, 2009, 11:13 AM > On Mon, Jul 6, 2009 at 8:33 PM, Peter > Kjellstrom > wrote: > > On Monday 06 July 2009, Devesh Sharma wrote: > >> Hello list. > >> > >> Is there any tool or method available to get some > performance numbers > >> related to Opensm? What are the major performance > parameters of OpenSM > >> which can > >> be used for performance measurement analaysis? > > > > Do you refer to the performance of the fabric as setup > by OpenSM or the > > performance of OpenSM itself? > Performance of OpenSM itself. There are many tool available > to measure > performance of Fabric. > > > > /Peter > > > >> Thanks in advance > >> regards > >> Devesh Sharma > > > > _______________________________________________ > > general mailing list > > general at lists.openfabrics.org > > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > Looking for local information? Find it on Yahoo! Local http://in.local.yahoo.com/ From ogerlitz at voltaire.com Tue Jul 7 01:15:06 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Tue, 07 Jul 2009 11:15:06 +0300 Subject: [ofa-general] Loopback connection not working on Chelsio In-Reply-To: <96f8e60e0907062036p4e4d8850yd700c90f0bf70069@mail.gmail.com> References: <96f8e60e0907062036p4e4d8850yd700c90f0bf70069@mail.gmail.com> Message-ID: <4A53040A.4020601@voltaire.com> pandit ib wrote: > I'm having trouble getting loopback rdma connections working on Chelsio. We were told that loopback connections are not supported. Is this an iWarp or Chelsio specific issue? What are the issues that is preventing it from working? Basically, thinking a bit about this, loopback communication is possible/trivial with IB since (I am quite sure) by spec HCA must/should support some sort of bridging, in other words if both QPs are on the same node then their communication doesn't go to the network (IB switches will never send a packet through the port it was received). Now, when your L2 network is Ethernet, you need to have someone let you do this U turn. When communication is going through the network stack, the latter takes care this, but when it doesn't you need to have support either at the NIC or the switch. Modern (H2/2009, later) NICs, such as those supporting SR-IOV and/or multiple physical functions would support some sort of bridging. I don't think its an iwarp limitation. Or. From ogerlitz at Voltaire.com Tue Jul 7 01:25:16 2009 From: ogerlitz at Voltaire.com (Or Gerlitz) Date: Tue, 07 Jul 2009 11:25:16 +0300 Subject: [ofa-general] 4GB physical memory limit on Chelsio In-Reply-To: <96f8e60e0907061932n19a8f91amb253a28a039b4d90@mail.gmail.com> References: <96f8e60e0907061932n19a8f91amb253a28a039b4d90@mail.gmail.com> Message-ID: <4A53066C.4070105@Voltaire.com> pandit ib wrote: > Is there a way to make it work on systems with > 4GB of physical memory. I think you want to look on zero STag, e.g see commit 96f15c0.... "RDMA/core: Add local DMA L_Key support" and bd7ed1... "RPC/RDMA: check selected memory registration mode at runtime" Or. From jackm at dev.mellanox.co.il Tue Jul 7 02:17:19 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Tue, 7 Jul 2009 12:17:19 +0300 Subject: [ofa-general] Compilation errors with OFED 1.4.1/ 1.4 In-Reply-To: <3307cdf90906212157o687a6929w49f9abc8b16fc131@mail.gmail.com> References: <9759F033B56A60469F217C49AC58E2BF085A541B@MAILUK2.rms.com> <4A3E9957.50104@mellanox.co.il> <3307cdf90906212157o687a6929w49f9abc8b16fc131@mail.gmail.com> Message-ID: <200907071217.19649.jackm@dev.mellanox.co.il> Looks like the OFED installation is faulty. The missing function declarations are all found in header files under directory (on your system) /usr/src/ofa_kernel/kernel_addons/backport/2.6.16_sles10_sp2. These header files must be taken before the regular kernel header files (they "include_next" to the regular files, and supplement these files with missing macros, static inline functions, etc.). For example, dma_map_single_attr() is found in file /usr/src/ofa_kernel/kernel_addons/backport/2.6.16_sles10_sp2/include/linux/dma-mapping.h For some reason, your compilation script is not taking directory /usr/src/ofa_kernel/kernel_addons/backport/2.6.16_sles10_sp2/include in the include path before the regular kernel includes. -Jack On Monday 22 June 2009 07:57, Rajouri Jammu wrote: > Tziporet, > > We (Mellanox) are running regression testing on SLES10 SP2 with OFED 1.4.1 > without any problem > > I don't think it's a runtime issue. > > The problem: > /usr/src/ofa_kernel/include/rdma/ib_verbs.h in OFED 1.4.1 is referring to > symbols like "dma_map_single_attrs" etc, but when I looked for those symbols > in /usr/src/linux-2.6.16.60-0.21 I did not find any references. > > Can you suggest a file that I should include that will resolve this issue? > > Thanks. > > On Sun, Jun 21, 2009 at 1:34 PM, Tziporet Koren > wrote: > > > Rajouri Jammu wrote: > > > >> Thanks for the response Evan. > >> > >> It appears to me that this is a problem in OFED 1.4.1 since the header > >> file in /usr/src/ofa_kernel/include/rdma/ib_verbs.h is referring to data > >> structures that are not present in the SLES 10 u 2 kernel. > >> > >> However, someone else indicated it was a problem in our code. > >> > >> Btw, my code compiled fine with OFED 1.3.1 on the same kernel. > >> > >> > > We (Mellanox) are running regression testing on SLES10 SP2 with OFED 1.4.1 > > without any problem > > So I guess its something specific in your system, or order of include files > > you use > > > > Tziporet > > > From vlad at lists.openfabrics.org Tue Jul 7 02:25:20 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Tue, 7 Jul 2009 02:25:20 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090707-0200 daily build status Message-ID: <20090707092520.7CD7DE283F8@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_probe': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/nes/nes.c:591: error: implicit declaration of function 'pcie_get_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/nes/nes_hw.h:1172: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_probe': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/nes/nes.c:591: error: implicit declaration of function 'pcie_get_readrq' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/nes/nes_hw.h:1172: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_probe': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/nes/nes.c:591: error: implicit declaration of function 'pcie_get_readrq' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.20_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_probe': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/nes/nes.c:591: error: implicit declaration of function 'pcie_get_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_x86_64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.27_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.27_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.27_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_x86_64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1411: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1430: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1411: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1430: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/nes/nes_hw.h:1172: error: field 'napi' has incomplete type /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.23_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_ia64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_probe': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_ia64_check/drivers/infiniband/hw/nes/nes.c:591: error: implicit declaration of function 'pcie_get_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_ia64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_ia64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_ia64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/common.h:645, from /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:60: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_ia64_check/drivers/net/cxgb3/adapter.h:198: error: field 'lro_frag_tbl' has incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_ia64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ppc64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/nes/nes.c:592: error: implicit declaration of function 'pcie_set_readrq' /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/nes/nes.c: In function 'nes_store_wqm_quanta': /home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/nes/nes.c:1119: error: implicit declaration of function 'strict_strtoul' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/nes/nes.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ppc64_check/drivers/infiniband/hw/nes] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090707-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From kliteyn at dev.mellanox.co.il Tue Jul 7 02:29:17 2009 From: kliteyn at dev.mellanox.co.il (Yevgeny Kliteynik) Date: Tue, 07 Jul 2009 12:29:17 +0300 Subject: Fwd: [ofa-general] Performance evaluation of Opensm In-Reply-To: <617255.53094.qm@web94203.mail.in2.yahoo.com> References: <617255.53094.qm@web94203.mail.in2.yahoo.com> Message-ID: <4A53156D.2040805@dev.mellanox.co.il> Hi Davesh, It's kind of hard to talk about "performance of OpenSM". Subnet Manager has different phases and modes of operation, each of them is completely separate issue: - Fabric discovery - Fabric ports/nodes configuration - Unicast routing calculation - Unicast routing configuration on fabric switches - Multicast routing calculation - Multicast routing configuration on fabric switches - SA queries processing - Memory consumption - Different routing algorithms consume different time and memory - QoS - etc, etc, etc Most of the above can be measured only on real cluster. Some (such as routing calculation and memory consumption) can be measured while OSM is running on top of the simulator. Some are very affected by the number of CPU cores that you have on the management node (e.g. SA queries processing), others mostly affected by the CPU frequency (unicast routing). Also, various OpenSM options can affect these phases, such as unicast routing cache may reduce routing calculation time to 0. Sorry that I'm not really answering your question :( I just want to point out the fact that there are many aspects that should be considered when talking about OpenSM performance. If what you're interested in is just "system-wide" numbers, then you'll probably want to know how much time it takes for the OpenSM to bring up cluster from scratch, or how much time it takes to reconfigure the fabric after some change. I think that these numbers can come only from the guys that administer IB clusters, and they would be highly dependent on the management node CPU. -- Yevgeny dhananjay tembe wrote: > Hi Devesh, > > OFED installer comes with a tool called as "osmtest" which is a test program for opensm and for subnet administrator that comes within the opensm itself. The command is /usr/sbin/osmtest. > > Another way to test it might be to try it in a large fabric. If you do not have a large fabric, try using the simulator that comes with ofed. > The command for the simulator is /usr/bin/ibsim > I have not used the simulator yet. So I do not know much about it. > > Thanks and regards, > ---Dhananjay. > > > --- On Tue, 7/7/09, Devesh Sharma wrote: > >> From: Devesh Sharma >> Subject: Fwd: [ofa-general] Performance evaluation of Opensm >> To: general at lists.openfabrics.org >> Date: Tuesday, 7 July, 2009, 11:13 AM >> On Mon, Jul 6, 2009 at 8:33 PM, Peter >> Kjellstrom >> wrote: >>> On Monday 06 July 2009, Devesh Sharma wrote: >>>> Hello list. >>>> >>>> Is there any tool or method available to get some >> performance numbers >>>> related to Opensm? What are the major performance >> parameters of OpenSM >>>> which can >>>> be used for performance measurement analaysis? >>> Do you refer to the performance of the fabric as setup >> by OpenSM or the >>> performance of OpenSM itself? >> Performance of OpenSM itself. There are many tool available >> to measure >> performance of Fabric. >>> /Peter >>> >>>> Thanks in advance >>>> regards >>>> Devesh Sharma >>> _______________________________________________ >>> general mailing list >>> general at lists.openfabrics.org >>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>> >>> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general >>> >> _______________________________________________ >> general mailing list >> general at lists.openfabrics.org >> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >> >> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general >> > > > Looking for local information? Find it on Yahoo! Local http://in.local.yahoo.com/ > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From monis at Voltaire.COM Tue Jul 7 02:32:51 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Tue, 07 Jul 2009 12:32:51 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: References: <4A521784.5090304@Voltaire.COM> Message-ID: <4A531643.70304@Voltaire.COM> > Wouldn't it make more sense to fix the bonding device or core networking > (wherever the problem is) so that it recomputes the multicast list when > the bonding hardware type changes? After this patch, do we end up with > an IPoIB interface that's not a member of the all hosts multicast group? > (That seems like it would lead to confusing breakage later) > > - R. You are right. The full fix to the example I brought needs to be higher in the stack. I'll see how can I fix it there also. But bonding is just an example (and I don't have any other yet). The fix in IPoIB prevents starvation of pending multicast addresses if one constantly fails to join. Now, IPoIB assumes that if a join for one mcast group fails then all other joins would fail or otherwise why not let groups a try? I think that this is a wrong assumption. From monis at Voltaire.COM Tue Jul 7 02:34:55 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Tue, 07 Jul 2009 12:34:55 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A525679.7090907@voltaire.com> References: <4A521784.5090304@Voltaire.COM> <4A525679.7090907@voltaire.com> Message-ID: <4A5316BF.6010300@Voltaire.COM> Yossi Etigin wrote: > Moni Shoua wrote: >> diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c >> index a0e9753..024fd18 100644 >> --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c >> +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c >> @@ -379,6 +379,7 @@ static int ipoib_mcast_join_complete(int status, >> struct ipoib_mcast *mcast = multicast->context; >> struct net_device *dev = mcast->dev; >> struct ipoib_dev_priv *priv = netdev_priv(dev); >> + struct ipoib_mcast *next_mcast; >> >> ipoib_dbg_mcast(priv, "join completion for %pI6 (status %d)\n", >> mcast->mcmember.mgid.raw, status); >> @@ -427,9 +428,16 @@ static int ipoib_mcast_join_complete(int status, >> >> mutex_lock(&mcast_mutex); >> spin_lock_irq(&priv->lock); >> - if (test_bit(IPOIB_MCAST_RUN, &priv->flags)) >> - queue_delayed_work(ipoib_workqueue, &priv->mcast_task, >> - mcast->backoff * HZ); >> + if (test_bit(IPOIB_MCAST_RUN, &priv->flags)) { >> + list_for_each_entry(next_mcast, &priv->multicast_list, list) { >> + if (!test_bit(IPOIB_MCAST_FLAG_SENDONLY, &next_mcast->flags) >> + && !test_bit(IPOIB_MCAST_FLAG_BUSY, &next_mcast->flags) >> + && !test_bit(IPOIB_MCAST_FLAG_ATTACHED, &next_mcast->flags)) >> + break; >> + } >> + queue_delayed_work(ipoib_workqueue, &priv->mcast_join_task, >> + next_mcast->backoff * HZ); >> + } >> spin_unlock_irq(&priv->lock); >> mutex_unlock(&mcast_mutex); >> > > The scan is only done to get the backoff to use, right? Yes > > You should also check (&next_mcast->list != &priv->multicast_list) to make sure > something is really found. For example the device might be flushed and priv->multicast_list > become empty (emptying the multicast list does not wait for the query to finish). You are right. > >> @@ -577,6 +585,9 @@ void ipoib_mcast_join_task(struct work_struct *work) >> break; >> } >> >> + if (!list_is_last(&mcast->list, &priv->multicast_list)) >> + list_move_tail(&mcast->list, &priv->multicast_list); >> + >> ipoib_mcast_join(dev, mcast, 1); >> return; >> } > > I don't think the list_is_last() check is really needed here. It's just to prevent from doing nothing. > And isn't priv->lock required for priv->multicast_list modifications? You are right again. > > --Yossi > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From ogerlitz at voltaire.com Tue Jul 7 03:03:14 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Tue, 07 Jul 2009 13:03:14 +0300 Subject: [ofa-general] Write combining support in OFED 1.5 In-Reply-To: <200907061753.40760.jackm@dev.mellanox.co.il> References: <200907061753.40760.jackm@dev.mellanox.co.il> Message-ID: <4A531D62.6040804@voltaire.com> Jack Morgenstein wrote: > I've been looking at the write-combining support in the 2.6.30 kernel, and it looks good [...] from the write-combining support in OFED 1.4: > Hi Jack, is there some WC related code which is in ofed but not in mainline? why? Or. From ogerlitz at voltaire.com Tue Jul 7 03:12:54 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Tue, 07 Jul 2009 13:12:54 +0300 Subject: [ofa-general] Re: [PATCH] rdma_cm: Add debugfs entries to monitor rdma_cm connections In-Reply-To: <4A520600.2080004@opengridcomputing.com> References: <4A3E45D3.3040405@Voltaire.COM> <4A50C13F.6070506@Voltaire.COM> <4A520600.2080004@opengridcomputing.com> Message-ID: <4A531FA6.6050803@voltaire.com> Steve Wise wrote: > I agree that this is useful for debugging. Roland, Sean, I agree with Steve and Moni. Today there's no way to know what rdma-cm connections/sessions are open now and with patch there is a way, so debugfs support is very useful for debugging rdma-cm ULPs, simple as that (e.g in the same manner ipoib has debugfs support), lets get this in for 2.6.32 Next, for more production ready monitoring support, netlink solutions could be great, Jason, thoughts? Or. From jackm at dev.mellanox.co.il Tue Jul 7 03:38:42 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Tue, 7 Jul 2009 13:38:42 +0300 Subject: [ofa-general] Write combining support in OFED 1.5 In-Reply-To: <4A531D62.6040804@voltaire.com> References: <200907061753.40760.jackm@dev.mellanox.co.il> <4A531D62.6040804@voltaire.com> Message-ID: <200907071338.43055.jackm@dev.mellanox.co.il> On Tuesday 07 July 2009 13:03, Or Gerlitz wrote: > Jack Morgenstein wrote: > > I've been looking at the write-combining support in the 2.6.30 kernel, and it looks good [...] from the write-combining support in OFED 1.4: > > > Hi Jack, is there some WC related code which is in ofed but not in > mainline? why? > > Or. Yes, there is (see kernel_patches/fixes/mlx4_0010_add_wc.patch). OFED 1.4 is based on kernel 2.6.27, which did not have kernel-based WC support -- this support was put in the kernel only in kernel 2.6.29. With OFED 1.5, I am trying to use the kernel-based WC support, and eliminate the patch. To do this, I need 2 things: 1. macro pgprot_writecombine to be defined for PPC/PPC64 under the arch directory (file arch/powerpc/include/asm/pgtable.h) 2. A way of determining (via the standard kernel) if write-combining is available on a given platform (to allow disabling use of ConnectX blueflame in userspace if WC is not available). -Jack From ogerlitz at voltaire.com Tue Jul 7 03:54:04 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Tue, 07 Jul 2009 13:54:04 +0300 Subject: [ofa-general] Write combining support in OFED 1.5 In-Reply-To: <200907071338.43055.jackm@dev.mellanox.co.il> References: <200907061753.40760.jackm@dev.mellanox.co.il> <4A531D62.6040804@voltaire.com> <200907071338.43055.jackm@dev.mellanox.co.il> Message-ID: <4A53294C.8020302@voltaire.com> Jack Morgenstein wrote: > Yes, see kernel_patches/fixes/mlx4_0010_add_wc.patch. With OFED 1.5, I am trying to use the kernel-based WC support, and eliminate the patch. To do this, I need 2 things: > 1. macro pgprot_writecombine to be defined for PPC/PPC64 under the arch directory (file arch/powerpc/include/asm/pgtable.h) > 2. A way of determining (via the standard kernel) if write-combining is available on a given platform (to allow disabling use of ConnectX blueflame in userspace if WC is not available). > It seems that none of the issues are IB specific, correct? As for PPC guys I think we have another open issue with them, regarding huge pages support, a patch which isn't mainline. Maybe you can use the opportunity to set http://lists.openfabrics.org/pipermail/general/2009-January/056812.html as well... Or. From devesh28 at gmail.com Tue Jul 7 03:58:55 2009 From: devesh28 at gmail.com (Devesh Sharma) Date: Tue, 7 Jul 2009 16:28:55 +0530 Subject: Fwd: [ofa-general] Performance evaluation of Opensm In-Reply-To: <4A53156D.2040805@dev.mellanox.co.il> References: <617255.53094.qm@web94203.mail.in2.yahoo.com> <4A53156D.2040805@dev.mellanox.co.il> Message-ID: <309a667c0907070358n53ad17lafcef8c3ef6e6ef2@mail.gmail.com> Thanks Yevgeny, for your valuable input. This will surly help for my work. On Tue, Jul 7, 2009 at 2:59 PM, Yevgeny Kliteynik wrote: > Hi Davesh, > > It's kind of hard to talk about "performance of OpenSM". > Subnet Manager has different phases and modes of operation, > each of them is completely separate issue: > > - Fabric discovery > - Fabric ports/nodes configuration > - Unicast routing calculation > - Unicast routing configuration on fabric switches > - Multicast routing calculation > - Multicast routing configuration on fabric switches > - SA queries processing > - Memory consumption > - Different routing algorithms consume different time and memory > - QoS > - etc, etc, etc > > Most of the above can be measured only on real cluster. But how these can be measured is there any compile time flag available in the Code? > Some (such as routing calculation and memory consumption) can > be measured while OSM is running on top of the simulator. Simulation results are far far away from real situation..:( I am interested in results with the real fabric. > Some are very affected by the number of CPU cores that you > have on the management node (e.g. SA queries processing), > others mostly affected by the CPU frequency (unicast routing). > Also, various OpenSM options can affect these phases, such as > unicast routing cache may reduce routing calculation time to 0. Hmm........correct. > > Sorry that I'm not really answering your question :( > I just want to point out the fact that there are many aspects > that should be considered when talking about OpenSM performance. Do we have any such tool with does profiling of all these phases of SM. Such tool will be helpful for the researcher working on different algorithms related to SM. > > If what you're interested in is just "system-wide" numbers, > then you'll probably want to know how much time it takes for > the OpenSM to bring up cluster from scratch, or how much time > it takes to reconfigure the fabric after some change. Will it be fine if I run OpenSM with "time" command and press Ctrl-C moment I see SUBNET UP msg. Of-course keeping some of the options and configurations as constant? # time opensm - SUBNET UP Ctrl-C > I think that these numbers can come only from the guys that > administer IB clusters, and they would be highly dependent > on the management node CPU. > > -- Yevgeny > > dhananjay tembe wrote: >> >> Hi Devesh, >> >> OFED installer comes with a tool called as "osmtest" which is a test >> program for opensm and for subnet administrator that comes within the opensm >> itself. The command is /usr/sbin/osmtest. >> >> Another way to test it might be to try it in a large fabric. If you >> do not have a large fabric, try using the simulator that comes with ofed. >> The command for the simulator is /usr/bin/ibsim >> I have not used the simulator yet. So I do not know much about it. >> >> Thanks and regards, >> ---Dhananjay. >> >> >> --- On Tue, 7/7/09, Devesh Sharma wrote: >> >>> From: Devesh Sharma >>> Subject: Fwd: [ofa-general] Performance evaluation of Opensm >>> To: general at lists.openfabrics.org >>> Date: Tuesday, 7 July, 2009, 11:13 AM >>> On Mon, Jul 6, 2009 at 8:33 PM, Peter >>> Kjellstrom >>> wrote: >>>> >>>> On Monday 06 July 2009, Devesh Sharma wrote: >>>>> >>>>> Hello list. >>>>> >>>>> Is there any tool or method available to get some >>> >>> performance numbers >>>>> >>>>> related to Opensm? What are the major performance >>> >>> parameters of OpenSM >>>>> >>>>> which can >>>>> be used for performance measurement analaysis? >>>> >>>> Do you refer to the performance of the fabric as setup >>> >>> by OpenSM or the >>>> >>>> performance of OpenSM itself? >>> >>> Performance of OpenSM itself. There are many tool available >>> to measure >>> performance of Fabric. >>>> >>>> /Peter >>>> >>>>> Thanks in advance >>>>> regards >>>>> Devesh Sharma >>>> >>>> _______________________________________________ >>>> general mailing list >>>> general at lists.openfabrics.org >>>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>>> >>>> To unsubscribe, please visit >>>> http://openib.org/mailman/listinfo/openib-general >>>> >>> _______________________________________________ >>> general mailing list >>> general at lists.openfabrics.org >>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>> >>> To unsubscribe, please visit >>> http://openib.org/mailman/listinfo/openib-general >>> >> >> >> Looking for local information? Find it on Yahoo! Local >> http://in.local.yahoo.com/ >> _______________________________________________ >> general mailing list >> general at lists.openfabrics.org >> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >> >> To unsubscribe, please visit >> http://openib.org/mailman/listinfo/openib-general >> > > From jackm at dev.mellanox.co.il Tue Jul 7 04:14:23 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Tue, 7 Jul 2009 14:14:23 +0300 Subject: [ofa-general] Write combining support in OFED 1.5 In-Reply-To: <4A53294C.8020302@voltaire.com> References: <200907061753.40760.jackm@dev.mellanox.co.il> <200907071338.43055.jackm@dev.mellanox.co.il> <4A53294C.8020302@voltaire.com> Message-ID: <200907071414.24006.jackm@dev.mellanox.co.il> Adding Eli Cohen (author of the huge-page support patch). Eli, what is missing on the PPC regarding huge page support? -Jack On Tuesday 07 July 2009 13:54, Or Gerlitz wrote: > Jack Morgenstein wrote: > > Yes, see kernel_patches/fixes/mlx4_0010_add_wc.patch. With OFED 1.5, I am trying to use the kernel-based WC support, and eliminate the patch. To do this, I need 2 things: > > 1. macro pgprot_writecombine to be defined for PPC/PPC64 under the arch directory (file arch/powerpc/include/asm/pgtable.h) > > 2. A way of determining (via the standard kernel) if write-combining is available on a given platform (to allow disabling use of ConnectX blueflame in userspace if WC is not available). > > > It seems that none of the issues are IB specific, correct? As for PPC > guys I think we have another open issue with them, regarding huge pages > support, a patch which isn't mainline. Maybe you can use the opportunity > to set > http://lists.openfabrics.org/pipermail/general/2009-January/056812.html > as well... > > Or. > > > From devel-ofed at morey-chaisemartin.com Tue Jul 7 04:13:25 2009 From: devel-ofed at morey-chaisemartin.com (Nicolas Morey-Chaisemartin) Date: Tue, 07 Jul 2009 13:13:25 +0200 Subject: Fwd: [ofa-general] Performance evaluation of Opensm In-Reply-To: <309a667c0907070358n53ad17lafcef8c3ef6e6ef2@mail.gmail.com> References: <617255.53094.qm@web94203.mail.in2.yahoo.com> <4A53156D.2040805@dev.mellanox.co.il> <309a667c0907070358n53ad17lafcef8c3ef6e6ef2@mail.gmail.com> Message-ID: <4A532DD5.1000702@morey-chaisemartin.com> Le 07/07/2009 12:58, Devesh Sharma a écrit : > Thanks Yevgeny, for your valuable input. This will surly help for my work. > > On Tue, Jul 7, 2009 at 2:59 PM, Yevgeny > Kliteynik wrote: >> Hi Davesh, >> >> It's kind of hard to talk about "performance of OpenSM". >> Subnet Manager has different phases and modes of operation, >> each of them is completely separate issue: >> >> - Fabric discovery >> - Fabric ports/nodes configuration >> - Unicast routing calculation >> - Unicast routing configuration on fabric switches >> - Multicast routing calculation >> - Multicast routing configuration on fabric switches >> - SA queries processing >> - Memory consumption >> - Different routing algorithms consume different time and memory >> - QoS >> - etc, etc, etc >> >> Most of the above can be measured only on real cluster. > But how these can be measured is there any compile time flag available > in the Code? >> Some (such as routing calculation and memory consumption) can >> be measured while OSM is running on top of the simulator. > Simulation results are far far away from real situation..:( I am > interested in results with the real fabric. Actually it's not. The scanning of the fabric is done before OpenSM calls the routing engine, so the routing engine is working from memory only anyway. So routing calculation time is exactly the same on a real fabric or a simulated one. However, fabric discover time and LFT update time will differ I agree. >> Some are very affected by the number of CPU cores that you >> have on the management node (e.g. SA queries processing), >> others mostly affected by the CPU frequency (unicast routing). >> Also, various OpenSM options can affect these phases, such as >> unicast routing cache may reduce routing calculation time to 0. > Hmm........correct. >> Sorry that I'm not really answering your question :( >> I just want to point out the fact that there are many aspects >> that should be considered when talking about OpenSM performance. > Do we have any such tool with does profiling of all these phases of > SM. Such tool will be > helpful for the researcher working on different algorithms related to SM. For internal actions, you can use valgrind --tool=callgrind It provides a full analysis of any program so you can find where bottlenecks are and pretty much any perf info you may need. However it does not allow to mesure times for network operations. >> If what you're interested in is just "system-wide" numbers, >> then you'll probably want to know how much time it takes for >> the OpenSM to bring up cluster from scratch, or how much time >> it takes to reconfigure the fabric after some change. > Will it be fine if I run OpenSM with "time" command and press Ctrl-C > moment I see > SUBNET UP msg. Of-course keeping some of the options and > configurations as constant? > > # time opensm - > SUBNET UP Ctrl-C > It should work. Problem is you won't have much granularity to know where the time is consumed. Plus Ctr-C doesn't kill OpenSM right away. If there are a lot of outstanding MAD, it can take few seconds before leaving. Nicolas From kliteyn at dev.mellanox.co.il Tue Jul 7 04:57:57 2009 From: kliteyn at dev.mellanox.co.il (Yevgeny Kliteynik) Date: Tue, 07 Jul 2009 14:57:57 +0300 Subject: Fwd: [ofa-general] Performance evaluation of Opensm In-Reply-To: <309a667c0907070358n53ad17lafcef8c3ef6e6ef2@mail.gmail.com> References: <617255.53094.qm@web94203.mail.in2.yahoo.com> <4A53156D.2040805@dev.mellanox.co.il> <309a667c0907070358n53ad17lafcef8c3ef6e6ef2@mail.gmail.com> Message-ID: <4A533845.4040101@dev.mellanox.co.il> Devesh Sharma wrote: > Thanks Yevgeny, for your valuable input. This will surly help for my work. > > On Tue, Jul 7, 2009 at 2:59 PM, Yevgeny > Kliteynik wrote: >> Hi Davesh, >> >> It's kind of hard to talk about "performance of OpenSM". >> Subnet Manager has different phases and modes of operation, >> each of them is completely separate issue: >> >> - Fabric discovery >> - Fabric ports/nodes configuration >> - Unicast routing calculation >> - Unicast routing configuration on fabric switches >> - Multicast routing calculation >> - Multicast routing configuration on fabric switches >> - SA queries processing >> - Memory consumption >> - Different routing algorithms consume different time and memory >> - QoS >> - etc, etc, etc >> >> Most of the above can be measured only on real cluster. > > But how these can be measured is there any compile time flag available > in the Code? You can run SM with higher verbosity, and then you will see more log messages in the SM log file (run 'opensm -h', or 'man opensm' to see its usage). Each log message has time stamp. Running 'opensm -D 7' will allow you to see SM state transitions. You can bump up the verbosity even more to see more details. However, note that high verbosity adds to the OSM runtime, so the best way to measure certain SM operations would be an old fashion way - add messages into the SM code and run SM with the usual verbosity. >> Some (such as routing calculation and memory consumption) can >> be measured while OSM is running on top of the simulator. > > Simulation results are far far away from real situation..:( I am > interested in results with the real fabric. Agree. >> Some are very affected by the number of CPU cores that you >> have on the management node (e.g. SA queries processing), >> others mostly affected by the CPU frequency (unicast routing). >> Also, various OpenSM options can affect these phases, such as >> unicast routing cache may reduce routing calculation time to 0. > Hmm........correct. >> Sorry that I'm not really answering your question :( >> I just want to point out the fact that there are many aspects >> that should be considered when talking about OpenSM performance. > > Do we have any such tool with does profiling of all these phases of > SM. Such tool will be > helpful for the researcher working on different algorithms related to SM. No tool (at least not that I'm aware of). Just examining the log, and/or adding log messages/printf's to the OpenSM code. >> If what you're interested in is just "system-wide" numbers, >> then you'll probably want to know how much time it takes for >> the OpenSM to bring up cluster from scratch, or how much time >> it takes to reconfigure the fabric after some change. > > Will it be fine if I run OpenSM with "time" command and press Ctrl-C > moment I see > SUBNET UP msg. Of-course keeping some of the options and > configurations as constant? > > # time opensm - > SUBNET UP Ctrl-C Sure, it's possible. You can run it as follows: 'time opensm -o' It will cause SM to run once and exit, so no need for ^C. It will add some overhead of cleaning up, but it's a small overhead. You can also run 'opensm -e -d2 &', and then examine OSM log - look for the "SUBNET UP" message in the log. This way you will get rid of the clean-up overhead. Check OpenSM usage - it might give you some more ideas. -- Yevgeny >> I think that these numbers can come only from the guys that >> administer IB clusters, and they would be highly dependent >> on the management node CPU. >> >> -- Yevgeny >> >> dhananjay tembe wrote: >>> Hi Devesh, >>> >>> OFED installer comes with a tool called as "osmtest" which is a test >>> program for opensm and for subnet administrator that comes within the opensm >>> itself. The command is /usr/sbin/osmtest. >>> >>> Another way to test it might be to try it in a large fabric. If you >>> do not have a large fabric, try using the simulator that comes with ofed. >>> The command for the simulator is /usr/bin/ibsim >>> I have not used the simulator yet. So I do not know much about it. >>> >>> Thanks and regards, >>> ---Dhananjay. >>> >>> >>> --- On Tue, 7/7/09, Devesh Sharma wrote: >>> >>>> From: Devesh Sharma >>>> Subject: Fwd: [ofa-general] Performance evaluation of Opensm >>>> To: general at lists.openfabrics.org >>>> Date: Tuesday, 7 July, 2009, 11:13 AM >>>> On Mon, Jul 6, 2009 at 8:33 PM, Peter >>>> Kjellstrom >>>> wrote: >>>>> On Monday 06 July 2009, Devesh Sharma wrote: >>>>>> Hello list. >>>>>> >>>>>> Is there any tool or method available to get some >>>> performance numbers >>>>>> related to Opensm? What are the major performance >>>> parameters of OpenSM >>>>>> which can >>>>>> be used for performance measurement analaysis? >>>>> Do you refer to the performance of the fabric as setup >>>> by OpenSM or the >>>>> performance of OpenSM itself? >>>> Performance of OpenSM itself. There are many tool available >>>> to measure >>>> performance of Fabric. >>>>> /Peter >>>>> >>>>>> Thanks in advance >>>>>> regards >>>>>> Devesh Sharma >>>>> _______________________________________________ >>>>> general mailing list >>>>> general at lists.openfabrics.org >>>>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>>>> >>>>> To unsubscribe, please visit >>>>> http://openib.org/mailman/listinfo/openib-general >>>>> >>>> _______________________________________________ >>>> general mailing list >>>> general at lists.openfabrics.org >>>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>>> >>>> To unsubscribe, please visit >>>> http://openib.org/mailman/listinfo/openib-general >>>> >>> >>> Looking for local information? Find it on Yahoo! Local >>> http://in.local.yahoo.com/ >>> _______________________________________________ >>> general mailing list >>> general at lists.openfabrics.org >>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>> >>> To unsubscribe, please visit >>> http://openib.org/mailman/listinfo/openib-general >>> >> > From chien.tin.tung at intel.com Tue Jul 7 05:07:20 2009 From: chien.tin.tung at intel.com (Tung, Chien Tin) Date: Tue, 7 Jul 2009 05:07:20 -0700 Subject: [ofa-general] Loopback connection not working on Chelsio In-Reply-To: <4A53040A.4020601@voltaire.com> References: <96f8e60e0907062036p4e4d8850yd700c90f0bf70069@mail.gmail.com> <4A53040A.4020601@voltaire.com> Message-ID: <60BEFF3FBD4C6047B0F13F205CAFA383035C15D361@azsmsx501.amr.corp.intel.com> >pandit ib wrote: >> I'm having trouble getting loopback rdma connections working >on Chelsio. We were told that loopback connections are not >supported. Is this an iWarp or Chelsio specific issue? What >are the issues that is preventing it from working? >I don't think its an iwarp limitation. > >Or. Intel/NetEffect NE020 supports loopback connections. It is not an iwarp limitation. Chien From eli at mellanox.co.il Tue Jul 7 05:24:25 2009 From: eli at mellanox.co.il (Eli Cohen) Date: Tue, 7 Jul 2009 15:24:25 +0300 Subject: [ofa-general] Write combining support in OFED 1.5 In-Reply-To: <200907071414.24006.jackm@dev.mellanox.co.il> References: <200907061753.40760.jackm@dev.mellanox.co.il> <200907071338.43055.jackm@dev.mellanox.co.il> <4A53294C.8020302@voltaire.com> <200907071414.24006.jackm@dev.mellanox.co.il> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD034721A4@mtlexch01.mtl.com> In PPC arch HPAGE_SHIFT is an unexported, variable (because huge page sizes are different there). The huge pages patch needs to reference this value (through HPAGE_SIZE). -----Original Message----- From: Jack Morgenstein [mailto:jackm at dev.mellanox.co.il] Sent: Tuesday, July 07, 2009 2:14 PM To: Or Gerlitz; Eli Cohen Cc: rdreier at cisco.com; general at lists.openfabrics.org Subject: Re: [ofa-general] Write combining support in OFED 1.5 Adding Eli Cohen (author of the huge-page support patch). Eli, what is missing on the PPC regarding huge page support? -Jack On Tuesday 07 July 2009 13:54, Or Gerlitz wrote: > Jack Morgenstein wrote: > > Yes, see kernel_patches/fixes/mlx4_0010_add_wc.patch. With OFED 1.5, I am trying to use the kernel-based WC support, and eliminate the patch. To do this, I need 2 things: > > 1. macro pgprot_writecombine to be defined for PPC/PPC64 under the > > arch directory (file arch/powerpc/include/asm/pgtable.h) > > 2. A way of determining (via the standard kernel) if write-combining is available on a given platform (to allow disabling use of ConnectX blueflame in userspace if WC is not available). > > > It seems that none of the issues are IB specific, correct? As for PPC > guys I think we have another open issue with them, regarding huge > pages support, a patch which isn't mainline. Maybe you can use the > opportunity to set > http://lists.openfabrics.org/pipermail/general/2009-January/056812.htm > l > as well... > > Or. > > > From lars.ellenberg at linbit.com Tue Jul 7 05:31:53 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Tue, 7 Jul 2009 14:31:53 +0200 Subject: [ofa-general] upstream stable merge request for OFED 1.4 In-Reply-To: References: <20090703144824.GA24820@barkeeper1-xen.linbit> Message-ID: <20090707123153.GD8065@barkeeper1-xen.linbit> On Fri, Jul 03, 2009 at 07:34:32PM +0200, Bart Van Assche wrote: > On Fri, Jul 3, 2009 at 4:48 PM, Lars Ellenberg wrote: > > maybe I'm doing things wrong. > > but the last merge with upstream for > > git://git.openfabrics.org/ofed_1_4/linux-2.6.git > > appears to have been 2.6.27. > > > > there is a reason for the kernel.org stable branches. > > > > if you do not plan to upgrade 2.6.28 -> 2.6.31 in the near future, > > please consider to merge with > > git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-2.6.27.y.git > > > > current upstream 2.6.27 stable is v2.6.27.26 > >  958 files changed, 13481 insertions(+), 6784 deletions(-) > > Hello Lars, > > As you probably know any patch submitted for inclusion in a kernel.org > stable branch must have been accepted first for inclusion in the > latest vanilla kernel. So the above question is equivalent to whether > the development of the OFED kernel components should continue as a > separate distribution or whether all patches should be submitted for > inclusion in the latest Linux kernel. This topic has been discussed > several times on this mailing list. You can find the most recent > discussion of this topic here: > http://lists.openfabrics.org/pipermail/general/2009-April/059080.html. Thanks for that link. But what I actually was referring to was the tree exposed at git://git.openfabrics.org/ofed_1_4/linux-2.6.git and why that one does neither track kernel.org upstream, nor the 2.6.27.y stable series, but is stuck on the known buggy 2.6.27. of course I can easily merge in the stable 2.6.27.26 in my local checkout of ofed_1_4. I already did. no problem there. but I was wondering about the status of the git://git.openfabrics.org/ofed_1_4/linux-2.6.git tree, whether that is supposed to be the "most uptodate official ofed_1_4" kernel, and whether or not it is going to be updated to either track some stable series, or linux-2.6 master. Thanks for any comments. -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. From bart.vanassche at gmail.com Tue Jul 7 05:58:56 2009 From: bart.vanassche at gmail.com (Bart Van Assche) Date: Tue, 7 Jul 2009 14:58:56 +0200 Subject: [ofa-general] upstream stable merge request for OFED 1.4 In-Reply-To: <20090707123153.GD8065@barkeeper1-xen.linbit> References: <20090703144824.GA24820@barkeeper1-xen.linbit> <20090707123153.GD8065@barkeeper1-xen.linbit> Message-ID: On Tue, Jul 7, 2009 at 2:31 PM, Lars Ellenberg wrote: > On Fri, Jul 03, 2009 at 07:34:32PM +0200, Bart Van Assche wrote: >> On Fri, Jul 3, 2009 at 4:48 PM, Lars Ellenberg wrote: >> > maybe I'm doing things wrong. >> > but the last merge with upstream for >> > git://git.openfabrics.org/ofed_1_4/linux-2.6.git >> > appears to have been 2.6.27. >> > >> > there is a reason for the kernel.org stable branches. >> > >> > if you do not plan to upgrade 2.6.28 -> 2.6.31 in the near future, >> > please consider to merge with >> > git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-2.6.27.y.git >> > >> > current upstream 2.6.27 stable is v2.6.27.26 >> >  958 files changed, 13481 insertions(+), 6784 deletions(-) >> >> Hello Lars, >> >> As you probably know any patch submitted for inclusion in a kernel.org >> stable branch must have been accepted first for inclusion in the >> latest vanilla kernel. So the above question is equivalent to whether >> the development of the OFED kernel components should continue as a >> separate distribution or whether all patches should be submitted for >> inclusion in the latest Linux kernel. This topic has been discussed >> several times on this mailing list. You can find the most recent >> discussion of this topic here: >> http://lists.openfabrics.org/pipermail/general/2009-April/059080.html. > > Thanks for that link. > > But what I actually was referring to was the tree exposed at > git://git.openfabrics.org/ofed_1_4/linux-2.6.git > > and why that one does neither track kernel.org upstream, > nor the 2.6.27.y stable series, > but is stuck on the known buggy 2.6.27. > > of course I can easily merge in the stable 2.6.27.26 in my > local checkout of ofed_1_4. I already did. no problem there. > > but I was wondering about the status of the > git://git.openfabrics.org/ofed_1_4/linux-2.6.git tree, > whether that is supposed to be the "most uptodate official ofed_1_4" > kernel, and whether or not it is going to be updated to either track > some stable series, or linux-2.6 master. Is anyone running a kernel compiled from git://git.openfabrics.org/ofed_1_4/linux-2.6.git ? AFAIK all InfiniBand users install the OFED-*.tgz distribution on top of an existing Linux distribution (RHEL, CentOS, Ubuntu, or vanilla 2.6.27.x kernel). It would be interesting to know which of the 2.6.27-stable patches affect any of the InfiniBand source files or kernel headers that are distributed through OFED, and whether any of these patches are not already included in the OFED distribution. Bart. From devesh28 at gmail.com Tue Jul 7 06:40:44 2009 From: devesh28 at gmail.com (Devesh Sharma) Date: Tue, 7 Jul 2009 19:10:44 +0530 Subject: Fwd: [ofa-general] Performance evaluation of Opensm In-Reply-To: <4A533845.4040101@dev.mellanox.co.il> References: <617255.53094.qm@web94203.mail.in2.yahoo.com> <4A53156D.2040805@dev.mellanox.co.il> <309a667c0907070358n53ad17lafcef8c3ef6e6ef2@mail.gmail.com> <4A533845.4040101@dev.mellanox.co.il> Message-ID: <309a667c0907070640h4ecc4817q683a4466c619d7b8@mail.gmail.com> On Tue, Jul 7, 2009 at 5:27 PM, Yevgeny Kliteynik wrote: > Devesh Sharma wrote: >> >> Thanks Yevgeny, for your valuable input. This will surly help for my work. >> >> On Tue, Jul 7, 2009 at 2:59 PM, Yevgeny >> Kliteynik wrote: >>> >>> Hi Davesh, >>> >>> It's kind of hard to talk about "performance of OpenSM". >>> Subnet Manager has different phases and modes of operation, >>> each of them is completely separate issue: >>> >>> - Fabric discovery >>> - Fabric ports/nodes configuration >>> - Unicast routing calculation >>> - Unicast routing configuration on fabric switches >>> - Multicast routing calculation >>> - Multicast routing configuration on fabric switches >>> - SA queries processing >>> - Memory consumption >>> - Different routing algorithms consume different time and memory >>> - QoS >>> - etc, etc, etc >>> >>> Most of the above can be measured only on real cluster. >> >> But how these can be measured is there any compile time flag available >> in the Code? > > You can run SM with higher verbosity, and then you will see more > log messages in the SM log file (run 'opensm -h', or 'man opensm' > to see its usage). Each log message has time stamp. > Running 'opensm -D 7' will allow you to see SM state transitions. I have used it with -V but its quite tedious to traverse through the log file..:( > You can bump up the verbosity even more to see more details. > However, note that high verbosity adds to the OSM runtime, so > the best way to measure certain SM operations would be an old fashion way - > add messages into the SM code and run SM with the > usual verbosity. Yes this is correct. I will try this also. > >>> Some (such as routing calculation and memory consumption) can >>> be measured while OSM is running on top of the simulator. >> >> Simulation results are far far away from real situation..:( I am >> interested in results with the real fabric. > > Agree. > >>> Some are very affected by the number of CPU cores that you >>> have on the management node (e.g. SA queries processing), >>> others mostly affected by the CPU frequency (unicast routing). >>> Also, various OpenSM options can affect these phases, such as >>> unicast routing cache may reduce routing calculation time to 0. >> >> Hmm........correct. >>> >>> Sorry that I'm not really answering your question :( >>> I just want to point out the fact that there are many aspects >>> that should be considered when talking about OpenSM performance. >> >> Do we have any such tool with does profiling of all these phases of >> SM. Such tool will be >> helpful for the researcher working on different algorithms related to SM. > > No tool (at least not that I'm aware of). > Just examining the log, and/or adding log messages/printf's to > the OpenSM code. ok...the old trusted method....:) > >>> If what you're interested in is just "system-wide" numbers, >>> then you'll probably want to know how much time it takes for >>> the OpenSM to bring up cluster from scratch, or how much time >>> it takes to reconfigure the fabric after some change. >> >> Will it be fine if I run OpenSM with "time" command and press Ctrl-C >> moment I see >> SUBNET UP msg. Of-course keeping some of the options and >> configurations as constant? >> >> # time opensm - >> SUBNET UP Ctrl-C > > Sure, it's possible. > You can run it as follows: 'time opensm -o' > It will cause SM to run once and exit, so no need for ^C. > It will add some overhead of cleaning up, but it's a small overhead. > You can also run 'opensm -e -d2 &', and then examine OSM log - look for the > "SUBNET UP" message in the log. This way you will > get rid of the clean-up overhead. > I think these options will give what I want. Thanks for your valuable inputs once again. > Check OpenSM usage - it might give you some more ideas. > > -- Yevgeny > >>> I think that these numbers can come only from the guys that >>> administer IB clusters, and they would be highly dependent >>> on the management node CPU. >>> >>> -- Yevgeny >>> >>> dhananjay tembe wrote: >>>> >>>> Hi Devesh, >>>> >>>> OFED installer comes with a tool called as "osmtest" which is a >>>> test >>>> program for opensm and for subnet administrator that comes within the >>>> opensm >>>> itself. The command is /usr/sbin/osmtest. >>>> >>>> Another way to test it might be to try it in a large fabric. If you >>>> do not have a large fabric, try using the simulator that comes with >>>> ofed. >>>> The command for the simulator is /usr/bin/ibsim >>>> I have not used the simulator yet. So I do not know much about it. >>>> >>>> Thanks and regards, >>>> ---Dhananjay. >>>> >>>> >>>> --- On Tue, 7/7/09, Devesh Sharma wrote: >>>> >>>>> From: Devesh Sharma >>>>> Subject: Fwd: [ofa-general] Performance evaluation of Opensm >>>>> To: general at lists.openfabrics.org >>>>> Date: Tuesday, 7 July, 2009, 11:13 AM >>>>> On Mon, Jul 6, 2009 at 8:33 PM, Peter >>>>> Kjellstrom >>>>> wrote: >>>>>> >>>>>> On Monday 06 July 2009, Devesh Sharma wrote: >>>>>>> >>>>>>> Hello list. >>>>>>> >>>>>>> Is there any tool or method available to get some >>>>> >>>>> performance numbers >>>>>>> >>>>>>> related to Opensm? What are the major performance >>>>> >>>>> parameters of OpenSM >>>>>>> >>>>>>> which can >>>>>>> be used for performance measurement analaysis? >>>>>> >>>>>> Do you refer to the performance of the fabric as setup >>>>> >>>>> by OpenSM or the >>>>>> >>>>>> performance of OpenSM itself? >>>>> >>>>> Performance of OpenSM itself. There are many tool available >>>>> to measure >>>>> performance of Fabric. >>>>>> >>>>>> /Peter >>>>>> >>>>>>> Thanks in advance >>>>>>> regards >>>>>>> Devesh Sharma >>>>>> >>>>>> _______________________________________________ >>>>>> general mailing list >>>>>> general at lists.openfabrics.org >>>>>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>>>>> >>>>>> To unsubscribe, please visit >>>>>> http://openib.org/mailman/listinfo/openib-general >>>>>> >>>>> _______________________________________________ >>>>> general mailing list >>>>> general at lists.openfabrics.org >>>>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>>>> >>>>> To unsubscribe, please visit >>>>> http://openib.org/mailman/listinfo/openib-general >>>>> >>>> >>>> Looking for local information? Find it on Yahoo! Local >>>> http://in.local.yahoo.com/ >>>> _______________________________________________ >>>> general mailing list >>>> general at lists.openfabrics.org >>>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>>> >>>> To unsubscribe, please visit >>>> http://openib.org/mailman/listinfo/openib-general >>>> >>> >> > > From devesh28 at gmail.com Tue Jul 7 06:43:41 2009 From: devesh28 at gmail.com (Devesh Sharma) Date: Tue, 7 Jul 2009 19:13:41 +0530 Subject: Fwd: [ofa-general] Performance evaluation of Opensm In-Reply-To: <4A532DD5.1000702@morey-chaisemartin.com> References: <617255.53094.qm@web94203.mail.in2.yahoo.com> <4A53156D.2040805@dev.mellanox.co.il> <309a667c0907070358n53ad17lafcef8c3ef6e6ef2@mail.gmail.com> <4A532DD5.1000702@morey-chaisemartin.com> Message-ID: <309a667c0907070643n5d824f25wcc74077266122eb6@mail.gmail.com> Thanks Nicolas for your response. On Tue, Jul 7, 2009 at 4:43 PM, Nicolas Morey-Chaisemartin wrote: > Le 07/07/2009 12:58, Devesh Sharma a écrit : >> Thanks Yevgeny, for your valuable input. This will surly help for my work. >> >> On Tue, Jul 7, 2009 at 2:59 PM, Yevgeny >> Kliteynik wrote: >>> Hi Davesh, >>> >>> It's kind of hard to talk about "performance of OpenSM". >>> Subnet Manager has different phases and modes of operation, >>> each of them is completely separate issue: >>> >>> - Fabric discovery >>> - Fabric ports/nodes configuration >>> - Unicast routing calculation >>> - Unicast routing configuration on fabric switches >>> - Multicast routing calculation >>> - Multicast routing configuration on fabric switches >>> - SA queries processing >>> - Memory consumption >>> - Different routing algorithms consume different time and memory >>> - QoS >>> - etc, etc, etc >>> >>> Most of the above can be measured only on real cluster. >> But how these can be measured is there any compile time flag available >> in the Code? >>> Some (such as routing calculation and memory consumption) can >>> be measured while OSM is running on top of the simulator. >> Simulation results are far far away from real situation..:( I am >> interested in results with the real fabric. > > Actually it's not. The scanning of the fabric is done before OpenSM calls the routing engine, so the routing engine is working from memory only anyway. So routing calculation time is exactly the same on a real fabric or a simulated one. Hmm....ok. > However, fabric discover time and LFT update time will differ I agree. >>> Some are very affected by the number of CPU cores that you >>> have on the management node (e.g. SA queries processing), >>> others mostly affected by the CPU frequency (unicast routing). >>> Also, various OpenSM options can affect these phases, such as >>> unicast routing cache may reduce routing calculation time to 0. >> Hmm........correct. >>> Sorry that I'm not really answering your question :( >>> I just want to point out the fact that there are many aspects >>> that should be considered when talking about OpenSM performance. >> Do we have any such tool with does profiling of all these phases of >> SM. Such tool will be >> helpful for the researcher working on different algorithms related to SM. > > For internal actions, you can use valgrind --tool=callgrind > It provides a full analysis of any program so you can find where bottlenecks are and pretty much any perf info you may need. However it does not allow to mesure times for network operations. > ok....I will try this. >>> If what you're interested in is just "system-wide" numbers, >>> then you'll probably want to know how much time it takes for >>> the OpenSM to bring up cluster from scratch, or how much time >>> it takes to reconfigure the fabric after some change. >> Will it be fine if I run OpenSM with "time" command and press Ctrl-C >> moment I see >> SUBNET UP msg. Of-course keeping some of the options and >> configurations as constant? >> >> # time opensm - >> SUBNET UP Ctrl-C >> > It should work. Problem is you won't have much granularity to know where the time is consumed. Plus Ctr-C doesn't kill OpenSM right away. If there are a lot of outstanding MAD, it can take few seconds before leaving. > Yevgeny has suggested a better way to do this. Will try that. > Nicolas > > From swise at opengridcomputing.com Tue Jul 7 06:49:05 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Tue, 07 Jul 2009 08:49:05 -0500 Subject: [ofa-general] Loopback connection not working on Chelsio In-Reply-To: <96f8e60e0907062036p4e4d8850yd700c90f0bf70069@mail.gmail.com> References: <96f8e60e0907062036p4e4d8850yd700c90f0bf70069@mail.gmail.com> Message-ID: <4A535251.9020700@opengridcomputing.com> pandit ib wrote: > Hi, > > I'm having trouble getting loopback rdma connections working on Chelsio. > > We were told that loopback connections are not supported. > > Is this an iWarp or Chelsio specific issue? > > Chelsio specific. I don't know if the NES/Intel rnic supports it. > What are the issues that is preventing it from working? > > Unfortunately, the Chelsio HW doesn't support RDMA loopback in HW. And there is no software rdma loopback driver in Linux. > Are there plans for supporting this capability? > > Although kinda painful, a SW loopback module could be developed. Anyone interested in doing this? Steve. From swise at opengridcomputing.com Tue Jul 7 06:52:45 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Tue, 07 Jul 2009 08:52:45 -0500 Subject: [ofa-general] 4GB physical memory limit on Chelsio In-Reply-To: <96f8e60e0907061932n19a8f91amb253a28a039b4d90@mail.gmail.com> References: <96f8e60e0907061932n19a8f91amb253a28a039b4d90@mail.gmail.com> Message-ID: <4A53532D.8090207@opengridcomputing.com> pandit ib wrote: > In iwch_provider.c, line 718, iwch_get_dma_mr() there is a comment > which seems to indicate that the Chelsio card supports registering > only 4GB of physical memory. > Is that correct? > > Most of our systems have way greater memory. In fact, the HPC7000 > blades we are testing on has 64GB of memory. > > The problem we are running into is that our driver calls > ib_get_dma_mr() to register all physical memory. > When the driver attempts to post recv buffers it's getting -EINVAL error. > > > I traced it down to line 253 in iwch_qp.c. > > if (sg_list[i].addr + ((u64) sg_list[i].length) > > mhp->attr.va_fbo + ((u64) mhp->attr.len)) { > PDBG("%s %d\n", __func__, __LINE__); > return -EINVAL; > } > > > when i printed out those values I got the following: > > Jul 6 19:13:17 lv2 kernel: sg_list[i].addr faca80000, > sg_list[i].length 7360 mhp->attr.va_fbo 0 mhp->attr.len 4294967295 > > The length of the registration indeed seems to be 4GB in size. > > Is there a way to make it work on systems with > 4GB of physical > memory. Is there a configuration parameter? > > Thanks, > Ranjit > Hey Ranjit, Sorry, this is limitation in the current Chelsio HW (T3). You cannot use get_dma_mr to register memory above 4GB. It just doesn't work for T3. In fact, get_dma_mr just maps dma addresses 0..4GB-1. So if your system is using a iommu and mapping bus addresses above 4GB, it won't work at all. Chelsio's next part will overcome this limitation. In the meantime, you have to use reg_phys_mr or the fast_register work request to register specific regions on demand. You can see how this is done by investigating the NFS RDMA transport code. Steve. > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From swise at opengridcomputing.com Tue Jul 7 06:53:57 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Tue, 07 Jul 2009 08:53:57 -0500 Subject: [ofa-general] 4GB physical memory limit on Chelsio In-Reply-To: <4A53066C.4070105@Voltaire.com> References: <96f8e60e0907061932n19a8f91amb253a28a039b4d90@mail.gmail.com> <4A53066C.4070105@Voltaire.com> Message-ID: <4A535375.9000302@opengridcomputing.com> Or Gerlitz wrote: > pandit ib wrote: > >> Is there a way to make it work on systems with > 4GB of physical memory. >> > > I think you want to look on zero STag, e.g see commit 96f15c0.... "RDMA/core: Add local DMA L_Key support" and bd7ed1... "RPC/RDMA: check selected memory registration mode at runtime" > > Yes, Zero stag aka the local dma lmr can be used, but only for local operations. You cannot use the local dma lkey for the source or sink of an rdma read... Steve. From hnrose at comcast.net Tue Jul 7 07:00:07 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Tue, 7 Jul 2009 10:00:07 -0400 Subject: [ofa-general] [PATCH] opensm/osm_mcast_tbl.c: In osm_mcast_tbl_init, use calloc Message-ID: <20090707140007.GA8954@comcast.net> rather than malloc/memset Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_mcast_tbl.c b/opensm/opensm/osm_mcast_tbl.c index 17fb69c..82850be 100644 --- a/opensm/opensm/osm_mcast_tbl.c +++ b/opensm/opensm/osm_mcast_tbl.c @@ -2,6 +2,7 @@ * Copyright (c) 2004-2006 Voltaire, Inc. All rights reserved. * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -93,16 +94,13 @@ osm_mcast_tbl_init(IN osm_mcast_tbl_t * const p_tbl, since it is (and must be) defined that way the table structure in order to create a pointer to a two dimensional array. */ - p_tbl->p_mask_tbl = malloc(p_tbl->num_entries * + p_tbl->p_mask_tbl = calloc(p_tbl->num_entries, (IB_MCAST_POSITION_MAX + 1) * IB_MCAST_MASK_SIZE / 8); if (p_tbl->p_mask_tbl == NULL) return (IB_INSUFFICIENT_MEMORY); - memset(p_tbl->p_mask_tbl, 0, - p_tbl->num_entries * (IB_MCAST_POSITION_MAX + - 1) * IB_MCAST_MASK_SIZE / 8); return (IB_SUCCESS); } From alexs at linux.vnet.ibm.com Tue Jul 7 07:06:39 2009 From: alexs at linux.vnet.ibm.com (Alexander Schmidt) Date: Tue, 7 Jul 2009 16:06:39 +0200 Subject: [ofa-general] [PATCH] ehca: use port autodetect mode as default Message-ID: <20090707160639.6b45bf4d@BL3D1974.boeblingen.de.ibm.com> This patch sets the port autodetect mode as default for the ehca driver. The autodetect code has been in the kernel for several releases now and has proved to be stable. --- Roland, please queue this change for 2.6.32 if you are okay with it. drivers/infiniband/hw/ehca/ehca_main.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) --- infiniband.git.orig/drivers/infiniband/hw/ehca/ehca_main.c +++ infiniband.git/drivers/infiniband/hw/ehca/ehca_main.c @@ -52,7 +52,7 @@ #include "ehca_tools.h" #include "hcp_if.h" -#define HCAD_VERSION "0028" +#define HCAD_VERSION "0029" MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Christoph Raisch "); @@ -64,7 +64,7 @@ static int ehca_hw_level = 0; static int ehca_poll_all_eqs = 1; int ehca_debug_level = 0; -int ehca_nr_ports = 2; +int ehca_nr_ports = -1; int ehca_use_hp_mr = 0; int ehca_port_act_time = 30; int ehca_static_rate = -1; @@ -95,8 +95,8 @@ MODULE_PARM_DESC(hw_level, "Hardware level (0: autosensing (default), " "0x10..0x14: eHCA, 0x20..0x23: eHCA2)"); MODULE_PARM_DESC(nr_ports, - "number of connected ports (-1: autodetect, 1: port one only, " - "2: two ports (default)"); + "number of connected ports (-1: autodetect (default), " + "1: port one only, 2: two ports)"); MODULE_PARM_DESC(use_hp_mr, "Use high performance MRs (default: no)"); MODULE_PARM_DESC(port_act_time, From Line.Holen at Sun.COM Tue Jul 7 07:07:52 2009 From: Line.Holen at Sun.COM (Line.Holen at Sun.COM) Date: Tue, 07 Jul 2009 16:07:52 +0200 Subject: [ofa-general] [PATCH] opensm: Add new Sun vendor ID Message-ID: <4A5356B8.1040803@Sun.COM> Signed-off-by: Line Holen --- diff --git a/opensm/include/opensm/osm_base.h b/opensm/include/opensm/osm_base.h index 03eed79..0537002 100644 --- a/opensm/include/opensm/osm_base.h +++ b/opensm/include/opensm/osm_base.h @@ -2,6 +2,7 @@ * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. * Copyright (c) 2002-2006 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. + * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -864,6 +865,7 @@ typedef enum _osm_sm_signal { #define OSM_VENDOR_ID_HP 0x001708 #define OSM_VENDOR_ID_RIOWORKS 0x005045 #define OSM_VENDOR_ID_SUN 0x0003BA +#define OSM_VENDOR_ID_SUN2 0x002128 #define OSM_VENDOR_ID_3LEAFNTWKS 0x0016A1 #define OSM_VENDOR_ID_XSIGO 0x001397 #define OSM_VENDOR_ID_HP2 0x0018FE diff --git a/opensm/opensm/osm_helper.c b/opensm/opensm/osm_helper.c index 0123edc..f3cffa8 100644 --- a/opensm/opensm/osm_helper.c +++ b/opensm/opensm/osm_helper.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009 HNR Consulting. All rights reserved. + * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -2272,6 +2273,7 @@ const char *osm_get_manufacturer_str(IN uint64_t const guid_ho) case OSM_VENDOR_ID_RIOWORKS: return (rioworks_str); case OSM_VENDOR_ID_SUN: + case OSM_VENDOR_ID_SUN2: return (sun_str); case OSM_VENDOR_ID_3LEAFNTWKS: return (leafntwks_str); From fenkes at de.ibm.com Tue Jul 7 07:20:19 2009 From: fenkes at de.ibm.com (Joachim Fenkes) Date: Tue, 7 Jul 2009 16:20:19 +0200 Subject: [ofa-general] [PATCH v4] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <20090701200043.GF20745@obsidianresearch.com> Message-ID: <200907071620.23003.fenkes@de.ibm.com> Previously, libibmad reacted to GSI MAD responses with a "redirect" status by throwing an error. IBM eHCA adapters use redirection, so most infiniband_diags tools didn't work against eHCA. Fix: Modify mad_rpc() so that it resends the request to the redirection target if a "redirect" GS response is received. This is repeated until no "redirect" response is received, allowing for multiple levels of indirection. The dport argument is updated with the redirection target, so subsequent MADs will not go through the redirection process again but reach the target directly. Tested using perfquery between ehca, mlx4 and mthca in all possible combinations. Signed-off-by: Joachim Fenkes --- After all has been said and done, here's the hopefully last iteration of the patch, with the hex display of the redirect LID replaced by decimal. Any objections against this patch? Regards, Joachim libibmad/include/infiniband/mad.h | 9 +++++ libibmad/src/gs.c | 6 ++- libibmad/src/rpc.c | 65 ++++++++++++++++++++++++++++-------- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/libibmad/include/infiniband/mad.h b/libibmad/include/infiniband/mad.h index aa27eb5..bdf5158 100644 --- a/libibmad/include/infiniband/mad.h +++ b/libibmad/include/infiniband/mad.h @@ -115,6 +115,8 @@ enum MAD_ATTR_ID { enum MAD_STATUS { IB_MAD_STS_OK = (0 << 2), + IB_MAD_STS_BUSY = (1 << 0), + IB_MAD_STS_REDIRECT = (1 << 1), IB_MAD_STS_BAD_BASE_VER_OR_CLASS = (1 << 2), IB_MAD_STS_METHOD_NOT_SUPPORTED = (2 << 2), IB_MAD_STS_METHOD_ATTR_NOT_SUPPORTED = (3 << 2), @@ -783,8 +785,15 @@ MAD_EXPORT int madrpc_set_timeout(int timeout); MAD_EXPORT struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port, int *mgmt_classes, int num_classes); MAD_EXPORT void mad_rpc_close_port(struct ibmad_port *srcport); + +/* + * On redirection, the dport argument is updated with the redirection target, + * so subsequent MADs will not go through the redirection process again but + * reach the target directly. + */ MAD_EXPORT void *mad_rpc(const struct ibmad_port *srcport, ib_rpc_t * rpc, ib_portid_t * dport, void *payload, void *rcvdata); + MAD_EXPORT void *mad_rpc_rmpp(const struct ibmad_port *srcport, ib_rpc_t * rpc, ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, void *data); diff --git a/libibmad/src/gs.c b/libibmad/src/gs.c index f3d245e..c7e4ff6 100644 --- a/libibmad/src/gs.c +++ b/libibmad/src/gs.c @@ -70,7 +70,8 @@ uint8_t *pma_query_via(void *rcvbuf, ib_portid_t * dest, int port, rpc.datasz = IB_PC_DATA_SZ; rpc.dataoffs = IB_PC_DATA_OFFS; - dest->qp = 1; + if (!dest->qp) + dest->qp = 1; if (!dest->qkey) dest->qkey = IB_DEFAULT_QP1_QKEY; @@ -109,7 +110,8 @@ uint8_t *performance_reset_via(void *rcvbuf, ib_portid_t * dest, rpc.timeout = timeout; rpc.datasz = IB_PC_DATA_SZ; rpc.dataoffs = IB_PC_DATA_OFFS; - dest->qp = 1; + if (!dest->qp) + dest->qp = 1; if (!dest->qkey) dest->qkey = IB_DEFAULT_QP1_QKEY; diff --git a/libibmad/src/rpc.c b/libibmad/src/rpc.c index 07b623d..efea1d3 100644 --- a/libibmad/src/rpc.c +++ b/libibmad/src/rpc.c @@ -183,33 +183,68 @@ _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, return -1; } +static int redirect_port(ib_portid_t *port, uint8_t *mad) +{ + port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); + if (!port->lid) { + IBWARN("GID-based redirection is not supported"); + return -1; + } + + port->qp = mad_get_field(mad, 64, IB_CPI_REDIRECT_QP_F); + port->qkey = mad_get_field(mad, 64, IB_CPI_REDIRECT_QKEY_F); + port->sl = mad_get_field(mad, 64, IB_CPI_REDIRECT_SL_F); + + /* TODO: Reverse map redirection P_Key to P_Key index */ + + if (ibdebug) + IBWARN("redirected to lid %d, qp 0x%x, qkey 0x%x, sl 0x%x", + port->lid, port->qp, port->qkey, port->sl); + + return 0; +} + void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc, ib_portid_t * dport, void *payload, void *rcvdata) { int status, len; uint8_t sndbuf[1024], rcvbuf[1024], *mad; int timeout, retries; + int redirect = 1; - len = 0; - memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); + while (redirect) { + len = 0; + memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); - if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) - return 0; + if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) + return 0; - timeout = rpc->timeout ? rpc->timeout : - port->timeout ? port->timeout : madrpc_timeout; - retries = port->retries ? port->retries : madrpc_retries; + timeout = rpc->timeout ? rpc->timeout : + port->timeout ? port->timeout : madrpc_timeout; + retries = port->retries ? port->retries : madrpc_retries; - if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf, - port->class_agents[rpc->mgtclass], - len, timeout, retries)) < 0) { - IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); - return 0; - } + if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf, + port->class_agents[rpc->mgtclass], + len, timeout, retries)) < 0) { + IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); + return 0; + } - mad = umad_get_mad(rcvbuf); + mad = umad_get_mad(rcvbuf); + status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F); + + /* check for exact match instead of only the redirect bit; + * that way, weird statuses cause an error, too */ + if (status == IB_MAD_STS_REDIRECT) { + /* update dport for next request and retry */ + /* bail if redirection fails */ + if (redirect_port(dport, mad)) + redirect = 0; + } else + redirect = 0; + } - if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F)) != 0) { + if (status != 0) { ERRS("MAD completed with error status 0x%x; dport (%s)", status, portid2str(dport)); return 0; -- 1.5.5 From monis at Voltaire.COM Tue Jul 7 07:53:33 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Tue, 07 Jul 2009 17:53:33 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A521784.5090304@Voltaire.COM> References: <4A521784.5090304@Voltaire.COM> Message-ID: <4A53616D.5000400@Voltaire.COM> Roland, this is the patch with changes according to Yossi's comments in case you are convinced that there is a place to a fix in IPoIB along with a fix in bonding or core network code. ------------------------------- Whenever an illegal multicast address is passed to IPoIB for it to join it stops all subsequent requests from being joined. That happens because IPoIB joins to multicast addresses in the order they arrived and doesn't handle the next group's join until the current join finishes with success. This phenomena happens a lot when a bonding interface enslaves IPoIB devices. Before enslaving IPoIB interfaces the bonding device acts like an Ethernet device, including the way it translates muticast IP addresses to HW addresses. When it comes up without slaves it translates the group 224.0.0.1 (all hosts) as if it were an Ethernet device and when it enslaves IPoIB devices this is the address that they get for joining (which is a garbage for them) This patch moves the multicast address to the end of the list after a join attempt. Even if the join fails then the next attempt will be for a different address. Signed-off-by: Moni Shoua -- drivers/infiniband/ulp/ipoib/ipoib_multicast.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c index a0e9753..3c3c63d 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c @@ -379,6 +379,7 @@ static int ipoib_mcast_join_complete(int status, struct ipoib_mcast *mcast = multicast->context; struct net_device *dev = mcast->dev; struct ipoib_dev_priv *priv = netdev_priv(dev); + struct ipoib_mcast *next_mcast; ipoib_dbg_mcast(priv, "join completion for %pI6 (status %d)\n", mcast->mcmember.mgid.raw, status); @@ -427,9 +428,17 @@ static int ipoib_mcast_join_complete(int status, mutex_lock(&mcast_mutex); spin_lock_irq(&priv->lock); - if (test_bit(IPOIB_MCAST_RUN, &priv->flags)) - queue_delayed_work(ipoib_workqueue, &priv->mcast_task, - mcast->backoff * HZ); + if (test_bit(IPOIB_MCAST_RUN, &priv->flags)) { + list_for_each_entry(next_mcast, &priv->multicast_list, list) { + if (!test_bit(IPOIB_MCAST_FLAG_SENDONLY, &next_mcast->flags) + && !test_bit(IPOIB_MCAST_FLAG_BUSY, &next_mcast->flags) + && !test_bit(IPOIB_MCAST_FLAG_ATTACHED, &next_mcast->flags)) + break; + } + if (&next_mcast->list != &priv->multicast_list) + queue_delayed_work(ipoib_workqueue, &priv->mcast_task, + next_mcast->backoff * HZ); + } spin_unlock_irq(&priv->lock); mutex_unlock(&mcast_mutex); @@ -570,13 +579,16 @@ void ipoib_mcast_join_task(struct work_struct *work) break; } } - spin_unlock_irq(&priv->lock); if (&mcast->list == &priv->multicast_list) { /* All done */ + spin_unlock_irq(&priv->lock); break; } + list_move_tail(&mcast->list, &priv->multicast_list); + spin_unlock_irq(&priv->lock); + ipoib_mcast_join(dev, mcast, 1); return; } From jackm at dev.mellanox.co.il Tue Jul 7 07:57:28 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Tue, 7 Jul 2009 17:57:28 +0300 Subject: [ofa-general] upstream stable merge request for OFED 1.4 In-Reply-To: <20090707123153.GD8065@barkeeper1-xen.linbit> References: <20090703144824.GA24820@barkeeper1-xen.linbit> <20090707123153.GD8065@barkeeper1-xen.linbit> Message-ID: <200907071757.29266.jackm@dev.mellanox.co.il> On Tuesday 07 July 2009 15:31, Lars Ellenberg wrote: > but I was wondering about the status of the > git://git.openfabrics.org/ofed_1_4/linux-2.6.git tree, > whether that is supposed to be the "most uptodate official ofed_1_4" > kernel, and whether or not it is going to be updated to either track > some stable series, or linux-2.6 master. > OFED users install the OFED package on top of an existing kernel (thus the need for backports). By install, I mean that the OFED "*.ko" modules are installed under /lib/modules//updates -- which supersede the modules under /lib/modules//kernel. These modules residing in updates are built entirely from code in the OFED source tree (which will also use header files from the kernel resident on the host). The only thing needed from the kernel on the host system is the API/prototypes, which I understand do not change in the stable series of a given kernel. There is no reason, therefore, to deal with merges to the most recent stable version of a given kernel. Furthermore, bug fixes to modules in a released OFED are put in as patches in the kernel_patches/fixes directory, and if there is demand for it another version of OFED is released with those fixes (e.g., OFED 1.4.1, for OFED 1.4). (For users who need fixes immediately, they can take the OFED daily build, which will incorporate any new patches in the kernel_patches/fixes directory. These patches are applied as part of the OFED installation process). OFED 1.4 will remain at kernel 2.6.27. OFED 1.5 is currently under development, and is based upon kernel 2.6.30. -Jack From hal.rosenstock at gmail.com Tue Jul 7 08:23:18 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Tue, 7 Jul 2009 11:23:18 -0400 Subject: [ofa-general] Re: [PATCH v4] libibmad: Handle MAD redirection In-Reply-To: <200907071620.23003.fenkes@de.ibm.com> References: <200906291410.33477.fenkes@de.ibm.com> <20090701200043.GF20745@obsidianresearch.com> <200907071620.23003.fenkes@de.ibm.com> Message-ID: On Tue, Jul 7, 2009 at 10:20 AM, Joachim Fenkes wrote: > Previously, libibmad reacted to GSI MAD responses with a "redirect" status > by throwing an error. IBM eHCA adapters use redirection, so most > infiniband_diags tools didn't work against eHCA. > > Fix: Modify mad_rpc() so that it resends the request to the redirection > target if a "redirect" GS response is received. This is repeated until no > "redirect" response is received, allowing for multiple levels of > indirection. > > The dport argument is updated with the redirection target, so subsequent > MADs will not go through the redirection process again but reach the target > directly. > > Tested using perfquery between ehca, mlx4 and mthca in all possible > combinations. > > Signed-off-by: Joachim Fenkes > --- > > After all has been said and done, here's the hopefully last iteration of the > patch, with the hex display of the redirect LID replaced by decimal. > > Any objections against this patch? See below for comment. > > Regards, >  Joachim > >  libibmad/include/infiniband/mad.h |    9 +++++ >  libibmad/src/gs.c                 |    6 ++- >  libibmad/src/rpc.c                |   65 ++++++++++++++++++++++++++++-------- >  3 files changed, 63 insertions(+), 17 deletions(-) > > diff --git a/libibmad/include/infiniband/mad.h b/libibmad/include/infiniband/mad.h > index aa27eb5..bdf5158 100644 > --- a/libibmad/include/infiniband/mad.h > +++ b/libibmad/include/infiniband/mad.h > @@ -115,6 +115,8 @@ enum MAD_ATTR_ID { > >  enum MAD_STATUS { >        IB_MAD_STS_OK                        = (0 << 2), > +       IB_MAD_STS_BUSY                      = (1 << 0), > +       IB_MAD_STS_REDIRECT                  = (1 << 1), >        IB_MAD_STS_BAD_BASE_VER_OR_CLASS     = (1 << 2), >        IB_MAD_STS_METHOD_NOT_SUPPORTED      = (2 << 2), >        IB_MAD_STS_METHOD_ATTR_NOT_SUPPORTED = (3 << 2), > @@ -783,8 +785,15 @@ MAD_EXPORT int madrpc_set_timeout(int timeout); >  MAD_EXPORT struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port, >                        int *mgmt_classes, int num_classes); >  MAD_EXPORT void mad_rpc_close_port(struct ibmad_port *srcport); > + > +/* > + * On redirection, the dport argument is updated with the redirection target, > + * so subsequent MADs will not go through the redirection process again but > + * reach the target directly. > + */ >  MAD_EXPORT void *mad_rpc(const struct ibmad_port *srcport, ib_rpc_t * rpc, >                        ib_portid_t * dport, void *payload, void *rcvdata); > + >  MAD_EXPORT void *mad_rpc_rmpp(const struct ibmad_port *srcport, ib_rpc_t * rpc, >                              ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, >                              void *data); > diff --git a/libibmad/src/gs.c b/libibmad/src/gs.c > index f3d245e..c7e4ff6 100644 > --- a/libibmad/src/gs.c > +++ b/libibmad/src/gs.c > @@ -70,7 +70,8 @@ uint8_t *pma_query_via(void *rcvbuf, ib_portid_t * dest, int port, >        rpc.datasz = IB_PC_DATA_SZ; >        rpc.dataoffs = IB_PC_DATA_OFFS; > > -       dest->qp = 1; > +       if (!dest->qp) > +               dest->qp = 1; >        if (!dest->qkey) >                dest->qkey = IB_DEFAULT_QP1_QKEY; > > @@ -109,7 +110,8 @@ uint8_t *performance_reset_via(void *rcvbuf, ib_portid_t * dest, >        rpc.timeout = timeout; >        rpc.datasz = IB_PC_DATA_SZ; >        rpc.dataoffs = IB_PC_DATA_OFFS; > -       dest->qp = 1; > +       if (!dest->qp) > +               dest->qp = 1; >        if (!dest->qkey) >                dest->qkey = IB_DEFAULT_QP1_QKEY; > > diff --git a/libibmad/src/rpc.c b/libibmad/src/rpc.c > index 07b623d..efea1d3 100644 > --- a/libibmad/src/rpc.c > +++ b/libibmad/src/rpc.c > @@ -183,33 +183,68 @@ _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, >        return -1; >  } > > +static int redirect_port(ib_portid_t *port, uint8_t *mad) > +{ > +       port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); > +       if (!port->lid) { > +               IBWARN("GID-based redirection is not supported"); > +               return -1; > +       } I hate to keep beating this horse but the lack of a LID certainly means GID based redirection when the GID is not 0, IMO this LID check is insufficient in general. I suppose this can be fixed down the road. -- Hal > + > +       port->qp = mad_get_field(mad, 64, IB_CPI_REDIRECT_QP_F); > +       port->qkey = mad_get_field(mad, 64, IB_CPI_REDIRECT_QKEY_F); > +       port->sl = mad_get_field(mad, 64, IB_CPI_REDIRECT_SL_F); > + > +       /* TODO: Reverse map redirection P_Key to P_Key index */ > + > +       if (ibdebug) > +               IBWARN("redirected to lid %d, qp 0x%x, qkey 0x%x, sl 0x%x", > +                      port->lid, port->qp, port->qkey, port->sl); > + > +       return 0; > +} > + >  void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc, >              ib_portid_t * dport, void *payload, void *rcvdata) >  { >        int status, len; >        uint8_t sndbuf[1024], rcvbuf[1024], *mad; >        int timeout, retries; > +       int redirect = 1; > > -       len = 0; > -       memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); > +       while (redirect) { > +               len = 0; > +               memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); > > -       if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) > -               return 0; > +               if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) > +                       return 0; > > -       timeout = rpc->timeout ? rpc->timeout : > -           port->timeout ? port->timeout : madrpc_timeout; > -       retries = port->retries ? port->retries : madrpc_retries; > +               timeout = rpc->timeout ? rpc->timeout : > +                       port->timeout ? port->timeout : madrpc_timeout; > +               retries = port->retries ? port->retries : madrpc_retries; > > -       if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf, > -                             port->class_agents[rpc->mgtclass], > -                             len, timeout, retries)) < 0) { > -               IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); > -               return 0; > -       } > +               if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf, > +                                     port->class_agents[rpc->mgtclass], > +                                     len, timeout, retries)) < 0) { > +                       IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); > +                       return 0; > +               } > > -       mad = umad_get_mad(rcvbuf); > +               mad = umad_get_mad(rcvbuf); > +               status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F); > + > +               /* check for exact match instead of only the redirect bit; > +                * that way, weird statuses cause an error, too */ > +               if (status == IB_MAD_STS_REDIRECT) { > +                       /* update dport for next request and retry */ > +                       /* bail if redirection fails */ > +                       if (redirect_port(dport, mad)) > +                               redirect = 0; > +               } else > +                       redirect = 0; > +       } > > -       if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F)) != 0) { > +       if (status != 0) { >                ERRS("MAD completed with error status 0x%x; dport (%s)", >                     status, portid2str(dport)); >                return 0; > -- > 1.5.5 > > > > From cfreese at super.org Tue Jul 7 09:11:13 2009 From: cfreese at super.org (Craig Reese) Date: Tue, 07 Jul 2009 12:11:13 -0400 Subject: [ofa-general] 1MB/sec IPoIB performance IBM Blade and RHEL 5.3? Message-ID: <4A5373A1.4010001@super.org> Not sure if this is a RedHat, IBM, or OpenFabrics issue, but thought this might be a good place to start.... Environment: IBM HS20 (8843), each with 1x IB card (26K6456: board_id: HCA.Cougar.A1) IBM/TopSpin switch module (26K6453) RHEL 5.3 Server w/ OFED 1.3.1 (as comes with the 5.3 distribution) (I know this is somewhat dated, but it's what I have to experiment with at the moment) I pretty much just installed things and started them up. No tuning or updating of software etc.... I've got interfaces setup on each machine: hs20[ab] = Ethernet hs20[ab]-ib = Infiniband both seem to be working for normal IP stuff (ssh, ping, etc...) I was curious about performance so I ran qperf: starting the server on hs20a with : hs20a> qperf running the client from hs20b: hs20b> qperf hs20a tcp_bw tcp_bw: bw = 118 MB/sec hs20b> qperf hs20a-ib tcp_bw tcp_bw: bw = 1.15 MB/sec <-- Not a typo! hs20b> qperf hs20a-ib sdp_bw sdp_bw: bw = 238 MB/sec I got similar results using netperf. Can anyone account for the poor performance in case 2? I'm unclear whether it's obsolete hardware, misconfigured software/hardware, deficiencies in 1.3.1 software, or my misinterpretation of the results, but 1.15MB/sec seems pretty bad. From lars.ellenberg at linbit.com Tue Jul 7 09:14:15 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Tue, 7 Jul 2009 18:14:15 +0200 Subject: [ofa-general] lockdep is unhappy with SDP Message-ID: <20090707161415.GE8065@barkeeper1-xen.linbit> [ 1947.003662] ============================================= [ 1947.006730] [ INFO: possible recursive locking detected ] [ 1947.020862] 2.6.27.26-d02 #2 which is to read: git://git.openfabrics.org/ofed_1_4/linux-2.6.git merged with upstream stable git://git4.kernel.org/pub/scm/linux/kernel/git/stable/linux-2.6.27.y.git both as of today: ofed_kernel 08acda8 sdp: Fix memory leak in bzcopy linux-v2.6.27.y/master 49cbf40 Linux 2.6.27.26 the relevant code section looks identically in ofed 1.5, though. [ 1947.020862] --------------------------------------------- [ 1947.020862] ib_cm/6/31070 is trying to acquire lock: [ 1947.054199] (sk_lock-27){--..}, at: [] sdp_connected_handler+0x1c0/0x2f0 [ib_sdp] [ 1947.054199] [ 1947.054199] but task is already holding lock: [ 1947.054199] (sk_lock-27){--..}, at: [] sdp_cma_handler+0x3c/0x15f0 [ib_sdp] [ 1947.054199] [ 1947.054199] other info that might help us debug this: [ 1947.054199] 4 locks held by ib_cm/6/31070: [ 1947.120855] #0: (ib_cm){--..}, at: [] run_workqueue+0x14d/0x250 [ 1947.120855] #1: (&(&work->work)->work){--..}, at: [] run_workqueue+0x14d/0x250 [ 1947.120855] #2: (&id_priv->handler_mutex){--..}, at: [] cma_disable_callback+0x2b/0x60 [rdma_cm] [ 1947.120855] #3: (sk_lock-27){--..}, at: [] sdp_cma_handler+0x3c/0x15f0 [ib_sdp] [ 1947.120855] [ 1947.120855] stack backtrace: [ 1947.120855] Pid: 31070, comm: ib_cm/6 Not tainted 2.6.27.26-d02 #2 [ 1947.120855] [ 1947.120855] Call Trace: [ 1947.120855] [] validate_chain+0xaaf/0x1040 [ 1947.120855] [] __lock_acquire+0x24a/0xa00 [ 1947.120855] [] lock_acquire+0x91/0xc0 [ 1947.250858] [] ? sdp_connected_handler+0x1c0/0x2f0 [ib_sdp] [ 1947.250858] [] lock_sock_nested+0x108/0x120 actually it uses lock_sock(parent), which is lock_sock_nested(parent, 0); maybe it is simply wrong? or maybe this is in fact ok, and you only need to tell lockdep about it, like - lock_sock(parent); + lock_sock_nested(parent, SINGLE_DEPTH_NESTING); [ 1947.250858] [] ? sdp_connected_handler+0x1c0/0x2f0 [ib_sdp] [ 1947.250858] [] ? trace_hardirqs_on+0xd/0x10 [ 1947.250858] [] ? trace_hardirqs_on_caller+0xda/0x180 [ 1947.250858] [] sdp_connected_handler+0x1c0/0x2f0 [ib_sdp] [ 1947.250858] [] sdp_cma_handler+0x6f1/0x15f0 [ib_sdp] [ 1947.250858] [] ? trace_hardirqs_on_caller+0xda/0x180 [ 1947.250858] [] ? mutex_lock_nested+0x1f3/0x300 [ 1947.250858] [] ? cma_disable_callback+0x2b/0x60 [rdma_cm] [ 1947.250858] [] ? trace_hardirqs_off+0xd/0x10 [ 1947.250858] [] ? cma_disable_callback+0x2b/0x60 [rdma_cm] [ 1947.250858] [] ? cm_work_handler+0x0/0xce0 [ib_cm] [ 1947.250858] [] ? cm_work_handler+0x0/0xce0 [ib_cm] [ 1947.250858] [] cma_ib_handler+0xcd/0x230 [rdma_cm] [ 1947.250858] [] ? trace_hardirqs_on+0xd/0x10 [ 1947.250858] [] cm_process_work+0x22/0xe0 [ib_cm] [ 1947.250858] [] ? cm_work_handler+0x0/0xce0 [ib_cm] [ 1947.250858] [] cm_rtu_handler+0xab/0x150 [ib_cm] [ 1947.499836] [] cm_work_handler+0xfc/0xce0 [ib_cm] [ 1947.507523] [] ? run_workqueue+0x14d/0x250 [ 1947.507523] [] ? cm_work_handler+0x0/0xce0 [ib_cm] [ 1947.507523] [] run_workqueue+0x19e/0x250 [ 1947.507523] [] ? run_workqueue+0x14d/0x250 [ 1947.507523] [] worker_thread+0xbf/0x120 [ 1947.507523] [] ? autoremove_wake_function+0x0/0x40 [ 1947.507523] [] ? worker_thread+0x0/0x120 [ 1947.507523] [] kthread+0x4d/0x80 [ 1947.507523] [] child_rip+0xa/0x11 [ 1947.507523] [] ? restore_args+0x0/0x30 [ 1947.507523] [] ? kthread+0x0/0x80 [ 1947.507523] [] ? child_rip+0x0/0x11 Thanks for feedback. -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. From jgunthorpe at obsidianresearch.com Tue Jul 7 09:48:03 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Tue, 7 Jul 2009 10:48:03 -0600 Subject: [ofa-general] Re: [PATCH] rdma_cm: Add debugfs entries to monitor rdma_cm connections In-Reply-To: <4A531FA6.6050803@voltaire.com> References: <4A3E45D3.3040405@Voltaire.COM> <4A50C13F.6070506@Voltaire.COM> <4A520600.2080004@opengridcomputing.com> <4A531FA6.6050803@voltaire.com> Message-ID: <20090707164803.GR20745@obsidianresearch.com> On Tue, Jul 07, 2009 at 01:12:54PM +0300, Or Gerlitz wrote: > Steve Wise wrote: > >I agree that this is useful for debugging. > Roland, Sean, I agree with Steve and Moni. Today there's no way to know > what rdma-cm connections/sessions are open now and with patch there is a > way, so debugfs support is very useful for debugging rdma-cm ULPs, > simple as that (e.g in the same manner ipoib has debugfs support), lets > get this in for 2.6.32 > > Next, for more production ready monitoring support, netlink solutions > could be great, Jason, thoughts? Well, thats the rub isn't it? debugfs is not 'production ready' (by definition) so why spend time on it? I honestly don't understand the reluctance to do a good job here. We all hear the same things from users - more visibility is needed - this debufs patch kinda addresses a very tiny portion of that visbility. Why the resistance to doing a proper job and solving the actual problem? debufs is entirely the wrong answer - this is not kernel debugging, this is required user space diagnostic information. Jason From lars.ellenberg at linbit.com Tue Jul 7 10:13:26 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Tue, 7 Jul 2009 19:13:26 +0200 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <20090702132304.GD9305@soda.linbit> References: <20090701133652.GG9112@soda.linbit> <4A4B947C.1030607@mellanox.co.il> <20090702081445.GA9118@soda.linbit> <4A4C7021.2090506@mellanox.co.il> <20090702132304.GD9305@soda.linbit> Message-ID: <20090707171326.GH8065@barkeeper1-xen.linbit> On Thu, Jul 02, 2009 at 03:23:04PM +0200, Lars Ellenberg wrote: > On Thu, Jul 02, 2009 at 11:30:25AM +0300, Amir Vadai wrote: > > Please attach the perl script to reproduce and I will check it. > > As to the second problem - I did notice such behavior but couldn't > > find a scenario to reproduce it. I guess it happen when the socket is > > closed due to error. > > > > Tell me if you notice any message in dmesg. > > My former test cluster has been reassigned. > The new test hardware will be available earliest tomorrow. > But I don't think there was anything relevant in dmesg. > > I'll try to reproduce on the new test hardware then, > and will get back to you as soon as possible. ok. finally new test hardware working. this is on debian lenny, userland ofed from http://pkg-ofed.alioth.debian.org/apt/ofed/ kernel git://git.openfabrics.org/ofed_1_4/linux-2.6.git merged with upstream stable git://git4.kernel.org/pub/scm/linux/kernel/git/stable/linux-2.6.27.y.git both as of today: ofed_kernel 08acda8 sdp: Fix memory leak in bzcopy linux-v2.6.27.y/master 49cbf40 Linux 2.6.27.26 kernel config: very many "kernel debugging" things enabled. if you want me to try a certain .config, or anything, this can be arranged. two very ugly perl scripts attached, one tcp server, one tcp client, adapted from the perlipc man page. client connects, sends a package, receives a package in an endless loop. package format: 4 byte magic, 2 byte ignored, 2 byte payload length indicated length of payload, all same bytes, but the trailing 4 byte, which again is a magic number. with ethernet, or IPoIB: runs endless. with LD_PRELOAD=libsdp.so runs for very few iterations, and errors out on one of the sanity checks. sample output: root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 2009-07-07 19:07:08 my_client.pl 4300: recv hdr [25]: invalid magic: 32 30 30 39 2d 30 37 2d which hapeens to be the hexdump of the string "2009-07-". where did it copy_user() that from? wtf? root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 2009-07-07 19:07:08 my_client.pl 4301: recv payload [35]: expected 12296, but received 12295 byte; last bytes received: 55 e4 e3 e2 ^^^^ pid, ^^ seq number. so for only 35 ping/pongs it did work ok. exactly: one byte too short. the trailing magic expected is e4 e3 e1 e1 root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 2009-07-07 19:07:09 my_client.pl 4302: recv payload [33]: expected 4131, but received 4130 byte; last bytes received: 55 e4 e3 e2 root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 2009-07-07 19:07:10 my_client.pl 4303: recv payload [29]: expected 16401, but received 16400 byte; last bytes received: 55 e4 e3 e2 root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 2009-07-07 19:07:12 my_client.pl 4304: recv payload [21]: expected 4110, but received 4109 byte; last bytes received: 55 e4 e3 e2 root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 2009-07-07 19:07:13 my_client.pl 4305: recv payload [4]: expected 20495, but received 20494 byte; last bytes received: 55 e4 e3 e2 root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 2009-07-07 19:07:14 my_client.pl 4306: recv payload [12]: expected 22530, but received 22529 byte; last bytes received: 55 e4 e3 e2 any suggestions how to proceed from here? -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. -------------- next part -------------- A non-text attachment was scrubbed... Name: my_client.pl Type: text/x-perl Size: 2726 bytes Desc: not available URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: my_server.pl Type: text/x-perl Size: 4012 bytes Desc: not available URL: From monis at Voltaire.COM Tue Jul 7 10:31:47 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Tue, 07 Jul 2009 20:31:47 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: References: <4A521784.5090304@Voltaire.COM> Message-ID: <4A538683.4030705@Voltaire.COM> Roland Dreier wrote: > After this patch, do we end up with > an IPoIB interface that's not a member of the all hosts multicast group? > (That seems like it would lead to confusing breakage later) > In my search for a solution to the issue with bonding I found the answer for this. I checked what will be the affect of delaying the decision of what is the device type until first the slave comes. This causes arp_mc_map() to fail and the address is not added to the mc list. On the other hand, the IPoIO will be a member of all hosts because dev_open()) does that for any net device that it is called for (see below). linux-ck8g:/root # ip m s dev ib0 16: ib0 link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:fb link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:ff:98:2f:c5 link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:01 users 2 link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:00:00:00:00:01 users 2 inet 224.0.0.1 inet6 ff02::1 However, there is a side affect. The bonding interface will be registered only to the inet address of all hosts but not to the link address. This is until dev_open is called again (see below again) linux-ck8g:/root # ip m s dev bond0 18: bond0 link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:fb link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:ff:98:2f:c5 inet 224.0.0.251 inet 224.0.0.1 inet6 ff02::1:ff98:2fc5 inet6 ff02::1 linux-ck8g:/root # ifconfig bond0 down linux-ck8g:/root # ifconfig bond0 up linux-ck8g:/root # ip m s dev bond0 18: bond0 link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:fb link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:ff:98:2f:c5 link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:00:00:00:00:01 link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:01 inet 224.0.0.251 inet 224.0.0.1 inet6 ff02::1:ff98:2fc5 inet6 ff02::1 linux-ck8g:/root # I don't want to leave it like that but just for curiosity, it is bad that the bond interface is not registered to the all hosts group if its slave is? thanks From chu11 at llnl.gov Tue Jul 7 15:24:12 2009 From: chu11 at llnl.gov (Al Chu) Date: Tue, 07 Jul 2009 15:24:12 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [0/5] libibnetdisc cleanup patches Message-ID: <1247005452.4730.38.camel@auk31.llnl.gov> Hey Sasha, There's a lot of minor things I wanted to cleanup/fix in the libibnetdisc library. Some have future features in mind, some were just general cleanup things I saw long the way. I've talked to Ira about it, and he concurs with the patches. These will probably the first 5 of many more to come. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory From chu11 at llnl.gov Tue Jul 7 15:24:26 2009 From: chu11 at llnl.gov (Al Chu) Date: Tue, 07 Jul 2009 15:24:26 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [1/5] libibnetdisc cleanup patches Message-ID: <1247005466.4730.39.camel@auk31.llnl.gov> Make api more consistent by removing convenience pointer from ibnd_node_t and requiring ibnd_fabric_t passed to all functions (in this case ibnd_update_node). Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- >From a6f7c6bea7f3549bb0bd24a4304f9dc3aa4b73ac Mon Sep 17 00:00:00 2001 From: Albert Chu Date: Tue, 7 Jul 2009 09:57:26 -0700 Subject: [PATCH] Make api more consistent by removing convenience pointer from ibnd_node_t and requiring ibnd_fabric_t passed to all functions (in this case ibnd_update_node). Signed-off-by: Albert Chu Signed-off-by: Ira Weiny --- .../libibnetdisc/include/infiniband/ibnetdisc.h | 3 +-- infiniband-diags/libibnetdisc/src/chassis.c | 7 +++---- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h index c7d293c..5f07805 100644 --- a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h +++ b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h @@ -47,7 +47,6 @@ struct port; /* forward declare */ */ typedef struct node { struct node *next; /* all node list in fabric */ - struct ib_fabric *fabric; /* the fabric node belongs to */ ib_portid_t path_portid; /* path from "from_node" */ int dist; /* num of hops from "from_node" */ @@ -161,7 +160,7 @@ MAD_EXPORT void ibnd_destroy_fabric(ibnd_fabric_t *fabric); */ MAD_EXPORT ibnd_node_t *ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid); MAD_EXPORT ibnd_node_t *ibnd_find_node_dr(ibnd_fabric_t *fabric, char *dr_str); -MAD_EXPORT ibnd_node_t *ibnd_update_node(ibnd_node_t *node); +MAD_EXPORT ibnd_node_t *ibnd_update_node(ibnd_fabric_t *fabric, ibnd_node_t *node); typedef void (*ibnd_iter_node_func_t)(ibnd_node_t *node, void *user_data); MAD_EXPORT void ibnd_iter_nodes(ibnd_fabric_t *fabric, diff --git a/infiniband-diags/libibnetdisc/src/chassis.c b/infiniband-diags/libibnetdisc/src/chassis.c index dbb0abe..78aee1f 100644 --- a/infiniband-diags/libibnetdisc/src/chassis.c +++ b/infiniband-diags/libibnetdisc/src/chassis.c @@ -197,9 +197,8 @@ static uint64_t get_chassisguid(ibnd_node_t *node) return sysimgguid; } -static ibnd_chassis_t *find_chassisguid(ibnd_node_t *node) +static ibnd_chassis_t *find_chassisguid(struct ibnd_fabric *f, ibnd_node_t *node) { - struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(node->fabric); ibnd_chassis_t *current; uint64_t chguid; @@ -790,7 +789,7 @@ ibnd_chassis_t *group_nodes(struct ibnd_fabric *fabric) if (mad_get_field(node->node.info, 0, IB_NODE_VENDORID_F) == VTR_VENDOR_ID) continue; if (mad_get_field64(node->node.info, 0, IB_NODE_SYSTEM_GUID_F)) { - chassis = find_chassisguid((ibnd_node_t *)node); + chassis = find_chassisguid(fabric, (ibnd_node_t *)node); if (chassis) chassis->nodecount++; else { @@ -811,7 +810,7 @@ ibnd_chassis_t *group_nodes(struct ibnd_fabric *fabric) if (mad_get_field(node->node.info, 0, IB_NODE_VENDORID_F) == VTR_VENDOR_ID) continue; if (mad_get_field64(node->node.info, 0, IB_NODE_SYSTEM_GUID_F)) { - chassis = find_chassisguid((ibnd_node_t *)node); + chassis = find_chassisguid(fabric, (ibnd_node_t *)node); if (chassis && chassis->nodecount > 1) { if (!chassis->chassisnum) chassis->chassisnum = ++chassisnum; diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index baea98e..b640bc1 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -245,12 +245,12 @@ ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid) } ibnd_node_t * -ibnd_update_node(ibnd_node_t *node) +ibnd_update_node(ibnd_fabric_t *fabric, ibnd_node_t *node) { char portinfo_port0[IB_SMP_DATA_SIZE]; void *nd = node->nodedesc; int p = 0; - struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(node->fabric); + struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric); struct ibnd_node *n = CONV_NODE_INTERNAL(node); if (query_node_info(f, n, &(n->node.path_portid))) @@ -377,7 +377,6 @@ create_node(struct ibnd_fabric *fabric, struct ibnd_node *temp, ib_portid_t *pat memcpy(node, temp, sizeof(*node)); node->node.dist = dist; node->node.path_portid = *path; - node->node.fabric = (ibnd_fabric_t *)fabric; add_to_nodeguid_hash(node, fabric->nodestbl); -- 1.5.4.5 From chu11 at llnl.gov Tue Jul 7 15:24:41 2009 From: chu11 at llnl.gov (Al Chu) Date: Tue, 07 Jul 2009 15:24:41 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [2/5] libibnetdisc cleanup patches Message-ID: <1247005481.4730.40.camel@auk31.llnl.gov> Rename internal libibnetdisc structs with _int suffix to differentiate code more cleanly against public structs. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- >From 0f2dba86d84bd96f7a24297c6fc2634f97d54f35 Mon Sep 17 00:00:00 2001 From: Albert Chu Date: Tue, 7 Jul 2009 11:18:05 -0700 Subject: [PATCH] Rename internal libibnetdisc structs with _int suffix to differentiate code more cleanly against public structs. Signed-off-by: Albert Chu Signed-off-by: Ira Weiny --- infiniband-diags/libibnetdisc/src/chassis.c | 48 +++++++------- infiniband-diags/libibnetdisc/src/chassis.h | 2 +- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 92 ++++++++++++------------ infiniband-diags/libibnetdisc/src/internal.h | 32 ++++---- 4 files changed, 87 insertions(+), 87 deletions(-) diff --git a/infiniband-diags/libibnetdisc/src/chassis.c b/infiniband-diags/libibnetdisc/src/chassis.c index 78aee1f..bec24bc 100644 --- a/infiniband-diags/libibnetdisc/src/chassis.c +++ b/infiniband-diags/libibnetdisc/src/chassis.c @@ -84,7 +84,7 @@ char *ibnd_get_chassis_slot_str(ibnd_node_t *node, char *str, size_t size) return (str); } -static ibnd_chassis_t *find_chassisnum(struct ibnd_fabric *fabric, unsigned char chassisnum) +static ibnd_chassis_t *find_chassisnum(struct ibnd_fabric_int *fabric, unsigned char chassisnum) { ibnd_chassis_t *current; @@ -197,7 +197,7 @@ static uint64_t get_chassisguid(ibnd_node_t *node) return sysimgguid; } -static ibnd_chassis_t *find_chassisguid(struct ibnd_fabric *f, ibnd_node_t *node) +static ibnd_chassis_t *find_chassisguid(struct ibnd_fabric_int *f, ibnd_node_t *node) { ibnd_chassis_t *current; uint64_t chguid; @@ -213,7 +213,7 @@ static ibnd_chassis_t *find_chassisguid(struct ibnd_fabric *f, ibnd_node_t *node uint64_t ibnd_get_chassis_guid(ibnd_fabric_t *fabric, unsigned char chassisnum) { - struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric); + struct ibnd_fabric_int *f = CONV_FABRIC_INTERNAL(fabric); ibnd_chassis_t *chassis; chassis = find_chassisnum(f, chassisnum); @@ -223,46 +223,46 @@ uint64_t ibnd_get_chassis_guid(ibnd_fabric_t *fabric, unsigned char chassisnum) return 0; } -static int is_router(struct ibnd_node *n) +static int is_router(struct ibnd_node_int *n) { uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F); return (devid == VTR_DEVID_IB_FC_ROUTER || devid == VTR_DEVID_IB_IP_ROUTER); } -static int is_spine_9096(struct ibnd_node *n) +static int is_spine_9096(struct ibnd_node_int *n) { uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F); return (devid == VTR_DEVID_SFB4 || devid == VTR_DEVID_SFB4_DDR); } -static int is_spine_9288(struct ibnd_node *n) +static int is_spine_9288(struct ibnd_node_int *n) { uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F); return (devid == VTR_DEVID_SFB12 || devid == VTR_DEVID_SFB12_DDR); } -static int is_spine_2004(struct ibnd_node *n) +static int is_spine_2004(struct ibnd_node_int *n) { uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F); return (devid == VTR_DEVID_SFB2004); } -static int is_spine_2012(struct ibnd_node *n) +static int is_spine_2012(struct ibnd_node_int *n) { uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F); return (devid == VTR_DEVID_SFB2012); } -static int is_spine(struct ibnd_node *n) +static int is_spine(struct ibnd_node_int *n) { return (is_spine_9096(n) || is_spine_9288(n) || is_spine_2004(n) || is_spine_2012(n)); } -static int is_line_24(struct ibnd_node *n) +static int is_line_24(struct ibnd_node_int *n) { uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F); return (devid == VTR_DEVID_SLB24 || @@ -270,24 +270,24 @@ static int is_line_24(struct ibnd_node *n) devid == VTR_DEVID_SRB2004); } -static int is_line_8(struct ibnd_node *n) +static int is_line_8(struct ibnd_node_int *n) { uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F); return (devid == VTR_DEVID_SLB8); } -static int is_line_2024(struct ibnd_node *n) +static int is_line_2024(struct ibnd_node_int *n) { uint32_t devid = mad_get_field(n->node.info, 0, IB_NODE_DEVID_F); return (devid == VTR_DEVID_SLB2024); } -static int is_line(struct ibnd_node *n) +static int is_line(struct ibnd_node_int *n) { return (is_line_24(n) || is_line_8(n) || is_line_2024(n)); } -int is_chassis_switch(struct ibnd_node *n) +int is_chassis_switch(struct ibnd_node_int *n) { return (is_spine(n) || is_line(n)); } @@ -308,7 +308,7 @@ char spine4_slot_2_slb[25] = { 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 0, 0 char anafa_spine4_slot_2_slb[25] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; /* reference { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 }; */ -static void get_sfb_slot(struct ibnd_node *node, ibnd_port_t *lineport) +static void get_sfb_slot(struct ibnd_node_int *node, ibnd_port_t *lineport) { ibnd_node_t *n = (ibnd_node_t *)node; @@ -335,7 +335,7 @@ static void get_sfb_slot(struct ibnd_node *node, ibnd_port_t *lineport) } } -static void get_router_slot(struct ibnd_node *node, ibnd_port_t *spineport) +static void get_router_slot(struct ibnd_node_int *node, ibnd_port_t *spineport) { ibnd_node_t *n = (ibnd_node_t *)node; uint64_t guessnum = 0; @@ -407,12 +407,12 @@ static void voltaire_portmap(ibnd_port_t *port); It could be optimized so, but time overhead is very small and its only diag.util */ -static void fill_voltaire_chassis_record(struct ibnd_node *node) +static void fill_voltaire_chassis_record(struct ibnd_node_int *node) { ibnd_node_t *n = (ibnd_node_t *)node; int p = 0; ibnd_port_t *port; - struct ibnd_node *remnode = 0; + struct ibnd_node_int *remnode = 0; if (node->ch_found) /* somehow this node has already been passed */ return; @@ -587,10 +587,10 @@ static void pass_on_spines_interpolate_chguid(ibnd_chassis_t *chassis) in that chassis chassis structure = structure of one standalone chassis */ -static void build_chassis(struct ibnd_node *node, ibnd_chassis_t *chassis) +static void build_chassis(struct ibnd_node_int *node, ibnd_chassis_t *chassis) { int p = 0; - struct ibnd_node *remnode = 0; + struct ibnd_node_int *remnode = 0; ibnd_port_t *port = 0; /* we get here with node = chassis_spine */ @@ -684,7 +684,7 @@ int int2ext_map_slb2024[2][25] = { static void voltaire_portmap(ibnd_port_t *port) { - struct ibnd_node *n = CONV_NODE_INTERNAL(port->node); + struct ibnd_node_int *n = CONV_NODE_INTERNAL(port->node); int portnum = port->portnum; int chipnum = 0; ibnd_node_t *node = port->node; @@ -709,7 +709,7 @@ voltaire_portmap(ibnd_port_t *port) port->ext_portnum = int2ext_map_slb8[chipnum][portnum]; } -static void add_chassis(struct ibnd_fabric *fabric) +static void add_chassis(struct ibnd_fabric_int *fabric) { if (!(fabric->current_chassis = calloc(1, sizeof(ibnd_chassis_t)))) IBPANIC("out of mem"); @@ -744,9 +744,9 @@ add_node_to_chassis(ibnd_chassis_t *chassis, ibnd_node_t *node) Pointer to the first chassis in a NULL terminated list of chassis in the fabric specified. */ -ibnd_chassis_t *group_nodes(struct ibnd_fabric *fabric) +ibnd_chassis_t *group_nodes(struct ibnd_fabric_int *fabric) { - struct ibnd_node *node; + struct ibnd_node_int *node; int dist; int chassisnum = 0; ibnd_chassis_t *chassis; diff --git a/infiniband-diags/libibnetdisc/src/chassis.h b/infiniband-diags/libibnetdisc/src/chassis.h index 16dad49..ca57a78 100644 --- a/infiniband-diags/libibnetdisc/src/chassis.h +++ b/infiniband-diags/libibnetdisc/src/chassis.h @@ -80,6 +80,6 @@ enum ibnd_chassis_type { UNRESOLVED_CT, ISR9288_CT, ISR9096_CT, ISR2012_CT, ISR2004_CT }; enum ibnd_chassis_slot_type { UNRESOLVED_CS, LINE_CS, SPINE_CS, SRBD_CS }; -ibnd_chassis_t *group_nodes(struct ibnd_fabric *fabric); +ibnd_chassis_t *group_nodes(struct ibnd_fabric_int *fabric); #endif /* _CHASSIS_H_ */ diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index b640bc1..a10dfcd 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -69,7 +69,7 @@ decode_port_info(ibnd_port_t *port) } static int -get_port_info(struct ibnd_fabric *fabric, struct ibnd_port *port, +get_port_info(struct ibnd_fabric_int *fabric, struct ibnd_port_int *port, int portnum, ib_portid_t *portid) { char width[64], speed[64]; @@ -99,7 +99,7 @@ get_port_info(struct ibnd_fabric *fabric, struct ibnd_port *port, * Returns -1 if error. */ static int -query_node_info(struct ibnd_fabric *fabric, struct ibnd_node *node, ib_portid_t *portid) +query_node_info(struct ibnd_fabric_int *fabric, struct ibnd_node_int *node, ib_portid_t *portid) { if (!smp_query_via(&(node->node.info), portid, IB_ATTR_NODE_INFO, 0, timeout_ms, fabric->fabric.ibmad_port)) @@ -118,8 +118,8 @@ query_node_info(struct ibnd_fabric *fabric, struct ibnd_node *node, ib_portid_t * Returns 0 if non switch node is found, 1 if switch is found, -1 if error. */ static int -query_node(struct ibnd_fabric *fabric, struct ibnd_node *inode, - struct ibnd_port *iport, ib_portid_t *portid) +query_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *inode, + struct ibnd_port_int *iport, ib_portid_t *portid) { ibnd_node_t *node = &(inode->node); ibnd_port_t *port = &(iport->port); @@ -177,7 +177,7 @@ add_port_to_dpath(ib_dr_path_t *path, int nextport) } static int -extend_dpath(struct ibnd_fabric *f, ib_portid_t *portid, int nextport) +extend_dpath(struct ibnd_fabric_int *f, ib_portid_t *portid, int nextport) { int rc = 0; @@ -201,7 +201,7 @@ extend_dpath(struct ibnd_fabric *f, ib_portid_t *portid, int nextport) static void dump_endnode(ib_portid_t *path, char *prompt, - struct ibnd_node *node, struct ibnd_port *port) + struct ibnd_node_int *node, struct ibnd_port_int *port) { char type[64]; if (!show_progress) @@ -217,11 +217,11 @@ dump_endnode(ib_portid_t *path, char *prompt, node->node.nodedesc); } -static struct ibnd_node * -find_existing_node(struct ibnd_fabric *fabric, struct ibnd_node *new) +static struct ibnd_node_int * +find_existing_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *new) { int hash = HASHGUID(new->node.guid) % HTSZ; - struct ibnd_node *node; + struct ibnd_node_int *node; for (node = fabric->nodestbl[hash]; node; node = node->htnext) if (node->node.guid == new->node.guid) @@ -233,9 +233,9 @@ find_existing_node(struct ibnd_fabric *fabric, struct ibnd_node *new) ibnd_node_t * ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid) { - struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric); + struct ibnd_fabric_int *f = CONV_FABRIC_INTERNAL(fabric); int hash = HASHGUID(guid) % HTSZ; - struct ibnd_node *node; + struct ibnd_node_int *node; for (node = f->nodestbl[hash]; node; node = node->htnext) if (node->node.guid == guid) @@ -250,8 +250,8 @@ ibnd_update_node(ibnd_fabric_t *fabric, ibnd_node_t *node) char portinfo_port0[IB_SMP_DATA_SIZE]; void *nd = node->nodedesc; int p = 0; - struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric); - struct ibnd_node *n = CONV_NODE_INTERNAL(node); + struct ibnd_fabric_int *f = CONV_FABRIC_INTERNAL(fabric); + struct ibnd_node_int *n = CONV_NODE_INTERNAL(node); if (query_node_info(f, n, &(n->node.path_portid))) return (NULL); @@ -288,7 +288,7 @@ done: ibnd_node_t * ibnd_find_node_dr(ibnd_fabric_t *fabric, char *dr_str) { - struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric); + struct ibnd_fabric_int *f = CONV_FABRIC_INTERNAL(fabric); int i = 0; ibnd_node_t *rc = f->fabric.from_node; ib_dr_path_t path; @@ -315,7 +315,7 @@ ibnd_find_node_dr(ibnd_fabric_t *fabric, char *dr_str) } static void -add_to_nodeguid_hash(struct ibnd_node *node, struct ibnd_node *hash[]) +add_to_nodeguid_hash(struct ibnd_node_int *node, struct ibnd_node_int *hash[]) { int hash_idx = HASHGUID(node->node.guid) % HTSZ; @@ -324,7 +324,7 @@ add_to_nodeguid_hash(struct ibnd_node *node, struct ibnd_node *hash[]) } static void -add_to_portguid_hash(struct ibnd_port *port, struct ibnd_port *hash[]) +add_to_portguid_hash(struct ibnd_port_int *port, struct ibnd_port_int *hash[]) { int hash_idx = HASHGUID(port->port.guid) % HTSZ; @@ -333,7 +333,7 @@ add_to_portguid_hash(struct ibnd_port *port, struct ibnd_port *hash[]) } static void -add_to_type_list(struct ibnd_node*node, struct ibnd_fabric *fabric) +add_to_type_list(struct ibnd_node_int *node, struct ibnd_fabric_int *fabric) { switch (node->node.type) { case IB_NODE_CA: @@ -352,7 +352,7 @@ add_to_type_list(struct ibnd_node*node, struct ibnd_fabric *fabric) } static void -add_to_nodedist(struct ibnd_node *node, struct ibnd_fabric *fabric) +add_to_nodedist(struct ibnd_node_int *node, struct ibnd_fabric_int *fabric) { int dist = node->node.dist; if (node->node.type != IB_NODE_SWITCH) @@ -363,10 +363,10 @@ add_to_nodedist(struct ibnd_node *node, struct ibnd_fabric *fabric) } -static struct ibnd_node * -create_node(struct ibnd_fabric *fabric, struct ibnd_node *temp, ib_portid_t *path, int dist) +static struct ibnd_node_int * +create_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *temp, ib_portid_t *path, int dist) { - struct ibnd_node *node; + struct ibnd_node_int *node; node = malloc(sizeof(*node)); if (!node) { @@ -390,8 +390,8 @@ create_node(struct ibnd_fabric *fabric, struct ibnd_node *temp, ib_portid_t *pat return node; } -static struct ibnd_port * -find_existing_port_node(struct ibnd_node *node, struct ibnd_port *port) +static struct ibnd_port_int * +find_existing_port_node(struct ibnd_node_int *node, struct ibnd_port_int *port) { if (port->port.portnum > node->node.numports || node->node.ports == NULL ) return (NULL); @@ -399,10 +399,10 @@ find_existing_port_node(struct ibnd_node *node, struct ibnd_port *port) return (CONV_PORT_INTERNAL(node->node.ports[port->port.portnum])); } -static struct ibnd_port * -add_port_to_node(struct ibnd_fabric *fabric, struct ibnd_node *node, struct ibnd_port *temp) +static struct ibnd_port_int * +add_port_to_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *node, struct ibnd_port_int *temp) { - struct ibnd_port *port; + struct ibnd_port_int *port; port = malloc(sizeof(*port)); if (!port) @@ -427,8 +427,8 @@ add_port_to_node(struct ibnd_fabric *fabric, struct ibnd_node *node, struct ibnd } static void -link_ports(struct ibnd_node *node, struct ibnd_port *port, - struct ibnd_node *remotenode, struct ibnd_port *remoteport) +link_ports(struct ibnd_node_int *node, struct ibnd_port_int *port, + struct ibnd_node_int *remotenode, struct ibnd_port_int *remoteport) { IBND_DEBUG("linking: 0x%" PRIx64 " %p->%p:%u and 0x%" PRIx64 " %p->%p:%u\n", node->node.guid, node, port, port->port.portnum, @@ -443,13 +443,13 @@ link_ports(struct ibnd_node *node, struct ibnd_port *port, } static int -get_remote_node(struct ibnd_fabric *fabric, struct ibnd_node *node, struct ibnd_port *port, ib_portid_t *path, - int portnum, int dist) +get_remote_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *node, struct ibnd_port_int *port, + ib_portid_t *path, int portnum, int dist) { - struct ibnd_node node_buf; - struct ibnd_port port_buf; - struct ibnd_node *remotenode, *oldnode; - struct ibnd_port *remoteport, *oldport; + struct ibnd_node_int node_buf; + struct ibnd_port_int port_buf; + struct ibnd_node_int *remotenode, *oldnode; + struct ibnd_port_int *remoteport, *oldport; memset(&node_buf, 0, sizeof(node_buf)); memset(&port_buf, 0, sizeof(port_buf)); @@ -495,12 +495,12 @@ ibnd_fabric_t * ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, ib_portid_t *from, int hops) { - struct ibnd_fabric *fabric = NULL; + struct ibnd_fabric_int *fabric = NULL; ib_portid_t my_portid = {0}; - struct ibnd_node node_buf; - struct ibnd_port port_buf; - struct ibnd_node *node; - struct ibnd_port *port; + struct ibnd_node_int node_buf; + struct ibnd_port_int port_buf; + struct ibnd_node_int *node; + struct ibnd_port_int *port; int i; int dist = 0; ib_portid_t *path; @@ -611,7 +611,7 @@ error: } static void -destroy_node(struct ibnd_node *node) +destroy_node(struct ibnd_node_int *node) { int p = 0; @@ -625,10 +625,10 @@ destroy_node(struct ibnd_node *node) void ibnd_destroy_fabric(ibnd_fabric_t *fabric) { - struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric); + struct ibnd_fabric_int *f = CONV_FABRIC_INTERNAL(fabric); int dist = 0; - struct ibnd_node *node = NULL; - struct ibnd_node *next = NULL; + struct ibnd_node_int *node = NULL; + struct ibnd_node_int *next = NULL; ibnd_chassis_t *ch, *ch_next; ch = f->first_chassis; @@ -687,9 +687,9 @@ ibnd_iter_nodes_type(ibnd_fabric_t *fabric, int node_type, void *user_data) { - struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric); - struct ibnd_node *list = NULL; - struct ibnd_node *cur = NULL; + struct ibnd_fabric_int *f = CONV_FABRIC_INTERNAL(fabric); + struct ibnd_node_int *list = NULL; + struct ibnd_node_int *cur = NULL; switch (node_type) { case IB_NODE_SWITCH: diff --git a/infiniband-diags/libibnetdisc/src/internal.h b/infiniband-diags/libibnetdisc/src/internal.h index 5785e33..bdef871 100644 --- a/infiniband-diags/libibnetdisc/src/internal.h +++ b/infiniband-diags/libibnetdisc/src/internal.h @@ -49,47 +49,47 @@ #define IBND_ERROR(fmt, ...) \ fprintf(stderr, "%s:%u; " fmt, __FILE__, __LINE__, ## __VA_ARGS__) -struct ibnd_node { +struct ibnd_node_int { /* This member MUST BE FIRST */ ibnd_node_t node; /* internal use only */ unsigned char ch_found; - struct ibnd_node *htnext; /* hash table list */ - struct ibnd_node *dnext; /* nodesdist next */ - struct ibnd_node *type_next; /* next based on type */ + struct ibnd_node_int *htnext; /* hash table list */ + struct ibnd_node_int *dnext; /* nodesdist next */ + struct ibnd_node_int *type_next; /* next based on type */ }; -#define CONV_NODE_INTERNAL(node) ((struct ibnd_node *)node) +#define CONV_NODE_INTERNAL(node) ((struct ibnd_node_int *)node) -struct ibnd_port { +struct ibnd_port_int { /* This member MUST BE FIRST */ ibnd_port_t port; /* internal use only */ - struct ibnd_port *htnext; + struct ibnd_port_int *htnext; }; -#define CONV_PORT_INTERNAL(port) ((struct ibnd_port *)port) +#define CONV_PORT_INTERNAL(port) ((struct ibnd_port_int *)port) /* HASH table defines */ #define HASHGUID(guid) ((uint32_t)(((uint32_t)(guid) * 101) ^ ((uint32_t)((guid) >> 32) * 103))) #define HTSZ 137 -struct ibnd_fabric { +struct ibnd_fabric_int { /* This member MUST BE FIRST */ ibnd_fabric_t fabric; /* internal use only */ - struct ibnd_node *nodestbl[HTSZ]; - struct ibnd_port *portstbl[HTSZ]; - struct ibnd_node *nodesdist[MAXHOPS+1]; + struct ibnd_node_int *nodestbl[HTSZ]; + struct ibnd_port_int *portstbl[HTSZ]; + struct ibnd_node_int *nodesdist[MAXHOPS+1]; ibnd_chassis_t *first_chassis; ibnd_chassis_t *current_chassis; ibnd_chassis_t *last_chassis; - struct ibnd_node *switches; - struct ibnd_node *ch_adapters; - struct ibnd_node *routers; + struct ibnd_node_int *switches; + struct ibnd_node_int *ch_adapters; + struct ibnd_node_int *routers; ib_portid_t selfportid; }; -#define CONV_FABRIC_INTERNAL(fabric) ((struct ibnd_fabric *)fabric) +#define CONV_FABRIC_INTERNAL(fabric) ((struct ibnd_fabric_int *)fabric) #endif /* _INTERNAL_H_ */ -- 1.5.4.5 From chu11 at llnl.gov Tue Jul 7 15:24:52 2009 From: chu11 at llnl.gov (Al Chu) Date: Tue, 07 Jul 2009 15:24:52 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [3/5] libibnetdisc cleanup patches Message-ID: <1247005492.4730.41.camel@auk31.llnl.gov> Properly prefix all libibnetdisc structs with ibnd_, to associate them with the libibnetdisc library and avoid potential name conflicts with other code. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- >From f1c04e6d38c9408b475274bbae6ef16be9e2d498 Mon Sep 17 00:00:00 2001 From: Albert Chu Date: Tue, 7 Jul 2009 11:25:29 -0700 Subject: [PATCH] Properly prefix all libibnetdisc structs with ibnd_, to associate them with the libibnetdisc library and avoid potential name conflicts with other code. Signed-off-by: Albert Chu Signed-off-by: Ira Weiny --- .../libibnetdisc/include/infiniband/ibnetdisc.h | 26 ++++++++++---------- 1 files changed, 13 insertions(+), 13 deletions(-) diff --git a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h index 5f07805..62d639f 100644 --- a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h +++ b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h @@ -38,15 +38,15 @@ #include #include -struct ib_fabric; /* forward declare */ -struct chassis; /* forward declare */ -struct port; /* forward declare */ +struct ibnd_fabric; /* forward declare */ +struct ibnd_chassis; /* forward declare */ +struct ibnd_port; /* forward declare */ /** ========================================================================= * Node */ -typedef struct node { - struct node *next; /* all node list in fabric */ +typedef struct ibnd_node { + struct ibnd_node *next; /* all node list in fabric */ ib_portid_t path_portid; /* path from "from_node" */ int dist; /* num of hops from "from_node" */ @@ -67,13 +67,13 @@ typedef struct node { char nodedesc[IB_SMP_DATA_SIZE]; - struct port **ports; /* in order array of port pointers */ + struct ibnd_port **ports; /* in order array of port pointers */ /* the size of this array is info.numports + 1 */ /* items MAY BE NULL! (ie 0 == switches only) */ /* chassis info */ - struct node *next_chassis_node; /* next node in ibnd_chassis_t->nodes */ - struct chassis *chassis; /* if != NULL the chassis this node belongs to */ + struct ibnd_node *next_chassis_node; /* next node in ibnd_chassis_t->nodes */ + struct ibnd_chassis *chassis; /* if != NULL the chassis this node belongs to */ unsigned char ch_type; unsigned char ch_anafanum; unsigned char ch_slotnum; @@ -83,12 +83,12 @@ typedef struct node { /** ========================================================================= * Port */ -typedef struct port { +typedef struct ibnd_port { uint64_t guid; int portnum; int ext_portnum; /* optional if != 0 external port num */ ibnd_node_t *node; /* node this port belongs to */ - struct port *remoteport; /* null if SMA, or does not exist */ + struct ibnd_port *remoteport; /* null if SMA, or does not exist */ /* quick cache of info below */ uint16_t base_lid; uint8_t lmc; @@ -100,8 +100,8 @@ typedef struct port { /** ========================================================================= * Chassis */ -typedef struct chassis { - struct chassis *next; +typedef struct ibnd_chassis { + struct ibnd_chassis *next; uint64_t chassisguid; unsigned char chassisnum; @@ -120,7 +120,7 @@ typedef struct chassis { * Fabric * Main fabric object which is returned and represents the data discovered */ -typedef struct ib_fabric { +typedef struct ibnd_fabric { struct ibmad_port *ibmad_port; /* the node the discover was initiated from * "from" parameter in ibnd_discover_fabric -- 1.5.4.5 From chu11 at llnl.gov Tue Jul 7 15:25:02 2009 From: chu11 at llnl.gov (Al Chu) Date: Tue, 07 Jul 2009 15:25:02 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [4/5] libibnetdisc cleanup patches Message-ID: <1247005503.4730.42.camel@auk31.llnl.gov> In libibnetdisc, do not automatically output messages to stderr/stdout for warnings or non-fatal errors. Use IBND_DEBUG instead of IBPANIC or IBWARN to allow users of the lib to decide output conditions. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- >From b33e01e4ab71e82265011b1941052584774c9b4b Mon Sep 17 00:00:00 2001 From: Albert Chu Date: Tue, 7 Jul 2009 11:31:56 -0700 Subject: [PATCH] In libibnetdisc, do not automatically output messages to stderr/stdout for warnings or non-fatal errors. Use IBND_DEBUG instead of IBPANIC or IBWARN to allow users of the lib to decide output conditions. Signed-off-by: Albert Chu Signed-off-by: Ira Weiny --- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index a10dfcd..d8bf200 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -464,8 +464,8 @@ get_remote_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *node, stru return -1; if (query_node(fabric, &node_buf, &port_buf, path)) { - IBWARN("NodeInfo on %s failed, skipping port", - portid2str(path)); + IBND_DEBUG("NodeInfo on %s failed, skipping port", + portid2str(path)); path->drpath.cnt--; /* restore path */ return -1; } @@ -507,15 +507,15 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, int max_hops = MAXHOPS-1; /* default find everything */ if (!ibmad_port) { - IBPANIC("ibmad_port must be specified to " - "ibnd_discover_fabric\n"); + IBND_DEBUG("ibmad_port must be specified to " + "ibnd_discover_fabric\n"); return (NULL); } if (mad_rpc_class_agent(ibmad_port, IB_SMI_CLASS) == -1 || mad_rpc_class_agent(ibmad_port, IB_SMI_DIRECT_CLASS) == -1) { - IBPANIC("ibmad_port must be opened with " - "IB_SMI_CLASS && IB_SMI_DIRECT_CLASS\n"); + IBND_DEBUG("ibmad_port must be opened with " + "IB_SMI_CLASS && IB_SMI_DIRECT_CLASS\n"); return (NULL); } @@ -545,7 +545,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, memset(&port_buf, 0, sizeof(port_buf)); if (query_node(fabric, &node_buf, &port_buf, from)) { - IBWARN("can't reach node %s\n", portid2str(from)); + IBND_DEBUG("can't reach node %s\n", portid2str(from)); goto error; } @@ -579,7 +579,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, continue; if (get_port_info(fabric, &port_buf, i, path)) { - IBWARN("can't reach node %s port %d", portid2str(path), i); + IBND_DEBUG("can't reach node %s port %d", portid2str(path), i); continue; } -- 1.5.4.5 From chu11 at llnl.gov Tue Jul 7 15:25:11 2009 From: chu11 at llnl.gov (Al Chu) Date: Tue, 07 Jul 2009 15:25:11 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [5/5] libibnetdisc cleanup patches Message-ID: <1247005511.4730.43.camel@auk31.llnl.gov> Use IBPANIC consistently in libibnetdisc, in particular, since IBPANIC calls exit, there's no use in returning a value after an error. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- >From ee8a77441551a94890fd6c553bb608510579abff Mon Sep 17 00:00:00 2001 From: Albert Chu Date: Tue, 7 Jul 2009 12:59:22 -0700 Subject: [PATCH] Use IBPANIC consistently in libibnetdisc, in particular, since IBPANIC calls exit, there's no use in returning a value after an error. Signed-off-by: Albert Chu Signed-off-by: Ira Weiny --- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 8 ++------ 1 files changed, 2 insertions(+), 6 deletions(-) diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index d8bf200..ffd8602 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -369,10 +369,8 @@ create_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *temp, ib_porti struct ibnd_node_int *node; node = malloc(sizeof(*node)); - if (!node) { + if (!node) IBPANIC("OOM: node creation failed\n"); - return NULL; - } memcpy(node, temp, sizeof(*node)); node->node.dist = dist; @@ -530,10 +528,8 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, fabric = malloc(sizeof(*fabric)); - if (!fabric) { + if (!fabric) IBPANIC("OOM: failed to malloc ibnd_fabric_t\n"); - return (NULL); - } memset(fabric, 0, sizeof(*fabric)); -- 1.5.4.5 From jgunthorpe at obsidianresearch.com Tue Jul 7 15:38:30 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Tue, 7 Jul 2009 16:38:30 -0600 Subject: [ofa-general] [infiniband-diags] [PATCH] [5/5] libibnetdisc cleanup patches In-Reply-To: <1247005511.4730.43.camel@auk31.llnl.gov> References: <1247005511.4730.43.camel@auk31.llnl.gov> Message-ID: <20090707223830.GL12693@obsidianresearch.com> On Tue, Jul 07, 2009 at 03:25:11PM -0700, Al Chu wrote: > Use IBPANIC consistently in libibnetdisc, in particular, since IBPANIC > calls exit, there's no use in returning a value after an error. Calling abort/exit from within a general use lib on error is quite unfriendly as well.. Jason From weiny2 at llnl.gov Tue Jul 7 15:43:13 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Tue, 7 Jul 2009 15:43:13 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [5/5] libibnetdisc cleanup patches In-Reply-To: <20090707223830.GL12693@obsidianresearch.com> References: <1247005511.4730.43.camel@auk31.llnl.gov> <20090707223830.GL12693@obsidianresearch.com> Message-ID: <20090707154313.86316b44.weiny2@llnl.gov> Yea we know. We are working on it... ;-) Ira On Tue, 7 Jul 2009 16:38:30 -0600 Jason Gunthorpe wrote: > On Tue, Jul 07, 2009 at 03:25:11PM -0700, Al Chu wrote: > > Use IBPANIC consistently in libibnetdisc, in particular, since IBPANIC > > calls exit, there's no use in returning a value after an error. > > Calling abort/exit from within a general use lib on error is quite > unfriendly as well.. > > Jason > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://*lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://*openib.org/mailman/listinfo/openib-general > -- Ira Weiny Math Programmer/Computer Scientist Lawrence Livermore National Lab weiny2 at llnl.gov From akepner at sgi.com Tue Jul 7 16:22:43 2009 From: akepner at sgi.com (akepner at sgi.com) Date: Tue, 7 Jul 2009 16:22:43 -0700 Subject: [ofa-general] [PATCH] ibutils: ibdiagnet -r "Dead end" errors Message-ID: <20090707232243.GJ15871@sgi.com> On a cluster running sles11 and OFED 1.4, we recently started seeing errors like this: # ibdiagnet -r ..... -I- -I- Verifying all CA to CA paths ... -E- Unassigned LFT for lid:1 Dead end at:S0800690000004057/U1 -E- Fail to find a path from:r1i0n9/U1/1 to:r1lead/U1/1 ... But the forwarding tables (obtained with dump_lfts.sh, and smpdump) are correct. The problem turned out to be that the string "-lft" was being interpreted as a port number, resulting in an off-by-one error. The following fixed it for us. Signed-off-by: Arthur Kepner --- ibdebug.tcl | 1 + 1 files changed, 1 insertion(+) diff -rup a/ibutils-1.2/ibdiag/src/ibdebug.tcl b/ibutils-1.2/ibdiag/src/ibdebug.tcl --- a/ibutils-1.2/ibdiag/src/ibdebug.tcl 2009-07-07 14:25:57.604185193 -0700 +++ b/ibutils-1.2/ibdiag/src/ibdebug.tcl 2009-07-07 14:26:35.694551424 -0700 @@ -5130,6 +5130,7 @@ proc writeFdbsFile { args } { [SmMadGetByDr LftBlock dump "$DirectPath" $I] }] { set FDBs [concat $FDBs [Bar "0xff " 64]] } else { + set NewFDBs [RemoveElementFromList $NewFDBs "-lft"] set FDBs [concat $FDBs $NewFDBs] } } From rdreier at cisco.com Tue Jul 7 21:14:59 2009 From: rdreier at cisco.com (Roland Dreier) Date: Tue, 07 Jul 2009 21:14:59 -0700 Subject: [ofa-general] Re: [PATCH 1/3] V2 - libmthca - Optimize memory allocation of QP buffers In-Reply-To: <20090528102249.2ca01866@frecb007965> (sebastien dugue's message of "Thu, 28 May 2009 10:22:49 +0200") References: <20090528102059.2fd85540@frecb007965> <20090528102249.2ca01866@frecb007965> Message-ID: So I'm finally looking at applying this patch. And I wonder why you only do this for QP buffers -- it seems every other object allocated with mthca_alloc_buf(), namely CQs, SRQs and mem-free doorbell records, are all also page aligned, and similarly waste memory. So should we not just replace the posix_memalign() in alloc_buf with anonymous mmap, and end up with a simpler patch that saves even more memory? - R. From rdreier at cisco.com Tue Jul 7 21:22:34 2009 From: rdreier at cisco.com (Roland Dreier) Date: Tue, 07 Jul 2009 21:22:34 -0700 Subject: [ofa-general] [PATCH v2 RESEND] rdma_cm: Add debugfs entries to monitor rdma_cm connections In-Reply-To: <4A50C13F.6070506@Voltaire.COM> (Moni Shoua's message of "Sun, 05 Jul 2009 18:05:35 +0300") References: <4A3E45D3.3040405@Voltaire.COM> <4A50C13F.6070506@Voltaire.COM> Message-ID: > It seems like a dead and and there won't be a consensus on how to implement this. > Without disrespecting Jason's opinion I still would like to see this patch get in. > What's the convention in such cases? In general when the maintainer (ie me) is not convinced about a patch and there is not a strong consensus from others then we don't merge the patch. And in this particular case I find myself agreeing more and more with Jason. The big issue seems to be your requirement that this debugging work with "cat" with no other tools needed; however sticking stuff in debugfs means also that "mount" as root is needed. And to be honest, putting hard-to-extend and hard-to-parse text blobs in debugfs does seem much less useful than a netlink interface (which is much easier to use programmatically and allows for extension), even if netlink requires someone to write a (simple) userspace tool to dump information. OFED gives you a short-term way to make sure that tool is available to your users, and longer-term integrating with existing tools will be easier if you use netlink. Really, is being able to dump with "cat" so important? Even if it is, I would still prefer to see the infrastructure for doing this properly through netlink go in first, and add the debugfs support on top of that. As an aside, I think adding ftrace events support to the cma might be useful for debugging as well. - R. From tziporet at dev.mellanox.co.il Tue Jul 7 23:14:35 2009 From: tziporet at dev.mellanox.co.il (Tziporet Koren) Date: Wed, 08 Jul 2009 09:14:35 +0300 Subject: [ofa-general] Re: [PATCH 1/3] V2 - libmthca - Optimize memory allocation of QP buffers In-Reply-To: References: <20090528102059.2fd85540@frecb007965> <20090528102249.2ca01866@frecb007965> Message-ID: <4A54394B.2060704@mellanox.co.il> Roland Dreier wrote: > So I'm finally looking at applying this patch. And I wonder why you > only do this for QP buffers -- it seems every other object allocated > with mthca_alloc_buf(), namely CQs, SRQs and mem-free doorbell records, > are all also page aligned, and similarly waste memory. > > So should we not just replace the posix_memalign() in alloc_buf with > anonymous mmap, and end up with a simpler patch that saves even more > memory? > > BTW - we tested these patches here and they are working and indeed save memory I think this idea is good. Tziporet From davem at systemfabricworks.com Wed Jul 8 00:18:01 2009 From: davem at systemfabricworks.com (David McMillen) Date: Wed, 8 Jul 2009 02:18:01 -0500 Subject: [ofa-general] Question on rdma_resolve_route and retries Message-ID: <5e3be0f30907080018p11376da8gd75ba9fd86f3acb1@mail.gmail.com> We are trying to use OpenMPI 1.3.2 with rdma_cm support on an Infiniband fabric using OFED 1.4.1. When the MPI jobs get large enough, the event response to rdma_resolve_route becomes RDMA_CM_EVENT_ROUTE_ERROR with a status of ETIMEDOUT. It seems pretty clear that the SA path record requests are being synchronized and bunching together, and in the end exhausting the resources of the subnet manager node so only the first N are actually received. The sequence seems to be: call librdmacm-1.0.8/src/cma.c's rdma_resolve_route which translates directly into a kernel call into infiniband/core/cma.c's rdma_resolve_route with an IB fabric becomes a call into cma_resolve_ib_route which leads to a call to cma_query_ib_route which gets to calling infiniband/core/sa_query.c's ib_sa_path_rec_get with the callback pointing to cma_query_handler When cma_query_handler gets a callaback with a bad status, it sets the returned event to RDMA_CM_EVENT_ROUTE_ERROR Nowhere in there do I see any retry attempts. If the SA path record query packet, or it's response packet, gets lost, then the timeout eventually happens and we see RDMA_CM_EVENT_ROUTE_ERROR with a status of ETIMEDOUT. First question: Did I miss a retry buried somewhere in all of that? Second question: How does somebody come up with a timeout value that makes sense? Assuming retries are the responsibility of the rdma_resolve_route caller, you would like to have a value that is long enough to avoid false timeouts when a response is eventually going to make it, but not any longer. This value seems like it would be dependent on the fabric and the capabilities of the node running the subnet manager, and should be a fabric-specific parameter instead of something chosen at random by each caller of rdma_resolve_route. There is probably some interesting discussion to have around the amount of time that the rdma_resolve_route caller should wait after the failure before retrying, so that time could be added to the base timeout and simplify the processing. This duration might also be different for each node and iteration of the retry in an attempt to avoid wave after wave of multiple requestors overwhelming the subnet manager. There is also the question of how many times this needs to be repeated before the rdma_resolve_route caller declares complete failure. Perhaps this is also a fabric-specific parameter? Finally, is there some way to tune the subnet manager node so that the number of requests that can be captured and processed is maximized? Thanks for any help or ideas. Dave McMillen -------------- next part -------------- An HTML attachment was scrubbed... URL: From sebastien.dugue at bull.net Wed Jul 8 00:19:41 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Wed, 8 Jul 2009 09:19:41 +0200 Subject: [ofa-general] Re: [PATCH 1/3] V2 - libmthca - Optimize memory allocation of QP buffers In-Reply-To: References: <20090528102059.2fd85540@frecb007965> <20090528102249.2ca01866@frecb007965> Message-ID: <20090708091941.17d21098@frecb007965> Hi Roland, On Tue, 07 Jul 2009 21:14:59 -0700 Roland Dreier wrote: > So I'm finally looking at applying this patch. And I wonder why you > only do this for QP buffers -- it seems every other object allocated > with mthca_alloc_buf(), namely CQs, SRQs and mem-free doorbell records, > are all also page aligned, and similarly waste memory. You're right here that all alignd buffers could be allocated with mmap(). As I didn't want to be too radical, I only changed the QP buffers as those are the most offending consumers of aligned memory. If everyone agrees on dropping posix_memalign() for good (which I think would be a good thing), then I'll respin those 2 patches. Sebastien. > > So should we not just replace the posix_memalign() in alloc_buf with > anonymous mmap, and end up with a simpler patch that saves even more > memory? > > - R. > From sean.hefty at intel.com Wed Jul 8 00:37:26 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Wed, 8 Jul 2009 00:37:26 -0700 Subject: [ofa-general] RE: Question on rdma_resolve_route and retries In-Reply-To: <5e3be0f30907080018p11376da8gd75ba9fd86f3acb1@mail.gmail.com> References: <5e3be0f30907080018p11376da8gd75ba9fd86f3acb1@mail.gmail.com> Message-ID: <7D7BD0783D5C47F19C315EE6EBBD6383@amr.corp.intel.com> >We are trying to use OpenMPI 1.3.2 with rdma_cm support on an Infiniband fabric >using OFED 1.4.1. When the MPI jobs get large enough, the event response to >rdma_resolve_route becomes RDMA_CM_EVENT_ROUTE_ERROR with a status of >ETIMEDOUT. Yep - you pretty much need to connect out of band with all large MPI jobs using made up path data, or enable some sort of PR caching. >It seems pretty clear that the SA path record requests are being synchronized >and bunching together, and in the end exhausting the resources of the subnet >manager node so only the first N are actually received. In our testing, we discovered that the SA almost never dropped any queries. The problem was that the backlog grew so huge, that all requests had timed out before they could be acted on. There's probably something that could be done here to avoid storing received MADs for extended periods of time. >The sequence seems to be: > >call librdmacm-1.0.8/src/cma.c's rdma_resolve_route > >which translates directly into a kernel call into infiniband/core/cma.c's >rdma_resolve_route > >with an IB fabric becomes a call into cma_resolve_ib_route > >which leads to a call to cma_query_ib_route > >which gets to calling infiniband/core/sa_query.c's ib_sa_path_rec_get with the >callback pointing to cma_query_handler > >When cma_query_handler gets a callaback with a bad status, it sets the returned >event to RDMA_CM_EVENT_ROUTE_ERROR > >Nowhere in there do I see any retry attempts. If the SA path record query >packet, or it's response packet, gets lost, then the timeout eventually happens >and we see RDMA_CM_EVENT_ROUTE_ERROR with a status of ETIMEDOUT. The kernel sa_query module does not issue retries. All retries are the responsibility of the caller. This gives greater flexibility to how timeouts are handled, but has the drawback that all 'retries' are really new transactions. >First question: Did I miss a retry buried somewhere in all of that? I don't believe so. >Second question: How does somebody come up with a timeout value that makes >sense? Assuming retries are the responsibility of the rdma_resolve_route >caller, you would like to have a value that is long enough to avoid false >timeouts when a response is eventually going to make it, but not any longer. >This value seems like it would be dependent on the fabric and the capabilities >of the node running the subnet manager, and should be a fabric-specific >parameter instead of something chosen at random by each caller of >rdma_resolve_route. The timeout is also dependent on the load hitting the SA. I don't know that a fabric-specific parameter can work. - Sean From davem at systemfabricworks.com Wed Jul 8 00:57:42 2009 From: davem at systemfabricworks.com (David McMillen) Date: Wed, 8 Jul 2009 02:57:42 -0500 Subject: [ofa-general] Re: Question on rdma_resolve_route and retries In-Reply-To: <7D7BD0783D5C47F19C315EE6EBBD6383@amr.corp.intel.com> References: <5e3be0f30907080018p11376da8gd75ba9fd86f3acb1@mail.gmail.com> <7D7BD0783D5C47F19C315EE6EBBD6383@amr.corp.intel.com> Message-ID: <5e3be0f30907080057y2406f0c0nca2fb84dbe069676@mail.gmail.com> Thanks for the information -- I have some follow-on inline below. On Wed, Jul 8, 2009 at 2:37 AM, Sean Hefty wrote: > >We are trying to use OpenMPI 1.3.2 with rdma_cm support on an Infiniband > fabric > >using OFED 1.4.1. When the MPI jobs get large enough, the event response > to > >rdma_resolve_route becomes RDMA_CM_EVENT_ROUTE_ERROR with a status of > >ETIMEDOUT. > > Yep - you pretty much need to connect out of band with all large MPI jobs > using > made up path data, or enable some sort of PR caching. I should have mentioned that the fabric is a large torus using LASH routing, and we need to get the live SL value to make deadlock-free connections. We are definitely thinking about PR caching, but that raises issues about how to manage the life of the cache entries. > > > >It seems pretty clear that the SA path record requests are being > synchronized > >and bunching together, and in the end exhausting the resources of the > subnet > >manager node so only the first N are actually received. > > In our testing, we discovered that the SA almost never dropped any queries. > The > problem was that the backlog grew so huge, that all requests had timed out > before they could be acted on. There's probably something that could be > done > here to avoid storing received MADs for extended periods of time. This is encouraging. I did try testing with 10,000 ms timeouts and still got the failure with only 800 different processes, so I jumped to the conclusion that the queries were being dropped. Do you have a guess as to a timeout value that would always succeed? Also, your testing suggests that the receive queue almost never gets exhausted. At least as I understand things, if the queue ends up empty then the HCA can dump packets at great speed. How does the system cope with a potential stream of requests arriving less than half a microsecond apart? (I should have mentioned that the fabric is QDR.) I guess this is another way of asking my question about how to maximize the ability of the subnet manager node to accept requests. > > > >The sequence seems to be: > > > >call librdmacm-1.0.8/src/cma.c's rdma_resolve_route > > > >which translates directly into a kernel call into infiniband/core/cma.c's > >rdma_resolve_route > > > >with an IB fabric becomes a call into cma_resolve_ib_route > > > >which leads to a call to cma_query_ib_route > > > >which gets to calling infiniband/core/sa_query.c's ib_sa_path_rec_get with > the > >callback pointing to cma_query_handler > > > >When cma_query_handler gets a callaback with a bad status, it sets the > returned > >event to RDMA_CM_EVENT_ROUTE_ERROR > > > >Nowhere in there do I see any retry attempts. If the SA path record query > >packet, or it's response packet, gets lost, then the timeout eventually > happens > >and we see RDMA_CM_EVENT_ROUTE_ERROR with a status of ETIMEDOUT. > > The kernel sa_query module does not issue retries. All retries are the > responsibility of the caller. This gives greater flexibility to how > timeouts > are handled, but has the drawback that all 'retries' are really new > transactions. > > >First question: Did I miss a retry buried somewhere in all of that? > > I don't believe so. Thanks for the confirmation. There have been several people telling me that it is in there, and I couldn't find it. > > > >Second question: How does somebody come up with a timeout value that makes > >sense? Assuming retries are the responsibility of the rdma_resolve_route > >caller, you would like to have a value that is long enough to avoid false > >timeouts when a response is eventually going to make it, but not any > longer. > >This value seems like it would be dependent on the fabric and the > capabilities > >of the node running the subnet manager, and should be a fabric-specific > >parameter instead of something chosen at random by each caller of > >rdma_resolve_route. > > The timeout is also dependent on the load hitting the SA. I don't know > that a > fabric-specific parameter can work. Maybe I should have come up with a better name. By fabric-specific, I meant a specific implentation of the fabric, including the capability of the subnet manager node. How does somebody writing rdma_cm code come up with a number? That particular program might not put much of a load on the SA, but could run concurrently with other jobs that do (or don't). It would be nice to have a way to set up the retry mechanism so that it would work on any system it ran on. > > > - Sean > -------------- next part -------------- An HTML attachment was scrubbed... URL: From amirv at mellanox.co.il Wed Jul 8 01:12:15 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Wed, 08 Jul 2009 11:12:15 +0300 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <20090707171326.GH8065@barkeeper1-xen.linbit> References: <20090701133652.GG9112@soda.linbit> <4A4B947C.1030607@mellanox.co.il> <20090702081445.GA9118@soda.linbit> <4A4C7021.2090506@mellanox.co.il> <20090702132304.GD9305@soda.linbit> <20090707171326.GH8065@barkeeper1-xen.linbit> Message-ID: <4A5454DF.1070101@mellanox.co.il> Lars Hi, I opened a bug in our bugzilla (https://bugs.openfabrics.org/show_bug.cgi?id=1672). I couldn't reproduce it on my setup: SLES 10SP2, stock kernel, same ofed git version. will try now to install 2.6.27 kernel and check again. BTW, what type of servers do you use? Are they low/high end server? - Amir On 07/07/2009 08:13 PM, Lars Ellenberg wrote: > this is on debian lenny, > userland ofed from http://pkg-ofed.alioth.debian.org/apt/ofed/ > > kernel git://git.openfabrics.org/ofed_1_4/linux-2.6.git > merged with upstream stable > git://git4.kernel.org/pub/scm/linux/kernel/git/stable/linux-2.6.27.y.git > > both as of today: > ofed_kernel 08acda8 sdp: Fix memory leak in bzcopy > linux-v2.6.27.y/master 49cbf40 Linux 2.6.27.26 > > > kernel config: very many "kernel debugging" things enabled. > if you want me to try a certain .config, or anything, > this can be arranged. > > two very ugly perl scripts attached, > one tcp server, > one tcp client, > adapted from the perlipc man page. > > client connects, > sends a package, > receives a package > in an endless loop. > > package format: > 4 byte magic, 2 byte ignored, 2 byte payload length > indicated length of payload, all same bytes, > but the trailing 4 byte, which again is a magic number. > > with ethernet, or IPoIB: runs endless. > > with LD_PRELOAD=libsdp.so runs for very few iterations, > and errors out on one of the sanity checks. > > > sample output: > root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 > 2009-07-07 19:07:08 my_client.pl 4300: recv hdr [25]: invalid magic: 32 30 30 39 2d 30 37 2d > > which hapeens to be the hexdump of the string "2009-07-". > where did it copy_user() that from? wtf? > > root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 > 2009-07-07 19:07:08 my_client.pl 4301: recv payload [35]: expected 12296, but received 12295 byte; last bytes received: 55 e4 e3 e2 > ^^^^ pid, ^^ seq number. > so for only 35 ping/pongs it did work ok. > > exactly: one byte too short. > the trailing magic expected is e4 e3 e1 e1 > > root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 > 2009-07-07 19:07:09 my_client.pl 4302: recv payload [33]: expected 4131, but received 4130 byte; last bytes received: 55 e4 e3 e2 > > root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 > 2009-07-07 19:07:10 my_client.pl 4303: recv payload [29]: expected 16401, but received 16400 byte; last bytes received: 55 e4 e3 e2 > > root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 > 2009-07-07 19:07:12 my_client.pl 4304: recv payload [21]: expected 4110, but received 4109 byte; last bytes received: 55 e4 e3 e2 > > root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 > 2009-07-07 19:07:13 my_client.pl 4305: recv payload [4]: expected 20495, but received 20494 byte; last bytes received: 55 e4 e3 e2 > > root at kugel:/home/lars/DRBD/IB_SDP# LD_PRELOAD=libsdp.so perl my_client.pl rum-ib0 > 2009-07-07 19:07:14 my_client.pl 4306: recv payload [12]: expected 22530, but received 22529 byte; last bytes received: 55 e4 e3 e2 > > > any suggestions how to proceed from here? > -- Amir Vadai Software Eng. Mellanox Technologies mailto: amirv at mellanox.co.il Tel +972-3-6259539 From vlad at lists.openfabrics.org Wed Jul 8 02:34:39 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Wed, 8 Jul 2009 02:34:39 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090708-0200 daily build status Message-ID: <20090708093439.4CEF5102032A@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -Werror-implicit-function-declaration -fno-strict-aliasing -fno-common -ffreestanding -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1411: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1430: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1411: error: 'struct ib_device' has no member named 'dev' /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c: In function 'iwch_unregister_device': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.c:1430: error: 'struct ib_device' has no member named 'dev' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3/iwch_provider.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/hw/cxgb3] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090708-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From ogerlitz at Voltaire.com Wed Jul 8 02:56:40 2009 From: ogerlitz at Voltaire.com (Or Gerlitz) Date: Wed, 08 Jul 2009 12:56:40 +0300 Subject: [ofa-general] Re: [PATCH] rdma_cm: Add debugfs entries to monitor rdma_cm connections In-Reply-To: <20090707164803.GR20745@obsidianresearch.com> References: <4A3E45D3.3040405@Voltaire.COM> <4A50C13F.6070506@Voltaire.COM> <4A520600.2080004@opengridcomputing.com> <4A531FA6.6050803@voltaire.com> <20090707164803.GR20745@obsidianresearch.com> Message-ID: <4A546D58.6090707@Voltaire.com> Jason Gunthorpe wrote: > Well, thats the rub isn't it? debugfs is not 'production ready' (by > definition) so why spend time on it? to allow debugging, diagnosing problems > Why the resistance to doing a proper job and solving the actual problem? I didn't see resistance to do a better job, > debufs is entirely the wrong answer - this is not kernel debugging, > this is required user space diagnostic information. said who? how are you suggesting me to know what kernel iser/lustre/rds/rnfs RDMA connections are open now? Or. From lars.ellenberg at linbit.com Wed Jul 8 03:17:15 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Wed, 8 Jul 2009 12:17:15 +0200 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <4A5454DF.1070101@mellanox.co.il> References: <20090701133652.GG9112@soda.linbit> <4A4B947C.1030607@mellanox.co.il> <20090702081445.GA9118@soda.linbit> <4A4C7021.2090506@mellanox.co.il> <20090702132304.GD9305@soda.linbit> <20090707171326.GH8065@barkeeper1-xen.linbit> <4A5454DF.1070101@mellanox.co.il> Message-ID: <20090708101715.GB8083@barkeeper1-xen.linbit> On Wed, Jul 08, 2009 at 11:12:15AM +0300, Amir Vadai wrote: > Lars Hi, > > I opened a bug in our bugzilla (https://bugs.openfabrics.org/show_bug.cgi?id=1672). > > I couldn't reproduce it on my setup: SLES 10SP2, stock kernel, same ofed git version. > will try now to install 2.6.27 kernel and check again. With a "normal" kernel config, I needed to do full load bi-directional network traffic on IPoIB as well as SDP, multiple stream sockets, to eventually actually trigger it after a few minutes (several hundered MegaByte per second). with the "debug" kernel config, I was able to reproduce with only one socket, within milliseconds. my .config is attached. > BTW, what type of servers do you use? Are they low/high end server? This is the second cluster that show this bug. I first experienced it when using SDP sockets from within kernel space. I was able to reproduce in userland, which I thought might make it easier for you to reproduce. The current test cluster is a slightly aged 2U supermicro dual quadcore, 4GB ram, and proved to be very reliable hardware in all test up to now. it may be a little slow on interrupts. tail of /proc/cpuinfo: processor : 7 vendor_id : GenuineIntel cpu family : 6 model : 15 model name : Intel(R) Xeon(R) CPU E5310 @ 1.60GHz stepping : 7 cpu MHz : 1599.984 cache size : 4096 KB physical id : 1 siblings : 4 core id : 3 cpu cores : 4 apicid : 7 initial apicid : 7 fpu : yes fpu_exception : yes cpuid level : 10 wp : yes flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx lm constant_tsc arch_perfmon pebs bts rep_good nopl pni monitor ds_cpl vmx tm2 ssse3 cx16 xtpr dca lahf_lm bogomips : 3201.35 clflush size : 64 cache_alignment : 64 address sizes : 36 bits physical, 48 bits virtual power management: the IB setup is direct link, lspci says: 09:00.0 InfiniBand: Mellanox Technologies MT26428 [ConnectX IB QDR, PCIe 2.0 5GT/s] (rev a0) because using IPoIB does work just fine, I don't think we have issues with IB setup, or the hardware in general. only when using SDP it is broken, "forgets" bytes, or corrupts data. what I do "different" than the (assumed to be) typical SDP user is: sending large-ish messages at once (up to ~32 kB), possibly unaligned. which apparently is a mode that SDP has not excercised much yet, otherwise the recently fixed page leak would have been noticed by someone much earlier. -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. -------------- next part -------------- A non-text attachment was scrubbed... Name: config-ofed-1.4-d02.gz Type: application/x-gzip Size: 13700 bytes Desc: not available URL: From amirv at mellanox.co.il Wed Jul 8 03:33:26 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Wed, 08 Jul 2009 13:33:26 +0300 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <20090708101715.GB8083@barkeeper1-xen.linbit> References: <20090701133652.GG9112@soda.linbit> <4A4B947C.1030607@mellanox.co.il> <20090702081445.GA9118@soda.linbit> <4A4C7021.2090506@mellanox.co.il> <20090702132304.GD9305@soda.linbit> <20090707171326.GH8065@barkeeper1-xen.linbit> <4A5454DF.1070101@mellanox.co.il> <20090708101715.GB8083@barkeeper1-xen.linbit> Message-ID: <4A5475F6.1060309@mellanox.co.il> see below On 07/08/2009 01:17 PM, Lars Ellenberg wrote: > On Wed, Jul 08, 2009 at 11:12:15AM +0300, Amir Vadai wrote: > >> Lars Hi, >> >> I opened a bug in our bugzilla (https://bugs.openfabrics.org/show_bug.cgi?id=1672). >> >> I couldn't reproduce it on my setup: SLES 10SP2, stock kernel, same ofed git version. >> will try now to install 2.6.27 kernel and check again. >> > With a "normal" kernel config, I needed to do full load bi-directional > network traffic on IPoIB as well as SDP, multiple stream sockets, > to eventually actually trigger it after a few minutes > (several hundered MegaByte per second). > > with the "debug" kernel config, > I was able to reproduce with only one socket, > within milliseconds. > > my .config is attached. > I will test it with your config and kernel version > >> BTW, what type of servers do you use? Are they low/high end server? >> > This is the second cluster that show this bug. I first experienced it > when using SDP sockets from within kernel space. > I was able to reproduce in userland, > which I thought might make it easier for you to reproduce. > > The current test cluster is a slightly aged 2U supermicro dual quadcore, > 4GB ram, and proved to be very reliable hardware in all test up to now. > it may be a little slow on interrupts. > > tail of /proc/cpuinfo: > processor : 7 > vendor_id : GenuineIntel > cpu family : 6 > model : 15 > model name : Intel(R) Xeon(R) CPU E5310 @ 1.60GHz > stepping : 7 > cpu MHz : 1599.984 > cache size : 4096 KB > physical id : 1 > siblings : 4 > core id : 3 > cpu cores : 4 > apicid : 7 > initial apicid : 7 > fpu : yes > fpu_exception : yes > cpuid level : 10 > wp : yes > flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge > mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe > syscall nx lm constant_tsc arch_perfmon pebs bts rep_good nopl pni > monitor ds_cpl vmx tm2 ssse3 cx16 xtpr dca lahf_lm > bogomips : 3201.35 > clflush size : 64 > cache_alignment : 64 > address sizes : 36 bits physical, 48 bits virtual > power management: > > the IB setup is direct link, lspci says: > 09:00.0 InfiniBand: Mellanox Technologies MT26428 [ConnectX IB QDR, PCIe 2.0 5GT/s] (rev a0) > > > because using IPoIB does work just fine, I don't think we have issues > with IB setup, or the hardware in general. > only when using SDP it is broken, "forgets" bytes, or corrupts data. > I'm testing on SDR low end machines - and sometimes we have bugs that we only see on high bandwidth setups. And you have such a setup. > what I do "different" than the (assumed to be) typical SDP user is: > sending large-ish messages at once (up to ~32 kB), possibly unaligned. > > which apparently is a mode that SDP has not excercised much yet, > otherwise the recently fixed page leak would have been noticed by > someone much earlier. > > - Amir From jackm at dev.mellanox.co.il Wed Jul 8 03:50:12 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Wed, 8 Jul 2009 13:50:12 +0300 Subject: [ofa-general] [PATCH V2] mlx4: check for FW version which properly supports resize_cq Message-ID: <200907081350.12462.jackm@dev.mellanox.co.il> If a ConnectX card has a FW version installed which does not support resize cq, the resize_cq command will return -ENOSYS. Fixes Bugzilla 1415. Signed-off-by: Jack Morgenstein --- Roland, I submitted this on 2008-12-03, and somehow it fell through the cracks. I've regenerated it for your for-next branch for kernel 2.6.31. We already do something similar for qp's (NoErrorCompletionBit). (V2 -- put the #define MLX4_FW_VER_RESIZE_CQ in file cq.h, not in cq.c). -Jack diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index de5263b..660c99a 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -349,6 +349,9 @@ int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) int outst_cqe; int err; + if (dev->dev->caps.fw_ver < MLX4_FW_VER_RESIZE_CQ) + return -ENOSYS; + mutex_lock(&cq->resize_mutex); if (entries < 1 || entries > dev->dev->caps.max_cqes) { diff --git a/include/linux/mlx4/cq.h b/include/linux/mlx4/cq.h index 6f65b2c..af6740d 100644 --- a/include/linux/mlx4/cq.h +++ b/include/linux/mlx4/cq.h @@ -64,6 +64,9 @@ struct mlx4_err_cqe { u8 owner_sr_opcode; }; +/* Which firmware version adds support for Resize CQ */ +#define MLX4_FW_VER_RESIZE_CQ mlx4_fw_ver(2, 5, 0) + enum { MLX4_CQE_VLAN_PRESENT_MASK = 1 << 29, MLX4_CQE_QPN_MASK = 0xffffff, From jackm at dev.mellanox.co.il Wed Jul 8 04:32:56 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Wed, 8 Jul 2009 14:32:56 +0300 Subject: [ofa-general] [PATCH] mlx4: print out returned raw FW command status if a non-zero status is returned Message-ID: <200907081432.56360.jackm@dev.mellanox.co.il> The returned FW raw command status is invaluable in troubleshooting, and if a FW command error status is returned, we need to be able to see it (along with the command which caused the non-zero status). Signed-off-by: Jack Morgenstein diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c index 65ec77d..571cb07 100644 --- a/drivers/net/mlx4/cmd.c +++ b/drivers/net/mlx4/cmd.c @@ -108,6 +108,7 @@ struct mlx4_cmd_context { int next; u64 out_param; u16 token; + u8 fw_status; }; static int mlx4_status_to_errno(u8 status) @@ -215,6 +216,7 @@ static int mlx4_cmd_poll(struct mlx4_dev *dev, u64 in_param, u64 *out_param, void __iomem *hcr = priv->cmd.hcr; int err = 0; unsigned long end; + u32 stat; down(&priv->cmd.poll_sem); @@ -238,9 +240,10 @@ static int mlx4_cmd_poll(struct mlx4_dev *dev, u64 in_param, u64 *out_param, __raw_readl(hcr + HCR_OUT_PARAM_OFFSET)) << 32 | (u64) be32_to_cpu((__force __be32) __raw_readl(hcr + HCR_OUT_PARAM_OFFSET + 4)); - - err = mlx4_status_to_errno(be32_to_cpu((__force __be32) - __raw_readl(hcr + HCR_STATUS_OFFSET)) >> 24); + stat = be32_to_cpu((__force __be32) __raw_readl(hcr + HCR_STATUS_OFFSET)) >> 24; + err = mlx4_status_to_errno(stat); + if (err) + mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", op, stat); out: up(&priv->cmd.poll_sem); @@ -257,6 +260,7 @@ void mlx4_cmd_event(struct mlx4_dev *dev, u16 token, u8 status, u64 out_param) if (token != context->token) return; + context->fw_status = status; context->result = mlx4_status_to_errno(status); context->out_param = out_param; @@ -291,8 +295,11 @@ static int mlx4_cmd_wait(struct mlx4_dev *dev, u64 in_param, u64 *out_param, } err = context->result; - if (err) + if (err) { + mlx4_err(dev, "command 0x%x failed: fw status = 0x%x\n", + op, context->fw_status); goto out; + } if (out_is_imm) *out_param = context->out_param; From amirv at mellanox.co.il Wed Jul 8 05:26:18 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Wed, 08 Jul 2009 15:26:18 +0300 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <4A5475F6.1060309@mellanox.co.il> References: <20090701133652.GG9112@soda.linbit> <4A4B947C.1030607@mellanox.co.il> <20090702081445.GA9118@soda.linbit> <4A4C7021.2090506@mellanox.co.il> <20090702132304.GD9305@soda.linbit> <20090707171326.GH8065@barkeeper1-xen.linbit> <4A5454DF.1070101@mellanox.co.il> <20090708101715.GB8083@barkeeper1-xen.linbit> <4A5475F6.1060309@mellanox.co.il> Message-ID: <4A54906A.7090801@mellanox.co.il> ok - bug is reproduced now on your kernel+config will check it now. On 07/08/2009 01:33 PM, Amir Vadai wrote: > see below > > On 07/08/2009 01:17 PM, Lars Ellenberg wrote: > >> On Wed, Jul 08, 2009 at 11:12:15AM +0300, Amir Vadai wrote: >> >> >>> Lars Hi, >>> >>> I opened a bug in our bugzilla (https://bugs.openfabrics.org/show_bug.cgi?id=1672). >>> >>> I couldn't reproduce it on my setup: SLES 10SP2, stock kernel, same ofed git version. >>> will try now to install 2.6.27 kernel and check again. >>> >>> >> With a "normal" kernel config, I needed to do full load bi-directional >> network traffic on IPoIB as well as SDP, multiple stream sockets, >> to eventually actually trigger it after a few minutes >> (several hundered MegaByte per second). >> >> with the "debug" kernel config, >> I was able to reproduce with only one socket, >> within milliseconds. >> >> my .config is attached. >> >> > I will test it with your config and kernel version > >> >> >>> BTW, what type of servers do you use? Are they low/high end server? >>> >>> >> This is the second cluster that show this bug. I first experienced it >> when using SDP sockets from within kernel space. >> I was able to reproduce in userland, >> which I thought might make it easier for you to reproduce. >> >> The current test cluster is a slightly aged 2U supermicro dual quadcore, >> 4GB ram, and proved to be very reliable hardware in all test up to now. >> it may be a little slow on interrupts. >> >> tail of /proc/cpuinfo: >> processor : 7 >> vendor_id : GenuineIntel >> cpu family : 6 >> model : 15 >> model name : Intel(R) Xeon(R) CPU E5310 @ 1.60GHz >> stepping : 7 >> cpu MHz : 1599.984 >> cache size : 4096 KB >> physical id : 1 >> siblings : 4 >> core id : 3 >> cpu cores : 4 >> apicid : 7 >> initial apicid : 7 >> fpu : yes >> fpu_exception : yes >> cpuid level : 10 >> wp : yes >> flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge >> mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe >> syscall nx lm constant_tsc arch_perfmon pebs bts rep_good nopl pni >> monitor ds_cpl vmx tm2 ssse3 cx16 xtpr dca lahf_lm >> bogomips : 3201.35 >> clflush size : 64 >> cache_alignment : 64 >> address sizes : 36 bits physical, 48 bits virtual >> power management: >> >> the IB setup is direct link, lspci says: >> 09:00.0 InfiniBand: Mellanox Technologies MT26428 [ConnectX IB QDR, PCIe 2.0 5GT/s] (rev a0) >> >> >> because using IPoIB does work just fine, I don't think we have issues >> with IB setup, or the hardware in general. >> only when using SDP it is broken, "forgets" bytes, or corrupts data. >> >> > I'm testing on SDR low end machines - and sometimes we have bugs that we > only see on high bandwidth setups. > And you have such a setup. > >> what I do "different" than the (assumed to be) typical SDP user is: >> sending large-ish messages at once (up to ~32 kB), possibly unaligned. >> >> which apparently is a mode that SDP has not excercised much yet, >> otherwise the recently fixed page leak would have been noticed by >> someone much earlier. >> >> >> > - Amir > From FENKES at de.ibm.com Wed Jul 8 05:28:02 2009 From: FENKES at de.ibm.com (Joachim Fenkes) Date: Wed, 8 Jul 2009 14:28:02 +0200 Subject: [ofa-general] Re: [PATCH v4] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <20090701200043.GF20745@obsidianresearch.com> <200907071620.23003.fenkes@de.ibm.com> Message-ID: Hal Rosenstock wrote on 07.07.2009 17:23:18: > > +static int redirect_port(ib_portid_t *port, uint8_t *mad) > > +{ > > + port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); > > + if (!port->lid) { > > + IBWARN("GID-based redirection is not supported"); > > + return -1; > > + } > > I hate to keep beating this horse but the lack of a LID certainly > means GID based redirection when the GID is not 0, IMO this LID check > is insufficient in general. If the LID is given, my code does the right thing by redirecting regardless of any GID, as the spec requires. If no LID is given, but a GID is, my code bails with an error stating that GID-based redirection is not supported. If both GID and LID are 0, that's an error and my code bails with an error message (which may or may not be misleading depending on your perspective, but frankly I couldn't care less about broken agents). Which of those three reactions do you think is insufficient? > I suppose this can be fixed down the road. Is that an ACK? ;) Cheers, Joachim From hal.rosenstock at gmail.com Wed Jul 8 06:00:15 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 8 Jul 2009 09:00:15 -0400 Subject: [ofa-general] RE: Question on rdma_resolve_route and retries In-Reply-To: <7D7BD0783D5C47F19C315EE6EBBD6383@amr.corp.intel.com> References: <5e3be0f30907080018p11376da8gd75ba9fd86f3acb1@mail.gmail.com> <7D7BD0783D5C47F19C315EE6EBBD6383@amr.corp.intel.com> Message-ID: On 7/8/09, Sean Hefty wrote: >>Nowhere in there do I see any retry attempts. If the SA path record query >>packet, or it's response packet, gets lost, then the timeout eventually >> happens >>and we see RDMA_CM_EVENT_ROUTE_ERROR with a status of ETIMEDOUT. > > The kernel sa_query module does not issue retries. All retries are the > responsibility of the caller. This gives greater flexibility to how > timeouts > are handled, but has the drawback that all 'retries' are really new > transactions. To elaborate on this, the downside of this apporach is that this is a different transaction which does not give the SA the ability to discard based on transaction ID so this increases the SA processing load. -- Hal From hal.rosenstock at gmail.com Wed Jul 8 06:24:53 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 8 Jul 2009 09:24:53 -0400 Subject: [ofa-general] Re: [PATCH v4] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <20090701200043.GF20745@obsidianresearch.com> <200907071620.23003.fenkes@de.ibm.com> Message-ID: On 7/8/09, Joachim Fenkes wrote: > Hal Rosenstock wrote on 07.07.2009 17:23:18: > >> > +static int redirect_port(ib_portid_t *port, uint8_t *mad) >> > +{ >> > + port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F); >> > + if (!port->lid) { >> > + IBWARN("GID-based redirection is not supported"); >> > + return -1; >> > + } >> >> I hate to keep beating this horse but the lack of a LID certainly >> means GID based redirection when the GID is not 0, IMO this LID check >> is insufficient in general. > > If the LID is given, my code does the right thing by redirecting > regardless > of any GID, as the spec requires. If no LID is given, but a GID is, my > code > bails with an error stating that GID-based redirection is not supported. > If > both GID and LID are 0, that's an error and my code bails with an error > message (which may or may not be misleading depending on your perspective, > but frankly I couldn't care less about broken agents). > > Which of those three reactions do you think is insufficient? It looks to me like both GID and LID are allowed to be specified in the redirect and if so, there is the possibility of GID based redirection there (as well as when LID is 0) and it is the requester which decides on GRH inclusion or not. >> I suppose this can be fixed down the road. > > Is that an ACK? ;) Indeed, a qualified ACK :-) This case that concerns me ends up entangled in the to be specified multiple IB subnet case (router spec). -- Hal > Cheers, > Joachim > From rdreier at cisco.com Wed Jul 8 08:58:59 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 08 Jul 2009 08:58:59 -0700 Subject: [ofa-general] Re: [PATCH V2] mlx4: check for FW version which properly supports resize_cq In-Reply-To: <200907081350.12462.jackm@dev.mellanox.co.il> (Jack Morgenstein's message of "Wed, 8 Jul 2009 13:50:12 +0300") References: <200907081350.12462.jackm@dev.mellanox.co.il> Message-ID: > @@ -349,6 +349,9 @@ int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) > int outst_cqe; > int err; > > + if (dev->dev->caps.fw_ver < MLX4_FW_VER_RESIZE_CQ) > + return -ENOSYS; This is kind of dopey, isn't it? Seems cleaner just to leave the resize_cq method unset if the hardware doesn't support it; then the core takes care of this check for us. - R. From FENKES at de.ibm.com Wed Jul 8 09:12:25 2009 From: FENKES at de.ibm.com (Joachim Fenkes) Date: Wed, 8 Jul 2009 18:12:25 +0200 Subject: [ofa-general] Re: [PATCH v4] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <20090701200043.GF20745@obsidianresearch.com> <200907071620.23003.fenkes@de.ibm.com> Message-ID: Hal Rosenstock wrote on 08.07.2009 15:24:53: > >> I suppose this can be fixed down the road. > > > > Is that an ACK? ;) > > Indeed, a qualified ACK :-) Cool, thanks! This patch should make its way into OFED 1.5... so who should pull it? You? Vlad? Someone not on CC? Whoever, please apply for OFED 1.5 -- thanks! Cheers Joachim From hal.rosenstock at gmail.com Wed Jul 8 09:48:29 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 8 Jul 2009 12:48:29 -0400 Subject: [ofa-general] Re: [PATCH v4] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <20090701200043.GF20745@obsidianresearch.com> <200907071620.23003.fenkes@de.ibm.com> Message-ID: On 7/8/09, Joachim Fenkes wrote: > Hal Rosenstock wrote on 08.07.2009 15:24:53: > >> >> I suppose this can be fixed down the road. >> > >> > Is that an ACK? ;) >> >> Indeed, a qualified ACK :-) > > Cool, thanks! > > This patch should make its way into OFED 1.5... so who should pull it? > You? Vlad? Someone not on CC? Whoever, please apply for OFED 1.5 -- > thanks! Sasha is the management maintainer. Userspace trees for OFED 1.5 haven't been created and I think this aspect is in transition. -- Hal > > Cheers > Joachim > From jgunthorpe at obsidianresearch.com Wed Jul 8 11:08:39 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Wed, 8 Jul 2009 12:08:39 -0600 Subject: [ofa-general] Re: [PATCH] rdma_cm: Add debugfs entries to monitor rdma_cm connections In-Reply-To: <4A546D58.6090707@Voltaire.com> References: <4A3E45D3.3040405@Voltaire.COM> <4A50C13F.6070506@Voltaire.COM> <4A520600.2080004@opengridcomputing.com> <4A531FA6.6050803@voltaire.com> <20090707164803.GR20745@obsidianresearch.com> <4A546D58.6090707@Voltaire.com> Message-ID: <20090708180839.GS20745@obsidianresearch.com> On Wed, Jul 08, 2009 at 12:56:40PM +0300, Or Gerlitz wrote: > Jason Gunthorpe wrote: > > Well, thats the rub isn't it? debugfs is not 'production ready' (by > > definition) so why spend time on it? > > to allow debugging, diagnosing problems > > > Why the resistance to doing a proper job and solving the actual problem? > > I didn't see resistance to do a better job, I don't see any patches.. > > debufs is entirely the wrong answer - this is not kernel debugging, > > this is required user space diagnostic information. > > said who? Read the kernel devs postings and papers on debugfs and its intended role in the system. > how are you suggesting me to know what kernel iser/lustre/rds/rnfs RDMA connections > are open now? A userspace program analogous to ss or netstat is the proper way to do this - exactly the same as how NFS/iSCSI/etc kernel TCP connections show up today. How do you propose to finish the job? RDMACM information is such a minor export - what about the rest of it: * Completion Channel status and information * CQ status and information, and mappings to CCs * QP status and information and mappings to CQs (Including GRH and LRHs used for the QP, RDMACM and IBCM meta information used to establish the connection, APM data and status from the CM, all the QP attributes, etc) * MR and PD information * Process and FD bindings for all of the above * lsof should be able to show every RDMA object related to a process Jason From swise at opengridcomputing.com Wed Jul 8 11:11:08 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Wed, 08 Jul 2009 13:11:08 -0500 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> Message-ID: <4A54E13C.7040508@opengridcomputing.com> I did this change and the hang went away as well. I think cmatose.c needs this fix. pandit ib wrote: > I'm still seeing the hang. > > >> I debugged this to see why it hangs. The problem, as I see it, is in the server side. The problem is shown in the diagram below (connection set up and tear down are omitted). Both sides post 10 receive buffers. >> >> 1. Server sends ten messages (not signaled) >> 2. Client polls recv cq and >> receives 10 messages >> 3. Client sends 10 messages >> 4. Server polls the send cq for completions >> 5. Server polls the recv cq for completions >> >> The server hangs in step 4 where it is looking for send completions but there will not be any because the signaled flag was not set in step 1. >> >> ucmatose completes when I change the following line: >> send_wr.send_flags = 0; >> to >> send_wr.send_flags = IBV_SEND_SIGNALED; >> >> Don Wood >> >> >> > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From sean.hefty at intel.com Wed Jul 8 11:14:03 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Wed, 8 Jul 2009 11:14:03 -0700 Subject: [ofa-general] RE: Question on rdma_resolve_route and retries In-Reply-To: <5e3be0f30907080057y2406f0c0nca2fb84dbe069676@mail.gmail.com> References: <5e3be0f30907080018p11376da8gd75ba9fd86f3acb1@mail.gmail.com> <7D7BD0783D5C47F19C315EE6EBBD6383@amr.corp.intel.com> <5e3be0f30907080057y2406f0c0nca2fb84dbe069676@mail.gmail.com> Message-ID: <7EC16629F98D40E89615E6CBFE4590B6@amr.corp.intel.com> >This is encouraging. I did try testing with 10,000 ms timeouts and still got >the failure with only 800 different processes, so I jumped to the conclusion >that the queries were being dropped. Do you have a guess as to a timeout value >that would always succeed? We ended up around a 60 second timeout based on the number of connections and how quickly our SM node could process queries. This was done a while ago, and there have been a lot of improvements to opensm since then. I don't know of an easy way to test the performance of the SM. It's also possible that our test staggered the queries just enough that the SM could keep up receiving them. >Maybe I should have come up with a better name. By fabric-specific, I meant a >specific implentation of the fabric, including the capability of the subnet >manager node. How does somebody writing rdma_cm code come up with a number? >That particular program might not put much of a load on the SA, but could run >concurrently with other jobs that do (or don't). It would be nice to have a >way to set up the retry mechanism so that it would work on any system it ran >on. Maybe the SA service could track the SA response time and adjust the timeout accordingly. E.g. guess = .2(last response) + .8(last guess). Users could indicate that the default timeout could be used. Apps could also help by staggering their start times to avoid hitting the SA with hundreds of thousands of queries at once. - Sean From sean.hefty at intel.com Wed Jul 8 11:20:02 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Wed, 8 Jul 2009 11:20:02 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <4A54E13C.7040508@opengridcomputing.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> Message-ID: >I did this change and the hang went away as well. > >I think cmatose.c needs this fix. > >>> >>> ucmatose completes when I change the following line: >>> send_wr.send_flags = 0; >>> to >>> send_wr.send_flags = IBV_SEND_SIGNALED; cmatose sets init_qp_attr.sq_sig_all = 1 when initializing the QP, so I wouldn't expect this flag to be used. - Sean From swise at opengridcomputing.com Wed Jul 8 11:28:49 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Wed, 08 Jul 2009 13:28:49 -0500 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> Message-ID: <4A54E561.6080804@opengridcomputing.com> Sean Hefty wrote: >> I did this change and the hang went away as well. >> >> I think cmatose.c needs this fix. >> >> >>>> ucmatose completes when I change the following line: >>>> send_wr.send_flags = 0; >>>> to >>>> send_wr.send_flags = IBV_SEND_SIGNALED; >>>> > > cmatose sets init_qp_attr.sq_sig_all = 1 when initializing the QP, so I wouldn't > expect this flag to be used. > > I see. Then it sounds like cxgb3 is busted because its not checking this attribute. :) Steve. From rdreier at cisco.com Wed Jul 8 12:14:51 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 08 Jul 2009 12:14:51 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <4A54E561.6080804@opengridcomputing.com> (Steve Wise's message of "Wed, 08 Jul 2009 13:28:49 -0500") References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> <4A54E561.6080804@opengridcomputing.com> Message-ID: > I see. Then it sounds like cxgb3 is busted because its not checking > this attribute. :) Yep... grep shows none of amso1100, cxgb3, nor nes look at sq_sig_type at all in the kernel driver. Looks like a common iWARP bug :) - R. From swise at opengridcomputing.com Wed Jul 8 12:23:53 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Wed, 08 Jul 2009 14:23:53 -0500 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> <4A54E561.6080804@opengridcomputing.com> Message-ID: <4A54F249.3080500@opengridcomputing.com> Roland Dreier wrote: > > I see. Then it sounds like cxgb3 is busted because its not checking > > this attribute. :) > > Yep... grep shows none of amso1100, cxgb3, nor nes look at sq_sig_type > at all in the kernel driver. Looks like a common iWARP bug :) > > Crazy iWARP cowboys... From chien.tin.tung at intel.com Wed Jul 8 13:04:42 2009 From: chien.tin.tung at intel.com (Tung, Chien Tin) Date: Wed, 8 Jul 2009 13:04:42 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> <4A54E561.6080804@opengridcomputing.com> Message-ID: <60BEFF3FBD4C6047B0F13F205CAFA383035C234459@azsmsx501.amr.corp.intel.com> > > I see. Then it sounds like cxgb3 is busted because its not checking > > this attribute. :) > >Yep... grep shows none of amso1100, cxgb3, nor nes look at sq_sig_type >at all in the kernel driver. Looks like a common iWARP bug :) As far as I can tell this is an infiniband only attribute. I do not see it in iWarp RDMA spec, if I am wrong please point me to the right place. How do we handle situation like this when there is a difference between the two? Chien From rdreier at cisco.com Wed Jul 8 13:30:26 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 08 Jul 2009 13:30:26 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <60BEFF3FBD4C6047B0F13F205CAFA383035C234459@azsmsx501.amr.corp.intel.com> (Chien Tin Tung's message of "Wed, 8 Jul 2009 13:04:42 -0700") References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> <4A54E561.6080804@opengridcomputing.com> <60BEFF3FBD4C6047B0F13F205CAFA383035C234459@azsmsx501.amr.corp.intel.com> Message-ID: > As far as I can tell this is an infiniband only attribute. I do not see > it in iWarp RDMA spec, if I am wrong please point me to the right place. > > How do we handle situation like this when there is a difference between > the two? Simplest thing to do in this case seems to be to handle it as an extension on the iWARP side; just keep track of what the consumer passed in for sq_sig_type and if it's "signal all" then never post unsignaled work requests. No reason to create gratuitous semantic differences when it's fairly easy to implement the union of IB and iWARP semantics. - R. From swise at opengridcomputing.com Wed Jul 8 13:48:03 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Wed, 08 Jul 2009 15:48:03 -0500 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <60BEFF3FBD4C6047B0F13F205CAFA383035C234459@azsmsx501.amr.corp.intel.com> References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> <4A54E561.6080804@opengridcomputing.com> <60BEFF3FBD4C6047B0F13F205CAFA383035C234459@azsmsx501.amr.corp.intel.com> Message-ID: <4A550603.5090206@opengridcomputing.com> Tung, Chien Tin wrote: >>> I see. Then it sounds like cxgb3 is busted because its not checking >>> this attribute. :) >>> >> Yep... grep shows none of amso1100, cxgb3, nor nes look at sq_sig_type >> at all in the kernel driver. Looks like a common iWARP bug :) >> > > > As far as I can tell this is an infiniband only attribute. I do not see > it in iWarp RDMA spec, if I am wrong please point me to the right place. > > How do we handle situation like this when there is a difference between > the two? > > IMO its an API issue and thus really outside the scope of the iWARP RDMA Verbs document, which doesn't define any APIs. Since it can easily be supported (I think), I plan to add support in cxgb3. IE just add additional logic in your post paths to check the attribute and set the appropriate "signaled" bit for your WRs. Unless I'm missing something here? Steve. From rdreier at cisco.com Wed Jul 8 15:26:13 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 08 Jul 2009 15:26:13 -0700 Subject: [ofa-general] Re: [PATCH] ehca: use port autodetect mode as default In-Reply-To: <20090707160639.6b45bf4d@BL3D1974.boeblingen.de.ibm.com> (Alexander Schmidt's message of "Tue, 7 Jul 2009 16:06:39 +0200") References: <20090707160639.6b45bf4d@BL3D1974.boeblingen.de.ibm.com> Message-ID: looks good, applied From rdreier at cisco.com Wed Jul 8 15:31:32 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 08 Jul 2009 15:31:32 -0700 Subject: [ofa-general] Re: A question about tx lock in ipoib_flush_paths In-Reply-To: <4A52930F.9050408@gmail.com> (Yossi Etigin's message of "Tue, 07 Jul 2009 03:13:03 +0300") References: <4A4E190D.7010004@voltaire.com> <4A52930F.9050408@gmail.com> Message-ID: > Couldn't ipoib_start_xmit() grab ipoib_neigh, and cause the same thing the > commit above was intended to fix? I don't see how -- either ipoib_start_xmit runs before the flush task grabs the tx lock, in which case it uses the path and exits before the flush task runs, or ipoib_start_xmit runs after the flush task releases the lock, in which case the path is already removed from the path rbtree and hence won't be found by ipoib_start_xmit. Of course I could easily be missing something. From weiny2 at llnl.gov Wed Jul 8 17:06:00 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Wed, 8 Jul 2009 17:06:00 -0700 Subject: [ofa-general] [PATCH] Fix error return code from mad_send_via Message-ID: <20090708170600.45d0b681.weiny2@llnl.gov> From: Ira Weiny Date: Wed, 8 Jul 2009 16:52:09 -0700 Subject: [PATCH] Fix error return code from mad_send_via Signed-off-by: Ira Weiny --- libibmad/src/serv.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/libibmad/src/serv.c b/libibmad/src/serv.c index ac3a177..c9a093a 100644 --- a/libibmad/src/serv.c +++ b/libibmad/src/serv.c @@ -64,7 +64,7 @@ int mad_send_via(ib_rpc_t * rpc, ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, DEBUG("rmpp %p data %p", rmpp, data); if (mad_build_pkt(umad, rpc, dport, rmpp, data) < 0) - return 0; + return -1; if (ibdebug) { IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz); -- 1.5.4.5 From chien.tin.tung at intel.com Wed Jul 8 14:06:50 2009 From: chien.tin.tung at intel.com (Tung, Chien Tin) Date: Wed, 8 Jul 2009 14:06:50 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> <4A54E561.6080804@opengridcomputing.com> <60BEFF3FBD4C6047B0F13F205CAFA383035C234459@azsmsx501.amr.corp.intel.com> Message-ID: <60BEFF3FBD4C6047B0F13F205CAFA383035C2345DD@azsmsx501.amr.corp.intel.com> > > As far as I can tell this is an infiniband only attribute. >I do not see > > it in iWarp RDMA spec, if I am wrong please point me to the >right place. > > > > How do we handle situation like this when there is a >difference between > > the two? > >Simplest thing to do in this case seems to be to handle it as an >extension on the iWARP side; just keep track of what the >consumer passed >in for sq_sig_type and if it's "signal all" then never post unsignaled >work requests. No reason to create gratuitous semantic >differences when >it's fairly easy to implement the union of IB and iWARP semantics. For this issue, it is easy to resolve the difference and I don't have a problem implementing the extension as suggested by Roland. At the same time, I don't want to turn it into if an iWarp device does not do what an Infiniband device would do then it is a bug. I didn't take this issue that way as everyone said it with a :-) I'm probably making it a bigger issue than it is so I will stop now. :-) Chien From weiny2 at llnl.gov Wed Jul 8 17:47:20 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Wed, 8 Jul 2009 17:47:20 -0700 Subject: [ofa-general] [PATCH] Lock _do_madrpc for thread safety Message-ID: <20090708174720.f61d44fc.weiny2@llnl.gov> Sasha, I am working on making libibnetdisc a parallel implementation. As a result I have found that _do_madrpc is not thread safe. The following patch fixes this. However, I don't know you want to do... If one only uses mad_rpc and mad_rpc_rmpp then the patch works. However, if someone is using mad_send_via at the same time _do_madrpc will still fail. Is it by design that some responses will be lost while _do_madrpc is looking for it's response via the TID? Also, according to C13-18.1.1 and C13-19.1.1 you must use the SGID (or SLID) and the MgmtClass in addition to the TID to determine the uniqueness of a message. The SGID (or SLID) is of course the same but should the MgmtClass be checked here as well? Finally, why does _do_madrpc cast the transaction id to a 32 bit value? Confused, Ira From: Ira Weiny Date: Wed, 8 Jul 2009 15:01:11 -0700 Subject: [PATCH] Lock _do_madrpc for thread safety Signed-off-by: Ira Weiny --- libibmad/configure.in | 2 ++ libibmad/src/rpc.c | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/libibmad/configure.in b/libibmad/configure.in index b4f5c41..5d3e304 100644 --- a/libibmad/configure.in +++ b/libibmad/configure.in @@ -43,6 +43,8 @@ then AC_CHECK_HEADER(infiniband/umad.h, [], AC_MSG_ERROR([ not found. libibmad requires libibumad.]) ) +AC_CHECK_HEADER(pthread.h, [], + AC_MSG_ERROR([ not found. libibmad requires pthread.h])) fi dnl Checks for library functions diff --git a/libibmad/src/rpc.c b/libibmad/src/rpc.c index 07b623d..83bd4f0 100644 --- a/libibmad/src/rpc.c +++ b/libibmad/src/rpc.c @@ -43,6 +43,7 @@ #include #include +#include #include "mad_internal.h" @@ -58,6 +59,8 @@ static int madrpc_timeout = MAD_DEF_TIMEOUT_MS; static void *save_mad; static int save_mad_len = 256; +static pthread_mutex_t snd_rcv_lock = PTHREAD_MUTEX_INITIALIZER; + #undef DEBUG #define DEBUG if (ibdebug) IBWARN #define ERRS(fmt, ...) do { \ @@ -127,6 +130,7 @@ static int _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, int timeout, int max_retries) { + uint32_t trid_rcv; uint32_t trid; /* only low 32 bits */ int retries; int length, status; @@ -150,6 +154,8 @@ _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, ERRS("retry %d (timeout %d ms)", retries, timeout); length = len; + + pthread_mutex_lock(&snd_rcv_lock); if (umad_send(port_id, agentid, sndbuf, length, timeout, 0) < 0) { IBWARN("send failed; %m"); return -1; @@ -159,6 +165,7 @@ _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, /* send packet is lost somewhere. */ do { if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) { + pthread_mutex_unlock(&snd_rcv_lock); IBWARN("recv failed: %m"); return -1; } @@ -168,9 +175,14 @@ _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, xdump(stderr, "rcv buf\n", umad_get_mad(rcvbuf), IB_MAD_SIZE); } - } while ((uint32_t) - mad_get_field64(umad_get_mad(rcvbuf), 0, - IB_MAD_TRID_F) != trid); + + trid_rcv = (uint32_t) mad_get_field64(umad_get_mad(rcvbuf), + 0, IB_MAD_TRID_F); + if (trid_rcv != trid) + IBWARN("trid_rcv(%x) != trid (%x)\n", trid_rcv, trid); + + } while (trid_rcv != trid); + pthread_mutex_unlock(&snd_rcv_lock); status = umad_status(rcvbuf); if (!status) -- 1.5.4.5 From umayr at gatech.edu Wed Jul 8 23:10:14 2009 From: umayr at gatech.edu (umayr at gatech.edu) Date: Thu, 9 Jul 2009 02:10:14 -0400 (EDT) Subject: [ofa-general] rdma_listen() backlog In-Reply-To: <65194529.845991247119600339.JavaMail.root@mail1.gatech.edu> Message-ID: <2119455328.846331247119814612.JavaMail.root@mail1.gatech.edu> Hi, Given RDMA CM (server) listening on a given IP address and port number with a backlog values greater than 1. The problem is that every time more than one connection requests are sent to the listening port, rdma_listen() traps the later request but gives an error. What might be the reason for this error. Given previous discussions on the mailing list about the meaning of the backlog value, it's unclear whether it indicates the number of connection waiting to be processed (is that after the currently accepted connection has completed processing?) or is it "just an attribute of the service/listening endpoint" (again it's unclear what is the function of this attribute). So, in general, how is it possible to listen for and deal with multiple connections request on the same IP/port value? Thanks. - Umayr From jackm at dev.mellanox.co.il Thu Jul 9 01:12:09 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Thu, 9 Jul 2009 11:12:09 +0300 Subject: [ofa-general] Re: [PATCH V2] mlx4: check for FW version which properly supports resize_cq In-Reply-To: References: <200907081350.12462.jackm@dev.mellanox.co.il> Message-ID: <200907091112.10027.jackm@dev.mellanox.co.il> On Wednesday 08 July 2009 18:58, Roland Dreier wrote: > This is kind of dopey, isn't it?  Seems cleaner just to leave the > resize_cq method unset if the hardware doesn't support it; then the core > takes care of this check for us. > Not so (unfortunately). The problem is that doing it the "correct" way (as you describe above) results in userspace apps getting "EINVAL" as the return value, and not ENOSYS. This means that there is no way to differentiate between a real error in the call, and simply a not-implemented verb. This really clobbers MPI. That was the original reason I submitted this patch as I did. (I had forgotten about all these gory details, and went back and reviewed this issue for my reply here). I submitted a patch for the uverbs EINVAL/ENOSYS issue on Dec 2, 2008 See the short thread for a discussion of all the messy details I allude to above: http://lists.openfabrics.org/pipermail/general/2008-December/055734.html [PATCH] uverbs: return ENOSYS for unimplemented commands (not EINVAL), and my short comment to you in original patch I submitted for the resize_cq fw test http://lists.openfabrics.org/pipermail/general/2008-December/055770.html [PATCH V2] mlx4: check for FW version which properly supports resize_cq Basically, I did not want to deal with the global issue that the uverbs patch raises. Since MPI badly needed to differentiate between ibv_resize_cq() returning ENOSYS and returning EINVAL, the resize_cq patch represents a "local" fix for this specific verb -- rather than a fix with far-reaching consequences for any unimplemented verb. (BTW, the uverbs EINVAL/ENOSYS problem still exists in your tree. I'm not sure that we want to change this) . -Jack From tziporet at dev.mellanox.co.il Thu Jul 9 01:15:39 2009 From: tziporet at dev.mellanox.co.il (Tziporet Koren) Date: Thu, 09 Jul 2009 11:15:39 +0300 Subject: [ofa-general] Re: [PATCH v4] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <20090701200043.GF20745@obsidianresearch.com> <200907071620.23003.fenkes@de.ibm.com> Message-ID: <4A55A72B.4040805@mellanox.co.il> Hal Rosenstock wrote: >> >> This patch should make its way into OFED 1.5... so who should pull it? >> You? Vlad? Someone not on CC? Whoever, please apply for OFED 1.5 -- >> thanks! >> > > Sasha is the management maintainer. Userspace trees for OFED 1.5 > haven't been created and I think this aspect is in transition. > > -- Hal > > > We take user space for ofed 1.5 from latest management tarball from Sasha. So when it will be in the tree it will be in Tziporet From vlad at lists.openfabrics.org Thu Jul 9 02:41:03 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Thu, 9 Jul 2009 02:41:03 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090709-0200 daily build status Message-ID: <20090709094103.A707D10202BC@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -Werror-implicit-function-declaration -fno-strict-aliasing -fno-common -ffreestanding -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: from /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.c:37: include/linux/skbuff.h: In function 'skb_add_data': include/linux/skbuff.h:1041: warning: pointer targets in passing argument 2 of 'csum_partial_copy_from_user' differ in signedness make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: from /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.c:37: include/linux/skbuff.h: In function 'skb_add_data': include/linux/skbuff.h:1041: warning: pointer targets in passing argument 2 of 'csum_partial_copy_from_user' differ in signedness make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090709-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From leo.tominna at oracle.com Wed Jul 8 16:11:29 2009 From: leo.tominna at oracle.com (Leo Tominna) Date: Wed, 08 Jul 2009 16:11:29 -0700 Subject: [ofa-general] patch to ib_addr for sending arps Message-ID: <4A5527A1.5010104@oracle.com> This patch appears to help when strict ARP handling is enabled or when non-standard routing tables are used. The ARP request is replied to through the device that will be used for subsequent communication, so the ARP entry gets associated with the correct device in the ARP cache. tcpdump shows consistent ARPs generated by similar arguments to ping and rds-ping. --- ofa_kernel-1.5/drivers/infiniband/core/addr.c.orig 2009-07-08 15:07:52.000000000 -0700 +++ ofa_kernel-1.5/drivers/infiniband/core/addr.c 2009-07-08 15:23:49.000000000 -0700 @@ -176,7 +176,7 @@ mutex_unlock(&lock); } -static void addr_send_arp(struct sockaddr *dst_in) +static void addr_send_arp(struct sockaddr *src_in, struct sockaddr *dst_in) { struct rtable *rt; struct flowi fl; @@ -185,6 +185,9 @@ switch (dst_in->sa_family) { case AF_INET: + if (src_in) + fl.nl_u.ip4_u.saddr = + ((struct sockaddr_in *) src_in)->sin_addr.s_addr; fl.nl_u.ip4_u.daddr = ((struct sockaddr_in *) dst_in)->sin_addr.s_addr; @@ -200,6 +203,9 @@ { struct dst_entry *dst; + if (src_in) + fl.nl_u.ip6_u.saddr = + ((struct sockaddr_in6 *) src_in)->sin6_addr; fl.nl_u.ip6_u.daddr = ((struct sockaddr_in6 *) dst_in)->sin6_addr; @@ -467,7 +473,7 @@ case -ENODATA: req->timeout = msecs_to_jiffies(timeout_ms) + jiffies; queue_req(req); - addr_send_arp(dst_in); + addr_send_arp(src_in, dst_in); break; default: ret = req->status; From swise at opengridcomputing.com Thu Jul 9 07:36:12 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Thu, 09 Jul 2009 09:36:12 -0500 Subject: [ofa-general] rdma_listen() backlog In-Reply-To: <2119455328.846331247119814612.JavaMail.root@mail1.gatech.edu> References: <2119455328.846331247119814612.JavaMail.root@mail1.gatech.edu> Message-ID: <4A56005C.5030709@opengridcomputing.com> umayr at gatech.edu wrote: > Hi, > > Given RDMA CM (server) listening on a given IP address and port number with a backlog values greater than 1. The problem is that every time more than one connection requests are sent to the listening port, rdma_listen() traps the later request but gives an error. What might be the reason for this error. > > Given previous discussions on the mailing list about the meaning of the backlog value, it's unclear whether it indicates the number of connection waiting to be processed (is that after the currently accepted connection has completed processing?) or is it "just an attribute of the service/listening endpoint" (again it's unclear what is the function of this attribute). > > The backlog dictates how many pending rdma connect requests the kernel will keep in-queue for your application to accept/reject. > So, in general, how is it possible to listen for and deal with multiple connections request on the same IP/port value? > > The app should set the backlog to something appropriate, issue the listen, and then process rdma cm connect request events from the rdma cm event channel associated with the listening cm_id. Its up to the app as to whether it will process and accept more than one connect request at a time. I suggest you look at rping for an example application. In its persistent server mode (-P), the server will handle multiple connections. You can fetch the rping source code from: http://www.openfabrics.org/downloads/rdmacm/librdmacm-1.0.8.tar.gz. Look in src/examples/rping.c. Steve. From rdreier at cisco.com Thu Jul 9 08:10:46 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 09 Jul 2009 08:10:46 -0700 Subject: [ofa-general] Re: [PATCH V2] mlx4: check for FW version which properly supports resize_cq In-Reply-To: <200907091112.10027.jackm@dev.mellanox.co.il> (Jack Morgenstein's message of "Thu, 9 Jul 2009 11:12:09 +0300") References: <200907081350.12462.jackm@dev.mellanox.co.il> <200907091112.10027.jackm@dev.mellanox.co.il> Message-ID: > Not so (unfortunately). The problem is that doing it the "correct" way > (as you describe above) results in userspace apps getting "EINVAL" as > the return value, and not ENOSYS. This means that there is no way to > differentiate between a real error in the call, and simply a not-implemented > verb. This really clobbers MPI. Hmm... seems like a generic problem, since eg nes doesn't implement resize_cq() ever. I guess the cleanest fix would be to add handling for unset device->resize_cq into uverbs_cmd.c, so drivers can add that to their allowed userspace commands without actually implementing that. We Or maybe it's cleaner to add a stub resize_cq method that just returns ENOSYS that drivers can set when they don't actually implement it... - R. From rdreier at cisco.com Thu Jul 9 08:14:03 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 09 Jul 2009 08:14:03 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: <60BEFF3FBD4C6047B0F13F205CAFA383035C2345DD@azsmsx501.amr.corp.intel.com> (Chien Tin Tung's message of "Wed, 8 Jul 2009 14:06:50 -0700") References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> <4A54E561.6080804@opengridcomputing.com> <60BEFF3FBD4C6047B0F13F205CAFA383035C234459@azsmsx501.amr.corp.intel.com> <60BEFF3FBD4C6047B0F13F205CAFA383035C2345DD@azsmsx501.amr.corp.intel.com> Message-ID: > At the same time, I don't want to turn it into if an iWarp device > does not do what an Infiniband device would do then it is a bug. > I didn't take this issue that way as everyone said it with a :-) > I'm probably making it a bigger issue than it is so I will stop now. I don't think we necessarily choose IB or iWARP semantics. In this case for historical reasons (IB was implemented first) we have an interface with two ways of saying whether sends should be signaled -- the iWARP way of having only one place to set this is probably cleaner but it's not a big enough deal to break the interface. In other places we've used iWARP semantics -- eg one could view the RDMA CM stuff as doing a *lot* of work to provide iWARP "connect by IP" semantics on top of IB. Basically we're trying to come up with a common set of "RDMA verbs" that can be used on both IB and iWARP hardware. - R. From chien.tin.tung at intel.com Thu Jul 9 08:48:37 2009 From: chien.tin.tung at intel.com (Tung, Chien Tin) Date: Thu, 9 Jul 2009 08:48:37 -0700 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> <4A54E561.6080804@opengridcomputing.com> <60BEFF3FBD4C6047B0F13F205CAFA383035C234459@azsmsx501.amr.corp.intel.com> <60BEFF3FBD4C6047B0F13F205CAFA383035C2345DD@azsmsx501.amr.corp.intel.com> Message-ID: <60BEFF3FBD4C6047B0F13F205CAFA383035C29C004@azsmsx501.amr.corp.intel.com> >Basically we're trying to come up with a common set of "RDMA >verbs" that >can be used on both IB and iWARP hardware. We are in agreement. Chien From swise at opengridcomputing.com Thu Jul 9 08:53:11 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Thu, 09 Jul 2009 10:53:11 -0500 Subject: [ofa-general] cmatose fails whereas rping passes on iWarp In-Reply-To: References: <96f8e60e0907021617q1073f9b5m1ee1277374c8a5af@mail.gmail.com> <19F156367B9D4538A1A4AAA7A0465B51@amr.corp.intel.com> <96f8e60e0907061139j78155c60p4a5500a3b74c6cce@mail.gmail.com> <588992150B702C48B3312184F1B810AD040A20AAE8@azsmsx501.amr.corp.intel.com> <96f8e60e0907062007i4e21c03ai7ef51bfedec532b7@mail.gmail.com> <4A54E13C.7040508@opengridcomputing.com> Message-ID: <4A561267.3030902@opengridcomputing.com> Sean Hefty wrote: >> I did this change and the hang went away as well. >> >> I think cmatose.c needs this fix. >> >> >>>> ucmatose completes when I change the following line: >>>> send_wr.send_flags = 0; >>>> to >>>> send_wr.send_flags = IBV_SEND_SIGNALED; >>>> > > cmatose sets init_qp_attr.sq_sig_all = 1 when initializing the QP, so I wouldn't > expect this flag to be used. > > - Sean > I opened bug 1674 to track this issue for cxgb3. Steve. From yosefe at voltaire.com Thu Jul 9 09:21:34 2009 From: yosefe at voltaire.com (Yossi Etigin) Date: Thu, 09 Jul 2009 19:21:34 +0300 Subject: [ofa-general] Re: A question about tx lock in ipoib_flush_paths In-Reply-To: References: <4A4E190D.7010004@voltaire.com> <4A52930F.9050408@gmail.com> Message-ID: <4A56190E.1080105@voltaire.com> Roland Dreier wrote: > > Couldn't ipoib_start_xmit() grab ipoib_neigh, and cause the same thing the > > commit above was intended to fix? > > I don't see how -- either ipoib_start_xmit runs before the flush task > grabs the tx lock, in which case it uses the path and exits before the > flush task runs, or ipoib_start_xmit runs after the flush task releases > the lock, in which case the path is already removed from the path rbtree > and hence won't be found by ipoib_start_xmit. Of course I could easily > be missing something. The path itself is rarely used, most of the time we take the ah from the ipoib_neigh which is stashed in the kernel neighbour. The scenario is that ipoib_start_xmit() runs after the flush task releases the lock, and does 'neigh = *to_ipoib_neigh(skb_dst(skb)->neighbour)'. Then the flush task continues to run and does path_free() which calls ipoib_neigh_free(), and kfree-s the neighbour. Then the xmit routine will go on using a stale neigh pointer. --Yossi From jackm at dev.mellanox.co.il Thu Jul 9 09:27:42 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Thu, 9 Jul 2009 19:27:42 +0300 Subject: [ofa-general] Re: [PATCH V2] mlx4: check for FW version which properly supports resize_cq In-Reply-To: References: <200907081350.12462.jackm@dev.mellanox.co.il> <200907091112.10027.jackm@dev.mellanox.co.il> Message-ID: <200907091927.42856.jackm@dev.mellanox.co.il> On Thursday 09 July 2009 18:10, Roland Dreier wrote: > Or maybe it's cleaner to add a stub resize_cq method that just returns > ENOSYS that drivers can set when they don't actually implement it... > Basically, that is what the patch I submitted to you does. Its just that instead of having a different call which returns -ENOSYS and doing the test in the mlx4_ib_add function, I do it in the resize_cq function. If you prefer (i.e., if you think it is cleaner), I can do what you suggest above (and submit a new patch) -- do the version test when populating the virtual function table), and simply link resize_cq to a stub function which returns -ENOSYS if the installed fw version does not support resize_cq. -Jack From amirv at mellanox.co.il Thu Jul 9 09:28:26 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Thu, 09 Jul 2009 19:28:26 +0300 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <20090708101715.GB8083@barkeeper1-xen.linbit> References: <20090701133652.GG9112@soda.linbit> <4A4B947C.1030607@mellanox.co.il> <20090702081445.GA9118@soda.linbit> <4A4C7021.2090506@mellanox.co.il> <20090702132304.GD9305@soda.linbit> <20090707171326.GH8065@barkeeper1-xen.linbit> <4A5454DF.1070101@mellanox.co.il> <20090708101715.GB8083@barkeeper1-xen.linbit> Message-ID: <4A561AAA.4050904@mellanox.co.il> Lars Hi, I'v found the bug. a fix patch is attached. I will commit it after the weekend (In Israel weekend is Friday-Saturday). - Amir On 07/08/2009 01:17 PM, Lars Ellenberg wrote: > On Wed, Jul 08, 2009 at 11:12:15AM +0300, Amir Vadai wrote: > >> Lars Hi, >> >> I opened a bug in our bugzilla (https://bugs.openfabrics.org/show_bug.cgi?id=1672). >> >> I couldn't reproduce it on my setup: SLES 10SP2, stock kernel, same ofed git version. >> will try now to install 2.6.27 kernel and check again. >> > With a "normal" kernel config, I needed to do full load bi-directional > network traffic on IPoIB as well as SDP, multiple stream sockets, > to eventually actually trigger it after a few minutes > (several hundered MegaByte per second). > > with the "debug" kernel config, > I was able to reproduce with only one socket, > within milliseconds. > > my .config is attached. > > >> BTW, what type of servers do you use? Are they low/high end server? >> > This is the second cluster that show this bug. I first experienced it > when using SDP sockets from within kernel space. > I was able to reproduce in userland, > which I thought might make it easier for you to reproduce. > > The current test cluster is a slightly aged 2U supermicro dual quadcore, > 4GB ram, and proved to be very reliable hardware in all test up to now. > it may be a little slow on interrupts. > > tail of /proc/cpuinfo: > processor : 7 > vendor_id : GenuineIntel > cpu family : 6 > model : 15 > model name : Intel(R) Xeon(R) CPU E5310 @ 1.60GHz > stepping : 7 > cpu MHz : 1599.984 > cache size : 4096 KB > physical id : 1 > siblings : 4 > core id : 3 > cpu cores : 4 > apicid : 7 > initial apicid : 7 > fpu : yes > fpu_exception : yes > cpuid level : 10 > wp : yes > flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge > mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe > syscall nx lm constant_tsc arch_perfmon pebs bts rep_good nopl pni > monitor ds_cpl vmx tm2 ssse3 cx16 xtpr dca lahf_lm > bogomips : 3201.35 > clflush size : 64 > cache_alignment : 64 > address sizes : 36 bits physical, 48 bits virtual > power management: > > the IB setup is direct link, lspci says: > 09:00.0 InfiniBand: Mellanox Technologies MT26428 [ConnectX IB QDR, PCIe 2.0 5GT/s] (rev a0) > > > because using IPoIB does work just fine, I don't think we have issues > with IB setup, or the hardware in general. > only when using SDP it is broken, "forgets" bytes, or corrupts data. > > what I do "different" than the (assumed to be) typical SDP user is: > sending large-ish messages at once (up to ~32 kB), possibly unaligned. > > which apparently is a mode that SDP has not excercised much yet, > otherwise the recently fixed page leak would have been noticed by > someone much earlier. > > -------------- next part -------------- A non-text attachment was scrubbed... Name: data_integrity_bug.patch Type: text/x-patch Size: 563 bytes Desc: not available URL: From korisk at yandex.ru Thu Jul 9 10:29:29 2009 From: korisk at yandex.ru (korisk) Date: Thu, 09 Jul 2009 21:29:29 +0400 Subject: [ofa-general] sending mad in parallel mode and perfquery Message-ID: <4A5628F9.6070500@yandex.ru> Hello, ibumad library has functions send_mad and recv_mad which should be send sequentially. Is it possible to create function which would send several MADs to several destinations and then waits for replies(in terms of ib driver)? May time spent to such function be useful? Next question about param "-a" of perfquery command. It doesn't work. Has anyone ideas about it? Thank you. Sincerely, Korisk. From sean.hefty at intel.com Thu Jul 9 10:56:47 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Thu, 9 Jul 2009 10:56:47 -0700 Subject: [ofa-general] sending mad in parallel mode and perfquery In-Reply-To: <4A5628F9.6070500@yandex.ru> References: <4A5628F9.6070500@yandex.ru> Message-ID: <423F353FA5DE4BD79D1EC432F040B80E@amr.corp.intel.com> >ibumad library has functions send_mad and recv_mad which should be send >sequentially. >Is it possible to create function which would send several MADs to >several destinations and then waits for replies(in terms of ib driver)? I'm not sure that send_mad and recv_mad don't do what you want. To send to multiple destinations, call send_mad multiple times. The call returns after posting or queuing the send operation to the QP. It does not wait for a response or guarantee that the send has actually been placed on the wire before returning. recv_mad blocks until any response is received, and it can be called from multiple threads. recv_mad only has multi-threaded issues if MADs > 256 bytes are received. - Sean From He.Huang at Sun.COM Thu Jul 9 10:57:27 2009 From: He.Huang at Sun.COM (Isaac Huang) Date: Thu, 09 Jul 2009 13:57:27 -0400 Subject: [ofa-general] rdma_listen() backlog In-Reply-To: <4A56005C.5030709@opengridcomputing.com> References: <2119455328.846331247119814612.JavaMail.root@mail1.gatech.edu> <4A56005C.5030709@opengridcomputing.com> Message-ID: <20090709175727.GE5662@sun.com> On Thu, Jul 09, 2009 at 09:36:12AM -0500, Steve Wise wrote: > ...... > The backlog dictates how many pending rdma connect requests the kernel > will keep in-queue for your application to accept/reject. Maybe I've missed something, but the last time I checked it appeared to me that for kernel RDMA CM the 'backlog' parameter was not used at all unless for iWarp transport. Isaac From swise at opengridcomputing.com Thu Jul 9 11:09:31 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Thu, 09 Jul 2009 13:09:31 -0500 Subject: [ofa-general] rdma_listen() backlog In-Reply-To: <20090709175727.GE5662@sun.com> References: <2119455328.846331247119814612.JavaMail.root@mail1.gatech.edu> <4A56005C.5030709@opengridcomputing.com> <20090709175727.GE5662@sun.com> Message-ID: <4A56325B.3010208@opengridcomputing.com> Isaac Huang wrote: > On Thu, Jul 09, 2009 at 09:36:12AM -0500, Steve Wise wrote: > >> ...... >> The backlog dictates how many pending rdma connect requests the kernel >> will keep in-queue for your application to accept/reject. >> > > Maybe I've missed something, but the last time I checked it appeared > to me that for kernel RDMA CM the 'backlog' parameter was not used at > all unless for iWarp transport. > True. I thought the question was about user mode. From sean.hefty at intel.com Thu Jul 9 11:25:59 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Thu, 9 Jul 2009 11:25:59 -0700 Subject: [ofa-general] rdma_listen() backlog In-Reply-To: <20090709175727.GE5662@sun.com> References: <2119455328.846331247119814612.JavaMail.root@mail1.gatech.edu> <4A56005C.5030709@opengridcomputing.com> <20090709175727.GE5662@sun.com> Message-ID: >Maybe I've missed something, but the last time I checked it appeared >to me that for kernel RDMA CM the 'backlog' parameter was not used at >all unless for iWarp transport. It's not used for kernel IB connections. Since connection requests are reported through a callback, there's nothing to queue and it's unneeded. It is used for userspace connections. From hnrose at comcast.net Thu Jul 9 12:39:15 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Thu, 9 Jul 2009 15:39:15 -0400 Subject: [ofa-general] [PATCH] infiniband-diags/ibroute.c: Fix format and typo in printf Message-ID: <20090709193915.GA19695@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/infiniband-diags/src/ibroute.c b/infiniband-diags/src/ibroute.c index 31b8e4c..ad6d94c 100644 --- a/infiniband-diags/src/ibroute.c +++ b/infiniband-diags/src/ibroute.c @@ -186,7 +186,7 @@ dump_multicast_tables(ib_portid_t *portid, unsigned startlid, unsigned endlid) printf(" MLid\n"); } if (ibverbose) - printf("Switch muticast mlids capability is 0x%d\n", cap); + printf("Switch multicast mlid capability is %d\n", cap); chunks = ALIGN(nports + 1, 16) / 16; From chu11 at llnl.gov Thu Jul 9 13:21:10 2009 From: chu11 at llnl.gov (Al Chu) Date: Thu, 09 Jul 2009 13:21:10 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [0/4] libibnetdisc cleanup patches round 2 Message-ID: <1247170870.16377.23.camel@auk31.llnl.gov> Hey Sasha, Here are some more bug fixes, cleanup fixes, etc. to libibnetdisc. These 4 patches are applied on top of the previous 5 patches I sent. Again, all already signed off my Ira. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory From chu11 at llnl.gov Thu Jul 9 13:21:13 2009 From: chu11 at llnl.gov (Al Chu) Date: Thu, 09 Jul 2009 13:21:13 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [1/4] libibnetdisc cleanup patches round 2 Message-ID: <1247170873.16377.24.camel@auk31.llnl.gov> Fix potential memleak in ibnd_discover_fabric error path. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- >From 254f8dd150ef2b3751ef98671ecbffefdc7618e6 Mon Sep 17 00:00:00 2001 From: Albert Chu Date: Tue, 7 Jul 2009 16:08:16 -0700 Subject: [PATCH] Fix potential memleak in ibnd_discover_fabric error path. Signed-off-by: Albert Chu Signed-off-by: Ira Weiny --- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index ffd8602..ea121dc 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -602,7 +602,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, return ((ibnd_fabric_t *)fabric); error: - free(fabric); + ibnd_destroy_fabric(fabric); return (NULL); } -- 1.5.4.5 From chu11 at llnl.gov Thu Jul 9 13:21:21 2009 From: chu11 at llnl.gov (Al Chu) Date: Thu, 09 Jul 2009 13:21:21 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [3/4] libibnetdisc cleanup patches round 2 Message-ID: <1247170881.16377.26.camel@auk31.llnl.gov> Check input parameters to libibnetdisc functions. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- >From 509983720abadf313f7068b2bbe2933bafeefb84 Mon Sep 17 00:00:00 2001 From: Albert Chu Date: Wed, 8 Jul 2009 16:47:33 -0700 Subject: [PATCH] Check input parameters to libibnetdisc functions Signed-off-by: Albert Chu Signed-off-by: Ira Weiny --- infiniband-diags/libibnetdisc/src/chassis.c | 15 ++++++++ infiniband-diags/libibnetdisc/src/ibnetdisc.c | 47 ++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletions(-) diff --git a/infiniband-diags/libibnetdisc/src/chassis.c b/infiniband-diags/libibnetdisc/src/chassis.c index bec24bc..064d0b1 100644 --- a/infiniband-diags/libibnetdisc/src/chassis.c +++ b/infiniband-diags/libibnetdisc/src/chassis.c @@ -54,6 +54,11 @@ static char *ChassisSlotTypeStr[4] = { "", "Line", "Spine", "SRBD" }; char *ibnd_get_chassis_type(ibnd_node_t *node) { + if (!node) { + IBND_DEBUG("node parameter NULL\n"); + return (NULL); + } + /* Currently, only if Voltaire chassis */ if (mad_get_field(node->info, 0, IB_NODE_VENDORID_F) != VTR_VENDOR_ID) return (NULL); @@ -67,6 +72,11 @@ char *ibnd_get_chassis_type(ibnd_node_t *node) char *ibnd_get_chassis_slot_str(ibnd_node_t *node, char *str, size_t size) { + if (!node) { + IBND_DEBUG("node parameter NULL\n"); + return (NULL); + } + /* Currently, only if Voltaire chassis */ if (mad_get_field(node->info, 0, IB_NODE_VENDORID_F) != VTR_VENDOR_ID) return (NULL); @@ -216,6 +226,11 @@ uint64_t ibnd_get_chassis_guid(ibnd_fabric_t *fabric, unsigned char chassisnum) struct ibnd_fabric_int *f = CONV_FABRIC_INTERNAL(fabric); ibnd_chassis_t *chassis; + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return 0; + } + chassis = find_chassisnum(f, chassisnum); if (chassis) return chassis->chassisguid; diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index 7d9069b..88fb7df 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -239,6 +239,11 @@ ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid) int hash = HASHGUID(guid) % HTSZ; struct ibnd_node_int *node; + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return (NULL); + } + for (node = f->nodestbl[hash]; node; node = node->htnext) if (node->node.guid == guid) return (ibnd_node_t *)node; @@ -275,6 +280,16 @@ ibnd_update_node(struct ibmad_port *ibmad_port, ibnd_fabric_t *fabric, ibnd_node if (_check_ibmad_port(ibmad_port) < 0) return (NULL); + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return (NULL); + } + + if (!node) { + IBND_DEBUG("node parameter NULL\n"); + return (NULL); + } + if (query_node_info(ibmad_port, f, n, &(n->node.path_portid))) return (NULL); @@ -313,9 +328,16 @@ ibnd_find_node_dr(ibnd_fabric_t *fabric, char *dr_str) { struct ibnd_fabric_int *f = CONV_FABRIC_INTERNAL(fabric); int i = 0; - ibnd_node_t *rc = f->fabric.from_node; + ibnd_node_t *rc; ib_dr_path_t path; + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return (NULL); + } + + rc = f->fabric.from_node; + if (str2drpath(&path, dr_str, 0, 0) == -1) { return (NULL); } @@ -640,6 +662,9 @@ ibnd_destroy_fabric(ibnd_fabric_t *fabric) struct ibnd_node_int *next = NULL; ibnd_chassis_t *ch, *ch_next; + if (!fabric) + return; + ch = f->first_chassis; while (ch) { ch_next = ch->next; @@ -684,6 +709,16 @@ ibnd_iter_nodes(ibnd_fabric_t *fabric, { ibnd_node_t *cur = NULL; + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return; + } + + if (!func) { + IBND_DEBUG("func parameter NULL\n"); + return; + } + for (cur = fabric->nodes; cur; cur = cur->next) { func(cur, user_data); } @@ -700,6 +735,16 @@ ibnd_iter_nodes_type(ibnd_fabric_t *fabric, struct ibnd_node_int *list = NULL; struct ibnd_node_int *cur = NULL; + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return; + } + + if (!func) { + IBND_DEBUG("func parameter NULL\n"); + return; + } + switch (node_type) { case IB_NODE_SWITCH: list = f->switches; -- 1.5.4.5 From chu11 at llnl.gov Thu Jul 9 13:21:17 2009 From: chu11 at llnl.gov (Al Chu) Date: Thu, 09 Jul 2009 13:21:17 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [2/4] libibnetdisc cleanup patches round 2 Message-ID: <1247170877.16377.25.camel@auk31.llnl.gov> Make api more consistent and make struct ibnd_fabric a struct that represents just fabric data by removing the ibmad_port and making it a function parameter in appropriate functions. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- >From 1426b5fbfa60cd28f4b3e52e32ae425b9199ee61 Mon Sep 17 00:00:00 2001 From: Albert Chu Date: Wed, 8 Jul 2009 10:03:28 -0700 Subject: [PATCH] Make api more consistent and make struct ibnd_fabric a struct that represents just fabric data by removing the ibmad_port and making it a function parameter in appropriate functions. Signed-off-by: Albert Chu Signed-off-by: Ira Weiny --- .../libibnetdisc/include/infiniband/ibnetdisc.h | 4 +- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 93 +++++++++++--------- 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h index 62d639f..e01ca6a 100644 --- a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h +++ b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h @@ -121,7 +121,6 @@ typedef struct ibnd_chassis { * Main fabric object which is returned and represents the data discovered */ typedef struct ibnd_fabric { - struct ibmad_port *ibmad_port; /* the node the discover was initiated from * "from" parameter in ibnd_discover_fabric * or by default the node you ar running on @@ -160,7 +159,8 @@ MAD_EXPORT void ibnd_destroy_fabric(ibnd_fabric_t *fabric); */ MAD_EXPORT ibnd_node_t *ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid); MAD_EXPORT ibnd_node_t *ibnd_find_node_dr(ibnd_fabric_t *fabric, char *dr_str); -MAD_EXPORT ibnd_node_t *ibnd_update_node(ibnd_fabric_t *fabric, ibnd_node_t *node); +MAD_EXPORT ibnd_node_t *ibnd_update_node(struct ibmad_port *ibmad_port, + ibnd_fabric_t *fabric, ibnd_node_t *node); typedef void (*ibnd_iter_node_func_t)(ibnd_node_t *node, void *user_data); MAD_EXPORT void ibnd_iter_nodes(ibnd_fabric_t *fabric, diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index ea121dc..7d9069b 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -69,8 +69,8 @@ decode_port_info(ibnd_port_t *port) } static int -get_port_info(struct ibnd_fabric_int *fabric, struct ibnd_port_int *port, - int portnum, ib_portid_t *portid) +get_port_info(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *fabric, + struct ibnd_port_int *port, int portnum, ib_portid_t *portid) { char width[64], speed[64]; int iwidth; @@ -81,7 +81,7 @@ get_port_info(struct ibnd_fabric_int *fabric, struct ibnd_port_int *port, ispeed = mad_get_field(port->port.info, 0, IB_PORT_LINK_SPEED_ACTIVE_F); if (!smp_query_via(port->port.info, portid, IB_ATTR_PORT_INFO, portnum, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) return -1; decode_port_info(&(port->port)); @@ -99,10 +99,11 @@ get_port_info(struct ibnd_fabric_int *fabric, struct ibnd_port_int *port, * Returns -1 if error. */ static int -query_node_info(struct ibnd_fabric_int *fabric, struct ibnd_node_int *node, ib_portid_t *portid) +query_node_info(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *fabric, + struct ibnd_node_int *node, ib_portid_t *portid) { if (!smp_query_via(&(node->node.info), portid, IB_ATTR_NODE_INFO, 0, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) return -1; /* decode just a couple of fields for quicker reference. */ @@ -118,25 +119,25 @@ query_node_info(struct ibnd_fabric_int *fabric, struct ibnd_node_int *node, ib_p * Returns 0 if non switch node is found, 1 if switch is found, -1 if error. */ static int -query_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *inode, - struct ibnd_port_int *iport, ib_portid_t *portid) +query_node(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *fabric, + struct ibnd_node_int *inode, struct ibnd_port_int *iport, ib_portid_t *portid) { ibnd_node_t *node = &(inode->node); ibnd_port_t *port = &(iport->port); void *nd = inode->node.nodedesc; - if (query_node_info(fabric, inode, portid)) + if (query_node_info(ibmad_port, fabric, inode, portid)) return -1; port->portnum = mad_get_field(node->info, 0, IB_NODE_LOCAL_PORT_F); port->guid = mad_get_field64(node->info, 0, IB_NODE_PORT_GUID_F); if (!smp_query_via(nd, portid, IB_ATTR_NODE_DESC, 0, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) return -1; if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, 0, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) return -1; decode_port_info(port); @@ -148,7 +149,7 @@ query_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *inode, /* after we have the sma information find out the real PortInfo for this port */ if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, port->portnum, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) return -1; decode_port_info(port); @@ -156,7 +157,7 @@ query_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *inode, port->lmc = (uint8_t) node->smalmc; if (!smp_query_via(node->switchinfo, portid, IB_ATTR_SWITCH_INFO, 0, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) node->smaenhsp0 = 0; /* assume base SP0 */ else mad_decode_field(node->switchinfo, IB_SW_ENHANCED_PORT0_F, &node->smaenhsp0); @@ -177,7 +178,8 @@ add_port_to_dpath(ib_dr_path_t *path, int nextport) } static int -extend_dpath(struct ibnd_fabric_int *f, ib_portid_t *portid, int nextport) +extend_dpath(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *f, + ib_portid_t *portid, int nextport) { int rc = 0; @@ -185,7 +187,7 @@ extend_dpath(struct ibnd_fabric_int *f, ib_portid_t *portid, int nextport) /* If we were LID routed we need to set up the drslid */ if (!f->selfportid.lid) if (ib_resolve_self_via(&f->selfportid, NULL, NULL, - f->fabric.ibmad_port) < 0) + ibmad_port) < 0) return -1; portid->drpath.drslid = (uint16_t) f->selfportid.lid; @@ -244,8 +246,25 @@ ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid) return NULL; } +static int +_check_ibmad_port(struct ibmad_port *ibmad_port) +{ + if (!ibmad_port) { + IBND_DEBUG("ibmad_port must be specified\n"); + return (-1); + } + if (mad_rpc_class_agent(ibmad_port, IB_SMI_CLASS) == -1 + || + mad_rpc_class_agent(ibmad_port, IB_SMI_DIRECT_CLASS) == -1) { + IBND_DEBUG("ibmad_port must be opened with " + "IB_SMI_CLASS && IB_SMI_DIRECT_CLASS\n"); + return (-1); + } + return (0); +} + ibnd_node_t * -ibnd_update_node(ibnd_fabric_t *fabric, ibnd_node_t *node) +ibnd_update_node(struct ibmad_port *ibmad_port, ibnd_fabric_t *fabric, ibnd_node_t *node) { char portinfo_port0[IB_SMP_DATA_SIZE]; void *nd = node->nodedesc; @@ -253,30 +272,34 @@ ibnd_update_node(ibnd_fabric_t *fabric, ibnd_node_t *node) struct ibnd_fabric_int *f = CONV_FABRIC_INTERNAL(fabric); struct ibnd_node_int *n = CONV_NODE_INTERNAL(node); - if (query_node_info(f, n, &(n->node.path_portid))) + if (_check_ibmad_port(ibmad_port) < 0) + return (NULL); + + if (query_node_info(ibmad_port, f, n, &(n->node.path_portid))) return (NULL); if (!smp_query_via(nd, &(n->node.path_portid), IB_ATTR_NODE_DESC, 0, timeout_ms, - f->fabric.ibmad_port)) + ibmad_port)) return (NULL); /* update all the port info's */ for (p = 1; p >= n->node.numports; p++) { - get_port_info(f, CONV_PORT_INTERNAL(n->node.ports[p]), p, &(n->node.path_portid)); + get_port_info(ibmad_port, f, CONV_PORT_INTERNAL(n->node.ports[p]), + p, &(n->node.path_portid)); } if (n->node.type != IB_NODE_SWITCH) goto done; if (!smp_query_via(portinfo_port0, &(n->node.path_portid), IB_ATTR_PORT_INFO, 0, timeout_ms, - f->fabric.ibmad_port)) + ibmad_port)) return (NULL); n->node.smalid = mad_get_field(portinfo_port0, 0, IB_PORT_LID_F); n->node.smalmc = mad_get_field(portinfo_port0, 0, IB_PORT_LMC_F); if (!smp_query_via(node->switchinfo, &(n->node.path_portid), IB_ATTR_SWITCH_INFO, 0, timeout_ms, - f->fabric.ibmad_port)) + ibmad_port)) node->smaenhsp0 = 0; /* assume base SP0 */ else mad_decode_field(node->switchinfo, IB_SW_ENHANCED_PORT0_F, &n->node.smaenhsp0); @@ -441,7 +464,8 @@ link_ports(struct ibnd_node_int *node, struct ibnd_port_int *port, } static int -get_remote_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *node, struct ibnd_port_int *port, +get_remote_node(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *fabric, + struct ibnd_node_int *node, struct ibnd_port_int *port, ib_portid_t *path, int portnum, int dist) { struct ibnd_node_int node_buf; @@ -458,10 +482,10 @@ get_remote_node(struct ibnd_fabric_int *fabric, struct ibnd_node_int *node, stru != IB_PORT_PHYS_STATE_LINKUP) return -1; - if (extend_dpath(fabric, path, portnum) < 0) + if (extend_dpath(ibmad_port, fabric, path, portnum) < 0) return -1; - if (query_node(fabric, &node_buf, &port_buf, path)) { + if (query_node(ibmad_port, fabric, &node_buf, &port_buf, path)) { IBND_DEBUG("NodeInfo on %s failed, skipping port", portid2str(path)); path->drpath.cnt--; /* restore path */ @@ -504,18 +528,8 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, ib_portid_t *path; int max_hops = MAXHOPS-1; /* default find everything */ - if (!ibmad_port) { - IBND_DEBUG("ibmad_port must be specified to " - "ibnd_discover_fabric\n"); - return (NULL); - } - if (mad_rpc_class_agent(ibmad_port, IB_SMI_CLASS) == -1 - || - mad_rpc_class_agent(ibmad_port, IB_SMI_DIRECT_CLASS) == -1) { - IBND_DEBUG("ibmad_port must be opened with " - "IB_SMI_CLASS && IB_SMI_DIRECT_CLASS\n"); + if (_check_ibmad_port(ibmad_port) < 0) return (NULL); - } /* if not everything how much? */ if (hops >= 0) { @@ -533,14 +547,12 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, memset(fabric, 0, sizeof(*fabric)); - fabric->fabric.ibmad_port = ibmad_port; - IBND_DEBUG("from %s\n", portid2str(from)); memset(&node_buf, 0, sizeof(node_buf)); memset(&port_buf, 0, sizeof(port_buf)); - if (query_node(fabric, &node_buf, &port_buf, from)) { + if (query_node(ibmad_port, fabric, &node_buf, &port_buf, from)) { IBND_DEBUG("can't reach node %s\n", portid2str(from)); goto error; } @@ -555,7 +567,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, if (!port) IBPANIC("out of memory"); - if(get_remote_node(fabric, node, port, from, + if(get_remote_node(ibmad_port, fabric, node, port, from, mad_get_field(node->node.info, 0, IB_NODE_LOCAL_PORT_F), 0) < 0) return ((ibnd_fabric_t *)fabric); @@ -574,7 +586,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, IB_NODE_LOCAL_PORT_F)) continue; - if (get_port_info(fabric, &port_buf, i, path)) { + if (get_port_info(ibmad_port, fabric, &port_buf, i, path)) { IBND_DEBUG("can't reach node %s port %d", portid2str(path), i); continue; } @@ -593,7 +605,8 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, 0, IB_NODE_PORT_GUID_F); } - get_remote_node(fabric, node, port, path, i, dist); + get_remote_node(ibmad_port, fabric, node, port, + path, i, dist); } } } -- 1.5.4.5 From chu11 at llnl.gov Thu Jul 9 13:21:23 2009 From: chu11 at llnl.gov (Al Chu) Date: Thu, 09 Jul 2009 13:21:23 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [4/4] libibnetdisc cleanup patches round 2 Message-ID: <1247170883.16377.27.camel@auk31.llnl.gov> Remove timeout_ms parameter to ibnd_discover_fabric, timeout parameter should be specified via the ibmad_port. Remove extraneous use of global timeout_ms in library. Adjust ibnetdiscover, ibqueryerrors, iblinkinfo, and test code appropriately for adjustment. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- >From 65ce1d297e46737fe74c0544da901ed96d72401f Mon Sep 17 00:00:00 2001 From: Albert Chu Date: Wed, 8 Jul 2009 17:36:37 -0700 Subject: [PATCH] Remove timeout_ms parameter to ibnd_discover_fabric, timeout parameter should be specified via the ibmad_port. Remove extraneous use of global timeout_ms in library. Adjust ibnetdiscover, ibqueryerrors, iblinkinfo, and test code appropriately for adjustment. Signed-off-by: Albert Chu Signed-off-by: Ira Weiny --- .../libibnetdisc/include/infiniband/ibnetdisc.h | 4 --- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 21 +++++++++---------- infiniband-diags/libibnetdisc/test/testleaks.c | 6 +++- infiniband-diags/src/iblinkinfo.c | 7 ++++- infiniband-diags/src/ibnetdiscover.c | 8 +++--- infiniband-diags/src/ibqueryerrors.c | 7 ++++- 6 files changed, 28 insertions(+), 25 deletions(-) diff --git a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h index e01ca6a..5fb5d7b 100644 --- a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h +++ b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h @@ -140,13 +140,9 @@ MAD_EXPORT void ibnd_debug(int i); MAD_EXPORT void ibnd_show_progress(int i); MAD_EXPORT ibnd_fabric_t *ibnd_discover_fabric(struct ibmad_port *ibmad_port, - int timeout_ms, ib_portid_t *from, int hops); /** * open: (required) ibmad_port object from libibmad - * timeout_ms: (required) gives the timeout for a _SINGLE_ query on - * the fabric. So if there are multiple nodes not - * responding this may result in a lengthy delay. * from: (optional) specify the node to start scanning from. * If NULL start from the node we are running on. * hops: (optional) Specify how much of the fabric to traverse. diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index 88fb7df..d33f5ad 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -57,7 +57,6 @@ #include "internal.h" #include "chassis.h" -static int timeout_ms = 2000; static int show_progress = 0; int ibdebug; @@ -80,7 +79,7 @@ get_port_info(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *fabric, iwidth = mad_get_field(port->port.info, 0, IB_PORT_LINK_WIDTH_ACTIVE_F); ispeed = mad_get_field(port->port.info, 0, IB_PORT_LINK_SPEED_ACTIVE_F); - if (!smp_query_via(port->port.info, portid, IB_ATTR_PORT_INFO, portnum, timeout_ms, + if (!smp_query_via(port->port.info, portid, IB_ATTR_PORT_INFO, portnum, 0, ibmad_port)) return -1; @@ -102,7 +101,7 @@ static int query_node_info(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *fabric, struct ibnd_node_int *node, ib_portid_t *portid) { - if (!smp_query_via(&(node->node.info), portid, IB_ATTR_NODE_INFO, 0, timeout_ms, + if (!smp_query_via(&(node->node.info), portid, IB_ATTR_NODE_INFO, 0, 0, ibmad_port)) return -1; @@ -132,11 +131,11 @@ query_node(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *fabric, port->portnum = mad_get_field(node->info, 0, IB_NODE_LOCAL_PORT_F); port->guid = mad_get_field64(node->info, 0, IB_NODE_PORT_GUID_F); - if (!smp_query_via(nd, portid, IB_ATTR_NODE_DESC, 0, timeout_ms, + if (!smp_query_via(nd, portid, IB_ATTR_NODE_DESC, 0, 0, ibmad_port)) return -1; - if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, 0, timeout_ms, + if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, 0, 0, ibmad_port)) return -1; decode_port_info(port); @@ -148,7 +147,7 @@ query_node(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *fabric, node->smalmc = port->lmc; /* after we have the sma information find out the real PortInfo for this port */ - if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, port->portnum, timeout_ms, + if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, port->portnum, 0, ibmad_port)) return -1; decode_port_info(port); @@ -156,7 +155,7 @@ query_node(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *fabric, port->base_lid = (uint16_t) node->smalid; /* LID is still defined by port 0 */ port->lmc = (uint8_t) node->smalmc; - if (!smp_query_via(node->switchinfo, portid, IB_ATTR_SWITCH_INFO, 0, timeout_ms, + if (!smp_query_via(node->switchinfo, portid, IB_ATTR_SWITCH_INFO, 0, 0, ibmad_port)) node->smaenhsp0 = 0; /* assume base SP0 */ else @@ -293,7 +292,7 @@ ibnd_update_node(struct ibmad_port *ibmad_port, ibnd_fabric_t *fabric, ibnd_node if (query_node_info(ibmad_port, f, n, &(n->node.path_portid))) return (NULL); - if (!smp_query_via(nd, &(n->node.path_portid), IB_ATTR_NODE_DESC, 0, timeout_ms, + if (!smp_query_via(nd, &(n->node.path_portid), IB_ATTR_NODE_DESC, 0, 0, ibmad_port)) return (NULL); @@ -306,14 +305,14 @@ ibnd_update_node(struct ibmad_port *ibmad_port, ibnd_fabric_t *fabric, ibnd_node if (n->node.type != IB_NODE_SWITCH) goto done; - if (!smp_query_via(portinfo_port0, &(n->node.path_portid), IB_ATTR_PORT_INFO, 0, timeout_ms, + if (!smp_query_via(portinfo_port0, &(n->node.path_portid), IB_ATTR_PORT_INFO, 0, 0, ibmad_port)) return (NULL); n->node.smalid = mad_get_field(portinfo_port0, 0, IB_PORT_LID_F); n->node.smalmc = mad_get_field(portinfo_port0, 0, IB_PORT_LMC_F); - if (!smp_query_via(node->switchinfo, &(n->node.path_portid), IB_ATTR_SWITCH_INFO, 0, timeout_ms, + if (!smp_query_via(node->switchinfo, &(n->node.path_portid), IB_ATTR_SWITCH_INFO, 0, 0, ibmad_port)) node->smaenhsp0 = 0; /* assume base SP0 */ else @@ -536,7 +535,7 @@ get_remote_node(struct ibmad_port *ibmad_port, struct ibnd_fabric_int *fabric, } ibnd_fabric_t * -ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, +ibnd_discover_fabric(struct ibmad_port *ibmad_port, ib_portid_t *from, int hops) { struct ibnd_fabric_int *fabric = NULL; diff --git a/infiniband-diags/libibnetdisc/test/testleaks.c b/infiniband-diags/libibnetdisc/test/testleaks.c index 0d009c3..a8f5300 100644 --- a/infiniband-diags/libibnetdisc/test/testleaks.c +++ b/infiniband-diags/libibnetdisc/test/testleaks.c @@ -161,11 +161,13 @@ main(int argc, char **argv) ibmad_port = mad_rpc_open_port(ca, ca_port, mgmt_classes, 2); + mad_rpc_set_timeout(ibmad_port, timeout_ms); + while (iters == -1 || iters-- > 0) { if (from) { /* only scan part of the fabric */ str2drpath(&(port_id.drpath), from, 0, 0); - if ((fabric = ibnd_discover_fabric(ibmad_port, timeout_ms, + if ((fabric = ibnd_discover_fabric(ibmad_port, &port_id, hops)) == NULL) { fprintf(stderr, "discover failed\n"); rc = 1; @@ -173,7 +175,7 @@ main(int argc, char **argv) } guid = 0; } else { - if ((fabric = ibnd_discover_fabric(ibmad_port, timeout_ms, NULL, -1)) == NULL) { + if ((fabric = ibnd_discover_fabric(ibmad_port, NULL, -1)) == NULL) { fprintf(stderr, "discover failed\n"); rc = 1; goto close_port; diff --git a/infiniband-diags/src/iblinkinfo.c b/infiniband-diags/src/iblinkinfo.c index bba5c5c..5dfadee 100644 --- a/infiniband-diags/src/iblinkinfo.c +++ b/infiniband-diags/src/iblinkinfo.c @@ -308,6 +308,9 @@ main(int argc, char **argv) exit(1); } + if (ibd_timeout) + mad_rpc_set_timeout(ibmad_port, ibd_timeout); + node_name_map = open_node_name_map(node_name_map_file); if (dr_path) { @@ -324,12 +327,12 @@ main(int argc, char **argv) } if (resolved >= 0) - if ((fabric = ibnd_discover_fabric(ibmad_port, ibd_timeout, &port_id, + if ((fabric = ibnd_discover_fabric(ibmad_port, &port_id, hops)) == NULL) IBWARN("Single node discover failed; attempting full scan\n"); if (!fabric) - if ((fabric = ibnd_discover_fabric(ibmad_port, ibd_timeout, NULL, -1)) == NULL) { + if ((fabric = ibnd_discover_fabric(ibmad_port, NULL, -1)) == NULL) { fprintf(stderr, "discover failed\n"); rc = 1; goto close_port; diff --git a/infiniband-diags/src/ibnetdiscover.c b/infiniband-diags/src/ibnetdiscover.c index 1339485..37f769c 100644 --- a/infiniband-diags/src/ibnetdiscover.c +++ b/infiniband-diags/src/ibnetdiscover.c @@ -670,9 +670,6 @@ int main(int argc, char **argv) argc -= optind; argv += optind; - if (ibd_timeout) - timeout = ibd_timeout; - if (ibverbose) ibnd_debug(1); @@ -680,12 +677,15 @@ int main(int argc, char **argv) if (!ibmad_port) IBERROR("Failed to open %s port %d", ibd_ca, ibd_ca_port); + if (ibd_timeout) + mad_rpc_set_timeout(ibmad_port, ibd_timeout); + if (argc && !(f = fopen(argv[0], "w"))) IBERROR("can't open file %s for writing", argv[0]); node_name_map = open_node_name_map(node_name_map_file); - if ((fabric = ibnd_discover_fabric(ibmad_port, ibd_timeout, NULL, -1)) == NULL) + if ((fabric = ibnd_discover_fabric(ibmad_port, NULL, -1)) == NULL) IBERROR("discover failed\n"); if (ports_report) diff --git a/infiniband-diags/src/ibqueryerrors.c b/infiniband-diags/src/ibqueryerrors.c index 09f57c5..2c85423 100644 --- a/infiniband-diags/src/ibqueryerrors.c +++ b/infiniband-diags/src/ibqueryerrors.c @@ -422,6 +422,9 @@ main(int argc, char **argv) if (!ibmad_port) IBERROR("Failed to open port; %s:%d\n", ibd_ca, ibd_ca_port); + if (ibd_timeout) + mad_rpc_set_timeout(ibmad_port, ibd_timeout); + node_name_map = open_node_name_map(node_name_map_file); /* limit the scan the fabric around the target */ @@ -437,12 +440,12 @@ main(int argc, char **argv) } if (resolved >= 0) - if ((fabric = ibnd_discover_fabric(ibmad_port, ibd_timeout, &portid, + if ((fabric = ibnd_discover_fabric(ibmad_port, &portid, 0)) == NULL) IBWARN("Single node discover failed; attempting full scan\n"); if (!fabric) /* do a full scan */ - if ((fabric = ibnd_discover_fabric(ibmad_port, ibd_timeout, NULL, -1)) == NULL) { + if ((fabric = ibnd_discover_fabric(ibmad_port, NULL, -1)) == NULL) { fprintf(stderr, "discover failed\n"); rc = 1; goto close_port; -- 1.5.4.5 From akepner at sgi.com Thu Jul 9 13:51:32 2009 From: akepner at sgi.com (akepner at sgi.com) Date: Thu, 9 Jul 2009 13:51:32 -0700 Subject: [ofa-general] Re: A question about tx lock in ipoib_flush_paths In-Reply-To: <4A56190E.1080105@voltaire.com> References: <4A4E190D.7010004@voltaire.com> <4A52930F.9050408@gmail.com> <4A56190E.1080105@voltaire.com> Message-ID: <20090709205132.GD12939@sgi.com> On Thu, Jul 09, 2009 at 07:21:34PM +0300, Yossi Etigin wrote: > .... > The path itself is rarely used, most of the time we take the ah from the > ipoib_neigh which is stashed in the kernel neighbour. > The scenario is that ipoib_start_xmit() runs after the flush task releases > the lock, and does 'neigh = *to_ipoib_neigh(skb_dst(skb)->neighbour)'. > Then the flush task continues to run and does path_free() which calls > ipoib_neigh_free(), and kfree-s the neighbour. Then the xmit routine > will go on using a stale neigh pointer. > This does seem possible, no? If so, it would be addressed by the patch I sent here: http://lists.openfabrics.org/pipermail/general/2009-July/060501.html because the ipoib_neigh structure wouldn't be freed until after ipoib_start_xmit() had done rcu_read_unlock(). -- Arthur From ranjit.pandit.ib at gmail.com Thu Jul 9 14:13:23 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Thu, 9 Jul 2009 14:13:23 -0700 Subject: [ofa-general] rdma/iWarp capability of an ethX interface Message-ID: <96f8e60e0907091413i92fabedo3255a211cbb2c44f@mail.gmail.com> Is there a way to query an ethX interface and find out if it's iWarp/RDMA capable? thanks, Ranjit From swise at opengridcomputing.com Thu Jul 9 15:05:04 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Thu, 09 Jul 2009 17:05:04 -0500 Subject: [ofa-general] rdma/iWarp capability of an ethX interface In-Reply-To: <96f8e60e0907091413i92fabedo3255a211cbb2c44f@mail.gmail.com> References: <96f8e60e0907091413i92fabedo3255a211cbb2c44f@mail.gmail.com> Message-ID: <4A566990.9030500@opengridcomputing.com> pandit ib wrote: > Is there a way to query an ethX interface and find out if it's > iWarp/RDMA capable? > > Unfortunately there isn't an easy way to do this (someone should implement this as part of the rdma services!). One way you can do it is to get all the IP addresses from each interface via SIOCGIFCONF and friends, and then attempt to rdma_bind() to each address/port 0. If it succeeds, then that interface supports rdma. OpenMPI does this... Steve. From rdreier at cisco.com Thu Jul 9 16:15:11 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 09 Jul 2009 16:15:11 -0700 Subject: [ofa-general] rdma/iWarp capability of an ethX interface In-Reply-To: <4A566990.9030500@opengridcomputing.com> (Steve Wise's message of "Thu, 09 Jul 2009 17:05:04 -0500") References: <96f8e60e0907091413i92fabedo3255a211cbb2c44f@mail.gmail.com> <4A566990.9030500@opengridcomputing.com> Message-ID: > > Is there a way to query an ethX interface and find out if it's > > iWarp/RDMA capable? > > > > > > Unfortunately there isn't an easy way to do this (someone should > implement this as part of the rdma services!). > > One way you can do it is to get all the IP addresses from each > interface via SIOCGIFCONF and friends, and then attempt to rdma_bind() > to each address/port 0. If it succeeds, then that interface supports > rdma. OpenMPI does this... Another semi-ugly way to do it is to look at the /sys/class/net/ethX/device link and then look at the /sys/class/infiniband/xxx/device links and see if it is the same. any From He.Huang at Sun.COM Thu Jul 9 19:56:30 2009 From: He.Huang at Sun.COM (Isaac Huang) Date: Thu, 09 Jul 2009 22:56:30 -0400 Subject: [ofa-general] rdma_listen() backlog In-Reply-To: References: <2119455328.846331247119814612.JavaMail.root@mail1.gatech.edu> <4A56005C.5030709@opengridcomputing.com> <20090709175727.GE5662@sun.com> Message-ID: <20090710025630.GM5662@sun.com> On Thu, Jul 09, 2009 at 11:25:59AM -0700, Sean Hefty wrote: > >Maybe I've missed something, but the last time I checked it appeared > >to me that for kernel RDMA CM the 'backlog' parameter was not used at > >all unless for iWarp transport. > > It's not used for kernel IB connections. Since connection requests are reported > through a callback, there's nothing to queue and it's unneeded. I see, thanks for the explanation. But then why would iWarp transport cares the 'backlog' while connection requests are also reported via a callback? iw_cm_listen() allocates resources based on 'backlog' and I saw this comment over it: @backlog: The maximum number of outstanding un-accepted inbound listen requests to queue. Thanks, Isaac From liranl at mellanox.co.il Thu Jul 9 23:30:08 2009 From: liranl at mellanox.co.il (Liran Liss) Date: Fri, 10 Jul 2009 09:30:08 +0300 Subject: [ofa-general] rdma/iWarp capability of an ethX interface In-Reply-To: References: <96f8e60e0907091413i92fabedo3255a211cbb2c44f@mail.gmail.com><4A566990.9030500@opengridcomputing.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD0350ED51@mtlexch01.mtl.com> I think that all iWARP devices encode their MAC in index0 of their gid table. You can look for a matching RDMA device by querying its gids. -----Original Message----- From: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Roland Dreier Sent: Friday, July 10, 2009 2:15 AM To: Steve Wise Cc: general at lists.openfabrics.org Subject: Re: [ofa-general] rdma/iWarp capability of an ethX interface > > Is there a way to query an ethX interface and find out if it's > > iWarp/RDMA capable? > > > > > > Unfortunately there isn't an easy way to do this (someone should > implement this as part of the rdma services!). > > One way you can do it is to get all the IP addresses from each > interface via SIOCGIFCONF and friends, and then attempt to rdma_bind() > to each address/port 0. If it succeeds, then that interface supports > rdma. OpenMPI does this... Another semi-ugly way to do it is to look at the /sys/class/net/ethX/device link and then look at the /sys/class/infiniband/xxx/device links and see if it is the same. any _______________________________________________ general mailing list general at lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From vlad at lists.openfabrics.org Fri Jul 10 02:38:31 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Fri, 10 Jul 2009 02:38:31 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090710-0200 daily build status Message-ID: <20090710093831.7568E102062E@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -Werror-implicit-function-declaration -fno-strict-aliasing -fno-common -ffreestanding -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: from /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.c:37: include/linux/skbuff.h: In function 'skb_add_data': include/linux/skbuff.h:1041: warning: pointer targets in passing argument 2 of 'csum_partial_copy_from_user' differ in signedness make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: from /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.c:37: include/linux/skbuff.h: In function 'skb_add_data': include/linux/skbuff.h:1041: warning: pointer targets in passing argument 2 of 'csum_partial_copy_from_user' differ in signedness make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090710-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From yosefe at voltaire.com Fri Jul 10 04:27:56 2009 From: yosefe at voltaire.com (Yossi Etigin) Date: Fri, 10 Jul 2009 14:27:56 +0300 Subject: [ofa-general] Re: A question about tx lock in ipoib_flush_paths In-Reply-To: <20090709205132.GD12939@sgi.com> References: <4A4E190D.7010004@voltaire.com> <4A52930F.9050408@gmail.com> <4A56190E.1080105@voltaire.com> <20090709205132.GD12939@sgi.com> Message-ID: <4A5725BC.70700@voltaire.com> akepner at sgi.com wrote: > On Thu, Jul 09, 2009 at 07:21:34PM +0300, Yossi Etigin wrote: >> .... >> The path itself is rarely used, most of the time we take the ah from the >> ipoib_neigh which is stashed in the kernel neighbour. >> The scenario is that ipoib_start_xmit() runs after the flush task releases >> the lock, and does 'neigh = *to_ipoib_neigh(skb_dst(skb)->neighbour)'. >> Then the flush task continues to run and does path_free() which calls >> ipoib_neigh_free(), and kfree-s the neighbour. Then the xmit routine >> will go on using a stale neigh pointer. >> > > This does seem possible, no? > > If so, it would be addressed by the patch I sent here: > > http://lists.openfabrics.org/pipermail/general/2009-July/060501.html > > because the ipoib_neigh structure wouldn't be freed until after > ipoib_start_xmit() had done rcu_read_unlock(). > Yes, I guess it would address the ipoib_neigh structure problem. But then neigh->ah might be freed while xmit is using it. From swise at opengridcomputing.com Fri Jul 10 06:40:40 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Fri, 10 Jul 2009 08:40:40 -0500 Subject: [ofa-general] rdma_listen() backlog In-Reply-To: <20090710025630.GM5662@sun.com> References: <2119455328.846331247119814612.JavaMail.root@mail1.gatech.edu> <4A56005C.5030709@opengridcomputing.com> <20090709175727.GE5662@sun.com> <20090710025630.GM5662@sun.com> Message-ID: <4A5744D8.2040204@opengridcomputing.com> Isaac Huang wrote: > On Thu, Jul 09, 2009 at 11:25:59AM -0700, Sean Hefty wrote: > >>> Maybe I've missed something, but the last time I checked it appeared >>> to me that for kernel RDMA CM the 'backlog' parameter was not used at >>> all unless for iWarp transport. >>> >> It's not used for kernel IB connections. Since connection requests are reported >> through a callback, there's nothing to queue and it's unneeded. >> > > I see, thanks for the explanation. But then why would iWarp transport > cares the 'backlog' while connection requests are also reported via a > callback? iw_cm_listen() allocates resources based on 'backlog' and I > saw this comment over it: > > @backlog: The maximum number of outstanding un-accepted inbound listen > requests to queue. > > The iWARP rnics also map this backlog to the tcp backlog used in listening for incoming tcp connections. From john.russo at qlogic.com Fri Jul 10 06:56:56 2009 From: john.russo at qlogic.com (John Russo) Date: Fri, 10 Jul 2009 08:56:56 -0500 Subject: [ofa-general] ***SPAM*** Request to track fixed PRs Message-ID: Can we add a "Fixed in" field to OFED's Bugzilla. Then when a PR gets fixed we would know what OFED release the fix was added to. Right now there doesn't seem to be a good way to identify the problems that have been corrected in a release without pouring through a ton of emails. This would be a quick and easy fix that would help a number of people. [cid:image001.jpg at 01CA0144.C2896AB0] John F. Russo Engineering Manager QLogic Corporation 780 Fifth Avenue Suite 140 King of Prussia, PA 19406 Direct: 610-233-4866 Fax: 610-233-4777 Cell: 610-246-9903 www.qlogic.com -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: image001.jpg Type: image/jpeg Size: 1173 bytes Desc: image001.jpg URL: From akepner at sgi.com Fri Jul 10 09:08:50 2009 From: akepner at sgi.com (akepner at sgi.com) Date: Fri, 10 Jul 2009 09:08:50 -0700 Subject: [ofa-general] Re: A question about tx lock in ipoib_flush_paths In-Reply-To: <4A5725BC.70700@voltaire.com> References: <4A4E190D.7010004@voltaire.com> <4A52930F.9050408@gmail.com> <4A56190E.1080105@voltaire.com> <20090709205132.GD12939@sgi.com> <4A5725BC.70700@voltaire.com> Message-ID: <20090710160850.GB29594@sgi.com> On Fri, Jul 10, 2009 at 02:27:56PM +0300, Yossi Etigin wrote: > akepner at sgi.com wrote: > > ..... > > If so, it would be addressed by the patch I sent here: > > > > http://lists.openfabrics.org/pipermail/general/2009-July/060501.html > > > > because the ipoib_neigh structure wouldn't be freed until after > > ipoib_start_xmit() had done rcu_read_unlock(). > > > > Yes, I guess it would address the ipoib_neigh structure problem. > But then neigh->ah might be freed while xmit is using it. > Maybe I'm missing something, but what you say doesn't look correct. path_free() now does: if (neigh->ah) ipoib_put_ah(neigh->ah); (with a comment explaining that it's OK to do that while holding priv->lock.) But with the patch above, ipoib_put_ah() is deferred to the rcu callback (just like the kfree() of the ipoib_neigh structure is). -- Arthur From yosefe at voltaire.com Fri Jul 10 09:27:00 2009 From: yosefe at voltaire.com (Yossi Etigin) Date: Fri, 10 Jul 2009 19:27:00 +0300 Subject: [ofa-general] Re: A question about tx lock in ipoib_flush_paths In-Reply-To: <20090710160850.GB29594@sgi.com> References: <4A4E190D.7010004@voltaire.com> <4A52930F.9050408@gmail.com> <4A56190E.1080105@voltaire.com> <20090709205132.GD12939@sgi.com> <4A5725BC.70700@voltaire.com> <20090710160850.GB29594@sgi.com> Message-ID: <4A576BD4.8050407@voltaire.com> akepner at sgi.com wrote: > On Fri, Jul 10, 2009 at 02:27:56PM +0300, Yossi Etigin wrote: >> akepner at sgi.com wrote: >>> ..... >>> If so, it would be addressed by the patch I sent here: >>> >>> http://lists.openfabrics.org/pipermail/general/2009-July/060501.html >>> >>> because the ipoib_neigh structure wouldn't be freed until after >>> ipoib_start_xmit() had done rcu_read_unlock(). >>> >> Yes, I guess it would address the ipoib_neigh structure problem. >> But then neigh->ah might be freed while xmit is using it. >> > > Maybe I'm missing something, but what you say doesn't look > correct. > > path_free() now does: > > if (neigh->ah) > ipoib_put_ah(neigh->ah); > > (with a comment explaining that it's OK to do that while holding > priv->lock.) > > But with the patch above, ipoib_put_ah() is deferred to the rcu > callback (just like the kfree() of the ipoib_neigh structure is). > I see, you are right. ipoib_put_ah is moved to neigh_free context. Looks like your patch does the job. From bart.vanassche at gmail.com Fri Jul 10 09:55:57 2009 From: bart.vanassche at gmail.com (Bart Van Assche) Date: Fri, 10 Jul 2009 18:55:57 +0200 Subject: [ofa-general] 2.6.30.1: possible irq lock inversion dependency detected Message-ID: It would be great if someone could have a look at this issue. See also http://bugzilla.kernel.org/show_bug.cgi?id=13757 ========================================================= [ INFO: possible irq lock inversion dependency detected ] 2.6.30.1-scst-debug #5 --------------------------------------------------------- swapper/0 just changed the state of lock: (&priv->lock){-.-...}, at: [] ipoib_cm_rx_event_handler+0x4f/0xa0 [ib_ipoib] but this lock took another, HARDIRQ-unsafe lock in the past: (&(&mad_agent_priv->timed_work)->timer){+.-...} and interrupts could create inverse lock ordering between them. other info that might help us debug this: no locks held by swapper/0. the first lock's dependencies: -> (&priv->lock){-.-...} ops: 0 { IN-HARDIRQ-W at: [] 0xffffffffffffffff IN-SOFTIRQ-W at: [] 0xffffffffffffffff INITIAL USE at: [] __lock_acquire+0x171/0x1b50 [] lock_acquire+0x108/0x150 [] _spin_lock_irq+0x3c/0x50 [] ipoib_mcast_join_task+0x1fe/0x380 [ib_ipoib] [] worker_thread+0x21d/0x390 [] kthread+0x56/0x90 [] child_rip+0xa/0x20 [] 0xffffffffffffffff } [ ... ] From ranjit.pandit.ib at gmail.com Fri Jul 10 10:10:53 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Fri, 10 Jul 2009 10:10:53 -0700 Subject: [ofa-general] rdma/iWarp capability of an ethX interface In-Reply-To: <5D49E7A8952DC44FB38C38FA0D758EAD0350ED51@mtlexch01.mtl.com> References: <96f8e60e0907091413i92fabedo3255a211cbb2c44f@mail.gmail.com> <4A566990.9030500@opengridcomputing.com> <5D49E7A8952DC44FB38C38FA0D758EAD0350ED51@mtlexch01.mtl.com> Message-ID: <96f8e60e0907101010l395a1e70jcb0152bb8506c68d@mail.gmail.com> Thanks for all the suggestions. I'm tending towards trying the rdma_bind approach. On Thu, Jul 9, 2009 at 11:30 PM, Liran Liss wrote: > I think that all iWARP devices encode their MAC in index0 of their gid > table. > You can look for a matching RDMA device by querying its gids. > > > -----Original Message----- > From: general-bounces at lists.openfabrics.org > [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Roland > Dreier > Sent: Friday, July 10, 2009 2:15 AM > To: Steve Wise > Cc: general at lists.openfabrics.org > Subject: Re: [ofa-general] rdma/iWarp capability of an ethX interface > > >  > > Is there a way to query an ethX interface and find out if it's  > > > iWarp/RDMA capable? >  > > >  > > >  > >  > Unfortunately there isn't an easy way to do this (someone should  > > implement this as part of the rdma services!). >  > >  > One way you can do it is to get all the IP addresses from each  > > interface via SIOCGIFCONF and friends, and then attempt to rdma_bind() >> to each address/port 0.  If it succeeds, then that interface supports >> rdma.  OpenMPI does this... > > Another semi-ugly way to do it is to look at the > /sys/class/net/ethX/device link and then look at the > /sys/class/infiniband/xxx/device links and see if it is the same. > any > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From ranjit.pandit.ib at gmail.com Fri Jul 10 11:15:09 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Fri, 10 Jul 2009 11:15:09 -0700 Subject: [ofa-general] Memory registration limit of 16GB with Chelsio Message-ID: <96f8e60e0907101115i11fcf051h8508fdc18282368b@mail.gmail.com> Hi, In our testing we are hitting a memory registration limit of 16GB with the Chelsio cards. Here are some sample results for different region sizes. region size Number of Successful registration Total 2MB 8178 ~16GB 16MB 1021 ~16GB 128MB 127 ~16GB Are there any knobs that we can use to raise that limit? For example, on mlx4 we use "log_num_mtt" module load parameter to increase the limit. Here is a sample output of the test: [root at lv2 ~]# ./mr-test 16777216 mr-test: bufsize 16777216 device # 0 name="cxgb3_0" guid="00074305ca890000" ibv_open_device() context=0x77b5100 ibv_alloc_pd() pd=0x77b50e0 alloc: 1021 ibv_reg_mr failed:: Cannot allocate memory fw_ver: 7.4.0 max_mr_size 0x100000000 max_mr: 32768, could only register 1021 regions; Region size 16MB sleep 5 sec free: 0 done From hal.rosenstock at gmail.com Fri Jul 10 11:27:51 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Fri, 10 Jul 2009 14:27:51 -0400 Subject: [ofa-general] sending mad in parallel mode and perfquery In-Reply-To: <4A5628F9.6070500@yandex.ru> References: <4A5628F9.6070500@yandex.ru> Message-ID: On 7/9/09, korisk wrote: > Next question about param "-a" of perfquery command. It doesn't work. > Has anyone ideas about it? Not all IB devices support "-a" (all ports). More recent versions of perfquery emulate all ports support for those devices which don't natively support this. -- Hal > Thank you. > > Sincerely, > Korisk. > > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > From hal.rosenstock at gmail.com Fri Jul 10 11:37:01 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Fri, 10 Jul 2009 14:37:01 -0400 Subject: [ofa-general] [PATCH] Lock _do_madrpc for thread safety In-Reply-To: <20090708174720.f61d44fc.weiny2@llnl.gov> References: <20090708174720.f61d44fc.weiny2@llnl.gov> Message-ID: Ira, On 7/8/09, Ira Weiny wrote: > Sasha, > > I am working on making libibnetdisc a parallel implementation. As a result > I > have found that _do_madrpc is not thread safe. Have you read Sasha's commit 51d25384626a7b4ba386c414ed56c647a7bf64df from 12/26/08 ? In it, he states " I think that it will be more robust for multithreaded application to use its own synchronization methods (pthread mutex or any other) for better control. " > The following patch fixes > this. However, I don't know you want to do... > > If one only uses mad_rpc and mad_rpc_rmpp then the patch works. However, if > someone is using mad_send_via at the same time _do_madrpc will still fail. > Is > it by design that some responses will be lost while _do_madrpc is looking > for > it's response via the TID? > > Also, according to C13-18.1.1 and C13-19.1.1 you must use the SGID (or SLID) > and the MgmtClass in addition to the TID to determine the uniqueness of a > message. The SGID (or SLID) is of course the same but should the MgmtClass > be checked here as well? > > Finally, why does _do_madrpc cast the transaction id to a 32 bit value? The kernel uses the high 32 bits for an agent ID. See kernel Documentation/infiniband/user_mad.txt "Transaction IDs". -- Hal > Confused, > Ira From swise at opengridcomputing.com Fri Jul 10 11:45:33 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Fri, 10 Jul 2009 13:45:33 -0500 Subject: [ofa-general] Memory registration limit of 16GB with Chelsio In-Reply-To: <96f8e60e0907101115i11fcf051h8508fdc18282368b@mail.gmail.com> References: <96f8e60e0907101115i11fcf051h8508fdc18282368b@mail.gmail.com> Message-ID: <4A578C4D.2060003@opengridcomputing.com> Looks like you're running out of adapter resources for storing the physical buffers lists (PBLs). The cards have 32MB of memory for storing PBLs. 32MB of memory can store 4MB of 8B physical addresses. For user mode, your page size is 4KB, so that maps out to 4MB * 4KB = 16GB of memory. Hey Felix, Can we somehow adjust the PMRX memory map dynamically? For instance, maybe drop the number of TPTs to allow for more PBL memory? Steve pandit ib wrote: > Hi, > > In our testing we are hitting a memory registration limit of 16GB with > the Chelsio cards. > > Here are some sample results for different region sizes. > > region size Number of Successful registration Total > 2MB 8178 > ~16GB > 16MB 1021 > ~16GB > 128MB 127 > ~16GB > > Are there any knobs that we can use to raise that limit? > > For example, on mlx4 we use "log_num_mtt" module load parameter to > increase the limit. > > Here is a sample output of the test: > > [root at lv2 ~]# ./mr-test 16777216 > > mr-test: bufsize 16777216 > device # 0 name="cxgb3_0" guid="00074305ca890000" > ibv_open_device() context=0x77b5100 > ibv_alloc_pd() pd=0x77b50e0 > alloc: 1021 > ibv_reg_mr failed:: Cannot allocate memory > fw_ver: 7.4.0 > max_mr_size 0x100000000 > max_mr: 32768, could only register 1021 regions; Region size 16MB > sleep 5 sec > free: 0 > done > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From rdreier at cisco.com Fri Jul 10 11:52:09 2009 From: rdreier at cisco.com (Roland Dreier) Date: Fri, 10 Jul 2009 11:52:09 -0700 Subject: [ofa-general] Re: 2.6.30.1: possible irq lock inversion dependency detected In-Reply-To: (Bart Van Assche's message of "Fri, 10 Jul 2009 18:55:57 +0200") References: Message-ID: Wow, that is crazy lockdep output -- I can't really figure out what it thinks is wrong exactly. Can you apply the following total hack and redo the test, and send the lockdep output? It should make the lockdep warning trigger sooner and, I hope, make the warning shorter and easier to read. Thanks, Roland diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index e319d91..9593be8 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -1028,6 +1028,17 @@ static const struct net_device_ops ipoib_netdev_ops = { .ndo_neigh_setup = ipoib_neigh_setup_dev, }; +extern void mlx4_hack_irq_set(void (*f)(void *), void *a); +static void irqfunc(void *priv_ptr) +{ + struct ipoib_dev_priv *priv = priv_ptr; + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + spin_unlock_irqrestore(&priv->lock, flags); + ipoib_warn(priv, "got irq\n"); +} + static void ipoib_setup(struct net_device *dev) { struct ipoib_dev_priv *priv = netdev_priv(dev); @@ -1064,6 +1075,7 @@ static void ipoib_setup(struct net_device *dev) ipoib_lro_setup(priv); spin_lock_init(&priv->lock); + mlx4_hack_irq_set(irqfunc, priv); mutex_init(&priv->vlan_mutex); diff --git a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c index b9ceddd..51572c2 100644 --- a/drivers/net/mlx4/eq.c +++ b/drivers/net/mlx4/eq.c @@ -136,6 +136,21 @@ struct mlx4_eqe { u8 owner; } __attribute__((packed)); +static void (*irqfunc)(void *); +static void *irqarg; +static DEFINE_SPINLOCK(irqlock); + +void mlx4_hack_irq_set(void (*f)(void *), void *a) +{ + unsigned long flags; + + spin_lock_irqsave(&irqlock, flags); + irqfunc = f; + irqarg = a; + spin_unlock_irqrestore(&irqlock, flags); +} +EXPORT_SYMBOL(mlx4_hack_irq_set); + static void eq_set_ci(struct mlx4_eq *eq, int req_not) { __raw_writel((__force u32) cpu_to_be32((eq->cons_index & 0xffffff) | @@ -278,6 +293,18 @@ static irqreturn_t mlx4_msi_x_interrupt(int irq, void *eq_ptr) { struct mlx4_eq *eq = eq_ptr; struct mlx4_dev *dev = eq->dev; + unsigned long flags; + void (*f)(void *); + void *a; + + spin_lock_irqsave(&irqlock, flags); + f = irqfunc; + a = irqarg; + irqfunc = NULL; + spin_unlock_irqrestore(&irqlock, flags); + + if (f) + f(a); mlx4_eq_int(dev, eq); From bart.vanassche at gmail.com Fri Jul 10 12:29:51 2009 From: bart.vanassche at gmail.com (Bart Van Assche) Date: Fri, 10 Jul 2009 21:29:51 +0200 Subject: [ofa-general] Re: 2.6.30.1: possible irq lock inversion dependency detected In-Reply-To: References: Message-ID: On Fri, Jul 10, 2009 at 8:52 PM, Roland Dreier wrote: > > Wow, that is crazy lockdep output -- I can't really figure out what it > thinks is wrong exactly. > > Can you apply the following total hack and redo the test, and send the > lockdep output?  It should make the lockdep warning trigger sooner and, > I hope, make the warning shorter and easier to read. Thanks for the patch. With the patch applied the lockdep warning indeed occurs sooner and the output is now indeed shorter. You can find the new lockdep output here: http://bugzilla.kernel.org/attachment.cgi?id=22305. Bart. From weiny2 at llnl.gov Fri Jul 10 13:26:33 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Fri, 10 Jul 2009 13:26:33 -0700 Subject: [ofa-general] [PATCH] Lock _do_madrpc for thread safety In-Reply-To: References: <20090708174720.f61d44fc.weiny2@llnl.gov> Message-ID: <20090710132633.e85c6181.weiny2@llnl.gov> On Fri, 10 Jul 2009 14:37:01 -0400 Hal Rosenstock wrote: > Ira, > > On 7/8/09, Ira Weiny wrote: > > Sasha, > > > > I am working on making libibnetdisc a parallel implementation. As a result > > I > > have found that _do_madrpc is not thread safe. > > Have you read Sasha's commit 51d25384626a7b4ba386c414ed56c647a7bf64df > from 12/26/08 ? In it, he states " I think that it will be more robust > for multithreaded > application to use its own synchronization methods (pthread mutex or any > other) for better control. " I missed that. Thanks. However, I am starting to lean towards putting something in libibmad. The problem for me is that in the ibnetdisc library I can't control what the application is doing. And more importantly I can't allow the application to do anything (at least at certain times). I will wait for Sasha to weigh in on this but I would rather not implement something in ibnetdisc to synchronize the calls of another library. For example I don't want to re-implement mad_rpc and/or mad_send_via. However, we must allow applications above ibnetdisc make calls on the wire. I have been thinking about this since I sent the last email and I have another idea. What if we made ibmad queue all responses it gets in _do_madrpc which are not the response it is looking for. When mad_receive_via or subsequent _do_madrpc calls are made they first look in this queue for messages and return them if they are there. That would keep things in sync and not lose messages. Anyone could call into the mad layer any way they want. Applications, or parts of applications, which want to do parallel queries would still have to match up their own responses to queries but they would not have to worry about losing messages in the ibmad layer. > > > The following patch fixes > > this. However, I don't know you want to do... > > > > If one only uses mad_rpc and mad_rpc_rmpp then the patch works. However, if > > someone is using mad_send_via at the same time _do_madrpc will still fail. > > Is > > it by design that some responses will be lost while _do_madrpc is looking > > for > > it's response via the TID? > > > > Also, according to C13-18.1.1 and C13-19.1.1 you must use the SGID (or SLID) > > and the MgmtClass in addition to the TID to determine the uniqueness of a > > message. The SGID (or SLID) is of course the same but should the MgmtClass > > be checked here as well? > > > > Finally, why does _do_madrpc cast the transaction id to a 32 bit value? > > The kernel uses the high 32 bits for an agent ID. See kernel > Documentation/infiniband/user_mad.txt "Transaction IDs". Ah got it, thanks, Ira > > -- Hal > > > Confused, > > Ira > > -- Ira Weiny Math Programmer/Computer Scientist Lawrence Livermore National Lab weiny2 at llnl.gov From rdreier at cisco.com Fri Jul 10 13:42:04 2009 From: rdreier at cisco.com (Roland Dreier) Date: Fri, 10 Jul 2009 13:42:04 -0700 Subject: [ofa-general] Re: 2.6.30.1: possible irq lock inversion dependency detected In-Reply-To: (Bart Van Assche's message of "Fri, 10 Jul 2009 21:29:51 +0200") References: Message-ID: > Thanks for the patch. With the patch applied the lockdep warning > indeed occurs sooner and the output is now indeed shorter. You can > find the new lockdep output here: > http://bugzilla.kernel.org/attachment.cgi?id=22305. Thanks, that actually looks like a completely different issue (that I can actually understand). I was able to reproduce that here: the issue is doing skb_orphan() inside of priv->lock, and the network stack locking is not irq-safe. So the following hacky patch fixes that. This would be a short-term solution for the immediate issue at least. A better solution would be if we didn't need to make priv->lock hardirq-safe: the only place that requires it is the QP event handler in ipoib_cm.c, and that might be a little dicy to fix. Need to think about that. However with this patch applied I don't see any further lockdep reports here. It would be great if you could retest yet again with this applied (on top of my earlier patch to make priv->lock hardirq-safe as early as possible). Thanks, Roland diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index e319d91..4d53011 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -604,8 +604,11 @@ static void neigh_add_path(struct sk_buff *skb, struct net_device *dev) skb_queue_len(&neigh->queue)); goto err_drop; } - } else + } else { ipoib_send(dev, skb, path->ah, IPOIB_QPN(skb_dst(skb)->neighbour->ha)); + spin_unlock_irqrestore(&priv->lock, flags); + return; + } } else { neigh->ah = NULL; @@ -689,6 +692,8 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev, be16_to_cpu(path->pathrec.dlid)); ipoib_send(dev, skb, path->ah, IPOIB_QPN(phdr->hwaddr)); + spin_unlock_irqrestore(&priv->lock, flags); + return; } else if ((path->query || !path_rec_start(dev, path)) && skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) { /* put pseudoheader back on for next time */ diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c index a0e9753..a0825fe 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c @@ -720,7 +720,9 @@ out: } } + spin_unlock_irqrestore(&priv->lock, flags); ipoib_send(dev, skb, mcast->ah, IB_MULTICAST_QPN); + return; } unlock: From rdreier at cisco.com Fri Jul 10 13:44:14 2009 From: rdreier at cisco.com (Roland Dreier) Date: Fri, 10 Jul 2009 13:44:14 -0700 Subject: [ofa-general] Re: A question about tx lock in ipoib_flush_paths In-Reply-To: <20090709205132.GD12939@sgi.com> (akepner@sgi.com's message of "Thu, 9 Jul 2009 13:51:32 -0700") References: <4A4E190D.7010004@voltaire.com> <4A52930F.9050408@gmail.com> <4A56190E.1080105@voltaire.com> <20090709205132.GD12939@sgi.com> Message-ID: > If so, it would be addressed by the patch I sent here: > > http://lists.openfabrics.org/pipermail/general/2009-July/060501.html > > because the ipoib_neigh structure wouldn't be freed until after > ipoib_start_xmit() had done rcu_read_unlock(). Sorry for not replying to this sooner. I'd prefer not to take that approach, because all this code is complex enough that it is hard to maintain already; adding RCU on top of that just seems like it will make things even harder to fix in the future. I wish someone would come up with a way to slice through the gordian knot and simplify the neighbour handling -- maybe the whole strategy of stashing things in the network stack's neighbour structure is wrong? Maybe the core network stack could be tweaked to make things easier for ipoib to deal with? - R. From chien.tin.tung at intel.com Fri Jul 10 13:45:06 2009 From: chien.tin.tung at intel.com (Chien Tung) Date: Fri, 10 Jul 2009 15:45:06 -0500 Subject: [ofa-general] [PATCH] RDMA/nes: map MTU to IB_MTU_* and correctly report link state Message-ID: <20090710204506.GA5060@ctung-MOBL> ibv_devinfo is displaying static MTU and link state. Map MTU to next largest IB_MTU_*. Correctly report link state. Signed-off-by: Chien Tung CC: Steve Wise Reported-by: Jeff Squyres --- drivers/infiniband/hw/nes/nes_verbs.c | 23 ++++++++++++++++++++--- 1 files changed, 20 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index 21e0fd3..5aaa946 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -667,15 +667,32 @@ static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *prop */ static int nes_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) { + struct nes_vnic *nesvnic = to_nesvnic(ibdev); + struct net_device *netdev = nesvnic->netdev; + memset(props, 0, sizeof(*props)); - props->max_mtu = IB_MTU_2048; - props->active_mtu = IB_MTU_2048; + props->max_mtu = IB_MTU_4096; + + if (netdev->mtu >= 4096) + props->active_mtu = IB_MTU_4096; + else if (netdev->mtu >= 2048) + props->active_mtu = IB_MTU_2048; + else if (netdev->mtu >= 1024) + props->active_mtu = IB_MTU_1024; + else if (netdev->mtu >= 512) + props->active_mtu = IB_MTU_512; + else + props->active_mtu = IB_MTU_256; + props->lid = 1; props->lmc = 0; props->sm_lid = 0; props->sm_sl = 0; - props->state = IB_PORT_ACTIVE; + if (nesvnic->linkup) + props->state = IB_PORT_ACTIVE; + else + props->state = IB_PORT_DOWN; props->phys_state = 0; props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; -- 1.5.3.3 From bugzilla-daemon at bugzilla.kernel.org Fri Jul 10 09:34:01 2009 From: bugzilla-daemon at bugzilla.kernel.org (bugzilla-daemon at bugzilla.kernel.org) Date: Fri, 10 Jul 2009 16:34:01 GMT Subject: [ofa-general] [Bug 13757] New: Lockdep complains about possible irq lock inversion dependency Message-ID: http://bugzilla.kernel.org/show_bug.cgi?id=13757 Summary: Lockdep complains about possible irq lock inversion dependency Product: Drivers Version: 2.5 Kernel Version: 2.6.30.1 Platform: All OS/Version: Linux Tree: Mainline Status: NEW Severity: normal Priority: P1 Component: Infiniband/RDMA AssignedTo: drivers_infiniband-rdma at kernel-bugs.osdl.org ReportedBy: bart.vanassche at gmail.com Regression: No Created an attachment (id=22299) --> (http://bugzilla.kernel.org/attachment.cgi?id=22299) Kernel config Kernel: 2.6.30.1 with SCST zero-copy transfer completion notification and scsi_execute_fifo patches applied. These two patches do not modify any InfiniBand code. Setup: - Two servers connected back-to-back via InfiniBand. - OpenSM is running on one of the two servers. After having shut down one of the two servers, lockdep complained about possible irq lock inversion. -- Configure bugmail: http://bugzilla.kernel.org/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are watching the assignee of the bug. From akepner at sgi.com Fri Jul 10 14:24:06 2009 From: akepner at sgi.com (akepner at sgi.com) Date: Fri, 10 Jul 2009 14:24:06 -0700 Subject: [ofa-general] Re: A question about tx lock in ipoib_flush_paths In-Reply-To: References: <4A4E190D.7010004@voltaire.com> <4A52930F.9050408@gmail.com> <4A56190E.1080105@voltaire.com> <20090709205132.GD12939@sgi.com> Message-ID: <20090710212406.GD29594@sgi.com> On Fri, Jul 10, 2009 at 01:44:14PM -0700, Roland Dreier wrote: > ... > Sorry for not replying to this sooner. I'd prefer not to take that > approach, because all this code is complex enough that it is hard to > maintain already; adding RCU on top of that just seems like it will make > things even harder to fix in the future. > Sigh. But I can understand that preference. > I wish someone would come up with a way to slice through the gordian > knot and simplify the neighbour handling -- maybe the whole strategy of > stashing things in the network stack's neighbour structure is wrong? Yes, I think it is basically wrong. It was a good hack, but it subtly broke when CM was added. > Maybe the core network stack could be tweaked to make things easier for > ipoib to deal with? > I can investigate a different approach (but I'd certainly appreciate any good ideas!) In the meantime, we have found this patch to be a very effective workaround in practice: http://lists.openfabrics.org/pipermail/general/2009-June/060205.html It's obviously not a complete fix, but it improves the situation, and is very simple. Could we add that as a stopgap measure for now? -- Arthur From rdreier at cisco.com Fri Jul 10 15:12:30 2009 From: rdreier at cisco.com (Roland Dreier) Date: Fri, 10 Jul 2009 15:12:30 -0700 Subject: [ofa-general] Re: [PATCH ] mlx4_core: Distinguish multiple IB cards in /proc/interrupts In-Reply-To: <4A4C23D9.1030400@sgi.com> (Arputham Benjamin's message of "Wed, 01 Jul 2009 20:04:57 -0700") References: <4A4C23D9.1030400@sgi.com> Message-ID: > diff -rup a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c > --- a/drivers/net/mlx4/eq.c 2009-06-24 16:28:12.273861234 -0700 > +++ b/drivers/net/mlx4/eq.c 2009-06-30 16:38:58.200069975 -0700 > @@ -609,14 +609,21 @@ int mlx4_init_eq_table(struct mlx4_dev * > for (i = 0; i < MLX4_EQ_COMP_CPU0 + > dev->caps.num_comp_vectors; ++i) { > if (i == 0) > - snprintf(eq_name[0], 20, DRV_NAME "(async)"); > + snprintf(&priv->irq_name[i][0], > + DEVICE_NAME_SIZE, > + DRV_NAME "(async)" "@pci:%s", > + pci_name(dev->pdev)); > else > - snprintf(eq_name[i], 20, "eth-mlx4-%d", > - i - 1); > + snprintf(&priv->irq_name[i][0], > + DEVICE_NAME_SIZE, > + "eth-mlx4-%d" "@pci:%s", > + i - 1, > + pci_name(dev->pdev)); What tree did you generate this patch against? The code in the mainline kernel looks nothing like this. - R. From rdreier at cisco.com Fri Jul 10 15:15:08 2009 From: rdreier at cisco.com (Roland Dreier) Date: Fri, 10 Jul 2009 15:15:08 -0700 Subject: [ofa-general] [PATCH v2] mthca: Distinguish multiple IB cards in /proc/interrupts In-Reply-To: <4A4C1F7D.9040101@sgi.com> (Arputham Benjamin's message of "Wed, 01 Jul 2009 19:46:21 -0700") References: <4A4C1F7D.9040101@sgi.com> Message-ID: > @@ -356,6 +356,7 @@ struct mthca_dev { > struct ib_ah *sm_ah[MTHCA_MAX_PORTS]; > spinlock_t sm_lock; > u8 rate[MTHCA_MAX_PORTS]; > + char irq_name[MTHCA_NUM_EQ][IB_DEVICE_NAME_MAX]; This patch is whitespace-mangled (tabs are gone). So it can't be applied. Please fix your MTA and resend so it will apply. > + snprintf(&dev->irq_name[i][0], IB_DEVICE_NAME_MAX, I would probably write &dev->irq_name[i][0] as just dev->irq_name[i] > + snprintf(&dev->irq_name, IB_DEVICE_NAME_MAX, The first parameter of snprintf is a char *. Doesn't "&dev->irq_name" have type char ***? - R. From abenjamin at sgi.com Fri Jul 10 15:57:26 2009 From: abenjamin at sgi.com (Arputham Benjamin) Date: Fri, 10 Jul 2009 17:57:26 -0500 Subject: [ofa-general] RE: [PATCH ] mlx4_core: Distinguish multiple IB cards in /proc/interrupts References: <4A4C23D9.1030400@sgi.com> Message-ID: <1AB9A794DBDDF54A8A81BE2296F7BDFE99271A@cf--amer001e--3.americas.sgi.com> >> diff -rup a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c >> --- a/drivers/net/mlx4/eq.c 2009-06-24 16:28:12.273861234 -0700 >> +++ b/drivers/net/mlx4/eq.c 2009-06-30 16:38:58.200069975 -0700 >> @@ -609,14 +609,21 @@ int mlx4_init_eq_table(struct mlx4_dev * >> for (i = 0; i < MLX4_EQ_COMP_CPU0 + >> dev->caps.num_comp_vectors; ++i) { >> if (i == 0) >> - snprintf(eq_name[0], 20, DRV_NAME "(async)"); >> + snprintf(&priv->irq_name[i][0], >> + DEVICE_NAME_SIZE, >> + DRV_NAME "(async)" "@pci:%s", >> + pci_name(dev->pdev)); >> else >> - snprintf(eq_name[i], 20, "eth-mlx4-%d", >> - i - 1); >> + snprintf(&priv->irq_name[i][0], >> + DEVICE_NAME_SIZE, >> + "eth-mlx4-%d" "@pci:%s", >> + i - 1, >> + pci_name(dev->pdev)); > What tree did you generate this patch against? The code in the mainline > kernel looks nothing like this. This patch was generated against SLES10SP2 ( kernel release - 2.6.16.60-0.21-default) and OFED 1.4 Regards, Benjamin -------------- next part -------------- An HTML attachment was scrubbed... URL: From rdreier at cisco.com Fri Jul 10 16:32:01 2009 From: rdreier at cisco.com (Roland Dreier) Date: Fri, 10 Jul 2009 16:32:01 -0700 Subject: [ofa-general] Re: [PATCH ] mlx4_core: Distinguish multiple IB cards in /proc/interrupts In-Reply-To: <1AB9A794DBDDF54A8A81BE2296F7BDFE99271A@cf--amer001e--3.americas.sgi.com> (Arputham Benjamin's message of "Fri, 10 Jul 2009 17:57:26 -0500") References: <4A4C23D9.1030400@sgi.com> <1AB9A794DBDDF54A8A81BE2296F7BDFE99271A@cf--amer001e--3.americas.sgi.com> Message-ID: > This patch was generated against SLES10SP2 ( kernel release - 2.6.16.60-0.21-default) > and OFED 1.4 If you want a patch merged to the mainline kernel, please generate and test it against that kernel. From vlad at lists.openfabrics.org Sat Jul 11 02:38:44 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Sat, 11 Jul 2009 02:38:44 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090711-0200 daily build status Message-ID: <20090711093844.63CC9F203AA@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -Werror-implicit-function-declaration -fno-strict-aliasing -fno-common -ffreestanding -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: from /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.c:37: include/linux/skbuff.h: In function 'skb_add_data': include/linux/skbuff.h:1041: warning: pointer targets in passing argument 2 of 'csum_partial_copy_from_user' differ in signedness make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: from /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.c:37: include/linux/skbuff.h: In function 'skb_add_data': include/linux/skbuff.h:1041: warning: pointer targets in passing argument 2 of 'csum_partial_copy_from_user' differ in signedness make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090711-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From bart.vanassche at gmail.com Sat Jul 11 03:43:19 2009 From: bart.vanassche at gmail.com (Bart Van Assche) Date: Sat, 11 Jul 2009 12:43:19 +0200 Subject: [ofa-general] Re: 2.6.30.1: possible irq lock inversion dependency detected In-Reply-To: References: Message-ID: On Fri, Jul 10, 2009 at 10:42 PM, Roland Dreier wrote: > >  > Thanks for the patch. With the patch applied the lockdep warning >  > indeed occurs sooner and the output is now indeed shorter. You can >  > find the new lockdep output here: >  > http://bugzilla.kernel.org/attachment.cgi?id=22305. > > Thanks, that actually looks like a completely different issue (that I > can actually understand).  I was able to reproduce that here: the issue > is doing skb_orphan() inside of priv->lock, and the network stack > locking is not irq-safe.  So the following hacky patch fixes that. > > This would be a short-term solution for the immediate issue at least.  A > better solution would be if we didn't need to make priv->lock > hardirq-safe: the only place that requires it is the QP event handler in > ipoib_cm.c, and that might be a little dicy to fix.  Need to think about that. > > However with this patch applied I don't see any further lockdep reports > here.  It would be great if you could retest yet again with this applied > (on top of my earlier patch to make priv->lock hardirq-safe as early as > possible). With the two patches applied on top of 2.6.31.1 lockdep stopped complaining on my setup. By the way, do you know whether this is a long-standing issue or a result of the changes between 2.6.29 and 2.6.30 ? Bart. From hnrose at comcast.net Sat Jul 11 04:48:15 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Sat, 11 Jul 2009 07:48:15 -0400 Subject: [ofa-general] [PATCH] opensm/osm_ucast_lash.c: Eliminate possible seg fault in get_osm_switch_from_port Message-ID: <20090711114815.GA12867@comcast.net> get_osm_switch_from_port can be called with NULL port pointer from osm_get_lash_sl Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 12b5e34..871a673 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -5,6 +5,7 @@ * Copyright (c) 2007 Simula Research Laboratory. All rights reserved. * Copyright (c) 2007 Silicon Graphics Inc. All rights reserved. * Copyright (c) 2008,2009 System Fabric Works, Inc. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -129,7 +130,11 @@ static void connect_switches(lash_t * p_lash, int sw1, int sw2, int phy_port_1) static osm_switch_t *get_osm_switch_from_port(const osm_port_t * port) { - osm_physp_t *p = port->p_physp; + osm_physp_t *p; + + if (!port) + return NULL; + p = port->p_physp; if (p->p_node->sw) return p->p_node->sw; else if (p->p_remote_physp->p_node->sw) From vlad at lists.openfabrics.org Sun Jul 12 02:37:18 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Sun, 12 Jul 2009 02:37:18 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090712-0200 daily build status Message-ID: <20090712093718.F134F102023F@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -Werror-implicit-function-declaration -fno-strict-aliasing -fno-common -ffreestanding -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: from /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.c:37: include/linux/skbuff.h: In function 'skb_add_data': include/linux/skbuff.h:1041: warning: pointer targets in passing argument 2 of 'csum_partial_copy_from_user' differ in signedness make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: from /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.c:37: include/linux/skbuff.h: In function 'skb_add_data': include/linux/skbuff.h:1041: warning: pointer targets in passing argument 2 of 'csum_partial_copy_from_user' differ in signedness make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090712-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From frederic.ciesielski at hp.com Sun Jul 12 03:20:58 2009 From: frederic.ciesielski at hp.com (Ciesielski, Frederic (EMEA HPC&OSLO CC)) Date: Sun, 12 Jul 2009 10:20:58 +0000 Subject: [ofa-general] OFED 1.4.1 breaks SLES10 SP2 NFS server ? Message-ID: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> As far as I could see, x86-64 systems running SLES10 SP2 + OFED 1.4.1 are not valid NFS servers anymore, even without trying to activate NFS-RDMA, even without using IB at all for the export. NFS clients see too frequently 'Stale NFS file handle' error messages for them to properly read or write anything. This happens using ethernet and IPoIB... and works fine without OFED, or when OFED 1.4 is installed instead (which is not what I want). Did anybody test that ? Any idea about how to get rid of this major side effect ? Thanks. Fred. -------------- next part -------------- An HTML attachment was scrubbed... URL: From ogerlitz at voltaire.com Sun Jul 12 04:13:58 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Sun, 12 Jul 2009 14:13:58 +0300 Subject: [ofa-general] patch to ib_addr for sending arps In-Reply-To: <4A5527A1.5010104@oracle.com> References: <4A5527A1.5010104@oracle.com> Message-ID: <4A59C576.5060709@voltaire.com> Leo Tominna wrote: > This patch appears to help when strict ARP handling is enabled or when > non-standard routing tables are used. The ARP request is replied to > through the device that will be used for subsequent communication, so > the ARP entry gets associated with the correct device in the ARP > cache. tcpdump shows consistent ARPs generated by similar arguments > to ping and rds-ping. Hi Loe, Does this patch comes to solve the problem discussed over the "pick the outgoing HCA based on the IP used for bind" threads dated to February this year at the general and rds-devel mailing lists (http://lists.openfabrics.org/pipermail/general/2009-February/057008.html)? also by "strict ARP handling" do you refer to the case where there are multiple NICs on the same L2 broadcast domain (VLAN/Partition)? Or. From jackm at dev.mellanox.co.il Sun Jul 12 06:28:08 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Sun, 12 Jul 2009 16:28:08 +0300 Subject: [ofa-general] [PATCH] mlx4: Do not allow ib userspace open while device is being removed Message-ID: <200907121628.08412.jackm@dev.mellanox.co.il> Userspace apps are supposed to release all ib device resources if they receive a fatal async event (IBV_EVENT_DEVICE_FATAL). However, the app has no way of knowing when the device has come back up, except to repeatedly attempt ibv_open_device() until it succeeds. However, currently there is no protection against open succeeding when the device is in the midst of the removal following the fatal event. In this case, the open will succeed, but as a result the device waits in the middle of its removal until the new app releases its ib resources -- and the new app will not do so, since the open succeeded at a point following the fatal event generation. This patch adds an "active" flag to the device. The active flag is set to false (in the interrupt context) before the "fatal" event is generated, so any subsequent ibv_dev_open() call to the device will fail until the device comes back up, thus preventing the above deadlock. Signed-off-by: Jack Morgenstein --- Roland, For good measure, I also set the active flag to false at mlx4_ib_remove() -- to give some measure of protection against opening a new userspace app while the driver is in the process of being removed. The main issue in this patch, though, is setting the active flag to false before triggering the fatal event in catas.c. This patch is for kernel 2.6.31. -Jack diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index ae3d759..571d231 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -342,6 +344,9 @@ static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev, struct mlx4_ib_alloc_ucontext_resp resp; int err; + if (!dev->dev->ib_active) + return ERR_PTR(-EAGAIN); + resp.qp_tab_size = dev->dev->caps.num_qps; resp.bf_reg_size = dev->dev->caps.bf_reg_size; resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; @@ -673,6 +686,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) goto err_reg; } + dev->ib_active = 1; + return ibdev; err_reg: @@ -698,6 +713,8 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr) struct mlx4_ib_dev *ibdev = ibdev_ptr; int p; + dev->ib_active = 0; + mlx4_ib_mad_cleanup(ibdev); ib_unregister_device(&ibdev->ib_dev); diff --git a/drivers/net/mlx4/catas.c b/drivers/net/mlx4/catas.c index aa9674b..7a23beb 100644 --- a/drivers/net/mlx4/catas.c +++ b/drivers/net/mlx4/catas.c @@ -69,6 +69,7 @@ static void poll_catas(unsigned long dev_ptr) if (readl(priv->catas_err.map)) { dump_err_buf(dev); + dev->ib_active = 0; mlx4_dispatch_event(dev, MLX4_DEV_EVENT_CATASTROPHIC_ERROR, 0); if (internal_err_reset) { diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h index ce7cc6c..7f6d800 100644 --- a/include/linux/mlx4/device.h +++ b/include/linux/mlx4/device.h @@ -379,6 +379,7 @@ struct mlx4_dev { struct radix_tree_root qp_table_tree; u32 rev_id; char board_id[MLX4_BOARD_ID_LEN]; + u32 ib_active; }; struct mlx4_init_port_param { From sdietrich at novell.com Sun Jul 12 11:09:14 2009 From: sdietrich at novell.com (Sven-Thorsten Dietrich) Date: Sun, 12 Jul 2009 11:09:14 -0700 Subject: [ofa-general] OFED 1.4.1 breaks SLES10 SP2 NFS server ? In-Reply-To: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> References: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> Message-ID: <76D6DD3B-26CE-4FCA-9CC0-9C30329D98BE@novell.com> Is your system up to date with patches? If so, please open a bug on bugzilla.Novell.com. Thanks, Sven + 1 (415) 694 2930 On Jul 12, 2009, at 3:20, "Ciesielski, Frederic (EMEA HPC&OSLO CC)" wrote: > As far as I could see, x86-64 systems running SLES10 SP2 + OFED > 1.4.1 are not valid NFS servers anymore, even without trying to > activate NFS-RDMA, even without using IB at all for the export. > > NFS clients see too frequently 'Stale NFS file handle' error > messages for them to properly read or write anything. > > This happens using ethernet and IPoIB... and works fine without > OFED, or when OFED 1.4 is installed instead (which is not what I > want). > > Did anybody test that ? > Any idea about how to get rid of this major side effect ? > > Thanks. > Fred. > > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general -------------- next part -------------- An HTML attachment was scrubbed... URL: From swise at opengridcomputing.com Sun Jul 12 12:26:53 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Sun, 12 Jul 2009 14:26:53 -0500 Subject: [ofa-general] OFED 1.4.1 breaks SLES10 SP2 NFS server ? In-Reply-To: <76D6DD3B-26CE-4FCA-9CC0-9C30329D98BE@novell.com> References: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> <76D6DD3B-26CE-4FCA-9CC0-9C30329D98BE@novell.com> Message-ID: <4A5A38FD.8010909@opengridcomputing.com> If you don't want/need NFSRDMA (especially since it appears to not work well on sles10sp2), you can build/install ofed-1.4.1 with nfsrdma turned off. Assuming you're installing "all" when you run install.pl, you'll have to run install.pl and choose a custom install and select everything except nfsrdma. Does that make sense? Steve. Sven-Thorsten Dietrich wrote: > Is your system up to date with patches? > > If so, please open a bug on bugzilla.Novell.com > . > Thanks, > Sven > > + 1 (415) 694 2930 > > On Jul 12, 2009, at 3:20, "Ciesielski, Frederic (EMEA HPC&OSLO CC)" > > wrote: > >> As far as I could see, x86-64 systems running SLES10 SP2 + OFED 1.4.1 >> are not valid NFS servers anymore, even without trying to activate >> NFS-RDMA, even without using IB at all for the export. >> >> NFS clients see too frequently 'Stale NFS file handle' error messages >> for them to properly read or write anything. >> >> This happens using ethernet and IPoIB... and works fine without OFED, >> or when OFED 1.4 is installed instead (which is not what I want). >> >> Did anybody test that ? >> Any idea about how to get rid of this major side effect ? >> >> Thanks. >> Fred. >> >> >> _______________________________________________ >> general mailing list >> general at lists.openfabrics.org >> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >> >> To unsubscribe, please visit >> http://openib.org/mailman/listinfo/openib-general > ------------------------------------------------------------------------ > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From tom at opengridcomputing.com Sun Jul 12 14:33:39 2009 From: tom at opengridcomputing.com (Tom Tucker) Date: Sun, 12 Jul 2009 16:33:39 -0500 Subject: [ofa-general] OFED 1.4.1 breaks SLES10 SP2 NFS server ? In-Reply-To: <4A5A38FD.8010909@opengridcomputing.com> References: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> <76D6DD3B-26CE-4FCA-9CC0-9C30329D98BE@novell.com> <4A5A38FD.8010909@opengridcomputing.com> Message-ID: <4A5A56B3.80200@opengridcomputing.com> Jeff: Can you answer Sven's question regarding the level of testing performed on your port of NFSRDMA to SLES10SP2? Thanks, Tom Steve Wise wrote: > If you don't want/need NFSRDMA (especially since it appears to not work > well on sles10sp2), you can build/install ofed-1.4.1 with nfsrdma turned > off. Assuming you're installing "all" when you run install.pl, you'll > have to run install.pl and choose a custom install and select everything > except nfsrdma. > > Does that make sense? > > Steve. > > Sven-Thorsten Dietrich wrote: >> Is your system up to date with patches? >> >> If so, please open a bug on bugzilla.Novell.com >> . >> Thanks, >> Sven >> >> + 1 (415) 694 2930 >> >> On Jul 12, 2009, at 3:20, "Ciesielski, Frederic (EMEA HPC&OSLO CC)" >> > wrote: >> >>> As far as I could see, x86-64 systems running SLES10 SP2 + OFED 1.4.1 >>> are not valid NFS servers anymore, even without trying to activate >>> NFS-RDMA, even without using IB at all for the export. >>> >>> NFS clients see too frequently 'Stale NFS file handle' error messages >>> for them to properly read or write anything. >>> >>> This happens using ethernet and IPoIB... and works fine without OFED, >>> or when OFED 1.4 is installed instead (which is not what I want). >>> >>> Did anybody test that ? >>> Any idea about how to get rid of this major side effect ? >>> >>> Thanks. >>> Fred. >>> >>> >>> _______________________________________________ >>> general mailing list >>> general at lists.openfabrics.org >>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>> >>> To unsubscribe, please visit >>> http://openib.org/mailman/listinfo/openib-general >> ------------------------------------------------------------------------ >> >> _______________________________________________ >> general mailing list >> general at lists.openfabrics.org >> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >> >> To unsubscribe, please visit >> http://openib.org/mailman/listinfo/openib-general From sashak at voltaire.com Sun Jul 12 15:52:05 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 01:52:05 +0300 Subject: [ofa-general] sending mad in parallel mode and perfquery In-Reply-To: <423F353FA5DE4BD79D1EC432F040B80E@amr.corp.intel.com> References: <4A5628F9.6070500@yandex.ru> <423F353FA5DE4BD79D1EC432F040B80E@amr.corp.intel.com> Message-ID: <20090712225204.GA12543@me> On 10:56 Thu 09 Jul , Sean Hefty wrote: > > recv_mad blocks until any response is received, and it can be called from > multiple threads. recv_mad only has multi-threaded issues if MADs > 256 bytes > are received. Right, it works in this way. For instance mcast_storm.c (test utility distributed as example with ibsim) uses such approach: Sasha From sashak at voltaire.com Sun Jul 12 15:52:13 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 01:52:13 +0300 Subject: [ofa-general] Re: [PATCH] infiniband-diags/ibroute.c: Fix format and typo in printf In-Reply-To: <20090709193915.GA19695@comcast.net> References: <20090709193915.GA19695@comcast.net> Message-ID: <20090712225213.GB12543@me> On 15:39 Thu 09 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 12 15:52:21 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 01:52:21 +0300 Subject: [ofa-general] Re: [PATCH] Fix error return code from mad_send_via In-Reply-To: <20090708170600.45d0b681.weiny2@llnl.gov> References: <20090708170600.45d0b681.weiny2@llnl.gov> Message-ID: <20090712225221.GC12543@me> On 17:06 Wed 08 Jul , Ira Weiny wrote: > From: Ira Weiny > Date: Wed, 8 Jul 2009 16:52:09 -0700 > Subject: [PATCH] Fix error return code from mad_send_via > > > Signed-off-by: Ira Weiny Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 12 15:52:29 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 01:52:29 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osmtest: Fix some typos In-Reply-To: <20090701111317.GA12638@comcast.net> References: <20090701111317.GA12638@comcast.net> Message-ID: <20090712225229.GD12543@me> On 07:13 Wed 01 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 12 15:52:38 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 01:52:38 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_mgr.c: Add error numbers to some error log messages In-Reply-To: <20090702194840.GA27859@comcast.net> References: <20090702194840.GA27859@comcast.net> Message-ID: <20090712225238.GE12543@me> On 15:48 Thu 02 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 12 17:07:56 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 03:07:56 +0300 Subject: [ofa-general] [PATCH] Lock _do_madrpc for thread safety In-Reply-To: <20090710132633.e85c6181.weiny2@llnl.gov> References: <20090708174720.f61d44fc.weiny2@llnl.gov> <20090710132633.e85c6181.weiny2@llnl.gov> Message-ID: <20090713000756.GI12543@me> Hi Ira, On 13:26 Fri 10 Jul , Ira Weiny wrote: > > > > Have you read Sasha's commit 51d25384626a7b4ba386c414ed56c647a7bf64df > > from 12/26/08 ? In it, he states " I think that it will be more robust > > for multithreaded > > application to use its own synchronization methods (pthread mutex or any > > other) for better control. " > > I missed that. Thanks. Yes. And also I guess this will break WinOF. > However, I am starting to lean towards putting something in libibmad. The > problem for me is that in the ibnetdisc library I can't control what the > application is doing. And more importantly I can't allow the application to > do anything (at least at certain times). You don't need to use madrpc in ibnetdisc library then. madrpc stuff is just simple interface for tools like smpquery. Make something smarter using umad_send/recv(). > I have been thinking about this since I sent the last email and I have another > idea. What if we made ibmad queue all responses it gets in _do_madrpc which > are not the response it is looking for. You can add such sort of API, but I think that it should be new (higher) API layer. > Applications, or parts of applications, which want to do parallel queries > would still have to match up their own responses to queries but they would not > have to worry about losing messages in the ibmad layer. madrpc() is too primitive interface for such applications. There would be better to use umad_send/recv() directly or may be mad_send_via(). Example is mcast_storm.c distributed with ibsim. Sasha From sashak at voltaire.com Sun Jul 12 17:11:26 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 03:11:26 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Eliminate possible seg fault in get_osm_switch_from_port In-Reply-To: <20090711114815.GA12867@comcast.net> References: <20090711114815.GA12867@comcast.net> Message-ID: <20090713001126.GJ12543@me> Hi Hal, On 07:48 Sat 11 Jul , Hal Rosenstock wrote: > > get_osm_switch_from_port can be called with NULL port pointer > from osm_get_lash_sl > > Signed-off-by: Hal Rosenstock > --- > diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c > index 12b5e34..871a673 100644 > --- a/opensm/opensm/osm_ucast_lash.c > +++ b/opensm/opensm/osm_ucast_lash.c > @@ -5,6 +5,7 @@ > * Copyright (c) 2007 Simula Research Laboratory. All rights reserved. > * Copyright (c) 2007 Silicon Graphics Inc. All rights reserved. > * Copyright (c) 2008,2009 System Fabric Works, Inc. All rights reserved. > + * Copyright (c) 2009 HNR Consulting. All rights reserved. > * > * This software is available to you under a choice of one of two > * licenses. You may choose to be licensed under the terms of the GNU > @@ -129,7 +130,11 @@ static void connect_switches(lash_t * p_lash, int sw1, int sw2, int phy_port_1) > > static osm_switch_t *get_osm_switch_from_port(const osm_port_t * port) > { > - osm_physp_t *p = port->p_physp; > + osm_physp_t *p; > + > + if (!port) > + return NULL; When such *legal* get_osm_switch_from_port() call with port = NULL is possible? Sasha > + p = port->p_physp; > if (p->p_node->sw) > return p->p_node->sw; > else if (p->p_remote_physp->p_node->sw) From sashak at voltaire.com Sun Jul 12 17:14:28 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 03:14:28 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_sa_(path multipath)_record.c: Fix typo in a couple of log messages In-Reply-To: <20090703032354.GA28557@comcast.net> References: <20090703032354.GA28557@comcast.net> Message-ID: <20090713001428.GK12543@me> On 23:23 Thu 02 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 12 17:22:00 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 03:22:00 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_ftree.c: Fixed issue with reverse_hops introduced by commit 1d7dd18b531c1d6370f80cc7303493d6f3e3e777. In-Reply-To: <4A48C616.1090402@ext.bull.net> References: <4A48C616.1090402@ext.bull.net> Message-ID: <20090713002200.GA20347@me> On 15:48 Mon 29 Jun , Nicolas Morey-Chaisemartin wrote: > When using reverse hop with nodes on root switches, origin switch is explored multiple times with wrong hop numbers creating > loops in the network. > This is fixed by checking before going up (even on the master path) that the remote switch is either not > configured or with a longer path (second test was missing). > > Signed-off-by: Nicolas Morey-Chaisemartin Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 12 17:22:47 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 03:22:47 +0300 Subject: [ofa-general] Re: [PATCH] osm_ucast_ftree.c Count number of hops instead of calculating it In-Reply-To: <4A4B1109.9010709@Sun.COM> References: <4A4B1109.9010709@Sun.COM> Message-ID: <20090713002247.GB20347@me> On 09:32 Wed 01 Jul , Line.Holen at Sun.COM wrote: > The ftree algorithm is currently calculating number of hops based > on switch ranks and reverse hops. This patch replaces this with a > simple counter. > > Signed-off-by: Frank Olaf Sem-Jacobsen > Signed-off-by: Line Holen Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 12 17:29:50 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 03:29:50 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_mcast_tbl.c: In osm_mcast_tbl_init, use calloc In-Reply-To: <20090707140007.GA8954@comcast.net> References: <20090707140007.GA8954@comcast.net> Message-ID: <20090713002950.GC20347@me> On 10:00 Tue 07 Jul , Hal Rosenstock wrote: > > rather than malloc/memset > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 12 17:30:34 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 03:30:34 +0300 Subject: [ofa-general] Re: [PATCH] opensm: Add new Sun vendor ID In-Reply-To: <4A5356B8.1040803@Sun.COM> References: <4A5356B8.1040803@Sun.COM> Message-ID: <20090713003034.GD20347@me> On 16:07 Tue 07 Jul , Line.Holen at Sun.COM wrote: > Signed-off-by: Line Holen Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 12 17:34:52 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 03:34:52 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_switch.c: In osm_switch_prepare_path_rebuild, use realloc In-Reply-To: <20090629184457.GA13787@comcast.net> References: <20090629184457.GA13787@comcast.net> Message-ID: <20090713003452.GE20347@me> On 14:44 Mon 29 Jun , Hal Rosenstock wrote: > > rather than malloc/memset/free > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 12 17:37:45 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 13 Jul 2009 03:37:45 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_ftree.c: Made error numbers unique in some log messages In-Reply-To: <20090703131805.GA29743@comcast.net> References: <20090703131805.GA29743@comcast.net> Message-ID: <20090713003745.GF20347@me> On 09:18 Fri 03 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From felix at chelsio.com Sun Jul 12 18:48:08 2009 From: felix at chelsio.com (Felix Marti) Date: Sun, 12 Jul 2009 18:48:08 -0700 Subject: [ofa-general] Memory registration limit of 16GB with Chelsio References: <96f8e60e0907101115i11fcf051h8508fdc18282368b@mail.gmail.com> <4A578C4D.2060003@opengridcomputing.com> Message-ID: <8A71B368A89016469F72CD08050AD33405E63DDE@maui.asicdesigners.com> -----Original Message----- From: Steve Wise [mailto:swise at opengridcomputing.com] Sent: Friday, July 10, 2009 11:46 AM To: pandit ib Cc: general at lists.openfabrics.org; Felix Marti Subject: Re: [ofa-general] Memory registration limit of 16GB with Chelsio Looks like you're running out of adapter resources for storing the physical buffers lists (PBLs). The cards have 32MB of memory for storing PBLs. 32MB of memory can store 4MB of 8B physical addresses. For user mode, your page size is 4KB, so that maps out to 4MB * 4KB = 16GB of memory. Hey Felix, Can we somehow adjust the PMRX memory map dynamically? For instance, maybe drop the number of TPTs to allow for more PBL memory? [felix] Not dynamically, but it can surely be changed. DM, can it be controlled through a cxgbtool? Steve pandit ib wrote: > Hi, > > In our testing we are hitting a memory registration limit of 16GB with > the Chelsio cards. > > Here are some sample results for different region sizes. > > region size Number of Successful registration Total > 2MB 8178 > ~16GB > 16MB 1021 > ~16GB > 128MB 127 > ~16GB > > Are there any knobs that we can use to raise that limit? > > For example, on mlx4 we use "log_num_mtt" module load parameter to > increase the limit. > > Here is a sample output of the test: > > [root at lv2 ~]# ./mr-test 16777216 > > mr-test: bufsize 16777216 > device # 0 name="cxgb3_0" guid="00074305ca890000" > ibv_open_device() context=0x77b5100 > ibv_alloc_pd() pd=0x77b50e0 > alloc: 1021 > ibv_reg_mr failed:: Cannot allocate memory > fw_ver: 7.4.0 > max_mr_size 0x100000000 > max_mr: 32768, could only register 1021 regions; Region size 16MB > sleep 5 sec > free: 0 > done > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From leo.tominna at oracle.com Sun Jul 12 20:38:38 2009 From: leo.tominna at oracle.com (leo.tominna at oracle.com) Date: Sun, 12 Jul 2009 20:38:38 -0700 Subject: [ofa-general] patch to ib_addr for sending arps In-Reply-To: <4A59C576.5060709@voltaire.com> References: <4A5527A1.5010104@oracle.com> <4A59C576.5060709@voltaire.com> Message-ID: <4A5AAC3E.2040609@oracle.com> Hi Or, Sorry, I had not seen this earlier thread. I think a combination of the two patches, and another change would address both our problems. The patch I sent ensures neigh_lookup and neigh_event_send lookup the same route if source based routing tables are being used. With Sean's patch, if that ip_dev_lookup is duplicated to the addr_send_arp, that would address the source based routing case as well I think: addr_send_arp: + if (src_ip) + oif = ib_dev_lookup(src_ip) + s_addr = src_ip; addr_resolve_remote: + if (src_ip) + oif = ib_dev_lookup(src_ip) s_addr = src_ip; /* this was already there */ Associating the device with the source IP seems to be the correct thing to do in general, but I initially avoided it in favor of source based routing rules/tables since Linux does not do this by default. Source based routing seems to be the only way to get load balancing right for regular IP traffic when two local IPs are on the same subnet, so I thought it would be better to have the src_ip alone cause the routing lookup to associate everything with the correct device, as that would works for regular IP traffic and anyone else, like ib_addr clients. The problems I was seeing with arp was that Linux associates arp entries with specific devices, so if source based routing is used, and the arp send does not take src_ip into account, the arp is sent from the default device, and thats the device that gets the arp entry, where as neigh_lookup was looking for an entry on the correct device, and was never finding the neighbor. I think arp_ignore=1 is also needed. From what I can tell, on HPUX there is no device association with arp entries. Does anyone know why Linux has this flexibility? It looks like this was added in 2.2 or something, and I can't see any useful applications for this. Second, HPUX sends a single reply for arps, with the correct hardware information, regardless of what device it is replied from (or maybe it always replies from the correct device, doesn't matter, it works). On Linux each device replies with its own hardware address (unless arp_ignore is used). I've also seen arp requests being sent with the wrong hardware address, mainly with ping initiated traffic. when sent from devX, "who has ipZ tell ipY" causes the node with ipZ to create an implicit arp entry associating ipY with devX (rather than devY). The initial patch I sent is less restrictive but relies on source based routing to get everything working, perhaps an explicit device mapping (as above) makes more sense for RDMA traffic. Please correct me if something I said is incorrect or if these changes conflict with other working configurations. Thanks for your help. Leo Tominna Disclaimer: The statements and opinions expressed here are my own and do not necessarily reflect those of my employer. On 7/12/2009 4:13 AM, Or Gerlitz wrote: > Leo Tominna wrote: >> This patch appears to help when strict ARP handling is enabled or >> when non-standard routing tables are used. The ARP request is >> replied to through the device that will be used for subsequent >> communication, so the ARP entry gets associated with the correct >> device in the ARP cache. tcpdump shows consistent ARPs generated by >> similar arguments to ping and rds-ping. > Hi Loe, > > Does this patch comes to solve the problem discussed over the "pick > the outgoing HCA based on the IP used for bind" threads dated to > February this year at the general and rds-devel mailing lists > (http://lists.openfabrics.org/pipermail/general/2009-February/057008.html)? > also by "strict ARP handling" do you refer to the case where there are > multiple NICs on the same L2 broadcast domain (VLAN/Partition)? > > Or. > > From vlad at lists.openfabrics.org Mon Jul 13 02:38:10 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Mon, 13 Jul 2009 02:38:10 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090713-0200 daily build status Message-ID: <20090713093810.324271020461@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -Werror-implicit-function-declaration -fno-strict-aliasing -fno-common -ffreestanding -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: -I/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5/arch//include \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Wstrict-prototypes -Wundef -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(iscsi_iser)" -D"KBUILD_MODNAME=KBUILD_STR(ib_iser)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/.tmp_iscsi_iser.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.c:601: error: unknown field 'eh_target_reset_handler' specified in initializer make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser/iscsi_iser.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2354: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2369: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:1848: warning: passing argument 2 of 'match_token' discards qualifiers from pointer target type /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c: In function 'srp_add_port': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2097: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: from /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.c:37: include/linux/skbuff.h: In function 'skb_add_data': include/linux/skbuff.h:1041: warning: pointer targets in passing argument 2 of 'csum_partial_copy_from_user' differ in signedness make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: from /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.c:37: include/linux/skbuff.h: In function 'skb_add_data': include/linux/skbuff.h:1041: warning: pointer targets in passing argument 2 of 'csum_partial_copy_from_user' differ in signedness make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser/iser_verbs.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/iser] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090713-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From hal.rosenstock at gmail.com Mon Jul 13 03:13:41 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 13 Jul 2009 06:13:41 -0400 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Eliminate possible seg fault in get_osm_switch_from_port In-Reply-To: <20090713001126.GJ12543@me> References: <20090711114815.GA12867@comcast.net> <20090713001126.GJ12543@me> Message-ID: Hi Sasha, On Sun, Jul 12, 2009 at 8:11 PM, Sasha Khapyorsky wrote: > Hi Hal, > > On 07:48 Sat 11 Jul     , Hal Rosenstock wrote: >> >> get_osm_switch_from_port can be called with NULL port pointer >> from osm_get_lash_sl >> >> Signed-off-by: Hal Rosenstock >> --- >> diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c >> index 12b5e34..871a673 100644 >> --- a/opensm/opensm/osm_ucast_lash.c >> +++ b/opensm/opensm/osm_ucast_lash.c >> @@ -5,6 +5,7 @@ >>   * Copyright (c) 2007      Simula Research Laboratory. All rights reserved. >>   * Copyright (c) 2007      Silicon Graphics Inc. All rights reserved. >>   * Copyright (c) 2008,2009 System Fabric Works, Inc. All rights reserved. >> + * Copyright (c) 2009      HNR Consulting. All rights reserved. >>   * >>   * This software is available to you under a choice of one of two >>   * licenses.  You may choose to be licensed under the terms of the GNU >> @@ -129,7 +130,11 @@ static void connect_switches(lash_t * p_lash, int sw1, int sw2, int phy_port_1) >> >>  static osm_switch_t *get_osm_switch_from_port(const osm_port_t * port) >>  { >> -     osm_physp_t *p = port->p_physp; >> +     osm_physp_t *p; >> + >> +     if (!port) >> +             return NULL; > > When such *legal* get_osm_switch_from_port() call with port = NULL is > possible? It's called with NULL port pointer from osm_get_lash_sl which is invoked via SA PathRecord flow. -- Hal > Sasha > >> +     p = port->p_physp; >>       if (p->p_node->sw) >>               return p->p_node->sw; >>       else if (p->p_remote_physp->p_node->sw) > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From frederic.ciesielski at hp.com Mon Jul 13 06:28:50 2009 From: frederic.ciesielski at hp.com (Ciesielski, Frederic (EMEA HPC&OSLO CC)) Date: Mon, 13 Jul 2009 13:28:50 +0000 Subject: [ofa-general] OFED 1.4.1 breaks SLES10 SP2 NFS server ? In-Reply-To: <4A5A38FD.8010909@opengridcomputing.com> References: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> <76D6DD3B-26CE-4FCA-9CC0-9C30329D98BE@novell.com> <4A5A38FD.8010909@opengridcomputing.com> Message-ID: <7391130E01ED404FBD7A3C86731EEB7D3F73DF8ABB@GVW1087EXB.americas.hpqcorp.net> Yes, that will do it; I think I already tried to remove this option from the 'install.pl menu and rebuild, but probably the previous rpms were kept. Restarting from scratch and removing the nfs-rdma option permits not to have this updated NFS kernel modules, and things work now... Thanks. Fred. -----Original Message----- From: Steve Wise [mailto:swise at opengridcomputing.com] Sent: Sunday, July 12, 2009 21:27 To: Sven-Thorsten Dietrich Cc: Ciesielski, Frederic (EMEA HPC&OSLO CC); general at lists.openfabrics.org; Jeff Becker; Jon Mason; 'Tom Tucker' Subject: Re: [ofa-general] OFED 1.4.1 breaks SLES10 SP2 NFS server ? If you don't want/need NFSRDMA (especially since it appears to not work well on sles10sp2), you can build/install ofed-1.4.1 with nfsrdma turned off. Assuming you're installing "all" when you run install.pl, you'll have to run install.pl and choose a custom install and select everything except nfsrdma. Does that make sense? Steve. Sven-Thorsten Dietrich wrote: > Is your system up to date with patches? > > If so, please open a bug on bugzilla.Novell.com > . > Thanks, > Sven > > + 1 (415) 694 2930 > > On Jul 12, 2009, at 3:20, "Ciesielski, Frederic (EMEA HPC&OSLO CC)" > > wrote: > >> As far as I could see, x86-64 systems running SLES10 SP2 + OFED 1.4.1 >> are not valid NFS servers anymore, even without trying to activate >> NFS-RDMA, even without using IB at all for the export. >> >> NFS clients see too frequently 'Stale NFS file handle' error messages >> for them to properly read or write anything. >> >> This happens using ethernet and IPoIB... and works fine without OFED, >> or when OFED 1.4 is installed instead (which is not what I want). >> >> Did anybody test that ? >> Any idea about how to get rid of this major side effect ? >> >> Thanks. >> Fred. >> >> >> _______________________________________________ >> general mailing list >> general at lists.openfabrics.org >> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >> >> To unsubscribe, please visit >> http://openib.org/mailman/listinfo/openib-general > ------------------------------------------------------------------------ > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From ogerlitz at voltaire.com Mon Jul 13 06:47:22 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Mon, 13 Jul 2009 16:47:22 +0300 Subject: [ofa-general] patch to ib_addr for sending arps In-Reply-To: <4A5AAC3E.2040609@oracle.com> References: <4A5527A1.5010104@oracle.com> <4A59C576.5060709@voltaire.com> <4A5AAC3E.2040609@oracle.com> Message-ID: <4A5B3AEA.1000400@voltaire.com> leo.tominna at oracle.com wrote: > I think a combination of the two patches, and another change would > address both our problems. The patch I sent ensures neigh_lookup and > neigh_event_send lookup the same route if source based routing tables > are being used. Hi Leo, I will be happy if you can send a few liner explaining that are the two problems you are trying to solve, reading your email I wasn't sure... also the other patch was sent by Rick not Sean Or, From jgunthorpe at obsidianresearch.com Mon Jul 13 06:49:06 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 13 Jul 2009 07:49:06 -0600 Subject: [ofa-general] patch to ib_addr for sending arps In-Reply-To: <4A5AAC3E.2040609@oracle.com> References: <4A5527A1.5010104@oracle.com> <4A59C576.5060709@voltaire.com> <4A5AAC3E.2040609@oracle.com> Message-ID: <20090713134906.GA19062@obsidianresearch.com> On Sun, Jul 12, 2009 at 08:38:38PM -0700, leo.tominna at oracle.com wrote: > Associating the device with the source IP seems to be the correct thing to > do in general, but I initially avoided it in favor of source based routing > rules/tables since Linux does not do this by default. Source based routing > seems to be the only way to get load balancing right for regular IP > traffic Right, this is the Linux Way. The route table associated the output device with the source/destination pair, and it is correct and necessary to have source route policy entries to do what you are talking about. As per the thread Or dug up older versions didn't do this right - did it ever get fixed? It is also necessary to use one of the arp_ignore settings otherwise the receiver side responds with the wrong physical address. Source routing fixes the transmitter, arp_ignore fixes the receiver. > when two local IPs are on the same subnet, so I thought it would be better > to have the src_ip alone cause the routing lookup to associate everything > with the correct device, as that would works for regular IP traffic and > anyone else, like ib_addr clients. > The problems I was seeing with arp was that Linux associates arp entries > with specific devices, so if source based routing is used, and the arp send > does not take src_ip into account, the arp is sent from the default device, > and thats the device that gets the arp entry, where as neigh_lookup was > looking for an entry on the correct device, and was never finding the > neighbor. The proper flow is that the source/dest pair (+ plus extra) are used to do a route lookup. The route lookup returns the output device to use. The arp is sent out that device with source/dest pair. The reciever will then receive the broadcast arp on all interfaces and you need the arp_ignore setting to cause only the interface bound to that IP to generate a response. > From what I can tell, on HPUX there is no device association with arp > entries. Does anyone know why Linux has this flexibility? It looks like > this was added in 2.2 or something, and I can't see any useful applications > for this. I can have multiple networks with overlapping IP spaces and using routing tricks like policy routing I can generate overlapping network specific ARP entries. Consider that I might have two ethernet ports per machine, and two networks, both with the same IP. I can use policy routing to direct all high priority traffic to one network, and low priority to the other. The IPs are the same, but the macs are different, thus the arp table must be keyed by interface and destination. Once you have complex routing stuff like policy routing this becomes necessary, presumably HPUX does not have this.. > The initial patch I sent is less restrictive but relies on source based > routing to get everything working, perhaps an explicit device mapping (as > above) makes more sense for RDMA traffic. Please correct me if something I > said is incorrect or if these changes conflict with other working > configurations. This is Linux, the RDMA IP behavior should exactly match the in-kernel IP behavior. It wasn't clear to me if your patch did that or not? On the surface it looks right, sending the arp from the device (and IP?) the requesting packet is going out is the right thing.. Jason From Brian.Murrell at Sun.COM Mon Jul 13 06:54:27 2009 From: Brian.Murrell at Sun.COM (Brian J. Murrell) Date: Mon, 13 Jul 2009 09:54:27 -0400 Subject: [ofa-general] OFED 1.4.1 breaks SLES10 SP2 NFS server ? In-Reply-To: <7391130E01ED404FBD7A3C86731EEB7D3F73DF8ABB@GVW1087EXB.americas.hpqcorp.net> References: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> <76D6DD3B-26CE-4FCA-9CC0-9C30329D98BE@novell.com> <4A5A38FD.8010909@opengridcomputing.com> <7391130E01ED404FBD7A3C86731EEB7D3F73DF8ABB@GVW1087EXB.americas.hpqcorp.net> Message-ID: <1247493267.6768.377.camel@pc.interlinx.bc.ca> On Mon, 2009-07-13 at 13:28 +0000, Ciesielski, Frederic (EMEA HPC&OSLO CC) wrote: > Yes, that will do it; > I think I already tried to remove this option from the 'install.pl menu and rebuild, but probably the previous rpms were kept. Restarting from scratch and removing the nfs-rdma option permits not to have this updated NFS kernel modules, and things work now... Please note, that there is a problem with this solution currently being discussed on the ewg list regarding this configuration, and that's if you disable the NFSRDMA build option and then try to use the /usr/src/ofa_kernel tree to build other modules that use RPC or NFS headers, you will get headers for the NFSRDMA implementation, not the kernel's NFS/RPC implementation. Unless you are using a recent kernel, the OFA headers will not match the kernel's NFS/RPC implementation. b. -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 197 bytes Desc: This is a digitally signed message part URL: From yevgenyp at mellanox.co.il Mon Jul 13 08:27:48 2009 From: yevgenyp at mellanox.co.il (Yevgeny Petrilin) Date: Mon, 13 Jul 2009 18:27:48 +0300 Subject: [ofa-general][PATCH] mlx4_core: Synch catastrophic flow with module unload Message-ID: <4A5B5274.2020801@mellanox.co.il> There is a race condition when the mlx4_core module is being unloaded during the execution of restart task due to catastrophic error. Added a global mutex that synchs those operations. If the catastrophic task tries to catch the mutex, and it is already taken, it means that somebody is unloading the module, and there is no point in executing the restart operation. If the unload function tries to catch the mutex and it is taken, it would wait for the catas task to finish and then unload the module. Signed-off-by: Yevgeny Petrilin --- drivers/net/mlx4/catas.c | 4 ++++ drivers/net/mlx4/main.c | 6 ++++++ drivers/net/mlx4/mlx4.h | 2 ++ 3 files changed, 12 insertions(+), 0 deletions(-) diff --git a/drivers/net/mlx4/catas.c b/drivers/net/mlx4/catas.c index aa9674b..e3aa7e9 100644 --- a/drivers/net/mlx4/catas.c +++ b/drivers/net/mlx4/catas.c @@ -91,6 +91,9 @@ static void catas_reset(struct work_struct *work) LIST_HEAD(tlist); int ret; + if (!mutex_trylock(&drv_mutex)) + return; + spin_lock_irq(&catas_lock); list_splice_init(&catas_list, &tlist); spin_unlock_irq(&catas_lock); @@ -103,6 +106,7 @@ static void catas_reset(struct work_struct *work) else mlx4_dbg(dev, "Reset succeeded\n"); } + mutex_unlock(&drv_mutex); } void mlx4_start_catas_poll(struct mlx4_dev *dev) diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c index dac621b..9cd5123 100644 --- a/drivers/net/mlx4/main.c +++ b/drivers/net/mlx4/main.c @@ -77,6 +77,8 @@ static char mlx4_version[] __devinitdata = DRV_NAME ": Mellanox ConnectX core driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; +struct mutex drv_mutex; + static struct mlx4_profile default_profile = { .num_qp = 1 << 17, .num_srq = 1 << 16, @@ -1325,6 +1327,8 @@ static int __init mlx4_init(void) { int ret; + mutex_init(&drv_mutex); + if (mlx4_verify_params()) return -EINVAL; @@ -1340,7 +1344,9 @@ static int __init mlx4_init(void) static void __exit mlx4_cleanup(void) { + mutex_lock(&drv_mutex); pci_unregister_driver(&mlx4_driver); + mutex_unlock(&drv_mutex); destroy_workqueue(mlx4_wq); } diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h index 5bd79c2..bd8fb43 100644 --- a/drivers/net/mlx4/mlx4.h +++ b/drivers/net/mlx4/mlx4.h @@ -284,6 +284,8 @@ struct mlx4_sense { struct delayed_work sense_poll; }; +extern struct mutex drv_mutex; + struct mlx4_priv { struct mlx4_dev dev; -- 1.6.0 From devel-ofed at morey-chaisemartin.com Mon Jul 13 07:41:13 2009 From: devel-ofed at morey-chaisemartin.com (Nicolas Morey-Chaisemartin) Date: Mon, 13 Jul 2009 16:41:13 +0200 Subject: [ofa-general] [PATCH] opensm/osm_ucast_ftree.cd: Added support for same level links Message-ID: <4A5B4789.7040305@morey-chaisemartin.com> This patch adds support for horizontal (sibling) links in the Fat-Tree algorithm. Horizontal links are handled as both upward and downward. They are treated as any other through the exploration but they are only considered last to ensure that for 2 paths of the same lengths the one without horizontal links is preferred. Also trying horizontal links after other links ensures that the horizontal path will be as far as possible in the created route (in the ftree exploration sense) thus reducing (removing?) the risk of credit loops. Several tests have been run with Sun QNEM switches and results are good. This patch may require some enhancements later (horizontal links in higher levels have not been fully validated) but it does not change the behaviour on topologies without horizontal links. Thus it should be safe to apply. Acked-by: Line Holen Signed-off-by: Nicolas Morey-Chaisemartin --- opensm/opensm/osm_ucast_ftree.c | 192 ++++++++++++++++++++++++++++++++------- 1 files changed, 159 insertions(+), 33 deletions(-) diff --git a/opensm/opensm/osm_ucast_ftree.c b/opensm/opensm/osm_ucast_ftree.c index 26cdcab..ecd614b 100644 --- a/opensm/opensm/osm_ucast_ftree.c +++ b/opensm/opensm/osm_ucast_ftree.c @@ -170,6 +170,8 @@ typedef struct ftree_sw_t_ { uint16_t base_lid; ftree_port_group_t **down_port_groups; uint8_t down_port_groups_num; + ftree_port_group_t **sibling_port_groups; + uint8_t sibling_port_groups_num; ftree_port_group_t **up_port_groups; uint8_t up_port_groups_num; boolean_t is_leaf; @@ -493,9 +495,11 @@ static void port_group_dump(IN ftree_fabric_t * p_ftree, " Local <--> Remote GUID (LID):" "0x%016" PRIx64 " (0x%04x) <--> 0x%016" PRIx64 " (0x%04x)\n", size, buff, - (direction == FTREE_DIRECTION_DOWN) ? "DOWN" : "UP", - cl_ntoh64(p_group->port_guid), p_group->base_lid, - cl_ntoh64(p_group->remote_port_guid), p_group->remote_base_lid); + (direction == FTREE_DIRECTION_DOWN) ? "DOWN" : (direction == + FTREE_DIRECTION_SAME) + ? "SIBLING" : "UP", cl_ntoh64(p_group->port_guid), + p_group->base_lid, cl_ntoh64(p_group->remote_port_guid), + p_group->remote_base_lid); } /* port_group_dump() */ @@ -552,7 +556,12 @@ static ftree_sw_t *sw_create(IN ftree_fabric_t * p_ftree, p_sw->up_port_groups = (ftree_port_group_t **) malloc(ports_num * sizeof(ftree_port_group_t *)); - if (!p_sw->down_port_groups || !p_sw->up_port_groups) + p_sw->sibling_port_groups = + (ftree_port_group_t **) malloc(ports_num * + sizeof(ftree_port_group_t *)); + + if (!p_sw->down_port_groups || !p_sw->up_port_groups + || !p_sw->sibling_port_groups) return NULL; /* initialize lft buffer */ @@ -577,10 +586,14 @@ static void sw_destroy(IN ftree_fabric_t * p_ftree, IN ftree_sw_t * p_sw) for (i = 0; i < p_sw->down_port_groups_num; i++) port_group_destroy(p_sw->down_port_groups[i]); + for (i = 0; i < p_sw->sibling_port_groups_num; i++) + port_group_destroy(p_sw->sibling_port_groups[i]); for (i = 0; i < p_sw->up_port_groups_num; i++) port_group_destroy(p_sw->up_port_groups[i]); if (p_sw->down_port_groups) free(p_sw->down_port_groups); + if (p_sw->sibling_port_groups) + free(p_sw->sibling_port_groups); if (p_sw->up_port_groups) free(p_sw->up_port_groups); @@ -617,13 +630,17 @@ static void sw_dump(IN ftree_fabric_t * p_ftree, IN ftree_sw_t * p_sw) OSM_LOG(&p_ftree->p_osm->log, OSM_LOG_DEBUG, "Switch index: %s, GUID: 0x%016" PRIx64 - ", Ports: %u DOWN, %u UP\n", + ", Ports: %u DOWN, %u SIBLINGS, %u UP\n", tuple_to_str(p_sw->tuple), sw_get_guid_ho(p_sw), - p_sw->down_port_groups_num, p_sw->up_port_groups_num); + p_sw->down_port_groups_num, p_sw->sibling_port_groups_num, + p_sw->up_port_groups_num); for (i = 0; i < p_sw->down_port_groups_num; i++) port_group_dump(p_ftree, p_sw->down_port_groups[i], FTREE_DIRECTION_DOWN); + for (i = 0; i < p_sw->sibling_port_groups_num; i++) + port_group_dump(p_ftree, p_sw->sibling_port_groups[i], + FTREE_DIRECTION_SAME); for (i = 0; i < p_sw->up_port_groups_num; i++) port_group_dump(p_ftree, p_sw->up_port_groups[i], FTREE_DIRECTION_UP); @@ -652,6 +669,9 @@ static ftree_port_group_t *sw_get_port_group_by_remote_lid(IN ftree_sw_t * p_sw, if (direction == FTREE_DIRECTION_UP) { port_groups = p_sw->up_port_groups; size = p_sw->up_port_groups_num; + } else if (direction == FTREE_DIRECTION_SAME) { + port_groups = p_sw->sibling_port_groups; + size = p_sw->sibling_port_groups_num; } else { port_groups = p_sw->down_port_groups; size = p_sw->down_port_groups_num; @@ -687,10 +707,14 @@ static void sw_add_port(IN ftree_sw_t * p_sw, IN uint8_t port_num, p_remote_hca_or_sw, FALSE, FALSE); CL_ASSERT(p_group); - if (direction == FTREE_DIRECTION_UP) + if (direction == FTREE_DIRECTION_UP) { p_sw->up_port_groups[p_sw->up_port_groups_num++] = p_group; - else + } else if (direction == FTREE_DIRECTION_SAME) { + p_sw->sibling_port_groups[p_sw-> + sibling_port_groups_num++] = + p_group; + } else p_sw->down_port_groups[p_sw->down_port_groups_num++] = p_group; } @@ -1997,6 +2021,34 @@ bubble_sort_up(ftree_port_group_t ** p_group_array, uint32_t nmemb) p_group_array[0]->hca_or_sw.p_sw->counter_up_changed = FALSE; } +static inline void +bubble_sort_siblings(ftree_port_group_t ** p_group_array, uint32_t nmemb) +{ + uint32_t i = 0; + uint32_t j = 0; + ftree_port_group_t *tmp = p_group_array[0]; + + /* While we did modifications on the array order */ + /* i may grew above array length but next loop will fail and tmp will be null for the next time + * this way we save a test i < nmemb for each pass through the loop */ + for (i = 0; tmp != NULL; i++) { + /* Assume the array is orderd */ + tmp = NULL; + /* Comparing elements j and j-1 */ + for (j = 1; j < (nmemb - i); j++) { + /* If they are the wrong way around */ + if (p_group_array[j]->counter_up < + p_group_array[j - 1]->counter_up) { + /* We invert them */ + tmp = p_group_array[j - 1]; + p_group_array[j - 1] = p_group_array[j]; + p_group_array[j] = tmp; + + } + } + } +} + /* * Function: Sorts an array of port group. Order is decide through * __osm_ftree_port_group_compare_load_down ( up counters, least load remote switch, biggest GUID) @@ -2085,9 +2137,24 @@ fabric_route_upgoing_by_going_down(IN ftree_fabric_t * p_ftree, /* foreach down-going port group (in indexing order) */ bubble_sort_up(p_sw->down_port_groups, p_sw->down_port_groups_num); - for (k = 0; k < p_sw->down_port_groups_num; k++) { - p_group = p_sw->down_port_groups[k]; + if (p_sw->sibling_port_groups_num > 0) + bubble_sort_siblings(p_sw->sibling_port_groups, + p_sw->sibling_port_groups_num); + + for (k = 0; + k < + (p_sw->down_port_groups_num + + (is_real_lid ? p_sw->sibling_port_groups_num : 0)); k++) { + + if (k < p_sw->down_port_groups_num) { + p_group = p_sw->down_port_groups[k]; + } else { + p_group = + p_sw->sibling_port_groups[k - + p_sw-> + down_port_groups_num]; + } /* If this port group doesn't point to a switch, mark that the route was created and skip to the next group */ @@ -2384,8 +2451,8 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, if (p_min_group->counter_down == (p_min_group->remote_hca_or_sw.p_sw->min_counter_down + 1)) { - recalculate_min_counter_down(p_min_group-> - remote_hca_or_sw.p_sw); + recalculate_min_counter_down + (p_min_group->remote_hca_or_sw.p_sw); } if (is_real_lid) { @@ -2533,10 +2600,86 @@ fabric_route_downgoing_by_going_up(IN ftree_fabric_t * p_ftree, created_route |= routed; } + /* TREATING SIBLINGS !!! */ + if (p_sw->sibling_port_groups_num > 0) + bubble_sort_down(p_sw->sibling_port_groups, + p_sw->sibling_port_groups_num); + + for (i = 0; i < p_sw->sibling_port_groups_num; i++) { + p_group = p_sw->sibling_port_groups[i]; + p_remote_sw = p_group->remote_hca_or_sw.p_sw; + + /* skip if target lid has been already set on remote switch fwd tbl (with a bigger hop count) */ + if (p_remote_sw->p_osm_sw->new_lft[target_lid] != OSM_NO_PATH) + if (current_hops + 1 >= + sw_get_least_hops(p_remote_sw, target_lid)) + continue; + + if (p_sw->is_leaf) { + OSM_LOG(&p_ftree->p_osm->log, OSM_LOG_DEBUG, + " - Routing SECONDARY path for LID %u: %s --> %s\n", + target_lid, + tuple_to_str(p_sw->tuple), + tuple_to_str(p_remote_sw->tuple)); + } + + /* Routing REAL lids on SECONDARY path means routing + switch-to-switch or switch-to-CA paths. + We can safely assume that switch will initiate very + few traffic, so there's no point waisting runtime on + trying to balance these routes - always pick port 0. */ + + p_min_port = NULL; + ports_num = (uint16_t) cl_ptr_vector_get_size(&p_group->ports); + for (j = 0; j < ports_num; j++) { + cl_ptr_vector_at(&p_group->ports, j, (void *)&p_port); + if (!p_min_port) { + /* first port that we're checking - use + it as a port with the lowest load */ + p_min_port = p_port; + } else if (p_port->counter_down < + p_min_port->counter_down) { + /* this port is less loaded - use it as min */ + p_min_port = p_port; + } + } + + p_port = p_min_port; + //cl_ptr_vector_at(&p_group->ports, 0, (void *)&p_port); + p_remote_sw->p_osm_sw->new_lft[target_lid] = + p_port->remote_port_num; + + /* On the remote switch that is pointed by the p_group, + set hops for ALL the ports in the remote group. */ + + set_hops_on_remote_sw(p_group, target_lid, + current_hops + 1, is_target_a_sw); + + /* Recursion step: + Assign downgoing ports by stepping up, starting on REMOTE switch. */ + routed = fabric_route_downgoing_by_going_up(p_ftree, p_remote_sw, /* remote switch - used as a route-downgoing alg. next step point */ + p_sw, /* this switch - prev. position switch for the function */ + target_lid, /* LID that we're routing to */ + TRUE, /* whether the target LID is real or dummy */ + FALSE, /* whether this is path to HCA that should by tracked by counters */ + is_target_a_sw, /* Wheter target lid is a switch or not */ + reverse_hop_credit, /* Remaining reverse_hops allowed */ + reverse_hops, /* Number of reverse_hops done up to this point */ + current_hops + 1); + created_route |= routed; + if (routed) { + p_min_group->counter_down++; + p_min_port->counter_down++; + } + } + /* If we don't have any reverse hop credits, we are done */ if (reverse_hop_credit == 0) return created_route; + if (p_sw->is_leaf) + return created_route; + /* We explore all the down group ports */ /* We try to reverse jump for each of them */ /* They already have a route to us from the upgoing_by_going_down started earlier */ @@ -2659,7 +2802,6 @@ static void fabric_route_to_cns(IN ftree_fabric_t * p_ftree) /* We're done with the real targets (all CNs) of this leaf switch. Now route the dummy HCAs that are missing or that are non-CNs. When routing to dummy HCAs we don't fill lid matrices. */ - if (p_ftree->max_cn_per_leaf > routed_targets_on_leaf) { OSM_LOG(&p_ftree->p_osm->log, OSM_LOG_DEBUG, "Routing %u dummy CAs\n", @@ -3240,27 +3382,11 @@ static int fabric_construct_sw_ports(IN ftree_fabric_t * p_ftree, p_remote_hca_or_sw = (void *)p_remote_sw; - if (abs(p_sw->rank - p_remote_sw->rank) != 1) { - OSM_LOG(&p_ftree->p_osm->log, OSM_LOG_ERROR, - "ERR AB16: " - "Illegal link between switches with ranks %u and %u:\n" - " GUID 0x%016" PRIx64 - ", LID %u, rank %u\n" - " GUID 0x%016" PRIx64 - ", LID %u, rank %u\n", p_sw->rank, - p_remote_sw->rank, - sw_get_guid_ho(p_sw), - p_sw->base_lid, p_sw->rank, - sw_get_guid_ho(p_remote_sw), - p_remote_sw->base_lid, - p_remote_sw->rank); - res = -1; - goto Exit; - } - - if (p_sw->rank > p_remote_sw->rank) + if (p_sw->rank > p_remote_sw->rank) { direction = FTREE_DIRECTION_UP; - else + } else if (p_sw->rank == p_remote_sw->rank) { + direction = FTREE_DIRECTION_SAME; + } else direction = FTREE_DIRECTION_DOWN; /* switch LID is only in port 0 port_info structure */ From tziporet at mellanox.co.il Mon Jul 13 07:46:24 2009 From: tziporet at mellanox.co.il (Tziporet Koren) Date: Mon, 13 Jul 2009 17:46:24 +0300 Subject: [ofa-general] EWG/OFED agenda for today (July 13, 2009) In-Reply-To: <5D49E7A8952DC44FB38C38FA0D758EAD0334A01D@mtlexch01.mtl.com> References: <5D49E7A8952DC44FB38C38FA0D758EAD0334A01D@mtlexch01.mtl.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD03565C67@mtlexch01.mtl.com> This is EWG/OFED the agenda for today (July 13, 2009) 1. OFED 1.5 status: all We should be ready for pre-alpha release tomorrow SRP and iSER still not compiling 2. OFED 1.4.2: We wait for the bugs to be resolved Need an update from Jon Mason & Brian Murrell Intel requested to update nes driver too Tziporet From jon at opengridcomputing.com Mon Jul 13 07:57:55 2009 From: jon at opengridcomputing.com (Jon Mason) Date: Mon, 13 Jul 2009 09:57:55 -0500 Subject: [ofa-general] OFED 1.4.1 breaks SLES10 SP2 NFS server ? In-Reply-To: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> References: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> Message-ID: <20090713145754.GA24846@opengridcomputing.com> On Sun, Jul 12, 2009 at 10:20:58AM +0000, Ciesielski, Frederic (EMEA HPC&OSLO CC) wrote: > As far as I could see, x86-64 systems running SLES10 SP2 + OFED 1.4.1 are not valid NFS servers anymore, even without trying to activate NFS-RDMA, even without using IB at all for the export. > > NFS clients see too frequently 'Stale NFS file handle' error messages for them to properly read or write anything. > > This happens using ethernet and IPoIB... and works fine without OFED, or when OFED 1.4 is installed instead (which is not what I want). > > Did anybody test that ? I see it as well when testing over IB, but I did not see it when testing iWARP before OFED 1.4.1 came out. I have patches queued to fix it for OFED 1.4.2. Thanks, Jon > Any idea about how to get rid of this major side effect ? > > Thanks. > Fred. > > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From tziporet at dev.mellanox.co.il Mon Jul 13 08:20:33 2009 From: tziporet at dev.mellanox.co.il (Tziporet Koren) Date: Mon, 13 Jul 2009 18:20:33 +0300 Subject: [ofa-general] Re: [ewg] EWG/OFED agenda for today (July 13, 2009) In-Reply-To: <5D49E7A8952DC44FB38C38FA0D758EAD03565C67@mtlexch01.mtl.com> References: <5D49E7A8952DC44FB38C38FA0D758EAD0334A01D@mtlexch01.mtl.com> <5D49E7A8952DC44FB38C38FA0D758EAD03565C67@mtlexch01.mtl.com> Message-ID: <4A5B50C1.9050209@mellanox.co.il> Tziporet Koren wrote: > This is EWG/OFED the agenda for today (July 13, 2009) > > 1. OFED 1.5 status: all > We should be ready for pre-alpha release tomorrow > SRP and iSER still not compiling > > 2. OFED 1.4.2: We wait for the bugs to be resolved > Need an update from Jon Mason & Brian Murrell > Intel requested to update nes driver too > > > 3. local SA pathces - do we wish to leave them? > Meeting details (from Jeff outlook): Topic: OFED bi-weekly teleconference Date: Every other Monday, from Monday, June 29, 2009 to Monday, August 24, 2009 Time: 12:00 pm, Eastern Daylight Time (GMT -04:00, New York) Meeting Number: 207 415 805 Meeting Password: ofed Please click the link below to see more information, or to join the meeting. ---------------------------------------------------------------- ALERT:Toll-Free Dial Restrictions for (408) and (919) Area Codes ---------------------------------------------------------------- As of April 9th, 2009, you can no longer dial toll free in the 408 or 919 area codes in the United States. The affected toll free numbers are: (866) 432-9903 for the San Jose/Milpitas area and (866) 349-3520 for the RTP area. Please dial the local access number for your area from the list below: - San Jose/Milpitas (408) area: 525-6800 - RTP (919) area: 392-3330 ------------------------------------------------------- To join the online meeting ------------------------------------------------------- 1. Go to https://cisco.webex.com/cisco/j.php?ED=121669737&UID=0&PW=5bc3eb20892616555c 2. Enter your name and email address. 3. Enter the meeting password: ofed 4. Click "Join Now". ------------------------------------------------------- To join the teleconference only ------------------------------------------------------- 1. Dial into Cisco WebEx (view all Global Access Numbers at http://cisco.com/en/US/about/doing_business/conferencing/index.html 2. Press 3 to attend the meeting. 3. Follow the prompts to enter the Meeting Number (listed above) or Access Code followed by the # sign. San Jose, CA: +1.408.525.6800 RTP: +1.919.392.3330 US/Canada: +1.866.432.9903 United Kingdom: +44.20.8824.0117 India: +91.80.4350.1111 Germany: +49.619.6773.9002 Japan: +81.3.5763.9394 China: +86.10.8515.5666 Israel: +972.9.892.7026 Others: http://cisco.com/en/US/about/doing_business/conferencing/ From leo.tominna at oracle.com Mon Jul 13 08:27:13 2009 From: leo.tominna at oracle.com (leo.tominna at oracle.com) Date: Mon, 13 Jul 2009 08:27:13 -0700 Subject: [ofa-general] patch to ib_addr for sending arps In-Reply-To: <4A5B3AEA.1000400@voltaire.com> References: <4A5527A1.5010104@oracle.com> <4A59C576.5060709@voltaire.com> <4A5AAC3E.2040609@oracle.com> <4A5B3AEA.1000400@voltaire.com> Message-ID: <4A5B5251.6050902@oracle.com> Hi Or, One problem is that policy routing (I referred to it as source based routing) doesn't work with ib_addr, that was the initial patch I sent out. The second problem is that on some platforms like HPUX, this stuff works even without special routing rules (that was the additional lines added in my previous reply email, setting oif explicitly). I'll reply to Jason's email, but it looks like this is not really a problem. Thanks, Leo Tominna On 7/13/2009 6:47 AM, Or Gerlitz wrote: > leo.tominna at oracle.com wrote: >> I think a combination of the two patches, and another change would >> address both our problems. The patch I sent ensures neigh_lookup and >> neigh_event_send lookup the same route if source based routing tables >> are being used. > Hi Leo, > > I will be happy if you can send a few liner explaining that are the > two problems you are trying to solve, reading your email I wasn't > sure... also the other patch was sent by Rick not Sean > > Or, > From lars.ellenberg at linbit.com Mon Jul 13 08:57:20 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Mon, 13 Jul 2009 17:57:20 +0200 Subject: [ofa-general] using SDP for block device traffic: several problems In-Reply-To: <4A561AAA.4050904@mellanox.co.il> References: <20090701133652.GG9112@soda.linbit> <4A4B947C.1030607@mellanox.co.il> <20090702081445.GA9118@soda.linbit> <4A4C7021.2090506@mellanox.co.il> <20090702132304.GD9305@soda.linbit> <20090707171326.GH8065@barkeeper1-xen.linbit> <4A5454DF.1070101@mellanox.co.il> <20090708101715.GB8083@barkeeper1-xen.linbit> <4A561AAA.4050904@mellanox.co.il> Message-ID: <20090713155720.GA8038@barkeeper1-xen.linbit> On Thu, Jul 09, 2009 at 07:28:26PM +0300, Amir Vadai wrote: > Lars Hi, > > I'v found the bug. > > a fix patch is attached. > > I will commit it after the weekend (In Israel weekend is Friday-Saturday). Great news. Fix Confirmed. Thanks for tracking this down. Will be back with stats and tuning questions soonish. Cheers, -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. From ranjit.pandit.ib at gmail.com Mon Jul 13 09:49:32 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Mon, 13 Jul 2009 09:49:32 -0700 Subject: [ofa-general] Memory registration limit of 16GB with Chelsio In-Reply-To: <8A71B368A89016469F72CD08050AD33405E63DDE@maui.asicdesigners.com> References: <96f8e60e0907101115i11fcf051h8508fdc18282368b@mail.gmail.com> <4A578C4D.2060003@opengridcomputing.com> <8A71B368A89016469F72CD08050AD33405E63DDE@maui.asicdesigners.com> Message-ID: <96f8e60e0907130949y7edf6926l6e70dafc44a7503a@mail.gmail.com> > Hey Felix, > > Can we somehow adjust the PMRX memory map dynamically?  For instance, > maybe drop the number of TPTs to allow for more PBL memory? > [felix] Not dynamically, but it can surely be changed. DM, can it be > controlled through a cxgbtool? > For now, is there a patch to the code that we can experiment with? thx, Ranjit From ranjit.pandit.ib at gmail.com Mon Jul 13 09:52:26 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Mon, 13 Jul 2009 09:52:26 -0700 Subject: [ofa-general] Compilation errors with OFED 1.4.1/ 1.4 In-Reply-To: <200907071217.19649.jackm@dev.mellanox.co.il> References: <9759F033B56A60469F217C49AC58E2BF085A541B@MAILUK2.rms.com> <4A3E9957.50104@mellanox.co.il> <3307cdf90906212157o687a6929w49f9abc8b16fc131@mail.gmail.com> <200907071217.19649.jackm@dev.mellanox.co.il> Message-ID: <96f8e60e0907130952p73f09362o5b75bd121080e963@mail.gmail.com> > Looks like the OFED installation is faulty. Can we fix this issue in the next release of OFED? > For some reason, your compilation script is not taking directory > /usr/src/ofa_kernel/kernel_addons/backport/2.6.16_sles10_sp2/include > in the include path before the regular kernel includes. Are you referring to the OFED compilation script or our module's? This problem happens on RedHat EL 5.0 as well. On Tue, Jul 7, 2009 at 2:17 AM, Jack Morgenstein wrote: > The missing function declarations are all found in header files under directory > (on your system) /usr/src/ofa_kernel/kernel_addons/backport/2.6.16_sles10_sp2. > These header files must be taken before the regular kernel header files > (they "include_next" to the regular files, and supplement these files > with missing macros, static inline functions, etc.). > > For example, dma_map_single_attr() is found in file > /usr/src/ofa_kernel/kernel_addons/backport/2.6.16_sles10_sp2/include/linux/dma-mapping.h > > For some reason, your compilation script is not taking directory > /usr/src/ofa_kernel/kernel_addons/backport/2.6.16_sles10_sp2/include > in the include path before the regular kernel includes. > > -Jack > > On Monday 22 June 2009 07:57, Rajouri Jammu wrote: > >> Tziporet, >> > We (Mellanox) are running regression testing on SLES10 SP2 with OFED 1.4.1 >> without any problem >> >> I don't think it's a runtime issue. >> >> The problem: >> /usr/src/ofa_kernel/include/rdma/ib_verbs.h in OFED 1.4.1 is referring to >> symbols like "dma_map_single_attrs" etc, but when I looked for those symbols >> in /usr/src/linux-2.6.16.60-0.21 I did not find any references. >> >> Can you suggest a file that I should include that will resolve this issue? >> >> Thanks. >> >> On Sun, Jun 21, 2009 at 1:34 PM, Tziporet Koren > > wrote: >> >> > Rajouri Jammu wrote: >> > >> >> Thanks for the response Evan. >> >> >> >> It appears to me that this is a problem in OFED 1.4.1 since the header >> >> file in /usr/src/ofa_kernel/include/rdma/ib_verbs.h is referring to data >> >> structures that are not present in the SLES 10 u 2 kernel. >> >> >> >> However, someone else indicated it was a problem in our code. >> >> >> >> Btw, my code compiled fine with OFED 1.3.1 on the same kernel. >> >> >> >> >> > We (Mellanox) are running regression testing on SLES10 SP2 with OFED 1.4.1 >> > without any problem >> > So I guess its something specific in your system, or order of include files >> > you use >> > >> > Tziporet >> > >> > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From Jeffrey.C.Becker at nasa.gov Mon Jul 13 09:55:54 2009 From: Jeffrey.C.Becker at nasa.gov (Jeff Becker) Date: Mon, 13 Jul 2009 09:55:54 -0700 Subject: [ofa-general] OFED 1.4.1 breaks SLES10 SP2 NFS server ? In-Reply-To: <4A5A56B3.80200@opengridcomputing.com> References: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> <76D6DD3B-26CE-4FCA-9CC0-9C30329D98BE@novell.com> <4A5A38FD.8010909@opengridcomputing.com> <4A5A56B3.80200@opengridcomputing.com> Message-ID: <4A5B671A.8070103@nasa.gov> Tom Tucker wrote: > Jeff: > > Can you answer Sven's question regarding the level of testing performed > on your port of NFSRDMA to SLES10SP2? > I tested NFSRDMA on SLES10SP2 server and client (both x86-64). Connectathon passed. Thanks -jeff > Thanks, > Tom > > > Steve Wise wrote: > >> If you don't want/need NFSRDMA (especially since it appears to not work >> well on sles10sp2), you can build/install ofed-1.4.1 with nfsrdma turned >> off. Assuming you're installing "all" when you run install.pl, you'll >> have to run install.pl and choose a custom install and select everything >> except nfsrdma. >> >> Does that make sense? >> >> Steve. >> >> Sven-Thorsten Dietrich wrote: >> >>> Is your system up to date with patches? >>> >>> If so, please open a bug on bugzilla.Novell.com >>> . >>> Thanks, >>> Sven >>> >>> + 1 (415) 694 2930 >>> >>> On Jul 12, 2009, at 3:20, "Ciesielski, Frederic (EMEA HPC&OSLO CC)" >>> > wrote: >>> >>> >>>> As far as I could see, x86-64 systems running SLES10 SP2 + OFED 1.4.1 >>>> are not valid NFS servers anymore, even without trying to activate >>>> NFS-RDMA, even without using IB at all for the export. >>>> >>>> NFS clients see too frequently 'Stale NFS file handle' error messages >>>> for them to properly read or write anything. >>>> >>>> This happens using ethernet and IPoIB... and works fine without OFED, >>>> or when OFED 1.4 is installed instead (which is not what I want). >>>> >>>> Did anybody test that ? >>>> Any idea about how to get rid of this major side effect ? >>>> >>>> Thanks. >>>> Fred. >>>> >>>> >>>> _______________________________________________ >>>> general mailing list >>>> general at lists.openfabrics.org >>>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>>> >>>> To unsubscribe, please visit >>>> http://openib.org/mailman/listinfo/openib-general >>>> >>> ------------------------------------------------------------------------ >>> >>> _______________________________________________ >>> general mailing list >>> general at lists.openfabrics.org >>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>> >>> To unsubscribe, please visit >>> http://openib.org/mailman/listinfo/openib-general >>> > > From leo.tominna at oracle.com Mon Jul 13 10:14:05 2009 From: leo.tominna at oracle.com (leo.tominna at oracle.com) Date: Mon, 13 Jul 2009 10:14:05 -0700 Subject: [ofa-general] patch to ib_addr for sending arps In-Reply-To: <20090713134906.GA19062@obsidianresearch.com> References: <4A5527A1.5010104@oracle.com> <4A59C576.5060709@voltaire.com> <4A5AAC3E.2040609@oracle.com> <20090713134906.GA19062@obsidianresearch.com> Message-ID: <4A5B6B5D.6070605@oracle.com> Hi Jason, Thanks for clearing up the use case. In that case doing ip_dev_find to set oif would be wrong since it would not work correctly in the case the same IP is associated with two devices. By just setting s_addr before calling ip_route_output_key in addr_send_arp, that should take care of it (the initial patch sent). From what I can tell, this just fixes the policy routing case, without affecting/addressing configurations that are using default routing. I need to see why RDS/IB gets stuck in this case. My guess is that hardware addresses don't get resolved correctly (as expected), and two sides of an IB connection trip over a mismatch in what hardware a peer thinks its using. But that is another issue that can be fixed independently. I'll add some prints to see what might be happening. Thanks, Leo Tominna On 7/13/2009 6:49 AM, Jason Gunthorpe wrote: > On Sun, Jul 12, 2009 at 08:38:38PM -0700, leo.tominna at oracle.com wrote: > > >> Associating the device with the source IP seems to be the correct thing to >> do in general, but I initially avoided it in favor of source based routing >> rules/tables since Linux does not do this by default. Source based routing >> seems to be the only way to get load balancing right for regular IP >> traffic >> > > Right, this is the Linux Way. The route table associated the output > device with the source/destination pair, and it is correct and > necessary to have source route policy entries to do what you are > talking about. As per the thread Or dug up older versions didn't do > this right - did it ever get fixed? > > It is also necessary to use one of the arp_ignore settings otherwise > the receiver side responds with the wrong physical address. Source > routing fixes the transmitter, arp_ignore fixes the receiver. > > >> when two local IPs are on the same subnet, so I thought it would be better >> to have the src_ip alone cause the routing lookup to associate everything >> with the correct device, as that would works for regular IP traffic and >> anyone else, like ib_addr clients. >> > > >> The problems I was seeing with arp was that Linux associates arp entries >> with specific devices, so if source based routing is used, and the arp send >> does not take src_ip into account, the arp is sent from the default device, >> and thats the device that gets the arp entry, where as neigh_lookup was >> looking for an entry on the correct device, and was never finding the >> neighbor. >> > > The proper flow is that the source/dest pair (+ plus extra) are used > to do a route lookup. The route lookup returns the output device to > use. The arp is sent out that device with source/dest pair. The > reciever will then receive the broadcast arp on all interfaces and you > need the arp_ignore setting to cause only the interface bound to that > IP to generate a response. > > >> From what I can tell, on HPUX there is no device association with arp >> entries. Does anyone know why Linux has this flexibility? It looks like >> this was added in 2.2 or something, and I can't see any useful applications >> for this. >> > > I can have multiple networks with overlapping IP spaces and using > routing tricks like policy routing I can generate overlapping network > specific ARP entries. Consider that I might have two ethernet ports > per machine, and two networks, both with the same IP. I can use policy > routing to direct all high priority traffic to one network, and low > priority to the other. The IPs are the same, but the macs are > different, thus the arp table must be keyed by interface and > destination. > > Once you have complex routing stuff like policy routing this becomes > necessary, presumably HPUX does not have this.. > > >> The initial patch I sent is less restrictive but relies on source based >> routing to get everything working, perhaps an explicit device mapping (as >> above) makes more sense for RDMA traffic. Please correct me if something I >> said is incorrect or if these changes conflict with other working >> configurations. >> > > This is Linux, the RDMA IP behavior should exactly match the in-kernel > IP behavior. It wasn't clear to me if your patch did that or not? > > On the surface it looks right, sending the arp from the device (and > IP?) the requesting packet is going out is the right thing.. > > Jason > -------------- next part -------------- An HTML attachment was scrubbed... URL: From eli at dev.mellanox.co.il Mon Jul 13 11:13:10 2009 From: eli at dev.mellanox.co.il (Eli Cohen) Date: Mon, 13 Jul 2009 21:13:10 +0300 Subject: [ofa-general] [PATCH 0/8 v3] RDMAoE support Message-ID: <20090713181310.GA31865@mtls03> RDMA over Ethernet (RDMAoE) allows running the IB transport protocol using Ethernet frames allowing the deployment of IB semantics on lossless Ethernet fabrics. RDMAoE packets are standard Ethernet frames with an IEEE assigned Ethertype, a GRH, unmodified IB transport headers and payload. Aside from the considerations pointed out below, RDMAoE ports are functionally equivalent to regular IB ports from the RDMA stack perspective. IB subnet management and SA services are not required for RDMAoE operation; Ethernet management practices are used instead. In Ethernet, nodes are commonly referred to by applications by means of an IP address. RDMAoE encodes the IP addresses that were assigned to the corresponding Ethernet port into its GIDs, and makes use of the IP stack to bind a destination address to the corresponding netdevice (just as the CMA does today for IB and iWARP) and to obtain its L2 MAC addresses. The RDMA Verbs API is syntactically unmodified. When referring to RDMAoE ports, Address handles are required to contain GIDs and the L2 address fields in the API are ignored. The Ethernet L2 information is then obtained by the vendor-specific driver (both in kernel- and user-space) while modifying QPs to RTR and creating address handles. In order to maximize transparency for applications, RDMAoE implements a dedicated API that provides services equivalent to some of those provided by the IB-SA. The current approach is strictly local but may evolve in the future. This API is implemented using an independent source code file which allows for seamless evolution of the code without affecting the IB native SA interfaces. We have successfully tested MPI, SDP, RDS, and native Verbs applications over RDMAoE. To enable RDMAoE with the mlx4 driver stack, both the mlx4_en and mlx4_ib drivers must be loaded, and the netdevice for the corresponding RDMAoE port must be running. Individual ports of a multi port HCA can be independently configured as Ethernet (with support for RDMAoE) or IB, as is already the case. Following is a series of 8 patches based on version 2.6.30 of the Linux kernel. This new series reflects changes based on feedback from the community on the previous set of patches. The whole series is tagged v3. Signed-off-by: Eli Cohen drivers/infiniband/core/Makefile | 2 drivers/infiniband/core/addr.c | 20 drivers/infiniband/core/agent.c | 12 drivers/infiniband/core/cma.c | 124 +++ drivers/infiniband/core/mad.c | 48 + drivers/infiniband/core/multicast.c | 43 - drivers/infiniband/core/multicast.h | 79 ++ drivers/infiniband/core/rdmaoe_sa.c | 942 ++++++++++++++++++++++++++++++ drivers/infiniband/core/sa.h | 24 drivers/infiniband/core/sa_query.c | 26 drivers/infiniband/core/ud_header.c | 111 +++ drivers/infiniband/core/uverbs.h | 1 drivers/infiniband/core/uverbs_cmd.c | 33 + drivers/infiniband/core/uverbs_main.c | 1 drivers/infiniband/core/verbs.c | 17 drivers/infiniband/hw/mlx4/ah.c | 228 ++++++- drivers/infiniband/hw/mlx4/main.c | 276 +++++++- drivers/infiniband/hw/mlx4/mlx4_ib.h | 30 drivers/infiniband/hw/mlx4/qp.c | 253 ++++++-- drivers/infiniband/ulp/ipoib/ipoib_main.c | 3 drivers/net/mlx4/cmd.c | 6 drivers/net/mlx4/en_main.c | 15 drivers/net/mlx4/en_port.c | 4 drivers/net/mlx4/en_port.h | 3 drivers/net/mlx4/intf.c | 20 drivers/net/mlx4/main.c | 6 drivers/net/mlx4/mlx4.h | 1 include/linux/mlx4/cmd.h | 1 include/linux/mlx4/device.h | 31 include/linux/mlx4/driver.h | 16 include/linux/mlx4/qp.h | 8 include/rdma/ib_addr.h | 53 + include/rdma/ib_pack.h | 26 include/rdma/ib_user_verbs.h | 21 include/rdma/ib_verbs.h | 22 include/rdma/rdmaoe_sa.h | 66 ++ 36 files changed, 2333 insertions(+), 239 deletions(-) From eli at mellanox.co.il Mon Jul 13 11:13:19 2009 From: eli at mellanox.co.il (Eli Cohen) Date: Mon, 13 Jul 2009 21:13:19 +0300 Subject: [ofa-general] [PATCH 1/8 v3] ib_core: Add API to support RDMAoE Message-ID: <20090713181319.GA738@mtls03> Add two API functions needed for RDMAoE. ib_get_port_link_type() returns the link type support by the given device's port. It can be either PORT_LINK_IB for IB link layer or PORT_LINK_ETH for Ethernet links. Link type is reported to in query_port verb. ib_get_mac() will return the Ethernet MAC address leading to the port whose GID is spcified. This function is exported to userspace applications. ABI version is incremented from 6 to 7. Signed-off-by: Eli Cohen --- drivers/infiniband/core/uverbs.h | 1 + drivers/infiniband/core/uverbs_cmd.c | 33 +++++++++++++++++++++++++++++++++ drivers/infiniband/core/uverbs_main.c | 1 + drivers/infiniband/core/verbs.c | 17 +++++++++++++++++ include/rdma/ib_user_verbs.h | 21 ++++++++++++++++++--- include/rdma/ib_verbs.h | 22 ++++++++++++++++++++++ 6 files changed, 92 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index b3ea958..e69b04c 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -194,5 +194,6 @@ IB_UVERBS_DECLARE_CMD(create_srq); IB_UVERBS_DECLARE_CMD(modify_srq); IB_UVERBS_DECLARE_CMD(query_srq); IB_UVERBS_DECLARE_CMD(destroy_srq); +IB_UVERBS_DECLARE_CMD(get_mac); #endif /* UVERBS_H */ diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 56feab6..eefc414 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -452,6 +452,7 @@ ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, resp.active_width = attr.active_width; resp.active_speed = attr.active_speed; resp.phys_state = attr.phys_state; + resp.link_type = attr.link_type; if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) @@ -1824,6 +1825,38 @@ err: return ret; } +ssize_t ib_uverbs_get_mac(struct ib_uverbs_file *file, + const char __user *buf, int in_len, + int out_len) +{ + struct ib_uverbs_get_mac cmd; + struct ib_uverbs_get_mac_resp resp; + int ret; + struct ib_pd *pd; + + if (out_len < sizeof resp) + return -ENOSPC; + + if (copy_from_user(&cmd, buf, sizeof cmd)) + return -EFAULT; + + pd = idr_read_pd(cmd.pd_handle, file->ucontext); + if (!pd) + return -EINVAL; + + ret = ib_get_mac(pd->device, cmd.port, cmd.gid, resp.mac); + put_pd_read(pd); + if (!ret) { + if (copy_to_user((void __user *) (unsigned long) cmd.response, + &resp, sizeof resp)) { + return -EFAULT; + } + return in_len; + } + + return ret; +} + ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index eb36a81..b2f148f 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -108,6 +108,7 @@ static ssize_t (*uverbs_cmd_table[])(struct ib_uverbs_file *file, [IB_USER_VERBS_CMD_MODIFY_SRQ] = ib_uverbs_modify_srq, [IB_USER_VERBS_CMD_QUERY_SRQ] = ib_uverbs_query_srq, [IB_USER_VERBS_CMD_DESTROY_SRQ] = ib_uverbs_destroy_srq, + [IB_USER_VERBS_CMD_GET_MAC] = ib_uverbs_get_mac }; static struct vfsmount *uverbs_event_mnt; diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index a7da9be..bde5b0d 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -94,6 +94,13 @@ rdma_node_get_transport(enum rdma_node_type node_type) } EXPORT_SYMBOL(rdma_node_get_transport); +enum ib_port_link_type ib_get_port_link_type(struct ib_device *device, u8 port_num) +{ + return device->get_port_link_type ? + device->get_port_link_type(device, port_num) : PORT_LINK_IB; +} +EXPORT_SYMBOL(ib_get_port_link_type); + /* Protection domains */ struct ib_pd *ib_alloc_pd(struct ib_device *device) @@ -904,3 +911,13 @@ int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) return qp->device->detach_mcast(qp, gid, lid); } EXPORT_SYMBOL(ib_detach_mcast); + +int ib_get_mac(struct ib_device *device, u8 port, u8 *gid, u8 *mac) +{ + if (!device->get_mac) + return -ENOSYS; + + return device->get_mac(device, port, gid, mac); +} +EXPORT_SYMBOL(ib_get_mac); + diff --git a/include/rdma/ib_user_verbs.h b/include/rdma/ib_user_verbs.h index a17f771..184203b 100644 --- a/include/rdma/ib_user_verbs.h +++ b/include/rdma/ib_user_verbs.h @@ -42,7 +42,7 @@ * Increment this value if any changes that break userspace ABI * compatibility are made. */ -#define IB_USER_VERBS_ABI_VERSION 6 +#define IB_USER_VERBS_ABI_VERSION 7 enum { IB_USER_VERBS_CMD_GET_CONTEXT, @@ -81,7 +81,8 @@ enum { IB_USER_VERBS_CMD_MODIFY_SRQ, IB_USER_VERBS_CMD_QUERY_SRQ, IB_USER_VERBS_CMD_DESTROY_SRQ, - IB_USER_VERBS_CMD_POST_SRQ_RECV + IB_USER_VERBS_CMD_POST_SRQ_RECV, + IB_USER_VERBS_CMD_GET_MAC }; /* @@ -205,7 +206,8 @@ struct ib_uverbs_query_port_resp { __u8 active_width; __u8 active_speed; __u8 phys_state; - __u8 reserved[3]; + __u8 link_type; + __u8 reserved[2]; }; struct ib_uverbs_alloc_pd { @@ -621,6 +623,19 @@ struct ib_uverbs_destroy_ah { __u32 ah_handle; }; +struct ib_uverbs_get_mac { + __u64 response; + __u32 pd_handle; + __u8 port; + __u8 reserved[3]; + __u8 gid[16]; +}; + +struct ib_uverbs_get_mac_resp { + __u8 mac[6]; + __u16 reserved; +}; + struct ib_uverbs_attach_mcast { __u8 gid[16]; __u32 qp_handle; diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index c179318..9b3d43b 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -72,6 +72,11 @@ enum rdma_transport_type { RDMA_TRANSPORT_IWARP }; +enum ib_port_link_type { + PORT_LINK_IB, + PORT_LINK_ETH +}; + enum rdma_transport_type rdma_node_get_transport(enum rdma_node_type node_type) __attribute_const__; @@ -298,6 +303,7 @@ struct ib_port_attr { u8 active_width; u8 active_speed; u8 phys_state; + enum ib_port_link_type link_type; }; enum ib_device_modify_flags { @@ -1003,6 +1009,8 @@ struct ib_device { int (*query_port)(struct ib_device *device, u8 port_num, struct ib_port_attr *port_attr); + enum ib_port_link_type (*get_port_link_type)(struct ib_device *device, + u8 port_num); int (*query_gid)(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); @@ -1130,6 +1138,8 @@ struct ib_device { struct ib_grh *in_grh, struct ib_mad *in_mad, struct ib_mad *out_mad); + int (*get_mac)(struct ib_device *device, u8 port, + u8 *gid, u8 *mac); struct ib_dma_mapping_ops *dma_ops; @@ -1213,6 +1223,9 @@ int ib_query_device(struct ib_device *device, int ib_query_port(struct ib_device *device, u8 port_num, struct ib_port_attr *port_attr); +enum ib_port_link_type ib_get_port_link_type(struct ib_device *device, + u8 port_num); + int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); @@ -2031,4 +2044,13 @@ int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); */ int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); +/** + * ib_get_mac - get the mac address for the specified gid + * @device: IB device used for traffic + * @port: port number used. + * @gid: gid to be resolved into mac + * @mac: mac of the port bearing this gid + */ +int ib_get_mac(struct ib_device *device, u8 port, u8 *gid, u8 *mac); + #endif /* IB_VERBS_H */ -- 1.6.3.3 From eli at mellanox.co.il Mon Jul 13 11:14:10 2009 From: eli at mellanox.co.il (Eli Cohen) Date: Mon, 13 Jul 2009 21:14:10 +0300 Subject: [ofa-general] [PATCH 2/8 v3] ib_core: RDMAoE support only QP1 Message-ID: <20090713181410.GA754@mtls03> Since RDMAoE is using Ethernet as its link layer, there is no need for QP0. QP1 is still needed since it handles communications between CM agents. This patch will create only QP1 for RDMAoE ports. Signed-off-by: Eli Cohen --- drivers/infiniband/core/agent.c | 12 ++++++--- drivers/infiniband/core/mad.c | 48 ++++++++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/drivers/infiniband/core/agent.c b/drivers/infiniband/core/agent.c index ae7c288..c3f2048 100644 --- a/drivers/infiniband/core/agent.c +++ b/drivers/infiniband/core/agent.c @@ -48,6 +48,8 @@ struct ib_agent_port_private { struct list_head port_list; struct ib_mad_agent *agent[2]; + struct ib_device *device; + u8 port_num; }; static DEFINE_SPINLOCK(ib_agent_port_list_lock); @@ -58,11 +60,10 @@ __ib_get_agent_port(struct ib_device *device, int port_num) { struct ib_agent_port_private *entry; - list_for_each_entry(entry, &ib_agent_port_list, port_list) { - if (entry->agent[0]->device == device && - entry->agent[0]->port_num == port_num) + list_for_each_entry(entry, &ib_agent_port_list, port_list) + if (entry->device == device && entry->port_num == port_num) return entry; - } + return NULL; } @@ -175,6 +176,9 @@ int ib_agent_port_open(struct ib_device *device, int port_num) goto error3; } + port_priv->device = device; + port_priv->port_num = port_num; + spin_lock_irqsave(&ib_agent_port_list_lock, flags); list_add_tail(&port_priv->port_list, &ib_agent_port_list); spin_unlock_irqrestore(&ib_agent_port_list_lock, flags); diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c index de922a0..3d5449f 100644 --- a/drivers/infiniband/core/mad.c +++ b/drivers/infiniband/core/mad.c @@ -199,6 +199,16 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device, unsigned long flags; u8 mgmt_class, vclass; + /* Validate device and port */ + port_priv = ib_get_mad_port(device, port_num); + if (!port_priv) { + ret = ERR_PTR(-ENODEV); + goto error1; + } + + if (!port_priv->qp_info[qp_type].qp) + return NULL; + /* Validate parameters */ qpn = get_spl_qp_index(qp_type); if (qpn == -1) @@ -260,13 +270,6 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device, goto error1; } - /* Validate device and port */ - port_priv = ib_get_mad_port(device, port_num); - if (!port_priv) { - ret = ERR_PTR(-ENODEV); - goto error1; - } - /* Allocate structures */ mad_agent_priv = kzalloc(sizeof *mad_agent_priv, GFP_KERNEL); if (!mad_agent_priv) { @@ -556,6 +559,9 @@ int ib_unregister_mad_agent(struct ib_mad_agent *mad_agent) struct ib_mad_agent_private *mad_agent_priv; struct ib_mad_snoop_private *mad_snoop_priv; + if (!mad_agent) + return 0; + /* If the TID is zero, the agent can only snoop. */ if (mad_agent->hi_tid) { mad_agent_priv = container_of(mad_agent, @@ -2602,6 +2608,9 @@ static void cleanup_recv_queue(struct ib_mad_qp_info *qp_info) struct ib_mad_private *recv; struct ib_mad_list_head *mad_list; + if (!qp_info->qp) + return; + while (!list_empty(&qp_info->recv_queue.list)) { mad_list = list_entry(qp_info->recv_queue.list.next, @@ -2643,6 +2652,9 @@ static int ib_mad_port_start(struct ib_mad_port_private *port_priv) for (i = 0; i < IB_MAD_QPS_CORE; i++) { qp = port_priv->qp_info[i].qp; + if (!qp) + continue; + /* * PKey index for QP1 is irrelevant but * one is needed for the Reset to Init transition @@ -2684,6 +2696,9 @@ static int ib_mad_port_start(struct ib_mad_port_private *port_priv) } for (i = 0; i < IB_MAD_QPS_CORE; i++) { + if (!port_priv->qp_info[i].qp) + continue; + ret = ib_mad_post_receive_mads(&port_priv->qp_info[i], NULL); if (ret) { printk(KERN_ERR PFX "Couldn't post receive WRs\n"); @@ -2762,6 +2777,9 @@ error: static void destroy_mad_qp(struct ib_mad_qp_info *qp_info) { + if (!qp_info->qp) + return; + ib_destroy_qp(qp_info->qp); kfree(qp_info->snoop_table); } @@ -2777,6 +2795,7 @@ static int ib_mad_port_open(struct ib_device *device, struct ib_mad_port_private *port_priv; unsigned long flags; char name[sizeof "ib_mad123"]; + int has_smi; /* Create new device info */ port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL); @@ -2793,6 +2812,10 @@ static int ib_mad_port_open(struct ib_device *device, init_mad_qp(port_priv, &port_priv->qp_info[1]); cq_size = (IB_MAD_QP_SEND_SIZE + IB_MAD_QP_RECV_SIZE) * 2; + has_smi = ib_get_port_link_type(device, port_num) == PORT_LINK_IB; + if (has_smi) + cq_size *= 2; + port_priv->cq = ib_create_cq(port_priv->device, ib_mad_thread_completion_handler, NULL, port_priv, cq_size, 0); @@ -2816,9 +2839,11 @@ static int ib_mad_port_open(struct ib_device *device, goto error5; } - ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI); - if (ret) - goto error6; + if (has_smi) { + ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI); + if (ret) + goto error6; + } ret = create_mad_qp(&port_priv->qp_info[1], IB_QPT_GSI); if (ret) goto error7; @@ -2852,7 +2877,8 @@ error9: error8: destroy_mad_qp(&port_priv->qp_info[1]); error7: - destroy_mad_qp(&port_priv->qp_info[0]); + if (has_smi) + destroy_mad_qp(&port_priv->qp_info[0]); error6: ib_dereg_mr(port_priv->mr); error5: -- 1.6.3.3 From eli at mellanox.co.il Mon Jul 13 11:14:57 2009 From: eli at mellanox.co.il (Eli Cohen) Date: Mon, 13 Jul 2009 21:14:57 +0300 Subject: [ofa-general] [PATCH 3/8 v3] ib_core: Add RDMAoE SA support Message-ID: <20090713181457.GA766@mtls03> Add support for resolving paths and joining multicast group for RDMAoE ports. For Eth links, path resolution will complete immediately but will call the callback from a workqueue context to avoid deadloks. Multicast joins are handled in nearly the same way as IB mulitcast joins are handled in multicast.c. However they are handled entirly at the host and no MADs are involved. This allows for a client to create groups and dictate the qkey to be used in that group. The code is put in rdmaoe_sa.c which handles both multicast joins/leaves and path resolution. The following files were added: drivers/infiniband/core/multicast.h drivers/infiniband/core/rdmaoe_sa.c include/rdma/rdmaoe_sa.h There are changes made in vers/infiniband/core/multicast.c, drivers/infiniband/core/sa_query.c and drivers/infiniband/core/sa.h to allow sharing of data structs. New API functions are added for RDMAoE and comsumenrs who want to use this API need to be changed. Signed-off-by: Eli Cohen --- drivers/infiniband/core/Makefile | 2 +- drivers/infiniband/core/multicast.c | 43 +-- drivers/infiniband/core/multicast.h | 79 +++ drivers/infiniband/core/rdmaoe_sa.c | 938 +++++++++++++++++++++++++++++++++++ drivers/infiniband/core/sa.h | 24 + drivers/infiniband/core/sa_query.c | 26 +- include/rdma/rdmaoe_sa.h | 66 +++ 7 files changed, 1113 insertions(+), 65 deletions(-) create mode 100644 drivers/infiniband/core/multicast.h create mode 100644 drivers/infiniband/core/rdmaoe_sa.c create mode 100644 include/rdma/rdmaoe_sa.h diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index cb1ab3e..96db705 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -2,7 +2,7 @@ infiniband-$(CONFIG_INFINIBAND_ADDR_TRANS) := ib_addr.o rdma_cm.o user_access-$(CONFIG_INFINIBAND_ADDR_TRANS) := rdma_ucm.o obj-$(CONFIG_INFINIBAND) += ib_core.o ib_mad.o ib_sa.o \ - ib_cm.o iw_cm.o $(infiniband-y) + ib_cm.o iw_cm.o rdmaoe_sa.o $(infiniband-y) obj-$(CONFIG_INFINIBAND_USER_MAD) += ib_umad.o obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o ib_ucm.o \ $(user_access-y) diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c index 107f170..727a55a 100644 --- a/drivers/infiniband/core/multicast.c +++ b/drivers/infiniband/core/multicast.c @@ -39,6 +39,7 @@ #include #include "sa.h" +#include "multicast.h" static void mcast_add_one(struct ib_device *device); static void mcast_remove_one(struct ib_device *device); @@ -72,52 +73,10 @@ struct mcast_device { struct mcast_port port[0]; }; -enum mcast_state { - MCAST_JOINING, - MCAST_MEMBER, - MCAST_ERROR, -}; - -enum mcast_group_state { - MCAST_IDLE, - MCAST_BUSY, - MCAST_GROUP_ERROR, - MCAST_PKEY_EVENT -}; - enum { MCAST_INVALID_PKEY_INDEX = 0xFFFF }; -struct mcast_member; - -struct mcast_group { - struct ib_sa_mcmember_rec rec; - struct rb_node node; - struct mcast_port *port; - spinlock_t lock; - struct work_struct work; - struct list_head pending_list; - struct list_head active_list; - struct mcast_member *last_join; - int members[3]; - atomic_t refcount; - enum mcast_group_state state; - struct ib_sa_query *query; - int query_id; - u16 pkey_index; -}; - -struct mcast_member { - struct ib_sa_multicast multicast; - struct ib_sa_client *client; - struct mcast_group *group; - struct list_head list; - enum mcast_state state; - atomic_t refcount; - struct completion comp; -}; - static void join_handler(int status, struct ib_sa_mcmember_rec *rec, void *context); static void leave_handler(int status, struct ib_sa_mcmember_rec *rec, diff --git a/drivers/infiniband/core/multicast.h b/drivers/infiniband/core/multicast.h new file mode 100644 index 0000000..17eb9fe --- /dev/null +++ b/drivers/infiniband/core/multicast.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2009 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MULTICAST_H +#define MULTICAST_H + +enum mcast_state { + MCAST_JOINING, + MCAST_MEMBER, + MCAST_ERROR, +}; + +enum mcast_group_state { + MCAST_IDLE, + MCAST_BUSY, + MCAST_GROUP_ERROR, + MCAST_PKEY_EVENT +}; + +struct mcast_member; + +struct mcast_group { + struct ib_sa_mcmember_rec rec; + struct rb_node node; + struct mcast_port *port; + spinlock_t lock; + struct work_struct work; + struct list_head pending_list; + struct list_head active_list; + struct mcast_member *last_join; + int members[3]; + atomic_t refcount; + enum mcast_group_state state; + struct ib_sa_query *query; + int query_id; + u16 pkey_index; +}; + +struct mcast_member { + struct ib_sa_multicast multicast; + struct ib_sa_client *client; + struct mcast_group *group; + struct list_head list; + enum mcast_state state; + atomic_t refcount; + struct completion comp; +}; + +#endif /* MULTICAST_H */ + diff --git a/drivers/infiniband/core/rdmaoe_sa.c b/drivers/infiniband/core/rdmaoe_sa.c new file mode 100644 index 0000000..3548a56 --- /dev/null +++ b/drivers/infiniband/core/rdmaoe_sa.c @@ -0,0 +1,938 @@ +/* + * Copyright (c) 2009 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "sa.h" +#include "multicast.h" + +MODULE_AUTHOR("Eli Cohen"); +MODULE_DESCRIPTION("RDMAoE SA emulation"); +MODULE_LICENSE("Dual BSD/GPL"); + +static void rdmaoe_sa_add_one(struct ib_device *device); +static void rdmaoe_sa_remove_one(struct ib_device *device); +static void rdmaoe_mcast_add_one(struct ib_device *device); +static void rdmaoe_mcast_remove_one(struct ib_device *device); + +static struct workqueue_struct *mcast_wq; +static union ib_gid mgid0; + +struct mcast_device; + +struct mcast_port { + struct mcast_device *dev; + spinlock_t lock; + struct rb_root table; + atomic_t refcount; + struct completion comp; + u8 port_num; + struct socket *sock; + int bound; + int ifidx; +}; + +struct mcast_device { + struct ib_device *device; + struct ib_event_handler event_handler; + int start_port; + int end_port; + struct mcast_port port[0]; +}; + +enum { + MCAST_INVALID_PKEY_INDEX = 0xFFFF +}; + +static void join_handler(int status, struct ib_sa_mcmember_rec *rec, + void *context); +static void leave_handler(int status, struct ib_sa_mcmember_rec *rec, + void *context); + +static struct ib_client sa_client = { + .name = "rdmaoe_sa", + .add = rdmaoe_sa_add_one, + .remove = rdmaoe_sa_remove_one +}; + +static struct ib_client mcast_client = { + .name = "rdmaoe_multicast", + .add = rdmaoe_mcast_add_one, + .remove = rdmaoe_mcast_remove_one +}; + +struct rdmaoe_sa_port { + u8 port_num; +}; + +struct rdmaoe_sa_device { + int start_port, end_port; + struct ib_event_handler event_handler; + struct rdmaoe_sa_port port[0]; +}; +struct eth_work { + struct work_struct work; + struct mcast_member *member; + struct ib_device *device; + u8 port_num; +}; + +static int start_igmp6(struct mcast_port *port) +{ + struct socket *sock; + int err; + + err = sock_create(AF_INET6, SOCK_DGRAM, 0, &sock); + if (err) + return err; + + port->sock = sock; + + return 0; +} + +static void stop_igmp6(struct mcast_port *port) +{ + if (port->sock) { + sock_release(port->sock); + port->sock = NULL; + } +} + +static int get_if_idx(union ib_gid *gid) +{ + struct net_device *dev; + struct in6_addr addr; + + memcpy(&addr, gid, sizeof *gid); + for_each_netdev(&init_net, dev) { + if (ipv6_chk_addr(&init_net, &addr, dev, 1)) + return dev->ifindex; + } + + return -1; +} + +static int attach_socket(struct mcast_port *port, union ib_gid *mgid) +{ + union ib_gid gid; + struct sockaddr_in6 addr = {0}; + int err; + struct ipv6_mreq mcast; + + if (!port->sock) + return -EINVAL; + + if (!port->bound) { + err = ib_query_gid(port->dev->device, port->port_num, 0, &gid); + if (err) + return err; + + memcpy(addr.sin6_addr.in6_u.u6_addr8, &gid, sizeof gid); + + addr.sin6_scope_id = get_if_idx(&gid); + if (addr.sin6_scope_id == -1) + return -EINVAL; + + addr.sin6_family = AF_INET6; + err = kernel_bind(port->sock, (struct sockaddr *)&addr, sizeof addr); + if (err) + return err; + else { + port->bound = 1; + port->ifidx = addr.sin6_scope_id; + } + } + mcast.ipv6mr_ifindex = port->ifidx; + memcpy(mcast.ipv6mr_multiaddr.s6_addr, mgid, sizeof *mgid); + err = kernel_setsockopt(port->sock, SOL_IPV6, + IPV6_ADD_MEMBERSHIP, (char *)&mcast, sizeof mcast); + + return err; +} + +static int dettach_socket(struct mcast_port *port, union ib_gid *mgid) +{ + struct ipv6_mreq mcast; + + if (!port->sock) + return -EINVAL; + + mcast.ipv6mr_ifindex = port->ifidx; + memcpy(mcast.ipv6mr_multiaddr.s6_addr, mgid, sizeof *mgid); + + return kernel_setsockopt(port->sock, SOL_IPV6, IPV6_DROP_MEMBERSHIP, + (char *)&mcast, sizeof mcast); +} + +static struct mcast_group *mcast_find(struct mcast_port *port, + union ib_gid *mgid) +{ + struct rb_node *node = port->table.rb_node; + struct mcast_group *group; + int ret; + + while (node) { + group = rb_entry(node, struct mcast_group, node); + ret = memcmp(mgid->raw, group->rec.mgid.raw, sizeof *mgid); + if (!ret) + return group; + + if (ret < 0) + node = node->rb_left; + else + node = node->rb_right; + } + return NULL; +} + +static struct mcast_group *mcast_insert(struct mcast_port *port, + struct mcast_group *group, + int allow_duplicates) +{ + struct rb_node **link = &port->table.rb_node; + struct rb_node *parent = NULL; + struct mcast_group *cur_group; + int ret; + + while (*link) { + parent = *link; + cur_group = rb_entry(parent, struct mcast_group, node); + + ret = memcmp(group->rec.mgid.raw, cur_group->rec.mgid.raw, + sizeof group->rec.mgid); + if (ret < 0) + link = &(*link)->rb_left; + else if (ret > 0) + link = &(*link)->rb_right; + else if (allow_duplicates) + link = &(*link)->rb_left; + else + return cur_group; + } + rb_link_node(&group->node, parent, link); + rb_insert_color(&group->node, &port->table); + return NULL; +} + +static void deref_port(struct mcast_port *port) +{ + if (atomic_dec_and_test(&port->refcount)) + complete(&port->comp); +} + +static void release_group(struct mcast_group *group) +{ + struct mcast_port *port = group->port; + unsigned long flags; + + spin_lock_irqsave(&port->lock, flags); + if (atomic_dec_and_test(&group->refcount)) { + rb_erase(&group->node, &port->table); + spin_unlock_irqrestore(&port->lock, flags); + kfree(group); + deref_port(port); + } else + spin_unlock_irqrestore(&port->lock, flags); +} + +static void deref_member(struct mcast_member *member) +{ + if (atomic_dec_and_test(&member->refcount)) + complete(&member->comp); +} + +/* + * If a multicast group has zero members left for a particular join state, but + * the group is still a member with the SA, we need to leave that join state. + * Determine which join states we still belong to, but that do not have any + * active members. + */ +static u8 get_leave_state(struct mcast_group *group) +{ + u8 leave_state = 0; + int i; + + for (i = 0; i < 3; i++) + if (!group->members[i]) + leave_state |= (0x1 << i); + + return leave_state & group->rec.join_state; +} + +static void queue_join(struct mcast_member *member) +{ + struct mcast_group *group = member->group; + unsigned long flags; + + spin_lock_irqsave(&group->lock, flags); + list_add_tail(&member->list, &group->pending_list); + if (group->state == MCAST_IDLE) { + group->state = MCAST_BUSY; + atomic_inc(&group->refcount); + queue_work(mcast_wq, &group->work); + } + spin_unlock_irqrestore(&group->lock, flags); +} + +/* + * A multicast group has three types of members: full member, non member, and + * send only member. We need to keep track of the number of members of each + * type based on their join state. Adjust the number of members the belong to + * the specified join states. + */ +static void adjust_membership(struct mcast_group *group, u8 join_state, int inc) +{ + int i; + + for (i = 0; i < 3; i++, join_state >>= 1) + if (join_state & 0x1) + group->members[i] += inc; +} + +static int check_selector(ib_sa_comp_mask comp_mask, + ib_sa_comp_mask selector_mask, + ib_sa_comp_mask value_mask, + u8 selector, u8 src_value, u8 dst_value) +{ + int err; + + if (!(comp_mask & selector_mask) || !(comp_mask & value_mask)) + return 0; + + switch (selector) { + case IB_SA_GT: + err = (src_value <= dst_value); + break; + case IB_SA_LT: + err = (src_value >= dst_value); + break; + case IB_SA_EQ: + err = (src_value != dst_value); + break; + default: + err = 0; + break; + } + + return err; +} + +static int cmp_rec(struct ib_sa_mcmember_rec *src, + struct ib_sa_mcmember_rec *dst, ib_sa_comp_mask comp_mask) +{ + /* MGID must already match */ + + if (comp_mask & IB_SA_MCMEMBER_REC_PORT_GID && + memcmp(&src->port_gid, &dst->port_gid, sizeof src->port_gid)) + return -EINVAL; + if (comp_mask & IB_SA_MCMEMBER_REC_QKEY && src->qkey != dst->qkey) + return -EINVAL; + if (comp_mask & IB_SA_MCMEMBER_REC_MLID && src->mlid != dst->mlid) + return -EINVAL; + if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_MTU_SELECTOR, + IB_SA_MCMEMBER_REC_MTU, dst->mtu_selector, + src->mtu, dst->mtu)) + return -EINVAL; + if (comp_mask & IB_SA_MCMEMBER_REC_TRAFFIC_CLASS && + src->traffic_class != dst->traffic_class) + return -EINVAL; + if (comp_mask & IB_SA_MCMEMBER_REC_PKEY && src->pkey != dst->pkey) + return -EINVAL; + if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_RATE_SELECTOR, + IB_SA_MCMEMBER_REC_RATE, dst->rate_selector, + src->rate, dst->rate)) + return -EINVAL; + if (check_selector(comp_mask, + IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR, + IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME, + dst->packet_life_time_selector, + src->packet_life_time, dst->packet_life_time)) + return -EINVAL; + if (comp_mask & IB_SA_MCMEMBER_REC_SL && src->sl != dst->sl) + return -EINVAL; + if (comp_mask & IB_SA_MCMEMBER_REC_FLOW_LABEL && + src->flow_label != dst->flow_label) + return -EINVAL; + if (comp_mask & IB_SA_MCMEMBER_REC_HOP_LIMIT && + src->hop_limit != dst->hop_limit) + return -EINVAL; + if (comp_mask & IB_SA_MCMEMBER_REC_SCOPE && src->scope != dst->scope) + return -EINVAL; + + /* join_state checked separately, proxy_join ignored */ + + return 0; +} + +static void join_group(struct mcast_group *group, struct mcast_member *member, + u8 join_state) +{ + member->state = MCAST_MEMBER; + adjust_membership(group, join_state, 1); + group->rec.join_state |= join_state; + member->multicast.rec = group->rec; + member->multicast.rec.join_state = join_state; + list_move(&member->list, &group->active_list); +} + +static int fail_join(struct mcast_group *group, struct mcast_member *member, + int status) +{ + spin_lock_irq(&group->lock); + list_del_init(&member->list); + spin_unlock_irq(&group->lock); + return member->multicast.callback(status, &member->multicast); +} + +static void process_group_error(struct mcast_group *group) +{ + struct mcast_member *member; + int ret = 0; + u16 pkey_index; + + if (group->state == MCAST_PKEY_EVENT) + ret = ib_find_pkey(group->port->dev->device, + group->port->port_num, + be16_to_cpu(group->rec.pkey), &pkey_index); + + spin_lock_irq(&group->lock); + if (group->state == MCAST_PKEY_EVENT && !ret && + group->pkey_index == pkey_index) + goto out; + + while (!list_empty(&group->active_list)) { + member = list_entry(group->active_list.next, + struct mcast_member, list); + atomic_inc(&member->refcount); + list_del_init(&member->list); + adjust_membership(group, member->multicast.rec.join_state, -1); + member->state = MCAST_ERROR; + spin_unlock_irq(&group->lock); + + ret = member->multicast.callback(-ENETRESET, + &member->multicast); + deref_member(member); + if (ret) + ib_sa_free_multicast(&member->multicast); + spin_lock_irq(&group->lock); + } + + group->rec.join_state = 0; +out: + group->state = MCAST_BUSY; + spin_unlock_irq(&group->lock); +} + +static int send_join(struct mcast_group *group, struct mcast_member *member) +{ + group->last_join = member; + member->multicast.rec.pkey = cpu_to_be16(0xffff); + join_handler(0, &member->multicast.rec, group); + return 0; +} + +static int send_leave(struct mcast_group *group, u8 leave_state) +{ + struct ib_sa_mcmember_rec rec; + + rec = group->rec; + rec.join_state = leave_state; + leave_handler(0, &rec, group); + return 0; +} + +static void mcast_work_handler(struct work_struct *work) +{ + struct mcast_group *group; + struct mcast_member *member; + struct ib_sa_multicast *multicast; + int status, ret; + u8 join_state; + + group = container_of(work, typeof(*group), work); +retest: + spin_lock_irq(&group->lock); + while (!list_empty(&group->pending_list) || + (group->state != MCAST_BUSY)) { + + if (group->state != MCAST_BUSY) { + spin_unlock_irq(&group->lock); + process_group_error(group); + goto retest; + } + + member = list_entry(group->pending_list.next, + struct mcast_member, list); + multicast = &member->multicast; + join_state = multicast->rec.join_state; + atomic_inc(&member->refcount); + + if (join_state == (group->rec.join_state & join_state)) { + status = cmp_rec(&group->rec, &multicast->rec, + multicast->comp_mask); + if (!status) + join_group(group, member, join_state); + else + list_del_init(&member->list); + spin_unlock_irq(&group->lock); + ret = multicast->callback(status, multicast); + } else { + spin_unlock_irq(&group->lock); + status = send_join(group, member); + if (!status) { + deref_member(member); + return; + } + ret = fail_join(group, member, status); + } + + deref_member(member); + if (ret) + ib_sa_free_multicast(&member->multicast); + spin_lock_irq(&group->lock); + } + + join_state = get_leave_state(group); + if (join_state) { + group->rec.join_state &= ~join_state; + spin_unlock_irq(&group->lock); + dettach_socket(group->port, &group->rec.mgid); + if (send_leave(group, join_state)) + goto retest; + } else { + group->state = MCAST_IDLE; + spin_unlock_irq(&group->lock); + release_group(group); + } +} + +static void join_handler(int status, struct ib_sa_mcmember_rec *rec, + void *context) +{ + struct mcast_group *group = context; + u16 pkey_index = MCAST_INVALID_PKEY_INDEX; + + ib_find_pkey(group->port->dev->device, group->port->port_num, + be16_to_cpu(rec->pkey), &pkey_index); + + spin_lock_irq(&group->port->lock); + group->rec = *rec; + if (group->state == MCAST_BUSY && + group->pkey_index == MCAST_INVALID_PKEY_INDEX) + group->pkey_index = pkey_index; + if (!memcmp(&mgid0, &group->rec.mgid, sizeof mgid0)) { + rb_erase(&group->node, &group->port->table); + mcast_insert(group->port, group, 1); + } + spin_unlock_irq(&group->port->lock); + mcast_work_handler(&group->work); +} + +static void leave_handler(int status, struct ib_sa_mcmember_rec *rec, + void *context) +{ + struct mcast_group *group = context; + + mcast_work_handler(&group->work); +} + +static struct mcast_group *acquire_group(struct mcast_port *port, + union ib_gid *mgid, gfp_t gfp_mask) +{ + struct mcast_group *group, *cur_group; + unsigned long flags; + int is_mgid0; + int need_attach = 0; + + is_mgid0 = !memcmp(&mgid0, mgid, sizeof mgid0); + if (!is_mgid0) { + spin_lock_irqsave(&port->lock, flags); + group = mcast_find(port, mgid); + if (group) + goto found; + spin_unlock_irqrestore(&port->lock, flags); + } + + group = kzalloc(sizeof *group, gfp_mask); + if (!group) + return NULL; + + group->port = port; + group->rec.mgid = *mgid; + group->pkey_index = MCAST_INVALID_PKEY_INDEX; + INIT_LIST_HEAD(&group->pending_list); + INIT_LIST_HEAD(&group->active_list); + INIT_WORK(&group->work, mcast_work_handler); + spin_lock_init(&group->lock); + + spin_lock_irqsave(&port->lock, flags); + cur_group = mcast_insert(port, group, is_mgid0); + if (cur_group) { + kfree(group); + group = cur_group; + } else + atomic_inc(&port->refcount); + + if (!is_mgid0) + need_attach = 1; + +found: + atomic_inc(&group->refcount); + spin_unlock_irqrestore(&port->lock, flags); + if (need_attach && attach_socket(port, mgid)) { + release_group(group); + group = NULL; + } + + return group; +} + +struct ib_sa_multicast * +rdmaoe_sa_join_multicast(struct ib_sa_client *client, + struct ib_device *device, u8 port_num, + struct ib_sa_mcmember_rec *rec, + ib_sa_comp_mask comp_mask, gfp_t gfp_mask, + int (*callback)(int status, + struct ib_sa_multicast *multicast), + void *context) +{ + struct mcast_device *dev; + struct mcast_member *member; + struct ib_sa_multicast *multicast; + int ret; + + dev = ib_get_client_data(device, &mcast_client); + if (!dev) + return ERR_PTR(-ENODEV); + + member = kmalloc(sizeof *member, gfp_mask); + if (!member) + return ERR_PTR(-ENOMEM); + + ib_sa_client_get(client); + member->client = client; + member->multicast.rec = *rec; + member->multicast.comp_mask = comp_mask; + member->multicast.callback = callback; + member->multicast.context = context; + init_completion(&member->comp); + atomic_set(&member->refcount, 1); + member->state = MCAST_JOINING; + + member->group = acquire_group(&dev->port[port_num - dev->start_port], + &rec->mgid, gfp_mask); + if (!member->group) { + ret = -ENOMEM; + goto err; + } + + /* + * The user will get the multicast structure in their callback. They + * could then free the multicast structure before we can return from + * this routine. So we save the pointer to return before queuing + * any callback. + */ + multicast = &member->multicast; + queue_join(member); + return multicast; + +err: + ib_sa_client_put(client); + kfree(member); + return ERR_PTR(ret); +} +EXPORT_SYMBOL(rdmaoe_sa_join_multicast); + +void rdmaoe_sa_free_multicast(struct ib_sa_multicast *multicast) +{ + struct mcast_member *member; + struct mcast_group *group; + + member = container_of(multicast, struct mcast_member, multicast); + group = member->group; + spin_lock_irq(&group->lock); + if (member->state == MCAST_MEMBER) + adjust_membership(group, multicast->rec.join_state, -1); + + list_del_init(&member->list); + + if (group->state == MCAST_IDLE) { + group->state = MCAST_BUSY; + spin_unlock_irq(&group->lock); + /* Continue to hold reference on group until callback */ + queue_work(mcast_wq, &group->work); + } else { + spin_unlock_irq(&group->lock); + release_group(group); + } + + deref_member(member); + wait_for_completion(&member->comp); + ib_sa_client_put(member->client); +} +EXPORT_SYMBOL(rdmaoe_sa_free_multicast); + +struct eth_work_container { + struct work_struct work; + struct ib_sa_path_query *query; +}; + +static void resolve_callback(struct work_struct *work) +{ + struct eth_work_container *eth = + container_of(work, struct eth_work_container, work); + struct ib_sa_path_query *query = eth->query; + struct ib_sa_path_rec res = {}; + + res.dgid = query->dgid; + res.sgid = query->sgid; + res.hop_limit = 2; /* TBD fix this */ + res.mtu = IB_MTU_1024; /* TBD fix me */ + query->callback(0, &res, query->context); + + ib_sa_client_put(query->sa_query.client); +} + +int rdmaoe_sa_path_rec_get(struct ib_sa_client *client, + struct ib_device *device, u8 port_num, + struct ib_sa_path_rec *rec, + ib_sa_comp_mask comp_mask, + int timeout_ms, gfp_t gfp_mask, + void (*callback)(int status, + struct ib_sa_path_rec *resp, + void *context), + void *context, + struct ib_sa_query **sa_query) +{ + struct ib_sa_path_query *query; + struct eth_work_container *eth; + + query = kzalloc(sizeof *query, gfp_mask); + if (!query) + return -ENOMEM; + + eth = kzalloc(sizeof *eth, gfp_mask); + if (!eth) { + kfree(query); + return -ENOMEM; + } + + ib_sa_client_get(client); + query->sa_query.client = client; + query->callback = callback; + query->context = context; + query->dgid = rec->dgid; + query->sgid = rec->sgid; + + *sa_query = &query->sa_query; + + eth->query = query; + INIT_WORK(ð->work, resolve_callback); + schedule_work(ð->work); + + return 0; +} +EXPORT_SYMBOL(rdmaoe_sa_path_rec_get); + +static void rdmaoe_sa_event(struct ib_event_handler *handler, struct ib_event *event) +{ + printk(KERN_NOTICE "%s: got event %d\n", __func__, event->event); +} + +static void rdmaoe_sa_add_one(struct ib_device *device) +{ + struct rdmaoe_sa_device *sa_dev; + int s, e, i; + + if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB) + return; + + if (device->node_type == RDMA_NODE_IB_SWITCH) + return; + + s = 1; + e = device->phys_port_cnt; + + sa_dev = kmalloc(sizeof *sa_dev + + (e - s + 1) * sizeof *sa_dev, GFP_KERNEL); + if (!sa_dev) + return; + + sa_dev->start_port = s; + sa_dev->end_port = e; + + for (i = 0; i <= e - s; ++i) + sa_dev->port[i].port_num = i + s; + + ib_set_client_data(device, &sa_client, sa_dev); + + /* + * We register our event handler after everything is set up, + * and then update our cached info after the event handler is + * registered to avoid any problems if a port changes state + * during our initialization. + */ + + INIT_IB_EVENT_HANDLER(&sa_dev->event_handler, device, rdmaoe_sa_event); + if (ib_register_event_handler(&sa_dev->event_handler)) + goto err; + + return; + +err: + kfree(sa_dev); + + return; +} + +static void rdmaoe_sa_remove_one(struct ib_device *device) +{ + struct rdmaoe_sa_device *sa_dev = ib_get_client_data(device, &sa_client); + + if (!sa_dev) + return; + + ib_unregister_event_handler(&sa_dev->event_handler); + + kfree(sa_dev); +} + +static void mcast_event_handler(struct ib_event_handler *handler, + struct ib_event *event) +{ +} + +static void rdmaoe_mcast_add_one(struct ib_device *device) +{ + struct mcast_device *dev; + struct mcast_port *port; + int i; + + if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB) + return; + + if (device->node_type == RDMA_NODE_IB_SWITCH) + return; + + dev = kzalloc(sizeof *dev + device->phys_port_cnt * sizeof *port, + GFP_KERNEL); + if (!dev) + return; + + dev->start_port = 1; + dev->end_port = device->phys_port_cnt; + + for (i = 0; i <= dev->end_port - dev->start_port; i++) { + port = &dev->port[i]; + port->dev = dev; + port->port_num = dev->start_port + i; + spin_lock_init(&port->lock); + port->table = RB_ROOT; + init_completion(&port->comp); + atomic_set(&port->refcount, 1); + start_igmp6(port); + } + + dev->device = device; + ib_set_client_data(device, &mcast_client, dev); + + INIT_IB_EVENT_HANDLER(&dev->event_handler, device, mcast_event_handler); + ib_register_event_handler(&dev->event_handler); +} + +static void rdmaoe_mcast_remove_one(struct ib_device *device) +{ + struct mcast_device *dev; + struct mcast_port *port; + int i; + + dev = ib_get_client_data(device, &mcast_client); + if (!dev) + return; + + ib_unregister_event_handler(&dev->event_handler); + flush_workqueue(mcast_wq); + + for (i = 0; i <= dev->end_port - dev->start_port; i++) { + port = &dev->port[i]; + deref_port(port); + wait_for_completion(&port->comp); + stop_igmp6(port); + } + + kfree(dev); +} + +static int __init rdmaoe_sa_init(void) +{ + int err; + + mcast_wq = create_singlethread_workqueue("rdmaoe_mcast"); + if (!mcast_wq) + return -ENOMEM; + + err = ib_register_client(&sa_client); + if (err) { + printk(KERN_ERR "Couldn't register rdmaoe_sa client\n"); + goto reg_fail; + } + + err = ib_register_client(&mcast_client); + if (err) + goto reg_mcast_fail; + + + return 0; + +reg_mcast_fail: + ib_unregister_client(&sa_client); +reg_fail: + destroy_workqueue(mcast_wq); + return err; +} + +static void __exit rdmaoe_sa_cleanup(void) +{ + ib_unregister_client(&mcast_client); + ib_unregister_client(&sa_client); + destroy_workqueue(mcast_wq); +} + +module_init(rdmaoe_sa_init); +module_exit(rdmaoe_sa_cleanup); diff --git a/drivers/infiniband/core/sa.h b/drivers/infiniband/core/sa.h index b1d4bbf..da780cf 100644 --- a/drivers/infiniband/core/sa.h +++ b/drivers/infiniband/core/sa.h @@ -37,6 +37,30 @@ #include +struct ib_sa_query { + void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *); + void (*release)(struct ib_sa_query *); + struct ib_sa_client *client; + struct ib_sa_port *port; + struct ib_mad_send_buf *mad_buf; + struct ib_sa_sm_ah *sm_ah; + int id; +}; + +struct ib_sa_service_query { + void (*callback)(int, struct ib_sa_service_rec *, void *); + void *context; + struct ib_sa_query sa_query; +}; + +struct ib_sa_path_query { + void (*callback)(int, struct ib_sa_path_rec *, void *); + void *context; + struct ib_sa_query sa_query; + union ib_gid dgid; + union ib_gid sgid; +}; + static inline void ib_sa_client_get(struct ib_sa_client *client) { atomic_inc(&client->users); diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c index 1865049..0625e10 100644 --- a/drivers/infiniband/core/sa_query.c +++ b/drivers/infiniband/core/sa_query.c @@ -46,6 +46,7 @@ #include #include #include "sa.h" +#include "multicast.h" MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand subnet administration query support"); @@ -72,28 +73,6 @@ struct ib_sa_device { struct ib_sa_port port[0]; }; -struct ib_sa_query { - void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *); - void (*release)(struct ib_sa_query *); - struct ib_sa_client *client; - struct ib_sa_port *port; - struct ib_mad_send_buf *mad_buf; - struct ib_sa_sm_ah *sm_ah; - int id; -}; - -struct ib_sa_service_query { - void (*callback)(int, struct ib_sa_service_rec *, void *); - void *context; - struct ib_sa_query sa_query; -}; - -struct ib_sa_path_query { - void (*callback)(int, struct ib_sa_path_rec *, void *); - void *context; - struct ib_sa_query sa_query; -}; - struct ib_sa_mcmember_query { void (*callback)(int, struct ib_sa_mcmember_rec *, void *); void *context; @@ -363,6 +342,9 @@ static void update_sm_ah(struct work_struct *work) struct ib_port_attr port_attr; struct ib_ah_attr ah_attr; + if (ib_get_port_link_type(port->agent->device, port->port_num) != PORT_LINK_IB) + return; + if (ib_query_port(port->agent->device, port->port_num, &port_attr)) { printk(KERN_WARNING "Couldn't query port\n"); return; diff --git a/include/rdma/rdmaoe_sa.h b/include/rdma/rdmaoe_sa.h new file mode 100644 index 0000000..2a93235 --- /dev/null +++ b/include/rdma/rdmaoe_sa.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2009 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef RDMAOE_SA_H +#define RDMAOE_SA_H + +#include + +struct rdmaoe_sa_client { + atomic_t users; + struct completion comp; +}; + +struct ib_sa_multicast * +rdmaoe_sa_join_multicast(struct ib_sa_client *client, + struct ib_device *device, u8 port_num, + struct ib_sa_mcmember_rec *rec, + ib_sa_comp_mask comp_mask, gfp_t gfp_mask, + int (*callback)(int status, + struct ib_sa_multicast *multicast), + void *context); + +void rdmaoe_sa_free_multicast(struct ib_sa_multicast *multicast); + +int rdmaoe_sa_path_rec_get(struct ib_sa_client *client, + struct ib_device *device, u8 port_num, + struct ib_sa_path_rec *rec, + ib_sa_comp_mask comp_mask, + int timeout_ms, gfp_t gfp_mask, + void (*callback)(int status, + struct ib_sa_path_rec *resp, + void *context), + void *context, + struct ib_sa_query **sa_query); + +#endif /* RDMAOE_SA_H */ + -- 1.6.3.3 From eli at mellanox.co.il Mon Jul 13 11:15:39 2009 From: eli at mellanox.co.il (Eli Cohen) Date: Mon, 13 Jul 2009 21:15:39 +0300 Subject: [ofa-general] [PATCH 4/8 v3] ib_core: CMA device binding Message-ID: <20090713181539.GA778@mtls03> Add support for RDMAoE device binding and IP --> GID resolution. Modify calls to the SA such that the link type is veirifed first and the appropriate call is made to either IB SA or RDMAoE SA. Signed-off-by: Eli Cohen --- drivers/infiniband/core/addr.c | 20 ++++--- drivers/infiniband/core/cma.c | 124 +++++++++++++++++++++++++++++++++------- include/rdma/ib_addr.h | 53 +++++++++++++++++ 3 files changed, 166 insertions(+), 31 deletions(-) diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index ce511d8..440e613 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -64,7 +64,7 @@ struct addr_req { static void process_req(struct work_struct *work); -static DEFINE_MUTEX(lock); +static DEFINE_SPINLOCK(lock); static LIST_HEAD(req_list); static DECLARE_DELAYED_WORK(work, process_req); static struct workqueue_struct *addr_wq; @@ -163,7 +163,7 @@ static void queue_req(struct addr_req *req) { struct addr_req *temp_req; - mutex_lock(&lock); + spin_lock(&lock); list_for_each_entry_reverse(temp_req, &req_list, list) { if (time_after_eq(req->timeout, temp_req->timeout)) break; @@ -173,7 +173,7 @@ static void queue_req(struct addr_req *req) if (req_list.next == &req->list) set_timeout(req->timeout); - mutex_unlock(&lock); + spin_unlock(&lock); } static void addr_send_arp(struct sockaddr *dst_in) @@ -207,7 +207,9 @@ static void addr_send_arp(struct sockaddr *dst_in) if (!dst) return; - neigh_event_send(dst->neighbour, NULL); + if (dst->neighbour) + neigh_event_send(dst->neighbour, NULL); + dst_release(dst); break; } @@ -322,7 +324,7 @@ static void process_req(struct work_struct *work) INIT_LIST_HEAD(&done_list); - mutex_lock(&lock); + spin_lock(&lock); list_for_each_entry_safe(req, temp_req, &req_list, list) { if (req->status == -ENODATA) { src_in = (struct sockaddr *) &req->src_addr; @@ -341,7 +343,7 @@ static void process_req(struct work_struct *work) req = list_entry(req_list.next, struct addr_req, list); set_timeout(req->timeout); } - mutex_unlock(&lock); + spin_unlock(&lock); list_for_each_entry_safe(req, temp_req, &done_list, list) { list_del(&req->list); @@ -439,7 +441,7 @@ int rdma_resolve_ip(struct rdma_addr_client *client, struct addr_req *req; int ret = 0; - req = kzalloc(sizeof *req, GFP_KERNEL); + req = kzalloc(sizeof *req, GFP_ATOMIC); if (!req) return -ENOMEM; @@ -483,7 +485,7 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr) { struct addr_req *req, *temp_req; - mutex_lock(&lock); + spin_lock(&lock); list_for_each_entry_safe(req, temp_req, &req_list, list) { if (req->addr == addr) { req->status = -ECANCELED; @@ -493,7 +495,7 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr) break; } } - mutex_unlock(&lock); + spin_unlock(&lock); } EXPORT_SYMBOL(rdma_addr_cancel); diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 851de83..6cf0f1b 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -49,6 +49,7 @@ #include #include #include +#include #include MODULE_AUTHOR("Sean Hefty"); @@ -69,6 +70,8 @@ static struct ib_client cma_client = { }; static struct ib_sa_client sa_client; +static struct ib_sa_client rdmaoe_sa_client; +static struct ib_sa_client rdmaoe_mcast_client; static struct rdma_addr_client addr_client; static LIST_HEAD(dev_list); static LIST_HEAD(listen_any_list); @@ -327,22 +330,30 @@ static int cma_acquire_dev(struct rdma_id_private *id_priv) { struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; struct cma_device *cma_dev; - union ib_gid gid; + union ib_gid gid, llgid, *tmp; int ret = -ENODEV; switch (rdma_node_get_transport(dev_addr->dev_type)) { case RDMA_TRANSPORT_IB: ib_addr_get_sgid(dev_addr, &gid); + tmp = &gid; break; case RDMA_TRANSPORT_IWARP: iw_addr_get_sgid(dev_addr, &gid); + rdma_mac_to_ll_addr(dev_addr->src_dev_addr, &llgid); break; default: return -ENODEV; } list_for_each_entry(cma_dev, &dev_list, list) { - ret = ib_find_cached_gid(cma_dev->device, &gid, + if (ib_get_port_link_type(cma_dev->device, id_priv->id.port_num) + == PORT_LINK_ETH && + rdma_node_get_transport(dev_addr->dev_type) == + RDMA_TRANSPORT_IWARP) + tmp = &llgid; + + ret = ib_find_cached_gid(cma_dev->device, tmp, &id_priv->id.port_num, NULL); if (!ret) { cma_attach_to_dev(id_priv, cma_dev); @@ -821,12 +832,18 @@ static void cma_release_port(struct rdma_id_private *id_priv) static void cma_leave_mc_groups(struct rdma_id_private *id_priv) { struct cma_multicast *mc; + enum ib_port_link_type lt; + lt = ib_get_port_link_type(id_priv->id.device, id_priv->id.port_num); while (!list_empty(&id_priv->mc_list)) { mc = container_of(id_priv->mc_list.next, struct cma_multicast, list); list_del(&mc->list); - ib_sa_free_multicast(mc->multicast.ib); + if (lt == PORT_LINK_IB) + ib_sa_free_multicast(mc->multicast.ib); + else + rdmaoe_sa_free_multicast(mc->multicast.ib); + kfree(mc); } } @@ -1032,7 +1049,19 @@ static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id, if (rt->num_paths == 2) rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; - ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); + + switch (ib_get_port_link_type(listen_id->device, + ib_event->param.req_rcvd.port)) { + case PORT_LINK_IB: + ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); + break; + case PORT_LINK_ETH: + iw_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); + break; + default: + printk(KERN_ERR "RDMA CMA: unknown rdma port link type\n"); + goto destroy_id; + } ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr, &id->route.addr.dev_addr); if (ret) @@ -1563,10 +1592,19 @@ static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms, struct ib_sa_path_rec path_rec; ib_sa_comp_mask comp_mask; struct sockaddr_in6 *sin6; + enum ib_port_link_type lt; + lt = ib_get_port_link_type(id_priv->id.device, id_priv->id.port_num); memset(&path_rec, 0, sizeof path_rec); - ib_addr_get_sgid(&addr->dev_addr, &path_rec.sgid); - ib_addr_get_dgid(&addr->dev_addr, &path_rec.dgid); + if (lt == PORT_LINK_IB || + (ib_addr_hw_addr_is_gid(addr->dev_addr.src_dev_addr) && + ib_addr_hw_addr_is_gid(addr->dev_addr.dst_dev_addr))) { + ib_addr_get_sgid(&addr->dev_addr, &path_rec.sgid); + ib_addr_get_dgid(&addr->dev_addr, &path_rec.dgid); + } else { + rdma_mac_to_ll_addr(addr->dev_addr.src_dev_addr, &path_rec.sgid); + rdma_mac_to_ll_addr(addr->dev_addr.dst_dev_addr, &path_rec.dgid); + } path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr)); path_rec.numb_path = 1; path_rec.reversible = 1; @@ -1586,11 +1624,18 @@ static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms, comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; } - id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, - id_priv->id.port_num, &path_rec, - comp_mask, timeout_ms, - GFP_KERNEL, cma_query_handler, - work, &id_priv->query); + if (lt == PORT_LINK_IB) + id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, + id_priv->id.port_num, &path_rec, + comp_mask, timeout_ms, + GFP_KERNEL, cma_query_handler, + work, &id_priv->query); + else + id_priv->query_id = rdmaoe_sa_path_rec_get(&rdmaoe_sa_client, id_priv->id.device, + id_priv->id.port_num, &path_rec, + comp_mask, timeout_ms, + GFP_KERNEL, cma_query_handler, + work, &id_priv->query); return (id_priv->query_id < 0) ? id_priv->query_id : 0; } @@ -2699,6 +2744,14 @@ static void cma_set_mgid(struct rdma_id_private *id_priv, } } +static union ib_gid rdmaoe_mgid6 = { + .raw = {0xff, 0x12, 0x60, 0x1b} +}; + +static union ib_gid rdmaoe_mgid4 = { + .raw = {0xff, 0x12, 0x40, 0x1b} +}; + static int cma_join_ib_multicast(struct rdma_id_private *id_priv, struct cma_multicast *mc) { @@ -2706,17 +2759,27 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv, struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; ib_sa_comp_mask comp_mask; int ret; + enum ib_port_link_type lt; - ib_addr_get_mgid(dev_addr, &rec.mgid); - ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, - &rec.mgid, &rec); - if (ret) - return ret; + lt = ib_get_port_link_type(id_priv->id.device, id_priv->id.port_num); + if (lt == PORT_LINK_IB) { + ib_addr_get_mgid(dev_addr, &rec.mgid); + ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, + &rec.mgid, &rec); + if (ret) + return ret; + + } else + ((struct sockaddr *)&mc->addr)->sa_family == AF_INET6 ? + rec.mgid = rdmaoe_mgid6 : rdmaoe_mgid4; cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid); if (id_priv->id.ps == RDMA_PS_UDP) rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); - ib_addr_get_sgid(dev_addr, &rec.port_gid); + if (lt == PORT_LINK_ETH) + rdma_mac_to_ll_addr(dev_addr->src_dev_addr, &rec.port_gid); + else + ib_addr_get_sgid(dev_addr, &rec.port_gid); rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); rec.join_state = 1; @@ -2730,10 +2793,16 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv, comp_mask |= IB_SA_MCMEMBER_REC_RATE | IB_SA_MCMEMBER_REC_RATE_SELECTOR; - mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device, - id_priv->id.port_num, &rec, - comp_mask, GFP_KERNEL, - cma_ib_mc_handler, mc); + mc->multicast.ib = lt == PORT_LINK_IB ? + ib_sa_join_multicast(&sa_client, id_priv->id.device, + id_priv->id.port_num, &rec, + comp_mask, GFP_KERNEL, + cma_ib_mc_handler, mc) : + rdmaoe_sa_join_multicast(&rdmaoe_mcast_client, id_priv->id.device, + id_priv->id.port_num, &rec, + comp_mask, GFP_KERNEL, + cma_ib_mc_handler, mc); + if (IS_ERR(mc->multicast.ib)) return PTR_ERR(mc->multicast.ib); @@ -2787,8 +2856,10 @@ void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) { struct rdma_id_private *id_priv; struct cma_multicast *mc; + enum ib_port_link_type lt; id_priv = container_of(id, struct rdma_id_private, id); + lt = ib_get_port_link_type(id_priv->id.device, id_priv->id.port_num); spin_lock_irq(&id_priv->lock); list_for_each_entry(mc, &id_priv->mc_list, list) { if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) { @@ -2799,7 +2870,10 @@ void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) ib_detach_mcast(id->qp, &mc->multicast.ib->rec.mgid, mc->multicast.ib->rec.mlid); - ib_sa_free_multicast(mc->multicast.ib); + if (lt == PORT_LINK_IB) + ib_sa_free_multicast(mc->multicast.ib); + else + rdmaoe_sa_free_multicast(mc->multicast.ib); kfree(mc); return; } @@ -2974,6 +3048,8 @@ static int cma_init(void) return -ENOMEM; ib_sa_register_client(&sa_client); + ib_sa_register_client(&rdmaoe_sa_client); + ib_sa_register_client(&rdmaoe_mcast_client); rdma_addr_register_client(&addr_client); register_netdevice_notifier(&cma_nb); @@ -2985,6 +3061,8 @@ static int cma_init(void) err: unregister_netdevice_notifier(&cma_nb); rdma_addr_unregister_client(&addr_client); + ib_sa_unregister_client(&rdmaoe_mcast_client); + ib_sa_unregister_client(&rdmaoe_sa_client); ib_sa_unregister_client(&sa_client); destroy_workqueue(cma_wq); return ret; @@ -2995,6 +3073,8 @@ static void cma_cleanup(void) ib_unregister_client(&cma_client); unregister_netdevice_notifier(&cma_nb); rdma_addr_unregister_client(&addr_client); + ib_sa_unregister_client(&rdmaoe_mcast_client); + ib_sa_unregister_client(&rdmaoe_sa_client); ib_sa_unregister_client(&sa_client); destroy_workqueue(cma_wq); idr_destroy(&sdp_ps); diff --git a/include/rdma/ib_addr.h b/include/rdma/ib_addr.h index 483057b..7438b89 100644 --- a/include/rdma/ib_addr.h +++ b/include/rdma/ib_addr.h @@ -151,10 +151,63 @@ static inline void iw_addr_get_sgid(struct rdma_dev_addr *dev_addr, memcpy(gid, dev_addr->src_dev_addr, sizeof *gid); } +static inline void iw_addr_set_dgid(struct rdma_dev_addr *dev_addr, + union ib_gid *gid) +{ + memcpy(dev_addr->dst_dev_addr, gid, sizeof *gid); +} + static inline void iw_addr_get_dgid(struct rdma_dev_addr *dev_addr, union ib_gid *gid) { memcpy(gid, dev_addr->dst_dev_addr, sizeof *gid); } +static inline int rdma_link_local_addr(struct in6_addr *addr) +{ + if (addr->s6_addr32[0] == cpu_to_be32(0xfe800000) && + addr->s6_addr32[1] == 0) + return 1; + else + return 0; +} + +static inline void rdma_mac_to_ll_addr(u8 *mac, union ib_gid *gid) +{ + memset(gid->raw, 0, 16); + *((u32 *)gid->raw) = cpu_to_be32(0xfe800000); + gid->raw[12] = 0xfe; + gid->raw[11] = 0xff; + memcpy(gid->raw + 13, mac + 3, 3); + memcpy(gid->raw + 8, mac, 3); + gid->raw[8] ^= 2; +} + +static inline int rdma_is_multicast_addr(struct in6_addr *addr) +{ + return addr->s6_addr[0] == 0xff ? 1 : 0; +} + +static inline void rdma_get_ll_mac(struct in6_addr *addr, u8 *mac) +{ + memcpy(mac, &addr->s6_addr[8], 3); + memcpy(mac + 3, &addr->s6_addr[13], 3); + mac[0] ^= 2; +} + +static inline void rdma_get_mcast_mac(struct in6_addr *addr, u8 *mac) +{ + int i; + + mac[0] = 0x33; + mac[1] = 0x33; + for (i = 2; i < 6; ++i) + mac[i] = addr->s6_addr[i + 10]; +} + + static inline int ib_addr_hw_addr_is_gid(u8 *addr) + { + return be16_to_cpu(*(u16 *)(addr + 4)) == 0xfe80; + } + #endif /* IB_ADDR_H */ -- 1.6.3.3 From eli at mellanox.co.il Mon Jul 13 11:16:14 2009 From: eli at mellanox.co.il (Eli Cohen) Date: Mon, 13 Jul 2009 21:16:14 +0300 Subject: [ofa-general] [PATCH 5/8 v3] ib_core: RDMAoE UD packet packing support Message-ID: <20090713181614.GA790@mtls03> Add support functions to aid in packing RDMAoE packets. Signed-off-by: Eli Cohen --- drivers/infiniband/core/ud_header.c | 111 +++++++++++++++++++++++++++++++++++ include/rdma/ib_pack.h | 26 ++++++++ 2 files changed, 137 insertions(+), 0 deletions(-) diff --git a/drivers/infiniband/core/ud_header.c b/drivers/infiniband/core/ud_header.c index 8ec7876..d04b6f2 100644 --- a/drivers/infiniband/core/ud_header.c +++ b/drivers/infiniband/core/ud_header.c @@ -80,6 +80,29 @@ static const struct ib_field lrh_table[] = { .size_bits = 16 } }; +static const struct ib_field eth_table[] = { + { STRUCT_FIELD(eth, dmac_h), + .offset_words = 0, + .offset_bits = 0, + .size_bits = 32 }, + { STRUCT_FIELD(eth, dmac_l), + .offset_words = 1, + .offset_bits = 0, + .size_bits = 16 }, + { STRUCT_FIELD(eth, smac_h), + .offset_words = 1, + .offset_bits = 16, + .size_bits = 16 }, + { STRUCT_FIELD(eth, smac_l), + .offset_words = 2, + .offset_bits = 0, + .size_bits = 32 }, + { STRUCT_FIELD(eth, type), + .offset_words = 3, + .offset_bits = 0, + .size_bits = 16 } +}; + static const struct ib_field grh_table[] = { { STRUCT_FIELD(grh, ip_version), .offset_words = 0, @@ -241,6 +264,53 @@ void ib_ud_header_init(int payload_bytes, EXPORT_SYMBOL(ib_ud_header_init); /** + * ib_rdmaoe_ud_header_init - Initialize UD header structure + * @payload_bytes:Length of packet payload + * @grh_present:GRH flag (if non-zero, GRH will be included) + * @header:Structure to initialize + * + * ib_rdmaoe_ud_header_init() initializes the grh.ip_version, grh.payload_length, + * grh.next_header, bth.opcode, bth.pad_count and + * bth.transport_header_version fields of a &struct eth_ud_header given + * the payload length and whether a GRH will be included. + */ +void ib_rdmaoe_ud_header_init(int payload_bytes, + int grh_present, + struct eth_ud_header *header) +{ + int header_len; + + memset(header, 0, sizeof *header); + + header_len = + sizeof header->eth + + IB_BTH_BYTES + + IB_DETH_BYTES; + if (grh_present) + header_len += IB_GRH_BYTES; + + header->grh_present = grh_present; + if (grh_present) { + header->grh.ip_version = 6; + header->grh.payload_length = + cpu_to_be16((IB_BTH_BYTES + + IB_DETH_BYTES + + payload_bytes + + 4 + /* ICRC */ + 3) & ~3); /* round up */ + header->grh.next_header = 0x1b; + } + + if (header->immediate_present) + header->bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; + else + header->bth.opcode = IB_OPCODE_UD_SEND_ONLY; + header->bth.pad_count = (4 - payload_bytes) & 3; + header->bth.transport_header_version = 0; +} +EXPORT_SYMBOL(ib_rdmaoe_ud_header_init); + +/** * ib_ud_header_pack - Pack UD header struct into wire format * @header:UD header struct * @buf:Buffer to pack into @@ -281,6 +351,47 @@ int ib_ud_header_pack(struct ib_ud_header *header, EXPORT_SYMBOL(ib_ud_header_pack); /** + * rdmaoe_ud_header_pack - Pack UD header struct into eth wire format + * @header:UD header struct + * @buf:Buffer to pack into + * + * ib_ud_header_pack() packs the UD header structure @header into wire + * format in the buffer @buf. + */ +int rdmaoe_ud_header_pack(struct eth_ud_header *header, + void *buf) +{ + int len = 0; + + ib_pack(eth_table, ARRAY_SIZE(eth_table), + &header->eth, buf); + len += IB_ETH_BYTES; + + if (header->grh_present) { + ib_pack(grh_table, ARRAY_SIZE(grh_table), + &header->grh, buf + len); + len += IB_GRH_BYTES; + } + + ib_pack(bth_table, ARRAY_SIZE(bth_table), + &header->bth, buf + len); + len += IB_BTH_BYTES; + + ib_pack(deth_table, ARRAY_SIZE(deth_table), + &header->deth, buf + len); + len += IB_DETH_BYTES; + + if (header->immediate_present) { + memcpy(buf + len, &header->immediate_data, + sizeof header->immediate_data); + len += sizeof header->immediate_data; + } + + return len; +} +EXPORT_SYMBOL(rdmaoe_ud_header_pack); + +/** * ib_ud_header_unpack - Unpack UD header struct from wire format * @header:UD header struct * @buf:Buffer to pack into diff --git a/include/rdma/ib_pack.h b/include/rdma/ib_pack.h index d7fc45c..bf199eb 100644 --- a/include/rdma/ib_pack.h +++ b/include/rdma/ib_pack.h @@ -37,6 +37,7 @@ enum { IB_LRH_BYTES = 8, + IB_ETH_BYTES = 14, IB_GRH_BYTES = 40, IB_BTH_BYTES = 12, IB_DETH_BYTES = 8 @@ -210,6 +211,14 @@ struct ib_unpacked_deth { __be32 source_qpn; }; +struct ib_unpacked_eth { + u8 dmac_h[4]; + u8 dmac_l[2]; + u8 smac_h[2]; + u8 smac_l[4]; + __be16 type; +}; + struct ib_ud_header { struct ib_unpacked_lrh lrh; int grh_present; @@ -220,6 +229,16 @@ struct ib_ud_header { __be32 immediate_data; }; +struct eth_ud_header { + struct ib_unpacked_eth eth; + int grh_present; + struct ib_unpacked_grh grh; + struct ib_unpacked_bth bth; + struct ib_unpacked_deth deth; + int immediate_present; + __be32 immediate_data; +}; + void ib_pack(const struct ib_field *desc, int desc_len, void *structure, @@ -234,10 +253,17 @@ void ib_ud_header_init(int payload_bytes, int grh_present, struct ib_ud_header *header); +void ib_rdmaoe_ud_header_init(int payload_bytes, + int grh_present, + struct eth_ud_header *header); + int ib_ud_header_pack(struct ib_ud_header *header, void *buf); int ib_ud_header_unpack(void *buf, struct ib_ud_header *header); +int rdmaoe_ud_header_pack(struct eth_ud_header *header, + void *buf); + #endif /* IB_PACK_H */ -- 1.6.3.3 From davem at davemloft.net Mon Jul 13 11:14:07 2009 From: davem at davemloft.net (David Miller) Date: Mon, 13 Jul 2009 11:14:07 -0700 (PDT) Subject: [ofa-general][PATCH] mlx4_core: Synch catastrophic flow with module unload In-Reply-To: <4A5B5274.2020801@mellanox.co.il> References: <4A5B5274.2020801@mellanox.co.il> Message-ID: <20090713.111407.196601373.davem@davemloft.net> From: Yevgeny Petrilin Date: Mon, 13 Jul 2009 18:27:48 +0300 > There is a race condition when the mlx4_core module is being unloaded > during the execution of restart task due to catastrophic error. > Added a global mutex that synchs those operations. If the catastrophic task > tries to catch the mutex, and it is already taken, it means that somebody is unloading the > module, and there is no point in executing the restart operation. > If the unload function tries to catch the mutex and it is taken, > it would wait for the catas task to finish and then unload the module. > > Signed-off-by: Yevgeny Petrilin Applied, thanks. From eli at mellanox.co.il Mon Jul 13 11:17:05 2009 From: eli at mellanox.co.il (Eli Cohen) Date: Mon, 13 Jul 2009 21:17:05 +0300 Subject: [ofa-general] [PATCH 6/8 v3] IB/ipoib: restrict IPoIB to work on IB ports only Message-ID: <20090713181705.GA802@mtls03> We don't want IPoIB to work over RDMAoE since it will give worse performance than working directly on Ethernet interfaces which are a prerequisite to RDMAoE anyway. Signed-off-by: Eli Cohen --- drivers/infiniband/ulp/ipoib/ipoib_main.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index ab2c192..f12884c 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -1228,6 +1228,9 @@ static struct net_device *ipoib_add_port(const char *format, struct ib_port_attr attr; int result = -ENOMEM; + if (ib_get_port_link_type(hca, port) != PORT_LINK_IB) + return ERR_PTR(-ENODEV); + priv = ipoib_intf_alloc(format); if (!priv) goto alloc_mem_failed; -- 1.6.3.3 From eli at mellanox.co.il Mon Jul 13 11:17:43 2009 From: eli at mellanox.co.il (Eli Cohen) Date: Mon, 13 Jul 2009 21:17:43 +0300 Subject: [ofa-general] [PATCH 7/8 v3] mlx4: Add support for RDMAoE - address resolution Message-ID: <20090713181743.GA904@mtls03> The following path handles address vectors creation for RDMAoE ports. mlx4 needs the MAC address of the remote node to include it in the WQE of a UD QP or in the QP context of connected QPs. Address resolution is done atomically in the case of a link local address or using service from core/addr.c to do that. mlx4 transport packets were changed too to accomodate for RDMAoE. Signed-off-by: Eli Cohen --- drivers/infiniband/hw/mlx4/ah.c | 228 ++++++++++++++++++++++++++----- drivers/infiniband/hw/mlx4/mlx4_ib.h | 30 ++++- drivers/infiniband/hw/mlx4/qp.c | 253 +++++++++++++++++++++++++++------- include/linux/mlx4/device.h | 31 ++++- include/linux/mlx4/qp.h | 8 +- 5 files changed, 463 insertions(+), 87 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c index c75ac94..c994e1f 100644 --- a/drivers/infiniband/hw/mlx4/ah.c +++ b/drivers/infiniband/hw/mlx4/ah.c @@ -31,63 +31,200 @@ */ #include "mlx4_ib.h" +#include +#include +#include -struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) +static struct rdma_addr_client addr_client; + +struct resolve_ctx { + struct completion done; + int status; +}; + + +static int status2err(int status) { - struct mlx4_dev *dev = to_mdev(pd->device)->dev; - struct mlx4_ib_ah *ah; + return status; /* TBD */ +} - ah = kmalloc(sizeof *ah, GFP_ATOMIC); - if (!ah) - return ERR_PTR(-ENOMEM); +static void resolve_callback(int status, struct sockaddr *src_addr, + struct rdma_dev_addr *addr, void *context) +{ + struct resolve_ctx *ctx = context; - memset(&ah->av, 0, sizeof ah->av); + ctx->status = status; + complete(&ctx->done); +} - ah->av.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24)); - ah->av.g_slid = ah_attr->src_path_bits; - ah->av.dlid = cpu_to_be16(ah_attr->dlid); - if (ah_attr->static_rate) { - ah->av.stat_rate = ah_attr->static_rate + MLX4_STAT_RATE_OFFSET; - while (ah->av.stat_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && - !(1 << ah->av.stat_rate & dev->caps.stat_rate_support)) - --ah->av.stat_rate; +int mlx4_ib_resolve_grh(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah_attr, + u8 *mac, int *is_mcast) +{ + struct mlx4_ib_rdmaoe *rdmaoe = &dev->rdmaoe; + struct sockaddr_in6 dst = {0}; + struct rdma_dev_addr addr; + struct resolve_ctx ctx; + struct net_device *netdev; + int err = 0; + int ifidx; + + *is_mcast = 0; + spin_lock(&rdmaoe->lock); + netdev = rdmaoe->netdevs[ah_attr->port_num - 1]; + if (!netdev) { + spin_unlock(&rdmaoe->lock); + return -EINVAL; + } + ifidx = netdev->ifindex; + spin_unlock(&rdmaoe->lock); + + init_completion(&ctx.done); + memcpy(dst.sin6_addr.s6_addr, ah_attr->grh.dgid.raw, sizeof ah_attr->grh); + dst.sin6_family = AF_INET6; + dst.sin6_scope_id = ifidx; + if (rdma_link_local_addr(&dst.sin6_addr)) + rdma_get_ll_mac(&dst.sin6_addr, mac); + else if (rdma_is_multicast_addr(&dst.sin6_addr)) { + rdma_get_mcast_mac(&dst.sin6_addr, mac); + *is_mcast = 1; + } else { + err = rdma_resolve_ip(&addr_client, NULL, (struct sockaddr *)&dst, &addr, + 2000, resolve_callback, &ctx); + if (!err) + wait_for_completion(&ctx.done); + else + ctx.status = err; + + err = status2err(ctx.status); } - ah->av.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); + + return err; +} + +static struct ib_ah *create_ib_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr, + struct mlx4_ib_ah *ah) +{ + struct mlx4_dev *dev = to_mdev(pd->device)->dev; + + ah->av.ib.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24)); + ah->av.ib.g_slid = ah_attr->src_path_bits; if (ah_attr->ah_flags & IB_AH_GRH) { - ah->av.g_slid |= 0x80; - ah->av.gid_index = ah_attr->grh.sgid_index; - ah->av.hop_limit = ah_attr->grh.hop_limit; - ah->av.sl_tclass_flowlabel |= + ah->av.ib.g_slid |= 0x80; + ah->av.ib.gid_index = ah_attr->grh.sgid_index; + ah->av.ib.hop_limit = ah_attr->grh.hop_limit; + ah->av.ib.sl_tclass_flowlabel |= cpu_to_be32((ah_attr->grh.traffic_class << 20) | ah_attr->grh.flow_label); - memcpy(ah->av.dgid, ah_attr->grh.dgid.raw, 16); + memcpy(ah->av.ib.dgid, ah_attr->grh.dgid.raw, 16); } + ah->av.ib.dlid = cpu_to_be16(ah_attr->dlid); + if (ah_attr->static_rate) { + ah->av.ib.stat_rate = ah_attr->static_rate + MLX4_STAT_RATE_OFFSET; + while (ah->av.ib.stat_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && + !(1 << ah->av.ib.stat_rate & dev->caps.stat_rate_support)) + --ah->av.ib.stat_rate; + } + ah->av.ib.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); + return &ah->ibah; } +static struct ib_ah *create_rdmaoe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr, + struct mlx4_ib_ah *ah) +{ + struct mlx4_ib_dev *ibdev = to_mdev(pd->device); + struct mlx4_dev *dev = ibdev->dev; + u8 mac[6]; + int err; + int is_mcast; + + err = mlx4_ib_resolve_grh(ibdev, ah_attr, mac, &is_mcast); + if (err) + return ERR_PTR(err); + + memcpy(ah->av.eth.mac_0_1, mac, 2); + memcpy(ah->av.eth.mac_2_5, mac + 2, 4); + ah->av.ib.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24)); + ah->av.ib.g_slid = 0x80; + if (ah_attr->static_rate) { + ah->av.ib.stat_rate = ah_attr->static_rate + MLX4_STAT_RATE_OFFSET; + while (ah->av.ib.stat_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET && + !(1 << ah->av.ib.stat_rate & dev->caps.stat_rate_support)) + --ah->av.ib.stat_rate; + } + + /* + * HW requires multicast LID so we just choose one. + */ + if (is_mcast) + ah->av.ib.dlid = cpu_to_be16(0xc000); + + memcpy(ah->av.ib.dgid, ah_attr->grh.dgid.raw, 16); + ah->av.ib.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); + + return &ah->ibah; +} + +struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) +{ + struct mlx4_ib_ah *ah; + enum ib_port_link_type link_type; + struct ib_ah *ret; + + ah = kzalloc(sizeof *ah, GFP_ATOMIC); + if (!ah) + return ERR_PTR(-ENOMEM); + + link_type = ib_get_port_link_type(pd->device, ah_attr->port_num); + if (link_type == PORT_LINK_ETH) { + if (!(ah_attr->ah_flags & IB_AH_GRH)) { + ret = ERR_PTR(-EINVAL); + goto out; + } else { + /* TBD: need to handle the case when we get called + in an atomic context and there we might sleep. We + don't expect this currently since we're working with + link local addresses which we can translate without + going to sleep */ + ret = create_rdmaoe_ah(pd, ah_attr, ah); + if (IS_ERR(ret)) + goto out; + else + return ret; + } + } else + return create_ib_ah(pd, ah_attr, ah); /* never fails */ + +out: + kfree(ah); + return ret; +} + int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) { struct mlx4_ib_ah *ah = to_mah(ibah); + enum ib_port_link_type lt; + lt = ib_get_port_link_type(ibah->device, ah_attr->port_num); memset(ah_attr, 0, sizeof *ah_attr); - ah_attr->dlid = be16_to_cpu(ah->av.dlid); - ah_attr->sl = be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 28; - ah_attr->port_num = be32_to_cpu(ah->av.port_pd) >> 24; - if (ah->av.stat_rate) - ah_attr->static_rate = ah->av.stat_rate - MLX4_STAT_RATE_OFFSET; - ah_attr->src_path_bits = ah->av.g_slid & 0x7F; + ah_attr->dlid = lt == PORT_LINK_IB ? be16_to_cpu(ah->av.ib.dlid) : 0; + ah_attr->sl = be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; + ah_attr->port_num = be32_to_cpu(ah->av.ib.port_pd) >> 24; + if (ah->av.ib.stat_rate) + ah_attr->static_rate = ah->av.ib.stat_rate - MLX4_STAT_RATE_OFFSET; + ah_attr->src_path_bits = ah->av.ib.g_slid & 0x7F; if (mlx4_ib_ah_grh_present(ah)) { ah_attr->ah_flags = IB_AH_GRH; ah_attr->grh.traffic_class = - be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 20; + be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20; ah_attr->grh.flow_label = - be32_to_cpu(ah->av.sl_tclass_flowlabel) & 0xfffff; - ah_attr->grh.hop_limit = ah->av.hop_limit; - ah_attr->grh.sgid_index = ah->av.gid_index; - memcpy(ah_attr->grh.dgid.raw, ah->av.dgid, 16); + be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) & 0xfffff; + ah_attr->grh.hop_limit = ah->av.ib.hop_limit; + ah_attr->grh.sgid_index = ah->av.ib.gid_index; + memcpy(ah_attr->grh.dgid.raw, ah->av.ib.dgid, 16); } return 0; @@ -98,3 +235,30 @@ int mlx4_ib_destroy_ah(struct ib_ah *ah) kfree(to_mah(ah)); return 0; } + +int mlx4_ib_get_mac(struct ib_device *device, u8 port, u8 *gid, u8 *mac) +{ + int err; + struct mlx4_ib_dev *ibdev = to_mdev(device); + struct ib_ah_attr ah_attr = { + .port_num = port, + }; + int is_mcast; + + memcpy(ah_attr.grh.dgid.raw, gid, 16); + err = mlx4_ib_resolve_grh(ibdev, &ah_attr, mac, &is_mcast); + if (err) + ERR_PTR(err); + + return 0; +} + +void mlx4_ib_addr_init(void) +{ + rdma_addr_register_client(&addr_client); +} + +void mlx4_ib_addr_cleanup(void) +{ + rdma_addr_unregister_client(&addr_client); +} diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 8a7dd67..e2d6e62 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -138,6 +138,7 @@ struct mlx4_ib_qp { u8 resp_depth; u8 sq_no_prefetch; u8 state; + int mlx_type; }; struct mlx4_ib_srq { @@ -157,7 +158,15 @@ struct mlx4_ib_srq { struct mlx4_ib_ah { struct ib_ah ibah; - struct mlx4_av av; + union mlx4_ext_av av; +}; + +struct mlx4_ib_rdmaoe { + spinlock_t lock; + struct net_device *netdevs[MLX4_MAX_PORTS]; + int enstate[MLX4_MAX_PORTS]; + int mtu[MLX4_MAX_PORTS]; + struct notifier_block nb; }; struct mlx4_ib_dev { @@ -175,6 +184,8 @@ struct mlx4_ib_dev { spinlock_t sm_lock; struct mutex cap_mask_mutex; + + struct mlx4_ib_rdmaoe rdmaoe; }; static inline struct mlx4_ib_dev *to_mdev(struct ib_device *ibdev) @@ -312,10 +323,25 @@ int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages, u64 iova); int mlx4_ib_unmap_fmr(struct list_head *fmr_list); int mlx4_ib_fmr_dealloc(struct ib_fmr *fmr); +void mlx4_ib_addr_init(void); +void mlx4_ib_addr_cleanup(void); + +int mlx4_ib_resolve_grh(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah_attr, + u8 *mac, int *is_mcast); + +int mlx4_ib_get_mac(struct ib_device *device, u8 port, u8 *gid, u8 *mac); static inline int mlx4_ib_ah_grh_present(struct mlx4_ib_ah *ah) { - return !!(ah->av.g_slid & 0x80); + /* + * port number is located at the same place for both IB and Eth + */ + u8 port = (ah->av.ib.port_pd >> 24) & 3; + + if (ib_get_port_link_type(ah->ibah.device, port) == PORT_LINK_ETH) + return 1; + else + return !!(ah->av.ib.g_slid & 0x80); } #endif /* MLX4_IB_H */ diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 20724ae..7a6b765 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -32,6 +32,7 @@ */ #include +#include #include #include @@ -47,14 +48,21 @@ enum { enum { MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83, - MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f + MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f, + MLX4_IB_LINK_TYPE_IB = 0, + MLX4_IB_LINK_TYPE_ETH = 1 }; enum { /* * Largest possible UD header: send with GRH and immediate data. + * 4 bytes added to accommodate for eth header instead of lrh */ - MLX4_IB_UD_HEADER_SIZE = 72 + MLX4_IB_UD_HEADER_SIZE = 76 +}; + +enum { + MLX4_RDMAOE_ETHERTYPE = 0x8915 }; struct mlx4_ib_sqp { @@ -62,7 +70,10 @@ struct mlx4_ib_sqp { int pkey_index; u32 qkey; u32 send_psn; - struct ib_ud_header ud_header; + union { + struct ib_ud_header ib; + struct eth_ud_header eth; + } hdr; u8 header_buf[MLX4_IB_UD_HEADER_SIZE]; }; @@ -782,18 +793,6 @@ int mlx4_ib_destroy_qp(struct ib_qp *qp) return 0; } -static int to_mlx4_st(enum ib_qp_type type) -{ - switch (type) { - case IB_QPT_RC: return MLX4_QP_ST_RC; - case IB_QPT_UC: return MLX4_QP_ST_UC; - case IB_QPT_UD: return MLX4_QP_ST_UD; - case IB_QPT_SMI: - case IB_QPT_GSI: return MLX4_QP_ST_MLX; - default: return -1; - } -} - static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr, int attr_mask) { @@ -843,6 +842,12 @@ static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port) static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah, struct mlx4_qp_path *path, u8 port) { + int err; + int is_eth = ib_get_port_link_type(&dev->ib_dev, port) == + PORT_LINK_ETH ? 1 : 0; + u8 mac[6]; + int is_mcast; + path->grh_mylmc = ah->src_path_bits & 0x7f; path->rlid = cpu_to_be16(ah->dlid); if (ah->static_rate) { @@ -873,9 +878,36 @@ static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah, path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((port - 1) << 6) | ((ah->sl & 0xf) << 2); + if (is_eth) { + if (!(ah->ah_flags & IB_AH_GRH)) + return -1; + + err = mlx4_ib_resolve_grh(dev, ah, mac, &is_mcast); + if (err) + return err; + + memcpy(path->dmac_h, mac, 2); + memcpy(path->dmac_l, mac + 2, 4); + path->ackto = MLX4_IB_LINK_TYPE_ETH; + /* use index 0 into MAC table for RDMAoE */ + path->grh_mylmc &= 0x80; + } + return 0; } +static int to_mlx4_st(enum ib_qp_type type) +{ + switch (type) { + case IB_QPT_RC: return MLX4_QP_ST_RC; + case IB_QPT_UC: return MLX4_QP_ST_UC; + case IB_QPT_UD: return MLX4_QP_ST_UD; + case IB_QPT_SMI: + case IB_QPT_GSI: return MLX4_QP_ST_MLX; + default: return -1; + } +} + static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, enum ib_qp_state cur_state, enum ib_qp_state new_state) @@ -972,7 +1004,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, } if (attr_mask & IB_QP_TIMEOUT) { - context->pri_path.ackto = attr->timeout << 3; + context->pri_path.ackto |= (attr->timeout << 3); optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT; } @@ -1206,8 +1238,8 @@ out: return err; } -static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr, - void *wqe, unsigned *mlx_seg_len) +static int build_ib_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr, + void *wqe, unsigned *mlx_seg_len) { struct ib_device *ib_dev = &to_mdev(sqp->qp.ibqp.device)->ib_dev; struct mlx4_wqe_mlx_seg *mlx = wqe; @@ -1223,61 +1255,171 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr, for (i = 0; i < wr->num_sge; ++i) send_size += wr->sg_list[i].length; - ib_ud_header_init(send_size, mlx4_ib_ah_grh_present(ah), &sqp->ud_header); + ib_ud_header_init(send_size, mlx4_ib_ah_grh_present(ah), &sqp->hdr.ib); - sqp->ud_header.lrh.service_level = - be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 28; - sqp->ud_header.lrh.destination_lid = ah->av.dlid; - sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.g_slid & 0x7f); + sqp->hdr.ib.lrh.service_level = + be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28; + sqp->hdr.ib.lrh.destination_lid = ah->av.ib.dlid; + sqp->hdr.ib.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f); if (mlx4_ib_ah_grh_present(ah)) { - sqp->ud_header.grh.traffic_class = - (be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 20) & 0xff; - sqp->ud_header.grh.flow_label = - ah->av.sl_tclass_flowlabel & cpu_to_be32(0xfffff); - sqp->ud_header.grh.hop_limit = ah->av.hop_limit; - ib_get_cached_gid(ib_dev, be32_to_cpu(ah->av.port_pd) >> 24, - ah->av.gid_index, &sqp->ud_header.grh.source_gid); - memcpy(sqp->ud_header.grh.destination_gid.raw, - ah->av.dgid, 16); + sqp->hdr.ib.grh.traffic_class = + (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff; + sqp->hdr.ib.grh.flow_label = + ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff); + sqp->hdr.ib.grh.hop_limit = ah->av.ib.hop_limit; + ib_get_cached_gid(ib_dev, be32_to_cpu(ah->av.ib.port_pd) >> 24, + ah->av.ib.gid_index, &sqp->hdr.ib.grh.source_gid); + memcpy(sqp->hdr.ib.grh.destination_gid.raw, + ah->av.ib.dgid, 16); } mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) | - (sqp->ud_header.lrh.destination_lid == + (sqp->hdr.ib.lrh.destination_lid == IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) | - (sqp->ud_header.lrh.service_level << 8)); - mlx->rlid = sqp->ud_header.lrh.destination_lid; + (sqp->hdr.ib.lrh.service_level << 8)); + mlx->rlid = sqp->hdr.ib.lrh.destination_lid; + + switch (wr->opcode) { + case IB_WR_SEND: + sqp->hdr.ib.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + sqp->hdr.ib.immediate_present = 0; + break; + case IB_WR_SEND_WITH_IMM: + sqp->hdr.ib.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; + sqp->hdr.ib.immediate_present = 1; + sqp->hdr.ib.immediate_data = wr->ex.imm_data; + break; + default: + return -EINVAL; + } + + sqp->hdr.ib.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0; + if (sqp->hdr.ib.lrh.destination_lid == IB_LID_PERMISSIVE) + sqp->hdr.ib.lrh.source_lid = IB_LID_PERMISSIVE; + sqp->hdr.ib.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED); + if (!sqp->qp.ibqp.qp_num) + ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey); + else + ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->wr.ud.pkey_index, &pkey); + sqp->hdr.ib.bth.pkey = cpu_to_be16(pkey); + sqp->hdr.ib.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); + sqp->hdr.ib.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); + sqp->hdr.ib.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ? + sqp->qkey : wr->wr.ud.remote_qkey); + sqp->hdr.ib.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); + + header_size = ib_ud_header_pack(&sqp->hdr.ib, sqp->header_buf); + + /* + * Inline data segments may not cross a 64 byte boundary. If + * our UD header is bigger than the space available up to the + * next 64 byte boundary in the WQE, use two inline data + * segments to hold the UD header. + */ + spc = MLX4_INLINE_ALIGN - + ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1)); + if (header_size <= spc) { + inl->byte_count = cpu_to_be32(1 << 31 | header_size); + memcpy(inl + 1, sqp->header_buf, header_size); + i = 1; + } else { + inl->byte_count = cpu_to_be32(1 << 31 | spc); + memcpy(inl + 1, sqp->header_buf, spc); + + inl = (void *) (inl + 1) + spc; + memcpy(inl + 1, sqp->header_buf + spc, header_size - spc); + /* + * Need a barrier here to make sure all the data is + * visible before the byte_count field is set. + * Otherwise the HCA prefetcher could grab the 64-byte + * chunk with this inline segment and get a valid (!= + * 0xffffffff) byte count but stale data, and end up + * generating a packet with bad headers. + * + * The first inline segment's byte_count field doesn't + * need a barrier, because it comes after a + * control/MLX segment and therefore is at an offset + * of 16 mod 64. + */ + wmb(); + inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc)); + i = 2; + } + + *mlx_seg_len = + ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16); + return 0; +} + +static int build_eth_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr, + void *wqe, unsigned *mlx_seg_len) +{ + struct ib_device *ib_dev = &to_mdev(sqp->qp.ibqp.device)->ib_dev; + struct mlx4_wqe_mlx_seg *mlx = wqe; + struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx; + struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah); + u16 pkey; + int send_size; + int header_size; + int spc; + int i; + void *tmp; + + send_size = 0; + for (i = 0; i < wr->num_sge; ++i) + send_size += wr->sg_list[i].length; + + ib_rdmaoe_ud_header_init(send_size, mlx4_ib_ah_grh_present(ah), &sqp->hdr.eth); + + if (mlx4_ib_ah_grh_present(ah)) { + sqp->hdr.eth.grh.traffic_class = + (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff; + sqp->hdr.eth.grh.flow_label = + ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff); + sqp->hdr.eth.grh.hop_limit = ah->av.ib.hop_limit; + ib_get_cached_gid(ib_dev, be32_to_cpu(ah->av.ib.port_pd) >> 24, + ah->av.ib.gid_index, &sqp->hdr.eth.grh.source_gid); + memcpy(sqp->hdr.eth.grh.destination_gid.raw, + ah->av.ib.dgid, 16); + } + + mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE); switch (wr->opcode) { case IB_WR_SEND: - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; - sqp->ud_header.immediate_present = 0; + sqp->hdr.eth.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + sqp->hdr.eth.immediate_present = 0; break; case IB_WR_SEND_WITH_IMM: - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; - sqp->ud_header.immediate_present = 1; - sqp->ud_header.immediate_data = wr->ex.imm_data; + sqp->hdr.eth.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; + sqp->hdr.eth.immediate_present = 1; + sqp->hdr.eth.immediate_data = wr->ex.imm_data; break; default: return -EINVAL; } - sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0; - if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE) - sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE; - sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED); + memcpy(sqp->hdr.eth.eth.dmac_h, ah->av.eth.mac_0_1, 2); + memcpy(sqp->hdr.eth.eth.dmac_h + 2, ah->av.eth.mac_2_5, 2); + memcpy(sqp->hdr.eth.eth.dmac_l, ah->av.eth.mac_2_5 + 2, 2); + tmp = to_mdev(sqp->qp.ibqp.device)->rdmaoe.netdevs[sqp->qp.port - 1]->dev_addr; + memcpy(sqp->hdr.eth.eth.smac_h, tmp, 2); + memcpy(sqp->hdr.eth.eth.smac_l, tmp + 2, 4); + sqp->hdr.eth.eth.type = cpu_to_be16(MLX4_RDMAOE_ETHERTYPE); + sqp->hdr.eth.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED); if (!sqp->qp.ibqp.qp_num) ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey); else ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->wr.ud.pkey_index, &pkey); - sqp->ud_header.bth.pkey = cpu_to_be16(pkey); - sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); - sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); - sqp->ud_header.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ? + sqp->hdr.eth.bth.pkey = cpu_to_be16(pkey); + sqp->hdr.eth.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); + sqp->hdr.eth.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); + sqp->hdr.eth.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ? sqp->qkey : wr->wr.ud.remote_qkey); - sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); + sqp->hdr.eth.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); - header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf); + header_size = rdmaoe_ud_header_pack(&sqp->hdr.eth, sqp->header_buf); if (0) { printk(KERN_ERR "built UD header of size %d:\n", header_size); @@ -1333,6 +1475,15 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr, return 0; } +static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr, + void *wqe, unsigned *mlx_seg_len) +{ + if (ib_get_port_link_type(sqp->qp.ibqp.device, sqp->qp.port) == PORT_LINK_IB) + return build_ib_mlx_header(sqp, wr, wqe, mlx_seg_len); + else + return build_eth_mlx_header(sqp, wr, wqe, mlx_seg_len); +} + static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq) { unsigned cur; @@ -1414,6 +1565,8 @@ static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg, memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av)); dseg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn); dseg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey); + dseg->vlan = to_mah(wr->wr.ud.ah)->av.eth.vlan; + memcpy(dseg->mac_0_1, to_mah(wr->wr.ud.ah)->av.eth.mac_0_1, 6); } static void set_mlx_icrc_seg(void *dseg) diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h index 3aff8a6..b73b5f0 100644 --- a/include/linux/mlx4/device.h +++ b/include/linux/mlx4/device.h @@ -66,7 +66,8 @@ enum { MLX4_DEV_CAP_FLAG_ATOMIC = 1 << 18, MLX4_DEV_CAP_FLAG_RAW_MCAST = 1 << 19, MLX4_DEV_CAP_FLAG_UD_AV_PORT = 1 << 20, - MLX4_DEV_CAP_FLAG_UD_MCAST = 1 << 21 + MLX4_DEV_CAP_FLAG_UD_MCAST = 1 << 21, + MLX4_DEV_CAP_FLAG_RDMAOE = 1 << 30 }; enum { @@ -371,6 +372,28 @@ struct mlx4_av { u8 dgid[16]; }; +struct mlx4_eth_av { + __be32 port_pd; + u8 reserved1; + u8 smac_idx; + u16 reserved2; + u8 reserved3; + u8 gid_index; + u8 stat_rate; + u8 hop_limit; + __be32 sl_tclass_flowlabel; + u8 dgid[16]; + u32 reserved4[2]; + __be16 vlan; + u8 mac_0_1[2]; + u8 mac_2_5[4]; +}; + +union mlx4_ext_av { + struct mlx4_av ib; + struct mlx4_eth_av eth; +}; + struct mlx4_dev { struct pci_dev *pdev; unsigned long flags; @@ -399,6 +422,12 @@ struct mlx4_init_port_param { if (((type) == MLX4_PORT_TYPE_IB ? (dev)->caps.port_mask : \ ~(dev)->caps.port_mask) & 1 << ((port) - 1)) +#define mlx4_foreach_ib_transport_port(port, dev) \ + for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \ + if (((dev)->caps.port_mask & 1 << ((port) - 1)) || \ + ((dev)->caps.flags & MLX4_DEV_CAP_FLAG_RDMAOE)) + + int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, struct mlx4_buf *buf); void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf); diff --git a/include/linux/mlx4/qp.h b/include/linux/mlx4/qp.h index bf8f119..d73534f 100644 --- a/include/linux/mlx4/qp.h +++ b/include/linux/mlx4/qp.h @@ -112,7 +112,9 @@ struct mlx4_qp_path { u8 snooper_flags; u8 reserved3[2]; u8 counter_index; - u8 reserved4[7]; + u8 reserved4; + u8 dmac_h[2]; + u8 dmac_l[4]; }; struct mlx4_qp_context { @@ -218,7 +220,9 @@ struct mlx4_wqe_datagram_seg { __be32 av[8]; __be32 dqpn; __be32 qkey; - __be32 reservd[2]; + __be16 vlan; + u8 mac_0_1[2]; + u8 mac_2_5[4]; }; struct mlx4_wqe_lso_seg { -- 1.6.3.3 From eli at mellanox.co.il Mon Jul 13 11:18:24 2009 From: eli at mellanox.co.il (Eli Cohen) Date: Mon, 13 Jul 2009 21:18:24 +0300 Subject: [ofa-general] [PATCH 8/8 v3] mlx4: Add RDMAoE support - allow interfaces to correspond to each other Message-ID: <20090713181824.GA916@mtls03> This patch add support RDMAoE for mlx4. Since mlx4_ib now needs to reference mlx4_en netdevices, a new mechanism was added. Two new fields were added to struct mlx4_interface to define a protocol and a get_prot_dev method to retrieve the corresponding protocol's net device. An implementation of the new verb ib_get_port_link_type() - mlx4_ib_get_port_link_type - was added. mlx4_ib_query_port() has been modified to support eth link types. An interface is considered to be active if its corresponding eth interface is active. Code for setting the GID table of a port has been added. Currently, each IB port has a single GID entry in its table and that GID entery equals the link local IPv6 address. Signed-off-by: Eli Cohen --- drivers/infiniband/hw/mlx4/main.c | 276 +++++++++++++++++++++++++++++++++---- drivers/net/mlx4/cmd.c | 6 + drivers/net/mlx4/en_main.c | 15 ++- drivers/net/mlx4/en_port.c | 4 +- drivers/net/mlx4/en_port.h | 3 +- drivers/net/mlx4/intf.c | 20 +++ drivers/net/mlx4/main.c | 6 + drivers/net/mlx4/mlx4.h | 1 + include/linux/mlx4/cmd.h | 1 + include/linux/mlx4/driver.h | 16 ++- 10 files changed, 310 insertions(+), 38 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index ae3d759..40442b8 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -152,28 +153,19 @@ out: return err; } -static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port, - struct ib_port_attr *props) +static enum ib_port_link_type mlx4_ib_get_port_link_type(struct ib_device *device, + u8 port_num) { - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int err = -ENOMEM; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - memset(props, 0, sizeof *props); - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; - in_mad->attr_mod = cpu_to_be32(port); + struct mlx4_dev *dev = to_mdev(device)->dev; - err = mlx4_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad); - if (err) - goto out; + return dev->caps.port_mask & (1 << (port_num - 1)) ? + PORT_LINK_IB : PORT_LINK_ETH; +} +static void ib_link_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props, + struct ib_smp *out_mad) +{ props->lid = be16_to_cpup((__be16 *) (out_mad->data + 16)); props->lmc = out_mad->data[34] & 0x7; props->sm_lid = be16_to_cpup((__be16 *) (out_mad->data + 18)); @@ -193,6 +185,67 @@ static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port, props->subnet_timeout = out_mad->data[51] & 0x1f; props->max_vl_num = out_mad->data[37] >> 4; props->init_type_reply = out_mad->data[41] >> 4; + props->link_type = PORT_LINK_IB; +} + +static void eth_link_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props, + struct ib_smp *out_mad) +{ + struct mlx4_ib_rdmaoe *rdmaoe = &to_mdev(ibdev)->rdmaoe; + struct net_device *ndev; + + props->port_cap_flags = be32_to_cpup((__be32 *) (out_mad->data + 20)); + props->gid_tbl_len = to_mdev(ibdev)->dev->caps.gid_table_len[port]; + props->max_msg_sz = to_mdev(ibdev)->dev->caps.max_msg_sz; + props->pkey_tbl_len = to_mdev(ibdev)->dev->caps.pkey_table_len[port]; + props->bad_pkey_cntr = be16_to_cpup((__be16 *) (out_mad->data + 46)); + props->qkey_viol_cntr = be16_to_cpup((__be16 *) (out_mad->data + 48)); + props->active_width = out_mad->data[31] & 0xf; + props->active_speed = out_mad->data[35] >> 4; + props->max_mtu = out_mad->data[41] & 0xf; + props->active_mtu = rdmaoe->mtu[port - 1]; + props->subnet_timeout = out_mad->data[51] & 0x1f; + props->max_vl_num = out_mad->data[37] >> 4; + props->init_type_reply = out_mad->data[41] >> 4; + props->link_type = PORT_LINK_ETH; + spin_lock(&rdmaoe->lock); + ndev = rdmaoe->netdevs[port - 1]; + if (!ndev) + goto out; + + props->state = netif_running(ndev) && netif_oper_up(ndev) ? + IB_PORT_ACTIVE : IB_PORT_DOWN; + props->phys_state = props->state; +out: + spin_unlock(&rdmaoe->lock); +} + +static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port, + struct ib_port_attr *props) +{ + struct ib_smp *in_mad = NULL; + struct ib_smp *out_mad = NULL; + int err = -ENOMEM; + + in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); + out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); + if (!in_mad || !out_mad) + goto out; + + memset(props, 0, sizeof *props); + + init_query_mad(in_mad); + in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; + in_mad->attr_mod = cpu_to_be32(port); + + err = mlx4_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad); + if (err) + goto out; + + mlx4_ib_get_port_link_type(ibdev, port) == PORT_LINK_IB ? + ib_link_query_port(ibdev, port, props, out_mad) : + eth_link_query_port(ibdev, port, props, out_mad); out: kfree(in_mad); @@ -287,6 +340,7 @@ static int mlx4_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols, { struct mlx4_cmd_mailbox *mailbox; int err; + u8 is_eth = dev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH; mailbox = mlx4_alloc_cmd_mailbox(dev->dev); if (IS_ERR(mailbox)) @@ -302,7 +356,7 @@ static int mlx4_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols, ((__be32 *) mailbox->buf)[1] = cpu_to_be32(cap_mask); } - err = mlx4_cmd(dev->dev, mailbox->dma, port, 0, MLX4_CMD_SET_PORT, + err = mlx4_cmd(dev->dev, mailbox->dma, port, is_eth, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B); mlx4_free_cmd_mailbox(dev->dev, mailbox); @@ -538,19 +592,153 @@ static struct device_attribute *mlx4_class_attributes[] = { &dev_attr_board_id }; +static void mlx4_addrconf_ifid_eui48(u8 *eui, struct net_device *dev) +{ + memcpy(eui, dev->dev_addr, 3); + memcpy(eui + 5, dev->dev_addr + 3, 3); + eui[3] = 0xFF; + eui[4] = 0xFE; + eui[0] ^= 2; +} + +static int update_ipv6_gids(struct mlx4_ib_dev *dev, int port, int clear) +{ + struct net_device *ndev = dev->rdmaoe.netdevs[port - 1]; + struct mlx4_cmd_mailbox *mailbox; + union ib_gid *gids, *tmpgids; + int err; + + tmpgids = kzalloc(128 * sizeof *gids, GFP_ATOMIC); + if (!tmpgids) + return -ENOMEM; + + if (!clear) { + mlx4_addrconf_ifid_eui48(&tmpgids[0].raw[8], ndev); + tmpgids[0].global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL); + } + spin_unlock(&dev->rdmaoe.lock); + + mailbox = mlx4_alloc_cmd_mailbox(dev->dev); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); + goto out; + } + + gids = mailbox->buf; + memcpy(gids, tmpgids, 128 * sizeof *gids); + + err = mlx4_cmd(dev->dev, mailbox->dma, MLX4_SET_PORT_GID_TABLE << 8 | port, + 1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B); + + mlx4_free_cmd_mailbox(dev->dev, mailbox); + +out: + kfree(tmpgids); + spin_lock(&dev->rdmaoe.lock); + return err; +} + +static void update_mtu(struct mlx4_ib_dev *dev, int port) +{ + struct net_device *ndev = dev->rdmaoe.netdevs[port - 1]; + + if (ndev->mtu >= 4096) + dev->rdmaoe.mtu[port - 1] = IB_MTU_4096; + else if (ndev->mtu >= 2048) + dev->rdmaoe.mtu[port - 1] = IB_MTU_2048; + else if (ndev->mtu >= 1024) + dev->rdmaoe.mtu[port - 1] = IB_MTU_1024; + else if (ndev->mtu >= 512) + dev->rdmaoe.mtu[port - 1] = IB_MTU_512; + else if (ndev->mtu >= 256) + dev->rdmaoe.mtu[port - 1] = IB_MTU_256; + else { + printk(KERN_WARNING "Eth MTU %d too small - IB may not work properly\n", + ndev->mtu); + dev->rdmaoe.mtu[port - 1] = IB_MTU_256; + } +} + +static void handle_en_event(struct mlx4_ib_dev *dev, int port, unsigned long event) +{ + switch (event) { + case NETDEV_UP: + update_ipv6_gids(dev, port, 0); + break; + + case NETDEV_DOWN: + update_ipv6_gids(dev, port, 1); + + case NETDEV_CHANGEMTU: + update_mtu(dev, port); + break; + } +} + +static void netdev_added(struct mlx4_ib_dev *dev, int port) +{ + update_ipv6_gids(dev, port, 0); + update_mtu(dev, port); +} + +static void netdev_removed(struct mlx4_ib_dev *dev, int port) +{ + update_ipv6_gids(dev, port, 1); +} + +static int mlx4_ib_netdev_event(struct notifier_block *this, unsigned long event, + void *ptr) +{ + struct net_device *dev = ptr; + struct mlx4_ib_dev *ibdev; + struct net_device *oldnd; + struct mlx4_ib_rdmaoe *rdmaoe; + int port; + + if (!net_eq(dev_net(dev), &init_net)) + return NOTIFY_DONE; + + ibdev = container_of(this, struct mlx4_ib_dev, rdmaoe.nb); + rdmaoe = &ibdev->rdmaoe; + + spin_lock(&rdmaoe->lock); + mlx4_foreach_ib_transport_port(port, ibdev->dev) { + oldnd = rdmaoe->netdevs[port - 1]; + rdmaoe->netdevs[port - 1] = mlx4_get_prot_dev(ibdev->dev, MLX4_PROT_EN, port); + if (oldnd != rdmaoe->netdevs[port - 1]) { + if (rdmaoe->netdevs[port - 1]) + netdev_added(ibdev, port); + else + netdev_removed(ibdev, port); + } + } + + if (dev == rdmaoe->netdevs[0]) + handle_en_event(ibdev, 1, event); + else if (dev == rdmaoe->netdevs[1]) + handle_en_event(ibdev, 2, event); + + spin_unlock(&rdmaoe->lock); + + return NOTIFY_DONE; +} + static void *mlx4_ib_add(struct mlx4_dev *dev) { static int mlx4_ib_version_printed; struct mlx4_ib_dev *ibdev; int num_ports = 0; int i; + int err; + int port; + struct mlx4_ib_rdmaoe *rdmaoe; if (!mlx4_ib_version_printed) { printk(KERN_INFO "%s", mlx4_ib_version); ++mlx4_ib_version_printed; } - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) + mlx4_foreach_ib_transport_port(i, dev) num_ports++; /* No point in registering a device with no ports... */ @@ -563,6 +751,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) return NULL; } + rdmaoe = &ibdev->rdmaoe; + if (mlx4_pd_alloc(dev, &ibdev->priv_pdn)) goto err_dealloc; @@ -607,10 +797,12 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | - (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ); + (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | + (1ull << IB_USER_VERBS_CMD_GET_MAC); ibdev->ib_dev.query_device = mlx4_ib_query_device; ibdev->ib_dev.query_port = mlx4_ib_query_port; + ibdev->ib_dev.get_port_link_type = mlx4_ib_get_port_link_type; ibdev->ib_dev.query_gid = mlx4_ib_query_gid; ibdev->ib_dev.query_pkey = mlx4_ib_query_pkey; ibdev->ib_dev.modify_device = mlx4_ib_modify_device; @@ -654,15 +846,26 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) ibdev->ib_dev.map_phys_fmr = mlx4_ib_map_phys_fmr; ibdev->ib_dev.unmap_fmr = mlx4_ib_unmap_fmr; ibdev->ib_dev.dealloc_fmr = mlx4_ib_fmr_dealloc; + ibdev->ib_dev.get_mac = mlx4_ib_get_mac; + + mlx4_foreach_ib_transport_port(port, dev) + rdmaoe->netdevs[port - 1] = mlx4_get_prot_dev(dev, MLX4_PROT_EN, port); + spin_lock_init(&rdmaoe->lock); + if (dev->caps.flags & MLX4_DEV_CAP_FLAG_RDMAOE && !rdmaoe->nb.notifier_call) { + rdmaoe->nb.notifier_call = mlx4_ib_netdev_event; + err = register_netdevice_notifier(&rdmaoe->nb); + if (err) + goto err_map; + } if (init_node_data(ibdev)) - goto err_map; + goto err_notif; spin_lock_init(&ibdev->sm_lock); mutex_init(&ibdev->cap_mask_mutex); if (ib_register_device(&ibdev->ib_dev)) - goto err_map; + goto err_notif; if (mlx4_ib_mad_init(ibdev)) goto err_reg; @@ -678,6 +881,9 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) err_reg: ib_unregister_device(&ibdev->ib_dev); +err_notif: + unregister_netdevice_notifier(&rdmaoe->nb); + err_map: iounmap(ibdev->uar_map); @@ -700,11 +906,15 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr) mlx4_ib_mad_cleanup(ibdev); ib_unregister_device(&ibdev->ib_dev); + if (ibdev->rdmaoe.nb.notifier_call) { + unregister_netdevice_notifier(&ibdev->rdmaoe.nb); + ibdev->rdmaoe.nb.notifier_call = NULL; + } + iounmap(ibdev->uar_map); - for (p = 1; p <= ibdev->num_ports; ++p) + mlx4_foreach_port(p, dev, MLX4_PORT_TYPE_IB) mlx4_CLOSE_PORT(dev, p); - iounmap(ibdev->uar_map); mlx4_uar_free(dev, &ibdev->priv_uar); mlx4_pd_free(dev, ibdev->priv_pdn); ib_dealloc_device(&ibdev->ib_dev); @@ -745,17 +955,27 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr, static struct mlx4_interface mlx4_ib_interface = { .add = mlx4_ib_add, .remove = mlx4_ib_remove, - .event = mlx4_ib_event + .event = mlx4_ib_event, + .protocol = MLX4_PROT_IB }; static int __init mlx4_ib_init(void) { - return mlx4_register_interface(&mlx4_ib_interface); + int err; + + mlx4_ib_addr_init(); + + err = mlx4_register_interface(&mlx4_ib_interface); + if (err) + mlx4_ib_addr_cleanup(); + + return err; } static void __exit mlx4_ib_cleanup(void) { mlx4_unregister_interface(&mlx4_ib_interface); + mlx4_ib_addr_cleanup(); } module_init(mlx4_ib_init); diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c index 2845a05..07a646b 100644 --- a/drivers/net/mlx4/cmd.c +++ b/drivers/net/mlx4/cmd.c @@ -103,6 +103,7 @@ enum { struct mlx4_cmd_context { struct completion done; int result; + u16 command; int next; u64 out_param; u16 token; @@ -255,6 +256,10 @@ void mlx4_cmd_event(struct mlx4_dev *dev, u16 token, u8 status, u64 out_param) return; context->result = mlx4_status_to_errno(status); + if (context->result) + mlx4_warn(dev, "FW command 0x%x failed with FW status = %d\n", + context->command, status); + context->out_param = out_param; complete(&context->done); @@ -274,6 +279,7 @@ static int mlx4_cmd_wait(struct mlx4_dev *dev, u64 in_param, u64 *out_param, BUG_ON(cmd->free_head < 0); context = &cmd->context[cmd->free_head]; context->token += cmd->token_mask + 1; + context->command = op; cmd->free_head = context->next; spin_unlock(&cmd->context_lock); diff --git a/drivers/net/mlx4/en_main.c b/drivers/net/mlx4/en_main.c index 510633f..6f30eca 100644 --- a/drivers/net/mlx4/en_main.c +++ b/drivers/net/mlx4/en_main.c @@ -51,6 +51,13 @@ static const char mlx4_en_version[] = DRV_NAME ": Mellanox ConnectX HCA Ethernet driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; +static void *get_netdev(struct mlx4_dev *dev, void *ctx, u8 port) +{ + struct mlx4_en_dev *endev = ctx; + + return endev->pndev[port]; +} + static void mlx4_en_event(struct mlx4_dev *dev, void *endev_ptr, enum mlx4_dev_event event, int port) { @@ -229,9 +236,11 @@ err_free_res: } static struct mlx4_interface mlx4_en_interface = { - .add = mlx4_en_add, - .remove = mlx4_en_remove, - .event = mlx4_en_event, + .add = mlx4_en_add, + .remove = mlx4_en_remove, + .event = mlx4_en_event, + .get_prot_dev = get_netdev, + .protocol = MLX4_PROT_EN, }; static int __init mlx4_en_init(void) diff --git a/drivers/net/mlx4/en_port.c b/drivers/net/mlx4/en_port.c index a29abe8..a249887 100644 --- a/drivers/net/mlx4/en_port.c +++ b/drivers/net/mlx4/en_port.c @@ -127,8 +127,8 @@ int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn, memset(context, 0, sizeof *context); context->base_qpn = cpu_to_be32(base_qpn); - context->promisc = cpu_to_be32(promisc << SET_PORT_PROMISC_SHIFT | base_qpn); - context->mcast = cpu_to_be32(1 << SET_PORT_PROMISC_SHIFT | base_qpn); + context->promisc = cpu_to_be32(promisc << SET_PORT_PROMISC_EN_SHIFT | base_qpn); + context->mcast = cpu_to_be32(1 << SET_PORT_PROMISC_MODE_SHIFT | base_qpn); context->intra_no_vlan = 0; context->no_vlan = MLX4_NO_VLAN_IDX; context->intra_vlan_miss = 0; diff --git a/drivers/net/mlx4/en_port.h b/drivers/net/mlx4/en_port.h index e6477f1..9354891 100644 --- a/drivers/net/mlx4/en_port.h +++ b/drivers/net/mlx4/en_port.h @@ -36,7 +36,8 @@ #define SET_PORT_GEN_ALL_VALID 0x7 -#define SET_PORT_PROMISC_SHIFT 31 +#define SET_PORT_PROMISC_EN_SHIFT 31 +#define SET_PORT_PROMISC_MODE_SHIFT 30 enum { MLX4_CMD_SET_VLAN_FLTR = 0x47, diff --git a/drivers/net/mlx4/intf.c b/drivers/net/mlx4/intf.c index 0e7eb10..d64530e 100644 --- a/drivers/net/mlx4/intf.c +++ b/drivers/net/mlx4/intf.c @@ -159,3 +159,23 @@ void mlx4_unregister_device(struct mlx4_dev *dev) mutex_unlock(&intf_mutex); } + +void *mlx4_find_get_prot_dev(struct mlx4_dev *dev, enum mlx4_prot proto, int port) +{ + struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_device_context *dev_ctx; + unsigned long flags; + void *result = NULL; + + spin_lock_irqsave(&priv->ctx_lock, flags); + + list_for_each_entry(dev_ctx, &priv->ctx_list, list) + if (dev_ctx->intf->protocol == proto && dev_ctx->intf->get_prot_dev) { + result = dev_ctx->intf->get_prot_dev(dev, dev_ctx->context, port); + break; + } + + spin_unlock_irqrestore(&priv->ctx_lock, flags); + + return result; +} diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c index 30bea96..c72af51 100644 --- a/drivers/net/mlx4/main.c +++ b/drivers/net/mlx4/main.c @@ -100,6 +100,12 @@ module_param_named(use_prio, use_prio, bool, 0444); MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports " "(0/1, default 0)"); +void *mlx4_get_prot_dev(struct mlx4_dev *dev, enum mlx4_prot proto, int port) +{ + return mlx4_find_get_prot_dev(dev, proto, port); +} +EXPORT_SYMBOL(mlx4_get_prot_dev); + int mlx4_check_port_params(struct mlx4_dev *dev, enum mlx4_port_type *port_type) { diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h index 5bd79c2..db068c9 100644 --- a/drivers/net/mlx4/mlx4.h +++ b/drivers/net/mlx4/mlx4.h @@ -364,6 +364,7 @@ int mlx4_restart_one(struct pci_dev *pdev); int mlx4_register_device(struct mlx4_dev *dev); void mlx4_unregister_device(struct mlx4_dev *dev); void mlx4_dispatch_event(struct mlx4_dev *dev, enum mlx4_dev_event type, int port); +void *mlx4_find_get_prot_dev(struct mlx4_dev *dev, enum mlx4_prot proto, int port); struct mlx4_dev_cap; struct mlx4_init_hca_param; diff --git a/include/linux/mlx4/cmd.h b/include/linux/mlx4/cmd.h index 0f82293..22bd8d3 100644 --- a/include/linux/mlx4/cmd.h +++ b/include/linux/mlx4/cmd.h @@ -140,6 +140,7 @@ enum { MLX4_SET_PORT_MAC_TABLE = 0x2, MLX4_SET_PORT_VLAN_TABLE = 0x3, MLX4_SET_PORT_PRIO_MAP = 0x4, + MLX4_SET_PORT_GID_TABLE = 0x5, }; struct mlx4_dev; diff --git a/include/linux/mlx4/driver.h b/include/linux/mlx4/driver.h index 53c5fdb..0083256 100644 --- a/include/linux/mlx4/driver.h +++ b/include/linux/mlx4/driver.h @@ -44,15 +44,23 @@ enum mlx4_dev_event { MLX4_DEV_EVENT_PORT_REINIT, }; +enum mlx4_prot { + MLX4_PROT_IB, + MLX4_PROT_EN, +}; + struct mlx4_interface { - void * (*add) (struct mlx4_dev *dev); - void (*remove)(struct mlx4_dev *dev, void *context); - void (*event) (struct mlx4_dev *dev, void *context, - enum mlx4_dev_event event, int port); + void * (*add) (struct mlx4_dev *dev); + void (*remove)(struct mlx4_dev *dev, void *context); + void (*event) (struct mlx4_dev *dev, void *context, + enum mlx4_dev_event event, int port); + void * (*get_prot_dev) (struct mlx4_dev *dev, void *context, u8 port); + enum mlx4_prot protocol; struct list_head list; }; int mlx4_register_interface(struct mlx4_interface *intf); void mlx4_unregister_interface(struct mlx4_interface *intf); +void *mlx4_get_prot_dev(struct mlx4_dev *dev, enum mlx4_prot proto, int port); #endif /* MLX4_DRIVER_H */ -- 1.6.3.3 From hal.rosenstock at gmail.com Mon Jul 13 12:26:06 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 13 Jul 2009 15:26:06 -0400 Subject: [ofa-general] Re: [ewg] [PATCH 1/8 v3] ib_core: Add API to support RDMAoE In-Reply-To: <20090713181319.GA738@mtls03> References: <20090713181319.GA738@mtls03> Message-ID: On Mon, Jul 13, 2009 at 2:13 PM, Eli Cohen wrote: > Add two API functions needed for RDMAoE. > > ib_get_port_link_type() returns the link type support by the given device's > port. It can be either PORT_LINK_IB for IB link layer or PORT_LINK_ETH for > Ethernet links. Link type is reported to in query_port verb. > > ib_get_mac() will return the Ethernet MAC address leading to the port whose GID > is spcified. This function is exported to userspace applications. > > ABI version is incremented from 6 to 7. > > Signed-off-by: Eli Cohen > --- >  drivers/infiniband/core/uverbs.h      |    1 + >  drivers/infiniband/core/uverbs_cmd.c  |   33 +++++++++++++++++++++++++++++++++ >  drivers/infiniband/core/uverbs_main.c |    1 + >  drivers/infiniband/core/verbs.c       |   17 +++++++++++++++++ >  include/rdma/ib_user_verbs.h          |   21 ++++++++++++++++++--- >  include/rdma/ib_verbs.h               |   22 ++++++++++++++++++++++ >  6 files changed, 92 insertions(+), 3 deletions(-) > > diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h > index b3ea958..e69b04c 100644 > --- a/drivers/infiniband/core/uverbs.h > +++ b/drivers/infiniband/core/uverbs.h > @@ -194,5 +194,6 @@ IB_UVERBS_DECLARE_CMD(create_srq); >  IB_UVERBS_DECLARE_CMD(modify_srq); >  IB_UVERBS_DECLARE_CMD(query_srq); >  IB_UVERBS_DECLARE_CMD(destroy_srq); > +IB_UVERBS_DECLARE_CMD(get_mac); > >  #endif /* UVERBS_H */ > diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c > index 56feab6..eefc414 100644 > --- a/drivers/infiniband/core/uverbs_cmd.c > +++ b/drivers/infiniband/core/uverbs_cmd.c > @@ -452,6 +452,7 @@ ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, >        resp.active_width    = attr.active_width; >        resp.active_speed    = attr.active_speed; >        resp.phys_state      = attr.phys_state; > +       resp.link_type       = attr.link_type; > >        if (copy_to_user((void __user *) (unsigned long) cmd.response, >                         &resp, sizeof resp)) > @@ -1824,6 +1825,38 @@ err: >        return ret; >  } > > +ssize_t ib_uverbs_get_mac(struct ib_uverbs_file *file, > +                         const char __user *buf, int in_len, > +                         int out_len) > +{ > +       struct ib_uverbs_get_mac        cmd; > +       struct ib_uverbs_get_mac_resp   resp; > +       int              ret; > +       struct ib_pd    *pd; > + > +       if (out_len < sizeof resp) > +               return -ENOSPC; > + > +       if (copy_from_user(&cmd, buf, sizeof cmd)) > +               return -EFAULT; > + > +       pd = idr_read_pd(cmd.pd_handle, file->ucontext); > +       if (!pd) > +               return -EINVAL; > + > +       ret = ib_get_mac(pd->device, cmd.port, cmd.gid, resp.mac); > +       put_pd_read(pd); > +       if (!ret) { > +               if (copy_to_user((void __user *) (unsigned long) cmd.response, > +                                &resp, sizeof resp)) { > +                       return -EFAULT; > +               } > +               return in_len; > +       } > + > +       return ret; > +} > + >  ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, >                             const char __user *buf, int in_len, int out_len) >  { > diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c > index eb36a81..b2f148f 100644 > --- a/drivers/infiniband/core/uverbs_main.c > +++ b/drivers/infiniband/core/uverbs_main.c > @@ -108,6 +108,7 @@ static ssize_t (*uverbs_cmd_table[])(struct ib_uverbs_file *file, >        [IB_USER_VERBS_CMD_MODIFY_SRQ]          = ib_uverbs_modify_srq, >        [IB_USER_VERBS_CMD_QUERY_SRQ]           = ib_uverbs_query_srq, >        [IB_USER_VERBS_CMD_DESTROY_SRQ]         = ib_uverbs_destroy_srq, > +       [IB_USER_VERBS_CMD_GET_MAC]             = ib_uverbs_get_mac >  }; > >  static struct vfsmount *uverbs_event_mnt; > diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c > index a7da9be..bde5b0d 100644 > --- a/drivers/infiniband/core/verbs.c > +++ b/drivers/infiniband/core/verbs.c > @@ -94,6 +94,13 @@ rdma_node_get_transport(enum rdma_node_type node_type) >  } >  EXPORT_SYMBOL(rdma_node_get_transport); > > +enum ib_port_link_type ib_get_port_link_type(struct ib_device *device, u8 port_num) > +{ > +       return device->get_port_link_type ? > +               device->get_port_link_type(device, port_num) : PORT_LINK_IB; So do iWARP devices return PORT_LINK_IB ? If so, that seems a little weird to me. -- Hal > +} > +EXPORT_SYMBOL(ib_get_port_link_type); > + >  /* Protection domains */ > >  struct ib_pd *ib_alloc_pd(struct ib_device *device) > @@ -904,3 +911,13 @@ int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) >        return qp->device->detach_mcast(qp, gid, lid); >  } >  EXPORT_SYMBOL(ib_detach_mcast); > + > +int ib_get_mac(struct ib_device *device, u8 port, u8 *gid, u8 *mac) > +{ > +       if (!device->get_mac) > +               return -ENOSYS; > + > +       return device->get_mac(device, port, gid, mac); > +} > +EXPORT_SYMBOL(ib_get_mac); > + > diff --git a/include/rdma/ib_user_verbs.h b/include/rdma/ib_user_verbs.h > index a17f771..184203b 100644 > --- a/include/rdma/ib_user_verbs.h > +++ b/include/rdma/ib_user_verbs.h > @@ -42,7 +42,7 @@ >  * Increment this value if any changes that break userspace ABI >  * compatibility are made. >  */ > -#define IB_USER_VERBS_ABI_VERSION      6 > +#define IB_USER_VERBS_ABI_VERSION      7 > >  enum { >        IB_USER_VERBS_CMD_GET_CONTEXT, > @@ -81,7 +81,8 @@ enum { >        IB_USER_VERBS_CMD_MODIFY_SRQ, >        IB_USER_VERBS_CMD_QUERY_SRQ, >        IB_USER_VERBS_CMD_DESTROY_SRQ, > -       IB_USER_VERBS_CMD_POST_SRQ_RECV > +       IB_USER_VERBS_CMD_POST_SRQ_RECV, > +       IB_USER_VERBS_CMD_GET_MAC >  }; > >  /* > @@ -205,7 +206,8 @@ struct ib_uverbs_query_port_resp { >        __u8  active_width; >        __u8  active_speed; >        __u8  phys_state; > -       __u8  reserved[3]; > +       __u8  link_type; > +       __u8  reserved[2]; >  }; > >  struct ib_uverbs_alloc_pd { > @@ -621,6 +623,19 @@ struct ib_uverbs_destroy_ah { >        __u32 ah_handle; >  }; > > +struct ib_uverbs_get_mac { > +       __u64 response; > +       __u32 pd_handle; > +       __u8  port; > +       __u8  reserved[3]; > +       __u8  gid[16]; > +}; > + > +struct ib_uverbs_get_mac_resp { > +       __u8    mac[6]; > +       __u16   reserved; > +}; > + >  struct ib_uverbs_attach_mcast { >        __u8  gid[16]; >        __u32 qp_handle; > diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h > index c179318..9b3d43b 100644 > --- a/include/rdma/ib_verbs.h > +++ b/include/rdma/ib_verbs.h > @@ -72,6 +72,11 @@ enum rdma_transport_type { >        RDMA_TRANSPORT_IWARP >  }; > > +enum ib_port_link_type { > +       PORT_LINK_IB, > +       PORT_LINK_ETH > +}; > + >  enum rdma_transport_type >  rdma_node_get_transport(enum rdma_node_type node_type) __attribute_const__; > > @@ -298,6 +303,7 @@ struct ib_port_attr { >        u8                      active_width; >        u8                      active_speed; >        u8                      phys_state; > +       enum ib_port_link_type  link_type; >  }; > >  enum ib_device_modify_flags { > @@ -1003,6 +1009,8 @@ struct ib_device { >        int                        (*query_port)(struct ib_device *device, >                                                 u8 port_num, >                                                 struct ib_port_attr *port_attr); > +       enum ib_port_link_type     (*get_port_link_type)(struct ib_device *device, > +                                                        u8 port_num); >        int                        (*query_gid)(struct ib_device *device, >                                                u8 port_num, int index, >                                                union ib_gid *gid); > @@ -1130,6 +1138,8 @@ struct ib_device { >                                                  struct ib_grh *in_grh, >                                                  struct ib_mad *in_mad, >                                                  struct ib_mad *out_mad); > +       int                        (*get_mac)(struct ib_device *device, u8 port, > +                                             u8 *gid, u8 *mac); > >        struct ib_dma_mapping_ops   *dma_ops; > > @@ -1213,6 +1223,9 @@ int ib_query_device(struct ib_device *device, >  int ib_query_port(struct ib_device *device, >                  u8 port_num, struct ib_port_attr *port_attr); > > +enum ib_port_link_type ib_get_port_link_type(struct ib_device *device, > +                                            u8 port_num); > + >  int ib_query_gid(struct ib_device *device, >                 u8 port_num, int index, union ib_gid *gid); > > @@ -2031,4 +2044,13 @@ int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); >  */ >  int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); > > +/** > + * ib_get_mac - get the mac address for the specified gid > + * @device: IB device used for traffic > + * @port: port number used. > + * @gid: gid to be resolved into mac > + * @mac: mac of the port bearing this gid > + */ > +int ib_get_mac(struct ib_device *device, u8 port, u8 *gid, u8 *mac); > + >  #endif /* IB_VERBS_H */ > -- > 1.6.3.3 > > _______________________________________________ > ewg mailing list > ewg at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg > From hal.rosenstock at gmail.com Mon Jul 13 12:26:34 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 13 Jul 2009 15:26:34 -0400 Subject: [ofa-general] Re: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support only QP1 In-Reply-To: <20090713181410.GA754@mtls03> References: <20090713181410.GA754@mtls03> Message-ID: On Mon, Jul 13, 2009 at 2:14 PM, Eli Cohen wrote: > Since RDMAoE is using Ethernet as its link layer, there is no need for QP0. QP1 > is still needed since it handles communications between CM agents. This patch > will create only QP1 for RDMAoE ports. What happens with other QP1 traffic (other than CM and SA) ? Userspace can access QP1 (and QP0). Does QP0 error out ? What about QP1 ? Does it just timeout ? If so, a direct error would be better. -- Hal > Signed-off-by: Eli Cohen > --- >  drivers/infiniband/core/agent.c |   12 ++++++--- >  drivers/infiniband/core/mad.c   |   48 ++++++++++++++++++++++++++++++--------- >  2 files changed, 45 insertions(+), 15 deletions(-) > > diff --git a/drivers/infiniband/core/agent.c b/drivers/infiniband/core/agent.c > index ae7c288..c3f2048 100644 > --- a/drivers/infiniband/core/agent.c > +++ b/drivers/infiniband/core/agent.c > @@ -48,6 +48,8 @@ >  struct ib_agent_port_private { >        struct list_head port_list; >        struct ib_mad_agent *agent[2]; > +       struct ib_device    *device; > +       u8                   port_num; >  }; > >  static DEFINE_SPINLOCK(ib_agent_port_list_lock); > @@ -58,11 +60,10 @@ __ib_get_agent_port(struct ib_device *device, int port_num) >  { >        struct ib_agent_port_private *entry; > > -       list_for_each_entry(entry, &ib_agent_port_list, port_list) { > -               if (entry->agent[0]->device == device && > -                   entry->agent[0]->port_num == port_num) > +       list_for_each_entry(entry, &ib_agent_port_list, port_list) > +               if (entry->device == device && entry->port_num == port_num) >                        return entry; > -       } > + >        return NULL; >  } > > @@ -175,6 +176,9 @@ int ib_agent_port_open(struct ib_device *device, int port_num) >                goto error3; >        } > > +       port_priv->device = device; > +       port_priv->port_num = port_num; > + >        spin_lock_irqsave(&ib_agent_port_list_lock, flags); >        list_add_tail(&port_priv->port_list, &ib_agent_port_list); >        spin_unlock_irqrestore(&ib_agent_port_list_lock, flags); > diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c > index de922a0..3d5449f 100644 > --- a/drivers/infiniband/core/mad.c > +++ b/drivers/infiniband/core/mad.c > @@ -199,6 +199,16 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device, >        unsigned long flags; >        u8 mgmt_class, vclass; > > +       /* Validate device and port */ > +       port_priv = ib_get_mad_port(device, port_num); > +       if (!port_priv) { > +               ret = ERR_PTR(-ENODEV); > +               goto error1; > +       } > + > +       if (!port_priv->qp_info[qp_type].qp) > +               return NULL; > + >        /* Validate parameters */ >        qpn = get_spl_qp_index(qp_type); >        if (qpn == -1) > @@ -260,13 +270,6 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device, >                        goto error1; >        } > > -       /* Validate device and port */ > -       port_priv = ib_get_mad_port(device, port_num); > -       if (!port_priv) { > -               ret = ERR_PTR(-ENODEV); > -               goto error1; > -       } > - >        /* Allocate structures */ >        mad_agent_priv = kzalloc(sizeof *mad_agent_priv, GFP_KERNEL); >        if (!mad_agent_priv) { > @@ -556,6 +559,9 @@ int ib_unregister_mad_agent(struct ib_mad_agent *mad_agent) >        struct ib_mad_agent_private *mad_agent_priv; >        struct ib_mad_snoop_private *mad_snoop_priv; > > +       if (!mad_agent) > +               return 0; > + >        /* If the TID is zero, the agent can only snoop. */ >        if (mad_agent->hi_tid) { >                mad_agent_priv = container_of(mad_agent, > @@ -2602,6 +2608,9 @@ static void cleanup_recv_queue(struct ib_mad_qp_info *qp_info) >        struct ib_mad_private *recv; >        struct ib_mad_list_head *mad_list; > > +       if (!qp_info->qp) > +               return; > + >        while (!list_empty(&qp_info->recv_queue.list)) { > >                mad_list = list_entry(qp_info->recv_queue.list.next, > @@ -2643,6 +2652,9 @@ static int ib_mad_port_start(struct ib_mad_port_private *port_priv) > >        for (i = 0; i < IB_MAD_QPS_CORE; i++) { >                qp = port_priv->qp_info[i].qp; > +               if (!qp) > +                       continue; > + >                /* >                 * PKey index for QP1 is irrelevant but >                 * one is needed for the Reset to Init transition > @@ -2684,6 +2696,9 @@ static int ib_mad_port_start(struct ib_mad_port_private *port_priv) >        } > >        for (i = 0; i < IB_MAD_QPS_CORE; i++) { > +               if (!port_priv->qp_info[i].qp) > +                       continue; > + >                ret = ib_mad_post_receive_mads(&port_priv->qp_info[i], NULL); >                if (ret) { >                        printk(KERN_ERR PFX "Couldn't post receive WRs\n"); > @@ -2762,6 +2777,9 @@ error: > >  static void destroy_mad_qp(struct ib_mad_qp_info *qp_info) >  { > +       if (!qp_info->qp) > +               return; > + >        ib_destroy_qp(qp_info->qp); >        kfree(qp_info->snoop_table); >  } > @@ -2777,6 +2795,7 @@ static int ib_mad_port_open(struct ib_device *device, >        struct ib_mad_port_private *port_priv; >        unsigned long flags; >        char name[sizeof "ib_mad123"]; > +       int has_smi; > >        /* Create new device info */ >        port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL); > @@ -2793,6 +2812,10 @@ static int ib_mad_port_open(struct ib_device *device, >        init_mad_qp(port_priv, &port_priv->qp_info[1]); > >        cq_size = (IB_MAD_QP_SEND_SIZE + IB_MAD_QP_RECV_SIZE) * 2; > +       has_smi = ib_get_port_link_type(device, port_num) == PORT_LINK_IB; > +       if (has_smi) > +               cq_size *= 2; > + >        port_priv->cq = ib_create_cq(port_priv->device, >                                     ib_mad_thread_completion_handler, >                                     NULL, port_priv, cq_size, 0); > @@ -2816,9 +2839,11 @@ static int ib_mad_port_open(struct ib_device *device, >                goto error5; >        } > > -       ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI); > -       if (ret) > -               goto error6; > +       if (has_smi) { > +               ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI); > +               if (ret) > +                       goto error6; > +       } >        ret = create_mad_qp(&port_priv->qp_info[1], IB_QPT_GSI); >        if (ret) >                goto error7; > @@ -2852,7 +2877,8 @@ error9: >  error8: >        destroy_mad_qp(&port_priv->qp_info[1]); >  error7: > -       destroy_mad_qp(&port_priv->qp_info[0]); > +       if (has_smi) > +               destroy_mad_qp(&port_priv->qp_info[0]); >  error6: >        ib_dereg_mr(port_priv->mr); >  error5: > -- > 1.6.3.3 > > _______________________________________________ > ewg mailing list > ewg at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg > From rdreier at cisco.com Mon Jul 13 12:45:32 2009 From: rdreier at cisco.com (Roland Dreier) Date: Mon, 13 Jul 2009 12:45:32 -0700 Subject: [ofa-general][PATCH] mlx4_core: Synch catastrophic flow with module unload In-Reply-To: <20090713.111407.196601373.davem@davemloft.net> (David Miller's message of "Mon, 13 Jul 2009 11:14:07 -0700 (PDT)") References: <4A5B5274.2020801@mellanox.co.il> <20090713.111407.196601373.davem@davemloft.net> Message-ID: > Applied, thanks. Dave, please don't apply mlx4_core patches without giving me a chance to review them. In this case the patch looks buggy to me: I don't see how it handles, say, hot remove of one device -- it only handles module removal. And I would hope we could fix this without adding a global symbol as namespace polluting as "drv_mutex". Yevgeny didn't even send this patch to you; he just cc'ed netdev as a courtesy. However I understand that the physical location of mlx4_core in drivers/net makes it easy to do this. Maybe this is the best argument in favor of moving the mlx4_core stuff to drivers/shared? - R. From jon at opengridcomputing.com Mon Jul 13 12:48:11 2009 From: jon at opengridcomputing.com (Jon Mason) Date: Mon, 13 Jul 2009 14:48:11 -0500 Subject: [ofa-general] OFED 1.4.1 breaks SLES10 SP2 NFS server ? In-Reply-To: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> References: <7391130E01ED404FBD7A3C86731EEB7D3F735FA85E@GVW1087EXB.americas.hpqcorp.net> Message-ID: <20090713194811.GM24846@opengridcomputing.com> On Sun, Jul 12, 2009 at 10:20:58AM +0000, Ciesielski, Frederic (EMEA HPC&OSLO CC) wrote: > As far as I could see, x86-64 systems running SLES10 SP2 + OFED 1.4.1 are not valid NFS servers anymore, even without trying to activate NFS-RDMA, even without using IB at all for the export. > > NFS clients see too frequently 'Stale NFS file handle' error messages for them to properly read or write anything. > > This happens using ethernet and IPoIB... and works fine without OFED, or when OFED 1.4 is installed instead (which is not what I want). > > Did anybody test that ? > Any idea about how to get rid of this major side effect ? The problem is specific to non-ext filesystems (specifically reiserfs in this case). There is a bug in the NFS backports for SLES10 and RHEL5, which should be fixed in OFED 1.4.2 and OFED 1.5. This issue was not detected due to testing only on ext3 on SLES10sp2. If you use ext3 and OFED 1.4.1, you should not see the issue. Thanks, Jon > > Thanks. > Fred. > > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From rdreier at cisco.com Mon Jul 13 12:51:06 2009 From: rdreier at cisco.com (Roland Dreier) Date: Mon, 13 Jul 2009 12:51:06 -0700 Subject: [ofa-general] Re: [PATCH] mlx4: Do not allow ib userspace open while device is being removed In-Reply-To: <200907121628.08412.jackm@dev.mellanox.co.il> (Jack Morgenstein's message of "Sun, 12 Jul 2009 16:28:08 +0300") References: <200907121628.08412.jackm@dev.mellanox.co.il> Message-ID: > Userspace apps are supposed to release all ib device resources if > they receive a fatal async event (IBV_EVENT_DEVICE_FATAL). However, > the app has no way of knowing when the device has come back up, except > to repeatedly attempt ibv_open_device() until it succeeds. > > However, currently there is no protection against open succeeding when > the device is in the midst of the removal following the fatal event. > In this case, the open will succeed, but as a result the device waits > in the middle of its removal until the new app releases its ib resources > -- and the new app will not do so, since the open succeeded at a point > following the fatal event generation. Does mthca have this bug too? > --- a/include/linux/mlx4/device.h > +++ b/include/linux/mlx4/device.h > @@ -379,6 +379,7 @@ struct mlx4_dev { > struct radix_tree_root qp_table_tree; > u32 rev_id; > char board_id[MLX4_BOARD_ID_LEN]; > + u32 ib_active; > }; > > struct mlx4_init_port_param { I don't much like this approach of putting an IB-driver-specific flag into the generic mlx4 data structure. Why can't the IB driver manage this in its own data structure? > @@ -698,6 +713,8 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr) > struct mlx4_ib_dev *ibdev = ibdev_ptr; > int p; > > + dev->ib_active = 0; > + > mlx4_ib_mad_cleanup(ibdev); > ib_unregister_device(&ibdev->ib_dev); This just seems silly -- what is setting ib_active to 0 protecting against? The ib_unregister_device() call is going make sure we have no clients left anyway, isn't it? - R. From davem at davemloft.net Mon Jul 13 13:09:36 2009 From: davem at davemloft.net (David Miller) Date: Mon, 13 Jul 2009 13:09:36 -0700 (PDT) Subject: [ofa-general][PATCH] mlx4_core: Synch catastrophic flow with module unload In-Reply-To: References: <4A5B5274.2020801@mellanox.co.il> <20090713.111407.196601373.davem@davemloft.net> Message-ID: <20090713.130936.41603615.davem@davemloft.net> From: Roland Dreier Date: Mon, 13 Jul 2009 12:45:32 -0700 > > Applied, thanks. > > Dave, please don't apply mlx4_core patches without giving me a chance to > review them. In this case the patch looks buggy to me: I don't see how > it handles, say, hot remove of one device -- it only handles module > removal. And I would hope we could fix this without adding a global > symbol as namespace polluting as "drv_mutex". > > Yevgeny didn't even send this patch to you; he just cc'ed netdev as a > courtesy. However I understand that the physical location of mlx4_core > in drivers/net makes it easy to do this. Maybe this is the best > argument in favor of moving the mlx4_core stuff to drivers/shared? If it gets sent to netdev, it's for a networking driver, and it says "PATCH" rather than "RFC" or "please review" or "don't apply" you cannot reasonably expect me to not look into applying the thing. And if you're saying that patches for this device should start not going through me, and the tactic to accomplish that is to move the bulk of the driver into some driver/shared area, that's really weird. Anyways I didn't push the patch out to kernel.org yet so it's easy for me to remove it. From jgunthorpe at obsidianresearch.com Mon Jul 13 13:20:06 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 13 Jul 2009 14:20:06 -0600 Subject: [ofa-general] patch to ib_addr for sending arps In-Reply-To: <4A5B6B5D.6070605@oracle.com> References: <4A5527A1.5010104@oracle.com> <4A59C576.5060709@voltaire.com> <4A5AAC3E.2040609@oracle.com> <20090713134906.GA19062@obsidianresearch.com> <4A5B6B5D.6070605@oracle.com> Message-ID: <20090713202006.GB21867@obsidianresearch.com> On Mon, Jul 13, 2009 at 10:14:05AM -0700, leo.tominna at oracle.com wrote: > Hi Jason, > > Thanks for clearing up the use case. In that case doing ip_dev_find to set oif > would be wrong since it would not work correctly in the case the same IP is > associated with two devices. By just setting s_addr before calling > ip_route_output_key in addr_send_arp, that should take care of it (the initial > patch sent). > > From what I can tell, this just fixes the policy routing case, without > affecting/addressing configurations that are using default routing. I need to > see why RDS/IB gets stuck in this case. My guess is that hardware addresses > don't get resolved correctly (as expected), and two sides of an IB connection > trip over a mismatch in what hardware a peer thinks its using. > > But that is another issue that can be fixed independently. I'll add some > prints to see what might be happening. So, I think they might be related, at least, the current arrangement seems straneg to my eyes. There should be only one route lookup and it should not be in the send_arp function. The ip_dev_find (and related) in addr_resolve_local is the main culprit.. As far as I can see there should be one call to the route function (ip_route_output_key??) and that result should replace ip_dev_find and the dst and fl, etc should be passed down to send_arp and the other places that are calling ip_route_output_key. (This is the original problem I noted in that old thread) Multiple lookups like this seem like they should be racy against table updates. So you patch makes the arp part use a potentialy different device than the bind part, which is no good.. (yes?) Really, the best thing to do here is carefully look at the tcp and udp call paths and duplicate their function calls into the route module. It should be the same logic flow. Jason From leo.tominna at oracle.com Mon Jul 13 13:35:08 2009 From: leo.tominna at oracle.com (leo.tominna at oracle.com) Date: Mon, 13 Jul 2009 13:35:08 -0700 Subject: [ofa-general] patch to ib_addr for sending arps In-Reply-To: <20090713202006.GB21867@obsidianresearch.com> References: <4A5527A1.5010104@oracle.com> <4A59C576.5060709@voltaire.com> <4A5AAC3E.2040609@oracle.com> <20090713134906.GA19062@obsidianresearch.com> <4A5B6B5D.6070605@oracle.com> <20090713202006.GB21867@obsidianresearch.com> Message-ID: <4A5B9A7C.4060000@oracle.com> Right, there should only be one route lookup call. And the send_arp should match what TCP/UDP are doing, I'm pretty sure they don't use neigh_event_send like ib_addr is, or if they do, they are not using ip_route_output_key to get the neighbor entry. I could not find the code that generates arps for these protocols. addr_resolve_local should do the same as you said, although I didn't exercise this code when testing, I was testing with a single/stable remote node's IP. I'll see if there's a better fix then. Thanks for your help. Leo Tominna On 7/13/2009 1:20 PM, Jason Gunthorpe wrote: > On Mon, Jul 13, 2009 at 10:14:05AM -0700, leo.tominna at oracle.com wrote: > >> Hi Jason, >> >> Thanks for clearing up the use case. In that case doing ip_dev_find to set oif >> would be wrong since it would not work correctly in the case the same IP is >> associated with two devices. By just setting s_addr before calling >> ip_route_output_key in addr_send_arp, that should take care of it (the initial >> patch sent). >> >> From what I can tell, this just fixes the policy routing case, without >> affecting/addressing configurations that are using default routing. I need to >> see why RDS/IB gets stuck in this case. My guess is that hardware addresses >> don't get resolved correctly (as expected), and two sides of an IB connection >> trip over a mismatch in what hardware a peer thinks its using. >> >> But that is another issue that can be fixed independently. I'll add some >> prints to see what might be happening. >> > > So, I think they might be related, at least, the current arrangement > seems straneg to my eyes. > > There should be only one route lookup and it should not be in the > send_arp function. > > The ip_dev_find (and related) in addr_resolve_local is the main > culprit.. As far as I can see there should be one call to the route > function (ip_route_output_key??) and that result should replace > ip_dev_find and the dst and fl, etc should be passed down to send_arp > and the other places that are calling ip_route_output_key. (This is > the original problem I noted in that old thread) > > Multiple lookups like this seem like they should be racy against table > updates. > > So you patch makes the arp part use a potentialy different device than > the bind part, which is no good.. (yes?) > > Really, the best thing to do here is carefully look at the tcp and udp > call paths and duplicate their function calls into the route > module. It should be the same logic flow. > > Jason > -------------- next part -------------- An HTML attachment was scrubbed... URL: From jon at opengridcomputing.com Mon Jul 13 13:40:49 2009 From: jon at opengridcomputing.com (Jon Mason) Date: Mon, 13 Jul 2009 15:40:49 -0500 Subject: [ofa-general] Installing nfs-rdma with OFED-1.4.1 In-Reply-To: <7b160d240906260731o4e47d19bwa492771a71258f32@mail.gmail.com> References: <7b160d240906260335l59ca2ff2j1be2df02c0f38c4a@mail.gmail.com> <7b160d240906260425i1ddf7083yc63c4e803d0cafe4@mail.gmail.com> <7b160d240906260606r2fd4c483lb66c248e3163bcd@mail.gmail.com> <7b160d240906260731o4e47d19bwa492771a71258f32@mail.gmail.com> Message-ID: <20090713204049.GN24846@opengridcomputing.com> On Fri, Jun 26, 2009 at 03:31:50PM +0100, Ross Smith wrote: > Latest news: I've bodged the installation, but I'm having problems > with nfs.mount now. > > Since everything worked with nfs-rdma on the development server, I > manually copied the kerel-ib rpm file from there, and installed that > on top of the default infiniband install on the live server, using the > command: > > # rpm -iv --replacepkgs kernel-ib....... > > With that done modprobe xprtrdma now works. > > Since the kernel-ib file appears to be the only one modified as > nfsrdma support is added, I assume this is all that's needed. Can > anybody confirm if this is ok, or whether I'm going to break things > horribly by doing this? > > I also have nfs v1.1.6 installed, but it doesn't seem to understand > the rdma option. The system is still using tcp by default, and if I > try to use rdma, I get this: > > # mount -o rdma, port=20049 server:path /mnt > mount.nfs: Unsupported nfs mount option: rdma > > However, nfs is definitely the correct version: > # mount.nfs -V > mount.nfs (linux nfs-utils 1.1.6) > > Even using the mount.rnfs utility supplied with OFED doesn't help: > # rpm -iv rnfs-utils-1.1.5-2.OFED.i386.rpm > # mount.rnfs server:path /mnt -o rdma,port=20049 > mount.rnfs: Unsupported nfs mount option: rdma You need the "-i" option for older kernels. In this case, you would want: mount.rnfs server:path /mnt -i -o rdma,port=20049 > > And if I use mount.rnfs without the rdma option, "cat /proc/mounts" > reports that it's just using tcp. > > I'm getting read speeds of around 400MB/s using regular NFS which > isn't bad, but I know I can achieve over 900MB/s with this server over > Infiniband, so there's still some way to go. > > Ross > > > > On Fri, Jun 26, 2009 at 2:06 PM, Ross Smith wrote: > > Ok, I've made a little more progress. > > > > The issue with nfs-utils was solved by installing efsprogs-devel, and > > mount.nfs is now reporting that it's running v1.1.6. > > > > I still can't install OFED with nfs-rdma on the live server though, it > > appears that it's unable to use the rpm package to install with nfs > > support. > > > > I've now discovered how to use the ofed.conf file, and this is the > > procedure I'm following: > > > > - On the development server, I run install.pl and perform a standard install > > - Once that finishes, I edit the resulting ofed.conf and add nfsrdma=y > > - Still on the development server I run: > >  # install.pl -c ofed.conf > > - That can be seen to install most items from the rpm, and to > > recompile kernel-ib with the nfsrdma option. > > - I then package up the OFED folder with tar, and send it to the live server > > - On the live server I run the same command: > >  # install.pl -c ofed.conf > > > > Despite it running with exactly the same options, I get the error: > > > > "/lib/modules/2.6.18......./build/scripts is required to build kernel-ib RPM. > > Please install the corresponding kernel-source or kerenel-devel RPM." > > > > If I exclude the nfsrdma option and repeat all the above steps, the > > script installs fine from the pre-compiled RPM. > > > > It looks to me like a problem with the install script when choosing nfs-rdma. > > > > Does anybody have any suggestions as to how I can work around this? > > Am I able to simply install the RPM files manually? > > > > Ross > > > > > > > > On Fri, Jun 26, 2009 at 12:25 PM, Ross Smith wrote: > >> Thanks Robert, > >> > >> Looking through your notes, we've followed similar steps.  I should > >> mention that I do have Infiniband working on this server, including > >> ipoib.  It's purely the nfs-rdma part I'm struggling with. > >> > >> My latest attempt has been to attempt a custom install, selecting > >> nfs-rdma manually.  That worked fine on the development server, but > >> when I attempted to run it on the live server I had an error: > >> > >> "/lib/modules/....../build/scripts is required to build kernel-ib RPM. > >> Please install the corresponding kernel-source or kernel-devel RPM. > >> tk rpm is required to install ibutils" > >> > >> The problem is, for XenServer, you can't get the kernel-source on the > >> live server, instead Citrix ship a DDK (driver development kit) > >> virtual machine, which is a pre-built system with all the kernel > >> sources you need. > >> > >> OFED builds fine on the DDK machine, and for the default installation, > >> I can compile everything there and then transfer the OFED folder (with > >> the compiled RPM files) to the live server.  At that point install.pl > >> runs fine, without needing to re-compile things. > >> > >> However, it appears that when I choose some elements in the custom > >> install, it is not using these RPM files, and is trying to compile > >> kernel-ib again from scratch. > >> > >> I have attached a file listing the custom modules I have chosen in > >> case it helps. > >> > >> Ross > >> > >> > >> > >> On Fri, Jun 26, 2009 at 11:42 AM, Robert Dunkley wrote: > >>> Hi Ross, > >>> > >>> I installed OFED 1.3.1 (Older I know) with other options OK on a Centos > >>> 5.2 Dom0 (Separate Xen install on Centos, not "Xen Server"), here is > >>> some notes I took (I used the GUI installer) > >>> > >>> Hope this helps, > >>> > >>> Rob > >>> > >>> 1.      Download the OFED package from > >>> http://www.openfabrics.org/downloads/OFED/ > >>> 2.      Extract it: tar -xvzf OFED-1.3.1.tgz > >>> 3.      Check for the dependencies by running ./install.pl and choosing > >>> documentation ("0"). > >>> 4.      Install all required dependencies using yum install package nam. > >>> For Centos dependencies try: yum install gcc libstdc++-devel > >>> libsysfs-devel tcl tcl-devel tk pciutils-devel kernel-devel rpm-build > >>> gcc libtool bison flex tcl-devel swig gcc-c++ libtool kernel-xen-devel > >>> zlib-devel You may need to run the yum command three times (It takes a > >>> few runs for all the dependencies to sort themselves out) > >>> 5.      It is recommended you now upgrade the kernel to make sure the > >>> sources match it. For centos do: yum upgrade kernel (Or yum upgrade > >>> kernel-xen for Xen setups) > >>> 6.      Run ./install.pl again but this time choose to install ("2") and > >>> choose custom install ("4").  See the appendix for a recommendation on > >>> which modules to install. > >>> 7.      Install should now complete, if it fails check the log (Most > >>> likely you have not installed a required software package - go back to > >>> step 3) > >>> 8.      Say yes to configuring IPOIB (If you plan to use it) and > >>> manually set the IP and subnet. When it asks for a network it wants the > >>> base network address (If given an IP of 192.168.10.12 on a 255.255.255.0 > >>> subnet then the "Network" would be 192.168.10.0) > >>> 9.      Reboot > >>> 10.     Check the install using /etc/init.d/openibd status. This should > >>> show one interface for each Infiniband port (First will be "ib0"). You > >>> should also see quite a few modules loaded, the important ones are > >>> (mlx4_ib & mlx4_core - QLogic drivers, mthca - Mellanox driver, ib_core > >>> & ib_addr - core services, rdma_ucm & rdma_cm - RDMA, ib_cm & ib_ipoib - > >>> IPOIB and ib_uverbs - Direct/Verb) > >>> 11.     Run ibstat - This should display the installed devices and their > >>> firmware versions and hardware / GUI IDs. If you need to update the > >>> firmware for Mellanox see > >>> http://www.mellanox.com/support/firmware_download.php I find it easier > >>> to update using Windows and the WinOF software (Windows equivalent of > >>> OFED - Available here: http://www.openfabrics.org/downloads/WinOF/ ). > >>> > >>> -----Original Message----- > >>> From: general-bounces at lists.openfabrics.org > >>> [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Ross Smith > >>> Sent: 26 June 2009 11:36 > >>> To: nfs-rdma-devel at lists.sourceforge.net; general at lists.openfabrics.org > >>> Cc: Yehonatan Yossef > >>> Subject: [ofa-general] Installing nfs-rdma with OFED-1.4.1 > >>> > >>> Hello everyone, > >>> > >>> Can anybody help me get nfs-rdma installed on CentOS 5.2 (actually > >>> Citrix XenServer), with OFED-1.4.1? > >>> > >>> The default install of OFED doesn't include nfs-rdma, and I'm > >>> struggling to work out which components I do and don't need for a > >>> custom build. > >>> > >>> What I would like to include is: > >>> - Core infiniband components > >>> - Drivers for Mellanox Infinihost III Lx (MHES14) > >>> - IPoIB > >>> - nfs-rdma > >>> - troubleshooting and diagnostic tools (ibstat, ibtracert, ibswitches, > >>> ibhosts, etc...) > >>> > >>> My last attempt was following the instructions from OFED 1.4 - I > >>> edited ofed.conf to add nfsrdma=y, and attempted a default install. > >>> However after running install.pl, that change appears to have been > >>> overwritten. > >>> > >>> I am also having problems updating mount.nfs on this machine.  Both > >>> nfs-utils-1.1.6 and nfs-utils-1.2.0 fail to install with the error: > >>> "blkid/blkid.h: No such file or directory", and I cannot yet find a > >>> source for this file. > >>> > >>> Finally, my understanding is that nfs-rdma is in beta status for > >>> CentOS 5.2, but I couldn't see any open bugs on the tracker, are there > >>> any issues I should be aware of? > >>> > >>> thanks, > >>> > >>> Ross > >>> _______________________________________________ > >>> general mailing list > >>> general at lists.openfabrics.org > >>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > >>> > >>> To unsubscribe, please visit > >>> http://openib.org/mailman/listinfo/openib-general > >>> > >>> The SAQ Group > >>> > >>> Registered Office: 18 Chapel Street, Petersfield, Hampshire GU32 3DZ > >>> SAQ is the trading name of SEMTEC Limited. Registered in England & Wales > >>> Company Number: 06481952 > >>> > >>> http://www.saqnet.co.uk AS29219 > >>> > >>> SAQ Group Delivers high quality, honestly priced communication and I.T. services to UK Business. > >>> > >>> Broadband : Domains : Email : Hosting : CoLo : Servers : Racks : Transit : Backups : Managed Networks : Remote Support. > >>> > >>> ISPA Member > >>> > >>> > >> > > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From robert.j.woodruff at intel.com Mon Jul 13 13:58:23 2009 From: robert.j.woodruff at intel.com (Woodruff, Robert J) Date: Mon, 13 Jul 2009 13:58:23 -0700 Subject: [ofa-general] RE: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support only QP1 In-Reply-To: <20090713181410.GA754@mtls03> References: <20090713181410.GA754@mtls03> Message-ID: <382A478CAD40FA4FB46605CF81FE39F435877A24@orsmsx507.amr.corp.intel.com> Eli Cohen wrote, >Since RDMAoE is using Ethernet as its link layer, there is no need for QP0. QP1 >is still needed since it handles communications between CM agents. This patch >will create only QP1 for RDMAoE ports. Trying to emulate IB for mad services is a total hack and not how this new transport should be added into the core. It should be it's own transport type, just like iWarp was added. You should start with adding a new transport type to ib_verbs.h, e.g., --- ib_verbs.h 2009-07-13 09:06:10.000000000 -0400 +++ ib_verbs_new.h 2009-07-14 03:00:23.000000000 -0400 @@ -64,12 +64,14 @@ enum rdma_node_type { RDMA_NODE_IB_CA = 1, RDMA_NODE_IB_SWITCH, RDMA_NODE_IB_ROUTER, - RDMA_NODE_RNIC + RDMA_NODE_RNIC, + RDMA_NODE_IBXOE }; enum rdma_transport_type { RDMA_TRANSPORT_IB, - RDMA_TRANSPORT_IWARP + RDMA_TRANSPORT_IWARP, + RDMA_TRANSPORT_IBXOE }; enum rdma_transport_type From robert.j.woodruff at intel.com Mon Jul 13 14:03:55 2009 From: robert.j.woodruff at intel.com (Woodruff, Robert J) Date: Mon, 13 Jul 2009 14:03:55 -0700 Subject: [ofa-general] RE: [ewg] [PATCH 6/8 v3] IB/ipoib: restrict IPoIB to work on IB ports only In-Reply-To: <20090713181705.GA802@mtls03> References: <20090713181705.GA802@mtls03> Message-ID: <382A478CAD40FA4FB46605CF81FE39F435877A39@orsmsx507.amr.corp.intel.com> Eli Cohen wrote, >We don't want IPoIB to work over RDMAoE since it will give worse performance >than working directly on Ethernet interfaces which are a prerequisite to RDMAoE >anyway. This is another reason why NOT to try to add IBxOE under the IB transport, but rather add it as it's own transport type. We should not need to hack all the InfiniBand ULPs to now have to know the difference between real IB and IBxOE. From jgunthorpe at obsidianresearch.com Mon Jul 13 14:14:47 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 13 Jul 2009 15:14:47 -0600 Subject: [ofa-general] patch to ib_addr for sending arps In-Reply-To: <4A5B9A7C.4060000@oracle.com> References: <4A5527A1.5010104@oracle.com> <4A59C576.5060709@voltaire.com> <4A5AAC3E.2040609@oracle.com> <20090713134906.GA19062@obsidianresearch.com> <4A5B6B5D.6070605@oracle.com> <20090713202006.GB21867@obsidianresearch.com> <4A5B9A7C.4060000@oracle.com> Message-ID: <20090713211447.GC21867@obsidianresearch.com> On Mon, Jul 13, 2009 at 01:35:08PM -0700, leo.tominna at oracle.com wrote: > Right, there should only be one route lookup call. And the send_arp should > match what TCP/UDP are doing, I'm pretty sure they don't use neigh_event_send > like ib_addr is, or if they do, they are not using ip_route_output_key to get > the neighbor entry. I could not find the code that generates arps for these > protocols. They do, but indirectly, dst_output() cals neigh_resolve_output() which calls neigh_event_send() - since there is no real skb in the RDMA routines it seems reasonable to call neigh_event_sent() directly.. ip_route_output_key is used to populate dst in the skb which is fetched by neigh_resolve_output to get the neigh and the neigh has the ofid. Jason From rdreier at cisco.com Mon Jul 13 14:53:59 2009 From: rdreier at cisco.com (Roland Dreier) Date: Mon, 13 Jul 2009 14:53:59 -0700 Subject: [ofa-general][PATCH] mlx4_core: Synch catastrophic flow with module unload In-Reply-To: <20090713.130936.41603615.davem@davemloft.net> (David Miller's message of "Mon, 13 Jul 2009 13:09:36 -0700 (PDT)") References: <4A5B5274.2020801@mellanox.co.il> <20090713.111407.196601373.davem@davemloft.net> <20090713.130936.41603615.davem@davemloft.net> Message-ID: > If it gets sent to netdev, it's for a networking driver, and it says > "PATCH" rather than "RFC" or "please review" or "don't apply" you > cannot reasonably expect me to not look into applying the thing. > And if you're saying that patches for this device should start not > going through me, and the tactic to accomplish that is to move the > bulk of the driver into some driver/shared area, that's really weird. Well, first of all, if a driver, networking or not, has an active maintainer, I would expect you to give that maintainer a chance to look at any not-totally-trivial patches affecting that driver. But in this case, mlx4_core (as opposed to mlx4_en from the same drivers/net/mlx4 directory) really is not a network driver -- it is a low-level multiplexer for access to hardware that really is more InfiniBand than ethernet (with a dash of Fibre Channel thrown in). And yes, I am saying that making it clearer that mlx4_core is not an network driver by moving the source to a more appropriate place does seem to make sense. > Anyways I didn't push the patch out to kernel.org yet so it's easy for > me to remove it. Thanks. - R. From leo.tominna at oracle.com Mon Jul 13 15:10:25 2009 From: leo.tominna at oracle.com (leo.tominna at oracle.com) Date: Mon, 13 Jul 2009 15:10:25 -0700 Subject: [ofa-general] patch to ib_addr for sending arps In-Reply-To: <20090713211447.GC21867@obsidianresearch.com> References: <4A5527A1.5010104@oracle.com> <4A59C576.5060709@voltaire.com> <4A5AAC3E.2040609@oracle.com> <20090713134906.GA19062@obsidianresearch.com> <4A5B6B5D.6070605@oracle.com> <20090713202006.GB21867@obsidianresearch.com> <4A5B9A7C.4060000@oracle.com> <20090713211447.GC21867@obsidianresearch.com> Message-ID: <4A5BB0D1.9090204@oracle.com> Ok thanks. I'll look at these see what is different. Leo Tominna On 7/13/2009 2:14 PM, Jason Gunthorpe wrote: > On Mon, Jul 13, 2009 at 01:35:08PM -0700, leo.tominna at oracle.com wrote: > >> Right, there should only be one route lookup call. And the send_arp should >> match what TCP/UDP are doing, I'm pretty sure they don't use neigh_event_send >> like ib_addr is, or if they do, they are not using ip_route_output_key to get >> the neighbor entry. I could not find the code that generates arps for these >> protocols. >> > > They do, but indirectly, dst_output() cals neigh_resolve_output() > which calls neigh_event_send() - since there is no real skb in the RDMA > routines it seems reasonable to call neigh_event_sent() directly.. > > ip_route_output_key is used to populate dst in the skb which is > fetched by neigh_resolve_output to get the neigh and the neigh has the > ofid. > > Jason > -------------- next part -------------- An HTML attachment was scrubbed... URL: From sashak at voltaire.com Mon Jul 13 18:30:45 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Tue, 14 Jul 2009 04:30:45 +0300 Subject: [ofa-general] [PATCH] infiniband-diags/libibnetdisc: add man pages to EXTRA_DIST In-Reply-To: <4A5B7A6D.6070107@mellanox.co.il> References: <20090713155624.GD24846@opengridcomputing.com> <4A5B7A6D.6070107@mellanox.co.il> Message-ID: <20090714013045.GK20347@me> Add man pages files to EXTRA_DIST list so 'make dist' will generate a valid tarball. Signed-off-by: Sasha Khapyorsky --- infiniband-diags/libibnetdisc/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/infiniband-diags/libibnetdisc/Makefile.am b/infiniband-diags/libibnetdisc/Makefile.am index ddcf053..abf581d 100644 --- a/infiniband-diags/libibnetdisc/Makefile.am +++ b/infiniband-diags/libibnetdisc/Makefile.am @@ -43,5 +43,5 @@ man_MANS = man/ibnd_debug.3 \ man/ibnd_show_progress.3 \ man/ibnd_update_node.3 -EXTRA_DIST = $(srcdir)/src/libibnetdisc.map libibnetdisc.ver +EXTRA_DIST = $(srcdir)/src/libibnetdisc.map libibnetdisc.ver $(man_MANS) -- 1.6.3.3 From sashak at voltaire.com Mon Jul 13 18:44:45 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Tue, 14 Jul 2009 04:44:45 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Eliminate possible seg fault in get_osm_switch_from_port In-Reply-To: References: <20090711114815.GA12867@comcast.net> <20090713001126.GJ12543@me> Message-ID: <20090714014445.GN20347@me> On 06:13 Mon 13 Jul , Hal Rosenstock wrote: > > It's called with NULL port pointer from osm_get_lash_sl which is > invoked via SA PathRecord flow. It is what I asked - why is NULL pointer used there (in SA PathRecord flow) and is is legal usage? As far as I can see in osm_sa_path_record.c it should be impossible, but maybe I'm missing something. Sasha From eli at dev.mellanox.co.il Mon Jul 13 23:35:51 2009 From: eli at dev.mellanox.co.il (Eli Cohen) Date: Tue, 14 Jul 2009 09:35:51 +0300 Subject: [ofa-general] Re: [ewg] [PATCH 1/8 v3] ib_core: Add API to support RDMAoE In-Reply-To: References: <20090713181319.GA738@mtls03> Message-ID: <20090714063551.GA16024@mtls03> On Mon, Jul 13, 2009 at 03:26:06PM -0400, Hal Rosenstock wrote: > > > > +enum ib_port_link_type ib_get_port_link_type(struct ib_device *device, u8 port_num) > > +{ > > +       return device->get_port_link_type ? > > +               device->get_port_link_type(device, port_num) : PORT_LINK_IB; > > So do iWARP devices return PORT_LINK_IB ? If so, that seems a little > weird to me. > > -- Hal > Maybe it's more appropriate to make this function mandatory and require all drivers to report the correct port type. What do you think? From eli at dev.mellanox.co.il Tue Jul 14 00:46:45 2009 From: eli at dev.mellanox.co.il (Eli Cohen) Date: Tue, 14 Jul 2009 10:46:45 +0300 Subject: [ofa-general] Re: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support only QP1 In-Reply-To: References: <20090713181410.GA754@mtls03> Message-ID: <20090714074645.GB16024@mtls03> On Mon, Jul 13, 2009 at 03:26:34PM -0400, Hal Rosenstock wrote: > On Mon, Jul 13, 2009 at 2:14 PM, Eli Cohen wrote: > > Since RDMAoE is using Ethernet as its link layer, there is no need for QP0. QP1 > > is still needed since it handles communications between CM agents. This patch > > will create only QP1 for RDMAoE ports. > > What happens with other QP1 traffic (other than CM and SA) ? I think it should work but I haven't tried that. > Userspace > can access QP1 (and QP0). QP0 is not accessible since ib_register_mad_agent() will fail for QP0 becuase of this: if (!port_priv->qp_info[qp_type].qp) return NULL; QP1 should work in the same way > Does QP0 error out ? What about QP1 ? Does > it just timeout ? If so, a direct error would be better. > See above - you can't access QP0. Do you know of a utility from userspace which sends/receives MADs on QP0 or QP1? From liranl at mellanox.co.il Tue Jul 14 01:15:47 2009 From: liranl at mellanox.co.il (Liran Liss) Date: Tue, 14 Jul 2009 11:15:47 +0300 Subject: [ofa-general] RE: [ewg] [PATCH 6/8 v3] IB/ipoib: restrict IPoIB to work on IB ports only In-Reply-To: <382A478CAD40FA4FB46605CF81FE39F435877A39@orsmsx507.amr.corp.intel.com> References: <20090713181705.GA802@mtls03> <382A478CAD40FA4FB46605CF81FE39F435877A39@orsmsx507.amr.corp.intel.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD03566278@mtlexch01.mtl.com> This exaclty the same as for iWARP: IPoIB checks the node transport, and if it is != IB, it exists. For RDMAoE, we do the same check but at the port level. -----Original Message----- From: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Woodruff, Robert J Sent: Tuesday, July 14, 2009 12:04 AM To: Eli Cohen; Hefty, Sean; Roland Dreier Cc: ewg; general-list Subject: [ofa-general] RE: [ewg] [PATCH 6/8 v3] IB/ipoib: restrict IPoIB to work on IB ports only Eli Cohen wrote, >We don't want IPoIB to work over RDMAoE since it will give worse >performance than working directly on Ethernet interfaces which are a >prerequisite to RDMAoE anyway. This is another reason why NOT to try to add IBxOE under the IB transport, but rather add it as it's own transport type. We should not need to hack all the InfiniBand ULPs to now have to know the difference between real IB and IBxOE._______________________________________________ general mailing list general at lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From liranl at mellanox.co.il Tue Jul 14 01:52:52 2009 From: liranl at mellanox.co.il (Liran Liss) Date: Tue, 14 Jul 2009 11:52:52 +0300 Subject: [ofa-general] RE: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support onlyQP1 In-Reply-To: <382A478CAD40FA4FB46605CF81FE39F435877A24@orsmsx507.amr.corp.intel.com> References: <20090713181410.GA754@mtls03> <382A478CAD40FA4FB46605CF81FE39F435877A24@orsmsx507.amr.corp.intel.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD03566312@mtlexch01.mtl.com> S.B. --Liran > Trying to emulate IB for mad services is a total hack and not how this new transport should be added into the core. It should be it's own transport type, just like iWarp was added. > You should start with adding a new transport type to ib_verbs.h, e.g., LL: it is not a hack: RDMAoE will probably use mad services at least for connection management, and additional ones in the future. --- ib_verbs.h 2009-07-13 09:06:10.000000000 -0400 +++ ib_verbs_new.h 2009-07-14 03:00:23.000000000 -0400 @@ -64,12 +64,14 @@ enum rdma_node_type { RDMA_NODE_IB_CA = 1, RDMA_NODE_IB_SWITCH, RDMA_NODE_IB_ROUTER, - RDMA_NODE_RNIC + RDMA_NODE_RNIC, + RDMA_NODE_IBXOE }; LL: a multi-port HCA can have both IB and Ethernet ports, so this is not a per-node thing. enum rdma_transport_type { RDMA_TRANSPORT_IB, - RDMA_TRANSPORT_IWARP + RDMA_TRANSPORT_IWARP, + RDMA_TRANSPORT_IBXOE }; LL: thanks, we will look into this. I am not sure that "transport" is the right terminology, since we are using the IB transport layer. enum rdma_transport_type_______________________________________________ general mailing list general at lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From FENKES at de.ibm.com Tue Jul 14 02:10:07 2009 From: FENKES at de.ibm.com (Joachim Fenkes) Date: Tue, 14 Jul 2009 11:10:07 +0200 Subject: [ofa-general] Re: [PATCH v4] libibmad: Handle MAD redirection In-Reply-To: References: <200906291410.33477.fenkes@de.ibm.com> <20090701200043.GF20745@obsidianresearch.com> <200907071620.23003.fenkes@de.ibm.com> Message-ID: Hal Rosenstock wrote on 08.07.2009 18:48:29: > > This patch should make its way into OFED 1.5... so who should pull it? > > You? Vlad? Someone not on CC? Whoever, please apply for OFED 1.5 -- > > thanks! > > Sasha is the management maintainer. Userspace trees for OFED 1.5 > haven't been created and I think this aspect is in transition. Sasha, can you apply this patch? Thanks! Here's a link to the patch: http://lists.openfabrics.org/pipermail/ewg/2009-July/013519.html Cheers, Joachim From jackm at dev.mellanox.co.il Tue Jul 14 02:35:43 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Tue, 14 Jul 2009 12:35:43 +0300 Subject: [ofa-general] Compilation errors with OFED 1.4.1/ 1.4 In-Reply-To: <96f8e60e0907130952p73f09362o5b75bd121080e963@mail.gmail.com> References: <9759F033B56A60469F217C49AC58E2BF085A541B@MAILUK2.rms.com> <200907071217.19649.jackm@dev.mellanox.co.il> <96f8e60e0907130952p73f09362o5b75bd121080e963@mail.gmail.com> Message-ID: <200907141235.43845.jackm@dev.mellanox.co.il> On Monday 13 July 2009 19:52, pandit ib wrote: > > Looks like the OFED installation is faulty. > > Can we fix this issue in the next release of OFED? This is not an OFED issue, you need to fix your compilation script. > > For some reason, your compilation script is not taking directory > > /usr/src/ofa_kernel/kernel_addons/backport/2.6.16_sles10_sp2/include > > in the include path before the regular kernel includes. > > Are you referring to the OFED compilation script or our module's? I am referring to your module's compilation script. You need to make sure that your module's compilation script will include directory /usr/src/ofa_kernel/kernel_addons/backport/2.6.16_sles10_sp2/include BEFORE the standard linux includes. > This problem happens on RedHat EL 5.0 as well. It will happen on ALL distributions and kernels which are before kernel 2.6.27 (for OFED 1.4) -- which does not have any kernel_addons/backport directory furnishing include files. From vlad at lists.openfabrics.org Tue Jul 14 02:39:32 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Tue, 14 Jul 2009 02:39:32 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090714-0200 daily build status Message-ID: <20090714093932.BC97110203FE@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_x86_64_check/net/rds/af_rds.c:419: warning: passing argument 3 of 'sk_alloc' makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_x86_64_check/net/rds/af_rds.c:419: warning: passing argument 4 of 'sk_alloc' makes integer from pointer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_x86_64_check/net/rds/af_rds.c: At top level: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_x86_64_check/net/rds/af_rds.c:438: warning: initialization from incompatible pointer type make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_x86_64_check/net/rds/af_rds.c:419: warning: passing argument 3 of 'sk_alloc' makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_x86_64_check/net/rds/af_rds.c:419: warning: passing argument 4 of 'sk_alloc' makes integer from pointer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_x86_64_check/net/rds/af_rds.c: At top level: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_x86_64_check/net/rds/af_rds.c:438: warning: initialization from incompatible pointer type make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -m64 -mno-red-zone -mcmodel=kernel -funit-at-a-time -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090714-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From ogerlitz at voltaire.com Tue Jul 14 02:43:14 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Tue, 14 Jul 2009 12:43:14 +0300 Subject: [ofa-general] Re: [PATCH 0/8 v3] RDMAoE support In-Reply-To: <20090713181310.GA31865@mtls03> References: <20090713181310.GA31865@mtls03> Message-ID: <4A5C5332.7020403@voltaire.com> Eli Cohen wrote: > RDMAoE allows running the IB transport protocol using Ethernet frames allowing the deployment of IB semantics on lossless Ethernet fabrics. RDMAoE packets are standard Ethernet frames with an IEEE assigned Ethertype, a GRH, unmodified IB transport headers and payload. Hi Eli and the team @ Mellanox Before going into more detailed review and comments, I'd like to try and clarify few issues. 1. GRH - is it a must per your design to have it also for unicast packets? or maybe it just simplifies things, or both? I assume you may need it for the CM logic to keep working the way it used to. 2. is there any reason not to restrict the design for supporting addr / rdma-cm based consumers? e.g in the same manner that iWARP is? 3. CM services - note that once the ULP did the address resolution, the addr / rdma-cm data structure can/has the src/dest MACs, so basically why not push all the changes to the mad code and eimplement the MADs over raw or even datagram socket? 4. regarding your rdmaoe_sa - aside from being non review-able! (since in the same patch 3/8 you move tons of code from one place to another and add changes) it seems to me an overkill. I would like to see a solution which takes advantage of the SA non existence under Ethernet, and hence route resolution becomes no-op as it is with iWARP, as for multicast join, since it is a local operation it should and can be done by the rdma-cm, no need to modify the layers below it, expect for exposing API for the rdma-cm to do so. 5. 8/8 says "Currently, each IB port has a single GID entry in its table and that GID entery equals the link local IPv6 address" - does your design support IPv4 as well, how? > To enable RDMAoE with the mlx4 driver stack, both the mlx4_en and mlx4_ib drivers must be loaded, and the netdevice for the corresponding RDMAoE port must be running. 6. it seems that your design is somehow too tightly coupled to the connectX hca and the mlx4 driver, please note that the design should allow for implementing software rdmaoe provider as well > Individual ports of a multi port HCA can be independently configured as Ethernet (with support for RDMAoE) or IB, as is already the case. [...] Following is a series of 8 patches based on version 2.6.30 of the Linux kernel > Does the mainline kernel has all the patches to do so - I wasn't sure if this is the case. can you send the instructions? > This new series reflects changes based on feedback from the community on the previous set of patches. The whole series is tagged v3 There's not a single mentioning of a change vs the previous versions, how do you expect someone not to treat it as v1?! For some reason you have chosen to use cross-posting, I don't think these patches need that. Or. From hal.rosenstock at gmail.com Tue Jul 14 04:03:51 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Tue, 14 Jul 2009 07:03:51 -0400 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Eliminate possible seg fault in get_osm_switch_from_port In-Reply-To: <20090714014445.GN20347@me> References: <20090711114815.GA12867@comcast.net> <20090713001126.GJ12543@me> <20090714014445.GN20347@me> Message-ID: On Mon, Jul 13, 2009 at 9:44 PM, Sasha Khapyorsky wrote: > On 06:13 Mon 13 Jul     , Hal Rosenstock wrote: >> >> It's called with NULL port pointer from osm_get_lash_sl which is >> invoked via SA PathRecord flow. > > It is what I asked - why is NULL pointer used there (in SA PathRecord > flow) and is is legal usage? It shouldn't be a valid use case. > As far as I can see in osm_sa_path_record.c it should be impossible, but > maybe I'm missing something. Yes, that's what I thought too but apparently it doesn't appear to be the case and I'm not sure how that could occur. Any ideas ? -- Hal > Sasha > From hal.rosenstock at gmail.com Tue Jul 14 04:07:52 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Tue, 14 Jul 2009 07:07:52 -0400 Subject: [ofa-general] Re: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support only QP1 In-Reply-To: <382A478CAD40FA4FB46605CF81FE39F435877A24@orsmsx507.amr.corp.intel.com> References: <20090713181410.GA754@mtls03> <382A478CAD40FA4FB46605CF81FE39F435877A24@orsmsx507.amr.corp.intel.com> Message-ID: On Mon, Jul 13, 2009 at 4:58 PM, Woodruff, Robert J wrote: > Eli Cohen wrote, > >>Since RDMAoE is using Ethernet as its link layer, there is no need for QP0. QP1 >>is still needed since it handles communications between CM agents. This patch >>will create only QP1 for RDMAoE ports. > > > Trying to emulate IB for mad services is a total hack and not how this > new transport should be added into the core. It should be it's own transport type, > just like iWarp was added. > You should start with adding a new transport type to ib_verbs.h, > e.g., > > > --- ib_verbs.h  2009-07-13 09:06:10.000000000 -0400 > +++ ib_verbs_new.h      2009-07-14 03:00:23.000000000 -0400 > @@ -64,12 +64,14 @@ enum rdma_node_type { >        RDMA_NODE_IB_CA         = 1, >        RDMA_NODE_IB_SWITCH, >        RDMA_NODE_IB_ROUTER, > -       RDMA_NODE_RNIC > +       RDMA_NODE_RNIC, > +       RDMA_NODE_IBXOE >  }; > >  enum rdma_transport_type { >        RDMA_TRANSPORT_IB, > -       RDMA_TRANSPORT_IWARP > +       RDMA_TRANSPORT_IWARP, > +       RDMA_TRANSPORT_IBXOE >  }; > >  enum rdma_transport_type Unfortunately I don't think it's this simple although I wish it were. IBXOE is on a per port rather than a per node basis which is a different model than we've used for IB or iWARP. -- Hal From hal.rosenstock at gmail.com Tue Jul 14 04:15:44 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Tue, 14 Jul 2009 07:15:44 -0400 Subject: [ofa-general] Re: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support only QP1 In-Reply-To: <20090714074645.GB16024@mtls03> References: <20090713181410.GA754@mtls03> <20090714074645.GB16024@mtls03> Message-ID: On Tue, Jul 14, 2009 at 3:46 AM, Eli Cohen wrote: > On Mon, Jul 13, 2009 at 03:26:34PM -0400, Hal Rosenstock wrote: >> On Mon, Jul 13, 2009 at 2:14 PM, Eli Cohen wrote: >> > Since RDMAoE is using Ethernet as its link layer, there is no need for QP0. QP1 >> > is still needed since it handles communications between CM agents. This patch >> > will create only QP1 for RDMAoE ports. >> >> What happens with other QP1 traffic (other than CM and SA) ? > I think it should work but I haven't tried that. Would you ? You could try tools from infiniband-diags or ibdiagnet. >> Userspace >> can access QP1 (and QP0). > QP0 is not accessible since ib_register_mad_agent() will fail for QP0 > becuase of this: > >        if (!port_priv->qp_info[qp_type].qp) >                return NULL; > > QP1 should work in the same way So what happens with things like PerfMgt class ? I think it ends up timing out if no receiver consumer is present. >> Does QP0 error out ? What about QP1 ? Does >> it just timeout ? If so, a direct error would be better. >> > > See above - you can't access QP0. Do you know of a utility from > userspace which sends/receives MADs on QP0 or QP1? Yes, opensm, infiniband-diags (various), and ibutils (ibdiagnet, etc). -- Hal From hal.rosenstock at gmail.com Tue Jul 14 04:18:04 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Tue, 14 Jul 2009 07:18:04 -0400 Subject: [ofa-general] Re: [ewg] [PATCH 1/8 v3] ib_core: Add API to support RDMAoE In-Reply-To: <20090714063551.GA16024@mtls03> References: <20090713181319.GA738@mtls03> <20090714063551.GA16024@mtls03> Message-ID: On Tue, Jul 14, 2009 at 2:35 AM, Eli Cohen wrote: > On Mon, Jul 13, 2009 at 03:26:06PM -0400, Hal Rosenstock wrote: >> > >> > +enum ib_port_link_type ib_get_port_link_type(struct ib_device *device, u8 port_num) >> > +{ >> > +       return device->get_port_link_type ? >> > +               device->get_port_link_type(device, port_num) : PORT_LINK_IB; >> >> So do iWARP devices return PORT_LINK_IB ? If so, that seems a little >> weird to me. >> >> -- Hal >> > > Maybe it's more appropriate to make this function mandatory and > require all drivers to report the correct port type. What do you > think? That seems better to me; another alternative would be to require this routine for either all iWARP or IB devices as well but that might be error prone. Maybe someone else has a better idea on this. -- Hal From acceptany at gmail.com Tue Jul 14 04:25:52 2009 From: acceptany at gmail.com (Jordan) Date: Tue, 14 Jul 2009 19:25:52 +0800 Subject: [ofa-general] Some problem about the LID selection. Message-ID: <91fe68d50907140425h53c57ab9r56c6a7049a12f531@mail.gmail.com> When LMC > 0 , there are 2^LMC LIDs assigned to a port. When choosing different LID, there maybe different paths to get to this port. In other words, different LIDs represent different paths. But which path should I choose to get to the destination? It seems that OpenSM does not provide such algorithm . Does OFED provide such algorithm to manage which path should to choose ? -------------- next part -------------- An HTML attachment was scrubbed... URL: From jgunthorpe at obsidianresearch.com Tue Jul 14 05:01:05 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Tue, 14 Jul 2009 06:01:05 -0600 Subject: [ofa-general] SDP and stock kernel gets BUG? Message-ID: <20090714120105.GA11987@obsidianresearch.com> Hey all, I'm trying to use SDP with stock 2.6.30.1 plus the 'drivers/infiniband/ulp/sdp' directory from ofa_kernel-1.5-ofed20090713.src.rpm and I get this BUG ON: BUG: scheduling while atomic: ib_cm/0/4209/0x00000003 Modules linked in: ib_sdp w83793 hwmon_vid rdma_ucm rdma_cm iw_cm ib_addr mlx4_ib ib_ipoib ib_cm ib_sa ib_uverbs ib_umad ib_mthca e1000e ib_mad ib_core mlx4_core i5k_amb hwmon i2c_i801 i2c_core Pid: 4209, comm: ib_cm/0 Not tainted 2.6.30.1 #1 Call Trace: [] __schedule_bug+0x65/0x6a [] __schedule+0x83/0x901 [] ? del_timer_sync+0x14/0x21 [] schedule+0x13/0x31 [] schedule_timeout+0x154/0x17e [] ? process_timeout+0x0/0xb [] ? wait_for_common+0xb9/0x12d [] wait_for_common+0xb9/0x12d [] ? default_wake_function+0x0/0xf [] wait_for_completion_timeout+0xe/0x10 [] __mlx4_cmd+0xf5/0x232 [mlx4_core] [] mlx4_mr_free+0x59/0xb7 [mlx4_core] [] mlx4_ib_dereg_mr+0x1f/0x3b [mlx4_ib] [] ib_dereg_mr+0x22/0x33 [ib_core] [] sdp_destroy_qp+0xcf/0x107 [ib_sdp] [] sdp_reset_sk+0x389/0x4ff [ib_sdp] [] ? sdp_xmit_poll+0x53/0x5c [ib_sdp] [] ? sdp_disconnected_handler+0x9a/0xc9 [ib_sdp] [] sdp_cma_handler+0x188b/0x1aa5 [ib_sdp] [] ? _spin_unlock_irqrestore+0x2d/0x48 [] ? _spin_lock_irq+0x1c/0x34 [] ? _spin_lock_irq+0x1c/0x34 [] ? sub_preempt_count+0x9e/0xbe [] ? _spin_unlock_irq+0x16/0x2f [] ? cm_work_handler+0x0/0xbf3 [ib_cm] [] cma_ib_handler+0x19b/0x209 [rdma_cm] [] cm_process_work+0x1c/0xaf [ib_cm] [] cm_work_handler+0x3be/0xbf3 [ib_cm] [] ? cm_work_handler+0x0/0xbf3 [ib_cm] [] ? cm_work_handler+0x0/0xbf3 [ib_cm] [] worker_thread+0x14b/0x1f5 [] ? autoremove_wake_function+0x0/0x38 [] ? worker_thread+0x0/0x1f5 [] kthread+0x56/0x85 [] child_rip+0xa/0x20 [] ? finish_task_switch+0xb5/0xc4 [] ? restore_args+0x0/0x30 [] ? kthread+0x0/0x85 [] ? child_rip+0x0/0x20 Anyone know what is up?? Does SDP work with the baseline OFED 1.5 kernel? This kernel has CONFIG_PREEMPT=y if that makes a difference. Thanks, Jason From todd.rimmer at qlogic.com Tue Jul 14 05:19:38 2009 From: todd.rimmer at qlogic.com (Todd Rimmer) Date: Tue, 14 Jul 2009 07:19:38 -0500 Subject: [ofa-general] Some problem about the LID selection. In-Reply-To: <91fe68d50907140425h53c57ab9r56c6a7049a12f531@mail.gmail.com> References: <91fe68d50907140425h53c57ab9r56c6a7049a12f531@mail.gmail.com> Message-ID: <5AEC2602AE03EB46BFC16C6B9B200DA815A8E6705E@MNEXMB2.qlogic.org> The intent of the specification was for the SM to provide such suggestions, hence fabric policies and config could be easily controlled and path selection can be coordinated with routing and topology knowledge (which the SM has). Generally Path Record queries should be made using SGID and DGID. In which case the SM can return multiple matching paths. It is up to the SM to return an appropriate subset of paths. The SM can also return multiple paths in "preferred order of use" (eg. Primary, then secondary path for failover, etc). The NumbPath parameter in the PathRecord query can specify the maximum number of such paths of interest. Typical use cases are: - Ask for 1 path (most applications) - Ask for 2 paths and perform a failover algorithm or use APM (I'm aware of quite a few applications which use failover) - Ask for many or all paths and perform a load balancing algorithm across them (I'm aware of a couple applications which do this) Todd Rimmer Chief Architect QLogic Network Systems Group Voice: 610-233-4852 Fax: 610-233-4777 Todd.Rimmer at QLogic.com www.QLogic.com From: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Jordan Sent: Tuesday, July 14, 2009 7:26 AM To: general at lists.openfabrics.org Subject: [ofa-general] Some problem about the LID selection. When LMC > 0 , there are 2^LMC LIDs assigned to a port. When choosing different LID, there maybe different paths to get to this port. In other words, different LIDs represent different paths. But which path should I choose to get to the destination? It seems that OpenSM does not provide such algorithm . Does OFED provide such algorithm to manage which path should to choose ? -------------- next part -------------- An HTML attachment was scrubbed... URL: From amirv at mellanox.co.il Tue Jul 14 06:14:59 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Tue, 14 Jul 2009 16:14:59 +0300 Subject: [ofa-general] SDP and stock kernel gets BUG? In-Reply-To: <20090714120105.GA11987@obsidianresearch.com> References: <20090714120105.GA11987@obsidianresearch.com> Message-ID: <4A5C84D3.9060901@mellanox.co.il> Hi, I will post a fix soon. -- Amir Vadai Software Eng. Mellanox Technologies mailto: amirv at mellanox.co.il Tel +972-3-6259539 On 07/14/2009 03:01 PM, Jason Gunthorpe wrote: > Hey all, > > I'm trying to use SDP with stock 2.6.30.1 plus the > 'drivers/infiniband/ulp/sdp' directory from > ofa_kernel-1.5-ofed20090713.src.rpm and I get this BUG ON: > > BUG: scheduling while atomic: ib_cm/0/4209/0x00000003 > Modules linked in: ib_sdp w83793 hwmon_vid rdma_ucm rdma_cm iw_cm > ib_addr mlx4_ib ib_ipoib ib_cm ib_sa ib_uverbs ib_umad ib_mthca e1000e > ib_mad ib_core mlx4_core i5k_amb hwmon i2c_i801 i2c_core > Pid: 4209, comm: ib_cm/0 Not tainted 2.6.30.1 #1 > Call Trace: > [] __schedule_bug+0x65/0x6a > [] __schedule+0x83/0x901 > [] ? del_timer_sync+0x14/0x21 > [] schedule+0x13/0x31 > [] schedule_timeout+0x154/0x17e > [] ? process_timeout+0x0/0xb > [] ? wait_for_common+0xb9/0x12d > [] wait_for_common+0xb9/0x12d > [] ? default_wake_function+0x0/0xf > [] wait_for_completion_timeout+0xe/0x10 > [] __mlx4_cmd+0xf5/0x232 [mlx4_core] > [] mlx4_mr_free+0x59/0xb7 [mlx4_core] > [] mlx4_ib_dereg_mr+0x1f/0x3b [mlx4_ib] > [] ib_dereg_mr+0x22/0x33 [ib_core] > [] sdp_destroy_qp+0xcf/0x107 [ib_sdp] > [] sdp_reset_sk+0x389/0x4ff [ib_sdp] > [] ? sdp_xmit_poll+0x53/0x5c [ib_sdp] > [] ? sdp_disconnected_handler+0x9a/0xc9 [ib_sdp] > [] sdp_cma_handler+0x188b/0x1aa5 [ib_sdp] > [] ? _spin_unlock_irqrestore+0x2d/0x48 > [] ? _spin_lock_irq+0x1c/0x34 > [] ? _spin_lock_irq+0x1c/0x34 > [] ? sub_preempt_count+0x9e/0xbe > [] ? _spin_unlock_irq+0x16/0x2f > [] ? cm_work_handler+0x0/0xbf3 [ib_cm] > [] cma_ib_handler+0x19b/0x209 [rdma_cm] > [] cm_process_work+0x1c/0xaf [ib_cm] > [] cm_work_handler+0x3be/0xbf3 [ib_cm] > [] ? cm_work_handler+0x0/0xbf3 [ib_cm] > [] ? cm_work_handler+0x0/0xbf3 [ib_cm] > [] worker_thread+0x14b/0x1f5 > [] ? autoremove_wake_function+0x0/0x38 > [] ? worker_thread+0x0/0x1f5 > [] kthread+0x56/0x85 > [] child_rip+0xa/0x20 > [] ? finish_task_switch+0xb5/0xc4 > [] ? restore_args+0x0/0x30 > [] ? kthread+0x0/0x85 > [] ? child_rip+0x0/0x20 > > Anyone know what is up?? Does SDP work with the baseline OFED 1.5 > kernel? This kernel has CONFIG_PREEMPT=y if that makes a difference. > > Thanks, > Jason > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From liranl at mellanox.co.il Tue Jul 14 06:31:42 2009 From: liranl at mellanox.co.il (Liran Liss) Date: Tue, 14 Jul 2009 16:31:42 +0300 Subject: [ofa-general] RE: [ewg] [PATCH 6/8 v3] IB/ipoib: restrict IPoIB to work on IB ports only References: <20090713181705.GA802@mtls03> <382A478CAD40FA4FB46605CF81FE39F435877A39@orsmsx507.amr.corp.intel.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD035ADC79@mtlexch01.mtl.com> Oops, I meant "exits" instead of "exists"... -----Original Message----- From: Liran Liss Sent: Tuesday, July 14, 2009 11:16 AM To: 'Woodruff, Robert J'; Eli Cohen; Hefty, Sean; Roland Dreier Cc: ewg; general-list Subject: RE: [ofa-general] RE: [ewg] [PATCH 6/8 v3] IB/ipoib: restrict IPoIB to work on IB ports only This exaclty the same as for iWARP: IPoIB checks the node transport, and if it is != IB, it exists. For RDMAoE, we do the same check but at the port level. -----Original Message----- From: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Woodruff, Robert J Sent: Tuesday, July 14, 2009 12:04 AM To: Eli Cohen; Hefty, Sean; Roland Dreier Cc: ewg; general-list Subject: [ofa-general] RE: [ewg] [PATCH 6/8 v3] IB/ipoib: restrict IPoIB to work on IB ports only Eli Cohen wrote, >We don't want IPoIB to work over RDMAoE since it will give worse >performance than working directly on Ethernet interfaces which are a >prerequisite to RDMAoE anyway. This is another reason why NOT to try to add IBxOE under the IB transport, but rather add it as it's own transport type. We should not need to hack all the InfiniBand ULPs to now have to know the difference between real IB and IBxOE._______________________________________________ general mailing list general at lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From eli at dev.mellanox.co.il Tue Jul 14 06:38:32 2009 From: eli at dev.mellanox.co.il (Eli Cohen) Date: Tue, 14 Jul 2009 16:38:32 +0300 Subject: [ofa-general] Re: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support only QP1 In-Reply-To: References: <20090713181410.GA754@mtls03> <20090714074645.GB16024@mtls03> Message-ID: <20090714133832.GB20264@mtls03> On Tue, Jul 14, 2009 at 07:15:44AM -0400, Hal Rosenstock wrote: > On Tue, Jul 14, 2009 at 3:46 AM, Eli Cohen wrote: > > On Mon, Jul 13, 2009 at 03:26:34PM -0400, Hal Rosenstock wrote: > >> On Mon, Jul 13, 2009 at 2:14 PM, Eli Cohen wrote: > >> > Since RDMAoE is using Ethernet as its link layer, there is no need for QP0. QP1 > >> > is still needed since it handles communications between CM agents. This patch > >> > will create only QP1 for RDMAoE ports. > >> > >> What happens with other QP1 traffic (other than CM and SA) ? > > I think it should work but I haven't tried that. > > Would you ? You could try tools from infiniband-diags or ibdiagnet. Yes I would try that. But I need something that will not fail because it could not open QP0. For example, something that uses only QP1. Are the any in ibutils? > > >> Userspace > >> can access QP1 (and QP0). > > QP0 is not accessible since ib_register_mad_agent() will fail for QP0 > > becuase of this: > > > >        if (!port_priv->qp_info[qp_type].qp) > >                return NULL; > > > > QP1 should work in the same way > > So what happens with things like PerfMgt class ? I think it ends up > timing out if no receiver consumer is present. > > >> Does QP0 error out ? What about QP1 ? Does > >> it just timeout ? If so, a direct error would be better. > >> > > > > See above - you can't access QP0. Do you know of a utility from > > userspace which sends/receives MADs on QP0 or QP1? > > Yes, opensm, infiniband-diags (various), and ibutils (ibdiagnet, etc). > > -- Hal From hal.rosenstock at gmail.com Tue Jul 14 06:40:33 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Tue, 14 Jul 2009 09:40:33 -0400 Subject: [ofa-general] Re: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support only QP1 In-Reply-To: <20090714133832.GB20264@mtls03> References: <20090713181410.GA754@mtls03> <20090714074645.GB16024@mtls03> <20090714133832.GB20264@mtls03> Message-ID: On Tue, Jul 14, 2009 at 9:38 AM, Eli Cohen wrote: > On Tue, Jul 14, 2009 at 07:15:44AM -0400, Hal Rosenstock wrote: >> On Tue, Jul 14, 2009 at 3:46 AM, Eli Cohen wrote: >> > On Mon, Jul 13, 2009 at 03:26:34PM -0400, Hal Rosenstock wrote: >> >> On Mon, Jul 13, 2009 at 2:14 PM, Eli Cohen wrote: >> >> > Since RDMAoE is using Ethernet as its link layer, there is no need for QP0. QP1 >> >> > is still needed since it handles communications between CM agents. This patch >> >> > will create only QP1 for RDMAoE ports. >> >> >> >> What happens with other QP1 traffic (other than CM and SA) ? >> > I think it should work but I haven't tried that. >> >> Would you ? You could try tools from infiniband-diags or ibdiagnet. > Yes I would try that. But I need something that will not fail because > it could not open QP0. So opensm, ibdiagnet, and smpquery fail (error out) ? > For example, something that uses only QP1. Are > the any in ibutils? In infiniband-diags, there are perfquery, saquery, vendstat, ibping, and ibssystat which only use QP1. The latter two run are client/server and can take GUID (not GID) as an argument. -- Hal >> >> >> Userspace >> >> can access QP1 (and QP0). >> > QP0 is not accessible since ib_register_mad_agent() will fail for QP0 >> > becuase of this: >> > >> >        if (!port_priv->qp_info[qp_type].qp) >> >                return NULL; >> > >> > QP1 should work in the same way >> >> So what happens with things like PerfMgt class ? I think it ends up >> timing out if no receiver consumer is present. >> >> >> Does QP0 error out ? What about QP1 ? Does >> >> it just timeout ? If so, a direct error would be better. >> >> >> > >> > See above - you can't access QP0. Do you know of a utility from >> > userspace which sends/receives MADs on QP0 or QP1? >> >> Yes, opensm, infiniband-diags (various), and ibutils (ibdiagnet, etc). >> >> -- Hal > From hnrose at comcast.net Tue Jul 14 07:04:27 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Tue, 14 Jul 2009 10:04:27 -0400 Subject: [ofa-general] [PATCH] opensm/osm_perfmgr.c: In perfmgr_send_pc_mad, only set CounterSelect when Set method is used Message-ID: <20090714140427.GA29521@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c index ecfdbda..0437d47 100644 --- a/opensm/opensm/osm_perfmgr.c +++ b/opensm/opensm/osm_perfmgr.c @@ -376,7 +376,8 @@ static ib_api_status_t perfmgr_send_pc_mad(osm_perfmgr_t * perfmgr, port_counter = (ib_port_counters_t *) & pm_mad->data; memset(port_counter, 0, sizeof(*port_counter)); port_counter->port_select = port; - port_counter->counter_select = 0xFFFF; + if (mad_method == IB_MAD_METHOD_SET) + port_counter->counter_select = 0xFFFF; p_madw->mad_addr.dest_lid = dest_lid; p_madw->mad_addr.addr_type.gsi.remote_qp = dest_qp; From jgunthorpe at obsidianresearch.com Tue Jul 14 07:31:53 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Tue, 14 Jul 2009 08:31:53 -0600 Subject: [ofa-general] SDP and stock kernel gets BUG? In-Reply-To: <4A5C84D3.9060901@mellanox.co.il> References: <20090714120105.GA11987@obsidianresearch.com> <4A5C84D3.9060901@mellanox.co.il> Message-ID: <20090714143152.GA12386@obsidianresearch.com> On Tue, Jul 14, 2009 at 04:14:59PM +0300, Amir Vadai wrote: > Hi, > > I will post a fix soon. Thanks Amir! BTW - we are testing SDP here and trying to track down a performance regression - using the 2.6.30.1 combined with OFED-1.5 SDP performs poorly while 2.6.27.10 combined with OFED-1.4 SDP performs well. We have not yet narrowed down what is going on.. Don't suppose you have any insight? Regards, Jason From amirv at mellanox.co.il Tue Jul 14 07:37:18 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Tue, 14 Jul 2009 17:37:18 +0300 Subject: [ofa-general] SDP and stock kernel gets BUG? In-Reply-To: <20090714143152.GA12386@obsidianresearch.com> References: <20090714120105.GA11987@obsidianresearch.com> <4A5C84D3.9060901@mellanox.co.il> <20090714143152.GA12386@obsidianresearch.com> Message-ID: <4A5C981E.1080108@mellanox.co.il> OFED-1.5 SDP has many changes in the data path. Performance should be improved - although it is still work in progress. You could use /proc/sdp/sdpstats and /proc/sdp/sdpprf facilities that has been added in 1.5. (enabled in sdp.h by defining macros SDPSTATS_ON and SDP_PROFILING). What packet sizes do you use? what is the setup in general? Let me know if you find something interesting. - Amir On 07/14/2009 05:31 PM, Jason Gunthorpe wrote: > On Tue, Jul 14, 2009 at 04:14:59PM +0300, Amir Vadai wrote: > >> Hi, >> >> I will post a fix soon. >> > Thanks Amir! > > BTW - we are testing SDP here and trying to track down a performance > regression - using the 2.6.30.1 combined with OFED-1.5 SDP performs > poorly while 2.6.27.10 combined with OFED-1.4 SDP performs well. We > have not yet narrowed down what is going on.. Don't suppose you have > any insight? > > Regards, > Jason > From sashak at voltaire.com Tue Jul 14 07:53:06 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Tue, 14 Jul 2009 17:53:06 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Eliminate possible seg fault in get_osm_switch_from_port In-Reply-To: References: <20090711114815.GA12867@comcast.net> <20090713001126.GJ12543@me> <20090714014445.GN20347@me> Message-ID: <20090714145306.GF15346@me> On 07:03 Tue 14 Jul , Hal Rosenstock wrote: > > > > It is what I asked - why is NULL pointer used there (in SA PathRecord > > flow) and is is legal usage? > > It shouldn't be a valid use case. So why do we need this patch then? > > As far as I can see in osm_sa_path_record.c it should be impossible, but > > maybe I'm missing something. > > Yes, that's what I thought too but apparently it doesn't appear to be > the case and I'm not sure how that could occur. Any ideas ? If you are able to reproduce such case try to find why this happens (crash OpenSM into gdb) - to hide bugs is not a good idea IMO. Sasha From michael.heinz at qlogic.com Tue Jul 14 07:54:46 2009 From: michael.heinz at qlogic.com (Mike Heinz) Date: Tue, 14 Jul 2009 09:54:46 -0500 Subject: [ofa-general] What is the purpose of libosmvendor? Message-ID: <4C2744E8AD2982428C5BFE523DF8CDCB453E3F28EC@MNEXMB1.qlogic.org> Reviewing the OFED stack, it appears that the appropriate method for user applications to access fabric information such as path records and port guids - but the name of the library itself leaves me a little nonplussed, rather than access to fabric information, it sounds like it's meant to provide a proprietary interface to the SM. Is this correct? Is there documentation for the library itself (beyond the sample applications)? -- Michael Heinz Principal Engineer, Qlogic Corporation King of Prussia, Pennsylvania From m.van.malland at linvision.com Tue Jul 14 08:08:31 2009 From: m.van.malland at linvision.com (Maarten van Malland) Date: Tue, 14 Jul 2009 17:08:31 +0200 Subject: [ofa-general] SDP and stock kernel gets BUG? In-Reply-To: <4A5C981E.1080108@mellanox.co.il> References: <20090714120105.GA11987@obsidianresearch.com> <4A5C84D3.9060901@mellanox.co.il> <20090714143152.GA12386@obsidianresearch.com> <4A5C981E.1080108@mellanox.co.il> Message-ID: <003701ca0494$f75368b0$e5fa3a10$@van.malland@linvision.com> Hi Amir, I'm testing with Jason to get this SDP working properly on a 2.6.30.1 stock kernel. With the SDP from OFED 1.5 this is the performance we're getting: inftsttwin03 ~ # LD_PRELOAD=/opt/ofa-1.5/lib/libsdp.so nttcp -r -T -l 128000 twin1-ibl Bytes Real s CPU s Real-MBit/s CPU-MBit/s Calls Real-C/s CPU-C/s l262144000 0.98 0.28 2142.7300 7599.5333 3167 3235.83 11476.4 1262144000 0.98 0.02 2143.1132 131096.5806 2048 2092.88 128024.0 inftsttwin03 ~ # LD_PRELOAD=/opt/ofa-1.5/lib/libsdp.so nttcp -r -T -l 128000 twin1-ibl Bytes Real s CPU s Real-MBit/s CPU-MBit/s Calls Real-C/s CPU-C/s l262144000 1.20 0.05 1748.5024 41948.9128 3345 2788.90 66909.4 1262144000 1.14 0.01 1847.3367 299678.7654 2048 1804.04 292655.0 However, with the SDP from OFED 1.4 we're getting this performance: inftsttwin03 ~ # LD_PRELOAD=/opt/ofa-1.5/lib/libsdp.so nttcp -r -T -l 128000 twin1-ibl Bytes Real s CPU s Real-MBit/s CPU-MBit/s Calls Real-C/s CPU-C/s l262144000 0.15 0.14 14411.9300 14565.7809 7007 48153.11 48667.2 1262144000 0.15 0.10 14410.4446 21848.5196 2048 14072.70 21336.4 inftsttwin03 ~ # LD_PRELOAD=/opt/ofa-1.5/lib/libsdp.so nttcp -r -T -l 128000 twin1-ibl Bytes Real s CPU s Real-MBit/s CPU-MBit/s Calls Real-C/s CPU-C/s l262144000 0.15 0.14 14286.1658 14465.3120 6614 45055.72 45620.7 1262144000 0.15 0.09 14312.5883 22798.5998 2048 13977.14 22264.3 The output from /proc/net/sdpstats is: inftsttwin03 ~ # cat /proc/net/sdpstats SDP statistics: sendmsg_seglen: 1 | - 0 2 | - 0 4 | - 0 8 | ************************************************** - 1 16 | - 0 32 | - 0 64 | ************************************************** - 1 128 | - 0 256 | - 0 512 | - 0 1024 | - 0 2048 | - 0 4096 | - 0 8192 | - 0 16384 | - 0 32768 | - 0 65536 | - 0 131072 | - 0 262144 | - 0 524288 | - 0 1048576 | - 0 2097152 | - 0 4194304 | - 0 8388608 | - 0 0 | - 0 send_size: 1 | - 0 2 | - 0 4 | - 0 8 | - 0 16 | ************************************************** - 325 32 | - 0 64 | - 1 128 | - 0 256 | - 0 512 | - 0 1024 | - 0 2048 | - 0 4096 | - 0 8192 | - 0 16384 | - 0 32768 | - 0 65536 | - 0 131072 | - 0 262144 | - 0 524288 | - 0 1048576 | - 0 2097152 | - 0 4194304 | - 0 8388608 | - 0 0 | - 0 credits_before_update: 0 | - 0 1 | - 0 2 | - 0 3 | - 0 4 | - 0 5 | - 0 6 | - 0 7 | - 0 8 | - 0 9 | - 0 10 | - 0 11 | - 0 12 | - 0 13 | - 0 14 | - 0 15 | - 0 16 | - 0 17 | - 0 18 | - 0 19 | - 0 20 | - 0 21 | - 0 22 | - 0 23 | - 0 24 | - 0 25 | - 0 26 | - 0 27 | - 0 28 | - 0 29 | - 0 30 | - 0 31 | - 0 32 | - 0 33 | - 0 34 | - 0 35 | - 0 36 | - 0 37 | - 0 38 | - 0 39 | - 0 40 | - 0 41 | - 0 42 | - 0 43 | - 0 44 | - 0 45 | - 0 46 | - 0 47 | - 0 48 | - 0 49 | - 0 50 | - 0 51 | - 0 52 | - 0 53 | - 1 54 | ************ - 1619 55 | ************************************************** - 6404 56 | - 0 57 | - 0 58 | - 0 59 | - 0 60 | - 0 61 | - 0 62 | - 0 63 | - 0 sdp_sendmsg() calls : 2 bcopy segments : 2 bzcopy segments : 0 post_send_credits : 322 memcpy_count : 98 post_send SDP_MID_HELLO : 0 post_send SDP_MID_HELLO_ACK : 0 post_send SDP_MID_DISCONN : 2 post_send SDP_MID_CHRCVBUF : 0 post_send SDP_MID_CHRCVBUF_ACK : 0 post_send SDP_MID_DATA : 324 post_recv : 8134 BZCopy poll miss : 0 send_wait_for_mem : 0 send_miss_no_credits : 0 rx_poll_miss : 0 tx_poll_miss : 3 tx_poll_busy : 0 tx_poll_hit : 3 CQ stats: - RX interrupts : 3151 - TX interrupts : 0 bz_clean : 180 bz_setup : 345 tx_copy : 2756 sendmsg : 535432 The first 100 lines from /proc/net/sdpprf: inftsttwin03 ~ # head -100 /proc/net/sdpprf 0 : [ 0.000000] TX: SDP_MID_DATA bufs: 55 mseq:1 ack:0 - [7916{3} 48477:5037] skb: ffff880123186d80 sdp_post_send:99 1 : [ 0.000248] tx completion. mseq:1 - [7916{3} 48477:5037] skb: ffff880123186d80 sdp_handle_send_comp:220 2 : [ 0.099838] TX: SDP_MID_DATA bufs: 55 mseq:2 ack:0 - [0{3} 48477:5037] skb: ffff880123186bc0 sdp_post_send:99 3 : [ 0.102805] RX SDP_MID_DATA +55 c:55->55 mseq:1 ack:0 - [0{0} 5038:51962] skb: ffff88011e055100 sdp_process_rx_skb:489 4 : [ 0.102807] Waking up sleepers - [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 5 : [ 0.102886] READ finished. mseq: 1 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011e055100 sdp_recvmsg:2146 6 : [ 0.102862] RX SDP_MID_DATA +55 c:55->55 mseq:2 ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78dc0 sdp_process_rx_skb:489 7 : [ 0.102862] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 8 : [ 0.102960] READ finished. mseq: 2 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78dc0 sdp_recvmsg:2146 9 : [ 0.102927] RX SDP_MID_DATA +55 c:55->55 mseq:3 ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78c00 sdp_process_rx_skb:489 10 : [ 0.102927] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 11 : [ 0.103024] READ finished. mseq: 3 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78c00 sdp_recvmsg:2146 12 : [ 0.103264] RX SDP_MID_DATA +55 c:55->55 mseq:4 ack:0 - [0{0} 5038:51962] skb: ffff88011dc78a40 sdp_process_rx_skb:489 13 : [ 0.103265] Waking up sleepers - [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 14 : [ 0.103320] READ finished. mseq: 4 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78a40 sdp_recvmsg:2146 15 : [ 0.103352] RX SDP_MID_DATA +55 c:55->55 mseq:5 ack:0 - [0{0} 5038:51962] skb: ffff88011dc78880 sdp_process_rx_skb:489 16 : [ 0.103353] Waking up sleepers - [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 17 : [ 0.103362] RX SDP_MID_DATA +55 c:55->55 mseq:6 ack:0 - [7916{0} 5038:51962] skb: ffff88011dc786c0 sdp_process_rx_skb:489 18 : [ 0.103363] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 19 : [ 0.103422] READ finished. mseq: 5 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78880 sdp_recvmsg:2146 20 : [ 0.103376] RX SDP_MID_DATA +55 c:55->55 mseq:7 ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78500 sdp_process_rx_skb:489 21 : [ 0.103377] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 22 : [ 0.103398] RX SDP_MID_DATA +55 c:55->55 mseq:8 ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78340 sdp_process_rx_skb:489 23 : [ 0.103398] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 24 : [ 0.103408] RX SDP_MID_DATA +55 c:55->55 mseq:9 ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78180 sdp_process_rx_skb:489 25 : [ 0.103408] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 26 : [ 0.103420] RX SDP_MID_DATA +55 c:55->55 mseq:10 ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3e00 sdp_process_rx_skb:489 27 : [ 0.103421] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 28 : [ 0.103429] RX SDP_MID_DATA +55 c:55->55 mseq:11 ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3c40 sdp_process_rx_skb:489 29 : [ 0.103429] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 30 : [ 0.103479] READ finished. mseq: 6 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011dc786c0 sdp_recvmsg:2146 31 : [ 0.103532] READ finished. mseq: 7 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78500 sdp_recvmsg:2146 32 : [ 0.103598] READ finished. mseq: 8 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78340 sdp_recvmsg:2146 33 : [ 0.103612] READ finished. mseq: 9 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011dc78180 sdp_recvmsg:2146 34 : [ 0.103626] READ finished. mseq: 10 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3e00 sdp_recvmsg:2146 35 : [ 0.103640] RX SDP_MID_DATA +55 c:55->55 mseq:12 ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3a80 sdp_process_rx_skb:489 36 : [ 0.103640] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 37 : [ 0.103644] READ finished. mseq: 11 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3c40 sdp_recvmsg:2146 38 : [ 0.103655] RX SDP_MID_DATA +55 c:55->55 mseq:13 ack:0 - [7916{0} 5038:51962] skb: ffff8801231b38c0 sdp_process_rx_skb:489 39 : [ 0.103656] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 40 : [ 0.103661] RX SDP_MID_DATA +55 c:55->55 mseq:14 ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3700 sdp_process_rx_skb:489 41 : [ 0.103661] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 42 : [ 0.103671] RX SDP_MID_DATA +55 c:55->55 mseq:15 ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3540 sdp_process_rx_skb:489 43 : [ 0.103671] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 44 : [ 0.103679] READ finished. mseq: 12 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3a80 sdp_recvmsg:2146 45 : [ 0.103695] READ finished. mseq: 13 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff8801231b38c0 sdp_recvmsg:2146 46 : [ 0.103712] READ finished. mseq: 14 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3700 sdp_recvmsg:2146 47 : [ 0.103728] READ finished. mseq: 15 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3540 sdp_recvmsg:2146 48 : [ 0.104631] RX SDP_MID_DATA +55 c:55->55 mseq:16 ack:0 - [0{0} 5038:51962] skb: ffff8801231b3380 sdp_process_rx_skb:489 49 : [ 0.104632] Waking up sleepers - [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 50 : [ 0.104641] RX SDP_MID_DATA +55 c:55->55 mseq:17 ack:0 - [7916{0} 5038:51962] skb: ffff8801231b31c0 sdp_process_rx_skb:489 51 : [ 0.104642] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 52 : [ 0.104653] RX SDP_MID_DATA +55 c:55->55 mseq:18 ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8cc0 sdp_process_rx_skb:489 53 : [ 0.104653] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 54 : [ 0.104657] READ finished. mseq: 16 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff8801231b3380 sdp_recvmsg:2146 55 : [ 0.104661] RX SDP_MID_DATA +55 c:55->55 mseq:19 ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8b00 sdp_process_rx_skb:489 56 : [ 0.104662] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 57 : [ 0.104674] READ finished. mseq: 17 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff8801231b31c0 sdp_recvmsg:2146 58 : [ 0.104688] READ finished. mseq: 18 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8cc0 sdp_recvmsg:2146 59 : [ 0.104703] READ finished. mseq: 19 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8b00 sdp_recvmsg:2146 60 : [ 0.105630] RX SDP_MID_DATA +55 c:55->55 mseq:20 ack:0 - [0{0} 5038:51962] skb: ffff88011e0b8940 sdp_process_rx_skb:489 61 : [ 0.105632] Waking up sleepers - [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 62 : [ 0.105640] RX SDP_MID_DATA +55 c:55->55 mseq:21 ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8780 sdp_process_rx_skb:489 63 : [ 0.105641] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 64 : [ 0.105652] RX SDP_MID_DATA +55 c:55->55 mseq:22 ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b85c0 sdp_process_rx_skb:489 65 : [ 0.105652] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 66 : [ 0.105656] READ finished. mseq: 20 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8940 sdp_recvmsg:2146 67 : [ 0.105661] RX SDP_MID_DATA +55 c:55->55 mseq:23 ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8400 sdp_process_rx_skb:489 68 : [ 0.105661] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 69 : [ 0.105673] READ finished. mseq: 21 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8780 sdp_recvmsg:2146 70 : [ 0.105687] READ finished. mseq: 22 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b85c0 sdp_recvmsg:2146 71 : [ 0.105702] READ finished. mseq: 23 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8400 sdp_recvmsg:2146 72 : [ 0.106630] RX SDP_MID_DATA +55 c:55->55 mseq:24 ack:0 - [0{0} 5038:51962] skb: ffff88011e0b8240 sdp_process_rx_skb:489 73 : [ 0.106631] Waking up sleepers - [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 74 : [ 0.106640] RX SDP_MID_DATA +55 c:55->55 mseq:25 ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8080 sdp_process_rx_skb:489 75 : [ 0.106640] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 76 : [ 0.106652] RX SDP_MID_DATA +55 c:55->55 mseq:26 ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a1d00 sdp_process_rx_skb:489 77 : [ 0.106653] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 78 : [ 0.106656] READ finished. mseq: 24 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8240 sdp_recvmsg:2146 79 : [ 0.106661] RX SDP_MID_DATA +55 c:55->55 mseq:27 ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a1b40 sdp_process_rx_skb:489 80 : [ 0.106661] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 81 : [ 0.106673] READ finished. mseq: 25 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011e0b8080 sdp_recvmsg:2146 82 : [ 0.106686] READ finished. mseq: 26 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a1d00 sdp_recvmsg:2146 83 : [ 0.106701] READ finished. mseq: 27 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a1b40 sdp_recvmsg:2146 84 : [ 0.107630] RX SDP_MID_DATA +55 c:55->55 mseq:28 ack:0 - [0{0} 5038:51962] skb: ffff88011b5a1980 sdp_process_rx_skb:489 85 : [ 0.107632] Waking up sleepers - [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 86 : [ 0.107640] RX SDP_MID_DATA +55 c:55->55 mseq:29 ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a17c0 sdp_process_rx_skb:489 87 : [ 0.107641] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 88 : [ 0.107652] RX SDP_MID_DATA +55 c:55->55 mseq:30 ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a1600 sdp_process_rx_skb:489 89 : [ 0.107653] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 90 : [ 0.107658] TX: SDP_MID_DATA bufs: 53 mseq:1 ack:30 - [7916{0} 5038:51962] skb: ffff88011b67c180 sdp_post_send:99 91 : [ 0.107662] READ finished. mseq: 28 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a1980 sdp_recvmsg:2146 92 : [ 0.107661] RX SDP_MID_DATA +55 c:54->54 mseq:31 ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a1440 sdp_process_rx_skb:489 93 : [ 0.107661] Waking up sleepers - [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 94 : [ 0.107677] tx completion. mseq:1 - [7916{0} 5038:51962] skb: ffff88011b67c180 sdp_handle_send_comp:220 95 : [ 0.107678] READ finished. mseq: 29 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a17c0 sdp_recvmsg:2146 96 : [ 0.107691] READ finished. mseq: 30 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a1600 sdp_recvmsg:2146 97 : [ 0.107706] READ finished. mseq: 31 mseq_ack:0 - [7916{0} 5038:51962] skb: ffff88011b5a1440 sdp_recvmsg:2146 98 : [ 0.107742] RX SDP_MID_DATA +55 c:54->55 mseq:32 ack:1 - [0{0} 5038:51962] skb: ffff88011b5a1280 sdp_process_rx_skb:489 99 : [ 0.107743] Waking up sleepers - [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 Hope this helps, Maarten -----Oorspronkelijk bericht----- Van: Amir Vadai [mailto:amirv at mellanox.co.il] Verzonden: dinsdag 14 juli 2009 16:37 Aan: Jason Gunthorpe CC: general-list Onderwerp: Re: [ofa-general] SDP and stock kernel gets BUG? OFED-1.5 SDP has many changes in the data path. Performance should be improved - although it is still work in progress. You could use /proc/sdp/sdpstats and /proc/sdp/sdpprf facilities that has been added in 1.5. (enabled in sdp.h by defining macros SDPSTATS_ON and SDP_PROFILING). What packet sizes do you use? what is the setup in general? Let me know if you find something interesting. - Amir On 07/14/2009 05:31 PM, Jason Gunthorpe wrote: > On Tue, Jul 14, 2009 at 04:14:59PM +0300, Amir Vadai wrote: > >> Hi, >> >> I will post a fix soon. >> > Thanks Amir! > > BTW - we are testing SDP here and trying to track down a performance > regression - using the 2.6.30.1 combined with OFED-1.5 SDP performs > poorly while 2.6.27.10 combined with OFED-1.4 SDP performs well. We > have not yet narrowed down what is going on.. Don't suppose you have > any insight? > > Regards, > Jason > From devel-ofed at morey-chaisemartin.com Tue Jul 14 08:26:46 2009 From: devel-ofed at morey-chaisemartin.com (Nicolas Morey-Chaisemartin) Date: Tue, 14 Jul 2009 17:26:46 +0200 Subject: [ofa-general] [Repost][PATCH] opensm: Added support for select counters (xmit_wait) Message-ID: <4A5CA3B6.9080809@morey-chaisemartin.com> Support for xmit_wait counters was missing in the perfmgr though it was read from the mad and event plugin interface already handles it. This patch adds support for it (tested and working with an event plugin) Tested-by: Jean-Vincent Ficet Signed-off-by: Nicolas Morey-Chaisemartin --- I think emails got mixed up the first time so here it is again. opensm/include/opensm/osm_perfmgr_db.h | 23 ++++++- opensm/opensm/osm_perfmgr.c | 30 +++++++- opensm/opensm/osm_perfmgr_db.c | 124 +++++++++++++++++++++++++++++-- 3 files changed, 166 insertions(+), 11 deletions(-) diff --git a/opensm/include/opensm/osm_perfmgr_db.h b/opensm/include/opensm/osm_perfmgr_db.h index 42a47bd..35b5ac3 100644 --- a/opensm/include/opensm/osm_perfmgr_db.h +++ b/opensm/include/opensm/osm_perfmgr_db.h @@ -109,6 +109,14 @@ typedef struct { } perfmgr_db_data_cnt_reading_t; /** ========================================================================= + * Port select count reading + */ +typedef struct { + uint64_t xmit_wait; + time_t time; +} perfmgr_db_sel_reading_t; + +/** ========================================================================= * Dump output options */ typedef enum { @@ -125,6 +133,8 @@ typedef struct db_port { perfmgr_db_err_reading_t err_previous; perfmgr_db_data_cnt_reading_t dc_total; perfmgr_db_data_cnt_reading_t dc_previous; + perfmgr_db_sel_reading_t ps_total; + perfmgr_db_sel_reading_t ps_previous; time_t last_reset; } db_port_t; @@ -179,7 +189,16 @@ perfmgr_db_err_t perfmgr_db_get_prev_dc(perfmgr_db_t * db, uint64_t guid, reading); perfmgr_db_err_t perfmgr_db_clear_prev_dc(perfmgr_db_t * db, uint64_t guid, uint8_t port); - +perfmgr_db_err_t perfmgr_db_add_ps_reading(perfmgr_db_t * db, uint64_t guid, + uint8_t port, + perfmgr_db_sel_reading_t * + reading); +perfmgr_db_err_t perfmgr_db_get_prev_ps(perfmgr_db_t * db, uint64_t guid, + uint8_t port, + perfmgr_db_sel_reading_t * + reading); +perfmgr_db_err_t perfmgr_db_clear_prev_ps(perfmgr_db_t * db, uint64_t guid, + uint8_t port); void perfmgr_db_clear_counters(perfmgr_db_t * db); perfmgr_db_err_t perfmgr_db_dump(perfmgr_db_t * db, char *file, perfmgr_db_dump_t dump_type); @@ -196,6 +215,8 @@ void perfmgr_db_fill_data_cnt_read_pc(ib_port_counters_t * wire_read, perfmgr_db_data_cnt_reading_t * reading); void perfmgr_db_fill_data_cnt_read_epc(ib_port_counters_ext_t * wire_read, perfmgr_db_data_cnt_reading_t * reading); +void perfmgr_db_fill_sel_read(ib_port_counters_t * wire_read, + perfmgr_db_sel_reading_t * reading); END_C_DECLS diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c index ecfdbda..8a9eb12 100644 --- a/opensm/opensm/osm_perfmgr.c +++ b/opensm/opensm/osm_perfmgr.c @@ -853,10 +853,12 @@ void osm_perfmgr_destroy(osm_perfmgr_t * pm) static void perfmgr_check_oob_clear(osm_perfmgr_t * pm, monitored_node_t * mon_node, uint8_t port, perfmgr_db_err_reading_t * cr, - perfmgr_db_data_cnt_reading_t * dc) + perfmgr_db_data_cnt_reading_t * dc, + perfmgr_db_sel_reading_t * ps) { perfmgr_db_err_reading_t prev_err; perfmgr_db_data_cnt_reading_t prev_dc; + perfmgr_db_sel_reading_t prev_ps; if (perfmgr_db_get_prev_err(pm->db, mon_node->guid, port, &prev_err) != PERFMGR_EVENT_DB_SUCCESS) { @@ -905,6 +907,23 @@ static void perfmgr_check_oob_clear(osm_perfmgr_t * pm, mon_node->name, mon_node->guid, port); perfmgr_db_clear_prev_dc(pm->db, mon_node->guid, port); } + + if (perfmgr_db_get_prev_ps(pm->db, mon_node->guid, port, &prev_ps) + != PERFMGR_EVENT_DB_SUCCESS) { + OSM_LOG(pm->log, OSM_LOG_VERBOSE, + "Failed to find previous select count " + "reading for %s (0x%" PRIx64 ") port %u\n", + mon_node->name, mon_node->guid, port); + return; + } + + if (ps->xmit_wait < prev_ps.xmit_wait) { + OSM_LOG(pm->log, OSM_LOG_ERROR, + "PerfMgr: ERR 4C17: Detected an out of band select counter " + "clear on node %s (0x%" PRIx64 ") port %u\n", + mon_node->name, mon_node->guid, port); + perfmgr_db_clear_prev_ps(pm->db, mon_node->guid, port); + } } /********************************************************************** @@ -1062,6 +1081,8 @@ static void pc_recv_process(void *context, void *data) uint8_t port = mad_context->perfmgr_context.port; perfmgr_db_err_reading_t err_reading; perfmgr_db_data_cnt_reading_t data_reading; + perfmgr_db_sel_reading_t select_reading; + cl_map_item_t *p_node; monitored_node_t *p_mon_node; @@ -1148,10 +1169,12 @@ static void pc_recv_process(void *context, void *data) */ perfmgr_db_fill_data_cnt_read_pc(wire_read, &data_reading); + perfmgr_db_fill_sel_read(wire_read, &select_reading); + /* detect an out of band clear on the port */ if (mad_context->perfmgr_context.mad_method != IB_MAD_METHOD_SET) perfmgr_check_oob_clear(pm, p_mon_node, port, &err_reading, - &data_reading); + &data_reading, &select_reading); /* log any critical events from this reading */ perfmgr_log_events(pm, p_mon_node, port, &err_reading); @@ -1161,9 +1184,12 @@ static void pc_recv_process(void *context, void *data) &err_reading); perfmgr_db_add_dc_reading(pm->db, node_guid, port, &data_reading); + perfmgr_db_add_ps_reading(pm->db, node_guid, port, + &select_reading); } else { perfmgr_db_clear_prev_err(pm->db, node_guid, port); perfmgr_db_clear_prev_dc(pm->db, node_guid, port); + perfmgr_db_clear_prev_ps(pm->db, node_guid, port); } perfmgr_check_overflow(pm, p_mon_node, port, wire_read); diff --git a/opensm/opensm/osm_perfmgr_db.c b/opensm/opensm/osm_perfmgr_db.c index e5dfc19..132c2fb 100644 --- a/opensm/opensm/osm_perfmgr_db.c +++ b/opensm/opensm/osm_perfmgr_db.c @@ -486,6 +486,102 @@ Exit: return (rc); } +static inline void +debug_dump_ps_reading(perfmgr_db_t * db, uint64_t guid, uint8_t port_num, + db_port_t * port, perfmgr_db_sel_reading_t * cur) +{ + osm_log_t *log = db->perfmgr->log; + if (!osm_log_is_active(log, OSM_LOG_DEBUG)) + return; + + osm_log(log, OSM_LOG_DEBUG, + "xd %" PRIu64 " <-- %" PRIu64 " (%" PRIu64 ")\n", + cur->xmit_wait, port->ps_previous.xmit_wait, + port->ps_total.xmit_wait); +} + +/********************************************************************** + * perfmgr_db_sel_reading_t functions + **********************************************************************/ +perfmgr_db_err_t +perfmgr_db_add_ps_reading(perfmgr_db_t * db, uint64_t guid, uint8_t port, + perfmgr_db_sel_reading_t * reading) +{ + db_port_t *p_port = NULL; + db_node_t *node = NULL; + perfmgr_db_sel_reading_t *previous = NULL; + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; + osm_epi_ps_event_t epi_ps_data; + + cl_plock_excl_acquire(&db->lock); + node = get(db, guid); + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) + goto Exit; + + p_port = &node->ports[port]; + previous = &node->ports[port].ps_previous; + + debug_dump_ps_reading(db, guid, port, p_port, reading); + + epi_ps_data.time_diff_s = reading->time - previous->time; + osm_epi_create_port_id(&epi_ps_data.port_id, guid, port, + node->node_name); + + /* calculate changes from previous reading */ + epi_ps_data.xmit_wait = reading->xmit_wait - previous->xmit_wait; + p_port->ps_total.xmit_wait += epi_ps_data.xmit_wait; + + p_port->ps_previous = *reading; + osm_opensm_report_event(db->perfmgr->osm, + OSM_EVENT_ID_PORT_SELECT, &epi_ps_data); + +Exit: + cl_plock_release(&db->lock); + return (rc); +} + +perfmgr_db_err_t perfmgr_db_get_prev_ps(perfmgr_db_t * db, uint64_t guid, + uint8_t port, + perfmgr_db_sel_reading_t * reading) +{ + db_node_t *node = NULL; + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; + + cl_plock_acquire(&db->lock); + + node = get(db, guid); + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) + goto Exit; + + *reading = node->ports[port].ps_previous; + +Exit: + cl_plock_release(&db->lock); + return (rc); +} + +perfmgr_db_err_t +perfmgr_db_clear_prev_ps(perfmgr_db_t * db, uint64_t guid, uint8_t port) +{ + db_node_t *node = NULL; + perfmgr_db_sel_reading_t *previous = NULL; + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; + + cl_plock_excl_acquire(&db->lock); + node = get(db, guid); + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) + goto Exit; + + previous = &node->ports[port].ps_previous; + + memset(previous, 0, sizeof(*previous)); + node->ports[port].ps_previous.time = time(NULL); + +Exit: + cl_plock_release(&db->lock); + return (rc); +} + static void clear_counters(cl_map_item_t * const p_map_item, void *context) { db_node_t *node = (db_node_t *) p_map_item; @@ -517,6 +613,8 @@ static void clear_counters(cl_map_item_t * const p_map_item, void *context) node->ports[i].dc_total.multicast_rcv_pkts = 0; node->ports[i].dc_total.time = ts; + node->ports[i].ps_total.xmit_wait = 0; + node->ports[i].last_reset = ts; } } @@ -546,7 +644,7 @@ static void dump_node_mr(db_node_t * node, FILE * fp) "%s\t%s\t" "%s\t%s\t%s\t%s\t%s\t%s\t%s\t" "%s\t%s\t%s\t%s\t%s\t%s\t%s\t" - "%s\t%s\t%s\t%s\n", + "%s\t%s\t%s\t%s\t%s\n", "symbol_err_cnt", "link_err_recover", "link_downed", @@ -565,8 +663,7 @@ static void dump_node_mr(db_node_t * node, FILE * fp) "rcv_pkts", "unicast_xmit_pkts", "unicast_rcv_pkts", - "multicast_xmit_pkts", - "multicast_rcv_pkts"); + "multicast_xmit_pkts", "multicast_rcv_pkts", "xmit_wait"); for (i = (node->esp0) ? 0 : 1; i < node->num_ports; i++) { char *since = ctime(&node->ports[i].last_reset); since[strlen(since) - 1] = '\0'; /* remove \n */ @@ -577,8 +674,8 @@ static void dump_node_mr(db_node_t * node, FILE * fp) "%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 - "\t%" PRIu64 "\t%" PRIu64 "\n", node->node_name, - node->node_guid, i, since, + "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\n", + node->node_name, node->node_guid, i, since, node->ports[i].err_total.symbol_err_cnt, node->ports[i].err_total.link_err_recover, node->ports[i].err_total.link_downed, @@ -598,7 +695,8 @@ static void dump_node_mr(db_node_t * node, FILE * fp) node->ports[i].dc_total.unicast_xmit_pkts, node->ports[i].dc_total.unicast_rcv_pkts, node->ports[i].dc_total.multicast_xmit_pkts, - node->ports[i].dc_total.multicast_rcv_pkts); + node->ports[i].dc_total.multicast_rcv_pkts, + node->ports[i].ps_total.xmit_wait); } } @@ -634,7 +732,8 @@ static void dump_node_hr(db_node_t * node, FILE * fp) " unicast_xmit_pkts : %" PRIu64 "\n" " unicast_rcv_pkts : %" PRIu64 "\n" " multicast_xmit_pkts : %" PRIu64 "\n" - " multicast_rcv_pkts : %" PRIu64 "\n", + " multicast_rcv_pkts : %" PRIu64 "\n" + " xmit_wait : %" PRIu64 "\n", node->node_name, node->node_guid, i, @@ -658,7 +757,8 @@ static void dump_node_hr(db_node_t * node, FILE * fp) node->ports[i].dc_total.unicast_xmit_pkts, node->ports[i].dc_total.unicast_rcv_pkts, node->ports[i].dc_total.multicast_xmit_pkts, - node->ports[i].dc_total.multicast_rcv_pkts); + node->ports[i].dc_total.multicast_rcv_pkts, + node->ports[i].ps_total.xmit_wait); } } @@ -809,4 +909,12 @@ perfmgr_db_fill_data_cnt_read_epc(ib_port_counters_ext_t * wire_read, reading->multicast_rcv_pkts = cl_ntoh64(wire_read->multicast_rcv_pkts); reading->time = time(NULL); } + +void +perfmgr_db_fill_sel_read(ib_port_counters_t * wire_read, + perfmgr_db_sel_reading_t * reading) +{ + reading->xmit_wait = cl_ntoh32(wire_read->xmit_wait); + reading->time = time(NULL); +} #endif /* ENABLE_OSM_PERF_MGR */ From hal.rosenstock at gmail.com Tue Jul 14 08:52:20 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Tue, 14 Jul 2009 11:52:20 -0400 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Eliminate possible seg fault in get_osm_switch_from_port In-Reply-To: <20090714145306.GF15346@me> References: <20090711114815.GA12867@comcast.net> <20090713001126.GJ12543@me> <20090714014445.GN20347@me> <20090714145306.GF15346@me> Message-ID: On Tue, Jul 14, 2009 at 10:53 AM, Sasha Khapyorsky wrote: > On 07:03 Tue 14 Jul     , Hal Rosenstock wrote: >> > >> > It is what I asked - why is NULL pointer used there (in SA PathRecord >> > flow) and is is legal usage? >> >> It shouldn't be a valid use case. > > So why do we need this patch then? How about a CL_ASSERT there ? > >> > As far as I can see in osm_sa_path_record.c it should be impossible, but >> > maybe I'm missing something. >> >> Yes, that's what I thought too but apparently it doesn't appear to be >> the case and I'm not sure how that could occur. Any ideas ? > > If you are able to reproduce such case try to find why this happens > (crash OpenSM into gdb) I put core into gdb but it doesn't show how p_port became NULL and test time to recreate is not easy to get but it's on the list... -- Hal > - to hide bugs is not a good idea IMO. > Sasha > From amirv at mellanox.co.il Tue Jul 14 09:02:50 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Tue, 14 Jul 2009 19:02:50 +0300 Subject: [ofa-general] SDP and stock kernel gets BUG? In-Reply-To: <003701ca0494$f75368b0$e5fa3a10$@van.malland@linvision.com> References: <20090714120105.GA11987@obsidianresearch.com> <4A5C84D3.9060901@mellanox.co.il> <20090714143152.GA12386@obsidianresearch.com> <4A5C981E.1080108@mellanox.co.il> <003701ca0494$f75368b0$e5fa3a10$@van.malland@linvision.com> Message-ID: <4A5CAC2A.5020000@mellanox.co.il> Thanks, I will check it. - Amir On 07/14/2009 06:08 PM, Maarten van Malland wrote: > Hi Amir, > > I'm testing with Jason to get this SDP working properly on a 2.6.30.1 stock > kernel. > > With the SDP from OFED 1.5 this is the performance we're getting: > > inftsttwin03 ~ # LD_PRELOAD=/opt/ofa-1.5/lib/libsdp.so nttcp -r -T -l 128000 > twin1-ibl > Bytes Real s CPU s Real-MBit/s CPU-MBit/s Calls Real-C/s > CPU-C/s > l262144000 0.98 0.28 2142.7300 7599.5333 3167 3235.83 > 11476.4 > 1262144000 0.98 0.02 2143.1132 131096.5806 2048 2092.88 > 128024.0 > inftsttwin03 ~ # LD_PRELOAD=/opt/ofa-1.5/lib/libsdp.so nttcp -r -T -l 128000 > twin1-ibl > Bytes Real s CPU s Real-MBit/s CPU-MBit/s Calls Real-C/s > CPU-C/s > l262144000 1.20 0.05 1748.5024 41948.9128 3345 2788.90 > 66909.4 > 1262144000 1.14 0.01 1847.3367 299678.7654 2048 1804.04 > 292655.0 > > However, with the SDP from OFED 1.4 we're getting this performance: > > inftsttwin03 ~ # LD_PRELOAD=/opt/ofa-1.5/lib/libsdp.so nttcp -r -T -l 128000 > twin1-ibl > Bytes Real s CPU s Real-MBit/s CPU-MBit/s Calls Real-C/s > CPU-C/s > l262144000 0.15 0.14 14411.9300 14565.7809 7007 48153.11 > 48667.2 > 1262144000 0.15 0.10 14410.4446 21848.5196 2048 14072.70 > 21336.4 > inftsttwin03 ~ # LD_PRELOAD=/opt/ofa-1.5/lib/libsdp.so nttcp -r -T -l 128000 > twin1-ibl > Bytes Real s CPU s Real-MBit/s CPU-MBit/s Calls Real-C/s > CPU-C/s > l262144000 0.15 0.14 14286.1658 14465.3120 6614 45055.72 > 45620.7 > 1262144000 0.15 0.09 14312.5883 22798.5998 2048 13977.14 > 22264.3 > > The output from /proc/net/sdpstats is: > > inftsttwin03 ~ # cat /proc/net/sdpstats > SDP statistics: > sendmsg_seglen: > 1 | - 0 > 2 | - 0 > 4 | - 0 > 8 | ************************************************** - 1 > 16 | - 0 > 32 | - 0 > 64 | ************************************************** - 1 > 128 | - 0 > 256 | - 0 > 512 | - 0 > 1024 | - 0 > 2048 | - 0 > 4096 | - 0 > 8192 | - 0 > 16384 | - 0 > 32768 | - 0 > 65536 | - 0 > 131072 | - 0 > 262144 | - 0 > 524288 | - 0 > 1048576 | - 0 > 2097152 | - 0 > 4194304 | - 0 > 8388608 | - 0 > 0 | - 0 > send_size: > 1 | - 0 > 2 | - 0 > 4 | - 0 > 8 | - 0 > 16 | ************************************************** - 325 > 32 | - 0 > 64 | - 1 > 128 | - 0 > 256 | - 0 > 512 | - 0 > 1024 | - 0 > 2048 | - 0 > 4096 | - 0 > 8192 | - 0 > 16384 | - 0 > 32768 | - 0 > 65536 | - 0 > 131072 | - 0 > 262144 | - 0 > 524288 | - 0 > 1048576 | - 0 > 2097152 | - 0 > 4194304 | - 0 > 8388608 | - 0 > 0 | - 0 > credits_before_update: > 0 | - 0 > 1 | - 0 > 2 | - 0 > 3 | - 0 > 4 | - 0 > 5 | - 0 > 6 | - 0 > 7 | - 0 > 8 | - 0 > 9 | - 0 > 10 | - 0 > 11 | - 0 > 12 | - 0 > 13 | - 0 > 14 | - 0 > 15 | - 0 > 16 | - 0 > 17 | - 0 > 18 | - 0 > 19 | - 0 > 20 | - 0 > 21 | - 0 > 22 | - 0 > 23 | - 0 > 24 | - 0 > 25 | - 0 > 26 | - 0 > 27 | - 0 > 28 | - 0 > 29 | - 0 > 30 | - 0 > 31 | - 0 > 32 | - 0 > 33 | - 0 > 34 | - 0 > 35 | - 0 > 36 | - 0 > 37 | - 0 > 38 | - 0 > 39 | - 0 > 40 | - 0 > 41 | - 0 > 42 | - 0 > 43 | - 0 > 44 | - 0 > 45 | - 0 > 46 | - 0 > 47 | - 0 > 48 | - 0 > 49 | - 0 > 50 | - 0 > 51 | - 0 > 52 | - 0 > 53 | - 1 > 54 | ************ - 1619 > 55 | ************************************************** - 6404 > 56 | - 0 > 57 | - 0 > 58 | - 0 > 59 | - 0 > 60 | - 0 > 61 | - 0 > 62 | - 0 > 63 | - 0 > sdp_sendmsg() calls : 2 > bcopy segments : 2 > bzcopy segments : 0 > post_send_credits : 322 > memcpy_count : 98 > post_send SDP_MID_HELLO : 0 > post_send SDP_MID_HELLO_ACK : 0 > post_send SDP_MID_DISCONN : 2 > post_send SDP_MID_CHRCVBUF : 0 > post_send SDP_MID_CHRCVBUF_ACK : 0 > post_send SDP_MID_DATA : 324 > > post_recv : 8134 > BZCopy poll miss : 0 > send_wait_for_mem : 0 > send_miss_no_credits : 0 > rx_poll_miss : 0 > tx_poll_miss : 3 > tx_poll_busy : 0 > tx_poll_hit : 3 > CQ stats: > - RX interrupts : 3151 > - TX interrupts : 0 > bz_clean : 180 > bz_setup : 345 > tx_copy : 2756 > sendmsg : 535432 > > The first 100 lines from /proc/net/sdpprf: > > inftsttwin03 ~ # head -100 /proc/net/sdpprf > 0 : [ 0.000000] TX: SDP_MID_DATA bufs: 55 mseq:1 ack:0 - > [7916{3} 48477:5037] skb: ffff880123186d80 sdp_post_send:99 > 1 : [ 0.000248] tx completion. mseq:1 - > [7916{3} 48477:5037] skb: ffff880123186d80 sdp_handle_send_comp:220 > 2 : [ 0.099838] TX: SDP_MID_DATA bufs: 55 mseq:2 ack:0 - > [0{3} 48477:5037] skb: ffff880123186bc0 sdp_post_send:99 > 3 : [ 0.102805] RX SDP_MID_DATA +55 c:55->55 mseq:1 ack:0 - > [0{0} 5038:51962] skb: ffff88011e055100 sdp_process_rx_skb:489 > 4 : [ 0.102807] Waking up sleepers - > [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 5 : [ 0.102886] READ finished. mseq: 1 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011e055100 sdp_recvmsg:2146 > 6 : [ 0.102862] RX SDP_MID_DATA +55 c:55->55 mseq:2 ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78dc0 sdp_process_rx_skb:489 > 7 : [ 0.102862] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 8 : [ 0.102960] READ finished. mseq: 2 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78dc0 sdp_recvmsg:2146 > 9 : [ 0.102927] RX SDP_MID_DATA +55 c:55->55 mseq:3 ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78c00 sdp_process_rx_skb:489 > 10 : [ 0.102927] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 11 : [ 0.103024] READ finished. mseq: 3 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78c00 sdp_recvmsg:2146 > 12 : [ 0.103264] RX SDP_MID_DATA +55 c:55->55 mseq:4 ack:0 - > [0{0} 5038:51962] skb: ffff88011dc78a40 sdp_process_rx_skb:489 > 13 : [ 0.103265] Waking up sleepers - > [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 14 : [ 0.103320] READ finished. mseq: 4 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78a40 sdp_recvmsg:2146 > 15 : [ 0.103352] RX SDP_MID_DATA +55 c:55->55 mseq:5 ack:0 - > [0{0} 5038:51962] skb: ffff88011dc78880 sdp_process_rx_skb:489 > 16 : [ 0.103353] Waking up sleepers - > [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 17 : [ 0.103362] RX SDP_MID_DATA +55 c:55->55 mseq:6 ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc786c0 sdp_process_rx_skb:489 > 18 : [ 0.103363] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 19 : [ 0.103422] READ finished. mseq: 5 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78880 sdp_recvmsg:2146 > 20 : [ 0.103376] RX SDP_MID_DATA +55 c:55->55 mseq:7 ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78500 sdp_process_rx_skb:489 > 21 : [ 0.103377] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 22 : [ 0.103398] RX SDP_MID_DATA +55 c:55->55 mseq:8 ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78340 sdp_process_rx_skb:489 > 23 : [ 0.103398] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 24 : [ 0.103408] RX SDP_MID_DATA +55 c:55->55 mseq:9 ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78180 sdp_process_rx_skb:489 > 25 : [ 0.103408] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 26 : [ 0.103420] RX SDP_MID_DATA +55 c:55->55 mseq:10 ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3e00 sdp_process_rx_skb:489 > 27 : [ 0.103421] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 28 : [ 0.103429] RX SDP_MID_DATA +55 c:55->55 mseq:11 ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3c40 sdp_process_rx_skb:489 > 29 : [ 0.103429] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 30 : [ 0.103479] READ finished. mseq: 6 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc786c0 sdp_recvmsg:2146 > 31 : [ 0.103532] READ finished. mseq: 7 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78500 sdp_recvmsg:2146 > 32 : [ 0.103598] READ finished. mseq: 8 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78340 sdp_recvmsg:2146 > 33 : [ 0.103612] READ finished. mseq: 9 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011dc78180 sdp_recvmsg:2146 > 34 : [ 0.103626] READ finished. mseq: 10 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3e00 sdp_recvmsg:2146 > 35 : [ 0.103640] RX SDP_MID_DATA +55 c:55->55 mseq:12 ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3a80 sdp_process_rx_skb:489 > 36 : [ 0.103640] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 37 : [ 0.103644] READ finished. mseq: 11 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3c40 sdp_recvmsg:2146 > 38 : [ 0.103655] RX SDP_MID_DATA +55 c:55->55 mseq:13 ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b38c0 sdp_process_rx_skb:489 > 39 : [ 0.103656] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 40 : [ 0.103661] RX SDP_MID_DATA +55 c:55->55 mseq:14 ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3700 sdp_process_rx_skb:489 > 41 : [ 0.103661] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 42 : [ 0.103671] RX SDP_MID_DATA +55 c:55->55 mseq:15 ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3540 sdp_process_rx_skb:489 > 43 : [ 0.103671] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 44 : [ 0.103679] READ finished. mseq: 12 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3a80 sdp_recvmsg:2146 > 45 : [ 0.103695] READ finished. mseq: 13 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b38c0 sdp_recvmsg:2146 > 46 : [ 0.103712] READ finished. mseq: 14 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3700 sdp_recvmsg:2146 > 47 : [ 0.103728] READ finished. mseq: 15 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3540 sdp_recvmsg:2146 > 48 : [ 0.104631] RX SDP_MID_DATA +55 c:55->55 mseq:16 ack:0 - > [0{0} 5038:51962] skb: ffff8801231b3380 sdp_process_rx_skb:489 > 49 : [ 0.104632] Waking up sleepers - > [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 50 : [ 0.104641] RX SDP_MID_DATA +55 c:55->55 mseq:17 ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b31c0 sdp_process_rx_skb:489 > 51 : [ 0.104642] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 52 : [ 0.104653] RX SDP_MID_DATA +55 c:55->55 mseq:18 ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8cc0 sdp_process_rx_skb:489 > 53 : [ 0.104653] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 54 : [ 0.104657] READ finished. mseq: 16 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b3380 sdp_recvmsg:2146 > 55 : [ 0.104661] RX SDP_MID_DATA +55 c:55->55 mseq:19 ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8b00 sdp_process_rx_skb:489 > 56 : [ 0.104662] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 57 : [ 0.104674] READ finished. mseq: 17 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff8801231b31c0 sdp_recvmsg:2146 > 58 : [ 0.104688] READ finished. mseq: 18 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8cc0 sdp_recvmsg:2146 > 59 : [ 0.104703] READ finished. mseq: 19 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8b00 sdp_recvmsg:2146 > 60 : [ 0.105630] RX SDP_MID_DATA +55 c:55->55 mseq:20 ack:0 - > [0{0} 5038:51962] skb: ffff88011e0b8940 sdp_process_rx_skb:489 > 61 : [ 0.105632] Waking up sleepers - > [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 62 : [ 0.105640] RX SDP_MID_DATA +55 c:55->55 mseq:21 ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8780 sdp_process_rx_skb:489 > 63 : [ 0.105641] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 64 : [ 0.105652] RX SDP_MID_DATA +55 c:55->55 mseq:22 ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b85c0 sdp_process_rx_skb:489 > 65 : [ 0.105652] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 66 : [ 0.105656] READ finished. mseq: 20 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8940 sdp_recvmsg:2146 > 67 : [ 0.105661] RX SDP_MID_DATA +55 c:55->55 mseq:23 ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8400 sdp_process_rx_skb:489 > 68 : [ 0.105661] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 69 : [ 0.105673] READ finished. mseq: 21 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8780 sdp_recvmsg:2146 > 70 : [ 0.105687] READ finished. mseq: 22 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b85c0 sdp_recvmsg:2146 > 71 : [ 0.105702] READ finished. mseq: 23 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8400 sdp_recvmsg:2146 > 72 : [ 0.106630] RX SDP_MID_DATA +55 c:55->55 mseq:24 ack:0 - > [0{0} 5038:51962] skb: ffff88011e0b8240 sdp_process_rx_skb:489 > 73 : [ 0.106631] Waking up sleepers - > [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 74 : [ 0.106640] RX SDP_MID_DATA +55 c:55->55 mseq:25 ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8080 sdp_process_rx_skb:489 > 75 : [ 0.106640] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 76 : [ 0.106652] RX SDP_MID_DATA +55 c:55->55 mseq:26 ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a1d00 sdp_process_rx_skb:489 > 77 : [ 0.106653] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 78 : [ 0.106656] READ finished. mseq: 24 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8240 sdp_recvmsg:2146 > 79 : [ 0.106661] RX SDP_MID_DATA +55 c:55->55 mseq:27 ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a1b40 sdp_process_rx_skb:489 > 80 : [ 0.106661] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 81 : [ 0.106673] READ finished. mseq: 25 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011e0b8080 sdp_recvmsg:2146 > 82 : [ 0.106686] READ finished. mseq: 26 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a1d00 sdp_recvmsg:2146 > 83 : [ 0.106701] READ finished. mseq: 27 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a1b40 sdp_recvmsg:2146 > 84 : [ 0.107630] RX SDP_MID_DATA +55 c:55->55 mseq:28 ack:0 - > [0{0} 5038:51962] skb: ffff88011b5a1980 sdp_process_rx_skb:489 > 85 : [ 0.107632] Waking up sleepers - > [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 86 : [ 0.107640] RX SDP_MID_DATA +55 c:55->55 mseq:29 ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a17c0 sdp_process_rx_skb:489 > 87 : [ 0.107641] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 88 : [ 0.107652] RX SDP_MID_DATA +55 c:55->55 mseq:30 ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a1600 sdp_process_rx_skb:489 > 89 : [ 0.107653] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 90 : [ 0.107658] TX: SDP_MID_DATA bufs: 53 mseq:1 ack:30 - > [7916{0} 5038:51962] skb: ffff88011b67c180 sdp_post_send:99 > 91 : [ 0.107662] READ finished. mseq: 28 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a1980 sdp_recvmsg:2146 > 92 : [ 0.107661] RX SDP_MID_DATA +55 c:54->54 mseq:31 ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a1440 sdp_process_rx_skb:489 > 93 : [ 0.107661] Waking up sleepers - > [7916{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > 94 : [ 0.107677] tx completion. mseq:1 - > [7916{0} 5038:51962] skb: ffff88011b67c180 sdp_handle_send_comp:220 > 95 : [ 0.107678] READ finished. mseq: 29 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a17c0 sdp_recvmsg:2146 > 96 : [ 0.107691] READ finished. mseq: 30 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a1600 sdp_recvmsg:2146 > 97 : [ 0.107706] READ finished. mseq: 31 mseq_ack:0 - > [7916{0} 5038:51962] skb: ffff88011b5a1440 sdp_recvmsg:2146 > 98 : [ 0.107742] RX SDP_MID_DATA +55 c:54->55 mseq:32 ack:1 - > [0{0} 5038:51962] skb: ffff88011b5a1280 sdp_process_rx_skb:489 > 99 : [ 0.107743] Waking up sleepers - > [0{0} 5038:51962] skb: (null) sdp_bzcopy_write_space:602 > > > Hope this helps, > > Maarten > > -----Oorspronkelijk bericht----- > Van: Amir Vadai [mailto:amirv at mellanox.co.il] > Verzonden: dinsdag 14 juli 2009 16:37 > Aan: Jason Gunthorpe > CC: general-list > Onderwerp: Re: [ofa-general] SDP and stock kernel gets BUG? > > OFED-1.5 SDP has many changes in the data path. > Performance should be improved - although it is still work in progress. > > You could use /proc/sdp/sdpstats and /proc/sdp/sdpprf facilities that > has been added in 1.5. > (enabled in sdp.h by defining macros SDPSTATS_ON and SDP_PROFILING). > > What packet sizes do you use? what is the setup in general? > > Let me know if you find something interesting. > > - Amir > > On 07/14/2009 05:31 PM, Jason Gunthorpe wrote: > >> On Tue, Jul 14, 2009 at 04:14:59PM +0300, Amir Vadai wrote: >> >> >>> Hi, >>> >>> I will post a fix soon. >>> >>> >> Thanks Amir! >> >> BTW - we are testing SDP here and trying to track down a performance >> regression - using the 2.6.30.1 combined with OFED-1.5 SDP performs >> poorly while 2.6.27.10 combined with OFED-1.4 SDP performs well. We >> have not yet narrowed down what is going on.. Don't suppose you have >> any insight? >> >> Regards, >> Jason >> >> > From sashak at voltaire.com Tue Jul 14 09:16:11 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Tue, 14 Jul 2009 19:16:11 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Eliminate possible seg fault in get_osm_switch_from_port In-Reply-To: References: <20090711114815.GA12867@comcast.net> <20090713001126.GJ12543@me> <20090714014445.GN20347@me> <20090714145306.GF15346@me> Message-ID: <20090714161611.GB22163@me> On 11:52 Tue 14 Jul , Hal Rosenstock wrote: > > How about a CL_ASSERT there ? Not much better than segfault :). I think that we don't need to patch the main stream code until we understand the problem. Sasha > > If you are able to reproduce such case try to find why this happens > > (crash OpenSM into gdb) > > I put core into gdb but it doesn't show how p_port became NULL try 'backtrace'. Sasha From ranjit.pandit.ib at gmail.com Tue Jul 14 09:57:05 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Tue, 14 Jul 2009 09:57:05 -0700 Subject: [ofa-general] Memory registration limit of 16GB with Chelsio In-Reply-To: <96f8e60e0907130949y7edf6926l6e70dafc44a7503a@mail.gmail.com> References: <96f8e60e0907101115i11fcf051h8508fdc18282368b@mail.gmail.com> <4A578C4D.2060003@opengridcomputing.com> <8A71B368A89016469F72CD08050AD33405E63DDE@maui.asicdesigners.com> <96f8e60e0907130949y7edf6926l6e70dafc44a7503a@mail.gmail.com> Message-ID: <96f8e60e0907140957k20675d18v284b4e1be4caa9ae@mail.gmail.com> Felix, >> [felix] Not dynamically, but it can surely be changed. DM, can it be >> controlled through a cxgbtool? >> Can you send me some pointers to changes I can make in the code to experiment with? For now, it might be OK for us to work around this issue by applying a patch. thanks, Ranjit From hal.rosenstock at gmail.com Tue Jul 14 10:05:32 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Tue, 14 Jul 2009 13:05:32 -0400 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Eliminate possible seg fault in get_osm_switch_from_port In-Reply-To: <20090714161611.GB22163@me> References: <20090711114815.GA12867@comcast.net> <20090713001126.GJ12543@me> <20090714014445.GN20347@me> <20090714145306.GF15346@me> <20090714161611.GB22163@me> Message-ID: On Tue, Jul 14, 2009 at 12:16 PM, Sasha Khapyorsky wrote: > On 11:52 Tue 14 Jul     , Hal Rosenstock wrote: >> >> How about a CL_ASSERT there ? > > Not much better than segfault :). Not much better but still slightly better. If not, then what about all the other CL_ASSERTs in the code ? > I think that we don't need to patch > the main stream code until we understand the problem. OK. > Sasha > >> > If you are able to reproduce such case try to find why this happens >> > (crash OpenSM into gdb) >> >> I put core into gdb but it doesn't show how p_port became NULL > > try 'backtrace'. Of course I did that. -- Hal > Sasha > From sashak at voltaire.com Tue Jul 14 10:18:25 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Tue, 14 Jul 2009 20:18:25 +0300 Subject: [ofa-general] Re: [PATCH] opensm/lash: Set minimum VL for LASH to use In-Reply-To: References: <20090618141430.GA11942@comcast.net> <20090623214928.GG4324@me> Message-ID: <20090714171825.GB27223@me> Hi Hal, On 11:53 Mon 06 Jul , Hal Rosenstock wrote: > >> @@ -1286,7 +1315,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port, > >> > >> ?? ?? ?? src_id = get_lash_id(p_sw); > >> ?? ?? ?? if (src_id == dst_id) > >> - ?? ?? ?? ?? ?? ?? return OSM_DEFAULT_SL; > >> + ?? ?? ?? ?? ?? ?? return opt->lash_start_vl; > > > > Is this correct? As far as I understand this is SL for paths between > > CA ports connected to the same switch (which should be safe in sense of > > credit loops). Should lash be involved here? > > Is there a reason not to use the LASH SL for this ? I think it's more > in the spirit of LASH to do this. OTOH it changes the default(current) behavior, and it would be nice to have a reason to do this. Sasha From swise at opengridcomputing.com Tue Jul 14 10:20:56 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Tue, 14 Jul 2009 12:20:56 -0500 Subject: [ofa-general] Memory registration limit of 16GB with Chelsio In-Reply-To: <96f8e60e0907140957k20675d18v284b4e1be4caa9ae@mail.gmail.com> References: <96f8e60e0907101115i11fcf051h8508fdc18282368b@mail.gmail.com> <4A578C4D.2060003@opengridcomputing.com> <8A71B368A89016469F72CD08050AD33405E63DDE@maui.asicdesigners.com> <96f8e60e0907130949y7edf6926l6e70dafc44a7503a@mail.gmail.com> <96f8e60e0907140957k20675d18v284b4e1be4caa9ae@mail.gmail.com> Message-ID: <4A5CBE78.9020000@opengridcomputing.com> You could try something like this. This removes the ISCSI and TOE/DDP memory usage, and bumps the PBL memory up to twice what it was. (this is untested). Steve. ---- --- drivers/net/cxgb3/t3_hw.c.org 2009-07-14 12:14:45.000000000 -0500 +++ drivers/net/cxgb3/t3_hw.c 2009-07-14 12:16:32.000000000 -0500 @@ -3084,13 +3084,13 @@ static void ulp_config(struct adapter *a { unsigned int m = p->chan_rx_size; - ulp_region(adap, ISCSI, m, p->chan_rx_size / 8); - ulp_region(adap, TDDP, m, p->chan_rx_size / 8); + ulp_region(adap, ISCSI, m, 0); + ulp_region(adap, TDDP, m, 0); ulptx_region(adap, TPT, m, p->chan_rx_size / 4); ulp_region(adap, STAG, m, p->chan_rx_size / 4); ulp_region(adap, RQ, m, p->chan_rx_size / 4); - ulptx_region(adap, PBL, m, p->chan_rx_size / 4); - ulp_region(adap, PBL, m, p->chan_rx_size / 4); + ulptx_region(adap, PBL, m, p->chan_rx_size / 2); + ulp_region(adap, PBL, m, p->chan_rx_size / 2); t3_write_reg(adap, A_ULPRX_TDDP_TAGMASK, 0xffffffff); } From sashak at voltaire.com Tue Jul 14 10:22:56 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Tue, 14 Jul 2009 20:22:56 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Eliminate possible seg fault in get_osm_switch_from_port In-Reply-To: References: <20090711114815.GA12867@comcast.net> <20090713001126.GJ12543@me> <20090714014445.GN20347@me> <20090714145306.GF15346@me> <20090714161611.GB22163@me> Message-ID: <20090714172256.GC27223@me> On 13:05 Tue 14 Jul , Hal Rosenstock wrote: > On Tue, Jul 14, 2009 at 12:16 PM, Sasha Khapyorsky wrote: > > On 11:52 Tue 14 Jul ?? ?? , Hal Rosenstock wrote: > >> > >> How about a CL_ASSERT there ? > > > > Not much better than segfault :). > > Not much better but still slightly better. If not, then what about all > the other CL_ASSERTs in the code ? I cannot change the history :) , but seriously it probably was helpful for somebody during a debug period. Sasha From hal.rosenstock at gmail.com Tue Jul 14 10:22:13 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Tue, 14 Jul 2009 13:22:13 -0400 Subject: [ofa-general] Re: [PATCH] opensm/lash: Set minimum VL for LASH to use In-Reply-To: <20090714171825.GB27223@me> References: <20090618141430.GA11942@comcast.net> <20090623214928.GG4324@me> <20090714171825.GB27223@me> Message-ID: On Tue, Jul 14, 2009 at 1:18 PM, Sasha Khapyorsky wrote: > Hi Hal, > > On 11:53 Mon 06 Jul     , Hal Rosenstock wrote: >> >> @@ -1286,7 +1315,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port, >> >> >> >> ?? ?? ?? src_id = get_lash_id(p_sw); >> >> ?? ?? ?? if (src_id == dst_id) >> >> - ?? ?? ?? ?? ?? ?? return OSM_DEFAULT_SL; >> >> + ?? ?? ?? ?? ?? ?? return opt->lash_start_vl; >> > >> > Is this correct? As far as I understand this is SL for paths between >> > CA ports connected to the same switch (which should be safe in sense of >> > credit loops). Should lash be involved here? >> >> Is there a reason not to use the LASH SL for this ? I think it's more >> in the spirit of LASH to do this. > > OTOH it changes the default(current) behavior, and it would be nice to > have a reason to do this. The reason is to use the QoS parameters of LASH rather than the default SL. If you want, this can be separate as I would like to move the rest of this ahead if possible. -- Hal > > Sasha > From robert.j.woodruff at intel.com Tue Jul 14 10:31:22 2009 From: robert.j.woodruff at intel.com (Woodruff, Robert J) Date: Tue, 14 Jul 2009 10:31:22 -0700 Subject: [ofa-general] RE: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support only QP1 In-Reply-To: References: <20090713181410.GA754@mtls03> <382A478CAD40FA4FB46605CF81FE39F435877A24@orsmsx507.amr.corp.intel.com> Message-ID: <382A478CAD40FA4FB46605CF81FE39F435878175@orsmsx507.amr.corp.intel.com> Hal wrote, >Unfortunately I don't think it's this simple although I wish it were. >IBXOE is on a per port rather than a per node basis which is a >different model than we've used for IB or iWARP. >-- Hal Yuk... Can the driver/hardware present itself to the upper core layers as two separate NICs, ever though the hardware has both an IB port and an Ethernet port on one card ? as a multi-function PCI device or something ? From rdreier at cisco.com Tue Jul 14 11:36:44 2009 From: rdreier at cisco.com (Roland Dreier) Date: Tue, 14 Jul 2009 11:36:44 -0700 Subject: [ofa-general] Re: 2.6.30.1: possible irq lock inversion dependency detected In-Reply-To: (Bart Van Assche's message of "Sat, 11 Jul 2009 12:43:19 +0200") References: Message-ID: > With the two patches applied on top of 2.6.31.1 lockdep stopped > complaining on my setup. By the way, do you know whether this is a > long-standing issue or a result of the changes between 2.6.29 and > 2.6.30 ? Odd. The problem I fixed with my second patch was long-standing I think; the change from .29 to .30 was the addition of lockdep debugging for timers, so I could see how the first report could pop up only under .30 (since that involved a timer). Oh well, I'll look at cleaning up the second patch and pushing it upstream. From rdreier at cisco.com Tue Jul 14 11:48:52 2009 From: rdreier at cisco.com (Roland Dreier) Date: Tue, 14 Jul 2009 11:48:52 -0700 Subject: [ofa-general] [GIT PULL] please pull infiniband.git Message-ID: Linus, please pull from master.kernel.org:/pub/scm/linux/kernel/git/roland/infiniband.git for-linus This tree is also available from kernel.org mirrors at: git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband.git for-linus This will get a new PCI ID for the mlx4 driver, which also requires handling one new firmware status code. The chance of regressions should be near-nil, since existing devices won't hit any of the new code (and the worst case seems to be printing an incorrect error message anyway). Yevgeny Petrilin (2): mlx4_core: Handle multi-physical function devices mlx4_core: Add new ConnectX EN PCI ID 0x6764 drivers/net/mlx4/cmd.c | 5 ++++- drivers/net/mlx4/main.c | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c index 2845a05..65ec77d 100644 --- a/drivers/net/mlx4/cmd.c +++ b/drivers/net/mlx4/cmd.c @@ -80,7 +80,9 @@ enum { /* Bad management packet (silently discarded): */ CMD_STAT_BAD_PKT = 0x30, /* More outstanding CQEs in CQ than new CQ size: */ - CMD_STAT_BAD_SIZE = 0x40 + CMD_STAT_BAD_SIZE = 0x40, + /* Multi Function device support required: */ + CMD_STAT_MULTI_FUNC_REQ = 0x50, }; enum { @@ -128,6 +130,7 @@ static int mlx4_status_to_errno(u8 status) [CMD_STAT_LAM_NOT_PRE] = -EAGAIN, [CMD_STAT_BAD_PKT] = -EINVAL, [CMD_STAT_BAD_SIZE] = -ENOMEM, + [CMD_STAT_MULTI_FUNC_REQ] = -EACCES, }; if (status >= ARRAY_SIZE(trans_table) || diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c index 018348c..dac621b 100644 --- a/drivers/net/mlx4/main.c +++ b/drivers/net/mlx4/main.c @@ -729,7 +729,10 @@ static int mlx4_init_hca(struct mlx4_dev *dev) err = mlx4_QUERY_FW(dev); if (err) { - mlx4_err(dev, "QUERY_FW command failed, aborting.\n"); + if (err == -EACCES) + mlx4_info(dev, "non-primary physical function, skipping.\n"); + else + mlx4_err(dev, "QUERY_FW command failed, aborting.\n"); return err; } @@ -1285,6 +1288,7 @@ static struct pci_device_id mlx4_pci_table[] = { { PCI_VDEVICE(MELLANOX, 0x6750) }, /* MT25408 "Hermon" EN 10GigE PCIe gen2 */ { PCI_VDEVICE(MELLANOX, 0x6372) }, /* MT25458 ConnectX EN 10GBASE-T 10GigE */ { PCI_VDEVICE(MELLANOX, 0x675a) }, /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */ + { PCI_VDEVICE(MELLANOX, 0x6764) }, /* MT26468 ConnectX EN 10GigE PCIe gen2*/ { 0, } }; From ijstokes at crystal.harvard.edu Tue Jul 14 12:04:37 2009 From: ijstokes at crystal.harvard.edu (Ian Stokes-Rees) Date: Tue, 14 Jul 2009 15:04:37 -0400 Subject: [ofa-general] IB introductory material Message-ID: <4A5CD6C5.7040509@crystal.harvard.edu> Hello, We have recently purchased a small infiniband-enabled cluster and switch. I had hoped that the ofa-general list might be somewhere suitable to find guidance on configuring an IB-based MPI cluster using (otherwise) commodity hardware, however 48 hours of posts suggests this is a list for pretty serious IB developers. Before I un-subscribe, could someone kindly point me towards some kind of basic introduction to getting an IB system up and running? I don't need background. I have real hardware plugged in, some drivers installed (but possibly not the right ones), and am now working on compiling OpenMPI and MVAPICH2 with IB support to allow me to then compile other MPI applications to use the IB interconnect. Guidance on how to do this properly, and possibly some sanity checks or benchmarks would be very nice. Regards, Ian From dhabal.k.panda at gmail.com Tue Jul 14 12:10:44 2009 From: dhabal.k.panda at gmail.com (Dhabaleswar K. Panda) Date: Tue, 14 Jul 2009 15:10:44 -0400 Subject: [ofa-general] IB introductory material In-Reply-To: <4A5CD6C5.7040509@crystal.harvard.edu> References: <4A5CD6C5.7040509@crystal.harvard.edu> Message-ID: <3333ea0907141210p312ce22bkad6e0db41f903cf@mail.gmail.com> Detailed user guides for installing and using MVAPICH and MVAPICH2 on IB clusters are available from MVAPICH project site at the following URL: http://mvapich.cse.ohio-state.edu/support/ This will be helpful for you. Thanks, DK On Tue, Jul 14, 2009 at 3:04 PM, Ian Stokes-Rees < ijstokes at crystal.harvard.edu> wrote: > Hello, > > We have recently purchased a small infiniband-enabled cluster and switch. > I had hoped that the ofa-general list might be somewhere suitable to find > guidance on configuring an IB-based MPI cluster using (otherwise) commodity > hardware, however 48 hours of posts suggests this is a list for pretty > serious IB developers. > > Before I un-subscribe, could someone kindly point me towards some kind of > basic introduction to getting an IB system up and running? I don't need > background. I have real hardware plugged in, some drivers installed (but > possibly not the right ones), and am now working on compiling OpenMPI and > MVAPICH2 with IB support to allow me to then compile other MPI applications > to use the IB interconnect. Guidance on how to do this properly, and > possibly some sanity checks or benchmarks would be very nice. > > Regards, > > Ian > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > -------------- next part -------------- An HTML attachment was scrubbed... URL: From sdake at redhat.com Tue Jul 14 12:38:36 2009 From: sdake at redhat.com (Steven Dake) Date: Tue, 14 Jul 2009 12:38:36 -0700 Subject: [ofa-general] IB introductory material In-Reply-To: <4A5CD6C5.7040509@crystal.harvard.edu> References: <4A5CD6C5.7040509@crystal.harvard.edu> Message-ID: <1247600316.2635.74.camel@localhost.localdomain> A teammate of mine recently setup an IB cluster for a project we are working on and used the following as a good resource for very basic "how to get going" type instructions. http://people.redhat.com/dledford/ Regards -steve On Tue, 2009-07-14 at 15:04 -0400, Ian Stokes-Rees wrote: > Hello, > > We have recently purchased a small infiniband-enabled cluster and > switch. I had hoped that the ofa-general list might be somewhere > suitable to find guidance on configuring an IB-based MPI cluster using > (otherwise) commodity hardware, however 48 hours of posts suggests this > is a list for pretty serious IB developers. > > Before I un-subscribe, could someone kindly point me towards some kind > of basic introduction to getting an IB system up and running? I don't > need background. I have real hardware plugged in, some drivers > installed (but possibly not the right ones), and am now working on > compiling OpenMPI and MVAPICH2 with IB support to allow me to then > compile other MPI applications to use the IB interconnect. Guidance on > how to do this properly, and possibly some sanity checks or benchmarks > would be very nice. > > Regards, > > Ian > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From gus at ldeo.columbia.edu Tue Jul 14 12:46:28 2009 From: gus at ldeo.columbia.edu (Gus Correa) Date: Tue, 14 Jul 2009 15:46:28 -0400 Subject: [ofa-general] IB introductory material In-Reply-To: <4A5CD6C5.7040509@crystal.harvard.edu> References: <4A5CD6C5.7040509@crystal.harvard.edu> Message-ID: <4A5CE094.6010302@ldeo.columbia.edu> Hi Ian and list I second Ian's request, as I am pretty much in the same situation. For what it is worth, I point out to Ian and other people who may find this material useful, a few IB basic resources which I list on this thread to the Torque Users list: http://www.supercluster.org/pipermail/torqueusers/2009-July/009275.html Ian's question pops up every so often on many list that deal with clusters, MPI, etc. A lot of people out there need basic guidance on IB. So far I have not found a basic tutorial, getting started guide, FAQ list, or other instructional material that would tell a general user or system administrator with no IB expertise how to setup, configure, monitor, and maintain Infiniband networks (IB). Something like that would be extremely helpful, and I wonder if OFA is preparing it or has plans to prepare an IB basic tutorial or equivalent in the near future. My $0.02 Gus Correa --------------------------------------------------------------------- Gustavo Correa Lamont-Doherty Earth Observatory - Columbia University Palisades, NY, 10964-8000 - USA --------------------------------------------------------------------- Ian Stokes-Rees wrote: > Hello, > > We have recently purchased a small infiniband-enabled cluster and > switch. I had hoped that the ofa-general list might be somewhere > suitable to find guidance on configuring an IB-based MPI cluster using > (otherwise) commodity hardware, however 48 hours of posts suggests this > is a list for pretty serious IB developers. > > Before I un-subscribe, could someone kindly point me towards some kind > of basic introduction to getting an IB system up and running? I don't > need background. I have real hardware plugged in, some drivers > installed (but possibly not the right ones), and am now working on > compiling OpenMPI and MVAPICH2 with IB support to allow me to then > compile other MPI applications to use the IB interconnect. Guidance on > how to do this properly, and possibly some sanity checks or benchmarks > would be very nice. > > Regards, > > Ian > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general From liranl at mellanox.co.il Tue Jul 14 13:20:11 2009 From: liranl at mellanox.co.il (Liran Liss) Date: Tue, 14 Jul 2009 23:20:11 +0300 Subject: [ofa-general] RE: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support onlyQP1 References: <20090713181410.GA754@mtls03> <382A478CAD40FA4FB46605CF81FE39F435877A24@orsmsx507.amr.corp.intel.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD035ADF6C@mtlexch01.mtl.com> Hi Robert, Your suggestion to represent RDMAoE as a transport indeed makes the code simpler. Thus, we will have: switch(port_transport) { case RDMA_TRANSPORT_IB: ... break; case RDMA_TRANSPORT_RDMAOE: ... break; case RDMA_TRANSPORT_IWARP: ... break; }; instead of: switch(port_transport) { case RDMA_TRANSPORT_IB: if (port_type == IB) { ... } else { ... } break; case RDMA_TRANSPORT_IWARP: ... break; }; which is cleaner. In addition, for places in which IB and RDMAOE behave the same, we will have: case RDMA_TRANSPORT_IB: case RDMA_TRANSPORT_RDMAOE: ... break; which will make this fact explicit. The only difference is that the switch() will operate on port-transport rather than node transport. (We can add a wrapper that if the ib_dev didn't regsiter a port-transport function, it will default to the node transport.) Thanks! --Liran -----Original Message----- From: Liran Liss Sent: Tuesday, July 14, 2009 11:53 AM To: 'Woodruff, Robert J'; Eli Cohen; Hefty, Sean; Roland Dreier Cc: ewg; general-list Subject: RE: [ofa-general] RE: [ewg] [PATCH 2/8 v3] ib_core: RDMAoE support onlyQP1 S.B. --Liran > Trying to emulate IB for mad services is a total hack and not how this new transport should be added into the core. It should be it's own transport type, just like iWarp was added. > You should start with adding a new transport type to ib_verbs.h, e.g., LL: it is not a hack: RDMAoE will probably use mad services at least for connection management, and additional ones in the future. --- ib_verbs.h 2009-07-13 09:06:10.000000000 -0400 +++ ib_verbs_new.h 2009-07-14 03:00:23.000000000 -0400 @@ -64,12 +64,14 @@ enum rdma_node_type { RDMA_NODE_IB_CA = 1, RDMA_NODE_IB_SWITCH, RDMA_NODE_IB_ROUTER, - RDMA_NODE_RNIC + RDMA_NODE_RNIC, + RDMA_NODE_IBXOE }; LL: a multi-port HCA can have both IB and Ethernet ports, so this is not a per-node thing. enum rdma_transport_type { RDMA_TRANSPORT_IB, - RDMA_TRANSPORT_IWARP + RDMA_TRANSPORT_IWARP, + RDMA_TRANSPORT_IBXOE }; LL: thanks, we will look into this. I am not sure that "transport" is the right terminology, since we are using the IB transport layer. enum rdma_transport_type_______________________________________________ general mailing list general at lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From liranl at mellanox.co.il Tue Jul 14 13:59:32 2009 From: liranl at mellanox.co.il (Liran Liss) Date: Tue, 14 Jul 2009 23:59:32 +0300 Subject: [ofa-general] RE: [PATCH 0/8 v3] RDMAoE support In-Reply-To: <4A5C5332.7020403@voltaire.com> References: <20090713181310.GA31865@mtls03> <4A5C5332.7020403@voltaire.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD035ADF95@mtlexch01.mtl.com> S.B. Regards, --Liran -----Original Message----- From: Or Gerlitz [mailto:ogerlitz at voltaire.com] Sent: Tuesday, July 14, 2009 12:43 PM To: Eli Cohen Cc: Sean Hefty; Roland Dreier; Jack Morgenstein; general-list; Liran Liss Subject: Re: [PATCH 0/8 v3] RDMAoE support Eli Cohen wrote: > RDMAoE allows running the IB transport protocol using Ethernet frames allowing the deployment of IB semantics on lossless Ethernet fabrics. RDMAoE packets are standard Ethernet frames with an IEEE assigned Ethertype, a GRH, unmodified IB transport headers and payload. Hi Eli and the team @ Mellanox Before going into more detailed review and comments, I'd like to try and clarify few issues. 1. GRH - is it a must per your design to have it also for unicast packets? or maybe it just simplifies things, or both? I assume you may need it for the CM logic to keep working the way it used to. LL: GRH is required for all packets; it keeps things simple and provides length information. 2. is there any reason not to restrict the design for supporting addr / rdma-cm based consumers? e.g in the same manner that iWARP is? LL: There is no reason to restrict RDMAoE to RDMACM apps: all existing IB applications can work over RDMAoE. 3. CM services - note that once the ULP did the address resolution, the addr / rdma-cm data structure can/has the src/dest MACs, so basically why not push all the changes to the mad code and eimplement the MADs over raw or even datagram socket? LL: we already have a great CM protocol that runs over QP1 and is a perfect fit for RDMAoE; we don't see any reason to change this. 4. regarding your rdmaoe_sa - aside from being non review-able! (since in the same patch 3/8 you move tons of code from one place to another and add changes) it seems to me an overkill. I would like to see a solution which takes advantage of the SA non existence under Ethernet, and hence route resolution becomes no-op as it is with iWARP, as for multicast join, since it is a local operation it should and can be done by the rdma-cm, no need to modify the layers below it, expect for exposing API for the rdma-cm to do so. LL: As I said earlier, we do *not* want to restrict ourselves artificially to RDMACM. The rmdaoa "sa" code does more than just path resolution - it provides SL, MTU, rate, packet lifetime, and other params. The same thing is also true for multicast joins. In terms of the code, it is very simple: - move useful definitions that were previously in multicast.c and sa_query.c to header files. - implement only what is needed for rdmaoe in rdmaoe_sa.c. Perhaps we can make better code reuse (calling the SA's cmp_rec(), for example). We will look into it; thanks! I would just like to note that there is a tradeoff here: keeping rdmaoe code separate for independent development vs. reducing code duplication. Perhaps, as rmdaoe matures, we can aggregate the code to reduce duplication. 5. 8/8 says "Currently, each IB port has a single GID entry in its table and that GID entery equals the link local IPv6 address" - does your design support IPv4 as well, how? LL: Yes, we intend to use standard IPv4-mapped addresses (::0xffff). > To enable RDMAoE with the mlx4 driver stack, both the mlx4_en and mlx4_ib drivers must be loaded, and the netdevice for the corresponding RDMAoE port must be running. 6. it seems that your design is somehow too tightly coupled to the connectX hca and the mlx4 driver, please note that the design should allow for implementing software rdmaoe provider as well LL: on the contrary, there is *nothing* in the architecture that is HW-specific. This is intentional --- SW rdmaoe will be straighforward, efficient (as is SW FCoE), and simple (this is rdma transport after all!) to implement over any nic. We already know of people that are looking into that. > Individual ports of a multi port HCA can be independently configured > as Ethernet (with support for RDMAoE) or IB, as is already the case. > [...] Following is a series of 8 patches based on version 2.6.30 of > the Linux kernel > Does the mainline kernel has all the patches to do so - I wasn't sure if this is the case. can you send the instructions? > This new series reflects changes based on feedback from the community > on the previous set of patches. The whole series is tagged v3 There's not a single mentioning of a change vs the previous versions, how do you expect someone not to treat it as v1?! LL: thanks for the remark, we will send a list of changes in our following patches. For some reason you have chosen to use cross-posting, I don't think these patches need that. Or. From tziporet at dev.mellanox.co.il Tue Jul 14 14:00:35 2009 From: tziporet at dev.mellanox.co.il (Tziporet Koren) Date: Wed, 15 Jul 2009 00:00:35 +0300 Subject: [ofa-general] EWG/OFED meeting minutes for July 13, 2009 In-Reply-To: <4A5B50C1.9050209@mellanox.co.il> References: <5D49E7A8952DC44FB38C38FA0D758EAD0334A01D@mtlexch01.mtl.com> <5D49E7A8952DC44FB38C38FA0D758EAD03565C67@mtlexch01.mtl.com> <4A5B50C1.9050209@mellanox.co.il> Message-ID: <4A5CF1F3.1010105@mellanox.co.il> These are the meeting minutes for EWG/OFED meeting for July 13 09 Meeting summary =============== 1. OFED 1.5 alpha release is still not ready due to compilation issues 2. OFED 1.4.2: Bug fixes will be submitted soon. RC is expected this week and GA next week. Details: ======== 1. OFED 1.5 alpha release * RHEL 5.4 - we will take backports * SLES 10 SP3 - Tziporet to ask Novell if the kernel available somewhere * New Qlogic driver will be sent to kernel soon * PPC64 - compilation fails on mthca - Mellanox to fix it * Management package compilation still fail - Sasha is working to fix * Decided to leave sa cache patches since Qlogic are using them 2. OFED 1.4.2: Brian found more bugs on 1.4.1 - 4 bugs are opened, fixes are under testing major change in the headers will not be done in 1.4.2 since it put release stability on a risk (can be done in 1.5) Discussion: Who will use 1.4.2: * Lustre customers * NFS/RDMA users 1.4.2 Quality disclaimer * We do not going to recommend the distros to take OFED 1.4.2 as package since it is less tested and have not passed interop and OFA logo testing * Note should be written in RN on this. Note: To prevent such case in the future EWG ask Sun to test Lustre on OFED 1.5 on the RCs Comments: I will be on vacation in time frame for next meeting. Betsy Zeller will replace me Tziporet From jgunthorpe at obsidianresearch.com Tue Jul 14 14:16:53 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Tue, 14 Jul 2009 15:16:53 -0600 Subject: [ofa-general] [PATCH] Allow paths to the device specific library to be absolute Message-ID: <20090714211653.GA13700@obsidianresearch.com> If the driver line starts with a / then no lib prefix is applied and the full path is passed to dlopen. This lets a completely contained installation exist that relies on RPATH for the binaries and this mechanism for the drivers. Signed-off-by: Jason Gunthorpe --- src/init.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) This is for libibverbs.. It makes things much easier if you do not install libibverbs into the system library search path (ie an installation under /opt) The revised device line would look like: device /opt/foo/lib/libml4 diff --git a/src/init.c b/src/init.c index 82dfae4..441f80a 100644 --- a/src/init.c +++ b/src/init.c @@ -184,7 +184,9 @@ static void load_driver(const char *name) #define IBV_QUOTE(x) __IBV_QUOTE(x) if (asprintf(&so_name, - "lib%s-" IBV_QUOTE(IBV_DEVICE_LIBRARY_EXTENSION) ".so", + (name[0] == '/'? + "%s-" IBV_QUOTE(IBV_DEVICE_LIBRARY_EXTENSION) ".so": + "lib%s-" IBV_QUOTE(IBV_DEVICE_LIBRARY_EXTENSION) ".so"), name) < 0) { fprintf(stderr, PFX "Warning: couldn't load driver '%s'.\n", name); -- 1.6.0.4 From rdreier at cisco.com Tue Jul 14 15:33:40 2009 From: rdreier at cisco.com (Roland Dreier) Date: Tue, 14 Jul 2009 15:33:40 -0700 Subject: [ofa-general] Re: [PATCH V2] mlx4: check for FW version which properly supports resize_cq In-Reply-To: <200907091927.42856.jackm@dev.mellanox.co.il> (Jack Morgenstein's message of "Thu, 9 Jul 2009 19:27:42 +0300") References: <200907081350.12462.jackm@dev.mellanox.co.il> <200907091112.10027.jackm@dev.mellanox.co.il> <200907091927.42856.jackm@dev.mellanox.co.il> Message-ID: It occurs to me that one change that makes sense and would help make this fix cleaner is the following -- since after all if a command # is out of range, that's really a different error than if a low-level driver just doesn't implement a certain method. What do you think? drivers/infiniband/core/uverbs_main.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index eb36a81..06b3e4d 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -584,14 +584,16 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf, if (hdr.command < 0 || hdr.command >= ARRAY_SIZE(uverbs_cmd_table) || - !uverbs_cmd_table[hdr.command] || - !(file->device->ib_dev->uverbs_cmd_mask & (1ull << hdr.command))) + !uverbs_cmd_table[hdr.command]) return -EINVAL; if (!file->ucontext && hdr.command != IB_USER_VERBS_CMD_GET_CONTEXT) return -EINVAL; + if (!(file->device->ib_dev->uverbs_cmd_mask & (1ull << hdr.command))) + return -ENOSYS; + return uverbs_cmd_table[hdr.command](file, buf + sizeof hdr, hdr.in_words * 4, hdr.out_words * 4); } From He.Huang at Sun.COM Tue Jul 14 19:52:15 2009 From: He.Huang at Sun.COM (Isaac Huang) Date: Tue, 14 Jul 2009 22:52:15 -0400 Subject: [ofa-general] can an ib-bonding slave work independently? Message-ID: <20090715025215.GA4322@sun.com> Hi list, We had a customer report that while an ib-bonding device over ib0 and ib2 (on different HCAs) worked well ib2 as an independent IPoIB device couldn't work (ICMP pings failed). It was CentOS 5.3, with ib-bonding-0.9.0-28. I've checked the release notes and ib-bonding.txt and couldn't find any mention of such a limitation. It might be a configuration error at the customer site but debugging data so far has been scarce so I'd like ask whether someone has worked with such configuration (both the master and its slaves have valid IP addresses, and while the master carries TCP/IP traffic only the slaves handle both RDMA-capable ULPs and IPoIB), and if yes any configuration gotchas? Thanks, Isaac From ogerlitz at Voltaire.com Wed Jul 15 00:47:11 2009 From: ogerlitz at Voltaire.com (Or Gerlitz) Date: Wed, 15 Jul 2009 10:47:11 +0300 Subject: [ofa-general] can an ib-bonding slave work independently? In-Reply-To: <20090715025215.GA4322@sun.com> References: <20090715025215.GA4322@sun.com> Message-ID: <4A5D897F.20708@Voltaire.com> Isaac Huang wrote: > [...] bonding device over ib0 and ib2 worked well ib2 as an independent IPoIB device couldn't work > (ICMP pings failed). It was CentOS 5.3, with ib-bonding-0.9.0-28. Generally speaking, assigning an IP address and hence a route entry to a slave is non recommended and doesn't come without pain, e.g see "Potential Sources of Trouble", section 8.1 "Adventures in Routing" of Documentation/networking/bonding.txt, so your problem might have nothing to do with IPoIB. What kernel does CentOS 5.3 comes with? you may be able to use the mainline bonding driver. Or. From gmpc at sanger.ac.uk Wed Jul 15 01:22:59 2009 From: gmpc at sanger.ac.uk (Guy Coates) Date: Wed, 15 Jul 2009 09:22:59 +0100 Subject: [ofa-general] IB introductory material In-Reply-To: <4A5CE094.6010302@ldeo.columbia.edu> References: <4A5CD6C5.7040509@crystal.harvard.edu> <4A5CE094.6010302@ldeo.columbia.edu> Message-ID: <4A5D91E3.4080306@sanger.ac.uk> I did a basic "howto" for the debian OFED packages; some parts are debian specific, but it contains some basic info on setting up an IB network, sanity checking the connections and getting openMPI working that should be applicable to other distributions. http://pkg-ofed.alioth.debian.org/howto/infiniband-howto.html Contributions to add more detail or generalise it for other OSs are more than welcome. Cheers, Guy -- Dr. Guy Coates, Informatics System Group The Wellcome Trust Sanger Institute, Hinxton, Cambridge, CB10 1HH, UK Tel: +44 (0)1223 834244 x 6925 Fax: +44 (0)1223 496802 -- The Wellcome Trust Sanger Institute is operated by Genome Research Limited, a charity registered in England with number 1021457 and a company registered in England with number 2742969, whose registered office is 215 Euston Road, London, NW1 2BE. From ogerlitz at voltaire.com Wed Jul 15 01:29:11 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Wed, 15 Jul 2009 11:29:11 +0300 Subject: [ofa-general] Re: [PATCH 0/8 v3] RDMAoE support In-Reply-To: <5D49E7A8952DC44FB38C38FA0D758EAD035ADF95@mtlexch01.mtl.com> References: <20090713181310.GA31865@mtls03> <4A5C5332.7020403@voltaire.com> <5D49E7A8952DC44FB38C38FA0D758EAD035ADF95@mtlexch01.mtl.com> Message-ID: <4A5D9357.5020808@voltaire.com> Liran Liss wrote: > GRH is required for all packets; it keeps things simple and provides length information yes, makes sense > There is no reason to restrict RDMAoE to RDMACM apps: all existing IB applications can work over RDMAoE Considering the fact that you've put away IPoIB and the non-existence of user space SA queries, the only remaining potential user is SRP which anyway uses sophisticated QP1 and SA services even before going to H.A and multi-pathing - so I don't see anyone using SRP with RDMAoE. As for MPIs which don't use rdma-cm, the reason for that has been the problem or perception of SA scalability which doesn't exist over Ethernet. So all-in-all (or maybe all-to-all...) the only remaining consumers are those MPIs which will insist on exchanging pairs the way they exchange pairs today (note that with IB GRH isn't a must so you should now go and patch all MPIs which don't exchange GIDs). These guys will simply create Adress handles with the pairs, and as such there's no need to have any layer below the rdma-cm and addr doing MAC or GID resolution. > we already have a great CM protocol that runs over QP1 and is a perfect fit for RDMAoE; we don't see any reason to change this sounds good, lets see what the other reviewers think, so you didn't change a bit in the CM? > The rmdaoa "sa" code does more than just path resolution - it provides SL, MTU, rate, packet lifetime, and other params. The same thing is also true for multicast joins. mmm, interesting, what is the criteria / logic to assign SL and rate? looking in rdmaoe_sa.c I didn't find any sign to such code, please point me to the function/s that implement this. This is before going to the way you've implemented the multicast joins, which I'll get to later. > In terms of the code, it is very simple: - move useful definitions that were previously in multicast.c/sa_query.c to header files. - implement only what is needed for rdmaoe in rdmaoe_sa.c. Perhaps we can make better code reuse (calling the SA's cmp_rec(), for example). We will look into it; thanks! I would just like to note that there is a tradeoff here what ever, however, going your way (i.e claim that there's a need to rdmaoe_sa.c, I still don't see why rdmaoe_sa.c is needed) I don't see how it can be reviewed this since you made all the above changes in one patch - if you want review, separate to at least two patches, the first moves a way code, the second add the rdmaoe_sa.c. BTW - same goes for the API changes, you should have one patch that relates to kernel only changes, and another patch which relates to the user space changes, and this patch should come in a series with changes to libibverbs so people can review both sides of the kernel/user channel. > Yes, we intend to use standard IPv4-mapped addresses (::0xffff) okay, any reason not to do this from day one? is there anyone in the IB stack or in your firmware that assumes a specific GID would always be in index 0? > on the contrary, there is *nothing* in the architecture that is HW-specific. This is intentional --- SW rdmaoe will be straighforward, efficient (as is SW FCoE), and simple (this is rdma transport after all!) to implement over any nic. We already know of people that are looking into that. I'm not convinced... e.g we'll get to this more when doing a detailed review on the multicast thing. Also, I haven't seen any comment from anyone on your API changes, can be nice if you act to make the sw rdmaoe developers comment here. Or. From vlad at lists.openfabrics.org Wed Jul 15 02:38:31 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Wed, 15 Jul 2009 02:38:31 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090715-0200 daily build status Message-ID: <20090715093832.0961B1020441@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_x86_64_check/net/rds/af_rds.c:419: warning: passing argument 3 of 'sk_alloc' makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_x86_64_check/net/rds/af_rds.c:419: warning: passing argument 4 of 'sk_alloc' makes integer from pointer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_x86_64_check/net/rds/af_rds.c: At top level: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_x86_64_check/net/rds/af_rds.c:438: warning: initialization from incompatible pointer type make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_x86_64_check/net/rds/af_rds.c:419: warning: passing argument 3 of 'sk_alloc' makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_x86_64_check/net/rds/af_rds.c:419: warning: passing argument 4 of 'sk_alloc' makes integer from pointer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_x86_64_check/net/rds/af_rds.c: At top level: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_x86_64_check/net/rds/af_rds.c:438: warning: initialization from incompatible pointer type make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -m64 -mno-red-zone -mcmodel=kernel -funit-at-a-time -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1705: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1727: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1726: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090715-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From amirv at mellanox.co.il Wed Jul 15 03:27:48 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Wed, 15 Jul 2009 13:27:48 +0300 Subject: [ofa-general] [PATCH] sdp: fix some warning and bugs in porting to ofed 1.5 Message-ID: <1247653668-21541-1-git-send-email-amirv@mellanox.co.il> Signed-off-by: Amir Vadai --- drivers/infiniband/ulp/sdp/sdp.h | 9 +++++++-- drivers/infiniband/ulp/sdp/sdp_cma.c | 2 +- drivers/infiniband/ulp/sdp/sdp_main.c | 17 ++++++++--------- drivers/infiniband/ulp/sdp/sdp_rx.c | 14 ++++---------- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/drivers/infiniband/ulp/sdp/sdp.h b/drivers/infiniband/ulp/sdp/sdp.h index d03d6af..ec8dbe1 100644 --- a/drivers/infiniband/ulp/sdp/sdp.h +++ b/drivers/infiniband/ulp/sdp/sdp.h @@ -11,12 +11,15 @@ #define SDPSTATS_ON /* #define SDP_PROFILING */ -#define _sdp_printk(func, line, level, sk, format, arg...) \ +#define _sdp_printk(func, line, level, sk, format, arg...) do { \ + preempt_disable(); \ printk(level "%s:%d sdp_sock(%5d:%d %d:%d): " format, \ func, line, \ current->pid, smp_processor_id(), \ (sk) ? inet_sk(sk)->num : -1, \ - (sk) ? ntohs(inet_sk(sk)->dport) : -1, ## arg) + (sk) ? ntohs(inet_sk(sk)->dport) : -1, ## arg); \ + preempt_enable(); \ +} while (0) #define sdp_printk(level, sk, format, arg...) \ _sdp_printk(__func__, __LINE__, level, sk, format, ## arg) #define sdp_warn(sk, format, arg...) \ @@ -71,6 +74,7 @@ static inline unsigned long long current_nsec(void) #define sdp_prf1(sk, s, format, arg...) ({ \ struct sdpprf_log *l = \ &sdpprf_log[sdpprf_log_count++ & (SDPPRF_LOG_SIZE - 1)]; \ + preempt_disable(); \ l->idx = sdpprf_log_count - 1; \ l->pid = current->pid; \ l->sk_num = (sk) ? inet_sk(sk)->num : -1; \ @@ -81,6 +85,7 @@ static inline unsigned long long current_nsec(void) l->time = current_nsec(); \ l->func = __func__; \ l->line = __LINE__; \ + preempt_enable(); \ 1; \ }) #define sdp_prf(sk, s, format, arg...) diff --git a/drivers/infiniband/ulp/sdp/sdp_cma.c b/drivers/infiniband/ulp/sdp/sdp_cma.c index 2354fff..f8a7303 100644 --- a/drivers/infiniband/ulp/sdp/sdp_cma.c +++ b/drivers/infiniband/ulp/sdp/sdp_cma.c @@ -308,7 +308,7 @@ int sdp_cma_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) -EINVAL : 0; } - lock_sock(sk); + lock_sock_nested(sk, SINGLE_DEPTH_NESTING); sdp_dbg(sk, "%s event %d id %p\n", __func__, event->event, id); if (!sdp_sk(sk)->id) { sdp_dbg(sk, "socket is being torn down\n"); diff --git a/drivers/infiniband/ulp/sdp/sdp_main.c b/drivers/infiniband/ulp/sdp/sdp_main.c index 48e43dc..ab49846 100644 --- a/drivers/infiniband/ulp/sdp/sdp_main.c +++ b/drivers/infiniband/ulp/sdp/sdp_main.c @@ -181,7 +181,6 @@ static int sdp_get_port(struct sock *sk, unsigned short snum) static void sdp_destroy_qp(struct sdp_sock *ssk) { struct ib_pd *pd = NULL; - unsigned long flags; sdp_dbg(&ssk->isk.sk, "destroying qp\n"); @@ -189,7 +188,6 @@ static void sdp_destroy_qp(struct sdp_sock *ssk) del_timer(&ssk->tx_ring.timer); - rx_ring_lock(ssk, flags); sdp_rx_ring_destroy(ssk); sdp_tx_ring_destroy(ssk); @@ -209,9 +207,6 @@ static void sdp_destroy_qp(struct sdp_sock *ssk) ib_dealloc_pd(pd); sdp_remove_large_sock(ssk); - - rx_ring_unlock(ssk, flags); - } static void sdp_reset_keepalive_timer(struct sock *sk, unsigned long len) @@ -453,10 +448,6 @@ void sdp_reset_sk(struct sock *sk, int rc) sdp_set_error(sk, rc); } - sdp_destroy_qp(ssk); - - memset((void *)&ssk->id, 0, sizeof(*ssk) - offsetof(typeof(*ssk), id)); - sk->sk_state_change(sk); /* Don't destroy socket before destroy work does its job */ @@ -976,6 +967,10 @@ void sdp_destroy_work(struct work_struct *work) struct sock *sk = &ssk->isk.sk; sdp_dbg(sk, "%s: refcnt %d\n", __func__, atomic_read(&sk->sk_refcnt)); + sdp_destroy_qp(ssk); + + memset((void *)&ssk->id, 0, sizeof(*ssk) - offsetof(typeof(*ssk), id)); + sdp_cancel_dreq_wait_timeout(ssk); if (sk->sk_state == TCP_TIME_WAIT) @@ -2559,6 +2554,10 @@ static void __exit sdp_exit(void) sdp_proc_unregister(); ib_unregister_client(&sdp_client); + + percpu_counter_destroy(sockets_allocated); + percpu_counter_destroy(orphan_count); + kfree(orphan_count); kfree(sockets_allocated); } diff --git a/drivers/infiniband/ulp/sdp/sdp_rx.c b/drivers/infiniband/ulp/sdp/sdp_rx.c index 749a83a..38417c4 100644 --- a/drivers/infiniband/ulp/sdp/sdp_rx.c +++ b/drivers/infiniband/ulp/sdp/sdp_rx.c @@ -447,12 +447,12 @@ static int sdp_process_rx_ctl_skb(struct sdp_sock *ssk, struct sk_buff *skb) break; case SDP_MID_CHRCVBUF: sdp_dbg_data(sk, "Handling RX CHRCVBUF\n"); - sdp_handle_resize_request(ssk, (struct sdp_chrecvbuf *)h); + sdp_handle_resize_request(ssk, (struct sdp_chrecvbuf *)(h+1)); __kfree_skb(skb); break; case SDP_MID_CHRCVBUF_ACK: sdp_dbg_data(sk, "Handling RX CHRCVBUF_ACK\n"); - sdp_handle_resize_ack(ssk, (struct sdp_chrecvbuf *)h); + sdp_handle_resize_ack(ssk, (struct sdp_chrecvbuf *)(h+1)); __kfree_skb(skb); break; default: @@ -787,9 +787,6 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device) { struct ib_cq *rx_cq; int rc = 0; - unsigned long flags; - - rx_ring_lock(ssk, flags); atomic_set(&ssk->rx_ring.head, 1); atomic_set(&ssk->rx_ring.tail, 1); @@ -797,12 +794,11 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device) ssk->rx_ring.buffer = kmalloc( sizeof *ssk->rx_ring.buffer * SDP_RX_SIZE, GFP_KERNEL); if (!ssk->rx_ring.buffer) { - rc = -ENOMEM; sdp_warn(&ssk->isk.sk, "Unable to allocate RX Ring size %zd.\n", sizeof(*ssk->rx_ring.buffer) * SDP_RX_SIZE); - goto out; + return -ENOMEM; } /* TODO: use vector=IB_CQ_VECTOR_LEAST_ATTACHED when implemented @@ -822,13 +818,11 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device) sdp_arm_rx_cq(&ssk->isk.sk); - goto out; + return 0; err_cq: kfree(ssk->rx_ring.buffer); ssk->rx_ring.buffer = NULL; -out: - rx_ring_unlock(ssk, flags); return rc; } -- 1.5.3.7 From Line.Holen at Sun.COM Wed Jul 15 04:37:17 2009 From: Line.Holen at Sun.COM (Line.Holen at Sun.COM) Date: Wed, 15 Jul 2009 13:37:17 +0200 Subject: [ofa-general] [PATCH] opensm/osm_ucast_ftree.c Fix assert comparing number of CAs to CN ports Message-ID: <4A5DBF6D.60508@Sun.COM> There is an assert in fabric_dump_general_info() comparing number of CAs to number of CNs. The latter is a port counter and can be larger than the number of CAs. Adding a CA port counter for proper comparison. Signed-off-by: Line Holen --- diff --git a/opensm/opensm/osm_ucast_ftree.c b/opensm/opensm/osm_ucast_ftree.c index 26cdcab..d13ddd0 100644 --- a/opensm/opensm/osm_ucast_ftree.c +++ b/opensm/opensm/osm_ucast_ftree.c @@ -207,6 +207,7 @@ typedef struct ftree_fabric_t_ { cl_qmap_t cn_guid_tbl; cl_qmap_t io_guid_tbl; unsigned cn_num; + unsigned ca_ports; uint8_t leaf_switch_rank; uint8_t max_switch_rank; ftree_sw_t **leaf_switches; @@ -1148,11 +1149,11 @@ static void fabric_dump_general_info(IN ftree_fabric_t * p_ftree) OSM_LOG(&p_ftree->p_osm->log, OSM_LOG_INFO, " - FatTree max switch rank: %u\n", p_ftree->max_switch_rank); OSM_LOG(&p_ftree->p_osm->log, OSM_LOG_INFO, - " - Fabric has %u CAs (%u of them CNs), %u switches\n", - cl_qmap_count(&p_ftree->hca_tbl), p_ftree->cn_num, - cl_qmap_count(&p_ftree->sw_tbl)); + " - Fabric has %u CAs, %u CA ports (%u of them CNs), %u switches\n", + cl_qmap_count(&p_ftree->hca_tbl), p_ftree->ca_ports, + p_ftree->cn_num, cl_qmap_count(&p_ftree->sw_tbl)); - CL_ASSERT(cl_qmap_count(&p_ftree->hca_tbl) >= p_ftree->cn_num); + CL_ASSERT(p_ftree->ca_ports >= p_ftree->cn_num); for (i = 0; i <= p_ftree->max_switch_rank; i++) { j = 0; @@ -3146,6 +3147,7 @@ fabric_construct_hca_ports(IN ftree_fabric_t * p_ftree, IN ftree_hca_t * p_hca) (p_osm_port))); } } + p_ftree->ca_ports++; hca_add_port(p_hca, /* local ftree_hca object */ i, /* local port number */ From hnrose at comcast.net Wed Jul 15 04:58:35 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Wed, 15 Jul 2009 07:58:35 -0400 Subject: [ofa-general] [PATCH 1/2] opensm/osm_qos_policy.c: Use proper size in malloc in osm_qos_policy_vlarb_scope_create Message-ID: <20090715115835.GA5229@comcast.net> Signed-off-by: Hal Rosenstock --- opensm/opensm/osm_qos_policy.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/opensm/opensm/osm_qos_policy.c b/opensm/opensm/osm_qos_policy.c index 094fef2..2002fd1 100644 --- a/opensm/opensm/osm_qos_policy.c +++ b/opensm/opensm/osm_qos_policy.c @@ -192,7 +192,7 @@ void osm_qos_policy_port_group_destroy(osm_qos_port_group_t * p) osm_qos_vlarb_scope_t *osm_qos_policy_vlarb_scope_create() { osm_qos_vlarb_scope_t *p = - (osm_qos_vlarb_scope_t *) malloc(sizeof(osm_qos_sl2vl_scope_t)); + (osm_qos_vlarb_scope_t *) malloc(sizeof(osm_qos_vlarb_scope_t)); if (!p) return NULL; -- 1.5.6.4 From hnrose at comcast.net Tue Jul 14 10:28:52 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Tue, 14 Jul 2009 13:28:52 -0400 Subject: [ofa-general] [PATCH] infiniband-diags/README: Update saquery dependencies Message-ID: <20090714172852.GA32137@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/infiniband-diags/README b/infiniband-diags/README index c251726..78e2a3f 100644 --- a/infiniband-diags/README +++ b/infiniband-diags/README @@ -542,7 +542,7 @@ Usage: saquery [-h -d -P -N -L -G -s -g][] -g get multicast group info Dependencies: -OpenSM libvendor, OpenSM libopensm, libibumad +OpenSM libosmcomp, libibumad, libibmad 29. ibsysstat From hnrose at comcast.net Wed Jul 15 04:59:50 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Wed, 15 Jul 2009 07:59:50 -0400 Subject: [ofa-general] [PATCH 2/2] opensm/osm_qos_policy.c: Use calloc rather than malloc/memset Message-ID: <20090715115950.GB5229@comcast.net> Signed-off-by: Hal Rosenstock --- opensm/opensm/osm_qos_policy.c | 26 ++++++++------------------ 1 files changed, 8 insertions(+), 18 deletions(-) diff --git a/opensm/opensm/osm_qos_policy.c b/opensm/opensm/osm_qos_policy.c index 2002fd1..7826578 100644 --- a/opensm/opensm/osm_qos_policy.c +++ b/opensm/opensm/osm_qos_policy.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -133,10 +134,9 @@ static void __free_single_element(void *p_element, void *context) osm_qos_port_t *osm_qos_policy_port_create(osm_physp_t *p_physp) { osm_qos_port_t *p = - (osm_qos_port_t *) malloc(sizeof(osm_qos_port_t)); + (osm_qos_port_t *) calloc(1, sizeof(osm_qos_port_t)); if (!p) return NULL; - memset(p, 0, sizeof(osm_qos_port_t)); p->p_physp = p_physp; return p; @@ -148,11 +148,10 @@ osm_qos_port_t *osm_qos_policy_port_create(osm_physp_t *p_physp) osm_qos_port_group_t *osm_qos_policy_port_group_create() { osm_qos_port_group_t *p = - (osm_qos_port_group_t *) malloc(sizeof(osm_qos_port_group_t)); + (osm_qos_port_group_t *) calloc(1, sizeof(osm_qos_port_group_t)); if (!p) return NULL; - memset(p, 0, sizeof(osm_qos_port_group_t)); cl_qmap_init(&p->port_map); return p; @@ -192,12 +191,10 @@ void osm_qos_policy_port_group_destroy(osm_qos_port_group_t * p) osm_qos_vlarb_scope_t *osm_qos_policy_vlarb_scope_create() { osm_qos_vlarb_scope_t *p = - (osm_qos_vlarb_scope_t *) malloc(sizeof(osm_qos_vlarb_scope_t)); + (osm_qos_vlarb_scope_t *) calloc(1, sizeof(osm_qos_vlarb_scope_t)); if (!p) return NULL; - memset(p, 0, sizeof(osm_qos_vlarb_scope_t)); - cl_list_init(&p->group_list, 10); cl_list_init(&p->across_list, 10); cl_list_init(&p->vlarb_high_list, 10); @@ -238,12 +235,10 @@ void osm_qos_policy_vlarb_scope_destroy(osm_qos_vlarb_scope_t * p) osm_qos_sl2vl_scope_t *osm_qos_policy_sl2vl_scope_create() { osm_qos_sl2vl_scope_t *p = - (osm_qos_sl2vl_scope_t *) malloc(sizeof(osm_qos_sl2vl_scope_t)); + (osm_qos_sl2vl_scope_t *) calloc(1, sizeof(osm_qos_sl2vl_scope_t)); if (!p) return NULL; - memset(p, 0, sizeof(osm_qos_sl2vl_scope_t)); - cl_list_init(&p->group_list, 10); cl_list_init(&p->across_from_list, 10); cl_list_init(&p->across_to_list, 10); @@ -280,10 +275,9 @@ void osm_qos_policy_sl2vl_scope_destroy(osm_qos_sl2vl_scope_t * p) osm_qos_level_t *osm_qos_policy_qos_level_create() { osm_qos_level_t *p = - (osm_qos_level_t *) malloc(sizeof(osm_qos_level_t)); + (osm_qos_level_t *) calloc(1, sizeof(osm_qos_level_t)); if (!p) return NULL; - memset(p, 0, sizeof(osm_qos_level_t)); return p; } @@ -360,12 +354,10 @@ ib_net16_t osm_qos_level_get_shared_pkey(IN const osm_qos_level_t * p_qos_level, osm_qos_match_rule_t *osm_qos_policy_match_rule_create() { osm_qos_match_rule_t *p = - (osm_qos_match_rule_t *) malloc(sizeof(osm_qos_match_rule_t)); + (osm_qos_match_rule_t *) calloc(1, sizeof(osm_qos_match_rule_t)); if (!p) return NULL; - memset(p, 0, sizeof(osm_qos_match_rule_t)); - cl_list_init(&p->source_list, 10); cl_list_init(&p->source_group_list, 10); cl_list_init(&p->destination_list, 10); @@ -426,12 +418,10 @@ void osm_qos_policy_match_rule_destroy(osm_qos_match_rule_t * p) osm_qos_policy_t * osm_qos_policy_create(osm_subn_t * p_subn) { - osm_qos_policy_t * p_qos_policy = (osm_qos_policy_t *)malloc(sizeof(osm_qos_policy_t)); + osm_qos_policy_t * p_qos_policy = (osm_qos_policy_t *)calloc(1, sizeof(osm_qos_policy_t)); if (!p_qos_policy) return NULL; - memset(p_qos_policy, 0, sizeof(osm_qos_policy_t)); - cl_list_construct(&p_qos_policy->port_groups); cl_list_init(&p_qos_policy->port_groups, 10); -- 1.5.6.4 From hal.rosenstock at gmail.com Wed Jul 15 05:01:31 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 15 Jul 2009 08:01:31 -0400 Subject: [ofa-general] What is the purpose of libosmvendor? In-Reply-To: <4C2744E8AD2982428C5BFE523DF8CDCB453E3F28EC@MNEXMB1.qlogic.org> References: <4C2744E8AD2982428C5BFE523DF8CDCB453E3F28EC@MNEXMB1.qlogic.org> Message-ID: On Tue, Jul 14, 2009 at 10:54 AM, Mike Heinz wrote: > Reviewing the OFED stack, it appears that the appropriate method for user applications to access fabric information such as path records and port guids - but the name of the library itself leaves me a little nonplussed, rather than access to fabric information, it sounds like it's meant to provide a proprietary interface to the SM. > > Is this correct? It's a porting layer for OpenSM. OpenSM is ported in a number of environments (not just OFED). Currently, two vendor layers are in play even within OFED although this may be heading towards one but not sure when we'll get there. The other vendor layers are mainly historical AFAIK. > Is there documentation for the library itself (beyond the sample applications)? Not AFAIK other than the API header files. Note that saquery no longer uses this library. -- Hal > -- > Michael Heinz > Principal Engineer, Qlogic Corporation > King of Prussia, Pennsylvania > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From m.van.malland at linvision.com Wed Jul 15 05:08:10 2009 From: m.van.malland at linvision.com (Maarten van Malland) Date: Wed, 15 Jul 2009 14:08:10 +0200 Subject: [ofa-general] [PATCH] sdp: fix some warning and bugs in porting to ofed 1.5 In-Reply-To: <1247653668-21541-1-git-send-email-amirv@mellanox.co.il> References: <1247653668-21541-1-git-send-email-amirv@mellanox.co.il> Message-ID: <003e01ca0544$ef95d710$cec18530$@van.malland@linvision.com> Hi Amir, We tried your patch and unfortunately it didn't help much. Actually, it made it worse. Here is the dmesg output when we ran nttcp over sdp: inftsttwin03 ~ # dmesg sdp_reset_sk:447 sdp_sock( 4453:0 5038:51621): setting state to error BUG: unable to handle kernel paging request at 0000000000001398 IP: [] sdp_post_recv+0x299/0x66b [ib_sdp] PGD 11a961067 PUD 12dcac067 PMD 0 Oops: 0002 [#1] PREEMPT SMP last sysfs file: /sys/devices/system/cpu/cpu7/cache/index2/shared_cpu_map CPU 5 Modules linked in: ib_sdp w83793 hwmon_vid rdma_ucm rdma_cm iw_cm ib_addr mlx4_ib ib_ipoib ib_cm ib_sa ib_ uverbs ib_umad ib_mthca ib_mad i2c_i801 ib_core mlx4_core e1000e i2c_core i5k_amb hwmon [last unloaded: ib _sdp] Pid: 12072, comm: nttcp Not tainted 2.6.30.1 #1 X7DWT RIP: 0010:[] [] sdp_post_recv+0x299/0x66b [ib_sdp] RSP: 0018:ffff88011b4d1ab8 EFLAGS: 00010202 RAX: 0000000000001398 RBX: 0000000000000008 RCX: ffffe20003df9d40 RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffffff80277789 RBP: ffff88011b4d1bf8 R08: 0000000000000000 R09: ffff880000015e00 R10: 0000000000000000 R11: ffffe20003df9d40 R12: ffff88011099f500 R13: ffff880110474080 R14: 00000000000000d2 R15: 0000000000000000 FS: 00007f337b97c6f0(0000) GS:ffff880028186000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b CR2: 0000000000001398 CR3: 000000012d8bb000 CR4: 00000000000406e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process nttcp (pid: 12072, threadinfo ffff88011b4d0000, task ffff88011b49e640) Stack: ffff88011b4d1af8 ffffffff802308a8 00000000ffffffff ffff88012d4b1000 0000000000001398 ffff88012d965800 0000000000000139 0000000000011a80 ffff88011b4d1b98 ffffffff80553141 000000011b4c8000 0000000000000001 Call Trace: [] ? finish_task_switch+0xb5/0xc4 [] ? thread_return+0x4e/0x9d [] ? local_bh_disable+0xd/0xf [] ? _spin_lock_bh+0x11/0x34 [] ? _spin_lock_irqsave+0x1d/0x39 [] ? sub_preempt_count+0x9e/0xbe [] sdp_do_posts+0xa71/0xa91 [ib_sdp] [] ? sk_wait_data+0xb8/0xcd [] ? autoremove_wake_function+0x0/0x38 [] sdp_recvmsg+0x360/0x5dc [ib_sdp] [] sock_common_recvmsg+0x32/0x47 [] sock_aio_read+0xe8/0xfc [] ? irq_exit+0x4f/0x51 [] ? do_sync_read+0x62/0x126 [] do_sync_read+0xe2/0x126 [] ? trace_hardirqs_on_thunk+0x3a/0x3c [] ? restore_args+0x0/0x30 [] ? autoremove_wake_function+0x0/0x38 [] ? sub_preempt_count+0x9e/0xbe [] ? _spin_unlock+0x10/0x29 [] vfs_read+0xbe/0x134 [] sys_read+0x47/0x70 [] system_call_fastpath+0x16/0x1b Code: b8 00 00 00 41 3b 9d 6c 05 00 00 7c 83 8b 85 f0 fe ff ff 49 8b 95 30 04 00 00 83 e0 3f 48 6b c0 58 4 8 01 d0 48 89 85 e0 fe ff ff <4c> 89 20 4d 8b bd d0 04 00 00 49 8b 87 98 02 00 00 48 85 c0 74 RIP [] sdp_post_recv+0x299/0x66b [ib_sdp] RSP CR2: 0000000000001398 ---[ end trace f3d85b6359a569b5 ]--- Perhaps you can take a second look at it, Best regards, Maarten -----Oorspronkelijk bericht----- Van: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] Namens Amir Vadai Verzonden: woensdag 15 juli 2009 12:28 Aan: general at lists.openfabrics.org CC: Amir Vadai Onderwerp: [ofa-general] [PATCH] sdp: fix some warning and bugs in porting to ofed 1.5 Signed-off-by: Amir Vadai --- drivers/infiniband/ulp/sdp/sdp.h | 9 +++++++-- drivers/infiniband/ulp/sdp/sdp_cma.c | 2 +- drivers/infiniband/ulp/sdp/sdp_main.c | 17 ++++++++--------- drivers/infiniband/ulp/sdp/sdp_rx.c | 14 ++++---------- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/drivers/infiniband/ulp/sdp/sdp.h b/drivers/infiniband/ulp/sdp/sdp.h index d03d6af..ec8dbe1 100644 --- a/drivers/infiniband/ulp/sdp/sdp.h +++ b/drivers/infiniband/ulp/sdp/sdp.h @@ -11,12 +11,15 @@ #define SDPSTATS_ON /* #define SDP_PROFILING */ -#define _sdp_printk(func, line, level, sk, format, arg...) \ +#define _sdp_printk(func, line, level, sk, format, arg...) do { \ + preempt_disable(); \ printk(level "%s:%d sdp_sock(%5d:%d %d:%d): " format, \ func, line, \ current->pid, smp_processor_id(), \ (sk) ? inet_sk(sk)->num : -1, \ - (sk) ? ntohs(inet_sk(sk)->dport) : -1, ## arg) + (sk) ? ntohs(inet_sk(sk)->dport) : -1, ## arg); \ + preempt_enable(); \ +} while (0) #define sdp_printk(level, sk, format, arg...) \ _sdp_printk(__func__, __LINE__, level, sk, format, ## arg) #define sdp_warn(sk, format, arg...) \ @@ -71,6 +74,7 @@ static inline unsigned long long current_nsec(void) #define sdp_prf1(sk, s, format, arg...) ({ \ struct sdpprf_log *l = \ &sdpprf_log[sdpprf_log_count++ & (SDPPRF_LOG_SIZE - 1)]; \ + preempt_disable(); \ l->idx = sdpprf_log_count - 1; \ l->pid = current->pid; \ l->sk_num = (sk) ? inet_sk(sk)->num : -1; \ @@ -81,6 +85,7 @@ static inline unsigned long long current_nsec(void) l->time = current_nsec(); \ l->func = __func__; \ l->line = __LINE__; \ + preempt_enable(); \ 1; \ }) #define sdp_prf(sk, s, format, arg...) diff --git a/drivers/infiniband/ulp/sdp/sdp_cma.c b/drivers/infiniband/ulp/sdp/sdp_cma.c index 2354fff..f8a7303 100644 --- a/drivers/infiniband/ulp/sdp/sdp_cma.c +++ b/drivers/infiniband/ulp/sdp/sdp_cma.c @@ -308,7 +308,7 @@ int sdp_cma_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) -EINVAL : 0; } - lock_sock(sk); + lock_sock_nested(sk, SINGLE_DEPTH_NESTING); sdp_dbg(sk, "%s event %d id %p\n", __func__, event->event, id); if (!sdp_sk(sk)->id) { sdp_dbg(sk, "socket is being torn down\n"); diff --git a/drivers/infiniband/ulp/sdp/sdp_main.c b/drivers/infiniband/ulp/sdp/sdp_main.c index 48e43dc..ab49846 100644 --- a/drivers/infiniband/ulp/sdp/sdp_main.c +++ b/drivers/infiniband/ulp/sdp/sdp_main.c @@ -181,7 +181,6 @@ static int sdp_get_port(struct sock *sk, unsigned short snum) static void sdp_destroy_qp(struct sdp_sock *ssk) { struct ib_pd *pd = NULL; - unsigned long flags; sdp_dbg(&ssk->isk.sk, "destroying qp\n"); @@ -189,7 +188,6 @@ static void sdp_destroy_qp(struct sdp_sock *ssk) del_timer(&ssk->tx_ring.timer); - rx_ring_lock(ssk, flags); sdp_rx_ring_destroy(ssk); sdp_tx_ring_destroy(ssk); @@ -209,9 +207,6 @@ static void sdp_destroy_qp(struct sdp_sock *ssk) ib_dealloc_pd(pd); sdp_remove_large_sock(ssk); - - rx_ring_unlock(ssk, flags); - } static void sdp_reset_keepalive_timer(struct sock *sk, unsigned long len) @@ -453,10 +448,6 @@ void sdp_reset_sk(struct sock *sk, int rc) sdp_set_error(sk, rc); } - sdp_destroy_qp(ssk); - - memset((void *)&ssk->id, 0, sizeof(*ssk) - offsetof(typeof(*ssk), id)); - sk->sk_state_change(sk); /* Don't destroy socket before destroy work does its job */ @@ -976,6 +967,10 @@ void sdp_destroy_work(struct work_struct *work) struct sock *sk = &ssk->isk.sk; sdp_dbg(sk, "%s: refcnt %d\n", __func__, atomic_read(&sk->sk_refcnt)); + sdp_destroy_qp(ssk); + + memset((void *)&ssk->id, 0, sizeof(*ssk) - offsetof(typeof(*ssk), id)); + sdp_cancel_dreq_wait_timeout(ssk); if (sk->sk_state == TCP_TIME_WAIT) @@ -2559,6 +2554,10 @@ static void __exit sdp_exit(void) sdp_proc_unregister(); ib_unregister_client(&sdp_client); + + percpu_counter_destroy(sockets_allocated); + percpu_counter_destroy(orphan_count); + kfree(orphan_count); kfree(sockets_allocated); } diff --git a/drivers/infiniband/ulp/sdp/sdp_rx.c b/drivers/infiniband/ulp/sdp/sdp_rx.c index 749a83a..38417c4 100644 --- a/drivers/infiniband/ulp/sdp/sdp_rx.c +++ b/drivers/infiniband/ulp/sdp/sdp_rx.c @@ -447,12 +447,12 @@ static int sdp_process_rx_ctl_skb(struct sdp_sock *ssk, struct sk_buff *skb) break; case SDP_MID_CHRCVBUF: sdp_dbg_data(sk, "Handling RX CHRCVBUF\n"); - sdp_handle_resize_request(ssk, (struct sdp_chrecvbuf *)h); + sdp_handle_resize_request(ssk, (struct sdp_chrecvbuf *)(h+1)); __kfree_skb(skb); break; case SDP_MID_CHRCVBUF_ACK: sdp_dbg_data(sk, "Handling RX CHRCVBUF_ACK\n"); - sdp_handle_resize_ack(ssk, (struct sdp_chrecvbuf *)h); + sdp_handle_resize_ack(ssk, (struct sdp_chrecvbuf *)(h+1)); __kfree_skb(skb); break; default: @@ -787,9 +787,6 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device) { struct ib_cq *rx_cq; int rc = 0; - unsigned long flags; - - rx_ring_lock(ssk, flags); atomic_set(&ssk->rx_ring.head, 1); atomic_set(&ssk->rx_ring.tail, 1); @@ -797,12 +794,11 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device) ssk->rx_ring.buffer = kmalloc( sizeof *ssk->rx_ring.buffer * SDP_RX_SIZE, GFP_KERNEL); if (!ssk->rx_ring.buffer) { - rc = -ENOMEM; sdp_warn(&ssk->isk.sk, "Unable to allocate RX Ring size %zd.\n", sizeof(*ssk->rx_ring.buffer) * SDP_RX_SIZE); - goto out; + return -ENOMEM; } /* TODO: use vector=IB_CQ_VECTOR_LEAST_ATTACHED when implemented @@ -822,13 +818,11 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device) sdp_arm_rx_cq(&ssk->isk.sk); - goto out; + return 0; err_cq: kfree(ssk->rx_ring.buffer); ssk->rx_ring.buffer = NULL; -out: - rx_ring_unlock(ssk, flags); return rc; } -- 1.5.3.7 _______________________________________________ general mailing list general at lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From amirv at mellanox.co.il Wed Jul 15 05:23:30 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Wed, 15 Jul 2009 15:23:30 +0300 Subject: [ofa-general] [PATCH] sdp: fix some warning and bugs in porting to ofed 1.5 In-Reply-To: <003e01ca0544$ef95d710$cec18530$@van.malland@linvision.com> References: <1247653668-21541-1-git-send-email-amirv@mellanox.co.il> <003e01ca0544$ef95d710$cec18530$@van.malland@linvision.com> Message-ID: <4A5DCA42.7020302@mellanox.co.il> I will check it and return to you. - Amir On 07/15/2009 03:08 PM, Maarten van Malland wrote: > Hi Amir, > > We tried your patch and unfortunately it didn't help much. Actually, it made > it worse. Here is the dmesg output when we ran nttcp over sdp: > > inftsttwin03 ~ # dmesg > sdp_reset_sk:447 sdp_sock( 4453:0 5038:51621): setting state to error > BUG: unable to handle kernel paging request at 0000000000001398 > IP: [] sdp_post_recv+0x299/0x66b [ib_sdp] > PGD 11a961067 PUD 12dcac067 PMD 0 > Oops: 0002 [#1] PREEMPT SMP > last sysfs file: /sys/devices/system/cpu/cpu7/cache/index2/shared_cpu_map > CPU 5 > Modules linked in: ib_sdp w83793 hwmon_vid rdma_ucm rdma_cm iw_cm ib_addr > mlx4_ib ib_ipoib ib_cm ib_sa ib_ > uverbs ib_umad ib_mthca ib_mad i2c_i801 ib_core mlx4_core e1000e i2c_core > i5k_amb hwmon [last unloaded: ib > _sdp] > Pid: 12072, comm: nttcp Not tainted 2.6.30.1 #1 X7DWT > RIP: 0010:[] [] > sdp_post_recv+0x299/0x66b [ib_sdp] > RSP: 0018:ffff88011b4d1ab8 EFLAGS: 00010202 > RAX: 0000000000001398 RBX: 0000000000000008 RCX: ffffe20003df9d40 > RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffffff80277789 > RBP: ffff88011b4d1bf8 R08: 0000000000000000 R09: ffff880000015e00 > R10: 0000000000000000 R11: ffffe20003df9d40 R12: ffff88011099f500 > R13: ffff880110474080 R14: 00000000000000d2 R15: 0000000000000000 > FS: 00007f337b97c6f0(0000) GS:ffff880028186000(0000) knlGS:0000000000000000 > CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b > CR2: 0000000000001398 CR3: 000000012d8bb000 CR4: 00000000000406e0 > DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 > Process nttcp (pid: 12072, threadinfo ffff88011b4d0000, task > ffff88011b49e640) > Stack: > ffff88011b4d1af8 ffffffff802308a8 00000000ffffffff ffff88012d4b1000 > 0000000000001398 ffff88012d965800 0000000000000139 0000000000011a80 > ffff88011b4d1b98 ffffffff80553141 000000011b4c8000 0000000000000001 > Call Trace: > [] ? finish_task_switch+0xb5/0xc4 > [] ? thread_return+0x4e/0x9d > [] ? local_bh_disable+0xd/0xf > [] ? _spin_lock_bh+0x11/0x34 > [] ? _spin_lock_irqsave+0x1d/0x39 > [] ? sub_preempt_count+0x9e/0xbe > [] sdp_do_posts+0xa71/0xa91 [ib_sdp] > [] ? sk_wait_data+0xb8/0xcd > [] ? autoremove_wake_function+0x0/0x38 > [] sdp_recvmsg+0x360/0x5dc [ib_sdp] > [] sock_common_recvmsg+0x32/0x47 > [] sock_aio_read+0xe8/0xfc > [] ? irq_exit+0x4f/0x51 > [] ? do_sync_read+0x62/0x126 > [] do_sync_read+0xe2/0x126 > [] ? trace_hardirqs_on_thunk+0x3a/0x3c > [] ? restore_args+0x0/0x30 > [] ? autoremove_wake_function+0x0/0x38 > [] ? sub_preempt_count+0x9e/0xbe > [] ? _spin_unlock+0x10/0x29 > [] vfs_read+0xbe/0x134 > [] sys_read+0x47/0x70 > [] system_call_fastpath+0x16/0x1b > Code: b8 00 00 00 41 3b 9d 6c 05 00 00 7c 83 8b 85 f0 fe ff ff 49 8b 95 30 > 04 00 00 83 e0 3f 48 6b c0 58 4 > 8 01 d0 48 89 85 e0 fe ff ff <4c> 89 20 4d 8b bd d0 04 00 00 49 8b 87 98 02 > 00 00 48 85 c0 74 > RIP [] sdp_post_recv+0x299/0x66b [ib_sdp] > RSP > CR2: 0000000000001398 > ---[ end trace f3d85b6359a569b5 ]--- > > Perhaps you can take a second look at it, > > Best regards, > > Maarten > > > -----Oorspronkelijk bericht----- > Van: general-bounces at lists.openfabrics.org > [mailto:general-bounces at lists.openfabrics.org] Namens Amir Vadai > Verzonden: woensdag 15 juli 2009 12:28 > Aan: general at lists.openfabrics.org > CC: Amir Vadai > Onderwerp: [ofa-general] [PATCH] sdp: fix some warning and bugs in porting > to ofed 1.5 > > Signed-off-by: Amir Vadai > --- > drivers/infiniband/ulp/sdp/sdp.h | 9 +++++++-- > drivers/infiniband/ulp/sdp/sdp_cma.c | 2 +- > drivers/infiniband/ulp/sdp/sdp_main.c | 17 ++++++++--------- > drivers/infiniband/ulp/sdp/sdp_rx.c | 14 ++++---------- > 4 files changed, 20 insertions(+), 22 deletions(-) > > diff --git a/drivers/infiniband/ulp/sdp/sdp.h > b/drivers/infiniband/ulp/sdp/sdp.h > index d03d6af..ec8dbe1 100644 > --- a/drivers/infiniband/ulp/sdp/sdp.h > +++ b/drivers/infiniband/ulp/sdp/sdp.h > @@ -11,12 +11,15 @@ > #define SDPSTATS_ON > /* #define SDP_PROFILING */ > > -#define _sdp_printk(func, line, level, sk, format, arg...) \ > +#define _sdp_printk(func, line, level, sk, format, arg...) do { > \ > + preempt_disable(); \ > printk(level "%s:%d sdp_sock(%5d:%d %d:%d): " format, \ > func, line, \ > current->pid, smp_processor_id(), \ > (sk) ? inet_sk(sk)->num : -1, \ > - (sk) ? ntohs(inet_sk(sk)->dport) : -1, ## arg) > + (sk) ? ntohs(inet_sk(sk)->dport) : -1, ## arg); \ > + preempt_enable(); \ > +} while (0) > #define sdp_printk(level, sk, format, arg...) \ > _sdp_printk(__func__, __LINE__, level, sk, format, ## arg) > #define sdp_warn(sk, format, arg...) \ > @@ -71,6 +74,7 @@ static inline unsigned long long current_nsec(void) > #define sdp_prf1(sk, s, format, arg...) ({ \ > struct sdpprf_log *l = \ > &sdpprf_log[sdpprf_log_count++ & (SDPPRF_LOG_SIZE - 1)]; \ > + preempt_disable(); \ > l->idx = sdpprf_log_count - 1; \ > l->pid = current->pid; \ > l->sk_num = (sk) ? inet_sk(sk)->num : -1; \ > @@ -81,6 +85,7 @@ static inline unsigned long long current_nsec(void) > l->time = current_nsec(); \ > l->func = __func__; \ > l->line = __LINE__; \ > + preempt_enable(); \ > 1; \ > }) > #define sdp_prf(sk, s, format, arg...) > diff --git a/drivers/infiniband/ulp/sdp/sdp_cma.c > b/drivers/infiniband/ulp/sdp/sdp_cma.c > index 2354fff..f8a7303 100644 > --- a/drivers/infiniband/ulp/sdp/sdp_cma.c > +++ b/drivers/infiniband/ulp/sdp/sdp_cma.c > @@ -308,7 +308,7 @@ int sdp_cma_handler(struct rdma_cm_id *id, struct > rdma_cm_event *event) > -EINVAL : 0; > } > > - lock_sock(sk); > + lock_sock_nested(sk, SINGLE_DEPTH_NESTING); > sdp_dbg(sk, "%s event %d id %p\n", __func__, event->event, id); > if (!sdp_sk(sk)->id) { > sdp_dbg(sk, "socket is being torn down\n"); > diff --git a/drivers/infiniband/ulp/sdp/sdp_main.c > b/drivers/infiniband/ulp/sdp/sdp_main.c > index 48e43dc..ab49846 100644 > --- a/drivers/infiniband/ulp/sdp/sdp_main.c > +++ b/drivers/infiniband/ulp/sdp/sdp_main.c > @@ -181,7 +181,6 @@ static int sdp_get_port(struct sock *sk, unsigned short > snum) > static void sdp_destroy_qp(struct sdp_sock *ssk) > { > struct ib_pd *pd = NULL; > - unsigned long flags; > > > sdp_dbg(&ssk->isk.sk, "destroying qp\n"); > @@ -189,7 +188,6 @@ static void sdp_destroy_qp(struct sdp_sock *ssk) > > del_timer(&ssk->tx_ring.timer); > > - rx_ring_lock(ssk, flags); > > sdp_rx_ring_destroy(ssk); > sdp_tx_ring_destroy(ssk); > @@ -209,9 +207,6 @@ static void sdp_destroy_qp(struct sdp_sock *ssk) > ib_dealloc_pd(pd); > > sdp_remove_large_sock(ssk); > - > - rx_ring_unlock(ssk, flags); > - > } > > static void sdp_reset_keepalive_timer(struct sock *sk, unsigned long len) > @@ -453,10 +448,6 @@ void sdp_reset_sk(struct sock *sk, int rc) > sdp_set_error(sk, rc); > } > > - sdp_destroy_qp(ssk); > - > - memset((void *)&ssk->id, 0, sizeof(*ssk) - offsetof(typeof(*ssk), > id)); > - > sk->sk_state_change(sk); > > /* Don't destroy socket before destroy work does its job */ > @@ -976,6 +967,10 @@ void sdp_destroy_work(struct work_struct *work) > struct sock *sk = &ssk->isk.sk; > sdp_dbg(sk, "%s: refcnt %d\n", __func__, > atomic_read(&sk->sk_refcnt)); > > + sdp_destroy_qp(ssk); > + > + memset((void *)&ssk->id, 0, sizeof(*ssk) - offsetof(typeof(*ssk), > id)); > + > sdp_cancel_dreq_wait_timeout(ssk); > > if (sk->sk_state == TCP_TIME_WAIT) > @@ -2559,6 +2554,10 @@ static void __exit sdp_exit(void) > sdp_proc_unregister(); > > ib_unregister_client(&sdp_client); > + > + percpu_counter_destroy(sockets_allocated); > + percpu_counter_destroy(orphan_count); > + > kfree(orphan_count); > kfree(sockets_allocated); > } > diff --git a/drivers/infiniband/ulp/sdp/sdp_rx.c > b/drivers/infiniband/ulp/sdp/sdp_rx.c > index 749a83a..38417c4 100644 > --- a/drivers/infiniband/ulp/sdp/sdp_rx.c > +++ b/drivers/infiniband/ulp/sdp/sdp_rx.c > @@ -447,12 +447,12 @@ static int sdp_process_rx_ctl_skb(struct sdp_sock > *ssk, struct sk_buff *skb) > break; > case SDP_MID_CHRCVBUF: > sdp_dbg_data(sk, "Handling RX CHRCVBUF\n"); > - sdp_handle_resize_request(ssk, (struct sdp_chrecvbuf *)h); > + sdp_handle_resize_request(ssk, (struct sdp_chrecvbuf > *)(h+1)); > __kfree_skb(skb); > break; > case SDP_MID_CHRCVBUF_ACK: > sdp_dbg_data(sk, "Handling RX CHRCVBUF_ACK\n"); > - sdp_handle_resize_ack(ssk, (struct sdp_chrecvbuf *)h); > + sdp_handle_resize_ack(ssk, (struct sdp_chrecvbuf *)(h+1)); > __kfree_skb(skb); > break; > default: > @@ -787,9 +787,6 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct > ib_device *device) > { > struct ib_cq *rx_cq; > int rc = 0; > - unsigned long flags; > - > - rx_ring_lock(ssk, flags); > > atomic_set(&ssk->rx_ring.head, 1); > atomic_set(&ssk->rx_ring.tail, 1); > @@ -797,12 +794,11 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct > ib_device *device) > ssk->rx_ring.buffer = kmalloc( > sizeof *ssk->rx_ring.buffer * SDP_RX_SIZE, > GFP_KERNEL); > if (!ssk->rx_ring.buffer) { > - rc = -ENOMEM; > sdp_warn(&ssk->isk.sk, > "Unable to allocate RX Ring size %zd.\n", > sizeof(*ssk->rx_ring.buffer) * SDP_RX_SIZE); > > - goto out; > + return -ENOMEM; > } > > /* TODO: use vector=IB_CQ_VECTOR_LEAST_ATTACHED when implemented > @@ -822,13 +818,11 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct > ib_device *device) > > sdp_arm_rx_cq(&ssk->isk.sk); > > - goto out; > + return 0; > > err_cq: > kfree(ssk->rx_ring.buffer); > ssk->rx_ring.buffer = NULL; > -out: > - rx_ring_unlock(ssk, flags); > return rc; > } > > From monis at Voltaire.COM Wed Jul 15 08:12:00 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Wed, 15 Jul 2009 18:12:00 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: References: <4A521784.5090304@Voltaire.COM> Message-ID: <4A5DF1C0.2060701@Voltaire.COM> > > Wouldn't it make more sense to fix the bonding device or core networking > (wherever the problem is) so that it recomputes the multicast list when > the bonding hardware type changes? After this patch, do we end up with > an IPoIB interface that's not a member of the all hosts multicast group? > (That seems like it would lead to confusing breakage later) > > - R. Roland, I took your advice and sent a patch to bonding to fix the issue there to which I am waiting for comment) but I still think the patch for IPoIB is still needed. Without it, IPoIB is exposed to a DoS attack by a module (that looks like bonding but with malicious intentions) that sends IPoIB a garbage multicast address and stops it from joining any other group for ever, even if it is a legal group. What do you think? thanks From liranl at mellanox.co.il Wed Jul 15 08:39:07 2009 From: liranl at mellanox.co.il (Liran Liss) Date: Wed, 15 Jul 2009 18:39:07 +0300 Subject: [ofa-general] RE: [PATCH 0/8 v3] RDMAoE support In-Reply-To: <4A5D9357.5020808@voltaire.com> References: <20090713181310.GA31865@mtls03> <4A5C5332.7020403@voltaire.com> <5D49E7A8952DC44FB38C38FA0D758EAD035ADF95@mtlexch01.mtl.com> <4A5D9357.5020808@voltaire.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD035AE999@mtlexch01.mtl.com> See below. --Liran Liran Liss wrote: > GRH is required for all packets; it keeps things simple and provides > length information yes, makes sense > There is no reason to restrict RDMAoE to RDMACM apps: all existing IB > applications can work over RDMAoE Considering the fact that you've put away IPoIB and the non-existence of user space SA queries, the only remaining potential user is SRP which anyway uses sophisticated QP1 and SA services even before going to H.A and multi-pathing - so I don't see anyone using SRP with RDMAoE. As for MPIs which don't use rdma-cm, the reason for that has been the problem or perception of SA scalability which doesn't exist over Ethernet. So all-in-all (or maybe all-to-all...) the only remaining consumers are those MPIs which will insist on exchanging pairs the way they exchange pairs today (note that with IB GRH isn't a must so you should now go and patch all MPIs which don't exchange GIDs). These guys will simply create Adress handles with the pairs, and as such there's no need to have any layer below the rdma-cm and addr doing MAC or GID resolution. LL: I think that not having a user-space SA API is a big hole - we should not accept this situation and work to fill in the gap. Once such an API exists, all non-rdmacm apps will probably use it - implementing path queries and multicast joins is a tedious effort, as the convergence in the kernel has shown. For this reason, I believe that eventually almost all MAD traffic (at least for conventional apps - not IB network tools which are irrelevant for RDMAoE anyhow) will go only through QP0 and QP1. So, we will add the same SA support for RDMAoE in user-space, and nearly all apps can make use of it. > we already have a great CM protocol that runs over QP1 and is a > perfect fit for RDMAoE; we don't see any reason to change this sounds good, lets see what the other reviewers think, so you didn't change a bit in the CM? LL: that's the beauty of it: not a single change! > The rmdaoa "sa" code does more than just path resolution - it provides SL, MTU, rate, packet lifetime, and other params. The same thing is also true for multicast joins. mmm, interesting, what is the criteria / logic to assign SL and rate? looking in rdmaoe_sa.c I didn't find any sign to such code, please point me to the function/s that implement this. This is before going to the way you've implemented the multicast joins, which I'll get to later. LL: this is still TBD. We will need to define reasonable policies to integrate with DCB features, such as PFC. Any ideas on this are more than welcome! > In terms of the code, it is very simple: - move useful definitions > that were previously in multicast.c/sa_query.c to header files. - > implement only what is needed for rdmaoe in rdmaoe_sa.c. Perhaps we > can make better code reuse (calling the SA's cmp_rec(), for example). > We will look into it; thanks! I would just like to note that there is > a tradeoff here what ever, however, going your way (i.e claim that there's a need to rdmaoe_sa.c, I still don't see why rdmaoe_sa.c is needed) I don't see how it can be reviewed this since you made all the above changes in one patch - if you want review, separate to at least two patches, the first moves a way code, the second add the rdmaoe_sa.c. LL: good idea, thanks. Regarding rmdaoe_sa.c, we are open to suggestions from the community. We can surely do without it: RDMAoE support will amount to a few 'if' statements in the existing SA code. We understood that most people want to see this separated until the standard is finalized. In the meantime, we will try to reduce code duplication as much as possible. BTW - same goes for the API changes, you should have one patch that relates to kernel only changes, and another patch which relates to the user space changes, and this patch should come in a series with changes to libibverbs so people can review both sides of the kernel/user channel. LL: point noted, thanks. > Yes, we intend to use standard IPv4-mapped addresses > (::0xffff) okay, any reason not to do this from day one? is there anyone in the IB stack or in your firmware that assumes a specific GID would always be in index 0? LL: not at all. Its just a matter of what comes first - implementation is ongoing. > on the contrary, there is *nothing* in the architecture that is HW-specific. This is intentional --- SW rdmaoe will be straighforward, efficient (as is SW FCoE), and simple (this is rdma transport after all!) to implement over any nic. We already know of people that are looking into that. I'm not convinced... e.g we'll get to this more when doing a detailed review on the multicast thing. Also, I haven't seen any comment from anyone on your API changes, can be nice if you act to make the sw rdmaoe developers comment here. LL: the only API changes are the sa-like path-queries and multicast joins; verbs are not changed in any way. In addition, we implemented the 'get_mac' method (for translating gids to macs) in the generic user-kernel ABI so any RDMAoE implementation can use it. I don't understand what concerns you; please explain. Or. From rdreier at cisco.com Wed Jul 15 09:01:05 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 15 Jul 2009 09:01:05 -0700 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A5DF1C0.2060701@Voltaire.COM> (Moni Shoua's message of "Wed, 15 Jul 2009 18:12:00 +0300") References: <4A521784.5090304@Voltaire.COM> <4A5DF1C0.2060701@Voltaire.COM> Message-ID: > I took your advice and sent a patch to bonding to fix the issue there to which I > am waiting for comment) but I still think the patch for IPoIB is still needed. > Without it, IPoIB is exposed to a DoS attack by a module (that looks like bonding but > with malicious intentions) that sends IPoIB a garbage multicast address and stops it from > joining any other group for ever, even if it is a legal group. If the attack vector is a malicous module, I'm not too worried about it -- after all, a malicious module could just overwrite the IPoIB module code with whatever it wants and break things that way. Is there any way userspace can inject a bogus multicast address? - R. From sashak at voltaire.com Wed Jul 15 10:01:29 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 15 Jul 2009 20:01:29 +0300 Subject: [ofa-general] Re: [PATCH] infiniband-diags/README: Update saquery dependencies In-Reply-To: <20090714172852.GA32137@comcast.net> References: <20090714172852.GA32137@comcast.net> Message-ID: <20090715170129.GB19023@me> On 13:28 Tue 14 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From ranjit.pandit.ib at gmail.com Wed Jul 15 10:42:50 2009 From: ranjit.pandit.ib at gmail.com (pandit ib) Date: Wed, 15 Jul 2009 10:42:50 -0700 Subject: [ofa-general] Memory registration limit of 16GB with Chelsio In-Reply-To: <4A5CBE78.9020000@opengridcomputing.com> References: <96f8e60e0907101115i11fcf051h8508fdc18282368b@mail.gmail.com> <4A578C4D.2060003@opengridcomputing.com> <8A71B368A89016469F72CD08050AD33405E63DDE@maui.asicdesigners.com> <96f8e60e0907130949y7edf6926l6e70dafc44a7503a@mail.gmail.com> <96f8e60e0907140957k20675d18v284b4e1be4caa9ae@mail.gmail.com> <4A5CBE78.9020000@opengridcomputing.com> Message-ID: <96f8e60e0907151042i5ccc40a9u4df6e0522896aa5b@mail.gmail.com> Thanks Steve. As you suggested, that increased the limit to 32GBytes. Cheers, Ranjit On Tue, Jul 14, 2009 at 10:20 AM, Steve Wise wrote: > You could try something like this.  This removes the ISCSI and TOE/DDP > memory usage, and bumps the PBL memory up to twice what it was. > > (this is untested). > > Steve. > > ---- > > --- drivers/net/cxgb3/t3_hw.c.org    2009-07-14 12:14:45.000000000 -0500 > +++ drivers/net/cxgb3/t3_hw.c    2009-07-14 12:16:32.000000000 -0500 > @@ -3084,13 +3084,13 @@ static void ulp_config(struct adapter *a > { >    unsigned int m = p->chan_rx_size; > > -    ulp_region(adap, ISCSI, m, p->chan_rx_size / 8); > -    ulp_region(adap, TDDP, m, p->chan_rx_size / 8); > +    ulp_region(adap, ISCSI, m, 0); > +    ulp_region(adap, TDDP, m, 0); >    ulptx_region(adap, TPT, m, p->chan_rx_size / 4); >    ulp_region(adap, STAG, m, p->chan_rx_size / 4); >    ulp_region(adap, RQ, m, p->chan_rx_size / 4); > -    ulptx_region(adap, PBL, m, p->chan_rx_size / 4); > -    ulp_region(adap, PBL, m, p->chan_rx_size / 4); > +    ulptx_region(adap, PBL, m, p->chan_rx_size / 2); > +    ulp_region(adap, PBL, m, p->chan_rx_size / 2); >    t3_write_reg(adap, A_ULPRX_TDDP_TAGMASK, 0xffffffff); > } > > From sashak at voltaire.com Wed Jul 15 11:06:29 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 15 Jul 2009 21:06:29 +0300 Subject: [ofa-general] Re: [PATCH 1/2] opensm/osm_qos_policy.c: Use proper size in malloc in osm_qos_policy_vlarb_scope_create In-Reply-To: <20090715115835.GA5229@comcast.net> References: <20090715115835.GA5229@comcast.net> Message-ID: <20090715180629.GC19023@me> On 07:58 Wed 15 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Wed Jul 15 11:08:24 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 15 Jul 2009 21:08:24 +0300 Subject: [ofa-general] Re: [PATCH 2/2] opensm/osm_qos_policy.c: Use calloc rather than malloc/memset In-Reply-To: <20090715115950.GB5229@comcast.net> References: <20090715115950.GB5229@comcast.net> Message-ID: <20090715180824.GD19023@me> On 07:59 Wed 15 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From hnrose at comcast.net Wed Jul 15 12:21:19 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Wed, 15 Jul 2009 15:21:19 -0400 Subject: [ofa-general] [PATCH] opensm: Use calloc rather than malloc/memset Message-ID: <20090715192118.GA11093@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_mcast_mgr.c b/opensm/opensm/osm_mcast_mgr.c index ea49588..a393608 100644 --- a/opensm/opensm/osm_mcast_mgr.c +++ b/opensm/opensm/osm_mcast_mgr.c @@ -74,11 +74,9 @@ static osm_mcast_work_obj_t *mcast_work_obj_new(IN const osm_port_t * p_port) qlist. see cl_qlist_insert_tail(): CL_ASSERT(p_list_item->p_list != p_list) */ - p_obj = malloc(sizeof(*p_obj)); - if (p_obj) { - memset(p_obj, 0, sizeof(*p_obj)); + p_obj = calloc(1, sizeof(*p_obj)); + if (p_obj) p_obj->p_port = (osm_port_t *) p_port; - } return (p_obj); } @@ -582,9 +580,9 @@ static osm_mtree_node_t *mcast_mgr_branch(osm_sm_t * sm, osm_mgrp_t * p_mgrp, /* Prepare an empty list for each port in the switch. TO DO - this list array could probably be moved - inside the switch element to save on malloc thrashing. + inside the switch element to save on calloc thrashing. */ - list_array = malloc(sizeof(cl_qlist_t) * max_children); + list_array = calloc(max_children, sizeof(cl_qlist_t)); if (list_array == NULL) { OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0A16: " "Unable to allocate list array\n"); @@ -592,8 +590,6 @@ static osm_mtree_node_t *mcast_mgr_branch(osm_sm_t * sm, osm_mgrp_t * p_mgrp, goto Exit; } - memset(list_array, 0, sizeof(cl_qlist_t) * max_children); - for (i = 0; i < max_children; i++) cl_qlist_init(&list_array[i]); diff --git a/opensm/opensm/osm_mtree.c b/opensm/opensm/osm_mtree.c index 4df7517..22b0adf 100644 --- a/opensm/opensm/osm_mtree.c +++ b/opensm/opensm/osm_mtree.c @@ -52,12 +52,11 @@ osm_mtree_node_t *osm_mtree_node_new(IN const osm_switch_t * p_sw) osm_mtree_node_t *p_mtn; uint32_t i; - p_mtn = malloc(sizeof(osm_mtree_node_t) + + p_mtn = calloc(1, sizeof(osm_mtree_node_t) + sizeof(void *) * (p_sw->num_ports - 1)); if (!p_mtn) return NULL; - memset(p_mtn, 0, sizeof(*p_mtn)); p_mtn->p_sw = p_sw; p_mtn->max_children = p_sw->num_ports; for (i = 0; i < p_mtn->max_children; i++) diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c index d2733c4..d254a03 100644 --- a/opensm/opensm/osm_multicast.c +++ b/opensm/opensm/osm_multicast.c @@ -79,11 +79,10 @@ osm_mgrp_t *osm_mgrp_new(IN const ib_net16_t mlid) { osm_mgrp_t *p_mgrp; - p_mgrp = (osm_mgrp_t *) malloc(sizeof(*p_mgrp)); + p_mgrp = (osm_mgrp_t *) calloc(1, sizeof(*p_mgrp)); if (!p_mgrp) return NULL; - memset(p_mgrp, 0, sizeof(*p_mgrp)); cl_qmap_init(&p_mgrp->mcm_port_tbl); p_mgrp->mlid = mlid; p_mgrp->last_change_id = 0; diff --git a/opensm/opensm/osm_node.c b/opensm/opensm/osm_node.c index ee2fbed..9288b30 100644 --- a/opensm/opensm/osm_node.c +++ b/opensm/opensm/osm_node.c @@ -93,11 +93,10 @@ osm_node_t *osm_node_new(IN const osm_madw_t * const p_madw) */ size = p_ni->num_ports; - p_node = malloc(sizeof(*p_node) + sizeof(osm_physp_t) * size); + p_node = calloc(1, sizeof(*p_node) + sizeof(osm_physp_t) * size); if (!p_node) return NULL; - memset(p_node, 0, sizeof(*p_node) + sizeof(osm_physp_t) * size); p_node->node_info = *p_ni; p_node->physp_tbl_size = size + 1; diff --git a/opensm/opensm/osm_opensm.c b/opensm/opensm/osm_opensm.c index 50d1349..dfc522a 100644 --- a/opensm/opensm/osm_opensm.c +++ b/opensm/opensm/osm_opensm.c @@ -160,13 +160,12 @@ static void setup_routing_engine(osm_opensm_t *osm, const char *name) for (m = routing_modules; m->name && *m->name; m++) { if (!strcmp(m->name, name)) { - re = malloc(sizeof(struct osm_routing_engine)); + re = calloc(1, sizeof(struct osm_routing_engine)); if (!re) { OSM_LOG(&osm->log, OSM_LOG_VERBOSE, "memory allocation failed\n"); return; } - memset(re, 0, sizeof(struct osm_routing_engine)); re->name = m->name; if (m->setup(re, osm)) { diff --git a/opensm/opensm/osm_pkey.c b/opensm/opensm/osm_pkey.c index ea918d6..62e97ae 100644 --- a/opensm/opensm/osm_pkey.c +++ b/opensm/opensm/osm_pkey.c @@ -144,10 +144,9 @@ ib_api_status_t osm_pkey_tbl_set(IN osm_pkey_tbl_t * p_pkey_tbl, if (!p_pkey_block) { p_pkey_block = - (ib_pkey_table_t *) malloc(sizeof(ib_pkey_table_t)); + (ib_pkey_table_t *) calloc(1, sizeof(ib_pkey_table_t)); if (!p_pkey_block) return (IB_ERROR); - memset(p_pkey_block, 0, sizeof(ib_pkey_table_t)); cl_ptr_vector_set(&p_pkey_tbl->blocks, block, p_pkey_block); } @@ -208,10 +207,9 @@ ib_api_status_t osm_pkey_tbl_set_new_entry(IN osm_pkey_tbl_t * p_pkey_tbl, ib_pkey_table_t *p_block; if (!(p_block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_idx))) { - p_block = (ib_pkey_table_t *) malloc(sizeof(ib_pkey_table_t)); + p_block = (ib_pkey_table_t *) calloc(1, sizeof(ib_pkey_table_t)); if (!p_block) return (IB_ERROR); - memset(p_block, 0, sizeof(ib_pkey_table_t)); cl_ptr_vector_set(&p_pkey_tbl->new_blocks, block_idx, p_block); } diff --git a/opensm/opensm/osm_port.c b/opensm/opensm/osm_port.c index 751c0f0..48b95fe 100644 --- a/opensm/opensm/osm_port.c +++ b/opensm/opensm/osm_port.c @@ -117,10 +117,9 @@ void osm_physp_init(IN osm_physp_t * p_physp, IN const ib_net64_t port_guid, cl_ptr_vector_init(&p_physp->slvl_by_port, num_slvl, 1); for (i = 0; i < num_slvl; i++) { - p_slvl = (ib_slvl_table_t *) malloc(sizeof(ib_slvl_table_t)); + p_slvl = (ib_slvl_table_t *) calloc(1, sizeof(ib_slvl_table_t)); if (!p_slvl) break; - memset(p_slvl, 0, sizeof(ib_slvl_table_t)); cl_ptr_vector_set(&p_physp->slvl_by_port, i, p_slvl); } @@ -148,11 +147,10 @@ osm_port_t *osm_port_new(IN const ib_node_info_t * p_ni, osm_physp_t *p_physp; uint8_t port_num; - p_port = malloc(sizeof(*p_port)); + p_port = calloc(1, sizeof(*p_port)); if (!p_port) return NULL; - memset(p_port, 0, sizeof(*p_port)); cl_qlist_init(&p_port->mcm_list); p_port->p_node = (struct osm_node *)p_parent_node; port_guid = p_ni->port_guid; diff --git a/opensm/opensm/osm_prtn.c b/opensm/opensm/osm_prtn.c index 3f4cf00..b9df7d3 100644 --- a/opensm/opensm/osm_prtn.c +++ b/opensm/opensm/osm_prtn.c @@ -62,11 +62,10 @@ static uint16_t global_pkey_counter; osm_prtn_t *osm_prtn_new(IN const char *name, IN const uint16_t pkey) { - osm_prtn_t *p = malloc(sizeof(*p)); + osm_prtn_t *p = calloc(1, sizeof(*p)); if (!p) return NULL; - memset(p, 0, sizeof(*p)); p->pkey = pkey; p->sl = OSM_DEFAULT_SL; cl_map_construct(&p->full_guid_tbl); diff --git a/opensm/opensm/osm_sa_guidinfo_record.c b/opensm/opensm/osm_sa_guidinfo_record.c index 362c2ac..de96bb3 100644 --- a/opensm/opensm/osm_sa_guidinfo_record.c +++ b/opensm/opensm/osm_sa_guidinfo_record.c @@ -85,7 +85,7 @@ static ib_api_status_t gir_rcv_new_gir(IN osm_sa_t * sa, OSM_LOG_ENTER(sa->p_log); - p_rec_item = malloc(sizeof(*p_rec_item)); + p_rec_item = calloc(1, sizeof(*p_rec_item)); if (p_rec_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 5102: " "rec_item alloc failed\n"); @@ -97,8 +97,6 @@ static ib_api_status_t gir_rcv_new_gir(IN osm_sa_t * sa, "New GUIDInfoRecord: lid %u, block num %d\n", cl_ntoh16(match_lid), block_num); - memset(p_rec_item, 0, sizeof(*p_rec_item)); - p_rec_item->rec.lid = match_lid; p_rec_item->rec.block_num = block_num; if (!block_num) diff --git a/opensm/opensm/osm_sa_lft_record.c b/opensm/opensm/osm_sa_lft_record.c index d092129..c57b839 100644 --- a/opensm/opensm/osm_sa_lft_record.c +++ b/opensm/opensm/osm_sa_lft_record.c @@ -79,7 +79,7 @@ static ib_api_status_t lftr_rcv_new_lftr(IN osm_sa_t * sa, OSM_LOG_ENTER(sa->p_log); - p_rec_item = malloc(sizeof(*p_rec_item)); + p_rec_item = calloc(1, sizeof(*p_rec_item)); if (p_rec_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 4402: " "rec_item alloc failed\n"); @@ -93,8 +93,6 @@ static ib_api_status_t lftr_rcv_new_lftr(IN osm_sa_t * sa, cl_ntoh64(osm_node_get_node_guid(p_sw->p_node)), block, cl_ntoh16(lid)); - memset(p_rec_item, 0, sizeof(*p_rec_item)); - p_rec_item->rec.lid = lid; p_rec_item->rec.block_num = cl_hton16(block); diff --git a/opensm/opensm/osm_sa_link_record.c b/opensm/opensm/osm_sa_link_record.c index bf0b5ee..cd63b45 100644 --- a/opensm/opensm/osm_sa_link_record.c +++ b/opensm/opensm/osm_sa_link_record.c @@ -68,7 +68,7 @@ static void lr_rcv_build_physp_link(IN osm_sa_t * sa, IN ib_net16_t from_lid, { osm_lr_item_t *p_lr_item; - p_lr_item = malloc(sizeof(*p_lr_item)); + p_lr_item = calloc(1, sizeof(*p_lr_item)); if (p_lr_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 1801: " "Unable to acquire link record\n" @@ -78,7 +78,6 @@ static void lr_rcv_build_physp_link(IN osm_sa_t * sa, IN ib_net16_t from_lid, cl_ntoh16(from_lid), cl_ntoh16(to_lid)); return; } - memset(p_lr_item, 0, sizeof(*p_lr_item)); p_lr_item->link_rec.from_port_num = from_port; p_lr_item->link_rec.to_port_num = to_port; diff --git a/opensm/opensm/osm_sa_mcmember_record.c b/opensm/opensm/osm_sa_mcmember_record.c index 5543221..7a44f79 100644 --- a/opensm/opensm/osm_sa_mcmember_record.c +++ b/opensm/opensm/osm_sa_mcmember_record.c @@ -1352,7 +1352,7 @@ static ib_api_status_t mcmr_rcv_new_mcmr(IN osm_sa_t * sa, OSM_LOG_ENTER(sa->p_log); - p_rec_item = malloc(sizeof(*p_rec_item)); + p_rec_item = calloc(1, sizeof(*p_rec_item)); if (p_rec_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 1B15: " "rec_item alloc failed\n"); @@ -1360,8 +1360,6 @@ static ib_api_status_t mcmr_rcv_new_mcmr(IN osm_sa_t * sa, goto Exit; } - memset(p_rec_item, 0, sizeof(*p_rec_item)); - /* HACK: Untrusted requesters should result with 0 Join State, Port Guid, and Proxy */ p_rec_item->rec = *p_rcvd_rec; diff --git a/opensm/opensm/osm_sa_mft_record.c b/opensm/opensm/osm_sa_mft_record.c index 841eb86..6e166fa 100644 --- a/opensm/opensm/osm_sa_mft_record.c +++ b/opensm/opensm/osm_sa_mft_record.c @@ -81,7 +81,7 @@ static ib_api_status_t mftr_rcv_new_mftr(IN osm_sa_t * sa, OSM_LOG_ENTER(sa->p_log); - p_rec_item = malloc(sizeof(*p_rec_item)); + p_rec_item = calloc(1, sizeof(*p_rec_item)); if (p_rec_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 4A02: " "rec_item alloc failed\n"); @@ -98,8 +98,6 @@ static ib_api_status_t mftr_rcv_new_mftr(IN osm_sa_t * sa, position_block_num = ((uint16_t) position << 12) | (block & IB_MCAST_BLOCK_ID_MASK_HO); - memset(p_rec_item, 0, sizeof(*p_rec_item)); - p_rec_item->rec.lid = lid; p_rec_item->rec.position_block_num = cl_hton16(position_block_num); diff --git a/opensm/opensm/osm_sa_multipath_record.c b/opensm/opensm/osm_sa_multipath_record.c index 6ea5ba0..a5ec071 100644 --- a/opensm/opensm/osm_sa_multipath_record.c +++ b/opensm/opensm/osm_sa_multipath_record.c @@ -812,13 +812,12 @@ static osm_mpr_item_t *mpr_rcv_get_lid_pair_path(IN osm_sa_t * sa, OSM_LOG(sa->p_log, OSM_LOG_DEBUG, "Src LID %u, Dest LID %u\n", src_lid_ho, dest_lid_ho); - p_pr_item = malloc(sizeof(*p_pr_item)); + p_pr_item = calloc(1, sizeof(*p_pr_item)); if (p_pr_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 4501: " "Unable to allocate path record\n"); goto Exit; } - memset(p_pr_item, 0, sizeof(*p_pr_item)); status = mpr_rcv_get_path_parms(sa, p_mpr, p_src_port, p_dest_port, dest_lid_ho, comp_mask, &path_parms); diff --git a/opensm/opensm/osm_sa_node_record.c b/opensm/opensm/osm_sa_node_record.c index 15fb763..18a3ae1 100644 --- a/opensm/opensm/osm_sa_node_record.c +++ b/opensm/opensm/osm_sa_node_record.c @@ -79,7 +79,7 @@ static ib_api_status_t nr_rcv_new_nr(osm_sa_t * sa, OSM_LOG_ENTER(sa->p_log); - p_rec_item = malloc(sizeof(*p_rec_item)); + p_rec_item = calloc(1, sizeof(*p_rec_item)); if (p_rec_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 1D02: " "rec_item alloc failed\n"); @@ -93,8 +93,6 @@ static ib_api_status_t nr_rcv_new_nr(osm_sa_t * sa, cl_ntoh64(osm_node_get_node_guid(p_node)), cl_ntoh64(port_guid), cl_ntoh16(lid)); - memset(p_rec_item, 0, sizeof(*p_rec_item)); - p_rec_item->rec.lid = lid; p_rec_item->rec.node_info = p_node->node_info; diff --git a/opensm/opensm/osm_sa_path_record.c b/opensm/opensm/osm_sa_path_record.c index 75d9516..43acf9f 100644 --- a/opensm/opensm/osm_sa_path_record.c +++ b/opensm/opensm/osm_sa_path_record.c @@ -839,13 +839,12 @@ static osm_pr_item_t *pr_rcv_get_lid_pair_path(IN osm_sa_t * sa, OSM_LOG(sa->p_log, OSM_LOG_DEBUG, "Src LID %u, Dest LID %u\n", src_lid_ho, dest_lid_ho); - p_pr_item = malloc(sizeof(*p_pr_item)); + p_pr_item = calloc(1, sizeof(*p_pr_item)); if (p_pr_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 1F01: " "Unable to allocate path record\n"); goto Exit; } - memset(p_pr_item, 0, sizeof(*p_pr_item)); status = pr_rcv_get_path_parms(sa, p_pr, p_src_port, p_dest_port, dest_lid_ho, comp_mask, &path_parms); @@ -1711,13 +1710,12 @@ McastDest: goto Unlock; } - p_pr_item = malloc(sizeof(*p_pr_item)); + p_pr_item = calloc(1, sizeof(*p_pr_item)); if (p_pr_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 1F18: " "Unable to allocate path record for MC group\n"); goto Unlock; } - memset(p_pr_item, 0, sizeof(*p_pr_item)); /* Copy PathRecord request into response */ p_sa_mad = osm_madw_get_sa_mad_ptr(p_madw); diff --git a/opensm/opensm/osm_sa_pkey_record.c b/opensm/opensm/osm_sa_pkey_record.c index 8e47745..bb7a9cf 100644 --- a/opensm/opensm/osm_sa_pkey_record.c +++ b/opensm/opensm/osm_sa_pkey_record.c @@ -76,7 +76,7 @@ static void sa_pkey_create(IN osm_sa_t * sa, IN osm_physp_t * p_physp, OSM_LOG_ENTER(sa->p_log); - p_rec_item = malloc(sizeof(*p_rec_item)); + p_rec_item = calloc(1, sizeof(*p_rec_item)); if (p_rec_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 4602: " "rec_item alloc failed\n"); @@ -95,8 +95,6 @@ static void sa_pkey_create(IN osm_sa_t * sa, IN osm_physp_t * p_physp, cl_ntoh64(osm_physp_get_port_guid(p_physp)), cl_ntoh16(lid), osm_physp_get_port_num(p_physp), block); - memset(p_rec_item, 0, sizeof(*p_rec_item)); - p_rec_item->rec.lid = lid; p_rec_item->rec.block_num = block; p_rec_item->rec.port_num = osm_physp_get_port_num(p_physp); diff --git a/opensm/opensm/osm_sa_portinfo_record.c b/opensm/opensm/osm_sa_portinfo_record.c index b5ef101..2add401 100644 --- a/opensm/opensm/osm_sa_portinfo_record.c +++ b/opensm/opensm/osm_sa_portinfo_record.c @@ -84,7 +84,7 @@ static ib_api_status_t pir_rcv_new_pir(IN osm_sa_t * sa, OSM_LOG_ENTER(sa->p_log); - p_rec_item = malloc(sizeof(*p_rec_item)); + p_rec_item = calloc(1, sizeof(*p_rec_item)); if (p_rec_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 2102: " "rec_item alloc failed\n"); @@ -98,8 +98,6 @@ static ib_api_status_t pir_rcv_new_pir(IN osm_sa_t * sa, cl_ntoh64(osm_physp_get_port_guid(p_physp)), cl_ntoh16(lid), osm_physp_get_port_num(p_physp)); - memset(p_rec_item, 0, sizeof(*p_rec_item)); - p_rec_item->rec.lid = lid; p_rec_item->rec.port_info = p_physp->port_info; p_rec_item->rec.port_num = osm_physp_get_port_num(p_physp); diff --git a/opensm/opensm/osm_sa_slvl_record.c b/opensm/opensm/osm_sa_slvl_record.c index 061d970..64d3da7 100644 --- a/opensm/opensm/osm_sa_slvl_record.c +++ b/opensm/opensm/osm_sa_slvl_record.c @@ -83,7 +83,7 @@ static void sa_slvl_create(IN osm_sa_t * sa, IN const osm_physp_t * p_physp, OSM_LOG_ENTER(sa->p_log); - p_rec_item = malloc(sizeof(*p_rec_item)); + p_rec_item = calloc(1, sizeof(*p_rec_item)); if (p_rec_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 2602: " "rec_item alloc failed\n"); @@ -102,8 +102,6 @@ static void sa_slvl_create(IN osm_sa_t * sa, IN const osm_physp_t * p_physp, cl_ntoh64(osm_physp_get_port_guid(p_physp)), cl_ntoh16(lid), osm_physp_get_port_num(p_physp), in_port_idx); - memset(p_rec_item, 0, sizeof(*p_rec_item)); - p_rec_item->rec.lid = lid; p_rec_item->rec.out_port_num = osm_physp_get_port_num(p_physp); p_rec_item->rec.in_port_num = in_port_idx; diff --git a/opensm/opensm/osm_sa_sw_info_record.c b/opensm/opensm/osm_sa_sw_info_record.c index 2ea8baf..04f0d07 100644 --- a/opensm/opensm/osm_sa_sw_info_record.c +++ b/opensm/opensm/osm_sa_sw_info_record.c @@ -79,7 +79,7 @@ static ib_api_status_t sir_rcv_new_sir(IN osm_sa_t * sa, OSM_LOG_ENTER(sa->p_log); - p_rec_item = malloc(sizeof(*p_rec_item)); + p_rec_item = calloc(1, sizeof(*p_rec_item)); if (p_rec_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 5308: " "rec_item alloc failed\n"); @@ -90,8 +90,6 @@ static ib_api_status_t sir_rcv_new_sir(IN osm_sa_t * sa, OSM_LOG(sa->p_log, OSM_LOG_DEBUG, "New SwitchInfoRecord: lid %u\n", cl_ntoh16(lid)); - memset(p_rec_item, 0, sizeof(*p_rec_item)); - p_rec_item->rec.lid = lid; p_rec_item->rec.switch_info = p_sw->switch_info; diff --git a/opensm/opensm/osm_sa_vlarb_record.c b/opensm/opensm/osm_sa_vlarb_record.c index f9f11b7..93850c7 100644 --- a/opensm/opensm/osm_sa_vlarb_record.c +++ b/opensm/opensm/osm_sa_vlarb_record.c @@ -83,7 +83,7 @@ static void sa_vl_arb_create(IN osm_sa_t * sa, IN osm_physp_t * p_physp, OSM_LOG_ENTER(sa->p_log); - p_rec_item = malloc(sizeof(*p_rec_item)); + p_rec_item = calloc(1, sizeof(*p_rec_item)); if (p_rec_item == NULL) { OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 2A02: " "rec_item alloc failed\n"); @@ -102,8 +102,6 @@ static void sa_vl_arb_create(IN osm_sa_t * sa, IN osm_physp_t * p_physp, cl_ntoh64(osm_physp_get_port_guid(p_physp)), cl_ntoh16(lid), osm_physp_get_port_num(p_physp), block); - memset(p_rec_item, 0, sizeof(*p_rec_item)); - p_rec_item->rec.lid = lid; p_rec_item->rec.port_num = osm_physp_get_port_num(p_physp); p_rec_item->rec.block_num = block; diff --git a/opensm/opensm/osm_service.c b/opensm/opensm/osm_service.c index cc834b2..0cb550b 100644 --- a/opensm/opensm/osm_service.c +++ b/opensm/opensm/osm_service.c @@ -79,11 +79,9 @@ osm_svcr_t *osm_svcr_new(IN const ib_service_record_t * p_svc_rec) CL_ASSERT(p_svc_rec); - p_svcr = (osm_svcr_t *) malloc(sizeof(*p_svcr)); - if (p_svcr) { - memset(p_svcr, 0, sizeof(*p_svcr)); + p_svcr = (osm_svcr_t *) calloc(1, sizeof(*p_svcr)); + if (p_svcr) osm_svcr_init(p_svcr, p_svc_rec); - } return (p_svcr); } diff --git a/opensm/opensm/osm_sm.c b/opensm/opensm/osm_sm.c index daa60ff..6b6d456 100644 --- a/opensm/opensm/osm_sm.c +++ b/opensm/opensm/osm_sm.c @@ -450,10 +450,9 @@ static ib_api_status_t sm_mgrp_process(IN osm_sm_t * p_sm, * 'Schedule' all the QP0 traffic for when the state manager * isn't busy trying to do something else. */ - ctx = malloc(sizeof(*ctx)); + ctx = calloc(1, sizeof(*ctx)); if (!ctx) return IB_ERROR; - memset(ctx, 0, sizeof(*ctx)); ctx->mlid = p_mgrp->mlid; cl_spinlock_acquire(&p_sm->mgrp_lock); diff --git a/opensm/opensm/osm_switch.c b/opensm/opensm/osm_switch.c index ce1ca63..6a06b05 100644 --- a/opensm/opensm/osm_switch.c +++ b/opensm/opensm/osm_switch.c @@ -121,12 +121,10 @@ osm_switch_t *osm_switch_new(IN osm_node_t * const p_node, if (!p_si->lin_cap) /* The switch doesn't support LFT */ return NULL; - p_sw = malloc(sizeof(*p_sw)); + p_sw = calloc(1, sizeof(*p_sw)); if (!p_sw) return NULL; - memset(p_sw, 0, sizeof(*p_sw)); - p_sw->p_node = p_node; p_sw->switch_info = *p_si; p_sw->num_ports = num_ports; @@ -138,12 +136,10 @@ osm_switch_t *osm_switch_new(IN osm_node_t * const p_node, memset(p_sw->lft, OSM_NO_PATH, IB_LID_UCAST_END_HO + 1); - p_sw->p_prof = malloc(sizeof(*p_sw->p_prof) * num_ports); + p_sw->p_prof = calloc(num_ports, sizeof(*p_sw->p_prof)); if (!p_sw->p_prof) goto err; - memset(p_sw->p_prof, 0, sizeof(*p_sw->p_prof) * num_ports); - if (osm_mcast_tbl_init(&p_sw->mcast_tbl, osm_node_get_num_physp(p_node), cl_ntoh16(p_si->mcast_cap))) goto err; @@ -523,10 +519,9 @@ osm_switch_prepare_path_rebuild(IN osm_switch_t * p_sw, IN uint16_t max_lids) memset(p_sw->new_lft, OSM_NO_PATH, IB_LID_UCAST_END_HO + 1); if (!p_sw->hops) { - hops = malloc((max_lids + 1) * sizeof(hops[0])); + hops = calloc(max_lids + 1, sizeof(hops[0])); if (!hops) return -1; - memset(hops, 0, (max_lids + 1) * sizeof(hops[0])); p_sw->hops = hops; p_sw->num_hops = max_lids + 1; } else if (max_lids + 1 > p_sw->num_hops) { diff --git a/opensm/opensm/osm_ucast_cache.c b/opensm/opensm/osm_ucast_cache.c index 216b496..33ca177 100644 --- a/opensm/opensm/osm_ucast_cache.c +++ b/opensm/opensm/osm_ucast_cache.c @@ -106,14 +106,11 @@ static void cache_sw_set_leaf(cache_switch_t * p_sw) static cache_switch_t *cache_sw_new(uint16_t lid_ho, unsigned num_ports) { - cache_switch_t *p_cache_sw = malloc(sizeof(cache_switch_t) + + cache_switch_t *p_cache_sw = calloc(1, sizeof(cache_switch_t) + num_ports * sizeof(cache_port_t)); if (!p_cache_sw) return NULL; - memset(p_cache_sw, 0, - sizeof(*p_cache_sw) + num_ports * sizeof(cache_port_t)); - p_cache_sw->num_ports = num_ports; /* port[0] fields represent this switch details - lid and type */ diff --git a/opensm/opensm/osm_ucast_ftree.c b/opensm/opensm/osm_ucast_ftree.c index 26cdcab..a2aa171 100644 --- a/opensm/opensm/osm_ucast_ftree.c +++ b/opensm/opensm/osm_ucast_ftree.c @@ -323,10 +323,9 @@ static inline void tuple_from_key(IN ftree_tuple_t tuple, static ftree_sw_tbl_element_t *sw_tbl_element_create(IN ftree_sw_t * p_sw) { ftree_sw_tbl_element_t *p_element = - (ftree_sw_tbl_element_t *) malloc(sizeof(ftree_sw_tbl_element_t)); + (ftree_sw_tbl_element_t *) calloc(1, sizeof(ftree_sw_tbl_element_t)); if (!p_element) return NULL; - memset(p_element, 0, sizeof(ftree_sw_tbl_element_t)); p_element->p_sw = p_sw; return p_element; @@ -350,10 +349,9 @@ static void sw_tbl_element_destroy(IN ftree_sw_tbl_element_t * p_element) static ftree_port_t *port_create(IN uint8_t port_num, IN uint8_t remote_port_num) { - ftree_port_t *p_port = (ftree_port_t *) malloc(sizeof(ftree_port_t)); + ftree_port_t *p_port = (ftree_port_t *) calloc(1, sizeof(ftree_port_t)); if (!p_port) return NULL; - memset(p_port, 0, sizeof(ftree_port_t)); p_port->port_num = port_num; p_port->remote_port_num = remote_port_num; @@ -389,10 +387,9 @@ static ftree_port_group_t *port_group_create(IN uint16_t base_lid, IN boolean_t is_io) { ftree_port_group_t *p_group = - (ftree_port_group_t *) malloc(sizeof(ftree_port_group_t)); + (ftree_port_group_t *) calloc(1, sizeof(ftree_port_group_t)); if (p_group == NULL) return NULL; - memset(p_group, 0, sizeof(ftree_port_group_t)); p_group->base_lid = base_lid; p_group->remote_base_lid = remote_base_lid; @@ -533,10 +530,9 @@ static ftree_sw_t *sw_create(IN ftree_fabric_t * p_ftree, if (p_osm_sw->num_ports == 1) return NULL; - p_sw = (ftree_sw_t *) malloc(sizeof(ftree_sw_t)); + p_sw = (ftree_sw_t *) calloc(1, sizeof(ftree_sw_t)); if (p_sw == NULL) return NULL; - memset(p_sw, 0, sizeof(ftree_sw_t)); p_sw->p_osm_sw = p_osm_sw; p_sw->rank = 0xFFFFFFFF; @@ -757,10 +753,9 @@ sw_get_least_hops(IN ftree_sw_t * p_sw, IN uint16_t target_lid) static ftree_hca_t *hca_create(IN osm_node_t * p_osm_node) { - ftree_hca_t *p_hca = (ftree_hca_t *) malloc(sizeof(ftree_hca_t)); + ftree_hca_t *p_hca = (ftree_hca_t *) calloc(1, sizeof(ftree_hca_t)); if (p_hca == NULL) return NULL; - memset(p_hca, 0, sizeof(ftree_hca_t)); p_hca->p_osm_node = p_osm_node; p_hca->up_port_groups = (ftree_port_group_t **) @@ -884,12 +879,10 @@ static void hca_add_port(IN ftree_hca_t * p_hca, IN uint8_t port_num, static ftree_fabric_t *fabric_create() { ftree_fabric_t *p_ftree = - (ftree_fabric_t *) malloc(sizeof(ftree_fabric_t)); + (ftree_fabric_t *) calloc(1, sizeof(ftree_fabric_t)); if (p_ftree == NULL) return NULL; - memset(p_ftree, 0, sizeof(ftree_fabric_t)); - cl_qmap_init(&p_ftree->hca_tbl); cl_qmap_init(&p_ftree->sw_tbl); cl_qmap_init(&p_ftree->sw_by_tuple_tbl); @@ -1579,15 +1572,13 @@ static int fabric_create_leaf_switch_array(IN ftree_fabric_t * p_ftree) /* create array of ALL the switches that have leaf rank */ all_switches_at_leaf_level = (ftree_sw_t **) - malloc(cl_qmap_count(&p_ftree->sw_tbl) * sizeof(ftree_sw_t *)); + calloc(cl_qmap_count(&p_ftree->sw_tbl), sizeof(ftree_sw_t *)); if (!all_switches_at_leaf_level) { osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS, "Fat-tree routing: Memory allocation failed\n"); res = -1; goto Exit; } - memset(all_switches_at_leaf_level, 0, - cl_qmap_count(&p_ftree->sw_tbl) * sizeof(ftree_sw_t *)); p_next_sw = (ftree_sw_t *) cl_qmap_head(&p_ftree->sw_tbl); while (p_next_sw != (ftree_sw_t *) cl_qmap_end(&p_ftree->sw_tbl)) { @@ -1698,13 +1689,12 @@ static boolean_t fabric_validate_topology(IN ftree_fabric_t * p_ftree) "Validating fabric topology\n"); reference_sw_arr = - (ftree_sw_t **) malloc(tree_rank * sizeof(ftree_sw_t *)); + (ftree_sw_t **) calloc(tree_rank, sizeof(ftree_sw_t *)); if (reference_sw_arr == NULL) { osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS, "Fat-tree routing: Memory allocation failed\n"); return FALSE; } - memset(reference_sw_arr, 0, tree_rank * sizeof(ftree_sw_t *)); p_next_sw = (ftree_sw_t *) cl_qmap_head(&p_ftree->sw_tbl); while (res && p_next_sw != (ftree_sw_t *) cl_qmap_end(&p_ftree->sw_tbl)) { diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c index 78a7031..d8b794c 100644 --- a/opensm/opensm/osm_ucast_mgr.c +++ b/opensm/opensm/osm_ucast_mgr.c @@ -444,7 +444,7 @@ static void alloc_ports_priv(osm_ucast_mgr_t * mgr) lmc = ib_port_info_get_lmc(&port->p_physp->port_info); if (!lmc) continue; - r = malloc(sizeof(*r) + sizeof(r->guids[0]) * (1 << lmc)); + r = calloc(1, sizeof(*r) + sizeof(r->guids[0]) * (1 << lmc)); if (!r) { OSM_LOG(mgr->p_log, OSM_LOG_ERROR, "ERR 3A09: " "cannot allocate memory to track remote" @@ -452,7 +452,6 @@ static void alloc_ports_priv(osm_ucast_mgr_t * mgr) port->priv = NULL; continue; } - memset(r, 0, sizeof(*r) + sizeof(r->guids[0]) * (1 << lmc)); port->priv = r; } } diff --git a/opensm/opensm/osm_ucast_updn.c b/opensm/opensm/osm_ucast_updn.c index bb9ccda..e4985e6 100644 --- a/opensm/opensm/osm_ucast_updn.c +++ b/opensm/opensm/osm_ucast_updn.c @@ -394,10 +394,9 @@ static struct updn_node *create_updn_node(osm_switch_t * sw) { struct updn_node *u; - u = malloc(sizeof(*u)); + u = calloc(1, sizeof(*u)); if (!u) return NULL; - memset(u, 0, sizeof(*u)); u->sw = sw; u->id = cl_ntoh64(osm_node_get_node_guid(sw->p_node)); u->rank = 0xffffffff; @@ -431,13 +430,12 @@ static void updn_find_root_nodes_by_min_hop(OUT updn_t * p_updn) "Current number of ports in the subnet is %d\n", cl_qmap_count(&p_osm->subn.port_guid_tbl)); - cas_per_sw = malloc((IB_LID_UCAST_END_HO + 1) * sizeof(*cas_per_sw)); + cas_per_sw = calloc(IB_LID_UCAST_END_HO + 1, sizeof(*cas_per_sw)); if (!cas_per_sw) { OSM_LOG(&p_osm->log, OSM_LOG_ERROR, "ERR AA14: " "cannot alloc mem for CAs per switch counter array\n"); goto _exit; } - memset(cas_per_sw, 0, (IB_LID_UCAST_END_HO + 1) * sizeof(*cas_per_sw)); /* Find the Maximum number of CAs (and routers) for histogram normalization */ OSM_LOG(&p_osm->log, OSM_LOG_VERBOSE, @@ -671,10 +669,9 @@ int osm_ucast_updn_setup(struct osm_routing_engine *r, osm_opensm_t *osm) { updn_t *updn; - updn = malloc(sizeof(updn_t)); + updn = calloc(1, sizeof(updn_t)); if (!updn) return -1; - memset(updn, 0, sizeof(updn_t)); updn->p_osm = osm; From sashak at voltaire.com Wed Jul 15 14:57:58 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Thu, 16 Jul 2009 00:57:58 +0300 Subject: Fwd: [ofa-general] Performance evaluation of Opensm In-Reply-To: <309a667c0907070643n5d824f25wcc74077266122eb6@mail.gmail.com> References: <617255.53094.qm@web94203.mail.in2.yahoo.com> <4A53156D.2040805@dev.mellanox.co.il> <309a667c0907070358n53ad17lafcef8c3ef6e6ef2@mail.gmail.com> <4A532DD5.1000702@morey-chaisemartin.com> <309a667c0907070643n5d824f25wcc74077266122eb6@mail.gmail.com> Message-ID: <20090715215758.GA5767@me> On 19:13 Tue 07 Jul , Devesh Sharma wrote: > > > > For internal actions, you can use valgrind --tool=callgrind > > It provides a full analysis of any program so you can find where bottlenecks are and pretty much any perf info you may need. However it does not allow to mesure times for network operations. > > > ok....I will try this. You can also use gprof for this you may build OpenSM with -pg CFLAGS. Sasha From sashak at voltaire.com Wed Jul 15 16:01:41 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Thu, 16 Jul 2009 02:01:41 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_ftree.cd: Added support for same level links In-Reply-To: <4A5B4789.7040305@morey-chaisemartin.com> References: <4A5B4789.7040305@morey-chaisemartin.com> Message-ID: <20090715230141.GA12078@me> On 16:41 Mon 13 Jul , Nicolas Morey-Chaisemartin wrote: > This patch adds support for horizontal (sibling) links in the Fat-Tree algorithm. > Horizontal links are handled as both upward and downward. They are treated as any other through the exploration but they are only considered last to ensure that for 2 paths of the same lengths the one without horizontal links is preferred. > > Also trying horizontal links after other links ensures that the horizontal path will be as far as possible in the created route (in the ftree exploration sense) thus reducing (removing?) the risk of credit loops. > Several tests have been run with Sun QNEM switches and results are good. > > This patch may require some enhancements later (horizontal links in higher levels have not been fully validated) but it does not change the behaviour on topologies without horizontal links. Thus it should be safe to apply. > > > Acked-by: Line Holen > Signed-off-by: Nicolas Morey-Chaisemartin Applied. Thanks. Sasha From sashak at voltaire.com Wed Jul 15 16:06:00 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Thu, 16 Jul 2009 02:06:00 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_perfmgr.c: In perfmgr_send_pc_mad, only set CounterSelect when Set method is used In-Reply-To: <20090714140427.GA29521@comcast.net> References: <20090714140427.GA29521@comcast.net> Message-ID: <20090715230600.GB12078@me> On 10:04 Tue 14 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock > --- > diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c > index ecfdbda..0437d47 100644 > --- a/opensm/opensm/osm_perfmgr.c > +++ b/opensm/opensm/osm_perfmgr.c > @@ -376,7 +376,8 @@ static ib_api_status_t perfmgr_send_pc_mad(osm_perfmgr_t * perfmgr, > port_counter = (ib_port_counters_t *) & pm_mad->data; > memset(port_counter, 0, sizeof(*port_counter)); > port_counter->port_select = port; > - port_counter->counter_select = 0xFFFF; > + if (mad_method == IB_MAD_METHOD_SET) > + port_counter->counter_select = 0xFFFF; Could you explain why? Sasha > > p_madw->mad_addr.dest_lid = dest_lid; > p_madw->mad_addr.addr_type.gsi.remote_qp = dest_qp; > From sashak at voltaire.com Wed Jul 15 16:07:27 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Thu, 16 Jul 2009 02:07:27 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_ftree.c Fix assert comparing number of CAs to CN ports In-Reply-To: <4A5DBF6D.60508@Sun.COM> References: <4A5DBF6D.60508@Sun.COM> Message-ID: <20090715230727.GC12078@me> On 13:37 Wed 15 Jul , Line.Holen at Sun.COM wrote: > There is an assert in fabric_dump_general_info() comparing number of > CAs to number of CNs. The latter is a port counter and can be larger > than the number of CAs. Adding a CA port counter for proper comparison. > > Signed-off-by: Line Holen Applied. Thanks. Sasha From hal.rosenstock at gmail.com Wed Jul 15 16:25:08 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 15 Jul 2009 19:25:08 -0400 Subject: [ofa-general] Re: [PATCH] opensm/osm_perfmgr.c: In perfmgr_send_pc_mad, only set CounterSelect when Set method is used In-Reply-To: <20090715230600.GB12078@me> References: <20090714140427.GA29521@comcast.net> <20090715230600.GB12078@me> Message-ID: On Wed, Jul 15, 2009 at 7:06 PM, Sasha Khapyorsky wrote: > On 10:04 Tue 14 Jul     , Hal Rosenstock wrote: >> >> Signed-off-by: Hal Rosenstock >> --- >> diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c >> index ecfdbda..0437d47 100644 >> --- a/opensm/opensm/osm_perfmgr.c >> +++ b/opensm/opensm/osm_perfmgr.c >> @@ -376,7 +376,8 @@ static ib_api_status_t perfmgr_send_pc_mad(osm_perfmgr_t * perfmgr, >>       port_counter = (ib_port_counters_t *) & pm_mad->data; >>       memset(port_counter, 0, sizeof(*port_counter)); >>       port_counter->port_select = port; >> -     port_counter->counter_select = 0xFFFF; >> +     if (mad_method == IB_MAD_METHOD_SET) >> +             port_counter->counter_select = 0xFFFF; > > Could you explain why? CounterSelect is only valid on a Set. -- Hal > > Sasha > >> >>       p_madw->mad_addr.dest_lid = dest_lid; >>       p_madw->mad_addr.addr_type.gsi.remote_qp = dest_qp; >> > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From rdreier at cisco.com Wed Jul 15 17:02:45 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 15 Jul 2009 17:02:45 -0700 Subject: [ofa-general] [PATCH/RFC] New version of ummunot patch In-Reply-To: (Roland Dreier's message of "Wed, 27 May 2009 14:36:01 -0700") References: <20090506214628.GM2590@obsidianresearch.com> <20090506222638.GA16280@obsidianresearch.com> <20090507000231.GB16280@obsidianresearch.com> <20090507224806.GF16280@obsidianresearch.com> <5019F239-149F-49E1-8C23-436DE6094AB2@cisco.com> Message-ID: OK, here's the latest version of my ummunot patch. This incorporates a couple of bug fixes in how events are reported, which could have caused incorrect data to be reported or events to be missed. The current version seems reasonably solid, and I expect to send this to the kernel community for review in a week or so, with an eye to getting this merged for 2.6.32. If you have any feedback or concerns, now would be a good time to send them. If this isn't going to work for you, please let me know and we can go back to the drawing board! === ummunot: Userspace support for MMU notifications As discussed in and follow-up messages, libraries using RDMA would like to track precisely when application code changes memory mapping via free(), munmap(), etc. Current pure-userspace solutions using malloc hooks and other tricks are not robust, and the feeling among experts is that the issue is unfixable without kernel help. We solve this not by implementing the full API proposed in the email linked above but rather with a simpler and more generic interface, which may be useful in other contexts. Specifically, we implement a new character device driver, ummunot, that creates a /dev/ummunot node. A userspace process can open this node read-only and use the fd as follows: 1. ioctl() to register/unregister an address range to watch in the kernel (cf struct ummunot_register_ioctl in ). 2. read() to retrieve events generated when a mapping in a watched address range is invalidated (cf struct ummunot_event in ). select()/poll()/epoll() and SIGIO are handled for this IO. 3. mmap() one page at offset 0 to map a kernel page that contains a generation counter that is incremented each time an event is generated. This allows userspace to have a fast path that checks that no events have occurred without a system call. Thanks to Jason Gunthorpe for suggestions on the interface design. Also thanks to Jeff Squyres for prototyping support for this in Open MPI, which helped find several bugs during development. Signed-off-by: Roland Dreier --- drivers/char/Kconfig | 12 ++ drivers/char/Makefile | 1 + drivers/char/ummunot.c | 477 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/ummunot.h | 85 +++++++++ 4 files changed, 575 insertions(+), 0 deletions(-) create mode 100644 drivers/char/ummunot.c create mode 100644 include/linux/ummunot.h diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 6a06913..8d3dd9d 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -1108,6 +1108,18 @@ config DEVPORT depends on ISA || PCI default y +config UMMUNOT + tristate "Userspace MMU notifications" + select MMU_NOTIFIER + help + The ummunot (userspace MMU notification) driver creates a + character device that can be used by userspace libraries to + get notifications when an application's memory mapping + changed. This is used, for example, by RDMA libraries to + improve the reliability of memory registration caching, since + the kernel's MMU notifications can be used to know precisely + when to shoot down a cached registration. + source "drivers/s390/char/Kconfig" endmenu diff --git a/drivers/char/Makefile b/drivers/char/Makefile index 66f779a..39ff0c3 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -97,6 +97,7 @@ obj-$(CONFIG_NSC_GPIO) += nsc_gpio.o obj-$(CONFIG_CS5535_GPIO) += cs5535_gpio.o obj-$(CONFIG_GPIO_TB0219) += tb0219.o obj-$(CONFIG_TELCLOCK) += tlclk.o +obj-$(CONFIG_UMMUNOT) += ummunot.o obj-$(CONFIG_MWAVE) += mwave/ obj-$(CONFIG_AGP) += agp/ diff --git a/drivers/char/ummunot.c b/drivers/char/ummunot.c new file mode 100644 index 0000000..52a0b6d --- /dev/null +++ b/drivers/char/ummunot.c @@ -0,0 +1,477 @@ +/* + * Copyright (c) 2009 Cisco Systems. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenFabrics BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +MODULE_AUTHOR("Roland Dreier"); +MODULE_DESCRIPTION("Userspace MMU notifiers"); +MODULE_LICENSE("Dual BSD/GPL"); + +enum { + UMMUNOT_FLAG_DIRTY = 1, + UMMUNOT_FLAG_HINT = 2, +}; + +struct ummunot_reg { + u64 user_cookie; + unsigned long start; + unsigned long end; + unsigned long hint_start; + unsigned long hint_end; + unsigned long flags; + struct rb_node node; + struct list_head list; +}; + +struct ummunot_file { + struct mmu_notifier mmu_notifier; + struct mm_struct *mm; + struct rb_root reg_tree; + struct list_head dirty_list; + u64 *counter; + spinlock_t lock; + wait_queue_head_t read_wait; + struct fasync_struct *async_queue; +}; + +static struct ummunot_file *to_ummunot_file(struct mmu_notifier *mn) +{ + return container_of(mn, struct ummunot_file, mmu_notifier); +} + +static void ummunot_handle_not(struct mmu_notifier *mn, + unsigned long start, unsigned long end) +{ + struct ummunot_file *priv = to_ummunot_file(mn); + struct rb_node *n; + struct ummunot_reg *reg; + unsigned long flags; + int hit = 0; + + spin_lock_irqsave(&priv->lock, flags); + + for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) { + reg = rb_entry(n, struct ummunot_reg, node); + + if (reg->start >= end) + break; + + /* + * Ranges overlap if they're not disjoint; and they're + * disjoint if the end of one is before the start of + * the other one. + */ + if (!(reg->end <= start || end <= reg->start)) { + hit = 1; + + if (!test_and_set_bit(UMMUNOT_FLAG_DIRTY, ®->flags)) + list_add_tail(®->list, &priv->dirty_list); + + if (test_bit(UMMUNOT_FLAG_HINT, ®->flags)) { + clear_bit(UMMUNOT_FLAG_HINT, ®->flags); + } else { + set_bit(UMMUNOT_FLAG_HINT, ®->flags); + reg->hint_start = start; + reg->hint_end = end; + } + } + } + + if (hit) { + ++(*priv->counter); + flush_dcache_page(virt_to_page(priv->counter)); + wake_up_interruptible(&priv->read_wait); + kill_fasync(&priv->async_queue, SIGIO, POLL_IN); + } + + spin_unlock_irqrestore(&priv->lock, flags); +} + +static void ummunot_inval_page(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long addr) +{ + ummunot_handle_not(mn, addr, addr + PAGE_SIZE); +} + +static void ummunot_inval_range_start(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + ummunot_handle_not(mn, start, end); +} + +static const struct mmu_notifier_ops ummunot_mmu_notifier_ops = { + .invalidate_page = ummunot_inval_page, + .invalidate_range_start = ummunot_inval_range_start, +}; + +static int ummunot_open(struct inode *inode, struct file *filp) +{ + struct ummunot_file *priv; + int ret; + + if (filp->f_mode & FMODE_WRITE) + return -EINVAL; + + priv = kmalloc(sizeof *priv, GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->counter = (void *) get_zeroed_page(GFP_KERNEL); + if (!priv->counter) { + ret = -ENOMEM; + goto err; + } + + priv->reg_tree = RB_ROOT; + INIT_LIST_HEAD(&priv->dirty_list); + spin_lock_init(&priv->lock); + init_waitqueue_head(&priv->read_wait); + priv->async_queue = NULL; + + priv->mmu_notifier.ops = &ummunot_mmu_notifier_ops; + /* + * Register notifier last, since notifications can occur as + * soon as we register.... + */ + ret = mmu_notifier_register(&priv->mmu_notifier, current->mm); + if (ret) + goto err_page; + + priv->mm = current->mm; + atomic_inc(&priv->mm->mm_count); + + filp->private_data = priv; + + return 0; + +err_page: + free_page((unsigned long) priv->counter); + +err: + kfree(priv); + return ret; +} + +static int ummunot_close(struct inode *inode, struct file *filp) +{ + struct ummunot_file *priv = filp->private_data; + struct rb_node *n; + struct ummunot_reg *reg; + + mmu_notifier_unregister(&priv->mmu_notifier, priv->mm); + mmdrop(priv->mm); + free_page((unsigned long) priv->counter); + + for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) { + reg = rb_entry(n, struct ummunot_reg, node); + kfree(reg); + } + + kfree(priv); + + return 0; +} + +static ssize_t ummunot_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct ummunot_file *priv = filp->private_data; + struct ummunot_reg *reg; + ssize_t ret; + struct ummunot_event *events; + int max; + int n; + + events = (void *) get_zeroed_page(GFP_KERNEL); + if (!events) { + ret = -ENOMEM; + goto out; + } + + spin_lock_irq(&priv->lock); + + while (list_empty(&priv->dirty_list)) { + spin_unlock_irq(&priv->lock); + + if (filp->f_flags & O_NONBLOCK) { + ret = -EAGAIN; + goto out; + } + + if (wait_event_interruptible(priv->read_wait, + !list_empty(&priv->dirty_list))) { + ret = -ERESTARTSYS; + goto out; + } + + spin_lock_irq(&priv->lock); + } + + max = min(PAGE_SIZE, count) / sizeof *events; + + for (n = 0; n < max; ++n) { + if (list_empty(&priv->dirty_list)) { + events[n].type = UMMUNOT_EVENT_TYPE_LAST; + events[n].user_cookie_counter = *priv->counter; + ++n; + break; + } + + reg = list_first_entry(&priv->dirty_list, struct ummunot_reg, + list); + + events[n].type = UMMUNOT_EVENT_TYPE_INVAL; + if (test_bit(UMMUNOT_FLAG_HINT, ®->flags)) { + events[n].flags = UMMUNOT_EVENT_FLAG_HINT; + events[n].hint_start = max(reg->start, reg->hint_start); + events[n].hint_end = min(reg->end, reg->hint_end); + } else { + events[n].hint_start = reg->start; + events[n].hint_end = reg->end; + } + events[n].user_cookie_counter = reg->user_cookie; + + list_del(®->list); + reg->flags = 0; + } + + spin_unlock_irq(&priv->lock); + + if (copy_to_user(buf, events, n * sizeof *events)) + ret = -EFAULT; + else + ret = n * sizeof *events; + +out: + free_page((unsigned long) events); + return ret; +} + +static unsigned int ummunot_poll(struct file *filp, struct poll_table_struct *wait) +{ + struct ummunot_file *priv = filp->private_data; + + poll_wait(filp, &priv->read_wait, wait); + + return list_empty(&priv->dirty_list) ? 0 : (POLLIN | POLLRDNORM); +} + +static long ummunot_register_region(struct ummunot_file *priv, + struct ummunot_register_ioctl __user *arg) +{ + struct ummunot_register_ioctl parm; + struct ummunot_reg *reg, *treg; + struct rb_node **n = &priv->reg_tree.rb_node; + struct rb_node *pn; + int ret = 0; + + if (copy_from_user(&parm, arg, sizeof parm)) + return -EFAULT; + + if (parm.intf_version != UMMUNOT_INTF_VERSION) + return -EINVAL; + + reg = kmalloc(sizeof *reg, GFP_KERNEL); + if (!reg) + return -ENOMEM; + + reg->user_cookie = parm.user_cookie; + reg->start = parm.start; + reg->end = parm.end; + reg->flags = 0; + + spin_lock_irq(&priv->lock); + + for (pn = rb_first(&priv->reg_tree); pn; pn = rb_next(pn)) { + treg = rb_entry(pn, struct ummunot_reg, node); + + if (treg->user_cookie == parm.user_cookie) { + kfree(reg); + ret = -EINVAL; + goto out; + } + } + + pn = NULL; + while (*n) { + pn = *n; + treg = rb_entry(pn, struct ummunot_reg, node); + + if (reg->start <= treg->start) + n = &pn->rb_left; + else + n = &pn->rb_right; + } + + rb_link_node(®->node, pn, n); + rb_insert_color(®->node, &priv->reg_tree); + +out: + spin_unlock_irq(&priv->lock); + + return ret; +} + +static long ummunot_unregister_region(struct ummunot_file *priv, + __u64 __user *arg) +{ + u64 user_cookie; + struct rb_node *n; + struct ummunot_reg *reg; + int ret = -EINVAL; + + if (get_user(user_cookie, arg)) + return -EFAULT; + + spin_lock_irq(&priv->lock); + + for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) { + reg = rb_entry(n, struct ummunot_reg, node); + + if (reg->user_cookie == user_cookie) { + rb_erase(n, &priv->reg_tree); + if (test_bit(UMMUNOT_FLAG_DIRTY, ®->flags)) + list_del(®->list); + kfree(reg); + ret = 0; + break; + } + } + + spin_unlock_irq(&priv->lock); + + return ret; +} + +static long ummunot_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + struct ummunot_file *priv = filp->private_data; + void __user *argp = (void __user *) arg; + + switch (cmd) { + case UMMUNOT_REGISTER_REGION: + return ummunot_register_region(priv, argp); + case UMMUNOT_UNREGISTER_REGION: + return ummunot_unregister_region(priv, argp); + default: + return -ENOIOCTLCMD; + } +} + +static int ummunot_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct ummunot_file *priv = vma->vm_private_data; + + if (vmf->pgoff != 0) + return VM_FAULT_SIGBUS; + + vmf->page = virt_to_page(priv->counter); + get_page(vmf->page); + + return 0; + +} + +static struct vm_operations_struct ummunot_vm_ops = { + .fault = ummunot_fault, +}; + +static int ummunot_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct ummunot_file *priv = filp->private_data; + + if (vma->vm_end - vma->vm_start != PAGE_SIZE || + vma->vm_pgoff != 0) + return -EINVAL; + + vma->vm_ops = &ummunot_vm_ops; + vma->vm_private_data = priv; + + return 0; +} + +static int ummunot_fasync(int fd, struct file *filp, int on) +{ + struct ummunot_file *priv = filp->private_data; + + return fasync_helper(fd, filp, on, &priv->async_queue); +} + +static const struct file_operations ummunot_fops = { + .owner = THIS_MODULE, + .open = ummunot_open, + .release = ummunot_close, + .read = ummunot_read, + .poll = ummunot_poll, + .unlocked_ioctl = ummunot_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ummunot_ioctl, +#endif + .mmap = ummunot_mmap, + .fasync = ummunot_fasync, +}; + +static struct miscdevice ummunot_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "ummunot", + .fops = &ummunot_fops, +}; + +static int __init ummunot_init(void) +{ + return misc_register(&ummunot_misc); +} + +static void __exit ummunot_cleanup(void) +{ + misc_deregister(&ummunot_misc); +} + +module_init(ummunot_init); +module_exit(ummunot_cleanup); diff --git a/include/linux/ummunot.h b/include/linux/ummunot.h new file mode 100644 index 0000000..14fc75e --- /dev/null +++ b/include/linux/ummunot.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2009 Cisco Systems. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenFabrics BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _LINUX_UMMUNOT_H +#define _LINUX_UMMUNOT_H + +#include +#include + +#define UMMUNOT_INTF_VERSION 1 + +enum { + UMMUNOT_EVENT_TYPE_INVAL = 0, + UMMUNOT_EVENT_TYPE_LAST = 1, +}; + +enum { + UMMUNOT_EVENT_FLAG_HINT = 1 << 0, +}; + +/* + * If type field is INVAL, then user_cookie_counter holds the + * user_cookie for the region being reported; if the HINT flag is set + * then hint_start/hint_end hold the start and end of the mapping that + * was invalidated. (If HINT is not set, then multiple events + * invalidated parts of the registered range and hint_start/hint_end + * and set to the start/end of the whole registered range) + * + * If type is LAST, then the read operation has emptied the list of + * invalidated regions, and user_cookie_counter holds the value of the + * kernel's generation counter when the empty list occurred. The + * other fields are not filled in for this event. + */ +struct ummunot_event { + __u32 type; + __u32 flags; + __u64 hint_start; + __u64 hint_end; + __u64 user_cookie_counter; +}; + +struct ummunot_register_ioctl { + __u32 intf_version; /* in */ + __u32 reserved1; + __u64 start; /* in */ + __u64 end; /* in */ + __u64 user_cookie; /* in */ +}; + +#define UMMUNOT_MAGIC 'U' + +#define UMMUNOT_REGISTER_REGION _IOWR(UMMUNOT_MAGIC, 1, \ + struct ummunot_register_ioctl) +#define UMMUNOT_UNREGISTER_REGION _IOW(UMMUNOT_MAGIC, 2, __u64) + +#endif /* _LINUX_UMMUNOT_H */ -- 1.6.3.3 From jackm at dev.mellanox.co.il Wed Jul 15 22:38:44 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Thu, 16 Jul 2009 08:38:44 +0300 Subject: [ofa-general] Re: [PATCH V2] mlx4: check for FW version which properly supports resize_cq In-Reply-To: References: <200907081350.12462.jackm@dev.mellanox.co.il> <200907091927.42856.jackm@dev.mellanox.co.il> Message-ID: <200907160838.44401.jackm@dev.mellanox.co.il> On Wednesday 15 July 2009 01:33, Roland Dreier wrote: > It occurs to me that one change that makes sense and would help make > this fix cleaner is the following -- since after all if a command # is > out of range, that's really a different error than if a low-level driver > just doesn't implement a certain method. > This is exactly the patch I posted on Dec 2, 2008: http://lists.openfabrics.org/pipermail/general/2008-December/055734.html However, see my comments accompanying the patch. This is a global change for all verbs -- and I'm not sure that it won't break some current applications. MPI is OK with this change, however (in fact, MPI needs this change -- see the next post in the openfabrics list thread above). I suggest that you post to ewg, indicating the change (-ENOSYS for unimplemented commands instead of -EINVAL) and see if anyone responds. If it is OK, then you should just do it. -Jack From ogerlitz at voltaire.com Thu Jul 16 00:02:34 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Thu, 16 Jul 2009 10:02:34 +0300 Subject: [ofa-general] Re: [PATCH 0/8 v3] RDMAoE support In-Reply-To: <5D49E7A8952DC44FB38C38FA0D758EAD035AE999@mtlexch01.mtl.com> References: <20090713181310.GA31865@mtls03> <4A5C5332.7020403@voltaire.com> <5D49E7A8952DC44FB38C38FA0D758EAD035ADF95@mtlexch01.mtl.com> <4A5D9357.5020808@voltaire.com> <5D49E7A8952DC44FB38C38FA0D758EAD035AE999@mtlexch01.mtl.com> Message-ID: <4A5ED08A.2010907@voltaire.com> Liran Liss wrote: > I think that not having a user-space SA API is a big hole - we should not accept this situation and work to fill in the gap. Once such an API exists, all non-rdmacm apps will probably use it - implementing path queries and multicast joins is a tedious effort, as > the convergence in the kernel has shown. In short: no (big hole) and no (all apps will run to use it). In more details, if its a big hole, how come almost no one asked for it? next, many (most, maybe all) of the MPI people claim that their biggest/sole problem with the rdma-cm is the non scalability of the SA being their bottleneck (personally, I am not convinced the SA is the bottleneck) and as such they are not going in that direction. So we are still remain with SRP, does Mellanox see SRP deployments running over rdmaoe? > this is still TBD. We will need to define reasonable policies to integrate with DCB features, such as PFC. Any ideas on this are more than welcome! In your previous post "the rmdaoa "sa" code does more than just path resolution - it provides SL, MTU, rate, packet lifetime, and other params." you were using present so I'm not sure how I was supposed to guess that this is TBD. On the other hand, thinking about the SA thing a bit further, this is something to take care of. Basically, I understand that the idea is to replace LRH/GRH/BTH/ETH with something like MAC/VLAN/GRH/BTH/ETH - so SL is PCP, PKEY is VLAN (here we are hit directly by the IB bug of putting the pkey in the L4 header). Note that such mechanics (e.g PCP, VLAN, and DCBX related things) can be implemented more naturally in the rdma-cm layer > We understood that most people want to see this separated until the standard is finalized. In the meantime, we will try to reduce code duplication as much as possible. its open source, either send me pointer to such postings, or tell them to comment on the list > not at all. Its just a matter of what comes first - implementation is ongoing so this path-set doesn't play with e.g RDS using IPv4 addresses? I understood it does, so please clarify. > the only API changes are the sa-like path-queries and multicast joins; verbs are not changed in any way. In addition, we implemented the 'get_mac' method (for translating gids to macs) in the generic user-kernel ABI so any RDMAoE implementation can use it. I don't understand what concerns you; please explain. Any change to ib_verbs.h is an API change, and indeed there were few comments from Woody and others. you were commenting back, etc. What I'm saying is that I'd like to see the group who works on the software rdmaoe driver commenting too. Or. From vlad at lists.openfabrics.org Thu Jul 16 02:39:10 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Thu, 16 Jul 2009 02:39:10 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090716-0200 daily build status Message-ID: <20090716093910.E6F96102054B@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -fasynchronous-unwind-tables -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_cma)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c: In function 'sdp_cma_handler': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c:312: error: implicit declaration of function 'lock_sock_nested' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -fasynchronous-unwind-tables -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_cma)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c: In function 'sdp_cma_handler': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c:312: error: implicit declaration of function 'lock_sock_nested' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -m64 -mno-red-zone -mcmodel=kernel -funit-at-a-time -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090716-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From tziporet at mellanox.co.il Thu Jul 16 02:55:07 2009 From: tziporet at mellanox.co.il (Tziporet Koren) Date: Thu, 16 Jul 2009 12:55:07 +0300 Subject: [ofa-general] OFED 1.4.1-rc2 is available In-Reply-To: <5D49E7A8952DC44FB38C38FA0D758EAD02D5F39D@mtlexch01.mtl.com> References: <5D49E7A8952DC44FB38C38FA0D758EAD02C12252@mtlexch01.mtl.com> <5D49E7A8952DC44FB38C38FA0D758EAD02D5F39D@mtlexch01.mtl.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD035F7BDA@mtlexch01.mtl.com> OFED 1.4.2-RC2 is available. Notes: 1. Any Lustre and NFS/RDMA users should use this release 2. This release is aimed for critical bug fixes only, and will pass only a short QA cycle 2. OFA interop and Logo program testing are not planned for 1.4.2 The tarball is available on: http://www.openfabrics.org/downloads/OFED/ofed-1.4.2/OFED-1.4.2-rc2.tgz To get BUILD_ID run ofed_info Please report any issues in bugzilla https://bugs.openfabrics.org/ for OFED 1.4.2 Vladimir & Tziporet ======================================================================== Release information: ------------------------------ Linux Operating Systems: - RedHat EL4 up4: 2.6.9-42.ELsmp * - RedHat EL4 up5: 2.6.9-55.ELsmp - RedHat EL4 up6: 2.6.9-67.ELsmp - RedHat EL4 up7: 2.6.9-78.ELsmp - RedHat EL5: 2.6.18-8.el5 - RedHat EL5 up1: 2.6.18-53.el5 - RedHat EL5 up2: 2.6.18-92.el5 - RedHat EL5 up3: 2.6.18-128.el5 - OEL 4.5: 2.6.9-55.ELsmp - OEL 5.2: 2.6.18-92.el5 - CentOS 5.2: 2.6.18-92.el5 - Fedora C9: 2.6.25-14.fc9 * - SLES10: 2.6.16.21-0.8-smp - SLES10 SP1: 2.6.16.46-0.12-smp - SLES10 SP1 up1: 2.6.16.53-0.16-smp - SLES10 SP2: 2.6.16.60-0.21-smp - SLES11 GA: 2.6.27.13-1-default - OpenSuSE 10.3: 2.6.22.5-31 * - kernel.org: 2.6.26 and 2.6.27 * Minimal QA for these versions Systems: * x86_64 * x86 * ia64 * ppc64 Changes from OFED-1.4.1 ======================== Bug fixes: ---------- SDP: - Fix memory leak in bzcopy (#1672) - Fix bad credits advertised when connection initiated (#1679) - Fix compilation on i386 with gcc 3.4 used in Redhat 4. (#1630) - Fix Data integrity error (#1672) Backports: - Fix clear-dirty-page accounting. This bug was hit in Lustre testing (#1650) - Fix NFS stale file handles (#1680) - Simple NFS file operations causes RHEL 5.3 server to hard hang. (#1676) - iozone direct write test causes OFED to hang/crash (#1675) - Module crash on server with multiple client simple load (#1677) mlx4 driver: - Fix post send of local invalidate and fast registration packets. This fixed nfsrdma server crash @test5 connectathon basic test. (#1571) nes driver: - fix qp refcount during disconnect Features: --------- nes driver: - Make LRO as default feature mlx4 driver: - Added a new device ID: 0x6764: MT26468 ConnectX EN 10GigE PCIe gen2 Tasks that should be completed for GA (July 22): ================================================ - Fix bug: 1678 simple RDMA NFSv4 traffic hangs the clients attempting to complete - Documents update From tziporet at mellanox.co.il Thu Jul 16 04:11:48 2009 From: tziporet at mellanox.co.il (Tziporet Koren) Date: Thu, 16 Jul 2009 14:11:48 +0300 Subject: [ofa-general] RE: OFED 1.4.2-rc2 is available References: <5D49E7A8952DC44FB38C38FA0D758EAD02C12252@mtlexch01.mtl.com> <5D49E7A8952DC44FB38C38FA0D758EAD02D5F39D@mtlexch01.mtl.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD035F7CD9@mtlexch01.mtl.com> Fixed subject - its 1.4.2 and not 1.4.1 -----Original Message----- From: Tziporet Koren Sent: Thursday, July 16, 2009 12:55 PM To: Tziporet Koren; ewg at lists.openfabrics.org Cc: general at lists.openfabrics.org Subject: OFED 1.4.1-rc2 is available OFED 1.4.2-RC2 is available. Notes: 1. Any Lustre and NFS/RDMA users should use this release 2. This release is aimed for critical bug fixes only, and will pass only a short QA cycle 2. OFA interop and Logo program testing are not planned for 1.4.2 The tarball is available on: http://www.openfabrics.org/downloads/OFED/ofed-1.4.2/OFED-1.4.2-rc2.tgz To get BUILD_ID run ofed_info Please report any issues in bugzilla https://bugs.openfabrics.org/ for OFED 1.4.2 Vladimir & Tziporet ======================================================================== Release information: ------------------------------ Linux Operating Systems: - RedHat EL4 up4: 2.6.9-42.ELsmp * - RedHat EL4 up5: 2.6.9-55.ELsmp - RedHat EL4 up6: 2.6.9-67.ELsmp - RedHat EL4 up7: 2.6.9-78.ELsmp - RedHat EL5: 2.6.18-8.el5 - RedHat EL5 up1: 2.6.18-53.el5 - RedHat EL5 up2: 2.6.18-92.el5 - RedHat EL5 up3: 2.6.18-128.el5 - OEL 4.5: 2.6.9-55.ELsmp - OEL 5.2: 2.6.18-92.el5 - CentOS 5.2: 2.6.18-92.el5 - Fedora C9: 2.6.25-14.fc9 * - SLES10: 2.6.16.21-0.8-smp - SLES10 SP1: 2.6.16.46-0.12-smp - SLES10 SP1 up1: 2.6.16.53-0.16-smp - SLES10 SP2: 2.6.16.60-0.21-smp - SLES11 GA: 2.6.27.13-1-default - OpenSuSE 10.3: 2.6.22.5-31 * - kernel.org: 2.6.26 and 2.6.27 * Minimal QA for these versions Systems: * x86_64 * x86 * ia64 * ppc64 Changes from OFED-1.4.1 ======================== Bug fixes: ---------- SDP: - Fix memory leak in bzcopy (#1672) - Fix bad credits advertised when connection initiated (#1679) - Fix compilation on i386 with gcc 3.4 used in Redhat 4. (#1630) - Fix Data integrity error (#1672) Backports: - Fix clear-dirty-page accounting. This bug was hit in Lustre testing (#1650) - Fix NFS stale file handles (#1680) - Simple NFS file operations causes RHEL 5.3 server to hard hang. (#1676) - iozone direct write test causes OFED to hang/crash (#1675) - Module crash on server with multiple client simple load (#1677) mlx4 driver: - Fix post send of local invalidate and fast registration packets. This fixed nfsrdma server crash @test5 connectathon basic test. (#1571) nes driver: - fix qp refcount during disconnect Features: --------- nes driver: - Make LRO as default feature mlx4 driver: - Added a new device ID: 0x6764: MT26468 ConnectX EN 10GigE PCIe gen2 Tasks that should be completed for GA (July 22): ================================================ - Fix bug: 1678 simple RDMA NFSv4 traffic hangs the clients attempting to complete - Documents update From liranl at mellanox.co.il Thu Jul 16 04:18:49 2009 From: liranl at mellanox.co.il (Liran Liss) Date: Thu, 16 Jul 2009 14:18:49 +0300 Subject: [ofa-general] RE: [PATCH 0/8 v3] RDMAoE support In-Reply-To: <4A5ED08A.2010907@voltaire.com> References: <20090713181310.GA31865@mtls03> <4A5C5332.7020403@voltaire.com> <5D49E7A8952DC44FB38C38FA0D758EAD035ADF95@mtlexch01.mtl.com> <4A5D9357.5020808@voltaire.com> <5D49E7A8952DC44FB38C38FA0D758EAD035AE999@mtlexch01.mtl.com> <4A5ED08A.2010907@voltaire.com> Message-ID: <5D49E7A8952DC44FB38C38FA0D758EAD0EE0DC@mtlexch01.mtl.com> S.B. --Liran In short: no (big hole) and no (all apps will run to use it). In more details, if its a big hole, how come almost no one asked for it? next, many (most, maybe all) of the MPI people claim that their biggest/sole problem with the rdma-cm is the non scalability of the SA being their bottleneck (personally, I am not convinced the SA is the bottleneck) and as such they are not going in that direction. So we are still remain with SRP, does Mellanox see SRP deployments running over rdmaoe? LL: we keep getting questions from customers using native IB regarding a sensible way to do path queries and multicast joins from user-space. The only answer I can currently give them is to use rdmacm. That said, you are probably correct that most IB apps (in general) will use rdmacm. Regarding SRP, it is definitely a possibility. so this path-set doesn't play with e.g RDS using IPv4 addresses? I understood it does, so please clarify. LL: RDS uses the CMA, which supports IPv4. The current rdmaoe CMA code uses IPv6 link-local addresses as GIDs; once we add IPv4 support, the CMA will encapsulate the given IP address in the GID. From klto at zhaw.ch Thu Jul 16 02:12:39 2009 From: klto at zhaw.ch (Tobias Klauser) Date: Thu, 16 Jul 2009 11:12:39 +0200 Subject: [ofa-general] [PATCH] infiniband: Use %pM conversion specifier Message-ID: <1247735559-21945-1-git-send-email-klto@zhaw.ch> Use the %pM conversion specifier to print a MAC address. Signed-off-by: Tobias Klauser --- drivers/infiniband/hw/amso1100/c2.c | 6 +----- 1 files changed, 1 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/hw/amso1100/c2.c b/drivers/infiniband/hw/amso1100/c2.c index 0cfbb6d..8250740 100644 --- a/drivers/infiniband/hw/amso1100/c2.c +++ b/drivers/infiniband/hw/amso1100/c2.c @@ -86,11 +86,7 @@ MODULE_DEVICE_TABLE(pci, c2_pci_table); static void c2_print_macaddr(struct net_device *netdev) { - pr_debug("%s: MAC %02X:%02X:%02X:%02X:%02X:%02X, " - "IRQ %u\n", netdev->name, - netdev->dev_addr[0], netdev->dev_addr[1], netdev->dev_addr[2], - netdev->dev_addr[3], netdev->dev_addr[4], netdev->dev_addr[5], - netdev->irq); + pr_debug("%s: MAC %pM, IRQ %u\n", netdev->name, netdev->dev_addr, netdev->irq); } static void c2_set_rxbufsize(struct c2_port *c2_port) -- 1.6.0.4 From amirv at mellanox.co.il Thu Jul 16 06:05:31 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Thu, 16 Jul 2009 16:05:31 +0300 Subject: [ofa-general] [PATCH] sdp: fix some warning and bugs in porting to ofed 1.5 In-Reply-To: <003e01ca0544$ef95d710$cec18530$@van.malland@linvision.com> References: <1247653668-21541-1-git-send-email-amirv@mellanox.co.il> <003e01ca0544$ef95d710$cec18530$@van.malland@linvision.com> Message-ID: <4A5F259B.1010701@mellanox.co.il> Maarten Hi, I checked it in my setup using kernel 2.6.30 and I got same results as in ofed-1.4. Since SDP in 1.5 is going to have lots of changes in the data path it might not be very stable those days. It should be stabilized a lot more around mid August. Anyway the exact version I used could be taken from: url: git://git.openfabrics.org/~amirv/ofed_1_5.git branch: zcopy There should not be differences between this and ofed_kernel_1_5 branch (which I guess you used). For some info about the tests I did and if you have something to add - please view and update bugzilla: https://bugs.openfabrics.org/show_bug.cgi?id=1682 - Amir On 07/15/2009 03:08 PM, Maarten van Malland wrote: > Hi Amir, > > We tried your patch and unfortunately it didn't help much. Actually, it made > it worse. Here is the dmesg output when we ran nttcp over sdp: > > inftsttwin03 ~ # dmesg > sdp_reset_sk:447 sdp_sock( 4453:0 5038:51621): setting state to error > BUG: unable to handle kernel paging request at 0000000000001398 > IP: [] sdp_post_recv+0x299/0x66b [ib_sdp] > PGD 11a961067 PUD 12dcac067 PMD 0 > Oops: 0002 [#1] PREEMPT SMP > last sysfs file: /sys/devices/system/cpu/cpu7/cache/index2/shared_cpu_map > CPU 5 > Modules linked in: ib_sdp w83793 hwmon_vid rdma_ucm rdma_cm iw_cm ib_addr > mlx4_ib ib_ipoib ib_cm ib_sa ib_ > uverbs ib_umad ib_mthca ib_mad i2c_i801 ib_core mlx4_core e1000e i2c_core > i5k_amb hwmon [last unloaded: ib > _sdp] > Pid: 12072, comm: nttcp Not tainted 2.6.30.1 #1 X7DWT > RIP: 0010:[] [] > sdp_post_recv+0x299/0x66b [ib_sdp] > RSP: 0018:ffff88011b4d1ab8 EFLAGS: 00010202 > RAX: 0000000000001398 RBX: 0000000000000008 RCX: ffffe20003df9d40 > RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffffff80277789 > RBP: ffff88011b4d1bf8 R08: 0000000000000000 R09: ffff880000015e00 > R10: 0000000000000000 R11: ffffe20003df9d40 R12: ffff88011099f500 > R13: ffff880110474080 R14: 00000000000000d2 R15: 0000000000000000 > FS: 00007f337b97c6f0(0000) GS:ffff880028186000(0000) knlGS:0000000000000000 > CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b > CR2: 0000000000001398 CR3: 000000012d8bb000 CR4: 00000000000406e0 > DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 > Process nttcp (pid: 12072, threadinfo ffff88011b4d0000, task > ffff88011b49e640) > Stack: > ffff88011b4d1af8 ffffffff802308a8 00000000ffffffff ffff88012d4b1000 > 0000000000001398 ffff88012d965800 0000000000000139 0000000000011a80 > ffff88011b4d1b98 ffffffff80553141 000000011b4c8000 0000000000000001 > Call Trace: > [] ? finish_task_switch+0xb5/0xc4 > [] ? thread_return+0x4e/0x9d > [] ? local_bh_disable+0xd/0xf > [] ? _spin_lock_bh+0x11/0x34 > [] ? _spin_lock_irqsave+0x1d/0x39 > [] ? sub_preempt_count+0x9e/0xbe > [] sdp_do_posts+0xa71/0xa91 [ib_sdp] > [] ? sk_wait_data+0xb8/0xcd > [] ? autoremove_wake_function+0x0/0x38 > [] sdp_recvmsg+0x360/0x5dc [ib_sdp] > [] sock_common_recvmsg+0x32/0x47 > [] sock_aio_read+0xe8/0xfc > [] ? irq_exit+0x4f/0x51 > [] ? do_sync_read+0x62/0x126 > [] do_sync_read+0xe2/0x126 > [] ? trace_hardirqs_on_thunk+0x3a/0x3c > [] ? restore_args+0x0/0x30 > [] ? autoremove_wake_function+0x0/0x38 > [] ? sub_preempt_count+0x9e/0xbe > [] ? _spin_unlock+0x10/0x29 > [] vfs_read+0xbe/0x134 > [] sys_read+0x47/0x70 > [] system_call_fastpath+0x16/0x1b > Code: b8 00 00 00 41 3b 9d 6c 05 00 00 7c 83 8b 85 f0 fe ff ff 49 8b 95 30 > 04 00 00 83 e0 3f 48 6b c0 58 4 > 8 01 d0 48 89 85 e0 fe ff ff <4c> 89 20 4d 8b bd d0 04 00 00 49 8b 87 98 02 > 00 00 48 85 c0 74 > RIP [] sdp_post_recv+0x299/0x66b [ib_sdp] > RSP > CR2: 0000000000001398 > ---[ end trace f3d85b6359a569b5 ]--- > > Perhaps you can take a second look at it, > > Best regards, > > Maarten > > > -----Oorspronkelijk bericht----- > Van: general-bounces at lists.openfabrics.org > [mailto:general-bounces at lists.openfabrics.org] Namens Amir Vadai > Verzonden: woensdag 15 juli 2009 12:28 > Aan: general at lists.openfabrics.org > CC: Amir Vadai > Onderwerp: [ofa-general] [PATCH] sdp: fix some warning and bugs in porting > to ofed 1.5 > > Signed-off-by: Amir Vadai > --- > drivers/infiniband/ulp/sdp/sdp.h | 9 +++++++-- > drivers/infiniband/ulp/sdp/sdp_cma.c | 2 +- > drivers/infiniband/ulp/sdp/sdp_main.c | 17 ++++++++--------- > drivers/infiniband/ulp/sdp/sdp_rx.c | 14 ++++---------- > 4 files changed, 20 insertions(+), 22 deletions(-) > > diff --git a/drivers/infiniband/ulp/sdp/sdp.h > b/drivers/infiniband/ulp/sdp/sdp.h > index d03d6af..ec8dbe1 100644 > --- a/drivers/infiniband/ulp/sdp/sdp.h > +++ b/drivers/infiniband/ulp/sdp/sdp.h > @@ -11,12 +11,15 @@ > #define SDPSTATS_ON > /* #define SDP_PROFILING */ > > -#define _sdp_printk(func, line, level, sk, format, arg...) \ > +#define _sdp_printk(func, line, level, sk, format, arg...) do { > \ > + preempt_disable(); \ > printk(level "%s:%d sdp_sock(%5d:%d %d:%d): " format, \ > func, line, \ > current->pid, smp_processor_id(), \ > (sk) ? inet_sk(sk)->num : -1, \ > - (sk) ? ntohs(inet_sk(sk)->dport) : -1, ## arg) > + (sk) ? ntohs(inet_sk(sk)->dport) : -1, ## arg); \ > + preempt_enable(); \ > +} while (0) > #define sdp_printk(level, sk, format, arg...) \ > _sdp_printk(__func__, __LINE__, level, sk, format, ## arg) > #define sdp_warn(sk, format, arg...) \ > @@ -71,6 +74,7 @@ static inline unsigned long long current_nsec(void) > #define sdp_prf1(sk, s, format, arg...) ({ \ > struct sdpprf_log *l = \ > &sdpprf_log[sdpprf_log_count++ & (SDPPRF_LOG_SIZE - 1)]; \ > + preempt_disable(); \ > l->idx = sdpprf_log_count - 1; \ > l->pid = current->pid; \ > l->sk_num = (sk) ? inet_sk(sk)->num : -1; \ > @@ -81,6 +85,7 @@ static inline unsigned long long current_nsec(void) > l->time = current_nsec(); \ > l->func = __func__; \ > l->line = __LINE__; \ > + preempt_enable(); \ > 1; \ > }) > #define sdp_prf(sk, s, format, arg...) > diff --git a/drivers/infiniband/ulp/sdp/sdp_cma.c > b/drivers/infiniband/ulp/sdp/sdp_cma.c > index 2354fff..f8a7303 100644 > --- a/drivers/infiniband/ulp/sdp/sdp_cma.c > +++ b/drivers/infiniband/ulp/sdp/sdp_cma.c > @@ -308,7 +308,7 @@ int sdp_cma_handler(struct rdma_cm_id *id, struct > rdma_cm_event *event) > -EINVAL : 0; > } > > - lock_sock(sk); > + lock_sock_nested(sk, SINGLE_DEPTH_NESTING); > sdp_dbg(sk, "%s event %d id %p\n", __func__, event->event, id); > if (!sdp_sk(sk)->id) { > sdp_dbg(sk, "socket is being torn down\n"); > diff --git a/drivers/infiniband/ulp/sdp/sdp_main.c > b/drivers/infiniband/ulp/sdp/sdp_main.c > index 48e43dc..ab49846 100644 > --- a/drivers/infiniband/ulp/sdp/sdp_main.c > +++ b/drivers/infiniband/ulp/sdp/sdp_main.c > @@ -181,7 +181,6 @@ static int sdp_get_port(struct sock *sk, unsigned short > snum) > static void sdp_destroy_qp(struct sdp_sock *ssk) > { > struct ib_pd *pd = NULL; > - unsigned long flags; > > > sdp_dbg(&ssk->isk.sk, "destroying qp\n"); > @@ -189,7 +188,6 @@ static void sdp_destroy_qp(struct sdp_sock *ssk) > > del_timer(&ssk->tx_ring.timer); > > - rx_ring_lock(ssk, flags); > > sdp_rx_ring_destroy(ssk); > sdp_tx_ring_destroy(ssk); > @@ -209,9 +207,6 @@ static void sdp_destroy_qp(struct sdp_sock *ssk) > ib_dealloc_pd(pd); > > sdp_remove_large_sock(ssk); > - > - rx_ring_unlock(ssk, flags); > - > } > > static void sdp_reset_keepalive_timer(struct sock *sk, unsigned long len) > @@ -453,10 +448,6 @@ void sdp_reset_sk(struct sock *sk, int rc) > sdp_set_error(sk, rc); > } > > - sdp_destroy_qp(ssk); > - > - memset((void *)&ssk->id, 0, sizeof(*ssk) - offsetof(typeof(*ssk), > id)); > - > sk->sk_state_change(sk); > > /* Don't destroy socket before destroy work does its job */ > @@ -976,6 +967,10 @@ void sdp_destroy_work(struct work_struct *work) > struct sock *sk = &ssk->isk.sk; > sdp_dbg(sk, "%s: refcnt %d\n", __func__, > atomic_read(&sk->sk_refcnt)); > > + sdp_destroy_qp(ssk); > + > + memset((void *)&ssk->id, 0, sizeof(*ssk) - offsetof(typeof(*ssk), > id)); > + > sdp_cancel_dreq_wait_timeout(ssk); > > if (sk->sk_state == TCP_TIME_WAIT) > @@ -2559,6 +2554,10 @@ static void __exit sdp_exit(void) > sdp_proc_unregister(); > > ib_unregister_client(&sdp_client); > + > + percpu_counter_destroy(sockets_allocated); > + percpu_counter_destroy(orphan_count); > + > kfree(orphan_count); > kfree(sockets_allocated); > } > diff --git a/drivers/infiniband/ulp/sdp/sdp_rx.c > b/drivers/infiniband/ulp/sdp/sdp_rx.c > index 749a83a..38417c4 100644 > --- a/drivers/infiniband/ulp/sdp/sdp_rx.c > +++ b/drivers/infiniband/ulp/sdp/sdp_rx.c > @@ -447,12 +447,12 @@ static int sdp_process_rx_ctl_skb(struct sdp_sock > *ssk, struct sk_buff *skb) > break; > case SDP_MID_CHRCVBUF: > sdp_dbg_data(sk, "Handling RX CHRCVBUF\n"); > - sdp_handle_resize_request(ssk, (struct sdp_chrecvbuf *)h); > + sdp_handle_resize_request(ssk, (struct sdp_chrecvbuf > *)(h+1)); > __kfree_skb(skb); > break; > case SDP_MID_CHRCVBUF_ACK: > sdp_dbg_data(sk, "Handling RX CHRCVBUF_ACK\n"); > - sdp_handle_resize_ack(ssk, (struct sdp_chrecvbuf *)h); > + sdp_handle_resize_ack(ssk, (struct sdp_chrecvbuf *)(h+1)); > __kfree_skb(skb); > break; > default: > @@ -787,9 +787,6 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct > ib_device *device) > { > struct ib_cq *rx_cq; > int rc = 0; > - unsigned long flags; > - > - rx_ring_lock(ssk, flags); > > atomic_set(&ssk->rx_ring.head, 1); > atomic_set(&ssk->rx_ring.tail, 1); > @@ -797,12 +794,11 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct > ib_device *device) > ssk->rx_ring.buffer = kmalloc( > sizeof *ssk->rx_ring.buffer * SDP_RX_SIZE, > GFP_KERNEL); > if (!ssk->rx_ring.buffer) { > - rc = -ENOMEM; > sdp_warn(&ssk->isk.sk, > "Unable to allocate RX Ring size %zd.\n", > sizeof(*ssk->rx_ring.buffer) * SDP_RX_SIZE); > > - goto out; > + return -ENOMEM; > } > > /* TODO: use vector=IB_CQ_VECTOR_LEAST_ATTACHED when implemented > @@ -822,13 +818,11 @@ int sdp_rx_ring_create(struct sdp_sock *ssk, struct > ib_device *device) > > sdp_arm_rx_cq(&ssk->isk.sk); > > - goto out; > + return 0; > > err_cq: > kfree(ssk->rx_ring.buffer); > ssk->rx_ring.buffer = NULL; > -out: > - rx_ring_unlock(ssk, flags); > return rc; > } > -- Amir Vadai Software Eng. Mellanox Technologies mailto: amirv at mellanox.co.il Tel +972-3-6259539 From hnrose at comcast.net Thu Jul 16 06:32:38 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Thu, 16 Jul 2009 09:32:38 -0400 Subject: [ofa-general] [PATCH] infiniband-diags/ibroute.c: Fix typo in IBWARN message Message-ID: <20090716133238.GB24200@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/infiniband-diags/src/ibroute.c b/infiniband-diags/src/ibroute.c index ad6d94c..a5df490 100644 --- a/infiniband-diags/src/ibroute.c +++ b/infiniband-diags/src/ibroute.c @@ -305,7 +305,7 @@ dump_unicast_tables(ib_portid_t *portid, int startlid, int endlid) endlid = top; if (endlid > IB_MAX_UCAST_LID) { - IBWARN("ilegal lft top %d, truncate to %d", endlid, IB_MAX_UCAST_LID); + IBWARN("illegal lft top %d, truncate to %d", endlid, IB_MAX_UCAST_LID); endlid = IB_MAX_UCAST_LID; } From hnrose at comcast.net Thu Jul 16 06:31:58 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Thu, 16 Jul 2009 09:31:58 -0400 Subject: [ofa-general] [PATCH] infiniband-diags/ibtracert.c: Fix comparison to SwitchInfo.MulticastFDBCap in switch_mclookup Message-ID: <20090716133157.GA24200@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/infiniband-diags/src/ibtracert.c b/infiniband-diags/src/ibtracert.c index a887157..dc05a29 100644 --- a/infiniband-diags/src/ibtracert.c +++ b/infiniband-diags/src/ibtracert.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire Inc. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -450,7 +451,7 @@ switch_mclookup(Node *node, ib_portid_t *portid, int mlid, char *map) mad_decode_field(si, IB_SW_MCAST_FDB_CAP_F, &sw.mccap); - if (mlid > sw.mccap) + if (mlid >= sw.mccap) return -1; block = mlid / 32; From hal.rosenstock at gmail.com Thu Jul 16 06:38:41 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Thu, 16 Jul 2009 09:38:41 -0400 Subject: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: <4A4747B2.2030907@Voltaire.COM> References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM> Message-ID: On Sun, Jun 28, 2009 at 6:36 AM, Slava Strebkov wrote: > [PATCH 1/4] Patch implements multicast multiplexing as proposed in the >  thread entitled "IPv6 and IPoIB scalability issue". Would you be more explicit about what is covered (and what remains to be covered) in that lengthy thread ? >  This first patch contains definitions for new data types >  and infrastructure functions. >  Signed-off-by: Slava Strebkov > > --- >  opensm/include/opensm/osm_multicast.h |   94 +++++++++++++++++++++++++++++++-- >  opensm/opensm/osm_multicast.c         |   82 ++++++++++++++++++++++++++++- >  2 files changed, 170 insertions(+), 6 deletions(-) > > diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h > index a871306..02b63bd 100644 > --- a/opensm/include/opensm/osm_multicast.h > +++ b/opensm/include/opensm/osm_multicast.h > @@ -1,5 +1,5 @@ >  /* > - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. > + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. >  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. >  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. >  * > @@ -45,6 +45,7 @@ > >  #include >  #include > +#include >  #include >  #include >  #include > @@ -93,7 +94,7 @@ BEGIN_C_DECLS >  * >  * SYNOPSIS >  */ > -typedef struct osm_mcast_mgr_ctxt { > +    typedef struct osm_mcast_mgr_ctxt { Is this (and other similar) formatting change(s) needed ? While not a problem here, it would be easier to read/review the patches if the formatting changes are in separate patches. >        cl_list_item_t list_item; >        ib_net16_t mlid; >  } osm_mcast_mgr_ctxt_t; > @@ -107,6 +108,89 @@ typedef struct osm_mcast_mgr_ctxt { >  * SEE ALSO >  *********/ > > +/****s* OpenSM: Multicast Group Holder/osm_mgrp_holder_t > +* NAME > +*       osm_mgrp_holder_t > +* > +* DESCRIPTION > +*       Holder for mgroups. > +* > +*       The osm_mgrp_t object should be treated as opaque and should > +*       be manipulated only through the provided functions. > +* > +* SYNOPSIS > +*/ > + > +typedef struct osm_mgrp_holder { > +       cl_fmap_t mgrp_map; > +       cl_qmap_t mgrp_port_map; > +       ib_gid_t common_mgid; > +       osm_mtree_node_t *p_root; > +       boolean_t to_be_deleted; > +       uint32_t last_tree_id; > +       uint32_t last_change_id; > +       ib_net16_t mlid; > +} osm_mgrp_holder_t; > + > +/* > +* FIELDS > +*       mgrp_map > +*               Map for mgroups.  Must be first element!! > +* > +*       mgrp_port_map > +*               Map of  all ports joined same mlid > +* > +*       common_mgid > +*               mgid of mgroup, ANDed with bitmask. > +*       mgid of each mgroup in mgrp_map, ANDed with bitmask, > +*       see osm_mgrp_holder_prepare_common_mgid > +* > +*       p_root > +*               Pointer to the root "tree node" in the single spanning tree > +*               for this multicast group holder.  The nodes of the tree represent > +*               switches.  Member ports are not represented in the tree. > +* > +*       to_be_deleted > +*               Since holders  are deleted when there are no mgroups in. > +* > +*       last_change_id > +*               a counter for the number of changes applied to the group in this holder. > +*               This counter shuold be incremented on any modification > +*               to the group: joining or leaving of ports. > +* > +*       last_tree_id > +*               the last change id used for building the current tree. > +* > +*       mlid > +*               mlid of current group holder > +*/ > + /****s* OpenSM: Multicast group Port /osm_mgrp_port _t > +* NAME > +*       osm_mgrp_port _t > +* > +* DESCRIPTION > +*       Holder for pointers to mgroups and port guid. > +* > +* > +* SYNOPSIS > +*/ > +typedef struct _osm_mgrp_port { > +       cl_map_item_t guid_item; > +       cl_qlist_t mgroups; > +       ib_net64_t port_guid; > +} osm_mgrp_port_t; > +/* > +* FIELDS > +*       guid_item > +*               Map for ports. Must be first element > +* > +*       mgroups > +*               Map  for  mgroups opened by this port. > +* > +*       portguid > +*               guid of  port representing current structure > +*/ > + >  /****s* OpenSM: Multicast Group/osm_mgrp_t >  * NAME >  *      osm_mgrp_t > @@ -355,7 +439,7 @@ static inline ib_net16_t osm_mgrp_get_mlid(IN const osm_mgrp_t * const p_mgrp) >  * >  * SYNOPSIS >  */ > -osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t *subn, osm_log_t *log, > +osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t * subn, osm_log_t * log, >                                  IN osm_mgrp_t * const p_mgrp, >                                  IN const ib_gid_t * const p_port_gid, >                                  IN const uint8_t join_state, > @@ -452,8 +536,8 @@ osm_mgrp_delete_port(IN osm_subn_t * const p_subn, >  * SEE ALSO >  *********/ > > -int osm_mgrp_remove_port(osm_subn_t *subn, osm_log_t *log, osm_mgrp_t *mgrp, > -                        osm_mcm_port_t *mcm, uint8_t join_state); > +int osm_mgrp_remove_port(osm_subn_t * subn, osm_log_t * log, osm_mgrp_t * mgrp, > +                        osm_mcm_port_t * mcm, uint8_t join_state); > >  /****f* OpenSM: Multicast Group/osm_mgrp_apply_func >  * NAME > diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c > index d2733c4..ae0a818 100644 > --- a/opensm/opensm/osm_multicast.c > +++ b/opensm/opensm/osm_multicast.c > @@ -1,5 +1,5 @@ >  /* > - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. > + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. >  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. >  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. >  * > @@ -48,6 +48,7 @@ >  #include >  #include >  #include > +#include > >  /********************************************************************** >  **********************************************************************/ > @@ -298,3 +299,82 @@ void osm_mgrp_apply_func(const osm_mgrp_t * p_mgrp, osm_mgrp_func_t p_func, >        if (p_mtn) >                mgrp_apply_func_sub(p_mgrp, p_mtn, p_func, context); >  } > + > +/********************************************************************** > + **********************************************************************/ > +#define PREFIX_MASK_IP CL_HTON64(0xff10ffff0000ffffULL) > +#define PREFIX_SIGNATURE_IPV4 CL_HTON64(0xff10401b00000000ULL) > +#define INTERFACE_ID_IPV4            CL_HTON64(0x0000000fffffffffULL) > +#define PREFIX_SIGNATURE_IPV6                    CL_HTON64(0xff10601b00000000ULL) > +#define INTERFACE_ID_ALL_NODES           CL_HTON64(0x00000000ffffffffULL) > +#define INTERFACE_ID_ALL_HOSTS           CL_HTON64(0x0000000000000001ULL) > +#define INTERFACE_ID_ALL_GATEWAYS                CL_HTON64(0x000000000000004dULL) > +#define INTERFACE_ID_ALL_ROUTERS                 CL_HTON64(0x0000000000000002ULL) > +#define PREFIX_PKEY_MASK_OFF          CL_HTON64(0xffffffff0000ffffULL) > +#define PREFIX_MASK CL_HTON64(0xff10ffff0000ffffULL) > +#define PREFIX_SIGNATURE CL_HTON64(0xff10601b00000000ULL) > +#define INT_ID_MASK CL_HTON64(0xfffffff1ff000000ULL) > +#define INT_ID_SIGNATURE CL_HTON64(0x00000001ff000000ULL) > + > +/********************************************************************** > + **********************************************************************/ > + /* mask used for multiplexing mgids to one mlid. Here default value (0xff) means that 8 lsb bits > +    of mgid will be masked off . > +  */ > +static uint64_t MGID_MUX_MLID_MASK = CL_HTON64(0x00000000000000ffULL); Is this only changed by source change and recompilation ? > +void osm_mgrp_holder_prepare_common_mgid(IN const ib_gid_t * const p_mgid, > +                                        OUT ib_gid_t * p_common_mgid) > +{ > +       memcpy(p_common_mgid, p_mgid, sizeof(ib_gid_t)); > +       /* Don't mux non IPoIB mgids. When mux mask is zero, no multiplexing occures */ > +       if ((((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) != > +             PREFIX_SIGNATURE_IPV4) > +            && ((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) != > +                PREFIX_SIGNATURE_IPV6)) || (!MGID_MUX_MLID_MASK)) > +               return; > + > +       if (((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) == > +            PREFIX_SIGNATURE_IPV4) > +           && ((p_common_mgid->unicast.interface_id == INTERFACE_ID_ALL_NODES) > +               || (p_common_mgid->unicast.interface_id == > +                   INTERFACE_ID_ALL_HOSTS) > +               || (p_common_mgid->unicast.interface_id == > +                   INTERFACE_ID_ALL_GATEWAYS) > +               || (p_common_mgid->unicast.interface_id == > +                   INTERFACE_ID_ALL_ROUTERS))) { > +               /* zero pkey bits - special groups will have same mlid for all PKEYs */ Is it a good idea to use the same MLID regardless of PKey ? > +               p_common_mgid->unicast.prefix &= PREFIX_PKEY_MASK_OFF; > +       } else if ((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) == > +                  PREFIX_SIGNATURE_IPV6 > +                  && (p_common_mgid->unicast.interface_id & INT_ID_MASK) == > +                  INT_ID_SIGNATURE) { > +               /* Special Case IPv6 Solicited Node Multicast (SNM) addresses */ > +               /* 0xff1Z601bXXXX0000 : 0x00000001ffYYYYYY */ > +               /* Where Z is the scope, XXXX is the P_Key, and > +                * YYYYYY is the last 24 bits of the port guid */ > +               p_common_mgid->unicast.prefix &= PREFIX_MASK_IP; > +               p_common_mgid->unicast.interface_id &= INT_ID_MASK; > +       } else { > +               /* zero mask bits */ > +               p_common_mgid->unicast.interface_id &= (~MGID_MUX_MLID_MASK); > +       } > +} > + > +static int64_t > +__mgid_cmp(IN const void *const p_gid1, IN const void *const p_gid2) > +{ > +       return memcmp(p_gid1, p_gid2, sizeof(ib_gid_t)); > +} > + > +static osm_mgrp_port_t *osm_mgrp_port_new(ib_net64_t port_guid) > +{ > +       osm_mgrp_port_t *p_mgrp_port = > +           (osm_mgrp_port_t *) malloc(sizeof(osm_mgrp_port_t)); > +       if (!p_mgrp_port) { > +               return NULL; > +       } > +       memset(p_mgrp_port, 0, sizeof(*p_mgrp_port)); Minor - could use calloc rather than malloc/memset here. -- Hal > +       p_mgrp_port->port_guid = port_guid; > +       cl_qlist_init(&p_mgrp_port->mgroups); > +       return p_mgrp_port; > +} > -- > 1.5.5 > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From slavas at voltaire.com Thu Jul 16 06:52:54 2009 From: slavas at voltaire.com (Slava Strebkov) Date: Thu, 16 Jul 2009 16:52:54 +0300 Subject: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM> Message-ID: <39C75744D164D948A170E9792AF8E7CA01F6F919@exil.voltaire.com> This thread covers implementation of compression multiple MGIDs to one MLID. The following groups of IP addresses will have same MLID (for all pkey): 1. FF12401bxxxx000000000000FFFFFFFF - All Nodes 2. FF12401bxxxx00000000000000000001 - All hosts 3. FF12401bffff0000000000000000004d - all Gateways 4. FF12401bxxxx00000000000000000002 - all routers 5. FF12601bxxxx000000000001ffxxxxxx - IPv6 SNM Slava -----Original Message----- From: Hal Rosenstock [mailto:hal.rosenstock at gmail.com] Sent: Thursday, July 16, 2009 4:39 PM To: Slava Strebkov Cc: Sasha Khapyorsky; general at lists.openfabrics.org Subject: Re: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions On Sun, Jun 28, 2009 at 6:36 AM, Slava Strebkov wrote: > [PATCH 1/4] Patch implements multicast multiplexing as proposed in the >  thread entitled "IPv6 and IPoIB scalability issue". Would you be more explicit about what is covered (and what remains to be covered) in that lengthy thread ? >  This first patch contains definitions for new data types >  and infrastructure functions. >  Signed-off-by: Slava Strebkov > > --- >  opensm/include/opensm/osm_multicast.h |   94 +++++++++++++++++++++++++++++++-- >  opensm/opensm/osm_multicast.c         |   82 ++++++++++++++++++++++++++++- >  2 files changed, 170 insertions(+), 6 deletions(-) > > diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h > index a871306..02b63bd 100644 > --- a/opensm/include/opensm/osm_multicast.h > +++ b/opensm/include/opensm/osm_multicast.h > @@ -1,5 +1,5 @@ >  /* > - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. > + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. >  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. >  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. >  * > @@ -45,6 +45,7 @@ > >  #include >  #include > +#include >  #include >  #include >  #include > @@ -93,7 +94,7 @@ BEGIN_C_DECLS >  * >  * SYNOPSIS >  */ > -typedef struct osm_mcast_mgr_ctxt { > +    typedef struct osm_mcast_mgr_ctxt { Is this (and other similar) formatting change(s) needed ? While not a problem here, it would be easier to read/review the patches if the formatting changes are in separate patches. >        cl_list_item_t list_item; >        ib_net16_t mlid; >  } osm_mcast_mgr_ctxt_t; > @@ -107,6 +108,89 @@ typedef struct osm_mcast_mgr_ctxt { >  * SEE ALSO >  *********/ > > +/****s* OpenSM: Multicast Group Holder/osm_mgrp_holder_t > +* NAME > +*       osm_mgrp_holder_t > +* > +* DESCRIPTION > +*       Holder for mgroups. > +* > +*       The osm_mgrp_t object should be treated as opaque and should > +*       be manipulated only through the provided functions. > +* > +* SYNOPSIS > +*/ > + > +typedef struct osm_mgrp_holder { > +       cl_fmap_t mgrp_map; > +       cl_qmap_t mgrp_port_map; > +       ib_gid_t common_mgid; > +       osm_mtree_node_t *p_root; > +       boolean_t to_be_deleted; > +       uint32_t last_tree_id; > +       uint32_t last_change_id; > +       ib_net16_t mlid; > +} osm_mgrp_holder_t; > + > +/* > +* FIELDS > +*       mgrp_map > +*               Map for mgroups.  Must be first element!! > +* > +*       mgrp_port_map > +*               Map of  all ports joined same mlid > +* > +*       common_mgid > +*               mgid of mgroup, ANDed with bitmask. > +*       mgid of each mgroup in mgrp_map, ANDed with bitmask, > +*       see osm_mgrp_holder_prepare_common_mgid > +* > +*       p_root > +*               Pointer to the root "tree node" in the single spanning tree > +*               for this multicast group holder.  The nodes of the tree represent > +*               switches.  Member ports are not represented in the tree. > +* > +*       to_be_deleted > +*               Since holders  are deleted when there are no mgroups in. > +* > +*       last_change_id > +*               a counter for the number of changes applied to the group in this holder. > +*               This counter shuold be incremented on any modification > +*               to the group: joining or leaving of ports. > +* > +*       last_tree_id > +*               the last change id used for building the current tree. > +* > +*       mlid > +*               mlid of current group holder > +*/ > + /****s* OpenSM: Multicast group Port /osm_mgrp_port _t > +* NAME > +*       osm_mgrp_port _t > +* > +* DESCRIPTION > +*       Holder for pointers to mgroups and port guid. > +* > +* > +* SYNOPSIS > +*/ > +typedef struct _osm_mgrp_port { > +       cl_map_item_t guid_item; > +       cl_qlist_t mgroups; > +       ib_net64_t port_guid; > +} osm_mgrp_port_t; > +/* > +* FIELDS > +*       guid_item > +*               Map for ports. Must be first element > +* > +*       mgroups > +*               Map  for  mgroups opened by this port. > +* > +*       portguid > +*               guid of  port representing current structure > +*/ > + >  /****s* OpenSM: Multicast Group/osm_mgrp_t >  * NAME >  *      osm_mgrp_t > @@ -355,7 +439,7 @@ static inline ib_net16_t osm_mgrp_get_mlid(IN const osm_mgrp_t * const p_mgrp) >  * >  * SYNOPSIS >  */ > -osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t *subn, osm_log_t *log, > +osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t * subn, osm_log_t * log, >                                  IN osm_mgrp_t * const p_mgrp, >                                  IN const ib_gid_t * const p_port_gid, >                                  IN const uint8_t join_state, > @@ -452,8 +536,8 @@ osm_mgrp_delete_port(IN osm_subn_t * const p_subn, >  * SEE ALSO >  *********/ > > -int osm_mgrp_remove_port(osm_subn_t *subn, osm_log_t *log, osm_mgrp_t *mgrp, > -                        osm_mcm_port_t *mcm, uint8_t join_state); > +int osm_mgrp_remove_port(osm_subn_t * subn, osm_log_t * log, osm_mgrp_t * mgrp, > +                        osm_mcm_port_t * mcm, uint8_t join_state); > >  /****f* OpenSM: Multicast Group/osm_mgrp_apply_func >  * NAME > diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c > index d2733c4..ae0a818 100644 > --- a/opensm/opensm/osm_multicast.c > +++ b/opensm/opensm/osm_multicast.c > @@ -1,5 +1,5 @@ >  /* > - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. > + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. >  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. >  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. >  * > @@ -48,6 +48,7 @@ >  #include >  #include >  #include > +#include > >  /********************************************************************** >  **********************************************************************/ > @@ -298,3 +299,82 @@ void osm_mgrp_apply_func(const osm_mgrp_t * p_mgrp, osm_mgrp_func_t p_func, >        if (p_mtn) >                mgrp_apply_func_sub(p_mgrp, p_mtn, p_func, context); >  } > + > +/********************************************************************** > + **********************************************************************/ > +#define PREFIX_MASK_IP CL_HTON64(0xff10ffff0000ffffULL) > +#define PREFIX_SIGNATURE_IPV4 CL_HTON64(0xff10401b00000000ULL) > +#define INTERFACE_ID_IPV4            CL_HTON64(0x0000000fffffffffULL) > +#define PREFIX_SIGNATURE_IPV6                    CL_HTON64(0xff10601b00000000ULL) > +#define INTERFACE_ID_ALL_NODES           CL_HTON64(0x00000000ffffffffULL) > +#define INTERFACE_ID_ALL_HOSTS           CL_HTON64(0x0000000000000001ULL) > +#define INTERFACE_ID_ALL_GATEWAYS                CL_HTON64(0x000000000000004dULL) > +#define INTERFACE_ID_ALL_ROUTERS                 CL_HTON64(0x0000000000000002ULL) > +#define PREFIX_PKEY_MASK_OFF          CL_HTON64(0xffffffff0000ffffULL) > +#define PREFIX_MASK CL_HTON64(0xff10ffff0000ffffULL) > +#define PREFIX_SIGNATURE CL_HTON64(0xff10601b00000000ULL) > +#define INT_ID_MASK CL_HTON64(0xfffffff1ff000000ULL) > +#define INT_ID_SIGNATURE CL_HTON64(0x00000001ff000000ULL) > + > +/********************************************************************** > + **********************************************************************/ > + /* mask used for multiplexing mgids to one mlid. Here default value (0xff) means that 8 lsb bits > +    of mgid will be masked off . > +  */ > +static uint64_t MGID_MUX_MLID_MASK = CL_HTON64(0x00000000000000ffULL); Is this only changed by source change and recompilation ? > +void osm_mgrp_holder_prepare_common_mgid(IN const ib_gid_t * const p_mgid, > +                                        OUT ib_gid_t * p_common_mgid) > +{ > +       memcpy(p_common_mgid, p_mgid, sizeof(ib_gid_t)); > +       /* Don't mux non IPoIB mgids. When mux mask is zero, no multiplexing occures */ > +       if ((((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) != > +             PREFIX_SIGNATURE_IPV4) > +            && ((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) != > +                PREFIX_SIGNATURE_IPV6)) || (!MGID_MUX_MLID_MASK)) > +               return; > + > +       if (((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) == > +            PREFIX_SIGNATURE_IPV4) > +           && ((p_common_mgid->unicast.interface_id == INTERFACE_ID_ALL_NODES) > +               || (p_common_mgid->unicast.interface_id == > +                   INTERFACE_ID_ALL_HOSTS) > +               || (p_common_mgid->unicast.interface_id == > +                   INTERFACE_ID_ALL_GATEWAYS) > +               || (p_common_mgid->unicast.interface_id == > +                   INTERFACE_ID_ALL_ROUTERS))) { > +               /* zero pkey bits - special groups will have same mlid for all PKEYs */ Is it a good idea to use the same MLID regardless of PKey ? > +               p_common_mgid->unicast.prefix &= PREFIX_PKEY_MASK_OFF; > +       } else if ((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) == > +                  PREFIX_SIGNATURE_IPV6 > +                  && (p_common_mgid->unicast.interface_id & INT_ID_MASK) == > +                  INT_ID_SIGNATURE) { > +               /* Special Case IPv6 Solicited Node Multicast (SNM) addresses */ > +               /* 0xff1Z601bXXXX0000 : 0x00000001ffYYYYYY */ > +               /* Where Z is the scope, XXXX is the P_Key, and > +                * YYYYYY is the last 24 bits of the port guid */ > +               p_common_mgid->unicast.prefix &= PREFIX_MASK_IP; > +               p_common_mgid->unicast.interface_id &= INT_ID_MASK; > +       } else { > +               /* zero mask bits */ > +               p_common_mgid->unicast.interface_id &= (~MGID_MUX_MLID_MASK); > +       } > +} > + > +static int64_t > +__mgid_cmp(IN const void *const p_gid1, IN const void *const p_gid2) > +{ > +       return memcmp(p_gid1, p_gid2, sizeof(ib_gid_t)); > +} > + > +static osm_mgrp_port_t *osm_mgrp_port_new(ib_net64_t port_guid) > +{ > +       osm_mgrp_port_t *p_mgrp_port = > +           (osm_mgrp_port_t *) malloc(sizeof(osm_mgrp_port_t)); > +       if (!p_mgrp_port) { > +               return NULL; > +       } > +       memset(p_mgrp_port, 0, sizeof(*p_mgrp_port)); Minor - could use calloc rather than malloc/memset here. -- Hal > +       p_mgrp_port->port_guid = port_guid; > +       cl_qlist_init(&p_mgrp_port->mgroups); > +       return p_mgrp_port; > +} > -- > 1.5.5 > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From hal.rosenstock at gmail.com Thu Jul 16 07:15:27 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Thu, 16 Jul 2009 10:15:27 -0400 Subject: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: <39C75744D164D948A170E9792AF8E7CA01F6F919@exil.voltaire.com> References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM> <39C75744D164D948A170E9792AF8E7CA01F6F919@exil.voltaire.com> Message-ID: On Thu, Jul 16, 2009 at 9:52 AM, Slava Strebkov wrote: > > This thread covers implementation of compression multiple MGIDs to one MLID. > The following groups of IP addresses will have same MLID (for all pkey): > 1. FF12401bxxxx000000000000FFFFFFFF - All Nodes > 2. FF12401bxxxx00000000000000000001 - All hosts > 3. FF12401bffff0000000000000000004d - all Gateways > 4. FF12401bxxxx00000000000000000002 - all routers > 5. FF12601bxxxx000000000001ffxxxxxx - IPv6 SNM OK; thanks. So to recap, it compresses the list above to a single common MGID/MLID (regardless of PKey) and also compresses IP groups other than the above and IPv6 SNM based on MGID_MUX_MLID_MASK. There were also some comments/questions embedded in the patch which you may have missed. -- Hal > Slava > > -----Original Message----- > From: Hal Rosenstock [mailto:hal.rosenstock at gmail.com] > Sent: Thursday, July 16, 2009 4:39 PM > To: Slava Strebkov > Cc: Sasha Khapyorsky; general at lists.openfabrics.org > Subject: Re: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions > > On Sun, Jun 28, 2009 at 6:36 AM, Slava Strebkov wrote: >> [PATCH 1/4] Patch implements multicast multiplexing as proposed in the >>  thread entitled "IPv6 and IPoIB scalability issue". > > Would you be more explicit about what is covered (and what remains to > be covered) in that lengthy thread ? > >>  This first patch contains definitions for new data types >>  and infrastructure functions. >>  Signed-off-by: Slava Strebkov >> >> --- >>  opensm/include/opensm/osm_multicast.h |   94 +++++++++++++++++++++++++++++++-- >>  opensm/opensm/osm_multicast.c         |   82 ++++++++++++++++++++++++++++- >>  2 files changed, 170 insertions(+), 6 deletions(-) >> >> diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h >> index a871306..02b63bd 100644 >> --- a/opensm/include/opensm/osm_multicast.h >> +++ b/opensm/include/opensm/osm_multicast.h >> @@ -1,5 +1,5 @@ >>  /* >> - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. >> + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. >>  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. >>  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. >>  * >> @@ -45,6 +45,7 @@ >> >>  #include >>  #include >> +#include >>  #include >>  #include >>  #include >> @@ -93,7 +94,7 @@ BEGIN_C_DECLS >>  * >>  * SYNOPSIS >>  */ >> -typedef struct osm_mcast_mgr_ctxt { >> +    typedef struct osm_mcast_mgr_ctxt { > > Is this (and other similar) formatting change(s) needed ? > > While not a problem here, it would be easier to read/review the > patches if the formatting changes are in separate patches. > >>        cl_list_item_t list_item; >>        ib_net16_t mlid; >>  } osm_mcast_mgr_ctxt_t; >> @@ -107,6 +108,89 @@ typedef struct osm_mcast_mgr_ctxt { >>  * SEE ALSO >>  *********/ >> >> +/****s* OpenSM: Multicast Group Holder/osm_mgrp_holder_t >> +* NAME >> +*       osm_mgrp_holder_t >> +* >> +* DESCRIPTION >> +*       Holder for mgroups. >> +* >> +*       The osm_mgrp_t object should be treated as opaque and should >> +*       be manipulated only through the provided functions. >> +* >> +* SYNOPSIS >> +*/ >> + >> +typedef struct osm_mgrp_holder { >> +       cl_fmap_t mgrp_map; >> +       cl_qmap_t mgrp_port_map; >> +       ib_gid_t common_mgid; >> +       osm_mtree_node_t *p_root; >> +       boolean_t to_be_deleted; >> +       uint32_t last_tree_id; >> +       uint32_t last_change_id; >> +       ib_net16_t mlid; >> +} osm_mgrp_holder_t; >> + >> +/* >> +* FIELDS >> +*       mgrp_map >> +*               Map for mgroups.  Must be first element!! >> +* >> +*       mgrp_port_map >> +*               Map of  all ports joined same mlid >> +* >> +*       common_mgid >> +*               mgid of mgroup, ANDed with bitmask. >> +*       mgid of each mgroup in mgrp_map, ANDed with bitmask, >> +*       see osm_mgrp_holder_prepare_common_mgid >> +* >> +*       p_root >> +*               Pointer to the root "tree node" in the single spanning tree >> +*               for this multicast group holder.  The nodes of the tree represent >> +*               switches.  Member ports are not represented in the tree. >> +* >> +*       to_be_deleted >> +*               Since holders  are deleted when there are no mgroups in. >> +* >> +*       last_change_id >> +*               a counter for the number of changes applied to the group in this holder. >> +*               This counter shuold be incremented on any modification >> +*               to the group: joining or leaving of ports. >> +* >> +*       last_tree_id >> +*               the last change id used for building the current tree. >> +* >> +*       mlid >> +*               mlid of current group holder >> +*/ >> + /****s* OpenSM: Multicast group Port /osm_mgrp_port _t >> +* NAME >> +*       osm_mgrp_port _t >> +* >> +* DESCRIPTION >> +*       Holder for pointers to mgroups and port guid. >> +* >> +* >> +* SYNOPSIS >> +*/ >> +typedef struct _osm_mgrp_port { >> +       cl_map_item_t guid_item; >> +       cl_qlist_t mgroups; >> +       ib_net64_t port_guid; >> +} osm_mgrp_port_t; >> +/* >> +* FIELDS >> +*       guid_item >> +*               Map for ports. Must be first element >> +* >> +*       mgroups >> +*               Map  for  mgroups opened by this port. >> +* >> +*       portguid >> +*               guid of  port representing current structure >> +*/ >> + >>  /****s* OpenSM: Multicast Group/osm_mgrp_t >>  * NAME >>  *      osm_mgrp_t >> @@ -355,7 +439,7 @@ static inline ib_net16_t osm_mgrp_get_mlid(IN const osm_mgrp_t * const p_mgrp) >>  * >>  * SYNOPSIS >>  */ >> -osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t *subn, osm_log_t *log, >> +osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t * subn, osm_log_t * log, >>                                  IN osm_mgrp_t * const p_mgrp, >>                                  IN const ib_gid_t * const p_port_gid, >>                                  IN const uint8_t join_state, >> @@ -452,8 +536,8 @@ osm_mgrp_delete_port(IN osm_subn_t * const p_subn, >>  * SEE ALSO >>  *********/ >> >> -int osm_mgrp_remove_port(osm_subn_t *subn, osm_log_t *log, osm_mgrp_t *mgrp, >> -                        osm_mcm_port_t *mcm, uint8_t join_state); >> +int osm_mgrp_remove_port(osm_subn_t * subn, osm_log_t * log, osm_mgrp_t * mgrp, >> +                        osm_mcm_port_t * mcm, uint8_t join_state); >> >>  /****f* OpenSM: Multicast Group/osm_mgrp_apply_func >>  * NAME >> diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c >> index d2733c4..ae0a818 100644 >> --- a/opensm/opensm/osm_multicast.c >> +++ b/opensm/opensm/osm_multicast.c >> @@ -1,5 +1,5 @@ >>  /* >> - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. >> + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. >>  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. >>  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. >>  * >> @@ -48,6 +48,7 @@ >>  #include >>  #include >>  #include >> +#include >> >>  /********************************************************************** >>  **********************************************************************/ >> @@ -298,3 +299,82 @@ void osm_mgrp_apply_func(const osm_mgrp_t * p_mgrp, osm_mgrp_func_t p_func, >>        if (p_mtn) >>                mgrp_apply_func_sub(p_mgrp, p_mtn, p_func, context); >>  } >> + >> +/********************************************************************** >> + **********************************************************************/ >> +#define PREFIX_MASK_IP CL_HTON64(0xff10ffff0000ffffULL) >> +#define PREFIX_SIGNATURE_IPV4 CL_HTON64(0xff10401b00000000ULL) >> +#define INTERFACE_ID_IPV4            CL_HTON64(0x0000000fffffffffULL) >> +#define PREFIX_SIGNATURE_IPV6                    CL_HTON64(0xff10601b00000000ULL) >> +#define INTERFACE_ID_ALL_NODES           CL_HTON64(0x00000000ffffffffULL) >> +#define INTERFACE_ID_ALL_HOSTS           CL_HTON64(0x0000000000000001ULL) >> +#define INTERFACE_ID_ALL_GATEWAYS                CL_HTON64(0x000000000000004dULL) >> +#define INTERFACE_ID_ALL_ROUTERS                 CL_HTON64(0x0000000000000002ULL) >> +#define PREFIX_PKEY_MASK_OFF          CL_HTON64(0xffffffff0000ffffULL) >> +#define PREFIX_MASK CL_HTON64(0xff10ffff0000ffffULL) >> +#define PREFIX_SIGNATURE CL_HTON64(0xff10601b00000000ULL) >> +#define INT_ID_MASK CL_HTON64(0xfffffff1ff000000ULL) >> +#define INT_ID_SIGNATURE CL_HTON64(0x00000001ff000000ULL) >> + >> +/********************************************************************** >> + **********************************************************************/ >> + /* mask used for multiplexing mgids to one mlid. Here default value (0xff) means that 8 lsb bits >> +    of mgid will be masked off . >> +  */ >> +static uint64_t MGID_MUX_MLID_MASK = CL_HTON64(0x00000000000000ffULL); > > Is this only changed by source change and recompilation ? > >> +void osm_mgrp_holder_prepare_common_mgid(IN const ib_gid_t * const p_mgid, >> +                                        OUT ib_gid_t * p_common_mgid) >> +{ >> +       memcpy(p_common_mgid, p_mgid, sizeof(ib_gid_t)); >> +       /* Don't mux non IPoIB mgids. When mux mask is zero, no multiplexing occures */ >> +       if ((((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) != >> +             PREFIX_SIGNATURE_IPV4) >> +            && ((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) != >> +                PREFIX_SIGNATURE_IPV6)) || (!MGID_MUX_MLID_MASK)) >> +               return; >> + >> +       if (((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) == >> +            PREFIX_SIGNATURE_IPV4) >> +           && ((p_common_mgid->unicast.interface_id == INTERFACE_ID_ALL_NODES) >> +               || (p_common_mgid->unicast.interface_id == >> +                   INTERFACE_ID_ALL_HOSTS) >> +               || (p_common_mgid->unicast.interface_id == >> +                   INTERFACE_ID_ALL_GATEWAYS) >> +               || (p_common_mgid->unicast.interface_id == >> +                   INTERFACE_ID_ALL_ROUTERS))) { >> +               /* zero pkey bits - special groups will have same mlid for all PKEYs */ > > Is it a good idea to use the same MLID regardless of PKey ? > >> +               p_common_mgid->unicast.prefix &= PREFIX_PKEY_MASK_OFF; >> +       } else if ((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) == >> +                  PREFIX_SIGNATURE_IPV6 >> +                  && (p_common_mgid->unicast.interface_id & INT_ID_MASK) == >> +                  INT_ID_SIGNATURE) { >> +               /* Special Case IPv6 Solicited Node Multicast (SNM) addresses */ >> +               /* 0xff1Z601bXXXX0000 : 0x00000001ffYYYYYY */ >> +               /* Where Z is the scope, XXXX is the P_Key, and >> +                * YYYYYY is the last 24 bits of the port guid */ >> +               p_common_mgid->unicast.prefix &= PREFIX_MASK_IP; >> +               p_common_mgid->unicast.interface_id &= INT_ID_MASK; >> +       } else { >> +               /* zero mask bits */ >> +               p_common_mgid->unicast.interface_id &= (~MGID_MUX_MLID_MASK); >> +       } >> +} >> + >> +static int64_t >> +__mgid_cmp(IN const void *const p_gid1, IN const void *const p_gid2) >> +{ >> +       return memcmp(p_gid1, p_gid2, sizeof(ib_gid_t)); >> +} >> + >> +static osm_mgrp_port_t *osm_mgrp_port_new(ib_net64_t port_guid) >> +{ >> +       osm_mgrp_port_t *p_mgrp_port = >> +           (osm_mgrp_port_t *) malloc(sizeof(osm_mgrp_port_t)); >> +       if (!p_mgrp_port) { >> +               return NULL; >> +       } >> +       memset(p_mgrp_port, 0, sizeof(*p_mgrp_port)); > > Minor - could use calloc rather than malloc/memset here. > > -- Hal > >> +       p_mgrp_port->port_guid = port_guid; >> +       cl_qlist_init(&p_mgrp_port->mgroups); >> +       return p_mgrp_port; >> +} >> -- >> 1.5.5 >> >> _______________________________________________ >> general mailing list >> general at lists.openfabrics.org >> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >> >> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general >> > From slavas at voltaire.com Thu Jul 16 07:39:42 2009 From: slavas at voltaire.com (Slava Strebkov) Date: Thu, 16 Jul 2009 17:39:42 +0300 Subject: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM><39C75744D164D948A170E9792AF8E7CA01F6F919@exil.voltaire.com> Message-ID: <39C75744D164D948A170E9792AF8E7CA01F6F91A@exil.voltaire.com> Please see my answers below your comments. Slava -----Original Message----- From: Hal Rosenstock [mailto:hal.rosenstock at gmail.com] Sent: Thursday, July 16, 2009 5:15 PM To: Slava Strebkov Cc: Sasha Khapyorsky; general at lists.openfabrics.org Subject: Re: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions On Thu, Jul 16, 2009 at 9:52 AM, Slava Strebkov wrote: > > This thread covers implementation of compression multiple MGIDs to one MLID. > The following groups of IP addresses will have same MLID (for all pkey): > 1. FF12401bxxxx000000000000FFFFFFFF - All Nodes > 2. FF12401bxxxx00000000000000000001 - All hosts > 3. FF12401bffff0000000000000000004d - all Gateways > 4. FF12401bxxxx00000000000000000002 - all routers > 5. FF12601bxxxx000000000001ffxxxxxx - IPv6 SNM OK; thanks. So to recap, it compresses the list above to a single common MGID/MLID (regardless of PKey) and also compresses IP groups other than the above and IPv6 SNM based on MGID_MUX_MLID_MASK. There were also some comments/questions embedded in the patch which you may have missed. -- Hal > Slava > > -----Original Message----- > From: Hal Rosenstock [mailto:hal.rosenstock at gmail.com] > Sent: Thursday, July 16, 2009 4:39 PM > To: Slava Strebkov > Cc: Sasha Khapyorsky; general at lists.openfabrics.org > Subject: Re: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions > > On Sun, Jun 28, 2009 at 6:36 AM, Slava Strebkov wrote: >> [PATCH 1/4] Patch implements multicast multiplexing as proposed in the >>  thread entitled "IPv6 and IPoIB scalability issue". > > Would you be more explicit about what is covered (and what remains to > be covered) in that lengthy thread ? > >>  This first patch contains definitions for new data types >>  and infrastructure functions. >>  Signed-off-by: Slava Strebkov >> >> --- >>  opensm/include/opensm/osm_multicast.h |   94 +++++++++++++++++++++++++++++++-- >>  opensm/opensm/osm_multicast.c         |   82 ++++++++++++++++++++++++++++- >>  2 files changed, 170 insertions(+), 6 deletions(-) >> >> diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h >> index a871306..02b63bd 100644 >> --- a/opensm/include/opensm/osm_multicast.h >> +++ b/opensm/include/opensm/osm_multicast.h >> @@ -1,5 +1,5 @@ >>  /* >> - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. >> + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. >>  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. >>  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. >>  * >> @@ -45,6 +45,7 @@ >> >>  #include >>  #include >> +#include >>  #include >>  #include >>  #include >> @@ -93,7 +94,7 @@ BEGIN_C_DECLS >>  * >>  * SYNOPSIS >>  */ >> -typedef struct osm_mcast_mgr_ctxt { >> +    typedef struct osm_mcast_mgr_ctxt { > > Is this (and other similar) formatting change(s) needed ? > - osm_indent performed this work > While not a problem here, it would be easier to read/review the > patches if the formatting changes are in separate patches. > >>        cl_list_item_t list_item; >>        ib_net16_t mlid; >>  } osm_mcast_mgr_ctxt_t; >> @@ -107,6 +108,89 @@ typedef struct osm_mcast_mgr_ctxt { >>  * SEE ALSO >>  *********/ >> >> +/****s* OpenSM: Multicast Group Holder/osm_mgrp_holder_t >> +* NAME >> +*       osm_mgrp_holder_t >> +* >> +* DESCRIPTION >> +*       Holder for mgroups. >> +* >> +*       The osm_mgrp_t object should be treated as opaque and should >> +*       be manipulated only through the provided functions. >> +* >> +* SYNOPSIS >> +*/ >> + >> +typedef struct osm_mgrp_holder { >> +       cl_fmap_t mgrp_map; >> +       cl_qmap_t mgrp_port_map; >> +       ib_gid_t common_mgid; >> +       osm_mtree_node_t *p_root; >> +       boolean_t to_be_deleted; >> +       uint32_t last_tree_id; >> +       uint32_t last_change_id; >> +       ib_net16_t mlid; >> +} osm_mgrp_holder_t; >> + >> +/* >> +* FIELDS >> +*       mgrp_map >> +*               Map for mgroups.  Must be first element!! >> +* >> +*       mgrp_port_map >> +*               Map of  all ports joined same mlid >> +* >> +*       common_mgid >> +*               mgid of mgroup, ANDed with bitmask. >> +*       mgid of each mgroup in mgrp_map, ANDed with bitmask, >> +*       see osm_mgrp_holder_prepare_common_mgid >> +* >> +*       p_root >> +*               Pointer to the root "tree node" in the single spanning tree >> +*               for this multicast group holder.  The nodes of the tree represent >> +*               switches.  Member ports are not represented in the tree. >> +* >> +*       to_be_deleted >> +*               Since holders  are deleted when there are no mgroups in. >> +* >> +*       last_change_id >> +*               a counter for the number of changes applied to the group in this holder. >> +*               This counter shuold be incremented on any modification >> +*               to the group: joining or leaving of ports. >> +* >> +*       last_tree_id >> +*               the last change id used for building the current tree. >> +* >> +*       mlid >> +*               mlid of current group holder >> +*/ >> + /****s* OpenSM: Multicast group Port /osm_mgrp_port _t >> +* NAME >> +*       osm_mgrp_port _t >> +* >> +* DESCRIPTION >> +*       Holder for pointers to mgroups and port guid. >> +* >> +* >> +* SYNOPSIS >> +*/ >> +typedef struct _osm_mgrp_port { >> +       cl_map_item_t guid_item; >> +       cl_qlist_t mgroups; >> +       ib_net64_t port_guid; >> +} osm_mgrp_port_t; >> +/* >> +* FIELDS >> +*       guid_item >> +*               Map for ports. Must be first element >> +* >> +*       mgroups >> +*               Map  for  mgroups opened by this port. >> +* >> +*       portguid >> +*               guid of  port representing current structure >> +*/ >> + >>  /****s* OpenSM: Multicast Group/osm_mgrp_t >>  * NAME >>  *      osm_mgrp_t >> @@ -355,7 +439,7 @@ static inline ib_net16_t osm_mgrp_get_mlid(IN const osm_mgrp_t * const p_mgrp) >>  * >>  * SYNOPSIS >>  */ >> -osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t *subn, osm_log_t *log, >> +osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t * subn, osm_log_t * log, >>                                  IN osm_mgrp_t * const p_mgrp, >>                                  IN const ib_gid_t * const p_port_gid, >>                                  IN const uint8_t join_state, >> @@ -452,8 +536,8 @@ osm_mgrp_delete_port(IN osm_subn_t * const p_subn, >>  * SEE ALSO >>  *********/ >> >> -int osm_mgrp_remove_port(osm_subn_t *subn, osm_log_t *log, osm_mgrp_t *mgrp, >> -                        osm_mcm_port_t *mcm, uint8_t join_state); >> +int osm_mgrp_remove_port(osm_subn_t * subn, osm_log_t * log, osm_mgrp_t * mgrp, >> +                        osm_mcm_port_t * mcm, uint8_t join_state); >> >>  /****f* OpenSM: Multicast Group/osm_mgrp_apply_func >>  * NAME >> diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c >> index d2733c4..ae0a818 100644 >> --- a/opensm/opensm/osm_multicast.c >> +++ b/opensm/opensm/osm_multicast.c >> @@ -1,5 +1,5 @@ >>  /* >> - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. >> + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. >>  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. >>  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. >>  * >> @@ -48,6 +48,7 @@ >>  #include >>  #include >>  #include >> +#include >> >>  /********************************************************************** >>  **********************************************************************/ >> @@ -298,3 +299,82 @@ void osm_mgrp_apply_func(const osm_mgrp_t * p_mgrp, osm_mgrp_func_t p_func, >>        if (p_mtn) >>                mgrp_apply_func_sub(p_mgrp, p_mtn, p_func, context); >>  } >> + >> +/********************************************************************** >> + **********************************************************************/ >> +#define PREFIX_MASK_IP CL_HTON64(0xff10ffff0000ffffULL) >> +#define PREFIX_SIGNATURE_IPV4 CL_HTON64(0xff10401b00000000ULL) >> +#define INTERFACE_ID_IPV4            CL_HTON64(0x0000000fffffffffULL) >> +#define PREFIX_SIGNATURE_IPV6                    CL_HTON64(0xff10601b00000000ULL) >> +#define INTERFACE_ID_ALL_NODES           CL_HTON64(0x00000000ffffffffULL) >> +#define INTERFACE_ID_ALL_HOSTS           CL_HTON64(0x0000000000000001ULL) >> +#define INTERFACE_ID_ALL_GATEWAYS                CL_HTON64(0x000000000000004dULL) >> +#define INTERFACE_ID_ALL_ROUTERS                 CL_HTON64(0x0000000000000002ULL) >> +#define PREFIX_PKEY_MASK_OFF          CL_HTON64(0xffffffff0000ffffULL) >> +#define PREFIX_MASK CL_HTON64(0xff10ffff0000ffffULL) >> +#define PREFIX_SIGNATURE CL_HTON64(0xff10601b00000000ULL) >> +#define INT_ID_MASK CL_HTON64(0xfffffff1ff000000ULL) >> +#define INT_ID_SIGNATURE CL_HTON64(0x00000001ff000000ULL) >> + >> +/********************************************************************** >> + **********************************************************************/ >> + /* mask used for multiplexing mgids to one mlid. Here default value (0xff) means that 8 lsb bits >> +    of mgid will be masked off . >> +  */ >> +static uint64_t MGID_MUX_MLID_MASK = CL_HTON64(0x00000000000000ffULL); > > Is this only changed by source change and recompilation ? > MGID_MUX_MLID_MASK will be a parameter in opensm.conf file. Here I placed it as an example. >> +void osm_mgrp_holder_prepare_common_mgid(IN const ib_gid_t * const p_mgid, >> +                                        OUT ib_gid_t * p_common_mgid) >> +{ >> +       memcpy(p_common_mgid, p_mgid, sizeof(ib_gid_t)); >> +       /* Don't mux non IPoIB mgids. When mux mask is zero, no multiplexing occures */ >> +       if ((((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) != >> +             PREFIX_SIGNATURE_IPV4) >> +            && ((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) != >> +                PREFIX_SIGNATURE_IPV6)) || (!MGID_MUX_MLID_MASK)) >> +               return; >> + >> +       if (((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) == >> +            PREFIX_SIGNATURE_IPV4) >> +           && ((p_common_mgid->unicast.interface_id == INTERFACE_ID_ALL_NODES) >> +               || (p_common_mgid->unicast.interface_id == >> +                   INTERFACE_ID_ALL_HOSTS) >> +               || (p_common_mgid->unicast.interface_id == >> +                   INTERFACE_ID_ALL_GATEWAYS) >> +               || (p_common_mgid->unicast.interface_id == >> +                   INTERFACE_ID_ALL_ROUTERS))) { >> +               /* zero pkey bits - special groups will have same mlid for all PKEYs */ > > Is it a good idea to use the same MLID regardless of PKey ? > - This is not so much groups, even for all PKeys. Providing different MLID for each special group we'll not achieve any compression. >> +               p_common_mgid->unicast.prefix &= PREFIX_PKEY_MASK_OFF; >> +       } else if ((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) == >> +                  PREFIX_SIGNATURE_IPV6 >> +                  && (p_common_mgid->unicast.interface_id & INT_ID_MASK) == >> +                  INT_ID_SIGNATURE) { >> +               /* Special Case IPv6 Solicited Node Multicast (SNM) addresses */ >> +               /* 0xff1Z601bXXXX0000 : 0x00000001ffYYYYYY */ >> +               /* Where Z is the scope, XXXX is the P_Key, and >> +                * YYYYYY is the last 24 bits of the port guid */ >> +               p_common_mgid->unicast.prefix &= PREFIX_MASK_IP; >> +               p_common_mgid->unicast.interface_id &= INT_ID_MASK; >> +       } else { >> +               /* zero mask bits */ >> +               p_common_mgid->unicast.interface_id &= (~MGID_MUX_MLID_MASK); >> +       } >> +} >> + >> +static int64_t >> +__mgid_cmp(IN const void *const p_gid1, IN const void *const p_gid2) >> +{ >> +       return memcmp(p_gid1, p_gid2, sizeof(ib_gid_t)); >> +} >> + >> +static osm_mgrp_port_t *osm_mgrp_port_new(ib_net64_t port_guid) >> +{ >> +       osm_mgrp_port_t *p_mgrp_port = >> +           (osm_mgrp_port_t *) malloc(sizeof(osm_mgrp_port_t)); >> +       if (!p_mgrp_port) { >> +               return NULL; >> +       } >> +       memset(p_mgrp_port, 0, sizeof(*p_mgrp_port)); > > Minor - could use calloc rather than malloc/memset here. > > -- Hal > >> +       p_mgrp_port->port_guid = port_guid; >> +       cl_qlist_init(&p_mgrp_port->mgroups); >> +       return p_mgrp_port; >> +} >> -- >> 1.5.5 >> >> _______________________________________________ >> general mailing list >> general at lists.openfabrics.org >> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >> >> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general >> > From swise at opengridcomputing.com Thu Jul 16 07:55:38 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Thu, 16 Jul 2009 09:55:38 -0500 Subject: [ofa-general] Re: [PATCH] infiniband: Use %pM conversion specifier In-Reply-To: <1247735559-21945-1-git-send-email-klto@zhaw.ch> References: <1247735559-21945-1-git-send-email-klto@zhaw.ch> Message-ID: <4A5F3F6A.10604@opengridcomputing.com> Acked-by: Steve Wise Tobias Klauser wrote: > Use the %pM conversion specifier to print a MAC address. > > Signed-off-by: Tobias Klauser > --- > drivers/infiniband/hw/amso1100/c2.c | 6 +----- > 1 files changed, 1 insertions(+), 5 deletions(-) > > diff --git a/drivers/infiniband/hw/amso1100/c2.c b/drivers/infiniband/hw/amso1100/c2.c > index 0cfbb6d..8250740 100644 > --- a/drivers/infiniband/hw/amso1100/c2.c > +++ b/drivers/infiniband/hw/amso1100/c2.c > @@ -86,11 +86,7 @@ MODULE_DEVICE_TABLE(pci, c2_pci_table); > > static void c2_print_macaddr(struct net_device *netdev) > { > - pr_debug("%s: MAC %02X:%02X:%02X:%02X:%02X:%02X, " > - "IRQ %u\n", netdev->name, > - netdev->dev_addr[0], netdev->dev_addr[1], netdev->dev_addr[2], > - netdev->dev_addr[3], netdev->dev_addr[4], netdev->dev_addr[5], > - netdev->irq); > + pr_debug("%s: MAC %pM, IRQ %u\n", netdev->name, netdev->dev_addr, netdev->irq); > } > > static void c2_set_rxbufsize(struct c2_port *c2_port) > From hal.rosenstock at gmail.com Thu Jul 16 09:59:33 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Thu, 16 Jul 2009 12:59:33 -0400 Subject: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: <39C75744D164D948A170E9792AF8E7CA01F6F91A@exil.voltaire.com> References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM> <39C75744D164D948A170E9792AF8E7CA01F6F919@exil.voltaire.com> <39C75744D164D948A170E9792AF8E7CA01F6F91A@exil.voltaire.com> Message-ID: On Thu, Jul 16, 2009 at 10:39 AM, Slava Strebkov wrote: > > Please see my answers below your comments. > Slava > > -----Original Message----- > From: Hal Rosenstock [mailto:hal.rosenstock at gmail.com] > Sent: Thursday, July 16, 2009 5:15 PM > To: Slava Strebkov > Cc: Sasha Khapyorsky; general at lists.openfabrics.org > Subject: Re: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions > > On Thu, Jul 16, 2009 at 9:52 AM, Slava Strebkov wrote: >> >> This thread covers implementation of compression multiple MGIDs to one MLID. >> The following groups of IP addresses will have same MLID (for all pkey): >> 1. FF12401bxxxx000000000000FFFFFFFF - All Nodes >> 2. FF12401bxxxx00000000000000000001 - All hosts >> 3. FF12401bffff0000000000000000004d - all Gateways >> 4. FF12401bxxxx00000000000000000002 - all routers >> 5. FF12601bxxxx000000000001ffxxxxxx - IPv6 SNM > > OK; thanks. So to recap, it compresses the list above to a single > common MGID/MLID (regardless of PKey) and also compresses IP groups > other than the above and IPv6 SNM based on MGID_MUX_MLID_MASK. > > There were also some comments/questions embedded in the patch which > you may have missed. > > -- Hal > >> Slava >> >> -----Original Message----- >> From: Hal Rosenstock [mailto:hal.rosenstock at gmail.com] >> Sent: Thursday, July 16, 2009 4:39 PM >> To: Slava Strebkov >> Cc: Sasha Khapyorsky; general at lists.openfabrics.org >> Subject: Re: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions >> >> On Sun, Jun 28, 2009 at 6:36 AM, Slava Strebkov wrote: >>> [PATCH 1/4] Patch implements multicast multiplexing as proposed in the >>>  thread entitled "IPv6 and IPoIB scalability issue". >> >> Would you be more explicit about what is covered (and what remains to >> be covered) in that lengthy thread ? >> >>>  This first patch contains definitions for new data types >>>  and infrastructure functions. >>>  Signed-off-by: Slava Strebkov >>> >>> --- >>>  opensm/include/opensm/osm_multicast.h |   94 +++++++++++++++++++++++++++++++-- >>>  opensm/opensm/osm_multicast.c         |   82 ++++++++++++++++++++++++++++- >>>  2 files changed, 170 insertions(+), 6 deletions(-) >>> >>> diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h >>> index a871306..02b63bd 100644 >>> --- a/opensm/include/opensm/osm_multicast.h >>> +++ b/opensm/include/opensm/osm_multicast.h >>> @@ -1,5 +1,5 @@ >>>  /* >>> - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. >>> + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. >>>  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. >>>  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. >>>  * >>> @@ -45,6 +45,7 @@ >>> >>>  #include >>>  #include >>> +#include >>>  #include >>>  #include >>>  #include >>> @@ -93,7 +94,7 @@ BEGIN_C_DECLS >>>  * >>>  * SYNOPSIS >>>  */ >>> -typedef struct osm_mcast_mgr_ctxt { >>> +    typedef struct osm_mcast_mgr_ctxt { >> >> Is this (and other similar) formatting change(s) needed ? >> - osm_indent performed this work osm_indent has a number of shortcomings :-( This is clearly one. Others have been noticed in other previous postings. IMO this and other similar changes should be removed. The reason it doesn't handle it well is that this osm_indent is basically the linux kernel Lindent and the coding styles are different (e.g. such a typedef is not acceptable in the kernel). >> While not a problem here, it would be easier to read/review the >> patches if the formatting changes are in separate patches. Would you repost the changes with the formatting changes separate ? >>>        cl_list_item_t list_item; >>>        ib_net16_t mlid; >>>  } osm_mcast_mgr_ctxt_t; >>> @@ -107,6 +108,89 @@ typedef struct osm_mcast_mgr_ctxt { >>>  * SEE ALSO >>>  *********/ >>> >>> +/****s* OpenSM: Multicast Group Holder/osm_mgrp_holder_t >>> +* NAME >>> +*       osm_mgrp_holder_t >>> +* >>> +* DESCRIPTION >>> +*       Holder for mgroups. >>> +* >>> +*       The osm_mgrp_t object should be treated as opaque and should >>> +*       be manipulated only through the provided functions. >>> +* >>> +* SYNOPSIS >>> +*/ >>> + >>> +typedef struct osm_mgrp_holder { >>> +       cl_fmap_t mgrp_map; >>> +       cl_qmap_t mgrp_port_map; >>> +       ib_gid_t common_mgid; >>> +       osm_mtree_node_t *p_root; >>> +       boolean_t to_be_deleted; >>> +       uint32_t last_tree_id; >>> +       uint32_t last_change_id; >>> +       ib_net16_t mlid; >>> +} osm_mgrp_holder_t; >>> + >>> +/* >>> +* FIELDS >>> +*       mgrp_map >>> +*               Map for mgroups.  Must be first element!! >>> +* >>> +*       mgrp_port_map >>> +*               Map of  all ports joined same mlid >>> +* >>> +*       common_mgid >>> +*               mgid of mgroup, ANDed with bitmask. >>> +*       mgid of each mgroup in mgrp_map, ANDed with bitmask, >>> +*       see osm_mgrp_holder_prepare_common_mgid >>> +* >>> +*       p_root >>> +*               Pointer to the root "tree node" in the single spanning tree >>> +*               for this multicast group holder.  The nodes of the tree represent >>> +*               switches.  Member ports are not represented in the tree. >>> +* >>> +*       to_be_deleted >>> +*               Since holders  are deleted when there are no mgroups in. >>> +* >>> +*       last_change_id >>> +*               a counter for the number of changes applied to the group in this holder. >>> +*               This counter shuold be incremented on any modification >>> +*               to the group: joining or leaving of ports. >>> +* >>> +*       last_tree_id >>> +*               the last change id used for building the current tree. >>> +* >>> +*       mlid >>> +*               mlid of current group holder >>> +*/ >>> + /****s* OpenSM: Multicast group Port /osm_mgrp_port _t >>> +* NAME >>> +*       osm_mgrp_port _t >>> +* >>> +* DESCRIPTION >>> +*       Holder for pointers to mgroups and port guid. >>> +* >>> +* >>> +* SYNOPSIS >>> +*/ >>> +typedef struct _osm_mgrp_port { >>> +       cl_map_item_t guid_item; >>> +       cl_qlist_t mgroups; >>> +       ib_net64_t port_guid; >>> +} osm_mgrp_port_t; >>> +/* >>> +* FIELDS >>> +*       guid_item >>> +*               Map for ports. Must be first element >>> +* >>> +*       mgroups >>> +*               Map  for  mgroups opened by this port. >>> +* >>> +*       portguid >>> +*               guid of  port representing current structure >>> +*/ >>> + >>>  /****s* OpenSM: Multicast Group/osm_mgrp_t >>>  * NAME >>>  *      osm_mgrp_t >>> @@ -355,7 +439,7 @@ static inline ib_net16_t osm_mgrp_get_mlid(IN const osm_mgrp_t * const p_mgrp) >>>  * >>>  * SYNOPSIS >>>  */ >>> -osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t *subn, osm_log_t *log, >>> +osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t * subn, osm_log_t * log, >>>                                  IN osm_mgrp_t * const p_mgrp, >>>                                  IN const ib_gid_t * const p_port_gid, >>>                                  IN const uint8_t join_state, >>> @@ -452,8 +536,8 @@ osm_mgrp_delete_port(IN osm_subn_t * const p_subn, >>>  * SEE ALSO >>>  *********/ >>> >>> -int osm_mgrp_remove_port(osm_subn_t *subn, osm_log_t *log, osm_mgrp_t *mgrp, >>> -                        osm_mcm_port_t *mcm, uint8_t join_state); >>> +int osm_mgrp_remove_port(osm_subn_t * subn, osm_log_t * log, osm_mgrp_t * mgrp, >>> +                        osm_mcm_port_t * mcm, uint8_t join_state); >>> >>>  /****f* OpenSM: Multicast Group/osm_mgrp_apply_func >>>  * NAME >>> diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c >>> index d2733c4..ae0a818 100644 >>> --- a/opensm/opensm/osm_multicast.c >>> +++ b/opensm/opensm/osm_multicast.c >>> @@ -1,5 +1,5 @@ >>>  /* >>> - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. >>> + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. >>>  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. >>>  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. >>>  * >>> @@ -48,6 +48,7 @@ >>>  #include >>>  #include >>>  #include >>> +#include >>> >>>  /********************************************************************** >>>  **********************************************************************/ >>> @@ -298,3 +299,82 @@ void osm_mgrp_apply_func(const osm_mgrp_t * p_mgrp, osm_mgrp_func_t p_func, >>>        if (p_mtn) >>>                mgrp_apply_func_sub(p_mgrp, p_mtn, p_func, context); >>>  } >>> + >>> +/********************************************************************** >>> + **********************************************************************/ >>> +#define PREFIX_MASK_IP CL_HTON64(0xff10ffff0000ffffULL) >>> +#define PREFIX_SIGNATURE_IPV4 CL_HTON64(0xff10401b00000000ULL) >>> +#define INTERFACE_ID_IPV4            CL_HTON64(0x0000000fffffffffULL) >>> +#define PREFIX_SIGNATURE_IPV6                    CL_HTON64(0xff10601b00000000ULL) >>> +#define INTERFACE_ID_ALL_NODES           CL_HTON64(0x00000000ffffffffULL) >>> +#define INTERFACE_ID_ALL_HOSTS           CL_HTON64(0x0000000000000001ULL) >>> +#define INTERFACE_ID_ALL_GATEWAYS                CL_HTON64(0x000000000000004dULL) >>> +#define INTERFACE_ID_ALL_ROUTERS                 CL_HTON64(0x0000000000000002ULL) >>> +#define PREFIX_PKEY_MASK_OFF          CL_HTON64(0xffffffff0000ffffULL) >>> +#define PREFIX_MASK CL_HTON64(0xff10ffff0000ffffULL) >>> +#define PREFIX_SIGNATURE CL_HTON64(0xff10601b00000000ULL) >>> +#define INT_ID_MASK CL_HTON64(0xfffffff1ff000000ULL) >>> +#define INT_ID_SIGNATURE CL_HTON64(0x00000001ff000000ULL) >>> + >>> +/********************************************************************** >>> + **********************************************************************/ >>> + /* mask used for multiplexing mgids to one mlid. Here default value (0xff) means that 8 lsb bits >>> +    of mgid will be masked off . >>> +  */ >>> +static uint64_t MGID_MUX_MLID_MASK = CL_HTON64(0x00000000000000ffULL); >> >> Is this only changed by source change and recompilation ? >>       MGID_MUX_MLID_MASK will be a parameter in opensm.conf file. >        Here I placed it as an example. Is this in a subsequent patch in this series ? What are the new options for this ? What happens with the old option ? >>> +void osm_mgrp_holder_prepare_common_mgid(IN const ib_gid_t * const p_mgid, >>> +                                        OUT ib_gid_t * p_common_mgid) >>> +{ >>> +       memcpy(p_common_mgid, p_mgid, sizeof(ib_gid_t)); >>> +       /* Don't mux non IPoIB mgids. When mux mask is zero, no multiplexing occures */ >>> +       if ((((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) != >>> +             PREFIX_SIGNATURE_IPV4) >>> +            && ((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) != >>> +                PREFIX_SIGNATURE_IPV6)) || (!MGID_MUX_MLID_MASK)) >>> +               return; >>> + >>> +       if (((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) == >>> +            PREFIX_SIGNATURE_IPV4) >>> +           && ((p_common_mgid->unicast.interface_id == INTERFACE_ID_ALL_NODES) >>> +               || (p_common_mgid->unicast.interface_id == >>> +                   INTERFACE_ID_ALL_HOSTS) >>> +               || (p_common_mgid->unicast.interface_id == >>> +                   INTERFACE_ID_ALL_GATEWAYS) >>> +               || (p_common_mgid->unicast.interface_id == >>> +                   INTERFACE_ID_ALL_ROUTERS))) { >>> +               /* zero pkey bits - special groups will have same mlid for all PKEYs */ >> >> Is it a good idea to use the same MLID regardless of PKey ? >> - This is not so much groups, even for all PKeys. Providing different MLID >    for each special group we'll not achieve any compression. Any compression or as much compression won't be achieved ? It's problematic to put multiple PKeys all on the same MLID. -- Hal >>> +               p_common_mgid->unicast.prefix &= PREFIX_PKEY_MASK_OFF; >>> +       } else if ((p_common_mgid->unicast.prefix & PREFIX_MASK_IP) == >>> +                  PREFIX_SIGNATURE_IPV6 >>> +                  && (p_common_mgid->unicast.interface_id & INT_ID_MASK) == >>> +                  INT_ID_SIGNATURE) { >>> +               /* Special Case IPv6 Solicited Node Multicast (SNM) addresses */ >>> +               /* 0xff1Z601bXXXX0000 : 0x00000001ffYYYYYY */ >>> +               /* Where Z is the scope, XXXX is the P_Key, and >>> +                * YYYYYY is the last 24 bits of the port guid */ >>> +               p_common_mgid->unicast.prefix &= PREFIX_MASK_IP; >>> +               p_common_mgid->unicast.interface_id &= INT_ID_MASK; >>> +       } else { >>> +               /* zero mask bits */ >>> +               p_common_mgid->unicast.interface_id &= (~MGID_MUX_MLID_MASK); >>> +       } >>> +} >>> + >>> +static int64_t >>> +__mgid_cmp(IN const void *const p_gid1, IN const void *const p_gid2) >>> +{ >>> +       return memcmp(p_gid1, p_gid2, sizeof(ib_gid_t)); >>> +} >>> + >>> +static osm_mgrp_port_t *osm_mgrp_port_new(ib_net64_t port_guid) >>> +{ >>> +       osm_mgrp_port_t *p_mgrp_port = >>> +           (osm_mgrp_port_t *) malloc(sizeof(osm_mgrp_port_t)); >>> +       if (!p_mgrp_port) { >>> +               return NULL; >>> +       } >>> +       memset(p_mgrp_port, 0, sizeof(*p_mgrp_port)); >> >> Minor - could use calloc rather than malloc/memset here. >> >> -- Hal >> >>> +       p_mgrp_port->port_guid = port_guid; >>> +       cl_qlist_init(&p_mgrp_port->mgroups); >>> +       return p_mgrp_port; >>> +} >>> -- >>> 1.5.5 >>> >>> _______________________________________________ >>> general mailing list >>> general at lists.openfabrics.org >>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >>> >>> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general >>> >> > From robert.j.woodruff at intel.com Thu Jul 16 10:28:10 2009 From: robert.j.woodruff at intel.com (Woodruff, Robert J) Date: Thu, 16 Jul 2009 10:28:10 -0700 Subject: [ofa-general] RE: OFED 1.4.2-rc2 is available In-Reply-To: <5D49E7A8952DC44FB38C38FA0D758EAD035F7CD9@mtlexch01.mtl.com> References: <5D49E7A8952DC44FB38C38FA0D758EAD02C12252@mtlexch01.mtl.com> <5D49E7A8952DC44FB38C38FA0D758EAD02D5F39D@mtlexch01.mtl.com> <5D49E7A8952DC44FB38C38FA0D758EAD035F7CD9@mtlexch01.mtl.com> Message-ID: <382A478CAD40FA4FB46605CF81FE39F43597FD2B@orsmsx507.amr.corp.intel.com> FYI - When I try to install this, I get this error on EL5.3. If I add the upgrade_open_iscsi=yes to the ofed.conf, it installs fine. However, I don't recall having to do this for OFED-1.4.1, but my testing of 1.4.1 was on an earlier version of RedHat. [root at det-15 OFED-1.4.2-rc2]# ./install.pl -c ofed.conf Please uninstall iscsi-initiator-utils before installing OFED with iSER support. Or put "upgrade_open_iscsi=yes" in the ofed.conf: Perhaps this upgrade_open_iscsi=y should be the default in the ofed.conf if someone chooses to install everything as I do ? -----Original Message----- From: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Tziporet Koren Sent: Thursday, July 16, 2009 4:12 AM To: Tziporet Koren; ewg at lists.openfabrics.org Cc: general at lists.openfabrics.org Subject: [ofa-general] RE: OFED 1.4.2-rc2 is available Fixed subject - its 1.4.2 and not 1.4.1 -----Original Message----- From: Tziporet Koren Sent: Thursday, July 16, 2009 12:55 PM To: Tziporet Koren; ewg at lists.openfabrics.org Cc: general at lists.openfabrics.org Subject: OFED 1.4.1-rc2 is available OFED 1.4.2-RC2 is available. Notes: 1. Any Lustre and NFS/RDMA users should use this release 2. This release is aimed for critical bug fixes only, and will pass only a short QA cycle 2. OFA interop and Logo program testing are not planned for 1.4.2 The tarball is available on: http://www.openfabrics.org/downloads/OFED/ofed-1.4.2/OFED-1.4.2-rc2.tgz To get BUILD_ID run ofed_info Please report any issues in bugzilla https://bugs.openfabrics.org/ for OFED 1.4.2 Vladimir & Tziporet ======================================================================== Release information: ------------------------------ Linux Operating Systems: - RedHat EL4 up4: 2.6.9-42.ELsmp * - RedHat EL4 up5: 2.6.9-55.ELsmp - RedHat EL4 up6: 2.6.9-67.ELsmp - RedHat EL4 up7: 2.6.9-78.ELsmp - RedHat EL5: 2.6.18-8.el5 - RedHat EL5 up1: 2.6.18-53.el5 - RedHat EL5 up2: 2.6.18-92.el5 - RedHat EL5 up3: 2.6.18-128.el5 - OEL 4.5: 2.6.9-55.ELsmp - OEL 5.2: 2.6.18-92.el5 - CentOS 5.2: 2.6.18-92.el5 - Fedora C9: 2.6.25-14.fc9 * - SLES10: 2.6.16.21-0.8-smp - SLES10 SP1: 2.6.16.46-0.12-smp - SLES10 SP1 up1: 2.6.16.53-0.16-smp - SLES10 SP2: 2.6.16.60-0.21-smp - SLES11 GA: 2.6.27.13-1-default - OpenSuSE 10.3: 2.6.22.5-31 * - kernel.org: 2.6.26 and 2.6.27 * Minimal QA for these versions Systems: * x86_64 * x86 * ia64 * ppc64 Changes from OFED-1.4.1 ======================== Bug fixes: ---------- SDP: - Fix memory leak in bzcopy (#1672) - Fix bad credits advertised when connection initiated (#1679) - Fix compilation on i386 with gcc 3.4 used in Redhat 4. (#1630) - Fix Data integrity error (#1672) Backports: - Fix clear-dirty-page accounting. This bug was hit in Lustre testing (#1650) - Fix NFS stale file handles (#1680) - Simple NFS file operations causes RHEL 5.3 server to hard hang. (#1676) - iozone direct write test causes OFED to hang/crash (#1675) - Module crash on server with multiple client simple load (#1677) mlx4 driver: - Fix post send of local invalidate and fast registration packets. This fixed nfsrdma server crash @test5 connectathon basic test. (#1571) nes driver: - fix qp refcount during disconnect Features: --------- nes driver: - Make LRO as default feature mlx4 driver: - Added a new device ID: 0x6764: MT26468 ConnectX EN 10GigE PCIe gen2 Tasks that should be completed for GA (July 22): ================================================ - Fix bug: 1678 simple RDMA NFSv4 traffic hangs the clients attempting to complete - Documents update _______________________________________________ general mailing list general at lists.openfabrics.org http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From He.Huang at Sun.COM Thu Jul 16 11:00:24 2009 From: He.Huang at Sun.COM (Isaac Huang) Date: Thu, 16 Jul 2009 14:00:24 -0400 Subject: [ofa-general] can an ib-bonding slave work independently? In-Reply-To: <4A5D897F.20708@Voltaire.com> References: <20090715025215.GA4322@sun.com> <4A5D897F.20708@Voltaire.com> Message-ID: <20090716180024.GB5930@sun.com> On Wed, Jul 15, 2009 at 10:47:11AM +0300, Or Gerlitz wrote: > Isaac Huang wrote: > > [...] bonding device over ib0 and ib2 worked well ib2 as an independent IPoIB device couldn't work > > (ICMP pings failed). It was CentOS 5.3, with ib-bonding-0.9.0-28. > > Generally speaking, assigning an IP address and hence a route entry to a slave is non recommended and My understanding, which I'd be happy to find false, was that RDMA cmid couldn't be created and bound to a bonding device. If true, then assigning IPs to slaves seemed to be the only way to get ULPs that rely on the RDMA CM API to work, while the master interface provides failover to TCP/IP applications. > doesn't come without pain, e.g see "Potential Sources of Trouble", section 8.1 "Adventures in Routing" of Documentation/networking/bonding.txt, so your problem might have nothing to do with IPoIB. What kernel does CentOS 5.3 comes with? you may be able to use the mainline bonding driver. Thanks for the pointer; our configuration looked good and the slave did not have routes that supersede routes of the master: # ip route show 10.0.0.0/16 dev bond0 proto kernel scope link src 10.0.13.49 10.1.0.0/16 dev ib2 proto kernel scope link src 10.1.13.49 It appeared that all ARP requests over the slave 'ib2' failed, which was why ICMP pings failed: # ip neigh show 10.0.1.111 dev bond0 lladdr 80:00:00:48:fe:80:00:00:00:00:00:10:00:03:ba:00:01:00:fc:05 REACHABLE 10.0.1.101 dev bond0 lladdr 80:00:00:48:fe:80:00:00:00:00:00:10:00:03:ba:00:01:00:fb:05 REACHABLE 10.1.1.112 dev ib2 FAILED 10.1.1.132 dev ib2 FAILED 10.1.1.131 dev ib2 FAILED Rdma_resolve_addr over a cmid bound to the slave also failed with RDMA_CM_EVENT_ADDR_ERROR status -ETIMEDOUT. But tcpdump output on 'ib2' did show the ARP request and response: 15:20:47.571428 arp who-has 10.1.1.132 tell 10.1.13.49 hardware #32 15:20:47.571631 arp reply 10.1.1.132 is-at 80:00:00:49:fe:80:00:00:00:00:00:10:00:03:ba:00:01:00:fb:8a hardware #32 The response seemed to have been dropped by ARP for some reason. The ARP code appears to match responses with outstanding requests on a per-interface basis, and drops responses without a matching request on its incoming interface. When a response arrives on a slave, would it be considered to have been received from the slave interface or its master interface? That seemed to me to be the only place where the responses could be dropped - it worked all fine if bonding was not enabled. Thanks, Isaac From rolandd at cisco.com Thu Jul 16 14:08:39 2009 From: rolandd at cisco.com (Roland Dreier) Date: Thu, 16 Jul 2009 14:08:39 -0700 Subject: [ofa-general] [PATCH 2/6] IB: Use DEFINE_SPINLOCK() for static spinlocks In-Reply-To: <1247778523-29524-2-git-send-email-rolandd@cisco.com> References: <1247778523-29524-1-git-send-email-rolandd@cisco.com> <1247778523-29524-2-git-send-email-rolandd@cisco.com> Message-ID: <1247778523-29524-3-git-send-email-rolandd@cisco.com> Rather than just defining static spinlock_t variables and then initializing them later in init functions, simply define them with DEFINE_SPINLOCK() and remove the calls to spin_lock_init(). This cleans up the source a tad and also shrinks the compiled code; eg on x86-64: add/remove: 0/0 grow/shrink: 0/3 up/down: 0/-40 (-40) function old new delta ib_uverbs_init 336 326 -10 ib_mad_init_module 147 137 -10 ib_sa_init 123 103 -20 Signed-off-by: Roland Dreier --- drivers/infiniband/core/mad.c | 6 +----- drivers/infiniband/core/sa_query.c | 7 ++----- drivers/infiniband/core/uverbs_main.c | 4 +--- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c index de922a0..5cef8f8 100644 --- a/drivers/infiniband/core/mad.c +++ b/drivers/infiniband/core/mad.c @@ -51,8 +51,7 @@ static struct list_head ib_mad_port_list; static u32 ib_mad_client_id = 0; /* Port list lock */ -static spinlock_t ib_mad_port_list_lock; - +static DEFINE_SPINLOCK(ib_mad_port_list_lock); /* Forward declarations */ static int method_in_use(struct ib_mad_mgmt_method_table **method, @@ -2984,8 +2983,6 @@ static int __init ib_mad_init_module(void) { int ret; - spin_lock_init(&ib_mad_port_list_lock); - ib_mad_cache = kmem_cache_create("ib_mad", sizeof(struct ib_mad_private), 0, @@ -3021,4 +3018,3 @@ static void __exit ib_mad_cleanup_module(void) module_init(ib_mad_init_module); module_exit(ib_mad_cleanup_module); - diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c index 1865049..8254371 100644 --- a/drivers/infiniband/core/sa_query.c +++ b/drivers/infiniband/core/sa_query.c @@ -109,10 +109,10 @@ static struct ib_client sa_client = { .remove = ib_sa_remove_one }; -static spinlock_t idr_lock; +static DEFINE_SPINLOCK(idr_lock); static DEFINE_IDR(query_idr); -static spinlock_t tid_lock; +static DEFINE_SPINLOCK(tid_lock); static u32 tid; #define PATH_REC_FIELD(field) \ @@ -1077,9 +1077,6 @@ static int __init ib_sa_init(void) { int ret; - spin_lock_init(&idr_lock); - spin_lock_init(&tid_lock); - get_random_bytes(&tid, sizeof tid); ret = ib_register_client(&sa_client); diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index eb36a81..1a3ac3d 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -73,7 +73,7 @@ DEFINE_IDR(ib_uverbs_cq_idr); DEFINE_IDR(ib_uverbs_qp_idr); DEFINE_IDR(ib_uverbs_srq_idr); -static spinlock_t map_lock; +static DEFINE_SPINLOCK(map_lock); static struct ib_uverbs_device *dev_table[IB_UVERBS_MAX_DEVICES]; static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES); @@ -836,8 +836,6 @@ static int __init ib_uverbs_init(void) { int ret; - spin_lock_init(&map_lock); - ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES, "infiniband_verbs"); if (ret) { -- 1.6.3.3 From rolandd at cisco.com Thu Jul 16 14:08:41 2009 From: rolandd at cisco.com (Roland Dreier) Date: Thu, 16 Jul 2009 14:08:41 -0700 Subject: [ofa-general] [PATCH 4/6] mlx4_core: Remove unnecessary includes of In-Reply-To: <1247778523-29524-4-git-send-email-rolandd@cisco.com> References: <1247778523-29524-1-git-send-email-rolandd@cisco.com> <1247778523-29524-2-git-send-email-rolandd@cisco.com> <1247778523-29524-3-git-send-email-rolandd@cisco.com> <1247778523-29524-4-git-send-email-rolandd@cisco.com> Message-ID: <1247778523-29524-5-git-send-email-rolandd@cisco.com> Lots of mlx4 files with no function annotations included for no reason. Signed-off-by: Roland Dreier --- drivers/net/mlx4/cq.c | 1 - drivers/net/mlx4/eq.c | 1 - drivers/net/mlx4/icm.c | 1 - drivers/net/mlx4/mcg.c | 1 - drivers/net/mlx4/mr.c | 1 - drivers/net/mlx4/pd.c | 1 - drivers/net/mlx4/profile.c | 2 -- drivers/net/mlx4/qp.c | 2 -- drivers/net/mlx4/reset.c | 1 - drivers/net/mlx4/srq.c | 2 -- 10 files changed, 0 insertions(+), 13 deletions(-) diff --git a/drivers/net/mlx4/cq.c b/drivers/net/mlx4/cq.c index ac57b6a..ccfe276 100644 --- a/drivers/net/mlx4/cq.c +++ b/drivers/net/mlx4/cq.c @@ -34,7 +34,6 @@ * SOFTWARE. */ -#include #include #include diff --git a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c index b9ceddd..c11a052 100644 --- a/drivers/net/mlx4/eq.c +++ b/drivers/net/mlx4/eq.c @@ -31,7 +31,6 @@ * SOFTWARE. */ -#include #include #include #include diff --git a/drivers/net/mlx4/icm.c b/drivers/net/mlx4/icm.c index baf4bf6..04b382f 100644 --- a/drivers/net/mlx4/icm.c +++ b/drivers/net/mlx4/icm.c @@ -31,7 +31,6 @@ * SOFTWARE. */ -#include #include #include #include diff --git a/drivers/net/mlx4/mcg.c b/drivers/net/mlx4/mcg.c index 6053c35..5ccbce9 100644 --- a/drivers/net/mlx4/mcg.c +++ b/drivers/net/mlx4/mcg.c @@ -31,7 +31,6 @@ * SOFTWARE. */ -#include #include #include diff --git a/drivers/net/mlx4/mr.c b/drivers/net/mlx4/mr.c index f96948b..ca7ab8e 100644 --- a/drivers/net/mlx4/mr.c +++ b/drivers/net/mlx4/mr.c @@ -32,7 +32,6 @@ * SOFTWARE. */ -#include #include #include diff --git a/drivers/net/mlx4/pd.c b/drivers/net/mlx4/pd.c index 26d1a7a..c4988d6 100644 --- a/drivers/net/mlx4/pd.c +++ b/drivers/net/mlx4/pd.c @@ -31,7 +31,6 @@ * SOFTWARE. */ -#include #include #include diff --git a/drivers/net/mlx4/profile.c b/drivers/net/mlx4/profile.c index bd22df9..ca25b9d 100644 --- a/drivers/net/mlx4/profile.c +++ b/drivers/net/mlx4/profile.c @@ -32,8 +32,6 @@ * SOFTWARE. */ -#include - #include "mlx4.h" #include "fw.h" diff --git a/drivers/net/mlx4/qp.c b/drivers/net/mlx4/qp.c index 1c565ef..42ab9fc 100644 --- a/drivers/net/mlx4/qp.c +++ b/drivers/net/mlx4/qp.c @@ -33,8 +33,6 @@ * SOFTWARE. */ -#include - #include #include diff --git a/drivers/net/mlx4/reset.c b/drivers/net/mlx4/reset.c index 3951b88..e5741da 100644 --- a/drivers/net/mlx4/reset.c +++ b/drivers/net/mlx4/reset.c @@ -31,7 +31,6 @@ * SOFTWARE. */ -#include #include #include #include diff --git a/drivers/net/mlx4/srq.c b/drivers/net/mlx4/srq.c index fe9f218..1377d0d 100644 --- a/drivers/net/mlx4/srq.c +++ b/drivers/net/mlx4/srq.c @@ -31,8 +31,6 @@ * SOFTWARE. */ -#include - #include #include "mlx4.h" -- 1.6.3.3 From rolandd at cisco.com Thu Jul 16 14:08:38 2009 From: rolandd at cisco.com (Roland Dreier) Date: Thu, 16 Jul 2009 14:08:38 -0700 Subject: [ofa-general] [PATCH 1/6] IPoIB: Remove unused includes In-Reply-To: <1247778523-29524-1-git-send-email-rolandd@cisco.com> References: <1247778523-29524-1-git-send-email-rolandd@cisco.com> Message-ID: <1247778523-29524-2-git-send-email-rolandd@cisco.com> Signed-off-by: Roland Dreier --- drivers/infiniband/ulp/ipoib/ipoib_cm.c | 1 - drivers/infiniband/ulp/ipoib/ipoib_ib.c | 1 - 2 files changed, 0 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c index 181b1f3..8f4b4fc 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c @@ -31,7 +31,6 @@ */ #include -#include #include #include #include diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c index e7e5adf..e35f4a0 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c @@ -36,7 +36,6 @@ #include #include -#include #include #include -- 1.6.3.3 From rolandd at cisco.com Thu Jul 16 14:08:42 2009 From: rolandd at cisco.com (Roland Dreier) Date: Thu, 16 Jul 2009 14:08:42 -0700 Subject: [ofa-general] [PATCH 5/6] IB/mthca: Remove unnecessary include of In-Reply-To: <1247778523-29524-5-git-send-email-rolandd@cisco.com> References: <1247778523-29524-1-git-send-email-rolandd@cisco.com> <1247778523-29524-2-git-send-email-rolandd@cisco.com> <1247778523-29524-3-git-send-email-rolandd@cisco.com> <1247778523-29524-4-git-send-email-rolandd@cisco.com> <1247778523-29524-5-git-send-email-rolandd@cisco.com> Message-ID: <1247778523-29524-6-git-send-email-rolandd@cisco.com> mthca_config_reg.h was including for no reason -- the whole file is just defines of constants, so it's entirely self-contained. Signed-off-by: Roland Dreier --- drivers/infiniband/hw/mthca/mthca_config_reg.h | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/mthca/mthca_config_reg.h b/drivers/infiniband/hw/mthca/mthca_config_reg.h index 75671f7..155bc66 100644 --- a/drivers/infiniband/hw/mthca/mthca_config_reg.h +++ b/drivers/infiniband/hw/mthca/mthca_config_reg.h @@ -34,8 +34,6 @@ #ifndef MTHCA_CONFIG_REG_H #define MTHCA_CONFIG_REG_H -#include - #define MTHCA_HCR_BASE 0x80680 #define MTHCA_HCR_SIZE 0x0001c #define MTHCA_ECR_BASE 0x80700 -- 1.6.3.3 From rolandd at cisco.com Thu Jul 16 14:08:37 2009 From: rolandd at cisco.com (Roland Dreier) Date: Thu, 16 Jul 2009 14:08:37 -0700 Subject: [ofa-general] [PATCH 0/6] Current queue of cleanup patches Message-ID: <1247778523-29524-1-git-send-email-rolandd@cisco.com> Here's a few small cleanup patches fixing things I've noticed in passing. I have them queued up to merge for 2.6.32; please let me know if you have any concerns about them. From rolandd at cisco.com Thu Jul 16 14:08:43 2009 From: rolandd at cisco.com (Roland Dreier) Date: Thu, 16 Jul 2009 14:08:43 -0700 Subject: [ofa-general] [PATCH 6/6] IB/mthca: Remove unnecessary include of In-Reply-To: <1247778523-29524-6-git-send-email-rolandd@cisco.com> References: <1247778523-29524-1-git-send-email-rolandd@cisco.com> <1247778523-29524-2-git-send-email-rolandd@cisco.com> <1247778523-29524-3-git-send-email-rolandd@cisco.com> <1247778523-29524-4-git-send-email-rolandd@cisco.com> <1247778523-29524-5-git-send-email-rolandd@cisco.com> <1247778523-29524-6-git-send-email-rolandd@cisco.com> Message-ID: <1247778523-29524-7-git-send-email-rolandd@cisco.com> mthca_reset.c doesn't have any function annotations, so there's no reason to include . Signed-off-by: Roland Dreier --- drivers/infiniband/hw/mthca/mthca_reset.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/drivers/infiniband/hw/mthca/mthca_reset.c b/drivers/infiniband/hw/mthca/mthca_reset.c index acb6817..2a13a16 100644 --- a/drivers/infiniband/hw/mthca/mthca_reset.c +++ b/drivers/infiniband/hw/mthca/mthca_reset.c @@ -30,7 +30,6 @@ * SOFTWARE. */ -#include #include #include #include -- 1.6.3.3 From rolandd at cisco.com Thu Jul 16 14:08:40 2009 From: rolandd at cisco.com (Roland Dreier) Date: Thu, 16 Jul 2009 14:08:40 -0700 Subject: [ofa-general] [PATCH 3/6] mlx4_core: Use pci_request_regions() In-Reply-To: <1247778523-29524-3-git-send-email-rolandd@cisco.com> References: <1247778523-29524-1-git-send-email-rolandd@cisco.com> <1247778523-29524-2-git-send-email-rolandd@cisco.com> <1247778523-29524-3-git-send-email-rolandd@cisco.com> Message-ID: <1247778523-29524-4-git-send-email-rolandd@cisco.com> The old code used two calls to pci_request_region() to get the two BARs for the mlx4 device, for no particularly good reason. Clean up the code a little by converting this to a single call to pci_request_regions(). Signed-off-by: Roland Dreier --- drivers/net/mlx4/main.c | 26 ++++++++------------------ 1 files changed, 8 insertions(+), 18 deletions(-) diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c index dac621b..5c1afe0 100644 --- a/drivers/net/mlx4/main.c +++ b/drivers/net/mlx4/main.c @@ -1070,18 +1070,12 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) goto err_disable_pdev; } - err = pci_request_region(pdev, 0, DRV_NAME); + err = pci_request_regions(pdev, DRV_NAME); if (err) { - dev_err(&pdev->dev, "Cannot request control region, aborting.\n"); + dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); goto err_disable_pdev; } - err = pci_request_region(pdev, 2, DRV_NAME); - if (err) { - dev_err(&pdev->dev, "Cannot request UAR region, aborting.\n"); - goto err_release_bar0; - } - pci_set_master(pdev); err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); @@ -1090,7 +1084,7 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n"); - goto err_release_bar2; + goto err_release_regions; } } err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); @@ -1101,7 +1095,7 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) if (err) { dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, " "aborting.\n"); - goto err_release_bar2; + goto err_release_regions; } } @@ -1110,7 +1104,7 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) dev_err(&pdev->dev, "Device struct alloc failed, " "aborting.\n"); err = -ENOMEM; - goto err_release_bar2; + goto err_release_regions; } dev = &priv->dev; @@ -1205,11 +1199,8 @@ err_cmd: err_free_dev: kfree(priv); -err_release_bar2: - pci_release_region(pdev, 2); - -err_release_bar0: - pci_release_region(pdev, 0); +err_release_regions: + pci_release_regions(pdev); err_disable_pdev: pci_disable_device(pdev); @@ -1265,8 +1256,7 @@ static void mlx4_remove_one(struct pci_dev *pdev) pci_disable_msix(pdev); kfree(priv); - pci_release_region(pdev, 2); - pci_release_region(pdev, 0); + pci_release_regions(pdev); pci_disable_device(pdev); pci_set_drvdata(pdev, NULL); } -- 1.6.3.3 From arlin.r.davis at intel.com Thu Jul 16 14:38:28 2009 From: arlin.r.davis at intel.com (Davis, Arlin R) Date: Thu, 16 Jul 2009 14:38:28 -0700 Subject: [ofa-general] [PATCH] uDAPL v2: dapltest: replace small malloc's with stack allocation, add completion vector threshold Message-ID: Implement a malloc() threshold for the completion reaping byte vector allocation in functions: DT_handle_send_op, DT_handle_rdma_op & DT_handle_recv_op. When allocation size is under the threshold, use a stack local allocation instead of malloc/free. Move redundant bzero() to be called only in the case of using local stack allocation as DT_Mdep_malloc() already does a bzero(). Consolidate error handling return and free()check to a single point by using goto. Signed-off-by: Stan Smith diff --git a/test/dapltest/test/dapl_transaction_util.c b/test/dapltest/test/dapl_transaction_util.c index ffe5d7d..14a14dd 100644 --- a/test/dapltest/test/dapl_transaction_util.c +++ b/test/dapltest/test/dapl_transaction_util.c @@ -30,6 +30,8 @@ #include "dapl_proto.h" +#define DT_LOCAL_COMPLETION_VECTOR_SIZE 32 + /* ----------------------------------------------------------- * Post a recv buffer on each of this thread's EPs. */ @@ -85,11 +87,19 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead, { unsigned int i, j; unsigned char *completion_reaped; + unsigned char lcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE]; + bool rc = false; - completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char)); - - if (!completion_reaped) { - return false; + if (num_eps <= DT_LOCAL_COMPLETION_VECTOR_SIZE) { + completion_reaped = lcomp; + bzero((void *)completion_reaped, + sizeof(unsigned char) * num_eps); + } + else { + completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char)); + if (!completion_reaped) { + return false; + } } for (i = 0; i < num_eps; i++) { @@ -120,8 +130,7 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead, "Test Error: dat_ep_post_send failed: %s\n", DT_RetToString(ret)); DT_Test_Error(); - DT_Mdep_Free(completion_reaped); - return false; + goto xit; } } @@ -130,13 +139,11 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead, if (op->reap_send_on_recv && !op->server_initiated) { /* we will reap the send on the recv (Client SR) */ - DT_Mdep_Free(completion_reaped); - return true; + rc = true; + goto xit; } } - bzero((void *)completion_reaped, sizeof(unsigned char) * num_eps); - /* reap the send completion */ for (i = 0; i < num_eps; i++) { Transaction_Test_Op_t *op; @@ -146,8 +153,7 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead, if (!DT_dto_event_reap (phead, ep_context[i].reqt_evd_hdl, poll, &dto_stat)) { - DT_Mdep_Free(completion_reaped); - return false; + goto xit; } epnum = dto_stat.user_cookie.as_64 >> 32; @@ -160,8 +166,7 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead, dto_stat.user_cookie.as_64, dto_stat.transfered_length); DT_Test_Error(); - DT_Mdep_Free(completion_reaped); - return false; + goto xit; } op = &ep_context[epnum].op[op_indx]; @@ -176,8 +181,7 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead, ep_context[epnum].ep_handle, op->num_segs * op->seg_size, dto_cookie, "Send")) { - DT_Mdep_Free(completion_reaped); - return false; + goto xit; } if (completion_reaped[epnum]) { @@ -185,8 +189,7 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead, "Test Error: Send: Secondary completion seen for endpoint 0x%p (%d)\n", ep_context[epnum].ep_handle, epnum); DT_Test_Error(); - DT_Mdep_Free(completion_reaped); - return (false); + goto xit; } completion_reaped[epnum] = 1; } @@ -197,13 +200,16 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead, "Test Error: Send: No completion seen for endpoint 0x%p (#%d)\n", ep_context[i].ep_handle, i); DT_Test_Error(); - DT_Mdep_Free(completion_reaped); - return (false); + goto xit; } } - DT_Mdep_Free(completion_reaped); - return true; + rc = true; + +xit: + if (completion_reaped != lcomp) + DT_Mdep_Free(completion_reaped); + return rc; } /* ----------------------------------------------------------- @@ -220,21 +226,32 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, unsigned int i; unsigned char *recv_completion_reaped; unsigned char *send_completion_reaped; - - recv_completion_reaped = DT_Mdep_Malloc(num_eps); - if (recv_completion_reaped == NULL) { - return false; + unsigned char rcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE]; + unsigned char lcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE]; + bool rc = false; + + if (num_eps <= DT_LOCAL_COMPLETION_VECTOR_SIZE ) { + recv_completion_reaped = rcomp; + send_completion_reaped = lcomp; + bzero((void *)recv_completion_reaped, + sizeof(unsigned char) * num_eps); + bzero((void *)send_completion_reaped, + sizeof(unsigned char) * num_eps); } + else { + recv_completion_reaped = DT_Mdep_Malloc(num_eps); + if (recv_completion_reaped == NULL) { + return false; + } - send_completion_reaped = DT_Mdep_Malloc(num_eps); - if (send_completion_reaped == NULL) { - DT_Mdep_Free(recv_completion_reaped); - return false; + send_completion_reaped = DT_Mdep_Malloc(num_eps); + if (send_completion_reaped == NULL) { + DT_Mdep_Free(recv_completion_reaped); + return false; + } } /* Foreach EP, reap */ - bzero((void *)recv_completion_reaped, sizeof(unsigned char) * num_eps); - bzero((void *)send_completion_reaped, sizeof(unsigned char) * num_eps); for (i = 0; i < num_eps; i++) { Transaction_Test_Op_t *op; DAT_DTO_COMPLETION_EVENT_DATA dto_stat; @@ -244,9 +261,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, /* First reap the recv DTO event */ if (!DT_dto_event_reap (phead, ep_context[i].recv_evd_hdl, poll, &dto_stat)) { - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return false; + goto xit; } epnum = dto_stat.user_cookie.as_64 >> 32; @@ -259,9 +274,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, dto_stat.user_cookie.as_64, dto_stat.transfered_length); DT_Test_Error(); - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return false; + goto xit; } op = &ep_context[epnum].op[op_indx]; @@ -278,9 +291,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, DT_Tdep_PT_Printf(phead, "Test Error: recv DTO problem\n"); DT_Test_Error(); - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return false; + goto xit; } if (recv_completion_reaped[epnum]) { @@ -288,9 +299,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, "Test Error: Receive: Secondary completion seen for endpoint 0x%p (%d)\n", ep_context[epnum].ep_handle, epnum); DT_Test_Error(); - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return (false); + goto xit; } recv_completion_reaped[epnum] = 1; @@ -306,17 +315,13 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, "Internal Error: reap_send_on_recv" " but current op == #%d\n", op_indx); - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return false; + goto xit; } if (!DT_dto_event_reap (phead, ep_context[i].reqt_evd_hdl, poll, &dto_stat)) { - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return false; + goto xit; } epnum = dto_stat.user_cookie.as_64 >> 32; @@ -329,9 +334,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, dto_stat.user_cookie.as_64, dto_stat.transfered_length); DT_Test_Error(); - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return false; + goto xit; } /* @@ -359,9 +362,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, DT_Tdep_PT_Printf(phead, "Test Error: send DTO problem\n"); DT_Test_Error(); - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return false; + goto xit; } if (send_completion_reaped[epnum]) { @@ -370,9 +371,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, ep_context[epnum].ep_handle, epnum); DT_Test_Error(); - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return (false); + goto xit; } send_completion_reaped[epnum] = 1; } @@ -384,9 +383,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, "Test Error: Receive: No completion seen for endpoint 0x%p (#%d)\n", ep_context[i].ep_handle, i); DT_Test_Error(); - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return (false); + goto xit; } } @@ -398,9 +395,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, "Test Error: Send (ror): No completion seen for endpoint 0x%p (#%d)\n", ep_context[i].ep_handle, i); DT_Test_Error(); - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return (false); + goto xit; } } } @@ -412,15 +407,16 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead, DT_Tdep_PT_Printf(phead, "Test Error: recv re-post problem\n"); DT_Test_Error(); - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return false; + goto xit; } } - - DT_Mdep_Free(recv_completion_reaped); - DT_Mdep_Free(send_completion_reaped); - return true; + rc = true; +xit: + if (send_completion_reaped != lcomp) { + DT_Mdep_Free(recv_completion_reaped); + DT_Mdep_Free(send_completion_reaped); + } + return rc; } /* ----------------------------------------------------------- @@ -435,11 +431,18 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead, unsigned int i, j; DAT_RETURN ret; unsigned char *completion_reaped; + unsigned char lcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE]; + bool rc = false; - completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char)); - - if (!completion_reaped) { - return false; + if (num_eps <= DT_LOCAL_COMPLETION_VECTOR_SIZE) { + completion_reaped = lcomp; + bzero((void *)completion_reaped, sizeof(unsigned char) * num_eps); + } + else { + completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char)); + if (!completion_reaped) { + return false; + } } /* Initiate the operation */ @@ -500,8 +503,7 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead, RDMA_WRITE ? "write" : "read"), DT_RetToString(ret)); DT_Test_Error(); - DT_Mdep_Free(completion_reaped); - return (false); + goto err; } else { DT_Tdep_PT_Debug(3, (phead, "Done dat_ep_post_rdma_%s %s\n", @@ -511,7 +513,6 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead, } } - bzero((void *)completion_reaped, sizeof(unsigned char) * num_eps); /* Wait for it to happen */ for (i = 0; i < num_eps; i++) { Transaction_Test_Op_t *op; @@ -521,8 +522,7 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead, if (!DT_dto_event_reap (phead, ep_context[i].reqt_evd_hdl, poll, &dto_stat)) { - DT_Mdep_Free(completion_reaped); - return (false); + goto err; } epnum = dto_stat.user_cookie.as_64 >> 32; @@ -537,8 +537,7 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead, dto_stat.user_cookie.as_64, dto_stat.transfered_length); DT_Test_Error(); - DT_Mdep_Free(completion_reaped); - return false; + goto err; } op = &ep_context[epnum].op[op_indx]; @@ -554,8 +553,7 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead, dto_cookie, (opcode == RDMA_WRITE ? "RDMA/WR" : "RDMA/RD"))) { - DT_Mdep_Free(completion_reaped); - return (false); + goto err; } if (completion_reaped[epnum]) { @@ -565,8 +563,7 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead, RDMA_WRITE ? "RDMA/WR" : "RDMA/RD", ep_context[epnum].ep_handle, epnum); DT_Test_Error(); - DT_Mdep_Free(completion_reaped); - return (false); + goto err; } completion_reaped[epnum] = 1; @@ -584,14 +581,17 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead, RDMA_WRITE ? "RDMA/WR" : "RDMA/RD", ep_context[i].ep_handle, i); DT_Test_Error(); - DT_Mdep_Free(completion_reaped); - return (false); + goto err; } } - DT_Mdep_Free(completion_reaped); + rc = true; + +err: + if (completion_reaped != lcomp) + DT_Mdep_Free(completion_reaped); - return (true); + return rc; } /* ----------------------------------------------------------- From rdreier at cisco.com Thu Jul 16 14:53:34 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 16 Jul 2009 14:53:34 -0700 Subject: [ofa-general] [PATCH] Allow paths to the device specific library to be absolute In-Reply-To: <20090714211653.GA13700@obsidianresearch.com> (Jason Gunthorpe's message of "Tue, 14 Jul 2009 15:16:53 -0600") References: <20090714211653.GA13700@obsidianresearch.com> Message-ID: Seems sane and quite useful, applied thanks From rdreier at cisco.com Thu Jul 16 14:57:20 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 16 Jul 2009 14:57:20 -0700 Subject: [ofa-general] Re: [PATCH] infiniband: Use %pM conversion specifier In-Reply-To: <1247735559-21945-1-git-send-email-klto@zhaw.ch> (Tobias Klauser's message of "Thu, 16 Jul 2009 11:12:39 +0200") References: <1247735559-21945-1-git-send-email-klto@zhaw.ch> Message-ID: thanks, applied From andy.grover at oracle.com Thu Jul 16 17:12:47 2009 From: andy.grover at oracle.com (Andy Grover) Date: Thu, 16 Jul 2009 17:12:47 -0700 Subject: [ofa-general] How to demand-load RDS transports? Message-ID: <4A5FC1FF.6090700@oracle.com> I am working on a version of the RDS socket protocol that supports both IB/iWARP and TCP as modularized transports. Thus instead of rds.ko there would be rds.ko, rds-tcp.ko and rds-rdma.ko. Right now it's possible to use rds and rds-tcp without rds-rdma module loaded, but the entire IB stack must still be loaded -- the dynamic transport-loading code in rds.ko must still use IB API to see if any IB adapters are present. Is it possible (with udev perhaps?) to remove the IB calls from rds.ko, and get rds-rdma.ko loaded some other way when rds.ko is loaded and IB hw is present? Any pointers appreciated. Thanks -- Regards -- Andy From vlad at lists.openfabrics.org Fri Jul 17 02:39:31 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Fri, 17 Jul 2009 02:39:31 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090717-0200 daily build status Message-ID: <20090717093931.DA4661020571@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -fasynchronous-unwind-tables -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_cma)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c: In function 'sdp_cma_handler': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c:312: error: implicit declaration of function 'lock_sock_nested' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -fasynchronous-unwind-tables -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_cma)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c: In function 'sdp_cma_handler': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c:312: error: implicit declaration of function 'lock_sock_nested' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -m64 -mno-red-zone -mcmodel=kernel -funit-at-a-time -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090717-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From yossi.openib at gmail.com Fri Jul 17 05:50:13 2009 From: yossi.openib at gmail.com (Yossi Etigin) Date: Fri, 17 Jul 2009 15:50:13 +0300 Subject: [ofa-general] Re: [ewg] [PATCH 0/8 v3] RDMAoE support In-Reply-To: <20090713181310.GA31865@mtls03> References: <20090713181310.GA31865@mtls03> Message-ID: <4A607385.9080309@gmail.com> Eli Cohen wrote: > RDMA over Ethernet (RDMAoE) allows running the IB transport protocol > using Ethernet frames allowing the deployment of IB semantics on > lossless Ethernet fabrics. RDMAoE packets are standard Ethernet frames > with an IEEE assigned Ethertype, a GRH, unmodified IB transport > headers and payload. Aside from the considerations pointed out below, > RDMAoE ports are functionally equivalent to regular IB ports from the > RDMA stack perspective. > > IB subnet management and SA services are not required for RDMAoE > operation; Ethernet management practices are used instead. In > Ethernet, nodes are commonly referred to by applications by means of > an IP address. RDMAoE encodes the IP addresses that were assigned to > the corresponding Ethernet port into its GIDs, and makes use of the IP > stack to bind a destination address to the corresponding netdevice > (just as the CMA does today for IB and iWARP) and to obtain its L2 MAC > addresses. > > The RDMA Verbs API is syntactically unmodified. When referring to > RDMAoE ports, Address handles are required to contain GIDs and the L2 > address fields in the API are ignored. The Ethernet L2 information is > then obtained by the vendor-specific driver (both in kernel- and > user-space) while modifying QPs to RTR and creating address handles. > > In order to maximize transparency for applications, RDMAoE implements > a dedicated API that provides services equivalent to some of those > provided by the IB-SA. The current approach is strictly local but may > evolve in the future. This API is implemented using an independent > source code file which allows for seamless evolution of the code > without affecting the IB native SA interfaces. We have successfully > tested MPI, SDP, RDS, and native Verbs applications over RDMAoE. > > To enable RDMAoE with the mlx4 driver stack, both the mlx4_en and > mlx4_ib drivers must be loaded, and the netdevice for the > corresponding RDMAoE port must be running. Individual ports of a multi > port HCA can be independently configured as Ethernet (with support for > RDMAoE) or IB, as is already the case. > > Following is a series of 8 patches based on version 2.6.30 of the > Linux kernel. This new series reflects changes based on feedback from > the community on the previous set of patches. The whole series is > tagged v3. > > Signed-off-by: Eli Cohen > I agree with Or here, I really do not think that making RDMAoE transparent to applications is worth pushing a lot of compatibility code to the kernel. The winner here is definitely rdmaoe_sa - 1000 lines of useless code which boils down to kernel_bind and kernel_setsockopt. Why do you need all this code to hold state, refcounts, whatever - if the kernel already does this for you? If an application uses IB - let it use real IB. If it uses RDMA - let it use all RDMA implementations out there (IB, iwarp, RDMAoE). Therefore, I think the correct place to add RDMAoE is under rdma_cm. If a consumer wants to use RDMAoE - it should use rdma_cm. Looks like you are trying to add something that is between RDMAoE and IBoE, and put a lot of hacky bypass logic in core and ulps. --Yossi From roel.kluin at gmail.com Fri Jul 17 06:06:28 2009 From: roel.kluin at gmail.com (Roel Kluin) Date: Fri, 17 Jul 2009 15:06:28 +0200 Subject: [ofa-general] [PATCH] ipath: strncpy does not null terminate string Message-ID: <4A607754.4040204@gmail.com> With `sizeof(lastcomm) - 1` strncpy() will null terminate the string. Signed-off-by: Roel Kluin --- To test this: #include #include char a[10]; char b[10]; int main() { const char* str = "0123456789012"; strncpy(a, str, sizeof(a)); strncpy(b, str, sizeof(b) - 1); printf("String a was %s, b was %s\n", a, b); return 0; } diff --git a/drivers/infiniband/hw/ipath/ipath_file_ops.c b/drivers/infiniband/hw/ipath/ipath_file_ops.c index 2317398..801b7c8 100644 --- a/drivers/infiniband/hw/ipath/ipath_file_ops.c +++ b/drivers/infiniband/hw/ipath/ipath_file_ops.c @@ -1616,7 +1616,7 @@ static int try_alloc_port(struct ipath_devdata *dd, int port, pd->port_cnt = 1; port_fp(fp) = pd; pd->port_pid = get_pid(task_pid(current)); - strncpy(pd->port_comm, current->comm, sizeof(pd->port_comm)); + strncpy(pd->port_comm, current->comm, sizeof(pd->port_comm) - 1); ipath_stats.sps_ports++; ret = 0; } else diff --git a/drivers/infiniband/hw/ipath/ipath_mad.c b/drivers/infiniband/hw/ipath/ipath_mad.c index 16a702d..9cd9d63 100644 --- a/drivers/infiniband/hw/ipath/ipath_mad.c +++ b/drivers/infiniband/hw/ipath/ipath_mad.c @@ -60,7 +60,7 @@ static int recv_subn_get_nodedescription(struct ib_smp *smp, if (smp->attr_mod) smp->status |= IB_SMP_INVALID_FIELD; - strncpy(smp->data, ibdev->node_desc, sizeof(smp->data)); + strncpy(smp->data, ibdev->node_desc, sizeof(smp->data) - 1); return reply(smp); } From roel.kluin at gmail.com Fri Jul 17 07:22:26 2009 From: roel.kluin at gmail.com (Roel Kluin) Date: Fri, 17 Jul 2009 16:22:26 +0200 Subject: [ofa-general] Re: [PATCH] ipath: strncpy does not null terminate string In-Reply-To: <4A607754.4040204@gmail.com> References: <4A607754.4040204@gmail.com> Message-ID: <4A608922.7060900@gmail.com> strlcpy() will always null terminate the string. Signed-off-by: Roel Kluin --- Please use this one instead diff --git a/drivers/infiniband/hw/ipath/ipath_file_ops.c b/drivers/infiniband/hw/ipath/ipath_file_ops.c index 2317398..38a2870 100644 --- a/drivers/infiniband/hw/ipath/ipath_file_ops.c +++ b/drivers/infiniband/hw/ipath/ipath_file_ops.c @@ -1616,7 +1616,7 @@ static int try_alloc_port(struct ipath_devdata *dd, int port, pd->port_cnt = 1; port_fp(fp) = pd; pd->port_pid = get_pid(task_pid(current)); - strncpy(pd->port_comm, current->comm, sizeof(pd->port_comm)); + strlcpy(pd->port_comm, current->comm, sizeof(pd->port_comm)); ipath_stats.sps_ports++; ret = 0; } else diff --git a/drivers/infiniband/hw/ipath/ipath_mad.c b/drivers/infiniband/hw/ipath/ipath_mad.c index 16a702d..458a918 100644 --- a/drivers/infiniband/hw/ipath/ipath_mad.c +++ b/drivers/infiniband/hw/ipath/ipath_mad.c @@ -60,7 +60,7 @@ static int recv_subn_get_nodedescription(struct ib_smp *smp, if (smp->attr_mod) smp->status |= IB_SMP_INVALID_FIELD; - strncpy(smp->data, ibdev->node_desc, sizeof(smp->data)); + strlcpy(smp->data, ibdev->node_desc, sizeof(smp->data)); return reply(smp); } From lars.ellenberg at linbit.com Fri Jul 17 08:25:56 2009 From: lars.ellenberg at linbit.com (Lars Ellenberg) Date: Fri, 17 Jul 2009 17:25:56 +0200 Subject: [ofa-general] SDP tuning: recv_poll? sdp_zcopy_thresh? other parameters? Message-ID: <20090717152556.GC8073@barkeeper1-xen.linbit> Lets say my typical usage pattern is: two nodes, alice and bob. two "tcp" connections via SDP, long lifetime, called the "data" and the "meta" socket. alice sends on the "data" socket, typically transfer message sizes of 512 byte to 32 KiB, say. for each message received on the data socket bob then sends an "ack" back on the "meta" socket, message sizes of ~32 byte. [*] there are a few other messages on both sockets. I'd like to have maximum throughput when streaming large messages, but (of course) at the same time I'd like to have minimum latency for the short messages, and when sending only single requests. obviously if the cpu overhead can be minimized, that won't hurt either ;) now, which tunables in /sys/modules/ib_sdp/parameters are those that will most likely have some effect here? an "I don't now what I am tuning here, but I try anyways" approach gave me some benefit from using recv_poll 200 sdp_zcopy_thresh 8192, all else left at what the module chose itself. pointers to an overview about what those tunables actually do, or any recommendations (also for tunables in other modules, potentially, or sysctls, tcp or other socket options or whatnot) gladly accepted. [*] this is, of course, in fact DRBD (see http://www.drbd.org), the large messages on the "data" socket are replicated block device writes, bob needs to submit that data to its local IO subsystem, and only send out the "ack" messages on the "meta" socket once the corresponding write has been signalled as completed. -- : Lars Ellenberg : LINBIT | Your Way to High Availability : DRBD/HA support and consulting http://www.linbit.com DRBD® and LINBIT® are registered trademarks of LINBIT, Austria. From ralph.campbell at qlogic.com Fri Jul 17 08:59:12 2009 From: ralph.campbell at qlogic.com (Ralph Campbell) Date: Fri, 17 Jul 2009 08:59:12 -0700 Subject: [ofa-general] Re: [PATCH] ipath: strncpy does not null terminate string In-Reply-To: <4A608922.7060900@gmail.com> References: <4A607754.4040204@gmail.com> <4A608922.7060900@gmail.com> Message-ID: <1247846352.23162.613.camel@chromite.mv.qlogic.com> Acked-by: Ralph Campbell On Fri, 2009-07-17 at 07:22 -0700, Roel Kluin wrote: > strlcpy() will always null terminate the string. > > Signed-off-by: Roel Kluin > --- > Please use this one instead > > diff --git a/drivers/infiniband/hw/ipath/ipath_file_ops.c b/drivers/infiniband/hw/ipath/ipath_file_ops.c > index 2317398..38a2870 100644 > --- a/drivers/infiniband/hw/ipath/ipath_file_ops.c > +++ b/drivers/infiniband/hw/ipath/ipath_file_ops.c > @@ -1616,7 +1616,7 @@ static int try_alloc_port(struct ipath_devdata *dd, int port, > pd->port_cnt = 1; > port_fp(fp) = pd; > pd->port_pid = get_pid(task_pid(current)); > - strncpy(pd->port_comm, current->comm, sizeof(pd->port_comm)); > + strlcpy(pd->port_comm, current->comm, sizeof(pd->port_comm)); > ipath_stats.sps_ports++; > ret = 0; > } else > diff --git a/drivers/infiniband/hw/ipath/ipath_mad.c b/drivers/infiniband/hw/ipath/ipath_mad.c > index 16a702d..458a918 100644 > --- a/drivers/infiniband/hw/ipath/ipath_mad.c > +++ b/drivers/infiniband/hw/ipath/ipath_mad.c > @@ -60,7 +60,7 @@ static int recv_subn_get_nodedescription(struct ib_smp *smp, > if (smp->attr_mod) > smp->status |= IB_SMP_INVALID_FIELD; > > - strncpy(smp->data, ibdev->node_desc, sizeof(smp->data)); > + strlcpy(smp->data, ibdev->node_desc, sizeof(smp->data)); > > return reply(smp); > } > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From jgunthorpe at obsidianresearch.com Fri Jul 17 14:08:17 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Fri, 17 Jul 2009 15:08:17 -0600 Subject: [ofa-general] [PATCH] Allow paths to the device specific library to be absolute In-Reply-To: References: <20090714211653.GA13700@obsidianresearch.com> Message-ID: <20090717210817.GF7320@obsidianresearch.com> On Thu, Jul 16, 2009 at 02:53:34PM -0700, Roland Dreier wrote: > Seems sane and quite useful, applied thanks Thanks - I have a few other little quibbles I've noticed from using verbs from C++: * Missing const's in some function parameters * Use of 'enum foo' as a type for a bit field. In C/C++ or'ing enum members results in a type that is the enum base integral type, not the enum itself. g++ produces a warning by default. Very annoying :) Are you interested in patches for these too? -- Jason Gunthorpe (780)4406067x832 Chief Technology Officer, Obsidian Research Corp Edmonton, Canada From jgunthorpe at obsidianresearch.com Fri Jul 17 14:15:21 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Fri, 17 Jul 2009 15:15:21 -0600 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: References: <4A521784.5090304@Voltaire.COM> <4A5DF1C0.2060701@Voltaire.COM> Message-ID: <20090717211521.GG7320@obsidianresearch.com> On Wed, Jul 15, 2009 at 09:01:05AM -0700, Roland Dreier wrote: > > > I took your advice and sent a patch to bonding to fix the issue there to which I > > am waiting for comment) but I still think the patch for IPoIB is still needed. > > Without it, IPoIB is exposed to a DoS attack by a module (that looks like bonding but > > with malicious intentions) that sends IPoIB a garbage multicast address and stops it from > > joining any other group for ever, even if it is a legal group. > > If the attack vector is a malicous module, I'm not too worried about > it -- after all, a malicious module could just overwrite the IPoIB > module code with whatever it wants and break things that way. > > Is there any way userspace can inject a bogus multicast address? Can you do it with netlink? ip maddr add address ... dev ib0 Jason From vlad at lists.openfabrics.org Sat Jul 18 02:37:50 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Sat, 18 Jul 2009 02:37:50 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090718-0200 daily build status Message-ID: <20090718093750.5A6E910205E3@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -fasynchronous-unwind-tables -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_cma)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c: In function 'sdp_cma_handler': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c:312: error: implicit declaration of function 'lock_sock_nested' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -fasynchronous-unwind-tables -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_cma)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c: In function 'sdp_cma_handler': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c:312: error: implicit declaration of function 'lock_sock_nested' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -m64 -mno-red-zone -mcmodel=kernel -funit-at-a-time -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090718-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From jsquyres at cisco.com Sat Jul 18 17:32:49 2009 From: jsquyres at cisco.com (Jeff Squyres) Date: Sat, 18 Jul 2009 20:32:49 -0400 Subject: [ofa-general] Re: [PATCH/RFC] New version of ummunot patch In-Reply-To: References: <20090506214628.GM2590@obsidianresearch.com><20090506222638.GA16280@obsidianresearch.com><20090507000231.GB16280@obsidianresearch.com><20090507224806.GF16280@obsidianresearch.com> <5019F239-149F-49E1-8C23-436DE6094AB2@cisco.com> Message-ID: <26DBDE97-821F-4F73-B901-75FDA85D8053@cisco.com> Thanks Roland! Also note that the current prototype Open MPI version is available here: http://bitbucket.org/jsquyres/ummunot/ It still needs a bit of polish and proper integration before merging into the Open MPI mainline trunk, but the ideas and proof of concept are there. On Jul 15, 2009, at 8:02 PM, Roland Dreier (rdreier) wrote: > OK, here's the latest version of my ummunot patch. This > incorporates a > couple of bug fixes in how events are reported, which could have > caused > incorrect data to be reported or events to be missed. The current > version seems reasonably solid, and I expect to send this to the > kernel > community for review in a week or so, with an eye to getting this > merged > for 2.6.32. If you have any feedback or concerns, now would be a good > time to send them. If this isn't going to work for you, please let me > know and we can go back to the drawing board! > > === > > ummunot: Userspace support for MMU notifications > > As discussed in > > and follow-up messages, libraries using RDMA would like to track > precisely when application code changes memory mapping via free(), > munmap(), etc. Current pure-userspace solutions using malloc hooks > and other tricks are not robust, and the feeling among experts is that > the issue is unfixable without kernel help. > > We solve this not by implementing the full API proposed in the email > linked above but rather with a simpler and more generic interface, > which may be useful in other contexts. Specifically, we implement a > new character device driver, ummunot, that creates a /dev/ummunot > node. A userspace process can open this node read-only and use the fd > as follows: > > 1. ioctl() to register/unregister an address range to watch in the > kernel (cf struct ummunot_register_ioctl in ). > > 2. read() to retrieve events generated when a mapping in a watched > address range is invalidated (cf struct ummunot_event in > ). select()/poll()/epoll() and SIGIO are handled > for this IO. > > 3. mmap() one page at offset 0 to map a kernel page that contains a > generation counter that is incremented each time an event is > generated. This allows userspace to have a fast path that checks > that no events have occurred without a system call. > > Thanks to Jason Gunthorpe for > suggestions on the interface design. Also thanks to Jeff Squyres > for prototyping support for this in Open MPI, > which > helped find several bugs during development. > > Signed-off-by: Roland Dreier > --- > drivers/char/Kconfig | 12 ++ > drivers/char/Makefile | 1 + > drivers/char/ummunot.c | 477 +++++++++++++++++++++++++++++++++++++ > ++++++++++ > include/linux/ummunot.h | 85 +++++++++ > 4 files changed, 575 insertions(+), 0 deletions(-) > create mode 100644 drivers/char/ummunot.c > create mode 100644 include/linux/ummunot.h > > diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig > index 6a06913..8d3dd9d 100644 > --- a/drivers/char/Kconfig > +++ b/drivers/char/Kconfig > @@ -1108,6 +1108,18 @@ config DEVPORT > depends on ISA || PCI > default y > > +config UMMUNOT > + tristate "Userspace MMU notifications" > + select MMU_NOTIFIER > + help > + The ummunot (userspace MMU notification) driver creates a > + character device that can be used by userspace libraries to > + get notifications when an application's memory mapping > + changed. This is used, for example, by RDMA libraries to > + improve the reliability of memory registration caching, > since > + the kernel's MMU notifications can be used to know precisely > + when to shoot down a cached registration. > + > source "drivers/s390/char/Kconfig" > > endmenu > diff --git a/drivers/char/Makefile b/drivers/char/Makefile > index 66f779a..39ff0c3 100644 > --- a/drivers/char/Makefile > +++ b/drivers/char/Makefile > @@ -97,6 +97,7 @@ obj-$(CONFIG_NSC_GPIO) += nsc_gpio.o > obj-$(CONFIG_CS5535_GPIO) += cs5535_gpio.o > obj-$(CONFIG_GPIO_TB0219) += tb0219.o > obj-$(CONFIG_TELCLOCK) += tlclk.o > +obj-$(CONFIG_UMMUNOT) += ummunot.o > > obj-$(CONFIG_MWAVE) += mwave/ > obj-$(CONFIG_AGP) += agp/ > diff --git a/drivers/char/ummunot.c b/drivers/char/ummunot.c > new file mode 100644 > index 0000000..52a0b6d > --- /dev/null > +++ b/drivers/char/ummunot.c > @@ -0,0 +1,477 @@ > +/* > + * Copyright (c) 2009 Cisco Systems. All rights reserved. > + * > + * This software is available to you under a choice of one of two > + * licenses. You may choose to be licensed under the terms of the > GNU > + * General Public License (GPL) Version 2, available from the file > + * COPYING in the main directory of this source tree, or the > + * OpenFabrics BSD license below: > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the > following > + * conditions are met: > + * > + * - Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the following > + * disclaimer. > + * > + * - Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, > + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF > + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND > + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT > HOLDERS > + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN > + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN > + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE > + * SOFTWARE. > + */ > + > +#include > +#include > +#include > +#include > +#include > +#include > +#include > +#include > +#include > +#include > +#include > +#include > +#include > + > +#include > + > +MODULE_AUTHOR("Roland Dreier"); > +MODULE_DESCRIPTION("Userspace MMU notifiers"); > +MODULE_LICENSE("Dual BSD/GPL"); > + > +enum { > + UMMUNOT_FLAG_DIRTY = 1, > + UMMUNOT_FLAG_HINT = 2, > +}; > + > +struct ummunot_reg { > + u64 user_cookie; > + unsigned long start; > + unsigned long end; > + unsigned long hint_start; > + unsigned long hint_end; > + unsigned long flags; > + struct rb_node node; > + struct list_head list; > +}; > + > +struct ummunot_file { > + struct mmu_notifier mmu_notifier; > + struct mm_struct *mm; > + struct rb_root reg_tree; > + struct list_head dirty_list; > + u64 *counter; > + spinlock_t lock; > + wait_queue_head_t read_wait; > + struct fasync_struct *async_queue; > +}; > + > +static struct ummunot_file *to_ummunot_file(struct mmu_notifier *mn) > +{ > + return container_of(mn, struct ummunot_file, mmu_notifier); > +} > + > +static void ummunot_handle_not(struct mmu_notifier *mn, > + unsigned long start, unsigned long end) > +{ > + struct ummunot_file *priv = to_ummunot_file(mn); > + struct rb_node *n; > + struct ummunot_reg *reg; > + unsigned long flags; > + int hit = 0; > + > + spin_lock_irqsave(&priv->lock, flags); > + > + for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) { > + reg = rb_entry(n, struct ummunot_reg, node); > + > + if (reg->start >= end) > + break; > + > + /* > + * Ranges overlap if they're not disjoint; and they're > + * disjoint if the end of one is before the start of > + * the other one. > + */ > + if (!(reg->end <= start || end <= reg->start)) { > + hit = 1; > + > + if (!test_and_set_bit(UMMUNOT_FLAG_DIRTY, > ®->flags)) > + list_add_tail(®->list, &priv- > >dirty_list); > + > + if (test_bit(UMMUNOT_FLAG_HINT, ®- > >flags)) { > + clear_bit(UMMUNOT_FLAG_HINT, ®- > >flags); > + } else { > + set_bit(UMMUNOT_FLAG_HINT, ®- > >flags); > + reg->hint_start = start; > + reg->hint_end = end; > + } > + } > + } > + > + if (hit) { > + ++(*priv->counter); > + flush_dcache_page(virt_to_page(priv->counter)); > + wake_up_interruptible(&priv->read_wait); > + kill_fasync(&priv->async_queue, SIGIO, POLL_IN); > + } > + > + spin_unlock_irqrestore(&priv->lock, flags); > +} > + > +static void ummunot_inval_page(struct mmu_notifier *mn, > + struct mm_struct *mm, > + unsigned long addr) > +{ > + ummunot_handle_not(mn, addr, addr + PAGE_SIZE); > +} > + > +static void ummunot_inval_range_start(struct mmu_notifier *mn, > + struct mm_struct *mm, > + unsigned long start, unsigned > long end) > +{ > + ummunot_handle_not(mn, start, end); > +} > + > +static const struct mmu_notifier_ops ummunot_mmu_notifier_ops = { > + .invalidate_page = ummunot_inval_page, > + .invalidate_range_start = ummunot_inval_range_start, > +}; > + > +static int ummunot_open(struct inode *inode, struct file *filp) > +{ > + struct ummunot_file *priv; > + int ret; > + > + if (filp->f_mode & FMODE_WRITE) > + return -EINVAL; > + > + priv = kmalloc(sizeof *priv, GFP_KERNEL); > + if (!priv) > + return -ENOMEM; > + > + priv->counter = (void *) get_zeroed_page(GFP_KERNEL); > + if (!priv->counter) { > + ret = -ENOMEM; > + goto err; > + } > + > + priv->reg_tree = RB_ROOT; > + INIT_LIST_HEAD(&priv->dirty_list); > + spin_lock_init(&priv->lock); > + init_waitqueue_head(&priv->read_wait); > + priv->async_queue = NULL; > + > + priv->mmu_notifier.ops = &ummunot_mmu_notifier_ops; > + /* > + * Register notifier last, since notifications can occur as > + * soon as we register.... > + */ > + ret = mmu_notifier_register(&priv->mmu_notifier, current->mm); > + if (ret) > + goto err_page; > + > + priv->mm = current->mm; > + atomic_inc(&priv->mm->mm_count); > + > + filp->private_data = priv; > + > + return 0; > + > +err_page: > + free_page((unsigned long) priv->counter); > + > +err: > + kfree(priv); > + return ret; > +} > + > +static int ummunot_close(struct inode *inode, struct file *filp) > +{ > + struct ummunot_file *priv = filp->private_data; > + struct rb_node *n; > + struct ummunot_reg *reg; > + > + mmu_notifier_unregister(&priv->mmu_notifier, priv->mm); > + mmdrop(priv->mm); > + free_page((unsigned long) priv->counter); > + > + for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) { > + reg = rb_entry(n, struct ummunot_reg, node); > + kfree(reg); > + } > + > + kfree(priv); > + > + return 0; > +} > + > +static ssize_t ummunot_read(struct file *filp, char __user *buf, > + size_t count, loff_t *pos) > +{ > + struct ummunot_file *priv = filp->private_data; > + struct ummunot_reg *reg; > + ssize_t ret; > + struct ummunot_event *events; > + int max; > + int n; > + > + events = (void *) get_zeroed_page(GFP_KERNEL); > + if (!events) { > + ret = -ENOMEM; > + goto out; > + } > + > + spin_lock_irq(&priv->lock); > + > + while (list_empty(&priv->dirty_list)) { > + spin_unlock_irq(&priv->lock); > + > + if (filp->f_flags & O_NONBLOCK) { > + ret = -EAGAIN; > + goto out; > + } > + > + if (wait_event_interruptible(priv->read_wait, > + !list_empty(&priv- > >dirty_list))) { > + ret = -ERESTARTSYS; > + goto out; > + } > + > + spin_lock_irq(&priv->lock); > + } > + > + max = min(PAGE_SIZE, count) / sizeof *events; > + > + for (n = 0; n < max; ++n) { > + if (list_empty(&priv->dirty_list)) { > + events[n].type = UMMUNOT_EVENT_TYPE_LAST; > + events[n].user_cookie_counter = *priv- > >counter; > + ++n; > + break; > + } > + > + reg = list_first_entry(&priv->dirty_list, struct > ummunot_reg, > + list); > + > + events[n].type = UMMUNOT_EVENT_TYPE_INVAL; > + if (test_bit(UMMUNOT_FLAG_HINT, ®->flags)) { > + events[n].flags = > UMMUNOT_EVENT_FLAG_HINT; > + events[n].hint_start = max(reg->start, > reg->hint_start); > + events[n].hint_end = min(reg->end, reg- > >hint_end); > + } else { > + events[n].hint_start = reg->start; > + events[n].hint_end = reg->end; > + } > + events[n].user_cookie_counter = reg->user_cookie; > + > + list_del(®->list); > + reg->flags = 0; > + } > + > + spin_unlock_irq(&priv->lock); > + > + if (copy_to_user(buf, events, n * sizeof *events)) > + ret = -EFAULT; > + else > + ret = n * sizeof *events; > + > +out: > + free_page((unsigned long) events); > + return ret; > +} > + > +static unsigned int ummunot_poll(struct file *filp, struct > poll_table_struct *wait) > +{ > + struct ummunot_file *priv = filp->private_data; > + > + poll_wait(filp, &priv->read_wait, wait); > + > + return list_empty(&priv->dirty_list) ? 0 : (POLLIN | > POLLRDNORM); > +} > + > +static long ummunot_register_region(struct ummunot_file *priv, > + struct ummunot_register_ioctl > __user *arg) > +{ > + struct ummunot_register_ioctl parm; > + struct ummunot_reg *reg, *treg; > + struct rb_node **n = &priv->reg_tree.rb_node; > + struct rb_node *pn; > + int ret = 0; > + > + if (copy_from_user(&parm, arg, sizeof parm)) > + return -EFAULT; > + > + if (parm.intf_version != UMMUNOT_INTF_VERSION) > + return -EINVAL; > + > + reg = kmalloc(sizeof *reg, GFP_KERNEL); > + if (!reg) > + return -ENOMEM; > + > + reg->user_cookie = parm.user_cookie; > + reg->start = parm.start; > + reg->end = parm.end; > + reg->flags = 0; > + > + spin_lock_irq(&priv->lock); > + > + for (pn = rb_first(&priv->reg_tree); pn; pn = rb_next(pn)) { > + treg = rb_entry(pn, struct ummunot_reg, node); > + > + if (treg->user_cookie == parm.user_cookie) { > + kfree(reg); > + ret = -EINVAL; > + goto out; > + } > + } > + > + pn = NULL; > + while (*n) { > + pn = *n; > + treg = rb_entry(pn, struct ummunot_reg, node); > + > + if (reg->start <= treg->start) > + n = &pn->rb_left; > + else > + n = &pn->rb_right; > + } > + > + rb_link_node(®->node, pn, n); > + rb_insert_color(®->node, &priv->reg_tree); > + > +out: > + spin_unlock_irq(&priv->lock); > + > + return ret; > +} > + > +static long ummunot_unregister_region(struct ummunot_file *priv, > + __u64 __user *arg) > +{ > + u64 user_cookie; > + struct rb_node *n; > + struct ummunot_reg *reg; > + int ret = -EINVAL; > + > + if (get_user(user_cookie, arg)) > + return -EFAULT; > + > + spin_lock_irq(&priv->lock); > + > + for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) { > + reg = rb_entry(n, struct ummunot_reg, node); > + > + if (reg->user_cookie == user_cookie) { > + rb_erase(n, &priv->reg_tree); > + if (test_bit(UMMUNOT_FLAG_DIRTY, ®->flags)) > + list_del(®->list); > + kfree(reg); > + ret = 0; > + break; > + } > + } > + > + spin_unlock_irq(&priv->lock); > + > + return ret; > +} > + > +static long ummunot_ioctl(struct file *filp, unsigned int cmd, > + unsigned long arg) > +{ > + struct ummunot_file *priv = filp->private_data; > + void __user *argp = (void __user *) arg; > + > + switch (cmd) { > + case UMMUNOT_REGISTER_REGION: > + return ummunot_register_region(priv, argp); > + case UMMUNOT_UNREGISTER_REGION: > + return ummunot_unregister_region(priv, argp); > + default: > + return -ENOIOCTLCMD; > + } > +} > + > +static int ummunot_fault(struct vm_area_struct *vma, struct > vm_fault *vmf) > +{ > + struct ummunot_file *priv = vma->vm_private_data; > + > + if (vmf->pgoff != 0) > + return VM_FAULT_SIGBUS; > + > + vmf->page = virt_to_page(priv->counter); > + get_page(vmf->page); > + > + return 0; > + > +} > + > +static struct vm_operations_struct ummunot_vm_ops = { > + .fault = ummunot_fault, > +}; > + > +static int ummunot_mmap(struct file *filp, struct vm_area_struct > *vma) > +{ > + struct ummunot_file *priv = filp->private_data; > + > + if (vma->vm_end - vma->vm_start != PAGE_SIZE || > + vma->vm_pgoff != 0) > + return -EINVAL; > + > + vma->vm_ops = &ummunot_vm_ops; > + vma->vm_private_data = priv; > + > + return 0; > +} > + > +static int ummunot_fasync(int fd, struct file *filp, int on) > +{ > + struct ummunot_file *priv = filp->private_data; > + > + return fasync_helper(fd, filp, on, &priv->async_queue); > +} > + > +static const struct file_operations ummunot_fops = { > + .owner = THIS_MODULE, > + .open = ummunot_open, > + .release = ummunot_close, > + .read = ummunot_read, > + .poll = ummunot_poll, > + .unlocked_ioctl = ummunot_ioctl, > +#ifdef CONFIG_COMPAT > + .compat_ioctl = ummunot_ioctl, > +#endif > + .mmap = ummunot_mmap, > + .fasync = ummunot_fasync, > +}; > + > +static struct miscdevice ummunot_misc = { > + .minor = MISC_DYNAMIC_MINOR, > + .name = "ummunot", > + .fops = &ummunot_fops, > +}; > + > +static int __init ummunot_init(void) > +{ > + return misc_register(&ummunot_misc); > +} > + > +static void __exit ummunot_cleanup(void) > +{ > + misc_deregister(&ummunot_misc); > +} > + > +module_init(ummunot_init); > +module_exit(ummunot_cleanup); > diff --git a/include/linux/ummunot.h b/include/linux/ummunot.h > new file mode 100644 > index 0000000..14fc75e > --- /dev/null > +++ b/include/linux/ummunot.h > @@ -0,0 +1,85 @@ > +/* > + * Copyright (c) 2009 Cisco Systems. All rights reserved. > + * > + * This software is available to you under a choice of one of two > + * licenses. You may choose to be licensed under the terms of the > GNU > + * General Public License (GPL) Version 2, available from the file > + * COPYING in the main directory of this source tree, or the > + * OpenFabrics BSD license below: > + * > + * Redistribution and use in source and binary forms, with or > + * without modification, are permitted provided that the > following > + * conditions are met: > + * > + * - Redistributions of source code must retain the above > + * copyright notice, this list of conditions and the following > + * disclaimer. > + * > + * - Redistributions in binary form must reproduce the above > + * copyright notice, this list of conditions and the following > + * disclaimer in the documentation and/or other materials > + * provided with the distribution. > + * > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, > + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF > + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND > + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT > HOLDERS > + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN > + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN > + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE > + * SOFTWARE. > + */ > + > +#ifndef _LINUX_UMMUNOT_H > +#define _LINUX_UMMUNOT_H > + > +#include > +#include > + > +#define UMMUNOT_INTF_VERSION 1 > + > +enum { > + UMMUNOT_EVENT_TYPE_INVAL = 0, > + UMMUNOT_EVENT_TYPE_LAST = 1, > +}; > + > +enum { > + UMMUNOT_EVENT_FLAG_HINT = 1 << 0, > +}; > + > +/* > + * If type field is INVAL, then user_cookie_counter holds the > + * user_cookie for the region being reported; if the HINT flag is set > + * then hint_start/hint_end hold the start and end of the mapping > that > + * was invalidated. (If HINT is not set, then multiple events > + * invalidated parts of the registered range and hint_start/hint_end > + * and set to the start/end of the whole registered range) > + * > + * If type is LAST, then the read operation has emptied the list of > + * invalidated regions, and user_cookie_counter holds the value of > the > + * kernel's generation counter when the empty list occurred. The > + * other fields are not filled in for this event. > + */ > +struct ummunot_event { > + __u32 type; > + __u32 flags; > + __u64 hint_start; > + __u64 hint_end; > + __u64 user_cookie_counter; > +}; > + > +struct ummunot_register_ioctl { > + __u32 intf_version; /* in */ > + __u32 reserved1; > + __u64 start; /* in */ > + __u64 end; /* in */ > + __u64 user_cookie; /* in */ > +}; > + > +#define UMMUNOT_MAGIC 'U' > + > +#define UMMUNOT_REGISTER_REGION _IOWR(UMMUNOT_MAGIC, > 1, \ > + struct > ummunot_register_ioctl) > +#define UMMUNOT_UNREGISTER_REGION _IOW(UMMUNOT_MAGIC, 2, __u64) > + > +#endif /* _LINUX_UMMUNOT_H */ > -- > 1.6.3.3 > > -- Jeff Squyres Cisco Systems From jgunthorpe at obsidianresearch.com Sat Jul 18 22:26:34 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Sat, 18 Jul 2009 23:26:34 -0600 Subject: [ofa-general] [PATCH ibverbs] Make the gid argument to ibv_attach_mcast and ibv_detach_mcast const Message-ID: <20090719052634.GA1149@obsidianresearch.com> This constness flows through to the driver call struct and into the drivers and back into ibv_cmd_attach_mcast/ibv_cmd_detach_mcast. Signed-off-by: Jason Gunthorpe --- include/infiniband/driver.h | 4 ++-- include/infiniband/verbs.h | 8 ++++---- man/ibv_attach_mcast.3 | 4 ++-- src/cmd.c | 4 ++-- src/verbs.c | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h index 67a3bf8..e54e0e3 100644 --- a/include/infiniband/driver.h +++ b/include/infiniband/driver.h @@ -129,8 +129,8 @@ int ibv_cmd_post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *wr, int ibv_cmd_create_ah(struct ibv_pd *pd, struct ibv_ah *ah, struct ibv_ah_attr *attr); int ibv_cmd_destroy_ah(struct ibv_ah *ah); -int ibv_cmd_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); -int ibv_cmd_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); +int ibv_cmd_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); +int ibv_cmd_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); int ibv_dontfork_range(void *base, size_t size); int ibv_dofork_range(void *base, size_t size); diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h index a04cc62..226d85e 100644 --- a/include/infiniband/verbs.h +++ b/include/infiniband/verbs.h @@ -676,9 +676,9 @@ struct ibv_context_ops { struct ibv_recv_wr **bad_wr); struct ibv_ah * (*create_ah)(struct ibv_pd *pd, struct ibv_ah_attr *attr); int (*destroy_ah)(struct ibv_ah *ah); - int (*attach_mcast)(struct ibv_qp *qp, union ibv_gid *gid, + int (*attach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); - int (*detach_mcast)(struct ibv_qp *qp, union ibv_gid *gid, + int (*detach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); void (*async_event)(struct ibv_async_event *event); }; @@ -1060,7 +1060,7 @@ int ibv_destroy_ah(struct ibv_ah *ah); * the fabric appropriately. The port associated with the specified * QP must also be a member of the multicast group. */ -int ibv_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); +int ibv_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); /** * ibv_detach_mcast - Detaches the specified QP from a multicast group. @@ -1068,7 +1068,7 @@ int ibv_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); * @gid: Multicast group GID. * @lid: Multicast group LID in host byte order. */ -int ibv_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); +int ibv_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); /** * ibv_fork_init - Prepare data structures so that fork() may be used diff --git a/man/ibv_attach_mcast.3 b/man/ibv_attach_mcast.3 index 7d83d56..722ac9f 100644 --- a/man/ibv_attach_mcast.3 +++ b/man/ibv_attach_mcast.3 @@ -8,10 +8,10 @@ ibv_attach_mcast, ibv_detach_mcast \- attach and detach a queue pair .nf .B #include .sp -.BI "int ibv_attach_mcast(struct ibv_qp " "*qp" ", union ibv_gid " "*gid" ", +.BI "int ibv_attach_mcast(struct ibv_qp " "*qp" ", const union ibv_gid " "*gid" ", .BI " uint16_t " "lid" "); .sp -.BI "int ibv_detach_mcast(struct ibv_qp " "*qp" ", union ibv_gid " "*gid" ", +.BI "int ibv_detach_mcast(struct ibv_qp " "*qp" ", const union ibv_gid " "*gid" ", .BI " uint16_t " "lid" "); .fi .SH "DESCRIPTION" diff --git a/src/cmd.c b/src/cmd.c index 66d7134..c2e76cf 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -1091,7 +1091,7 @@ int ibv_cmd_destroy_qp(struct ibv_qp *qp) return 0; } -int ibv_cmd_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) +int ibv_cmd_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) { struct ibv_attach_mcast cmd; @@ -1107,7 +1107,7 @@ int ibv_cmd_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) return 0; } -int ibv_cmd_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) +int ibv_cmd_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) { struct ibv_detach_mcast cmd; diff --git a/src/verbs.c b/src/verbs.c index 9e370ce..477e412 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -532,13 +532,13 @@ int __ibv_destroy_ah(struct ibv_ah *ah) } default_symver(__ibv_destroy_ah, ibv_destroy_ah); -int __ibv_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) +int __ibv_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) { return qp->context->ops.attach_mcast(qp, gid, lid); } default_symver(__ibv_attach_mcast, ibv_attach_mcast); -int __ibv_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) +int __ibv_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) { return qp->context->ops.detach_mcast(qp, gid, lid); } -- 1.6.0.4 From amirv at mellanox.co.il Sun Jul 19 00:25:36 2009 From: amirv at mellanox.co.il (Amir Vadai) Date: Sun, 19 Jul 2009 10:25:36 +0300 Subject: [ofa-general] SDP tuning: recv_poll? sdp_zcopy_thresh? other parameters? In-Reply-To: <20090717152556.GC8073@barkeeper1-xen.linbit> References: <20090717152556.GC8073@barkeeper1-xen.linbit> Message-ID: <4A62CA70.60503@mellanox.co.il> Lars Hi, There are many changes between SDP in ofed 1.5 and 1.4, So I need to know which version do you intend to use. Some important parameters in ofed 1.4 that effect the latency, are: 1. Nagle - If you use Nagle, first packet is sent with no delay but next packets within a period of time, are collected before being sent - this is good for BW oriented traffic but not for latency. You could set socket option TCP_NODELAY on "meta" socket. and this should disable nagle there. 2. BCopy/BZcopy - Using bzcopy is good for CPU utilization on the send side (It means sending the buffer directly from the user buffer using the ib SEND verb) but only effective for big packets. Pay attention that when using bzcopy send, send() is blocked till buffer is sent on the wire and when using regular bcopy send, the send() command is returned almost immediately (after copying buffer into driver's private buffer).bzcopy threshold is controlled by setting sdp_zcopy_thresh. 3. recv_poll - if I am not mistaken (it was added to SDP before my time), it is a poll that is done before recvmsg is going to sleep waiting for data. 4. send_poll - This is used mainly for ping pong traffic. It tells how many polls to do after a send, and if set right - will reduce number of interrupts on RX. Other stuff that influence the BW is send buffer and receive buffer. I've seen places where it changed the BW in an order of magnitude. But here it is hard to tell the exact rule since it is a bit chaotic. This was relevant mainly for ofed 1.4 - in 1.5 logic was changed in some places and more tuneables where added. In 1.5 there is /proc/net/sdpstats that can give you important information about the patterns of traffic and SDP handling. As you can see there are many tuneables and many of them are changed those days. Bye, Amir. -- Amir Vadai Software Eng. Mellanox Technologies mailto: amirv at mellanox.co.il Tel +972-3-6259539 On 07/17/2009 06:25 PM, Lars Ellenberg wrote: > > Lets say my typical usage pattern is: > > two nodes, alice and bob. > two "tcp" connections via SDP, long lifetime, > called the "data" and the "meta" socket. > > alice sends on the "data" socket, typically transfer > message sizes of 512 byte to 32 KiB, say. > > for each message received on the data socket bob then sends an "ack" > back on the "meta" socket, message sizes of ~32 byte. [*] > there are a few other messages on both sockets. > > > I'd like to have maximum throughput when streaming large messages, but > (of course) at the same time I'd like to have minimum latency for the > short messages, and when sending only single requests. obviously if the > cpu overhead can be minimized, that won't hurt either ;) > > > now, which tunables in /sys/modules/ib_sdp/parameters are those > that will most likely have some effect here? > > an "I don't now what I am tuning here, but I try anyways" approach > gave me some benefit from using > recv_poll 200 > sdp_zcopy_thresh 8192, > all else left at what the module chose itself. > > > pointers to an overview about what those tunables actually do, or any > recommendations (also for tunables in other modules, potentially, or > sysctls, tcp or other socket options or whatnot) gladly accepted. > > > [*] > this is, of course, in fact DRBD (see http://www.drbd.org), the large > messages on the "data" socket are replicated block device writes, > bob needs to submit that data to its local IO subsystem, > and only send out the "ack" messages on the "meta" socket > once the corresponding write has been signalled as completed. > > From monis at Voltaire.COM Sun Jul 19 00:57:26 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Sun, 19 Jul 2009 10:57:26 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <20090717211521.GG7320@obsidianresearch.com> References: <4A521784.5090304@Voltaire.COM> <4A5DF1C0.2060701@Voltaire.COM> <20090717211521.GG7320@obsidianresearch.com> Message-ID: <4A62D1E6.30800@Voltaire.COM> >> Is there any way userspace can inject a bogus multicast address? > > Can you do it with netlink? > > ip maddr add address ... dev ib0 > > Jason Thanks Jason. This is an example that shows it can be done from userspace. Roland, I think that the output below answers your question. Before ip maddr show ------------------------------- linux:/etc/sysconfig/network # ip m s dev ib0 9: ib0 link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:00:00:00:00 link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 inet 224.0.0.1 inet6 ff02::1:ff96:ca05 inet6 ff02::1 linux:/etc/sysconfig/network # ip maddr add 33:33:00:00:00:01 dev ib0 linux:/etc/sysconfig/network # ip m s dev ib0 Before ip maddr show ------------------------------- 9: ib0 link 33:33:00:00:00:01:00:00:00:00:00:00:00:00:00:00:00:00:00:00 static link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:00:00:00:00 link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 inet 224.0.0.1 inet6 ff02::1:ff96:ca05 inet6 ff02::1 >From dmesg (22 means EINVAL) ------------------------------- ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 From ogerlitz at voltaire.com Sun Jul 19 01:35:25 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Sun, 19 Jul 2009 11:35:25 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <20090717211521.GG7320@obsidianresearch.com> References: <4A521784.5090304@Voltaire.COM> <4A5DF1C0.2060701@Voltaire.COM> <20090717211521.GG7320@obsidianresearch.com> Message-ID: <4A62DACD.1060403@voltaire.com> Jason Gunthorpe wrote: >> Is there any way userspace can inject a bogus multicast address? >> > Can you do it with netlink? ip maddr add address ... dev ib0 > aren't the permissions needed for this being the same as for those needed to probe into the kernel a module that crashes the system? Or. From ogerlitz at voltaire.com Sun Jul 19 01:47:27 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Sun, 19 Jul 2009 11:47:27 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A62D1E6.30800@Voltaire.COM> References: <4A521784.5090304@Voltaire.COM> <4A5DF1C0.2060701@Voltaire.COM> <20090717211521.GG7320@obsidianresearch.com> <4A62D1E6.30800@Voltaire.COM> Message-ID: <4A62DD9F.3000609@voltaire.com> Moni Shoua wrote: > This is an example that shows it can be done from userspace. Roland, I think that the output below answers your question. > > > Before ip maddr show > ------------------------------- > linux:/etc/sysconfig/network # ip m s dev ib0 > 9: ib0 > link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:00:00:00:00 > link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 > link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 Note that these are all junk mgids as well, with the problem being in the iproute2 package, its been fixed since Novmeber last year (see 7f71c0cae2db61890474e04ba3a26e40219e5561 "ip maddr show” on an infiniband address causes a stack corruption") > linux:/etc/sysconfig/network # ip maddr add 33:33:00:00:00:01 dev ib0 permissions for that? Or. From vlad at lists.openfabrics.org Sun Jul 19 02:37:59 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Sun, 19 Jul 2009 02:37:59 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090719-0200 daily build status Message-ID: <20090719093759.9D0E710201A1@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -fasynchronous-unwind-tables -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_cma)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c: In function 'sdp_cma_handler': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c:312: error: implicit declaration of function 'lock_sock_nested' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -fasynchronous-unwind-tables -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_cma)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c: In function 'sdp_cma_handler': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.c:312: error: implicit declaration of function 'lock_sock_nested' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp/sdp_cma.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -fno-reorder-blocks -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -m64 -mno-red-zone -mcmodel=kernel -funit-at-a-time -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090719-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From monis at Voltaire.COM Sun Jul 19 03:21:14 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Sun, 19 Jul 2009 13:21:14 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A62DACD.1060403@voltaire.com> References: <4A521784.5090304@Voltaire.COM> <4A5DF1C0.2060701@Voltaire.COM> <20090717211521.GG7320@obsidianresearch.com> <4A62DACD.1060403@voltaire.com> Message-ID: <4A62F39A.3010609@Voltaire.COM> Or Gerlitz wrote: > Jason Gunthorpe wrote: >>> Is there any way userspace can inject a bogus multicast address? >>> >> Can you do it with netlink? ip maddr add address ... dev ib0 > aren't the permissions needed for this being the same as for those > needed to probe into the kernel a module that crashes the system? > > Or. > Generally I think you are right. But what if a non root user has root permissions only for the command 'ip'? I'd also like to add that the issue is not just a bad intension but a mistake as well (with even more chances of this to happen). From monis at Voltaire.COM Sun Jul 19 03:31:46 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Sun, 19 Jul 2009 13:31:46 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A62DD9F.3000609@voltaire.com> References: <4A521784.5090304@Voltaire.COM> <4A5DF1C0.2060701@Voltaire.COM> <20090717211521.GG7320@obsidianresearch.com> <4A62D1E6.30800@Voltaire.COM> <4A62DD9F.3000609@voltaire.com> Message-ID: <4A62F612.70901@Voltaire.COM> Or Gerlitz wrote: > Moni Shoua wrote: >> This is an example that shows it can be done from userspace. Roland, I >> think that the output below answers your question. >> >> >> Before ip maddr show >> ------------------------------- >> linux:/etc/sysconfig/network # ip m s dev ib0 >> 9: ib0 >> link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:00:00:00:00 >> link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 >> link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 > Note that these are all junk mgids as well, with the problem being in > the iproute2 package, its been fixed since Novmeber last year (see > 7f71c0cae2db61890474e04ba3a26e40219e5561 "ip maddr show” on an > infiniband address causes a stack corruption") > Thanks. I intend to look at it. >> linux:/etc/sysconfig/network # ip maddr add 33:33:00:00:00:01 dev ib0 > permissions for that? > Needs root but as I already answered I don't think that this is a reason to ignore the example. From sashak at voltaire.com Sun Jul 19 06:21:14 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sun, 19 Jul 2009 16:21:14 +0300 Subject: [ofa-general] Re: [PATCH] infiniband-diags/ibtracert.c: Fix comparison to SwitchInfo.MulticastFDBCap in switch_mclookup In-Reply-To: <20090716133157.GA24200@comcast.net> References: <20090716133157.GA24200@comcast.net> Message-ID: <20090719132114.GD22967@me> On 09:31 Thu 16 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 19 06:21:38 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sun, 19 Jul 2009 16:21:38 +0300 Subject: [ofa-general] Re: [PATCH] infiniband-diags/ibroute.c: Fix typo in IBWARN message In-Reply-To: <20090716133238.GB24200@comcast.net> References: <20090716133238.GB24200@comcast.net> Message-ID: <20090719132138.GE22967@me> On 09:32 Thu 16 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 19 06:32:28 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sun, 19 Jul 2009 16:32:28 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_perfmgr.c: In perfmgr_send_pc_mad, only set CounterSelect when Set method is used In-Reply-To: References: <20090714140427.GA29521@comcast.net> <20090715230600.GB12078@me> Message-ID: <20090719133228.GF22967@me> On 19:25 Wed 15 Jul , Hal Rosenstock wrote: > On Wed, Jul 15, 2009 at 7:06 PM, Sasha Khapyorsky wrote: > > On 10:04 Tue 14 Jul ?? ?? , Hal Rosenstock wrote: > >> > >> Signed-off-by: Hal Rosenstock > >> --- > >> diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c > >> index ecfdbda..0437d47 100644 > >> --- a/opensm/opensm/osm_perfmgr.c > >> +++ b/opensm/opensm/osm_perfmgr.c > >> @@ -376,7 +376,8 @@ static ib_api_status_t perfmgr_send_pc_mad(osm_perfmgr_t * perfmgr, > >> ?? ?? ?? port_counter = (ib_port_counters_t *) & pm_mad->data; > >> ?? ?? ?? memset(port_counter, 0, sizeof(*port_counter)); > >> ?? ?? ?? port_counter->port_select = port; > >> - ?? ?? port_counter->counter_select = 0xFFFF; > >> + ?? ?? if (mad_method == IB_MAD_METHOD_SET) > >> + ?? ?? ?? ?? ?? ?? port_counter->counter_select = 0xFFFF; > > > > Could you explain why? > > CounterSelect is only valid on a Set. Then what was wrong with an initialization on a Get? Sasha From sashak at voltaire.com Sun Jul 19 06:53:34 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sun, 19 Jul 2009 16:53:34 +0300 Subject: [ofa-general] Re: [Repost][PATCH] opensm: Added support for select counters (xmit_wait) In-Reply-To: <4A5CA3B6.9080809@morey-chaisemartin.com> References: <4A5CA3B6.9080809@morey-chaisemartin.com> Message-ID: <20090719135334.GG22967@me> Hi Nicolas, On 17:26 Tue 14 Jul , Nicolas Morey-Chaisemartin wrote: > Support for xmit_wait counters was missing in the perfmgr though it was read from the mad and event plugin interface already handles it. > This patch adds support for it (tested and working with an event plugin) > > Tested-by: Jean-Vincent Ficet > Signed-off-by: Nicolas Morey-Chaisemartin > --- > I think emails got mixed up the first time so here it is again. > > opensm/include/opensm/osm_perfmgr_db.h | 23 ++++++- > opensm/opensm/osm_perfmgr.c | 30 +++++++- > opensm/opensm/osm_perfmgr_db.c | 124 +++++++++++++++++++++++++++++-- > 3 files changed, 166 insertions(+), 11 deletions(-) > > diff --git a/opensm/include/opensm/osm_perfmgr_db.h b/opensm/include/opensm/osm_perfmgr_db.h > index 42a47bd..35b5ac3 100644 > --- a/opensm/include/opensm/osm_perfmgr_db.h > +++ b/opensm/include/opensm/osm_perfmgr_db.h > @@ -109,6 +109,14 @@ typedef struct { > } perfmgr_db_data_cnt_reading_t; > > /** ========================================================================= > + * Port select count reading > + */ > +typedef struct { > + uint64_t xmit_wait; > + time_t time; > +} perfmgr_db_sel_reading_t; > + Why do we need a separate structure for this counter? Sasha > +/** ========================================================================= > * Dump output options > */ > typedef enum { > @@ -125,6 +133,8 @@ typedef struct db_port { > perfmgr_db_err_reading_t err_previous; > perfmgr_db_data_cnt_reading_t dc_total; > perfmgr_db_data_cnt_reading_t dc_previous; > + perfmgr_db_sel_reading_t ps_total; > + perfmgr_db_sel_reading_t ps_previous; > time_t last_reset; > } db_port_t; > > @@ -179,7 +189,16 @@ perfmgr_db_err_t perfmgr_db_get_prev_dc(perfmgr_db_t * db, uint64_t guid, > reading); > perfmgr_db_err_t perfmgr_db_clear_prev_dc(perfmgr_db_t * db, uint64_t guid, > uint8_t port); > - > +perfmgr_db_err_t perfmgr_db_add_ps_reading(perfmgr_db_t * db, uint64_t guid, > + uint8_t port, > + perfmgr_db_sel_reading_t * > + reading); > +perfmgr_db_err_t perfmgr_db_get_prev_ps(perfmgr_db_t * db, uint64_t guid, > + uint8_t port, > + perfmgr_db_sel_reading_t * > + reading); > +perfmgr_db_err_t perfmgr_db_clear_prev_ps(perfmgr_db_t * db, uint64_t guid, > + uint8_t port); > void perfmgr_db_clear_counters(perfmgr_db_t * db); > perfmgr_db_err_t perfmgr_db_dump(perfmgr_db_t * db, char *file, > perfmgr_db_dump_t dump_type); > @@ -196,6 +215,8 @@ void perfmgr_db_fill_data_cnt_read_pc(ib_port_counters_t * wire_read, > perfmgr_db_data_cnt_reading_t * reading); > void perfmgr_db_fill_data_cnt_read_epc(ib_port_counters_ext_t * wire_read, > perfmgr_db_data_cnt_reading_t * reading); > +void perfmgr_db_fill_sel_read(ib_port_counters_t * wire_read, > + perfmgr_db_sel_reading_t * reading); > > END_C_DECLS > > diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c > index ecfdbda..8a9eb12 100644 > --- a/opensm/opensm/osm_perfmgr.c > +++ b/opensm/opensm/osm_perfmgr.c > @@ -853,10 +853,12 @@ void osm_perfmgr_destroy(osm_perfmgr_t * pm) > static void perfmgr_check_oob_clear(osm_perfmgr_t * pm, > monitored_node_t * mon_node, uint8_t port, > perfmgr_db_err_reading_t * cr, > - perfmgr_db_data_cnt_reading_t * dc) > + perfmgr_db_data_cnt_reading_t * dc, > + perfmgr_db_sel_reading_t * ps) > { > perfmgr_db_err_reading_t prev_err; > perfmgr_db_data_cnt_reading_t prev_dc; > + perfmgr_db_sel_reading_t prev_ps; > > if (perfmgr_db_get_prev_err(pm->db, mon_node->guid, port, &prev_err) > != PERFMGR_EVENT_DB_SUCCESS) { > @@ -905,6 +907,23 @@ static void perfmgr_check_oob_clear(osm_perfmgr_t * pm, > mon_node->name, mon_node->guid, port); > perfmgr_db_clear_prev_dc(pm->db, mon_node->guid, port); > } > + > + if (perfmgr_db_get_prev_ps(pm->db, mon_node->guid, port, &prev_ps) > + != PERFMGR_EVENT_DB_SUCCESS) { > + OSM_LOG(pm->log, OSM_LOG_VERBOSE, > + "Failed to find previous select count " > + "reading for %s (0x%" PRIx64 ") port %u\n", > + mon_node->name, mon_node->guid, port); > + return; > + } > + > + if (ps->xmit_wait < prev_ps.xmit_wait) { > + OSM_LOG(pm->log, OSM_LOG_ERROR, > + "PerfMgr: ERR 4C17: Detected an out of band select counter " > + "clear on node %s (0x%" PRIx64 ") port %u\n", > + mon_node->name, mon_node->guid, port); > + perfmgr_db_clear_prev_ps(pm->db, mon_node->guid, port); > + } > } > > /********************************************************************** > @@ -1062,6 +1081,8 @@ static void pc_recv_process(void *context, void *data) > uint8_t port = mad_context->perfmgr_context.port; > perfmgr_db_err_reading_t err_reading; > perfmgr_db_data_cnt_reading_t data_reading; > + perfmgr_db_sel_reading_t select_reading; > + > cl_map_item_t *p_node; > monitored_node_t *p_mon_node; > > @@ -1148,10 +1169,12 @@ static void pc_recv_process(void *context, void *data) > */ > perfmgr_db_fill_data_cnt_read_pc(wire_read, &data_reading); > > + perfmgr_db_fill_sel_read(wire_read, &select_reading); > + > /* detect an out of band clear on the port */ > if (mad_context->perfmgr_context.mad_method != IB_MAD_METHOD_SET) > perfmgr_check_oob_clear(pm, p_mon_node, port, &err_reading, > - &data_reading); > + &data_reading, &select_reading); > > /* log any critical events from this reading */ > perfmgr_log_events(pm, p_mon_node, port, &err_reading); > @@ -1161,9 +1184,12 @@ static void pc_recv_process(void *context, void *data) > &err_reading); > perfmgr_db_add_dc_reading(pm->db, node_guid, port, > &data_reading); > + perfmgr_db_add_ps_reading(pm->db, node_guid, port, > + &select_reading); > } else { > perfmgr_db_clear_prev_err(pm->db, node_guid, port); > perfmgr_db_clear_prev_dc(pm->db, node_guid, port); > + perfmgr_db_clear_prev_ps(pm->db, node_guid, port); > } > > perfmgr_check_overflow(pm, p_mon_node, port, wire_read); > diff --git a/opensm/opensm/osm_perfmgr_db.c b/opensm/opensm/osm_perfmgr_db.c > index e5dfc19..132c2fb 100644 > --- a/opensm/opensm/osm_perfmgr_db.c > +++ b/opensm/opensm/osm_perfmgr_db.c > @@ -486,6 +486,102 @@ Exit: > return (rc); > } > > +static inline void > +debug_dump_ps_reading(perfmgr_db_t * db, uint64_t guid, uint8_t port_num, > + db_port_t * port, perfmgr_db_sel_reading_t * cur) > +{ > + osm_log_t *log = db->perfmgr->log; > + if (!osm_log_is_active(log, OSM_LOG_DEBUG)) > + return; > + > + osm_log(log, OSM_LOG_DEBUG, > + "xd %" PRIu64 " <-- %" PRIu64 " (%" PRIu64 ")\n", > + cur->xmit_wait, port->ps_previous.xmit_wait, > + port->ps_total.xmit_wait); > +} > + > +/********************************************************************** > + * perfmgr_db_sel_reading_t functions > + **********************************************************************/ > +perfmgr_db_err_t > +perfmgr_db_add_ps_reading(perfmgr_db_t * db, uint64_t guid, uint8_t port, > + perfmgr_db_sel_reading_t * reading) > +{ > + db_port_t *p_port = NULL; > + db_node_t *node = NULL; > + perfmgr_db_sel_reading_t *previous = NULL; > + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; > + osm_epi_ps_event_t epi_ps_data; > + > + cl_plock_excl_acquire(&db->lock); > + node = get(db, guid); > + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) > + goto Exit; > + > + p_port = &node->ports[port]; > + previous = &node->ports[port].ps_previous; > + > + debug_dump_ps_reading(db, guid, port, p_port, reading); > + > + epi_ps_data.time_diff_s = reading->time - previous->time; > + osm_epi_create_port_id(&epi_ps_data.port_id, guid, port, > + node->node_name); > + > + /* calculate changes from previous reading */ > + epi_ps_data.xmit_wait = reading->xmit_wait - previous->xmit_wait; > + p_port->ps_total.xmit_wait += epi_ps_data.xmit_wait; > + > + p_port->ps_previous = *reading; > + osm_opensm_report_event(db->perfmgr->osm, > + OSM_EVENT_ID_PORT_SELECT, &epi_ps_data); > + > +Exit: > + cl_plock_release(&db->lock); > + return (rc); > +} > + > +perfmgr_db_err_t perfmgr_db_get_prev_ps(perfmgr_db_t * db, uint64_t guid, > + uint8_t port, > + perfmgr_db_sel_reading_t * reading) > +{ > + db_node_t *node = NULL; > + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; > + > + cl_plock_acquire(&db->lock); > + > + node = get(db, guid); > + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) > + goto Exit; > + > + *reading = node->ports[port].ps_previous; > + > +Exit: > + cl_plock_release(&db->lock); > + return (rc); > +} > + > +perfmgr_db_err_t > +perfmgr_db_clear_prev_ps(perfmgr_db_t * db, uint64_t guid, uint8_t port) > +{ > + db_node_t *node = NULL; > + perfmgr_db_sel_reading_t *previous = NULL; > + perfmgr_db_err_t rc = PERFMGR_EVENT_DB_SUCCESS; > + > + cl_plock_excl_acquire(&db->lock); > + node = get(db, guid); > + if ((rc = bad_node_port(node, port)) != PERFMGR_EVENT_DB_SUCCESS) > + goto Exit; > + > + previous = &node->ports[port].ps_previous; > + > + memset(previous, 0, sizeof(*previous)); > + node->ports[port].ps_previous.time = time(NULL); > + > +Exit: > + cl_plock_release(&db->lock); > + return (rc); > +} > + > static void clear_counters(cl_map_item_t * const p_map_item, void *context) > { > db_node_t *node = (db_node_t *) p_map_item; > @@ -517,6 +613,8 @@ static void clear_counters(cl_map_item_t * const p_map_item, void *context) > node->ports[i].dc_total.multicast_rcv_pkts = 0; > node->ports[i].dc_total.time = ts; > > + node->ports[i].ps_total.xmit_wait = 0; > + > node->ports[i].last_reset = ts; > } > } > @@ -546,7 +644,7 @@ static void dump_node_mr(db_node_t * node, FILE * fp) > "%s\t%s\t" > "%s\t%s\t%s\t%s\t%s\t%s\t%s\t" > "%s\t%s\t%s\t%s\t%s\t%s\t%s\t" > - "%s\t%s\t%s\t%s\n", > + "%s\t%s\t%s\t%s\t%s\n", > "symbol_err_cnt", > "link_err_recover", > "link_downed", > @@ -565,8 +663,7 @@ static void dump_node_mr(db_node_t * node, FILE * fp) > "rcv_pkts", > "unicast_xmit_pkts", > "unicast_rcv_pkts", > - "multicast_xmit_pkts", > - "multicast_rcv_pkts"); > + "multicast_xmit_pkts", "multicast_rcv_pkts", "xmit_wait"); > for (i = (node->esp0) ? 0 : 1; i < node->num_ports; i++) { > char *since = ctime(&node->ports[i].last_reset); > since[strlen(since) - 1] = '\0'; /* remove \n */ > @@ -577,8 +674,8 @@ static void dump_node_mr(db_node_t * node, FILE * fp) > "%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 > "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 > "\t%" PRIu64 "\t%" PRIu64 "\t" "%" PRIu64 "\t%" PRIu64 > - "\t%" PRIu64 "\t%" PRIu64 "\n", node->node_name, > - node->node_guid, i, since, > + "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\n", > + node->node_name, node->node_guid, i, since, > node->ports[i].err_total.symbol_err_cnt, > node->ports[i].err_total.link_err_recover, > node->ports[i].err_total.link_downed, > @@ -598,7 +695,8 @@ static void dump_node_mr(db_node_t * node, FILE * fp) > node->ports[i].dc_total.unicast_xmit_pkts, > node->ports[i].dc_total.unicast_rcv_pkts, > node->ports[i].dc_total.multicast_xmit_pkts, > - node->ports[i].dc_total.multicast_rcv_pkts); > + node->ports[i].dc_total.multicast_rcv_pkts, > + node->ports[i].ps_total.xmit_wait); > } > } > > @@ -634,7 +732,8 @@ static void dump_node_hr(db_node_t * node, FILE * fp) > " unicast_xmit_pkts : %" PRIu64 "\n" > " unicast_rcv_pkts : %" PRIu64 "\n" > " multicast_xmit_pkts : %" PRIu64 "\n" > - " multicast_rcv_pkts : %" PRIu64 "\n", > + " multicast_rcv_pkts : %" PRIu64 "\n" > + " xmit_wait : %" PRIu64 "\n", > node->node_name, > node->node_guid, > i, > @@ -658,7 +757,8 @@ static void dump_node_hr(db_node_t * node, FILE * fp) > node->ports[i].dc_total.unicast_xmit_pkts, > node->ports[i].dc_total.unicast_rcv_pkts, > node->ports[i].dc_total.multicast_xmit_pkts, > - node->ports[i].dc_total.multicast_rcv_pkts); > + node->ports[i].dc_total.multicast_rcv_pkts, > + node->ports[i].ps_total.xmit_wait); > } > } > > @@ -809,4 +909,12 @@ perfmgr_db_fill_data_cnt_read_epc(ib_port_counters_ext_t * wire_read, > reading->multicast_rcv_pkts = cl_ntoh64(wire_read->multicast_rcv_pkts); > reading->time = time(NULL); > } > + > +void > +perfmgr_db_fill_sel_read(ib_port_counters_t * wire_read, > + perfmgr_db_sel_reading_t * reading) > +{ > + reading->xmit_wait = cl_ntoh32(wire_read->xmit_wait); > + reading->time = time(NULL); > +} > #endif /* ENABLE_OSM_PERF_MGR */ > From sashak at voltaire.com Sun Jul 19 06:58:05 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sun, 19 Jul 2009 16:58:05 +0300 Subject: [ofa-general] What is the purpose of libosmvendor? In-Reply-To: <4C2744E8AD2982428C5BFE523DF8CDCB453E3F28EC@MNEXMB1.qlogic.org> References: <4C2744E8AD2982428C5BFE523DF8CDCB453E3F28EC@MNEXMB1.qlogic.org> Message-ID: <20090719135805.GH22967@me> On 09:54 Tue 14 Jul , Mike Heinz wrote: > Reviewing the OFED stack, it appears that the appropriate method for user applications to access fabric information such as path records and port guids - but the name of the library itself leaves me a little nonplussed, rather than access to fabric information, it sounds like it's meant to provide a proprietary interface to the SM. > > Is this correct? This was the case some days. Now it is mostly legacy stuff. OpenSM uses ibumad vendor layer and WinOF still use AL. If win guys will switch to ibumad (WinOF has libibumad implementation already). This layer could be removed. Sasha From sashak at voltaire.com Sun Jul 19 07:09:02 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sun, 19 Jul 2009 17:09:02 +0300 Subject: [ofa-general] Re: [infiniband-diags] [PATCH] [1/5] libibnetdisc cleanup patches In-Reply-To: <1247005466.4730.39.camel@auk31.llnl.gov> References: <1247005466.4730.39.camel@auk31.llnl.gov> Message-ID: <20090719140902.GJ22967@me> On 15:24 Tue 07 Jul , Al Chu wrote: > Make api more consistent by removing convenience pointer from > ibnd_node_t and requiring ibnd_fabric_t passed to all functions (in this > case ibnd_update_node). > > Al > > -- > Albert Chu > chu11 at llnl.gov > Computer Scientist > High Performance Systems Division > Lawrence Livermore National Laboratory > From a6f7c6bea7f3549bb0bd24a4304f9dc3aa4b73ac Mon Sep 17 00:00:00 2001 Al, Please don't include this line ("^From ") to patch body. Such line is used in mbox format as message separator and as result breaks many git tools which refer to mbox format (such as 'git rebase'). Finally I need to fix this by hand for each patch. > From: Albert Chu > Date: Tue, 7 Jul 2009 09:57:26 -0700 > Subject: [PATCH] Make api more consistent by removing convenience pointer from ibnd_node_t and requiring ibnd_fabric_t passed to all functions (in this case ibnd_update_node). > > Signed-off-by: Albert Chu > Signed-off-by: Ira Weiny Applied. Thanks. Sasha From sashak at voltaire.com Sun Jul 19 07:12:10 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sun, 19 Jul 2009 17:12:10 +0300 Subject: [ofa-general] Re: [infiniband-diags] [PATCH] [2/5] libibnetdisc cleanup patches In-Reply-To: <1247005481.4730.40.camel@auk31.llnl.gov> References: <1247005481.4730.40.camel@auk31.llnl.gov> Message-ID: <20090719141210.GK22967@me> On 15:24 Tue 07 Jul , Al Chu wrote: > Rename internal libibnetdisc structs with _int suffix to differentiate > code more cleanly against public structs. Frankly I don't see any good reason to have internal structure at all. I would better to convert everything to use a single ones. Sasha From sashak at voltaire.com Sun Jul 19 08:14:23 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sun, 19 Jul 2009 18:14:23 +0300 Subject: [ofa-general] Re: [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: <4A4747B2.2030907@Voltaire.COM> References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM> Message-ID: <20090719151423.GL22967@me> Hi Slava, On 13:36 Sun 28 Jun , Slava Strebkov wrote: > [PATCH 1/4] Patch implements multicast multiplexing as proposed in the > thread entitled "IPv6 and IPoIB scalability issue". > This first patch contains definitions for new data types > and infrastructure functions. > Signed-off-by: Slava Strebkov It looks for me that this patch defines random non-used yet structures and functions from the follow stuff, it makes it really hard to review without seeing what purpose of this. I would suggest to prepare this patch series in functional order - one thing per patch. For example using fleximap for mgid resolution can be completely isolated, and mgid to mlid compression algorithm (which still be discussable) of course too. I will comment some technical things over patches. > > --- > opensm/include/opensm/osm_multicast.h | 94 +++++++++++++++++++++++++++++++-- > opensm/opensm/osm_multicast.c | 82 ++++++++++++++++++++++++++++- > 2 files changed, 170 insertions(+), 6 deletions(-) > > diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h > index a871306..02b63bd 100644 > --- a/opensm/include/opensm/osm_multicast.h > +++ b/opensm/include/opensm/osm_multicast.h > @@ -1,5 +1,5 @@ > /* > - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. > + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. > * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. > * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. > * > @@ -45,6 +45,7 @@ > > #include > #include > +#include > #include > #include > #include > @@ -93,7 +94,7 @@ BEGIN_C_DECLS > * > * SYNOPSIS > */ > -typedef struct osm_mcast_mgr_ctxt { > + typedef struct osm_mcast_mgr_ctxt { In please don't mix formatting and functional changes in one patch. In particular this change looks like indent bug :) > cl_list_item_t list_item; > ib_net16_t mlid; > } osm_mcast_mgr_ctxt_t; > @@ -107,6 +108,89 @@ typedef struct osm_mcast_mgr_ctxt { > * SEE ALSO > *********/ > > +/****s* OpenSM: Multicast Group Holder/osm_mgrp_holder_t > +* NAME > +* osm_mgrp_holder_t > +* > +* DESCRIPTION > +* Holder for mgroups. > +* > +* The osm_mgrp_t object should be treated as opaque and should > +* be manipulated only through the provided functions. > +* > +* SYNOPSIS > +*/ > + > +typedef struct osm_mgrp_holder { > + cl_fmap_t mgrp_map; > + cl_qmap_t mgrp_port_map; > + ib_gid_t common_mgid; > + osm_mtree_node_t *p_root; > + boolean_t to_be_deleted; > + uint32_t last_tree_id; > + uint32_t last_change_id; > + ib_net16_t mlid; > +} osm_mgrp_holder_t; > + > +/* > +* FIELDS > +* mgrp_map > +* Map for mgroups. Must be first element!! > +* > +* mgrp_port_map > +* Map of all ports joined same mlid > +* > +* common_mgid > +* mgid of mgroup, ANDed with bitmask. > +* mgid of each mgroup in mgrp_map, ANDed with bitmask, > +* see osm_mgrp_holder_prepare_common_mgid Bad formatted comment. > +* > +* p_root > +* Pointer to the root "tree node" in the single spanning tree > +* for this multicast group holder. The nodes of the tree represent > +* switches. Member ports are not represented in the tree. > +* > +* to_be_deleted > +* Since holders are deleted when there are no mgroups in. > +* > +* last_change_id > +* a counter for the number of changes applied to the group in this holder. 80 characters per line please. > +* This counter shuold be incremented on any modification > +* to the group: joining or leaving of ports. > +* > +* last_tree_id > +* the last change id used for building the current tree. > +* > +* mlid > +* mlid of current group holder > +*/ > + /****s* OpenSM: Multicast group Port /osm_mgrp_port _t > +* NAME > +* osm_mgrp_port _t > +* > +* DESCRIPTION > +* Holder for pointers to mgroups and port guid. > +* > +* > +* SYNOPSIS > +*/ > +typedef struct _osm_mgrp_port { > + cl_map_item_t guid_item; > + cl_qlist_t mgroups; > + ib_net64_t port_guid; > +} osm_mgrp_port_t; > +/* > +* FIELDS > +* guid_item > +* Map for ports. Must be first element > +* > +* mgroups > +* Map for mgroups opened by this port. > +* > +* portguid > +* guid of port representing current structure > +*/ > + > /****s* OpenSM: Multicast Group/osm_mgrp_t > * NAME > * osm_mgrp_t > @@ -355,7 +439,7 @@ static inline ib_net16_t osm_mgrp_get_mlid(IN const osm_mgrp_t * const p_mgrp) > * > * SYNOPSIS > */ > -osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t *subn, osm_log_t *log, > +osm_mcm_port_t *osm_mgrp_add_port(osm_subn_t * subn, osm_log_t * log, > IN osm_mgrp_t * const p_mgrp, > IN const ib_gid_t * const p_port_gid, > IN const uint8_t join_state, > @@ -452,8 +536,8 @@ osm_mgrp_delete_port(IN osm_subn_t * const p_subn, > * SEE ALSO > *********/ > > -int osm_mgrp_remove_port(osm_subn_t *subn, osm_log_t *log, osm_mgrp_t *mgrp, > - osm_mcm_port_t *mcm, uint8_t join_state); > +int osm_mgrp_remove_port(osm_subn_t * subn, osm_log_t * log, osm_mgrp_t * mgrp, > + osm_mcm_port_t * mcm, uint8_t join_state); > > /****f* OpenSM: Multicast Group/osm_mgrp_apply_func > * NAME > diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c > index d2733c4..ae0a818 100644 > --- a/opensm/opensm/osm_multicast.c > +++ b/opensm/opensm/osm_multicast.c > @@ -1,5 +1,5 @@ > /* > - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. > + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. > * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. > * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. > * > @@ -48,6 +48,7 @@ > #include > #include > #include > +#include > > /********************************************************************** > **********************************************************************/ > @@ -298,3 +299,82 @@ void osm_mgrp_apply_func(const osm_mgrp_t * p_mgrp, osm_mgrp_func_t p_func, > if (p_mtn) > mgrp_apply_func_sub(p_mgrp, p_mtn, p_func, context); > } > + > +/********************************************************************** > + **********************************************************************/ > +#define PREFIX_MASK_IP CL_HTON64(0xff10ffff0000ffffULL) > +#define PREFIX_SIGNATURE_IPV4 CL_HTON64(0xff10401b00000000ULL) > +#define INTERFACE_ID_IPV4 CL_HTON64(0x0000000fffffffffULL) > +#define PREFIX_SIGNATURE_IPV6 CL_HTON64(0xff10601b00000000ULL) > +#define INTERFACE_ID_ALL_NODES CL_HTON64(0x00000000ffffffffULL) > +#define INTERFACE_ID_ALL_HOSTS CL_HTON64(0x0000000000000001ULL) > +#define INTERFACE_ID_ALL_GATEWAYS CL_HTON64(0x000000000000004dULL) > +#define INTERFACE_ID_ALL_ROUTERS CL_HTON64(0x0000000000000002ULL) > +#define PREFIX_PKEY_MASK_OFF CL_HTON64(0xffffffff0000ffffULL) > +#define PREFIX_MASK CL_HTON64(0xff10ffff0000ffffULL) > +#define PREFIX_SIGNATURE CL_HTON64(0xff10601b00000000ULL) > +#define INT_ID_MASK CL_HTON64(0xfffffff1ff000000ULL) > +#define INT_ID_SIGNATURE CL_HTON64(0x00000001ff000000ULL) Please use spaces for offsetting (and tabs for indentation). Sasha From sashak at voltaire.com Sun Jul 19 08:15:27 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sun, 19 Jul 2009 18:15:27 +0300 Subject: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM> Message-ID: <20090719151527.GM22967@me> On 09:38 Thu 16 Jul , Hal Rosenstock wrote: > > +static osm_mgrp_port_t *osm_mgrp_port_new(ib_net64_t port_guid) > > +{ > > + ?? ?? ?? osm_mgrp_port_t *p_mgrp_port = > > + ?? ?? ?? ?? ?? (osm_mgrp_port_t *) malloc(sizeof(osm_mgrp_port_t)); > > + ?? ?? ?? if (!p_mgrp_port) { > > + ?? ?? ?? ?? ?? ?? ?? return NULL; > > + ?? ?? ?? } > > + ?? ?? ?? memset(p_mgrp_port, 0, sizeof(*p_mgrp_port)); > > Minor - could use calloc rather than malloc/memset here. BTW what is the clear benefit? Sasha From sashak at voltaire.com Sun Jul 19 08:50:56 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sun, 19 Jul 2009 18:50:56 +0300 Subject: [ofa-general] Re: [PATCH 2/4 v2] multicast multiplexing - implementation of new infrastructure In-Reply-To: <4A474C81.2020602@Voltaire.COM> References: <4A3A387A.2020509@Voltaire.COM> <4A474C81.2020602@Voltaire.COM> Message-ID: <20090719155056.GN22967@me> On 13:57 Sun 28 Jun , Slava Strebkov wrote: > > diff --git a/opensm/include/opensm/osm_mcm_info.h b/opensm/include/opensm/osm_mcm_info.h > index dec607f..2310d25 100644 > --- a/opensm/include/opensm/osm_mcm_info.h > +++ b/opensm/include/opensm/osm_mcm_info.h > @@ -1,5 +1,5 @@ > /* > - * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved. > + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. > * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. > * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. > * > @@ -47,6 +47,7 @@ > #include > #include > #include > +#include > > #ifdef __cplusplus > # define BEGIN_C_DECLS extern "C" { > @@ -71,7 +72,7 @@ BEGIN_C_DECLS > * > * SYNOPSIS > */ > -typedef struct osm_mcm_info { > + typedef struct osm_mcm_info { > cl_list_item_t list_item; > ib_net16_t mlid; > } osm_mcm_info_t; > @@ -131,6 +132,34 @@ void osm_mcm_info_delete(IN osm_mcm_info_t * const p_mcm); > * > * SEE ALSO > *********/ > - > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_get_mlid_by_mgid > +* NAME > +* osm_mgrp_holder_get_mlid_by_mgid > +* > +* DESCRIPTION > +* Searches holder which contains mgroup with given mgid > +* Returns mlid of the found holder > +* > +* SYNOPSIS > +*/ > +ib_net16_t osm_mgrp_holder_get_mlid_by_mgid(IN osm_sa_t * sa, > + IN const ib_gid_t * const p_mgid); Why *sa and not *subn? And why should it be in osm_mcm_info.h (which just describes *port*'s mcm related list)? > +/* > +* PARAMETERS > +* > +* p_sa > +* [in] Pointer to sa object > +* > +* p_mgid > +* [in] pointer to mgid > +* > +* RETURN VALUES > +* mlid of found holder, or zero. > +* > +* NOTES > +* > +* SEE ALSO > +* > +*********/ > END_C_DECLS > #endif /* _OSM_MCM_INFO_H_ */ > diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h > index 02b63bd..9f0cd96 100644 > --- a/opensm/include/opensm/osm_multicast.h > +++ b/opensm/include/opensm/osm_multicast.h > @@ -214,6 +214,8 @@ typedef struct osm_mgrp { > uint32_t last_change_id; > uint32_t last_tree_id; > unsigned full_members; > + cl_fmap_item_t mgid_item; > + cl_list_item_t mgrp_item; > } osm_mgrp_t; > /* > * FIELDS > @@ -254,6 +256,12 @@ typedef struct osm_mgrp { > * last_tree_id > * the last change id used for building the current tree. > * > +* mgid_item > +* fleximap item for map linkage, sorted by mgid. > +* > +* mgrp_item > +* list item for mgroups pointer list. > +* > * SEE ALSO > *********/ > > @@ -572,6 +580,295 @@ osm_mgrp_apply_func(const osm_mgrp_t * const p_mgrp, > * SEE ALSO > * Multicast Group > *********/ > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_new > +* NAME > +* osm_mgrp_holder_new > +* > +* DESCRIPTION > +* Allocates and initializes a Multicast Group Holder for use. > +* > +* SYNOPSIS > +*/ > +osm_mgrp_holder_t *osm_mgrp_holder_new(IN osm_subn_t * p_subn, > + IN ib_gid_t * p_mgid, > + IN ib_net16_t mlid); > +/* > +* PARAMETERS > +* p_subn > +* (in) pointer to osm_subnet > +* p_mgid > +* (in) pointer to mgid What is "mgid" there? "Common" or "real"? > +* mlid > +* [in] Multicast LID for this multicast group holder. > +* > +* RETURN VALUES > +* pointer to initialized osm_mgrp_holder_t > +* or NULL, if unsuccessful > +* > +* SEE ALSO > +* Multicast Group Holder, osm_mgrp_holder_delete > +*********/ > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_delete > +* NAME > +* osm_mgrp_holder_delete > +* > +* DESCRIPTION > +* Removes entry from array of holders > +* Removes port from mgroup port list > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_delete(IN osm_subn_t * p_subn, IN ib_net16_t mlid); > +/* > +* PARAMETERS > +* > +* p_subn > +* [in] Pointer to osm_subnet > +* > +* mlid > +* [in] holder's mlid > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* > +* SEE ALSO > +* > +*********/ > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_add_mgrp_port > +* NAME > +* osm_mgrp_holder_add_mgrp_port > +* > +* DESCRIPTION > +* Allocates osm_mgrp_port_t for new port joined to mgroup with mlid of this holder, > +* (or / and) adds mgroup to mgroup map of existed osm_mgrp_port_t object. > +* > +* SYNOPSIS > +*/ > +ib_api_status_t osm_mgrp_holder_add_mgrp_port(IN osm_mgrp_holder_t * > + p_mgrp_holder, > + IN osm_mgrp_t * p_mgrp, > + IN ib_net64_t port_guid); Why do you need both mgrp_holder and mgrp? > +/* > +* PARAMETERS > +* p_mgrp_holder > +* (in) pointer to osm_mgrp_holder_t > +* p_mgrp > +* (in) pointer to osm_mgrp_t > +* > +* RETURN VALUES > +* IB_SUCCESS or > +* IB_INSUFFICIENT_MEMORY > +* > +* SEE ALSO > +* Multicast Group Holder, osm_mgrp_holder_delete_mgrp_port > +*********/ > + > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_delete_mgrp_port > +* NAME > +* osm_mgrp_holder_delete_mgrp_port > +* > +* DESCRIPTION > +* Deletes osm_mgrp_port_t for specified port > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_delete_mgrp_port(IN osm_mgrp_holder_t * p_mgrp_holder, > + IN osm_mgrp_t * p_mgrp, > + IN ib_net64_t port_guid); Ditto. > +/* > +* PARAMETERS > +* p_mgrp_holder > +* [in] Pointer to an osm_mgrp_holder_t object. > +* > +* p_mgrp > +* (in) Pointer to osm_mgrp_t object > +* > +* port_guid > +* [in] Port guid of the departing port. > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* > +* SEE ALSO > + Multicast Group Holder,osm_holder_add_mgrp_port > +*********/ > + > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_remove_port > +* NAME > +* osm_mgrp_holder_remove_port > +* > +* DESCRIPTION > +* Removes osm_mgrp_port_t from mgrp_port_map of holder > +* Removes port from mgroup port list > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_remove_port(IN osm_subn_t * const p_subn, > + IN osm_log_t * const p_log, > + IN osm_mgrp_holder_t * const p_mgrp_holder, > + IN const ib_net64_t port_guid); > +/* > +* PARAMETERS > +* > +* p_subn > +* [in] Pointer to the subnet object > +* > +* p_log > +* [in] The log object pointer > +* > +* p_mgrp_holder > +* [in] Pointer to an osm_mgrp_holder_t object. > +* > +* port_guid > +* [in] Port guid of the departing port. > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* > +* SEE ALSO > +* > +*********/ > + > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_add_mgrp > +* NAME > +* osm_mgrp_holder_add_mgrp > +* > +* DESCRIPTION > +* Adds mgroup to holder according to its mgid > +* > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_add_mgrp(IN osm_mgrp_holder_t * p_mgrp_holder, > + IN osm_mgrp_t * p_mgrp, > + IN osm_log_t * const p_log); > +/* > +* PARAMETERS > +* > +* p_mgrp_holder > +* [in] Pointer to an osm_mgrp_holder_t object. > +* > +* p_mgrp > +* [in] mgroup to add. > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* Updates common_mgid when holder is being reused > +* SEE ALSO > +* Multicast Group Holder,osm_mgrp_holder_delete_mgrp > +*********/ > + > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_delete_mgrp > +* NAME > +* osm_mgrp_holder_delete_mgrp > +* > +* DESCRIPTION > +* Deletes mgroup from holder according to its mgid > +* > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_delete_mgrp(IN osm_mgrp_holder_t * p_mgrp_holder, > + IN osm_mgrp_t * p_mgrp); > +/* > +* PARAMETERS > +* > +* p_mgrp_holder > +* [in] Pointer to an osm_mgrp_holder_t object. > +* > +* p_mgrp > +* [in] mgroup to delete. > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* > +* SEE ALSO > +* Multicast Group Holder,osm_mgrp_holder_add_mgrp > +*********/ > + > +/* Multicast Group Holder, osm_mgrp_holder_delete_port > +*********/ To what is this comment related to? > + > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_prepare_common_mgid > +* NAME > +* osm_mgrp_holder_prepare_common_mgid > +* > +* DESCRIPTION > +* Prepares mgid, which is common for all mgroups in this holder > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_prepare_common_mgid(IN const ib_gid_t * const p_mgid, > + OUT ib_gid_t * p_common_mgid); > +/* > +* PARAMETERS > +* > +* p_mgid > +* [in] Pointer to mgid > +* > +* p_common_mgid > +* [out] common mgid > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* > +* SEE ALSO > +* > +*********/ > +/****f* OpenSM: Subnet/osm_get_mgrp_by_mlid > +* NAME > +* osm_get_mgrp_by_mlid > +* > +* DESCRIPTION > +* The looks for the given multicast group in the subnet table by mlid. > +* NOTE: this code is not thread safe. Need to grab the lock before > +* calling it. > +* > +* SYNOPSIS > +*/ > +static inline > + struct osm_mgrp_holder *osm_get_mgrp_holder_by_mlid(osm_subn_t const > + *p_subn, > + ib_net16_t mlid) > +{ > + return p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO]; > +} > + > +/* > +* PARAMETERS > +* p_subn > +* [in] Pointer to an osm_subn_t object > +* > +* mlid > +* [in] The multicast group mlid in network order > +* > +* RETURN VALUES > +* The multicast group structure pointer if found. NULL otherwise. > +*********/ > + > +static inline ib_net16_t osm_mgrp_holder_get_mlid(IN osm_mgrp_holder_t * > + const p_mgrp_holder) > +{ > + return (p_mgrp_holder->mlid); > +} > + > +static inline boolean_t osm_mgrp_holder_is_empty(IN const osm_mgrp_holder_t * > + const p_mgrp_holder) > +{ > + return (cl_qmap_count(&p_mgrp_holder->mgrp_port_map) == 0); > +} > > END_C_DECLS > #endif /* _OSM_MULTICAST_H_ */ > diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h > index 59a32ad..6713d4d 100644 > --- a/opensm/include/opensm/osm_subnet.h > +++ b/opensm/include/opensm/osm_subnet.h > @@ -1,5 +1,5 @@ > /* > - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. > + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. > * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. > * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. > * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. > @@ -509,6 +509,7 @@ typedef struct osm_subn { > boolean_t coming_out_of_standby; > unsigned need_update; > void *mgroups[IB_LID_MCAST_END_HO - IB_LID_MCAST_START_HO + 1]; > + void *mgroup_holders[IB_LID_MCAST_END_HO - IB_LID_MCAST_START_HO + 1]; > } osm_subn_t; > /* > * FIELDS > @@ -633,6 +634,10 @@ typedef struct osm_subn { > * Array of pointers to all Multicast Group objects in the subnet. > * Indexed by MLID offset from base MLID. > * > +* mgroup_holders > +* Array of pointers to all Multicast Group Holder objects in the subnet. > +* Indexed by MLID offset from base MLID. > +* > * SEE ALSO > * Subnet object > *********/ > diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c > index ae0a818..d2a19ea 100644 > --- a/opensm/opensm/osm_multicast.c > +++ b/opensm/opensm/osm_multicast.c > @@ -44,10 +44,12 @@ > > #include > #include > +#include > #include > #include > #include > #include > +#include > #include > > /********************************************************************** > @@ -74,6 +76,31 @@ void osm_mgrp_delete(IN osm_mgrp_t * p_mgrp) > free(p_mgrp); > } > > +void osm_mgrp_delete_group(IN osm_subn_t * p_subn, IN osm_mgrp_t * p_mgrp) > +{ > + osm_mcm_port_t *p_mcm_port; > + osm_mcm_port_t *p_next_mcm_port; > + > + CL_ASSERT(p_mgrp); > + > + osm_mgrp_holder_t *p_mgrp_holder = > + osm_get_mgrp_holder_by_mlid(p_subn, p_mgrp->mlid); > + p_next_mcm_port = > + (osm_mcm_port_t *) cl_qmap_head(&p_mgrp->mcm_port_tbl); > + while (p_next_mcm_port != > + (osm_mcm_port_t *) cl_qmap_end(&p_mgrp->mcm_port_tbl)) { > + p_mcm_port = p_next_mcm_port; > + p_next_mcm_port = > + (osm_mcm_port_t *) cl_qmap_next(&p_mcm_port->map_item); > + osm_mgrp_holder_delete_mgrp_port(p_mgrp_holder, p_mgrp, > + p_mcm_port->port_gid.unicast. > + interface_id); > + osm_mcm_port_delete(p_mcm_port); > + } > + osm_mgrp_holder_delete_mgrp(p_mgrp_holder, p_mgrp); > + free(p_mgrp); > +} > + > /********************************************************************** > **********************************************************************/ > osm_mgrp_t *osm_mgrp_new(IN const ib_net16_t mlid) > @@ -378,3 +405,203 @@ static osm_mgrp_port_t *osm_mgrp_port_new(ib_net64_t port_guid) > cl_qlist_init(&p_mgrp_port->mgroups); > return p_mgrp_port; > } > + > +void osm_mgrp_holder_delete(IN osm_subn_t * p_subn, ib_net16_t mlid) > +{ > + osm_mgrp_port_t *p_osm_mgr_port; > + cl_map_item_t *p_item; > + cl_fmap_item_t *p_fitem; > + osm_mgrp_t *p_mgrp; > + > + osm_mgrp_holder_t *p_mgrp_holder = > + p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO]; > + p_item = cl_qmap_head(&p_mgrp_holder->mgrp_port_map); > + while (p_item != cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { > + p_osm_mgr_port = (osm_mgrp_port_t *) p_item; > + cl_qlist_remove_all(&p_osm_mgr_port->mgroups); > + cl_qmap_remove_item(&p_mgrp_holder->mgrp_port_map, p_item); > + p_item = cl_qmap_head(&p_mgrp_holder->mgrp_port_map); > + free(p_osm_mgr_port); > + } I'm not sure that I understand your data model very well (maybe it would be a good idea to describe and implement this as separate patch). But do you have port lists in both mgrp and mgrp_holder? Assuming so, why? This code looks pretty overloaded. > + p_fitem = cl_fmap_head(&p_mgrp_holder->mgrp_map); > + while (p_fitem != cl_fmap_end(&p_mgrp_holder->mgrp_map)) { > + p_mgrp = > + (osm_mgrp_t *) PARENT_STRUCT(p_fitem, osm_mgrp_t, > + mgid_item); > + osm_mgrp_delete_group(p_subn, p_mgrp); > + p_fitem = cl_fmap_head(&p_mgrp_holder->mgrp_map); > + } > + /* destroy the mtree_node structure */ > + osm_mtree_destroy(p_mgrp_holder->p_root); > + p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO] = NULL; > + free(p_mgrp_holder); > +} > + > +osm_mgrp_holder_t *osm_mgrp_holder_new(IN osm_subn_t * p_subn, > + ib_gid_t * p_mgid, ib_net16_t mlid) > +{ > + osm_mgrp_holder_t *p_mgrp_holder; > + p_mgrp_holder = > + p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO] = > + (osm_mgrp_holder_t *) malloc(sizeof(*p_mgrp_holder)); > + if (!p_mgrp_holder) > + return NULL; > + > + memset(p_mgrp_holder, 0, sizeof(*p_mgrp_holder)); > + p_mgrp_holder->mlid = mlid; > + cl_fmap_init(&p_mgrp_holder->mgrp_map, __mgid_cmp); > + cl_qmap_init(&p_mgrp_holder->mgrp_port_map); > + osm_mgrp_holder_prepare_common_mgid(p_mgid, > + &p_mgrp_holder->common_mgid); > + return p_mgrp_holder; > +} > + > +ib_net16_t osm_mgrp_holder_get_mlid_by_mgid(IN osm_sa_t * sa, > + IN const ib_gid_t * const p_mgid) > +{ > + int i; > + ib_gid_t common_mgid; > + osm_mgrp_holder_t *p_mgrp_holder; > + > + OSM_LOG_ENTER(sa->p_log); > + > + osm_mgrp_holder_prepare_common_mgid(p_mgid, &common_mgid); > + for (i = 0; i <= sa->p_subn->max_mcast_lid_ho - IB_LID_MCAST_START_HO; > + i++) { > + if (sa->p_subn->mgroup_holders[i]) { > + p_mgrp_holder = > + (osm_mgrp_holder_t *) sa->p_subn->mgroup_holders[i]; > + if (!memcmp > + (&p_mgrp_holder->common_mgid, &common_mgid, > + sizeof(ib_gid_t))) { > + char gid_str[INET6_ADDRSTRLEN]; > + OSM_LOG(sa->p_log, OSM_LOG_DEBUG, > + "Found holder 0x%X for MGID %s\n", > + cl_ntoh16(p_mgrp_holder->mlid), > + inet_ntop(AF_INET6, p_mgid->raw, > + gid_str, sizeof(gid_str))); > + OSM_LOG_EXIT(sa->p_log); > + return p_mgrp_holder->mlid; > + } > + } > + } Hmm, why do we need linear search her? What was a purpose of storing mgrps in fleximap? > + OSM_LOG_EXIT(sa->p_log); > + return 0; > +} > + > +void osm_mgrp_holder_remove_port(osm_subn_t * subn, osm_log_t * p_log, > + osm_mgrp_holder_t * p_mgrp_holder, > + ib_net64_t port_guid) > +{ > + osm_mgrp_t *p_mgrp; > + cl_list_item_t *p_item; > + > + OSM_LOG_ENTER(p_log); > + > + osm_mgrp_port_t *p_mgrp_port = (osm_mgrp_port_t *) > + cl_qmap_remove(&p_mgrp_holder->mgrp_port_map, port_guid); > + if (p_mgrp_port != > + (osm_mgrp_port_t *) cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { > + char gid_str[INET6_ADDRSTRLEN]; > + OSM_LOG(p_log, OSM_LOG_DEBUG, > + "port 0x%" PRIx64 " removed from mlid 0x%X\n", > + port_guid, cl_ntoh16(p_mgrp_holder->mlid)); > + while ((p_item = > + cl_qlist_remove_head(&p_mgrp_port->mgroups)) != > + cl_qlist_end(&p_mgrp_port->mgroups)) { > + p_mgrp = > + (osm_mgrp_t *) PARENT_STRUCT(p_item, osm_mgrp_t, > + mgrp_item); > + OSM_LOG(p_log, OSM_LOG_DEBUG, > + "removing mgrp mgid %s from port 0x%" PRIx64 > + "\n", inet_ntop(AF_INET6, > + p_mgrp->mcmember_rec.mgid.raw, > + gid_str, sizeof(gid_str)), > + cl_ntoh64(port_guid)); > + osm_mgrp_delete_port(subn, p_log, p_mgrp, port_guid); > + } > + free(p_mgrp_port); > + } > + OSM_LOG_EXIT(p_log); > +} > + > +void osm_mgrp_holder_add_mgrp(osm_mgrp_holder_t * p_mgrp_holder, > + osm_mgrp_t * p_mgrp, osm_log_t * p_log) > +{ > + cl_fmap_item_t *p_fitem; > + char gid_str[INET6_ADDRSTRLEN]; > + > + OSM_LOG_ENTER(p_log); > + > + if (p_mgrp_holder->to_be_deleted) { > + /* this is re-used mgrp_holder, need to update common_mgid */ > + osm_mgrp_holder_prepare_common_mgid(&p_mgrp->mcmember_rec.mgid, > + &p_mgrp_holder-> > + common_mgid); > + } > + p_fitem = cl_fmap_insert(&p_mgrp_holder->mgrp_map, > + &p_mgrp->mcmember_rec.mgid, > + &p_mgrp->mgid_item); Ok, starting to understand (hopefully :)) - you have fleximap only per mgrp_holder. Why? Wouldn't it better to have single (per subnet) mgrp fleximap? I guess this will cover all common search cases and will simplify a data model dramatically. Also (unrelated directly to this patch series). I have some local cleanup/simplification changes in multicast stuff, in particular it eliminates the need in all those 'to_be_deleted', 'tree_id', etc. flags. I think this could help you with simpler implementation. Sasha > + CL_ASSERT(p_item == &p_mgrp->mgid_item); > + > + OSM_LOG(p_log, OSM_LOG_DEBUG, > + "mgrp with MGID:%s added to holder with mlid = 0x%X\n", > + inet_ntop(AF_INET6, p_mgrp->mcmember_rec.mgid.raw, gid_str, > + sizeof(gid_str)), cl_ntoh16(p_mgrp_holder->mlid)); > + p_mgrp_holder->last_change_id++; > + OSM_LOG_EXIT(p_log); > +} > + > +ib_api_status_t osm_mgrp_holder_add_mgrp_port(osm_mgrp_holder_t * p_mgrp_holder, > + osm_mgrp_t * p_mgrp, > + ib_net64_t port_guid) > +{ > + osm_mgrp_port_t *p_mgrp_port = (osm_mgrp_port_t *) > + cl_qmap_get(&p_mgrp_holder->mgrp_port_map, port_guid); > + if (p_mgrp_port == > + (osm_mgrp_port_t *) cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { > + /* new port to mlid */ > + p_mgrp_port = osm_mgrp_port_new(port_guid); > + if (!p_mgrp_port) { > + return IB_INSUFFICIENT_MEMORY; > + } > + cl_qmap_insert(&p_mgrp_holder->mgrp_port_map, > + p_mgrp_port->port_guid, &p_mgrp_port->guid_item); > + cl_qlist_insert_tail(&p_mgrp_port->mgroups, &p_mgrp->mgrp_item); > + } else { > + cl_qlist_insert_tail(&p_mgrp_port->mgroups, &p_mgrp->mgrp_item); > + } > + return IB_SUCCESS; > +} > + > +void osm_mgrp_holder_delete_mgrp_port(osm_mgrp_holder_t * p_mgrp_holder, > + osm_mgrp_t * p_mgrp, ib_net64_t port_guid) > +{ > + osm_mgrp_port_t *p_mgrp_port = (osm_mgrp_port_t *) > + cl_qmap_get(&p_mgrp_holder->mgrp_port_map, port_guid); > + if (p_mgrp_port != > + (osm_mgrp_port_t *) cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { > + cl_qlist_remove_item(&p_mgrp_port->mgroups, &p_mgrp->mgrp_item); > + if (0 == cl_qlist_count(&p_mgrp_port->mgroups)) { > + /* No mgroups registered on this port for current mlid */ > + cl_qmap_remove_item(&p_mgrp_holder->mgrp_port_map, > + &p_mgrp_port->guid_item); > + free(p_mgrp_port); > + } > + p_mgrp_holder->last_change_id++; > + } > +} > + > +void osm_mgrp_holder_delete_mgrp(osm_mgrp_holder_t * p_mgrp_holder, > + osm_mgrp_t * p_mgrp) > +{ > + p_mgrp->to_be_deleted = 1; > + cl_fmap_remove_item(&p_mgrp_holder->mgrp_map, &p_mgrp->mgid_item); > + if (0 == cl_fmap_count(&p_mgrp_holder->mgrp_map)) { > + /* No more mgroups on this mlid */ > + p_mgrp_holder->to_be_deleted = 1; > + p_mgrp_holder->last_tree_id = 0; > + p_mgrp_holder->last_change_id = 0; > + memset(&p_mgrp_holder->common_mgid, 0, sizeof(ib_gid_t)); > + } > +} > -- > 1.5.5 > From chu11 at llnl.gov Sun Jul 19 09:24:55 2009 From: chu11 at llnl.gov (Al Chu11) Date: Sun, 19 Jul 2009 09:24:55 -0700 Subject: [ofa-general] Re: [infiniband-diags] [PATCH] [2/5] libibnetdisc cleanup patches In-Reply-To: <20090719141210.GK22967@me> References: <1247005481.4730.40.camel@auk31.llnl.gov> <20090719141210.GK22967@me> Message-ID: <1248020695.3685.5.camel@whatsup.llnl.gov> Hey Sasha, I personally agree and I (believe now) convinced Ira of it. This patch and patch 3/5 are a bit to prepare for it b/c the internal vs. external structs were not named properly/consistently. But I could do it all in one big patch instead. Al On Sun, 2009-07-19 at 17:12 +0300, Sasha Khapyorsky wrote: > On 15:24 Tue 07 Jul , Al Chu wrote: > > Rename internal libibnetdisc structs with _int suffix to differentiate > > code more cleanly against public structs. > > Frankly I don't see any good reason to have internal structure at all. > I would better to convert everything to use a single ones. > > Sasha -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory From jgunthorpe at obsidianresearch.com Sun Jul 19 09:56:04 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Sun, 19 Jul 2009 10:56:04 -0600 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A62DACD.1060403@voltaire.com> References: <4A521784.5090304@Voltaire.COM> <4A5DF1C0.2060701@Voltaire.COM> <20090717211521.GG7320@obsidianresearch.com> <4A62DACD.1060403@voltaire.com> Message-ID: <20090719165604.GA16017@obsidianresearch.com> On Sun, Jul 19, 2009 at 11:35:25AM +0300, Or Gerlitz wrote: > Jason Gunthorpe wrote: > >>Is there any way userspace can inject a bogus multicast address? > >> > >Can you do it with netlink? ip maddr add address ... dev ib0 > > > aren't the permissions needed for this being the same as for those > needed to probe into the kernel a module that crashes the system? No, you only need CAP_NET_ADMIN. Jason From devel-ofed at morey-chaisemartin.com Sun Jul 19 11:40:56 2009 From: devel-ofed at morey-chaisemartin.com (Nicolas Morey-Chaisemartin) Date: Sun, 19 Jul 2009 20:40:56 +0200 Subject: [ofa-general] Re: [Repost][PATCH] opensm: Added support for select counters (xmit_wait) In-Reply-To: <20090719135334.GG22967@me> References: <4A5CA3B6.9080809@morey-chaisemartin.com> <20090719135334.GG22967@me> Message-ID: <4A6368B8.6050007@morey-chaisemartin.com> Le 19/07/2009 15:53, Sasha Khapyorsky a écrit : > Hi Nicolas, > > On 17:26 Tue 14 Jul , Nicolas Morey-Chaisemartin wrote: >> Support for xmit_wait counters was missing in the perfmgr though it was read from the mad and event plugin interface already handles it. >> This patch adds support for it (tested and working with an event plugin) >> >> Tested-by: Jean-Vincent Ficet >> Signed-off-by: Nicolas Morey-Chaisemartin >> --- >> I think emails got mixed up the first time so here it is again. >> >> opensm/include/opensm/osm_perfmgr_db.h | 23 ++++++- >> opensm/opensm/osm_perfmgr.c | 30 +++++++- >> opensm/opensm/osm_perfmgr_db.c | 124 +++++++++++++++++++++++++++++-- >> 3 files changed, 166 insertions(+), 11 deletions(-) >> >> diff --git a/opensm/include/opensm/osm_perfmgr_db.h b/opensm/include/opensm/osm_perfmgr_db.h >> index 42a47bd..35b5ac3 100644 >> --- a/opensm/include/opensm/osm_perfmgr_db.h >> +++ b/opensm/include/opensm/osm_perfmgr_db.h >> @@ -109,6 +109,14 @@ typedef struct { >> } perfmgr_db_data_cnt_reading_t; >> >> /** ========================================================================= >> + * Port select count reading >> + */ >> +typedef struct { >> + uint64_t xmit_wait; >> + time_t time; >> +} perfmgr_db_sel_reading_t; >> + > > Why do we need a separate structure for this counter? > > Sasha > I don't think we really need one but in the event plugin interface xmit_wait counters are defined using another struct so I used a new structure to keep the whole thing homogeneous. Nicolas From sashak at voltaire.com Sun Jul 19 15:34:09 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 01:34:09 +0300 Subject: [ofa-general] Re: [Repost][PATCH] opensm: Added support for select counters (xmit_wait) In-Reply-To: <4A6368B8.6050007@morey-chaisemartin.com> References: <4A5CA3B6.9080809@morey-chaisemartin.com> <20090719135334.GG22967@me> <4A6368B8.6050007@morey-chaisemartin.com> Message-ID: <20090719223409.GR22967@me> On 20:40 Sun 19 Jul , Nicolas Morey-Chaisemartin wrote: > >> + * Port select count reading > >> + */ > >> +typedef struct { > >> + uint64_t xmit_wait; > >> + time_t time; > >> +} perfmgr_db_sel_reading_t; > >> + > > > > Why do we need a separate structure for this counter? > > > > Sasha > > > > I don't think we really need one but in the event plugin interface xmit_wait counters are defined using another struct so I used a new structure to keep the whole thing homogeneous. I guess that separate xmit wait counter in event plugin was done assuming proprietary (vendor specific) counters implemented in some switches and I think that it was done even before IBA 1.2.1 where xmit_wait counter was included in a standard set. Sasha From acceptany at gmail.com Sun Jul 19 18:09:10 2009 From: acceptany at gmail.com (Jordan) Date: Mon, 20 Jul 2009 09:09:10 +0800 Subject: [ofa-general] How to choose a path through many of them? Message-ID: <91fe68d50907191809o58f4fd3did3580af7d67b7232@mail.gmail.com> When LMC > 0 , there are 2^LMC LIDs assigned to a port. When choosing different LID, there maybe different paths to get to this port.(all these paths are valid) In other words, different LIDs represent different paths. But which path should I choose to get to the destination? It seems that OpenSM does not provide such algorithm . Does OFED provide such algorithm to manage which path should to choose ? -------------- next part -------------- An HTML attachment was scrubbed... URL: From devel-ofed at morey-chaisemartin.com Sun Jul 19 23:39:14 2009 From: devel-ofed at morey-chaisemartin.com (Nicolas Morey-Chaisemartin) Date: Mon, 20 Jul 2009 08:39:14 +0200 Subject: [ofa-general] Re: [Repost][PATCH] opensm: Added support for select counters (xmit_wait) In-Reply-To: <20090719223409.GR22967@me> References: <4A5CA3B6.9080809@morey-chaisemartin.com> <20090719135334.GG22967@me> <4A6368B8.6050007@morey-chaisemartin.com> <20090719223409.GR22967@me> Message-ID: <4A641112.3070800@morey-chaisemartin.com> Sasha Khapyorsky wrote: > On 20:40 Sun 19 Jul , Nicolas Morey-Chaisemartin wrote: > >>>> + * Port select count reading >>>> + */ >>>> +typedef struct { >>>> + uint64_t xmit_wait; >>>> + time_t time; >>>> +} perfmgr_db_sel_reading_t; >>>> + >>>> >>> Why do we need a separate structure for this counter? >>> >>> Sasha >>> >>> >> I don't think we really need one but in the event plugin interface xmit_wait counters are defined using another struct so I used a new structure to keep the whole thing homogeneous. >> > > I guess that separate xmit wait counter in event plugin was done > assuming proprietary (vendor specific) counters implemented in some > switches and I think that it was done even before IBA 1.2.1 where > xmit_wait counter was included in a standard set. > > Sasha > > > So what is best in your mind? Bring everything path into the perf counter struct (meaning all plugin will have to change to fit the new API) or keeping the approach I proposed. I'd rather choose the second solution not only because it's already done, but in the future I'd like to have as many information as possible on contention related counters (xmit_wait per VL for examples) and probably some day vendor specific info will appear in this struct. SO we're probably better off keeping it splitted from the beginning. Anyway it's your call so if you'd rather merge everything ion the existing struct, I'll rewrite the patch. Nicolas From xiaoshuangx at gmail.com Mon Jul 20 01:36:01 2009 From: xiaoshuangx at gmail.com (xiaoshuang xia) Date: Mon, 20 Jul 2009 16:36:01 +0800 Subject: [ofa-general] Some question about opensm Message-ID: Hi, I recently want to implement a routing scheme using opensm, but I face some problem. Help me pls!! THX!! The problem is 1. In the opensm, if the multi-path between src and dst is denoted by the multiple dst-lid. 2. If there are multi-path between the src and dst , what mechanism does client choose the one of them. If this mechanism is implemented by opensm, which file do this job? If choose one prefered path doesn 't implement by opensm, what unit(call it "the choosing unit") finish this job, what data structure does opensm hold to exchange the multi-path information to "the choosing unit" and what file does this exchanging job in opensm. Thank you again!!! Your helps are very important to me!!! Xiaoshuang -------------- next part -------------- An HTML attachment was scrubbed... URL: From vlad at lists.openfabrics.org Mon Jul 20 02:29:45 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Mon, 20 Jul 2009 02:29:45 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090720-0200 daily build status Message-ID: <20090720092945.28E951020470@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: include/net/ip6_route.h: In function 'ip6_dst_store': include/net/ip6_route.h:156: error: dereferencing pointer to incomplete type include/net/ip6_route.h:158: error: dereferencing pointer to incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: include/net/ip6_route.h: In function 'ip6_dst_store': include/net/ip6_route.h:177: error: dereferencing pointer to incomplete type include/net/ip6_route.h:179: error: dereferencing pointer to incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: include/net/ip6_route.h: In function 'ip6_dst_store': include/net/ip6_route.h:177: error: dereferencing pointer to incomplete type include/net/ip6_route.h:179: error: dereferencing pointer to incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: include/net/ip6_route.h: In function 'ip6_dst_store': include/net/ip6_route.h:158: error: dereferencing pointer to incomplete type include/net/ip6_route.h:160: error: dereferencing pointer to incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2092: error: 'class_device_attr_ibdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2094: error: 'class_device_attr_port' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2100: error: implicit declaration of function 'class_device_unregister' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -m64 -mno-red-zone -mcmodel=kernel -funit-at-a-time -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2092: error: 'class_device_attr_ibdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2094: error: 'class_device_attr_port' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2100: error: implicit declaration of function 'class_device_unregister' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.27_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: include/net/ip6_route.h: In function 'ip6_dst_store': include/net/ip6_route.h:158: error: dereferencing pointer to incomplete type include/net/ip6_route.h:160: error: dereferencing pointer to incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_ia64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_ia64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: include/net/ip6_route.h: In function 'ip6_dst_store': include/net/ip6_route.h:156: error: dereferencing pointer to incomplete type include/net/ip6_route.h:158: error: dereferencing pointer to incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_ia64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_ia64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: include/net/ip6_route.h: In function 'ip6_dst_store': include/net/ip6_route.h:156: error: dereferencing pointer to incomplete type include/net/ip6_route.h:158: error: dereferencing pointer to incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_ppc64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_ppc64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: include/net/ip6_route.h: In function 'ip6_dst_store': include/net/ip6_route.h:158: error: dereferencing pointer to incomplete type include/net/ip6_route.h:160: error: dereferencing pointer to incomplete type make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_ppc64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_ppc64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090720-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From ogerlitz at voltaire.com Mon Jul 20 03:03:51 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Mon, 20 Jul 2009 13:03:51 +0300 Subject: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: <39C75744D164D948A170E9792AF8E7CA01F6F919@exil.voltaire.com> References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM> <39C75744D164D948A170E9792AF8E7CA01F6F919@exil.voltaire.com> Message-ID: <4A644107.1010404@voltaire.com> Slava Strebkov wrote: > This thread covers implementation of compression multiple MGIDs to one MLID. The following groups of IP addresses will have same MLID (for all pkey): > 1. FF12401bxxxx000000000000FFFFFFFF - All Nodes > 2. FF12401bxxxx00000000000000000001 - All hosts > 3. FF12401bffff0000000000000000004d - all Gateways > 4. FF12401bxxxx00000000000000000002 - all routers > 5. FF12601bxxxx000000000001ffxxxxxx - IPv6 SNM > I think we better avoid compressing the pkey as well. Think what happens when a packet is sent to the all hosts and with your patch is routed to N partitions each one having M hosts, (N-1) * M hosts will generate invalid pkey trap! IB partitioning is a means to separate nodes, and here you attempt to enforce a violation... Or. From slavas at voltaire.com Mon Jul 20 03:07:00 2009 From: slavas at voltaire.com (Slava Strebkov) Date: Mon, 20 Jul 2009 13:07:00 +0300 Subject: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: <4A644107.1010404@voltaire.com> References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM> <39C75744D164D948A170E9792AF8E7CA01F6F919@exil.voltaire.com> <4A644107.1010404@voltaire.com> Message-ID: <39C75744D164D948A170E9792AF8E7CA01F6F921@exil.voltaire.com> -Agreed, special groups (except SNM) will not be compressed into same MLID. Slava -----Original Message----- From: Or Gerlitz Sent: Monday, July 20, 2009 1:04 PM To: Slava Strebkov Cc: Eli Dorfman; general at lists.openfabrics.org Subject: Re: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions Slava Strebkov wrote: > This thread covers implementation of compression multiple MGIDs to one MLID. The following groups of IP addresses will have same MLID (for all pkey): > 1. FF12401bxxxx000000000000FFFFFFFF - All Nodes > 2. FF12401bxxxx00000000000000000001 - All hosts > 3. FF12401bffff0000000000000000004d - all Gateways > 4. FF12401bxxxx00000000000000000002 - all routers > 5. FF12601bxxxx000000000001ffxxxxxx - IPv6 SNM > I think we better avoid compressing the pkey as well. Think what happens when a packet is sent to the all hosts and with your patch is routed to N partitions each one having M hosts, (N-1) * M hosts will generate invalid pkey trap! IB partitioning is a means to separate nodes, and here you attempt to enforce a violation... Or. From slavas at voltaire.com Mon Jul 20 03:10:58 2009 From: slavas at voltaire.com (Slava Strebkov) Date: Mon, 20 Jul 2009 13:10:58 +0300 Subject: [ofa-general] RE: [PATCH 2/4 v2] multicast multiplexing - implementation of newinfrastructure In-Reply-To: <20090719155056.GN22967@me> References: <4A3A387A.2020509@Voltaire.COM> <4A474C81.2020602@Voltaire.COM> <20090719155056.GN22967@me> Message-ID: <39C75744D164D948A170E9792AF8E7CA01F6F922@exil.voltaire.com> -----Original Message----- From: Sasha Khapyorsky [mailto:sashakvolt at gmail.com] On Behalf Of Sasha Khapyorsky Sent: Sunday, July 19, 2009 6:51 PM To: Slava Strebkov Cc: general at lists.openfabrics.org Subject: Re: [PATCH 2/4 v2] multicast multiplexing - implementation of newinfrastructure On 13:57 Sun 28 Jun , Slava Strebkov wrote: > > diff --git a/opensm/include/opensm/osm_mcm_info.h b/opensm/include/opensm/osm_mcm_info.h > index dec607f..2310d25 100644 > --- a/opensm/include/opensm/osm_mcm_info.h > +++ b/opensm/include/opensm/osm_mcm_info.h > @@ -1,5 +1,5 @@ > /* > - * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved. > + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. > * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. > * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. > * > @@ -47,6 +47,7 @@ > #include > #include > #include > +#include > > #ifdef __cplusplus > # define BEGIN_C_DECLS extern "C" { > @@ -71,7 +72,7 @@ BEGIN_C_DECLS > * > * SYNOPSIS > */ > -typedef struct osm_mcm_info { > + typedef struct osm_mcm_info { > cl_list_item_t list_item; > ib_net16_t mlid; > } osm_mcm_info_t; > @@ -131,6 +132,34 @@ void osm_mcm_info_delete(IN osm_mcm_info_t * const p_mcm); > * > * SEE ALSO > *********/ > - > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_get_mlid_by_mgid > +* NAME > +* osm_mgrp_holder_get_mlid_by_mgid > +* > +* DESCRIPTION > +* Searches holder which contains mgroup with given mgid > +* Returns mlid of the found holder > +* > +* SYNOPSIS > +*/ > +ib_net16_t osm_mgrp_holder_get_mlid_by_mgid(IN osm_sa_t * sa, > + IN const ib_gid_t * const p_mgid); Why *sa and not *subn? And why should it be in osm_mcm_info.h (which just describes *port*'s mcm related list)? -This may be moved to another file, or perhaps it's better to create a new file for this stuff? > +/* > +* PARAMETERS > +* > +* p_sa > +* [in] Pointer to sa object > +* > +* p_mgid > +* [in] pointer to mgid > +* > +* RETURN VALUES > +* mlid of found holder, or zero. > +* > +* NOTES > +* > +* SEE ALSO > +* > +*********/ > END_C_DECLS > #endif /* _OSM_MCM_INFO_H_ */ > diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h > index 02b63bd..9f0cd96 100644 > --- a/opensm/include/opensm/osm_multicast.h > +++ b/opensm/include/opensm/osm_multicast.h > @@ -214,6 +214,8 @@ typedef struct osm_mgrp { > uint32_t last_change_id; > uint32_t last_tree_id; > unsigned full_members; > + cl_fmap_item_t mgid_item; > + cl_list_item_t mgrp_item; > } osm_mgrp_t; > /* > * FIELDS > @@ -254,6 +256,12 @@ typedef struct osm_mgrp { > * last_tree_id > * the last change id used for building the current tree. > * > +* mgid_item > +* fleximap item for map linkage, sorted by mgid. > +* > +* mgrp_item > +* list item for mgroups pointer list. > +* > * SEE ALSO > *********/ > > @@ -572,6 +580,295 @@ osm_mgrp_apply_func(const osm_mgrp_t * const p_mgrp, > * SEE ALSO > * Multicast Group > *********/ > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_new > +* NAME > +* osm_mgrp_holder_new > +* > +* DESCRIPTION > +* Allocates and initializes a Multicast Group Holder for use. > +* > +* SYNOPSIS > +*/ > +osm_mgrp_holder_t *osm_mgrp_holder_new(IN osm_subn_t * p_subn, > + IN ib_gid_t * p_mgid, > + IN ib_net16_t mlid); > +/* > +* PARAMETERS > +* p_subn > +* (in) pointer to osm_subnet > +* p_mgid > +* (in) pointer to mgid What is "mgid" there? "Common" or "real"? > +* mlid > +* [in] Multicast LID for this multicast group holder. > +* > +* RETURN VALUES > +* pointer to initialized osm_mgrp_holder_t > +* or NULL, if unsuccessful > +* > +* SEE ALSO > +* Multicast Group Holder, osm_mgrp_holder_delete > +*********/ > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_delete > +* NAME > +* osm_mgrp_holder_delete > +* > +* DESCRIPTION > +* Removes entry from array of holders > +* Removes port from mgroup port list > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_delete(IN osm_subn_t * p_subn, IN ib_net16_t mlid); > +/* > +* PARAMETERS > +* > +* p_subn > +* [in] Pointer to osm_subnet > +* > +* mlid > +* [in] holder's mlid > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* > +* SEE ALSO > +* > +*********/ > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_add_mgrp_port > +* NAME > +* osm_mgrp_holder_add_mgrp_port > +* > +* DESCRIPTION > +* Allocates osm_mgrp_port_t for new port joined to mgroup with mlid of this holder, > +* (or / and) adds mgroup to mgroup map of existed osm_mgrp_port_t object. > +* > +* SYNOPSIS > +*/ > +ib_api_status_t osm_mgrp_holder_add_mgrp_port(IN osm_mgrp_holder_t * > + p_mgrp_holder, > + IN osm_mgrp_t * p_mgrp, > + IN ib_net64_t port_guid); Why do you need both mgrp_holder and mgrp? > +/* > +* PARAMETERS > +* p_mgrp_holder > +* (in) pointer to osm_mgrp_holder_t > +* p_mgrp > +* (in) pointer to osm_mgrp_t > +* > +* RETURN VALUES > +* IB_SUCCESS or > +* IB_INSUFFICIENT_MEMORY > +* > +* SEE ALSO > +* Multicast Group Holder, osm_mgrp_holder_delete_mgrp_port > +*********/ > + > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_delete_mgrp_port > +* NAME > +* osm_mgrp_holder_delete_mgrp_port > +* > +* DESCRIPTION > +* Deletes osm_mgrp_port_t for specified port > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_delete_mgrp_port(IN osm_mgrp_holder_t * p_mgrp_holder, > + IN osm_mgrp_t * p_mgrp, > + IN ib_net64_t port_guid); Ditto. > +/* > +* PARAMETERS > +* p_mgrp_holder > +* [in] Pointer to an osm_mgrp_holder_t object. > +* > +* p_mgrp > +* (in) Pointer to osm_mgrp_t object > +* > +* port_guid > +* [in] Port guid of the departing port. > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* > +* SEE ALSO > + Multicast Group Holder,osm_holder_add_mgrp_port > +*********/ > + > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_remove_port > +* NAME > +* osm_mgrp_holder_remove_port > +* > +* DESCRIPTION > +* Removes osm_mgrp_port_t from mgrp_port_map of holder > +* Removes port from mgroup port list > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_remove_port(IN osm_subn_t * const p_subn, > + IN osm_log_t * const p_log, > + IN osm_mgrp_holder_t * const p_mgrp_holder, > + IN const ib_net64_t port_guid); > +/* > +* PARAMETERS > +* > +* p_subn > +* [in] Pointer to the subnet object > +* > +* p_log > +* [in] The log object pointer > +* > +* p_mgrp_holder > +* [in] Pointer to an osm_mgrp_holder_t object. > +* > +* port_guid > +* [in] Port guid of the departing port. > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* > +* SEE ALSO > +* > +*********/ > + > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_add_mgrp > +* NAME > +* osm_mgrp_holder_add_mgrp > +* > +* DESCRIPTION > +* Adds mgroup to holder according to its mgid > +* > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_add_mgrp(IN osm_mgrp_holder_t * p_mgrp_holder, > + IN osm_mgrp_t * p_mgrp, > + IN osm_log_t * const p_log); > +/* > +* PARAMETERS > +* > +* p_mgrp_holder > +* [in] Pointer to an osm_mgrp_holder_t object. > +* > +* p_mgrp > +* [in] mgroup to add. > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* Updates common_mgid when holder is being reused > +* SEE ALSO > +* Multicast Group Holder,osm_mgrp_holder_delete_mgrp > +*********/ > + > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_delete_mgrp > +* NAME > +* osm_mgrp_holder_delete_mgrp > +* > +* DESCRIPTION > +* Deletes mgroup from holder according to its mgid > +* > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_delete_mgrp(IN osm_mgrp_holder_t * p_mgrp_holder, > + IN osm_mgrp_t * p_mgrp); > +/* > +* PARAMETERS > +* > +* p_mgrp_holder > +* [in] Pointer to an osm_mgrp_holder_t object. > +* > +* p_mgrp > +* [in] mgroup to delete. > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* > +* SEE ALSO > +* Multicast Group Holder,osm_mgrp_holder_add_mgrp > +*********/ > + > +/* Multicast Group Holder, osm_mgrp_holder_delete_port > +*********/ To what is this comment related to? -to be deleted > + > +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_prepare_common_mgid > +* NAME > +* osm_mgrp_holder_prepare_common_mgid > +* > +* DESCRIPTION > +* Prepares mgid, which is common for all mgroups in this holder > +* > +* SYNOPSIS > +*/ > +void osm_mgrp_holder_prepare_common_mgid(IN const ib_gid_t * const p_mgid, > + OUT ib_gid_t * p_common_mgid); > +/* > +* PARAMETERS > +* > +* p_mgid > +* [in] Pointer to mgid > +* > +* p_common_mgid > +* [out] common mgid > +* > +* RETURN VALUES > +* None. > +* > +* NOTES > +* > +* SEE ALSO > +* > +*********/ > +/****f* OpenSM: Subnet/osm_get_mgrp_by_mlid > +* NAME > +* osm_get_mgrp_by_mlid > +* > +* DESCRIPTION > +* The looks for the given multicast group in the subnet table by mlid. > +* NOTE: this code is not thread safe. Need to grab the lock before > +* calling it. > +* > +* SYNOPSIS > +*/ > +static inline > + struct osm_mgrp_holder *osm_get_mgrp_holder_by_mlid(osm_subn_t const > + *p_subn, > + ib_net16_t mlid) > +{ > + return p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO]; > +} > + > +/* > +* PARAMETERS > +* p_subn > +* [in] Pointer to an osm_subn_t object > +* > +* mlid > +* [in] The multicast group mlid in network order > +* > +* RETURN VALUES > +* The multicast group structure pointer if found. NULL otherwise. > +*********/ > + > +static inline ib_net16_t osm_mgrp_holder_get_mlid(IN osm_mgrp_holder_t * > + const p_mgrp_holder) > +{ > + return (p_mgrp_holder->mlid); > +} > + > +static inline boolean_t osm_mgrp_holder_is_empty(IN const osm_mgrp_holder_t * > + const p_mgrp_holder) > +{ > + return (cl_qmap_count(&p_mgrp_holder->mgrp_port_map) == 0); > +} > > END_C_DECLS > #endif /* _OSM_MULTICAST_H_ */ > diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h > index 59a32ad..6713d4d 100644 > --- a/opensm/include/opensm/osm_subnet.h > +++ b/opensm/include/opensm/osm_subnet.h > @@ -1,5 +1,5 @@ > /* > - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. > + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. > * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. > * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. > * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. > @@ -509,6 +509,7 @@ typedef struct osm_subn { > boolean_t coming_out_of_standby; > unsigned need_update; > void *mgroups[IB_LID_MCAST_END_HO - IB_LID_MCAST_START_HO + 1]; > + void *mgroup_holders[IB_LID_MCAST_END_HO - IB_LID_MCAST_START_HO + 1]; > } osm_subn_t; > /* > * FIELDS > @@ -633,6 +634,10 @@ typedef struct osm_subn { > * Array of pointers to all Multicast Group objects in the subnet. > * Indexed by MLID offset from base MLID. > * > +* mgroup_holders > +* Array of pointers to all Multicast Group Holder objects in the subnet. > +* Indexed by MLID offset from base MLID. > +* > * SEE ALSO > * Subnet object > *********/ > diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c > index ae0a818..d2a19ea 100644 > --- a/opensm/opensm/osm_multicast.c > +++ b/opensm/opensm/osm_multicast.c > @@ -44,10 +44,12 @@ > > #include > #include > +#include > #include > #include > #include > #include > +#include > #include > > /********************************************************************** > @@ -74,6 +76,31 @@ void osm_mgrp_delete(IN osm_mgrp_t * p_mgrp) > free(p_mgrp); > } > > +void osm_mgrp_delete_group(IN osm_subn_t * p_subn, IN osm_mgrp_t * p_mgrp) > +{ > + osm_mcm_port_t *p_mcm_port; > + osm_mcm_port_t *p_next_mcm_port; > + > + CL_ASSERT(p_mgrp); > + > + osm_mgrp_holder_t *p_mgrp_holder = > + osm_get_mgrp_holder_by_mlid(p_subn, p_mgrp->mlid); > + p_next_mcm_port = > + (osm_mcm_port_t *) cl_qmap_head(&p_mgrp->mcm_port_tbl); > + while (p_next_mcm_port != > + (osm_mcm_port_t *) cl_qmap_end(&p_mgrp->mcm_port_tbl)) { > + p_mcm_port = p_next_mcm_port; > + p_next_mcm_port = > + (osm_mcm_port_t *) cl_qmap_next(&p_mcm_port->map_item); > + osm_mgrp_holder_delete_mgrp_port(p_mgrp_holder, p_mgrp, > + p_mcm_port->port_gid.unicast. > + interface_id); > + osm_mcm_port_delete(p_mcm_port); > + } > + osm_mgrp_holder_delete_mgrp(p_mgrp_holder, p_mgrp); > + free(p_mgrp); > +} > + > /********************************************************************** > **********************************************************************/ > osm_mgrp_t *osm_mgrp_new(IN const ib_net16_t mlid) > @@ -378,3 +405,203 @@ static osm_mgrp_port_t *osm_mgrp_port_new(ib_net64_t port_guid) > cl_qlist_init(&p_mgrp_port->mgroups); > return p_mgrp_port; > } > + > +void osm_mgrp_holder_delete(IN osm_subn_t * p_subn, ib_net16_t mlid) > +{ > + osm_mgrp_port_t *p_osm_mgr_port; > + cl_map_item_t *p_item; > + cl_fmap_item_t *p_fitem; > + osm_mgrp_t *p_mgrp; > + > + osm_mgrp_holder_t *p_mgrp_holder = > + p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO]; > + p_item = cl_qmap_head(&p_mgrp_holder->mgrp_port_map); > + while (p_item != cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { > + p_osm_mgr_port = (osm_mgrp_port_t *) p_item; > + cl_qlist_remove_all(&p_osm_mgr_port->mgroups); > + cl_qmap_remove_item(&p_mgrp_holder->mgrp_port_map, p_item); > + p_item = cl_qmap_head(&p_mgrp_holder->mgrp_port_map); > + free(p_osm_mgr_port); > + } I'm not sure that I understand your data model very well (maybe it would be a good idea to describe and implement this as separate patch). But do you have port lists in both mgrp and mgrp_holder? Assuming so, why? This code looks pretty overloaded. > + p_fitem = cl_fmap_head(&p_mgrp_holder->mgrp_map); > + while (p_fitem != cl_fmap_end(&p_mgrp_holder->mgrp_map)) { > + p_mgrp = > + (osm_mgrp_t *) PARENT_STRUCT(p_fitem, osm_mgrp_t, > + mgid_item); > + osm_mgrp_delete_group(p_subn, p_mgrp); > + p_fitem = cl_fmap_head(&p_mgrp_holder->mgrp_map); > + } > + /* destroy the mtree_node structure */ > + osm_mtree_destroy(p_mgrp_holder->p_root); > + p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO] = NULL; > + free(p_mgrp_holder); > +} > + > +osm_mgrp_holder_t *osm_mgrp_holder_new(IN osm_subn_t * p_subn, > + ib_gid_t * p_mgid, ib_net16_t mlid) > +{ > + osm_mgrp_holder_t *p_mgrp_holder; > + p_mgrp_holder = > + p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO] = > + (osm_mgrp_holder_t *) malloc(sizeof(*p_mgrp_holder)); > + if (!p_mgrp_holder) > + return NULL; > + > + memset(p_mgrp_holder, 0, sizeof(*p_mgrp_holder)); > + p_mgrp_holder->mlid = mlid; > + cl_fmap_init(&p_mgrp_holder->mgrp_map, __mgid_cmp); > + cl_qmap_init(&p_mgrp_holder->mgrp_port_map); > + osm_mgrp_holder_prepare_common_mgid(p_mgid, > + &p_mgrp_holder->common_mgid); > + return p_mgrp_holder; > +} > + > +ib_net16_t osm_mgrp_holder_get_mlid_by_mgid(IN osm_sa_t * sa, > + IN const ib_gid_t * const p_mgid) > +{ > + int i; > + ib_gid_t common_mgid; > + osm_mgrp_holder_t *p_mgrp_holder; > + > + OSM_LOG_ENTER(sa->p_log); > + > + osm_mgrp_holder_prepare_common_mgid(p_mgid, &common_mgid); > + for (i = 0; i <= sa->p_subn->max_mcast_lid_ho - IB_LID_MCAST_START_HO; > + i++) { > + if (sa->p_subn->mgroup_holders[i]) { > + p_mgrp_holder = > + (osm_mgrp_holder_t *) sa->p_subn->mgroup_holders[i]; > + if (!memcmp > + (&p_mgrp_holder->common_mgid, &common_mgid, > + sizeof(ib_gid_t))) { > + char gid_str[INET6_ADDRSTRLEN]; > + OSM_LOG(sa->p_log, OSM_LOG_DEBUG, > + "Found holder 0x%X for MGID %s\n", > + cl_ntoh16(p_mgrp_holder->mlid), > + inet_ntop(AF_INET6, p_mgid->raw, > + gid_str, sizeof(gid_str))); > + OSM_LOG_EXIT(sa->p_log); > + return p_mgrp_holder->mlid; > + } > + } > + } Hmm, why do we need linear search her? What was a purpose of storing mgrps in fleximap? -Each fleximap contains mgroups with same MLID. Linear search compares "common" mgid only. > + OSM_LOG_EXIT(sa->p_log); > + return 0; +} > + > +void osm_mgrp_holder_remove_port(osm_subn_t * subn, osm_log_t * p_log, > + osm_mgrp_holder_t * p_mgrp_holder, > + ib_net64_t port_guid) > +{ > + osm_mgrp_t *p_mgrp; > + cl_list_item_t *p_item; > + > + OSM_LOG_ENTER(p_log); > + > + osm_mgrp_port_t *p_mgrp_port = (osm_mgrp_port_t *) > + cl_qmap_remove(&p_mgrp_holder->mgrp_port_map, port_guid); > + if (p_mgrp_port != > + (osm_mgrp_port_t *) cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { > + char gid_str[INET6_ADDRSTRLEN]; > + OSM_LOG(p_log, OSM_LOG_DEBUG, > + "port 0x%" PRIx64 " removed from mlid 0x%X\n", > + port_guid, cl_ntoh16(p_mgrp_holder->mlid)); > + while ((p_item = > + cl_qlist_remove_head(&p_mgrp_port->mgroups)) != > + cl_qlist_end(&p_mgrp_port->mgroups)) { > + p_mgrp = > + (osm_mgrp_t *) PARENT_STRUCT(p_item, osm_mgrp_t, > + mgrp_item); > + OSM_LOG(p_log, OSM_LOG_DEBUG, > + "removing mgrp mgid %s from port 0x%" PRIx64 > + "\n", inet_ntop(AF_INET6, > + p_mgrp->mcmember_rec.mgid.raw, > + gid_str, sizeof(gid_str)), > + cl_ntoh64(port_guid)); > + osm_mgrp_delete_port(subn, p_log, p_mgrp, port_guid); > + } > + free(p_mgrp_port); > + } > + OSM_LOG_EXIT(p_log); > +} > + > +void osm_mgrp_holder_add_mgrp(osm_mgrp_holder_t * p_mgrp_holder, > + osm_mgrp_t * p_mgrp, osm_log_t * p_log) > +{ > + cl_fmap_item_t *p_fitem; > + char gid_str[INET6_ADDRSTRLEN]; > + > + OSM_LOG_ENTER(p_log); > + > + if (p_mgrp_holder->to_be_deleted) { > + /* this is re-used mgrp_holder, need to update common_mgid */ > + osm_mgrp_holder_prepare_common_mgid(&p_mgrp->mcmember_rec.mgid, > + &p_mgrp_holder-> > + common_mgid); > + } > + p_fitem = cl_fmap_insert(&p_mgrp_holder->mgrp_map, > + &p_mgrp->mcmember_rec.mgid, > + &p_mgrp->mgid_item); Ok, starting to understand (hopefully :)) - you have fleximap only per mgrp_holder. Why? Wouldn't it better to have single (per subnet) mgrp fleximap? I guess this will cover all common search cases and will simplify a data model dramatically. - I don't think so. In case of single fleximap search will always take logN. Having fleximap divided into groups per MLID will give logN in a worst case. Also, keeping mgroups per MLID simplifies queries like saquery -m Also (unrelated directly to this patch series). I have some local cleanup/simplification changes in multicast stuff, in particular it eliminates the need in all those 'to_be_deleted', 'tree_id', etc. flags. I think this could help you with simpler implementation. Sasha -Your changes are already in git? Slava > + CL_ASSERT(p_item == &p_mgrp->mgid_item); > + > + OSM_LOG(p_log, OSM_LOG_DEBUG, > + "mgrp with MGID:%s added to holder with mlid = 0x%X\n", > + inet_ntop(AF_INET6, p_mgrp->mcmember_rec.mgid.raw, gid_str, > + sizeof(gid_str)), cl_ntoh16(p_mgrp_holder->mlid)); > + p_mgrp_holder->last_change_id++; > + OSM_LOG_EXIT(p_log); > +} > + > +ib_api_status_t osm_mgrp_holder_add_mgrp_port(osm_mgrp_holder_t * p_mgrp_holder, > + osm_mgrp_t * p_mgrp, > + ib_net64_t port_guid) > +{ > + osm_mgrp_port_t *p_mgrp_port = (osm_mgrp_port_t *) > + cl_qmap_get(&p_mgrp_holder->mgrp_port_map, port_guid); > + if (p_mgrp_port == > + (osm_mgrp_port_t *) cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { > + /* new port to mlid */ > + p_mgrp_port = osm_mgrp_port_new(port_guid); > + if (!p_mgrp_port) { > + return IB_INSUFFICIENT_MEMORY; > + } > + cl_qmap_insert(&p_mgrp_holder->mgrp_port_map, > + p_mgrp_port->port_guid, &p_mgrp_port->guid_item); > + cl_qlist_insert_tail(&p_mgrp_port->mgroups, &p_mgrp->mgrp_item); > + } else { > + cl_qlist_insert_tail(&p_mgrp_port->mgroups, &p_mgrp->mgrp_item); > + } > + return IB_SUCCESS; > +} > + > +void osm_mgrp_holder_delete_mgrp_port(osm_mgrp_holder_t * p_mgrp_holder, > + osm_mgrp_t * p_mgrp, ib_net64_t port_guid) > +{ > + osm_mgrp_port_t *p_mgrp_port = (osm_mgrp_port_t *) > + cl_qmap_get(&p_mgrp_holder->mgrp_port_map, port_guid); > + if (p_mgrp_port != > + (osm_mgrp_port_t *) cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { > + cl_qlist_remove_item(&p_mgrp_port->mgroups, &p_mgrp->mgrp_item); > + if (0 == cl_qlist_count(&p_mgrp_port->mgroups)) { > + /* No mgroups registered on this port for current mlid */ > + cl_qmap_remove_item(&p_mgrp_holder->mgrp_port_map, > + &p_mgrp_port->guid_item); > + free(p_mgrp_port); > + } > + p_mgrp_holder->last_change_id++; > + } > +} > + > +void osm_mgrp_holder_delete_mgrp(osm_mgrp_holder_t * p_mgrp_holder, > + osm_mgrp_t * p_mgrp) > +{ > + p_mgrp->to_be_deleted = 1; > + cl_fmap_remove_item(&p_mgrp_holder->mgrp_map, &p_mgrp->mgid_item); > + if (0 == cl_fmap_count(&p_mgrp_holder->mgrp_map)) { > + /* No more mgroups on this mlid */ > + p_mgrp_holder->to_be_deleted = 1; > + p_mgrp_holder->last_tree_id = 0; > + p_mgrp_holder->last_change_id = 0; > + memset(&p_mgrp_holder->common_mgid, 0, sizeof(ib_gid_t)); > + } > +} > -- > 1.5.5 > From sashak at voltaire.com Mon Jul 20 03:17:11 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 13:17:11 +0300 Subject: [ofa-general] Re: [Repost][PATCH] opensm: Added support for select counters (xmit_wait) In-Reply-To: <4A641112.3070800@morey-chaisemartin.com> References: <4A5CA3B6.9080809@morey-chaisemartin.com> <20090719135334.GG22967@me> <4A6368B8.6050007@morey-chaisemartin.com> <20090719223409.GR22967@me> <4A641112.3070800@morey-chaisemartin.com> Message-ID: <20090720101711.GS22967@me> On 08:39 Mon 20 Jul , Nicolas Morey-Chaisemartin wrote: > So what is best in your mind? > Bring everything path into the perf counter struct (meaning all plugin > will have to change to fit the new API) or keeping the approach I proposed. We can add new counter to existing structure at the end and to not break API. The better option is to unify API (counter structures) of course, but it is different change. Basically using current (modified) plugin API it is not required to put a counters to plugin, some sort of "counters ready" event would be enough. > I'd rather choose the second solution not only because it's already > done, but in the future I'd like to have as many information as possible > on contention related counters (xmit_wait per VL for examples) and > probably some day vendor specific info will appear in this struct. Yes, but it is different counters and you will need to add something anyway. Sasha From sashak at voltaire.com Mon Jul 20 03:22:23 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 13:22:23 +0300 Subject: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: <39C75744D164D948A170E9792AF8E7CA01F6F921@exil.voltaire.com> References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM> <39C75744D164D948A170E9792AF8E7CA01F6F919@exil.voltaire.com> <4A644107.1010404@voltaire.com> <39C75744D164D948A170E9792AF8E7CA01F6F921@exil.voltaire.com> Message-ID: <20090720102223.GU22967@me> On 13:07 Mon 20 Jul , Slava Strebkov wrote: > -Agreed, special groups (except SNM) will not be compressed into same > MLID. May be I missed the point, but for me it looks that idea was to differentiate MLIDs by PKey value as well. Sasha From sashak at voltaire.com Mon Jul 20 03:47:13 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 13:47:13 +0300 Subject: [ofa-general] Re: [PATCH 2/4 v2] multicast multiplexing - implementation of newinfrastructure In-Reply-To: <39C75744D164D948A170E9792AF8E7CA01F6F922@exil.voltaire.com> References: <4A3A387A.2020509@Voltaire.COM> <4A474C81.2020602@Voltaire.COM> <20090719155056.GN22967@me> <39C75744D164D948A170E9792AF8E7CA01F6F922@exil.voltaire.com> Message-ID: <20090720104713.GV22967@me> On 13:10 Mon 20 Jul , Slava Strebkov wrote: > > > +ib_net16_t osm_mgrp_holder_get_mlid_by_mgid(IN osm_sa_t * sa, > > + IN const ib_gid_t * const > p_mgid); > > Why *sa and not *subn? And why should it be in osm_mcm_info.h (which > just describes *port*'s mcm related list)? > -This may be moved to another file, or perhaps it's better to create a > new file for this stuff? I don't think that we need new file (we have enough already), move it to osm_subnet.c (where all osm_get_*_by_*() are sitting) or osm_multicast.c. > > + osm_mgrp_holder_prepare_common_mgid(p_mgid, &common_mgid); > > + for (i = 0; i <= sa->p_subn->max_mcast_lid_ho - > IB_LID_MCAST_START_HO; > > + i++) { > > + if (sa->p_subn->mgroup_holders[i]) { > > + p_mgrp_holder = > > + (osm_mgrp_holder_t *) > sa->p_subn->mgroup_holders[i]; > > + if (!memcmp > > + (&p_mgrp_holder->common_mgid, &common_mgid, > > + sizeof(ib_gid_t))) { > > + char gid_str[INET6_ADDRSTRLEN]; > > + OSM_LOG(sa->p_log, OSM_LOG_DEBUG, > > + "Found holder 0x%X for MGID > %s\n", > > + cl_ntoh16(p_mgrp_holder->mlid), > > + inet_ntop(AF_INET6, p_mgid->raw, > > + gid_str, > sizeof(gid_str))); > > + OSM_LOG_EXIT(sa->p_log); > > + return p_mgrp_holder->mlid; > > + } > > + } > > + } > > Hmm, why do we need linear search her? What was a purpose of storing > mgrps in fleximap? > -Each fleximap contains mgroups with same MLID. > Linear search compares "common" mgid only. So why to not use common (per subnet) fleximap where mgrp will be mapped by mgid ("real" one)? I think it will simplify your data model and speedup most common searches. > > +void osm_mgrp_holder_add_mgrp(osm_mgrp_holder_t * p_mgrp_holder, > > + osm_mgrp_t * p_mgrp, osm_log_t * p_log) > > +{ > > + cl_fmap_item_t *p_fitem; > > + char gid_str[INET6_ADDRSTRLEN]; > > + > > + OSM_LOG_ENTER(p_log); > > + > > + if (p_mgrp_holder->to_be_deleted) { > > + /* this is re-used mgrp_holder, need to update > common_mgid */ > > + > osm_mgrp_holder_prepare_common_mgid(&p_mgrp->mcmember_rec.mgid, > > + &p_mgrp_holder-> > > + common_mgid); > > + } > > + p_fitem = cl_fmap_insert(&p_mgrp_holder->mgrp_map, > > + &p_mgrp->mcmember_rec.mgid, > > + &p_mgrp->mgid_item); > > Ok, starting to understand (hopefully :)) - you have fleximap only per > mgrp_holder. Why? Wouldn't it better to have single (per subnet) mgrp > fleximap? I guess this will cover all common search cases and will > simplify a data model dramatically. > - I don't think so. In case of single fleximap search will always take > logN. > Having fleximap divided into groups per MLID will give logN in a worst > case. logN + MLID detection (which is slow). No? Another downside of proposed data model is its compression algorithm awareness - it assumes MGID is compressed using signature. As I said somewhere in previous comment it would be really nice to have many MGIDs per MLID data transparent to used compression algorithm (it can change in a future, be configurable, etc.). And also I suppose that using common fleximap will require less code and it will not be overcomplicated as it is purposed now. > Also, keeping mgroups per MLID simplifies queries like saquery -m Had nothing against this. > Also (unrelated directly to this patch series). I have some local > cleanup/simplification changes in multicast stuff, in particular it > eliminates the need in all those 'to_be_deleted', 'tree_id', etc. flags. > I think this could help you with simpler implementation. > Sasha > -Your changes are already in git? In my local branch, I didn't publish it yet. Sasha From sashak at voltaire.com Mon Jul 20 04:57:28 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 14:57:28 +0300 Subject: [ofa-general] Re: [PATCHv2] opensm/lash: Set minimum VL for LASH to use In-Reply-To: <20090706154643.GA17356@comcast.net> References: <20090706154643.GA17356@comcast.net> Message-ID: <20090720115728.GA22967@me> Hi Hal, On 11:46 Mon 06 Jul , Hal Rosenstock wrote: > > diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c > index 12b5e34..3db75a0 100644 > --- a/opensm/opensm/osm_ucast_lash.c > +++ b/opensm/opensm/osm_ucast_lash.c > @@ -478,7 +478,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) > cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix; > int *num_mst_in_lane = p_lash->num_mst_in_lane; > int ***virtual_location = p_lash->virtual_location; > - int min_filled_lane, max_filled_lane, trials; > + int min_filled_lane, max_filled_lane, trials, max_vl; > int old_min_filled_lane, old_max_filled_lane, new_num_min_lane, > new_num_max_lane; > unsigned int i, j; > @@ -486,9 +486,11 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) > int next_switch2, output_link2; > int stop = 0, cycle_found; > int cycle_found2; > + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; > > - max_filled_lane = 0; > - min_filled_lane = lanes_needed - 1; > + max_filled_lane = start_vl; > + max_vl = lanes_needed + start_vl; > + min_filled_lane = max_vl - 1; > > trials = num_mst_in_lane[max_filled_lane]; > if (lanes_needed == 1) > @@ -590,7 +592,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) > new_num_min_lane = MAX_INT; > new_num_max_lane = 0; > > - for (i = 0; i < lanes_needed; i++) { > + for (i = start_vl; i < max_vl; i++) { > > if (num_mst_in_lane[i] < new_num_min_lane) { > new_num_min_lane = num_mst_in_lane[i]; > @@ -674,11 +676,12 @@ static void free_lash_structures(lash_t * p_lash) > unsigned int i, j, k; > unsigned num_switches = p_lash->num_switches; > osm_log_t *p_log = &p_lash->p_osm->log; > + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; > > OSM_LOG_ENTER(p_log); > > /* free cdg_vertex_matrix */ > - for (i = 0; i < p_lash->vl_min; i++) { > + for (i = start_vl; i < p_lash->vl_min; i++) { > for (j = 0; j < num_switches; j++) { > for (k = 0; k < num_switches; k++) > if (p_lash->cdg_vertex_matrix[i][j][k]) > @@ -715,13 +718,14 @@ static int init_lash_structures(lash_t * p_lash) > osm_log_t *p_log = &p_lash->p_osm->log; > int status = 0; > unsigned int i, j, k; > + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; > > OSM_LOG_ENTER(p_log); > > /* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */ > p_lash->cdg_vertex_matrix = > - (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****)); > - for (i = 0; i < vl_min; i++) { > + (cdg_vertex_t ****) calloc(vl_min, sizeof(cdg_vertex_t ****)); You don't need to use calloc() when memory initialization is not required - malloc() is always faster. Wouldn't it be better instead of allocating extra memory for "disabled" (0 - lash_vl_start) VLs range and tracking 'max_vl' in many places just to setup lash->vl_min at end of discover_network_properties() as vl_min - lash_vl_start and to increase routing_table[i].lane value by lash_vl_start at end of lash cycle? Sasha From sashak at voltaire.com Mon Jul 20 04:34:26 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 14:34:26 +0300 Subject: [ofa-general] Re: [PATCHv2] opensm/osm_mesh.c: Improve VL utilization In-Reply-To: <20090703133440.GA31663@comcast.net> References: <20090703133440.GA31663@comcast.net> Message-ID: <20090720113426.GZ22967@me> On 09:34 Fri 03 Jul , Hal Rosenstock wrote: > > opensm/osm_mesh.c: Improve VL utilization > > Signed-off-by: Robert Pearson > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Mon Jul 20 05:23:24 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 15:23:24 +0300 Subject: [ofa-general] Re: libibmad: ib_resolve_portid_str_via; Bug? In-Reply-To: <20090422170307.548e8a55.weiny2@llnl.gov> References: <20090422170307.548e8a55.weiny2@llnl.gov> Message-ID: <20090720122324.GA31735@me> Hi Ira, On 17:03 Wed 22 Apr , Ira Weiny wrote: > > Below is a patch which fixes an issue I had when using > ib_resolve_portid_str_via. When resolving via IB_DEST_GUID the > ib_resolve_guid_via function optionally uses the portid to attempt to set a > different subnet prefix. > > IMO I don't think portid should be an in/out parameter in > ib_resolve_portid_str_via. I happened to pass a portid object which was on > the stack and had some garbage data in it. It took me a while to figure out > that ib_resolve_portid_str_via was attempting to use that garbage data. > > To make this more clear I added ib_resolve_gid_via and another MAD_DEST type. > > What do you think? Right now the gid resolving is untested. I'm not against such addition, but this patch is malformed. Sasha From sashak at voltaire.com Mon Jul 20 06:29:59 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 16:29:59 +0300 Subject: [ofa-general] [PATCH v4] libibmad: Handle MAD redirection In-Reply-To: <200907071620.23003.fenkes@de.ibm.com> References: <200906291410.33477.fenkes@de.ibm.com> <20090701200043.GF20745@obsidianresearch.com> <200907071620.23003.fenkes@de.ibm.com> Message-ID: <20090720132959.GB31735@me> On 16:20 Tue 07 Jul , Joachim Fenkes wrote: > Previously, libibmad reacted to GSI MAD responses with a "redirect" status > by throwing an error. IBM eHCA adapters use redirection, so most > infiniband_diags tools didn't work against eHCA. > > Fix: Modify mad_rpc() so that it resends the request to the redirection > target if a "redirect" GS response is received. This is repeated until no > "redirect" response is received, allowing for multiple levels of > indirection. > > The dport argument is updated with the redirection target, so subsequent > MADs will not go through the redirection process again but reach the target > directly. > > Tested using perfquery between ehca, mlx4 and mthca in all possible > combinations. > > Signed-off-by: Joachim Fenkes Applied. Thanks. Sasha From hal.rosenstock at gmail.com Mon Jul 20 06:32:08 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 20 Jul 2009 09:32:08 -0400 Subject: [ofa-general] Re: [PATCHv2] opensm/lash: Set minimum VL for LASH to use In-Reply-To: <20090720115728.GA22967@me> References: <20090706154643.GA17356@comcast.net> <20090720115728.GA22967@me> Message-ID: Hi Sasha, On Mon, Jul 20, 2009 at 7:57 AM, Sasha Khapyorsky wrote: > Hi Hal, > > On 11:46 Mon 06 Jul     , Hal Rosenstock wrote: >> >> diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c >> index 12b5e34..3db75a0 100644 >> --- a/opensm/opensm/osm_ucast_lash.c >> +++ b/opensm/opensm/osm_ucast_lash.c >> @@ -478,7 +478,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) >>       cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix; >>       int *num_mst_in_lane = p_lash->num_mst_in_lane; >>       int ***virtual_location = p_lash->virtual_location; >> -     int min_filled_lane, max_filled_lane, trials; >> +     int min_filled_lane, max_filled_lane, trials, max_vl; >>       int old_min_filled_lane, old_max_filled_lane, new_num_min_lane, >>           new_num_max_lane; >>       unsigned int i, j; >> @@ -486,9 +486,11 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) >>       int next_switch2, output_link2; >>       int stop = 0, cycle_found; >>       int cycle_found2; >> +     unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; >> >> -     max_filled_lane = 0; >> -     min_filled_lane = lanes_needed - 1; >> +     max_filled_lane = start_vl; >> +     max_vl = lanes_needed + start_vl; >> +     min_filled_lane = max_vl - 1; >> >>       trials = num_mst_in_lane[max_filled_lane]; >>       if (lanes_needed == 1) >> @@ -590,7 +592,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) >>               new_num_min_lane = MAX_INT; >>               new_num_max_lane = 0; >> >> -             for (i = 0; i < lanes_needed; i++) { >> +             for (i = start_vl; i < max_vl; i++) { >> >>                       if (num_mst_in_lane[i] < new_num_min_lane) { >>                               new_num_min_lane = num_mst_in_lane[i]; >> @@ -674,11 +676,12 @@ static void free_lash_structures(lash_t * p_lash) >>       unsigned int i, j, k; >>       unsigned num_switches = p_lash->num_switches; >>       osm_log_t *p_log = &p_lash->p_osm->log; >> +     unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; >> >>       OSM_LOG_ENTER(p_log); >> >>       /* free cdg_vertex_matrix */ >> -     for (i = 0; i < p_lash->vl_min; i++) { >> +     for (i = start_vl; i < p_lash->vl_min; i++) { >>               for (j = 0; j < num_switches; j++) { >>                       for (k = 0; k < num_switches; k++) >>                               if (p_lash->cdg_vertex_matrix[i][j][k]) >> @@ -715,13 +718,14 @@ static int init_lash_structures(lash_t * p_lash) >>       osm_log_t *p_log = &p_lash->p_osm->log; >>       int status = 0; >>       unsigned int i, j, k; >> +     unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; >> >>       OSM_LOG_ENTER(p_log); >> >>       /* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */ >>       p_lash->cdg_vertex_matrix = >> -         (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****)); >> -     for (i = 0; i < vl_min; i++) { >> +         (cdg_vertex_t ****) calloc(vl_min, sizeof(cdg_vertex_t ****)); > > You don't need to use calloc() when memory initialization is not > required - malloc() is always faster. I restored the mallocs in the next version of this patch. > Wouldn't it be better instead of allocating extra memory for "disabled" > (0 - lash_vl_start) VLs range and tracking 'max_vl' in many places just > to setup lash->vl_min at end of discover_network_properties() as vl_min - > lash_vl_start and to increase routing_table[i].lane value by > lash_vl_start at end of lash cycle? Something like that could be done. Does it save significant memory (haven't looked yet to see exactly) ? OK as a subsequent patch to this change ? -- Hal > Sasha > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From hnrose at comcast.net Mon Jul 20 05:58:47 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Mon, 20 Jul 2009 08:58:47 -0400 Subject: [ofa-general] [PATCH] infiniband-diags/ibtracert.c: Fix linearcap test in switch_lookup Message-ID: <20090720125847.GA9987@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/infiniband-diags/src/ibtracert.c b/infiniband-diags/src/ibtracert.c index dc05a29..f1ca310 100644 --- a/infiniband-diags/src/ibtracert.c +++ b/infiniband-diags/src/ibtracert.c @@ -160,7 +160,7 @@ switch_lookup(Switch *sw, ib_portid_t *portid, int lid) mad_decode_field(si, IB_SW_LINEAR_FDB_CAP_F, &sw->linearcap); mad_decode_field(si, IB_SW_LINEAR_FDB_TOP_F, &sw->linearFDBtop); - if (lid > sw->linearcap && lid > sw->linearFDBtop) + if (lid >= sw->linearcap && lid > sw->linearFDBtop) return -1; if (!smp_query_via(fdb, portid, IB_ATTR_LINEARFORWTBL, lid / 64, From hnrose at comcast.net Mon Jul 20 06:20:17 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Mon, 20 Jul 2009 09:20:17 -0400 Subject: [ofa-general] [PATCH v3] opensm/lash: Set minimum VL for LASH to use Message-ID: <20090720132016.GA10346@comcast.net> rather than starting from VL 0 Signed-off-by: Robert Pearson Signed-off-by: Hal Rosenstock --- Changes since v2: Restored malloc use (rather than calloc) as pointed out by Sasha Changes since v1: Fixed comparisons with maximum VL Better lash_start_vl option handling Both as pointed out by Sasha diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 59a32ad..da8cc5e 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -221,6 +222,7 @@ typedef struct osm_subn_opt { char *prefix_routes_file; boolean_t consolidate_ipv6_snm_req; struct osm_subn_opt *file_opts; /* used for update */ + uint8_t lash_start_vl; /* starting vl to use in lash */ } osm_subn_opt_t; /* * FIELDS diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in index 66d2fe6..e8801c9 100644 --- a/opensm/man/opensm.8.in +++ b/opensm/man/opensm.8.in @@ -1,4 +1,4 @@ -.TH OPENSM 8 "April 22, 2009" "OpenIB" "OpenIB Management" +.TH OPENSM 8 "May 28, 2009" "OpenIB" "OpenIB Management" .SH NAME opensm \- InfiniBand subnet manager and administration (SM/SA) @@ -15,6 +15,7 @@ opensm \- InfiniBand subnet manager and administration (SM/SA) [\-r(eassign_lids)] [\-R | \-\-routing_engine ] [\-\-do_mesh_analysis] +[\-\-lash_start_vl ] [\-A | \-\-ucast_cache] [\-z | \-\-connect_roots] [\-M | \-\-lid_matrix_file ] @@ -147,6 +148,10 @@ This option enables additional analysis for the lash routing engine to precondition switch port assignments in regular cartesian meshes which may reduce the number of SLs required to give a deadlock free routing. .TP +\fB\-\-lash_start_vl\fR +This option sets the starting VL to use for the lash routing algorithm. +Defaults to 0. +.TP \fB\-A\fR, \fB\-\-ucast_cache\fR This option enables unicast routing cache and prevents routing recalculation (which is a heavy task in a large cluster) when diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c index 296d5d5..d682ff5 100644 --- a/opensm/opensm/main.c +++ b/opensm/opensm/main.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009 HNR Consulting. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -181,6 +182,9 @@ static void show_usage(void) " routing engine to precondition switch port assignments\n" " in regular cartesian meshes which may reduce the number\n" " of SLs required to give a deadlock free routing\n\n"); + printf("--lash_start_vl \n" + " Sets the starting VL to use for the lash routing algorithm.\n" + " Defaults to 0.\n"); printf("--connect_roots, -z\n" " This option enforces a routing engine (currently\n" " up/down only) to make connectivity between root switches\n" @@ -601,6 +605,7 @@ int main(int argc, char *argv[]) {"prefix_routes_file", 1, NULL, 3}, {"consolidate_ipv6_snm_req", 0, NULL, 4}, {"do_mesh_analysis", 0, NULL, 5}, + {"lash_start_vl", 1, NULL, 6}, {NULL, 0, NULL, 0} /* Required at the end of the array */ }; @@ -951,6 +956,15 @@ int main(int argc, char *argv[]) case 5: opt.do_mesh_analysis = TRUE; break; + case 6: + temp = strtol(optarg, NULL, 0); + if (temp < 0 || temp >= IB_MAX_NUM_VLS) { + fprintf(stderr, + "ERROR: starting lash vl must be between 0 and 15\n"); + return (-1); + } + opt.lash_start_vl = (uint8_t) temp; + break; case 'h': case '?': case ':': diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index ec15f8a..fda2eb0 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -389,6 +390,7 @@ static const opt_rec_t opt_tbl[] = { { "no_clients_rereg", OPT_OFFSET(no_clients_rereg), opts_parse_boolean, NULL, 1 }, { "prefix_routes_file", OPT_OFFSET(prefix_routes_file), opts_parse_charp, NULL, 0 }, { "consolidate_ipv6_snm_req", OPT_OFFSET(consolidate_ipv6_snm_req), opts_parse_boolean, NULL, 1 }, + { "lash_start_vl", OPT_OFFSET(lash_start_vl), opts_parse_uint8, NULL, 1 }, {0} }; @@ -749,6 +751,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt) p_opt->no_clients_rereg = FALSE; p_opt->prefix_routes_file = strdup(OSM_DEFAULT_PREFIX_ROUTES_FILE); p_opt->consolidate_ipv6_snm_req = FALSE; + p_opt->lash_start_vl = 0; subn_init_qos_options(&p_opt->qos_options, NULL); subn_init_qos_options(&p_opt->qos_ca_options, NULL); subn_init_qos_options(&p_opt->qos_sw0_options, NULL); @@ -1432,6 +1435,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) p_opts->do_mesh_analysis ? "TRUE" : "FALSE"); fprintf(out, + "# Starting VL for LASH algorithm\n" + "lash_start_vl %d\n\n", + p_opts->lash_start_vl); + + fprintf(out, "# SA database file name\nsa_db_file %s\n\n", p_opts->sa_db_file ? p_opts->sa_db_file : null_str); diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 12b5e34..5b63fd3 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -478,7 +478,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix; int *num_mst_in_lane = p_lash->num_mst_in_lane; int ***virtual_location = p_lash->virtual_location; - int min_filled_lane, max_filled_lane, trials; + int min_filled_lane, max_filled_lane, trials, max_vl; int old_min_filled_lane, old_max_filled_lane, new_num_min_lane, new_num_max_lane; unsigned int i, j; @@ -486,9 +486,11 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) int next_switch2, output_link2; int stop = 0, cycle_found; int cycle_found2; + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; - max_filled_lane = 0; - min_filled_lane = lanes_needed - 1; + max_filled_lane = start_vl; + max_vl = lanes_needed + start_vl; + min_filled_lane = max_vl - 1; trials = num_mst_in_lane[max_filled_lane]; if (lanes_needed == 1) @@ -590,7 +592,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) new_num_min_lane = MAX_INT; new_num_max_lane = 0; - for (i = 0; i < lanes_needed; i++) { + for (i = start_vl; i < max_vl; i++) { if (num_mst_in_lane[i] < new_num_min_lane) { new_num_min_lane = num_mst_in_lane[i]; @@ -674,11 +676,12 @@ static void free_lash_structures(lash_t * p_lash) unsigned int i, j, k; unsigned num_switches = p_lash->num_switches; osm_log_t *p_log = &p_lash->p_osm->log; + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; OSM_LOG_ENTER(p_log); /* free cdg_vertex_matrix */ - for (i = 0; i < p_lash->vl_min; i++) { + for (i = start_vl; i < p_lash->vl_min; i++) { for (j = 0; j < num_switches; j++) { for (k = 0; k < num_switches; k++) if (p_lash->cdg_vertex_matrix[i][j][k]) @@ -715,13 +718,14 @@ static int init_lash_structures(lash_t * p_lash) osm_log_t *p_log = &p_lash->p_osm->log; int status = 0; unsigned int i, j, k; + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; OSM_LOG_ENTER(p_log); /* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */ p_lash->cdg_vertex_matrix = (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****)); - for (i = 0; i < vl_min; i++) { + for (i = start_vl; i < vl_min; i++) { p_lash->cdg_vertex_matrix[i] = (cdg_vertex_t ***) malloc(num_switches * sizeof(cdg_vertex_t ***)); @@ -730,7 +734,7 @@ static int init_lash_structures(lash_t * p_lash) goto Exit_Mem_Error; } - for (i = 0; i < vl_min; i++) { + for (i = start_vl; i < vl_min; i++) { for (j = 0; j < num_switches; j++) { p_lash->cdg_vertex_matrix[i][j] = (cdg_vertex_t **) malloc(num_switches * @@ -804,6 +808,8 @@ static int lash_core(lash_t * p_lash) int cycle_found2 = 0; int status = 0; int *switch_bitmap = NULL; /* Bitmap to check if we have processed this pair */ + int max_vl; + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; OSM_LOG_ENTER(p_log); @@ -838,11 +844,14 @@ static int lash_core(lash_t * p_lash) } for (i = 0; i < num_switches; i++) { - for (dest_switch = 0; dest_switch < num_switches; dest_switch++) + for (dest_switch = 0; dest_switch < num_switches; dest_switch++) { + max_vl = lanes_needed + start_vl; + if (max_vl > p_lash->vl_min) + goto Error_Not_Enough_Lanes; if (dest_switch != i && switch_bitmap[i * num_switches + dest_switch] == 0) { - v_lane = 0; + v_lane = start_vl; stop = 0; - while (v_lane < lanes_needed && stop == 0) { + while (v_lane < max_vl && stop == 0) { generate_cdg_for_sp(p_lash, i, dest_switch, v_lane); generate_cdg_for_sp(p_lash, dest_switch, i, v_lane); @@ -906,7 +915,8 @@ static int lash_core(lash_t * p_lash) switches[dest_switch]->routing_table[i].lane = v_lane; if (cycle_found == 1 || cycle_found2 == 1) { - if (++lanes_needed > p_lash->vl_min) + lanes_needed++; + if (start_vl + lanes_needed > p_lash->vl_min) goto Error_Not_Enough_Lanes; generate_cdg_for_sp(p_lash, i, dest_switch, v_lane); @@ -926,19 +936,24 @@ static int lash_core(lash_t * p_lash) switch_bitmap[i * num_switches + dest_switch] = 1; switch_bitmap[dest_switch * num_switches + i] = 1; } + } } - OSM_LOG(p_log, OSM_LOG_INFO, - "Lanes needed: %d, Balancing\n", lanes_needed); + max_vl = lanes_needed + start_vl; + if (max_vl > p_lash->vl_min) + goto Error_Not_Enough_Lanes; - for (i = 0; i < lanes_needed; i++) { + for (i = start_vl; i < max_vl; i++) { OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", i, p_lash->num_mst_in_lane[i]); } + OSM_LOG(p_log, OSM_LOG_INFO, + "Lanes needed: %d, Balancing\n", lanes_needed); + balance_virtual_lanes(p_lash, lanes_needed); - for (i = 0; i < lanes_needed; i++) { + for (i = start_vl; i < max_vl; i++) { OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", i, p_lash->num_mst_in_lane[i]); } @@ -948,8 +963,9 @@ static int lash_core(lash_t * p_lash) Error_Not_Enough_Lanes: status = -1; OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4D02: " - "Lane requirements (%d) exceed available lanes (%d)\n", - lanes_needed, p_lash->vl_min); + "Lane requirements (%d) exceed available lanes (%d)" + " with starting lane (%d)\n", + lanes_needed, p_lash->vl_min, start_vl); Exit: if (switch_bitmap) free(switch_bitmap); @@ -1286,7 +1302,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port, src_id = get_lash_id(p_sw); if (src_id == dst_id) - return OSM_DEFAULT_SL; + return p_osm->subn.opt.lash_start_vl; return (uint8_t) ((switch_t *) p_sw->priv)->routing_table[dst_id].lane; } From hnrose at comcast.net Mon Jul 20 05:46:47 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Mon, 20 Jul 2009 08:46:47 -0400 Subject: [ofa-general] [PATCH] opensm/osm_mesh.c: Reorder switches for lash Message-ID: <20090720124647.GA6093@comcast.net> Signed-off-by: Robert Pearson Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_mesh.c b/opensm/opensm/osm_mesh.c index 23fad87..a463ca9 100644 --- a/opensm/opensm/osm_mesh.c +++ b/opensm/opensm/osm_mesh.c @@ -1272,6 +1272,80 @@ static int reorder_links(lash_t *p_lash, mesh_t *mesh) } /* + * compare two switches in a sort + */ + +/* Sort switches never gets called more than once so we can use a + static structure to keep our context. */ +static struct { + lash_t *p_lash; + mesh_t *mesh; +} sort_ctx; + +static int compare_switch(const void *p1, const void *p2) +{ + int i, j, d; + int dimension = sort_ctx.mesh->dimension; + switch_t *s1 = sort_ctx.p_lash->switches[*(int *)p1]; + switch_t *s2 = sort_ctx.p_lash->switches[*(int *)p2]; + + for (i = 0; i < dimension; i++) { + j = sort_ctx.mesh->dim_order[i]; + d = s1->node->coord[j] - s2->node->coord[j]; + + if (d > 0) + return 1; + + if (d < 0) + return -1; + } + + return 0; +} + +/* + * sort_switches - reorder switch array in p_lash + */ +static void sort_switches(lash_t *p_lash, mesh_t *mesh) +{ + int i, j; + int num_switches = p_lash->num_switches; + int *index, *reverse; + switch_t *s; + switch_t **switches; + + index = calloc(num_switches, sizeof(int)); + reverse = calloc(num_switches, sizeof(int)); + switches = calloc(num_switches, sizeof(switch_t *)); + + for (i = 0; i < num_switches; i++) + index[i] = i; + + sort_ctx.mesh = mesh; + sort_ctx.p_lash = p_lash; + qsort(index, num_switches, sizeof(int), compare_switch); + + for (i = 0; i < num_switches; i++) + reverse[index[i]] = i; + + for (i = 0; i < num_switches; i++) { + s = p_lash->switches[index[i]]; + switches[i] = s; + s->id = i; + for (j = 0; j < s->node->num_links; j++) + s->node->links[j]->switch_id = + reverse[s->node->links[j]->switch_id]; + } + + for (i = 0; i < num_switches; i++) + p_lash->switches[i] = switches[i]; + + free(switches); + free(index); + free(reverse); +} + +/* * osm_mesh_delete - free per mesh resources */ static void mesh_delete(mesh_t *mesh) @@ -1470,6 +1544,8 @@ int osm_do_mesh_analysis(lash_t *p_lash) if (reorder_links(p_lash, mesh)) goto err; + sort_switches(p_lash, mesh); + p = buf; p += sprintf(p, "found "); for (i = 0; i < mesh->dimension; i++) From hal.rosenstock at gmail.com Mon Jul 20 07:17:47 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 20 Jul 2009 10:17:47 -0400 Subject: [ofa-general] [PATCH 1/4] multicast multiplexing - definitions for new data types, and infrastructure functions In-Reply-To: <20090719151527.GM22967@me> References: <4A3A387A.2020509@Voltaire.COM> <4A4747B2.2030907@Voltaire.COM> <20090719151527.GM22967@me> Message-ID: On Sun, Jul 19, 2009 at 11:15 AM, Sasha Khapyorsky wrote: > On 09:38 Thu 16 Jul     , Hal Rosenstock wrote: >> > +static osm_mgrp_port_t *osm_mgrp_port_new(ib_net64_t port_guid) >> > +{ >> > + ?? ?? ?? osm_mgrp_port_t *p_mgrp_port = >> > + ?? ?? ?? ?? ?? (osm_mgrp_port_t *) malloc(sizeof(osm_mgrp_port_t)); >> > + ?? ?? ?? if (!p_mgrp_port) { >> > + ?? ?? ?? ?? ?? ?? ?? return NULL; >> > + ?? ?? ?? } >> > + ?? ?? ?? memset(p_mgrp_port, 0, sizeof(*p_mgrp_port)); >> >> Minor - could use calloc rather than malloc/memset here. > > BTW what is the clear benefit? It's minor but it does save the overhead of an extra call into library. -- Hal > Sasha From hal.rosenstock at gmail.com Mon Jul 20 07:20:07 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 20 Jul 2009 10:20:07 -0400 Subject: [ofa-general] Re: [PATCH] opensm/osm_perfmgr.c: In perfmgr_send_pc_mad, only set CounterSelect when Set method is used In-Reply-To: <20090719133228.GF22967@me> References: <20090714140427.GA29521@comcast.net> <20090715230600.GB12078@me> <20090719133228.GF22967@me> Message-ID: On Sun, Jul 19, 2009 at 9:32 AM, Sasha Khapyorsky wrote: > On 19:25 Wed 15 Jul     , Hal Rosenstock wrote: >> On Wed, Jul 15, 2009 at 7:06 PM, Sasha Khapyorsky wrote: >> > On 10:04 Tue 14 Jul ?? ?? , Hal Rosenstock wrote: >> >> >> >> Signed-off-by: Hal Rosenstock >> >> --- >> >> diff --git a/opensm/opensm/osm_perfmgr.c b/opensm/opensm/osm_perfmgr.c >> >> index ecfdbda..0437d47 100644 >> >> --- a/opensm/opensm/osm_perfmgr.c >> >> +++ b/opensm/opensm/osm_perfmgr.c >> >> @@ -376,7 +376,8 @@ static ib_api_status_t perfmgr_send_pc_mad(osm_perfmgr_t * perfmgr, >> >> ?? ?? ?? port_counter = (ib_port_counters_t *) & pm_mad->data; >> >> ?? ?? ?? memset(port_counter, 0, sizeof(*port_counter)); >> >> ?? ?? ?? port_counter->port_select = port; >> >> - ?? ?? port_counter->counter_select = 0xFFFF; >> >> + ?? ?? if (mad_method == IB_MAD_METHOD_SET) >> >> + ?? ?? ?? ?? ?? ?? port_counter->counter_select = 0xFFFF; >> > >> > Could you explain why? >> >> CounterSelect is only valid on a Set. > > Then what was wrong with an initialization on a Get? Your words not mine :-) I didn't say anything was wrong. Both are right but why initialize something that doesn't need it. Also, this is future looking as I expect more of such shortly so at what point is it worth it ? -- Hal > Sasha > From sashak at voltaire.com Mon Jul 20 09:24:48 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 19:24:48 +0300 Subject: [ofa-general] Re: [PATCHv2] opensm/lash: Set minimum VL for LASH to use In-Reply-To: References: <20090706154643.GA17356@comcast.net> <20090720115728.GA22967@me> Message-ID: <20090720162448.GD31735@me> On 09:32 Mon 20 Jul , Hal Rosenstock wrote: > > > Wouldn't it be better instead of allocating extra memory for "disabled" > > (0 - lash_vl_start) VLs range and tracking 'max_vl' in many places just > > to setup lash->vl_min at end of discover_network_properties() as vl_min - > > lash_vl_start and to increase routing_table[i].lane value by > > lash_vl_start at end of lash cycle? > > Something like that could be done. Does it save significant memory > (haven't looked yet to see exactly) ? Not sure about memory saving significance, but guess that this will simplify (actually leave "as is") the flows and will keep lash core unaffected by this addition. > OK as a subsequent patch to this > change ? I don't think - this seems like a different approach for me. Sasha From sashak at voltaire.com Mon Jul 20 09:29:36 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 19:29:36 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_perfmgr.c: In perfmgr_send_pc_mad, only set CounterSelect when Set method is used In-Reply-To: References: <20090714140427.GA29521@comcast.net> <20090715230600.GB12078@me> <20090719133228.GF22967@me> Message-ID: <20090720162936.GE31735@me> On 10:20 Mon 20 Jul , Hal Rosenstock wrote: > > > > Then what was wrong with an initialization on a Get? > > Your words not mine :-) I didn't say anything was wrong. Both are > right but why initialize something that doesn't need it. In order to save not actually "if ()" statement. > Also, this is > future looking as I expect more of such shortly so at what point is it > worth it ? What do you mean? Sasha From sashak at voltaire.com Mon Jul 20 09:32:55 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 19:32:55 +0300 Subject: [ofa-general] Re: [PATCH] infiniband-diags/ibtracert.c: Fix linearcap test in switch_lookup In-Reply-To: <20090720125847.GA9987@comcast.net> References: <20090720125847.GA9987@comcast.net> Message-ID: <20090720163255.GF31735@me> On 08:58 Mon 20 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From hal.rosenstock at gmail.com Mon Jul 20 09:33:59 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 20 Jul 2009 12:33:59 -0400 Subject: [ofa-general] Re: [Repost][PATCH] opensm: Added support for select counters (xmit_wait) In-Reply-To: <20090720101711.GS22967@me> References: <4A5CA3B6.9080809@morey-chaisemartin.com> <20090719135334.GG22967@me> <4A6368B8.6050007@morey-chaisemartin.com> <20090719223409.GR22967@me> <4A641112.3070800@morey-chaisemartin.com> <20090720101711.GS22967@me> Message-ID: On Mon, Jul 20, 2009 at 6:17 AM, Sasha Khapyorsky wrote: > On 08:39 Mon 20 Jul     , Nicolas Morey-Chaisemartin wrote: >> So what is best in your mind? >> Bring everything path into the perf counter struct (meaning all plugin >> will have to change to fit the new API) or keeping the approach I proposed. > > We can add new counter to existing structure at the end and to not break > API. > > The better option is to unify API (counter structures) of course, but it > is different change. Basically using current (modified) plugin API it is > not required to put a counters to plugin, some sort of "counters ready" > event would be enough. There's also a counter validity issue (in terms of PortXmitWait) if the difference between 0 and not supported is to be indicated. -- Hal >> I'd rather choose the second solution not only because it's already >> done, but in the future I'd like to have as many information as possible >> on contention related counters (xmit_wait per VL for examples) and >> probably some day vendor specific info will appear in this struct. > > Yes, but it is different counters and you will need to add something > anyway. > > Sasha > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From sashak at voltaire.com Mon Jul 20 10:16:34 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 20 Jul 2009 20:16:34 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_mesh.c: Reorder switches for lash In-Reply-To: <20090720124647.GA6093@comcast.net> References: <20090720124647.GA6093@comcast.net> Message-ID: <20090720171634.GG31735@me> On 08:46 Mon 20 Jul , Hal Rosenstock wrote: > Any comment about what this patch is supposed to do? > Signed-off-by: Robert Pearson > Signed-off-by: Hal Rosenstock > --- > diff --git a/opensm/opensm/osm_mesh.c b/opensm/opensm/osm_mesh.c > index 23fad87..a463ca9 100644 > --- a/opensm/opensm/osm_mesh.c > +++ b/opensm/opensm/osm_mesh.c > @@ -1272,6 +1272,80 @@ static int reorder_links(lash_t *p_lash, mesh_t *mesh) > } > > /* > + * compare two switches in a sort > + */ > + > +/* Sort switches never gets called more than once so we can use a > + static structure to keep our context. */ > +static struct { > + lash_t *p_lash; > + mesh_t *mesh; > +} sort_ctx; This breaks reenterability. Sasha > + > +static int compare_switch(const void *p1, const void *p2) > +{ > + int i, j, d; > + int dimension = sort_ctx.mesh->dimension; > + switch_t *s1 = sort_ctx.p_lash->switches[*(int *)p1]; > + switch_t *s2 = sort_ctx.p_lash->switches[*(int *)p2]; > + > + for (i = 0; i < dimension; i++) { > + j = sort_ctx.mesh->dim_order[i]; > + d = s1->node->coord[j] - s2->node->coord[j]; > + > + if (d > 0) > + return 1; > + > + if (d < 0) > + return -1; > + } > + > + return 0; > +} > + > +/* > + * sort_switches - reorder switch array in p_lash > + */ > +static void sort_switches(lash_t *p_lash, mesh_t *mesh) > +{ > + int i, j; > + int num_switches = p_lash->num_switches; > + int *index, *reverse; > + switch_t *s; > + switch_t **switches; > + > + index = calloc(num_switches, sizeof(int)); > + reverse = calloc(num_switches, sizeof(int)); > + switches = calloc(num_switches, sizeof(switch_t *)); > + > + for (i = 0; i < num_switches; i++) > + index[i] = i; > + > + sort_ctx.mesh = mesh; > + sort_ctx.p_lash = p_lash; > + qsort(index, num_switches, sizeof(int), compare_switch); > + > + for (i = 0; i < num_switches; i++) > + reverse[index[i]] = i; > + > + for (i = 0; i < num_switches; i++) { > + s = p_lash->switches[index[i]]; > + switches[i] = s; > + s->id = i; > + for (j = 0; j < s->node->num_links; j++) > + s->node->links[j]->switch_id = > + reverse[s->node->links[j]->switch_id]; > + } > + > + for (i = 0; i < num_switches; i++) > + p_lash->switches[i] = switches[i]; > + > + free(switches); > + free(index); > + free(reverse); > +} > + > +/* > * osm_mesh_delete - free per mesh resources > */ > static void mesh_delete(mesh_t *mesh) > @@ -1470,6 +1544,8 @@ int osm_do_mesh_analysis(lash_t *p_lash) > if (reorder_links(p_lash, mesh)) > goto err; > > + sort_switches(p_lash, mesh); > + > p = buf; > p += sprintf(p, "found "); > for (i = 0; i < mesh->dimension; i++) > From jgunthorpe at obsidianresearch.com Mon Jul 20 10:19:00 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 20 Jul 2009 11:19:00 -0600 Subject: [ofa-general] [PATCH] Clarify the syntax of the hop_weights_file Message-ID: <20090720171900.GS12693@obsidianresearch.com> - GUID is a port guid, and is specified in hex with 0x prefix - Lines with # are comments - Weights are simplex not duplex. --- opensm/man/opensm.8.in | 13 +++++++++---- 1 files changed, 9 insertions(+), 4 deletions(-) I met some people who were trying to use this and we had to look through the source to figure it out. diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in index 66d2fe6..dbf4b73 100644 --- a/opensm/man/opensm.8.in +++ b/opensm/man/opensm.8.in @@ -247,10 +247,15 @@ This option provides the means to define a set of ports equalization algorithm. .TP \fB\-w\fR, \fB\-\-hop_weights_file\fR -This option provides weighting factors per port representing a hop -cost in computing the lid matrix. The file consists of lines -containing a switch GUID, output port, and weighting factor. Any port -not listed in the file defaults to a weighting factor of 1. + +This option provides weighting factors per port representing a hop cost in +computing the lid matrix. The file consists of lines containing a switch port +GUID (specified as a 64 bit hex number, with leading 0x), output port number, +and weighting factor. Any port not listed in the file defaults to a weighting +factor of 1. Lines starting with # are comments. Weights affect only the +output route from the port, so many useful configurations will require weights +to be specified in pairs. + .TP \fB\-x\fR, \fB\-\-honor_guid2lid\fR This option forces OpenSM to honor the guid2lid file, -- 1.5.4.2 From hal.rosenstock at gmail.com Mon Jul 20 10:20:44 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 20 Jul 2009 13:20:44 -0400 Subject: [ofa-general] Re: [PATCH] opensm/osm_perfmgr.c: In perfmgr_send_pc_mad, only set CounterSelect when Set method is used In-Reply-To: <20090720162936.GE31735@me> References: <20090714140427.GA29521@comcast.net> <20090715230600.GB12078@me> <20090719133228.GF22967@me> <20090720162936.GE31735@me> Message-ID: On Mon, Jul 20, 2009 at 12:29 PM, Sasha Khapyorsky wrote: > On 10:20 Mon 20 Jul     , Hal Rosenstock wrote: >> > >> > Then what was wrong with an initialization on a Get? >> >> Your words not mine :-) I didn't say anything was wrong. Both are >> right but why initialize something that doesn't need it. > > In order to save not actually "if ()" statement. Right; it's the tradeoff of the if v. the additional initialization(s). >> Also, this is >> future looking as I expect more of such shortly so at what point is it >> worth it ? > > What do you mean? I expect more initializations like this shortly based on the patches currently being discussed on the list. -- Hal > Sasha > From hal.rosenstock at gmail.com Mon Jul 20 10:42:25 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 20 Jul 2009 13:42:25 -0400 Subject: [ofa-general] Re: [PATCHv2] opensm/lash: Set minimum VL for LASH to use In-Reply-To: <20090720162448.GD31735@me> References: <20090706154643.GA17356@comcast.net> <20090720115728.GA22967@me> <20090720162448.GD31735@me> Message-ID: On Mon, Jul 20, 2009 at 12:24 PM, Sasha Khapyorsky wrote: > On 09:32 Mon 20 Jul     , Hal Rosenstock wrote: >> >> > Wouldn't it be better instead of allocating extra memory for "disabled" >> > (0 - lash_vl_start) VLs range and tracking 'max_vl' in many places just >> > to setup lash->vl_min at end of discover_network_properties() as vl_min - >> > lash_vl_start and to increase routing_table[i].lane value by >> > lash_vl_start at end of lash cycle? >> >> Something like that could be done. Does it save significant memory >> (haven't looked yet to see exactly) ? > > Not sure about memory saving significance, but guess that this will > simplify (actually leave "as is") the flows and will keep lash core > unaffected by this addition. Are you sure about the simplification ? It might leave lash_core flows as is but I think that the start VL pops up and complicates things elsewhere. This seemed like a pretty straightforward change to me. -- Hal >> OK as a subsequent patch to this >> change ? > > I don't think - this seems like a different approach for me. > > Sasha > From rdreier at cisco.com Mon Jul 20 10:48:55 2009 From: rdreier at cisco.com (Roland Dreier) Date: Mon, 20 Jul 2009 10:48:55 -0700 Subject: [ofa-general] Re: [PATCH ibverbs] Make the gid argument to ibv_attach_mcast and ibv_detach_mcast const In-Reply-To: <20090719052634.GA1149@obsidianresearch.com> (Jason Gunthorpe's message of "Sat, 18 Jul 2009 23:26:34 -0600") References: <20090719052634.GA1149@obsidianresearch.com> Message-ID: > -int ibv_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); > +int ibv_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); Seems fine to me... I can't think of any risk of this breaking compatibility anywhere, but maybe I'm not being paranoid enough? - R. From sdake at redhat.com Mon Jul 20 10:55:59 2009 From: sdake at redhat.com (Steven Dake) Date: Mon, 20 Jul 2009 10:55:59 -0700 Subject: [ofa-general] [patch] add pkgconfig support to librdma Message-ID: <1248112559.2625.210.camel@localhost.localdomain> Hi, I cloned Sean's librdmacm git repo and added pkgconfig support to it. This is useful for other downstream projects to use the pkgconfig infrastructure to detect cflag and ldflag options rather then using a bunch of complex magic incantations to determine where the proper library and header files are located. patch attached. Passes make distcheck and make install. Regards -steve -------------- next part -------------- A non-text attachment was scrubbed... Name: 0001-Add-pkgconfig-support-to-rdmacm-library.patch Type: text/x-patch Size: 3172 bytes Desc: not available URL: From sdake at redhat.com Mon Jul 20 11:17:29 2009 From: sdake at redhat.com (Steven Dake) Date: Mon, 20 Jul 2009 11:17:29 -0700 Subject: [ofa-general] [PATCH] add pkgconfig support to ibverbs library Message-ID: <1248113849.2625.216.camel@localhost.localdomain> The attached patch adds support for pkgconfig to the ibverbs library. Cloned from Roland's kernel.org git tree. Regards -steve -------------- next part -------------- A non-text attachment was scrubbed... Name: 0001-Add-pkgconfig-support-to-ibverbs-library.patch Type: text/x-patch Size: 3253 bytes Desc: not available URL: From jgunthorpe at obsidianresearch.com Mon Jul 20 11:43:51 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 20 Jul 2009 12:43:51 -0600 Subject: [ofa-general] [PATCH] add pkgconfig support to ibverbs library In-Reply-To: <1248113849.2625.216.camel@localhost.localdomain> References: <1248113849.2625.216.camel@localhost.localdomain> Message-ID: <20090720184351.GT12693@obsidianresearch.com> On Mon, Jul 20, 2009 at 11:17:29AM -0700, Steven Dake wrote: > The attached patch adds support for pkgconfig to the ibverbs library. > > Cloned from Roland's kernel.org git tree. Erm, am I missing something? Shouldn't your patches include some use of the .pc files for libraries downstream of libibverbs, like librdmacm? I often install all this stuff in a prefix and it sure would be nice if the compilations didn't need as much hand holding beyond --prefix to get it all setup. > diff --git a/configure.in b/configure.in > index 374aae7..c54ca4b 100644 > +++ b/configure.in > @@ -6,6 +6,7 @@ AC_CONFIG_SRCDIR([src/ibverbs.h]) > AC_CONFIG_AUX_DIR(config) > AM_CONFIG_HEADER(config.h) > AM_INIT_AUTOMAKE(libibverbs, 1.1.2) > +AC_CHECK_PROGS([PKGCONFIG], [pkg-config]) Is that necessary, pkg-config is never called? Jason From jgunthorpe at obsidianresearch.com Mon Jul 20 11:53:33 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 20 Jul 2009 12:53:33 -0600 Subject: [ofa-general] Re: [PATCH ibverbs] Make the gid argument to ibv_attach_mcast and ibv_detach_mcast const In-Reply-To: References: <20090719052634.GA1149@obsidianresearch.com> Message-ID: <20090720185333.GU12693@obsidianresearch.com> On Mon, Jul 20, 2009 at 10:48:55AM -0700, Roland Dreier wrote: > > > -int ibv_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); > > +int ibv_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); > > Seems fine to me... I can't think of any risk of this breaking > compatibility anywhere, but maybe I'm not being paranoid enough? In C constness has no effect on the symbol binding or calling convention, so it is ok - AFAIK. I have also patches for mlx4 and mthca to suppress the compiler warning that results from this patch. ipath is OK as is, and I'm not sure where the iwarp stuff lives.. Jason From sdake at redhat.com Mon Jul 20 12:07:19 2009 From: sdake at redhat.com (Steven Dake) Date: Mon, 20 Jul 2009 12:07:19 -0700 Subject: [ofa-general] [PATCH] add pkgconfig support to ibverbs library In-Reply-To: <20090720184351.GT12693@obsidianresearch.com> References: <1248113849.2625.216.camel@localhost.localdomain> <20090720184351.GT12693@obsidianresearch.com> Message-ID: <1248116839.2625.240.camel@localhost.localdomain> On Mon, 2009-07-20 at 12:43 -0600, Jason Gunthorpe wrote: > On Mon, Jul 20, 2009 at 11:17:29AM -0700, Steven Dake wrote: > > The attached patch adds support for pkgconfig to the ibverbs library. > > > > Cloned from Roland's kernel.org git tree. > > Erm, am I missing something? Shouldn't your patches include some use > of the .pc files for libraries downstream of libibverbs, like > librdmacm? > yes librdmacm should use PKG_CHECK_MODULES to determine cflags and ldflags and I will submit a patch for this capability. I was uncertain of project policy, however. One problem with using PKG_CHECK_MODULES is that it only works properly in recent pkgconfig m4 files. In projects I maintain, we don't expect our users to run ./autogen.sh to parse the configure.ac/in file. Instead we distribute a already-autogenned version in our tarball (similar to the output from make dist). Using the PKG_CHECK_MODULES then requires up to date upstream packages of autotools for developers, but not for users (which simply run configure where the m4 macro has already been processed). Such a patch then won't also won't build on older distros (if you have concerns about autogenning your library against older versions of autotools). This issue caused significant problems in our developer community (downstream user of ibverbs) as everyone had to upgrade to recent autotools. There are mechanisms to use pkg-config directly in the configure script to avoid this defect, but I don't know how to do this (C jockey here, not too experienced with autotools). > I often install all this stuff in a prefix and it sure would be nice > if the compilations didn't need as much hand holding beyond --prefix > to get it all setup. > agree > > diff --git a/configure.in b/configure.in > > index 374aae7..c54ca4b 100644 > > +++ b/configure.in > > @@ -6,6 +6,7 @@ AC_CONFIG_SRCDIR([src/ibverbs.h]) > > AC_CONFIG_AUX_DIR(config) > > AM_CONFIG_HEADER(config.h) > > AM_INIT_AUTOMAKE(libibverbs, 1.1.2) > > +AC_CHECK_PROGS([PKGCONFIG], [pkg-config]) > > Is that necessary, pkg-config is never called? > good catch new patch attached > Jason -------------- next part -------------- A non-text attachment was scrubbed... Name: pkgconfig-ibverbs-take2.patch Type: text/x-patch Size: 2475 bytes Desc: not available URL: From jgunthorpe at obsidianresearch.com Mon Jul 20 15:34:29 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 20 Jul 2009 16:34:29 -0600 Subject: [ofa-general] [PATCH] add pkgconfig support to ibverbs library In-Reply-To: <1248116839.2625.240.camel@localhost.localdomain> References: <1248113849.2625.216.camel@localhost.localdomain> <20090720184351.GT12693@obsidianresearch.com> <1248116839.2625.240.camel@localhost.localdomain> Message-ID: <20090720223428.GW12693@obsidianresearch.com> On Mon, Jul 20, 2009 at 12:07:19PM -0700, Steven Dake wrote: > On Mon, 2009-07-20 at 12:43 -0600, Jason Gunthorpe wrote: > > On Mon, Jul 20, 2009 at 11:17:29AM -0700, Steven Dake wrote: > > > The attached patch adds support for pkgconfig to the ibverbs library. > > > > > > Cloned from Roland's kernel.org git tree. > > > > Erm, am I missing something? Shouldn't your patches include some use > > of the .pc files for libraries downstream of libibverbs, like > > librdmacm? > > > > yes librdmacm should use PKG_CHECK_MODULES to determine cflags and > ldflags and I will submit a patch for this capability. Hmm.. Is this actually usefull? It looks to me like even pkg-config doesn't interact with --prefix in a sane way, so instead of doing: CPPFLAGS="-I PREFIX" LDFLAGS="-L PREFIX" ./configure --prefix PREFIX I'd now do: PKG_CONFIG_LIBDIR=PREFIX ./configure --prefix PREFIX Is this an improvement worth the mess you went on to describe? Given that this software is completely Linux only it doesn't seem pkg-config will add much. libtool .la files already take care of the details for static linking cases, the libraries in the OFA family all have correct DT_NEEDED's and libtool adds DT_RPATH for the dynamic case, and -I is all that is needed for gcc. I've always regarded the autoconf behavior around --prefix as an annoying mis-feature.. Regards, Jason From jgunthorpe at obsidianresearch.com Mon Jul 20 15:36:02 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 20 Jul 2009 16:36:02 -0600 Subject: [ofa-general] [PATCH mthca] Remove empty stubs for detach/attach_mcast Message-ID: <20090720223602.GX12693@obsidianresearch.com> Just use ibv_cmd_* directly. Solves const correctness warnings due to changes in libibverbs Signed-off-by: Jason Gunthorpe --- src/mthca.c | 4 ++-- src/mthca.h | 2 -- src/verbs.c | 10 ---------- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/mthca.c b/src/mthca.c index e00c4ee..e33bf7f 100644 --- a/src/mthca.c +++ b/src/mthca.c @@ -127,8 +127,8 @@ static struct ibv_context_ops mthca_ctx_ops = { .destroy_qp = mthca_destroy_qp, .create_ah = mthca_create_ah, .destroy_ah = mthca_destroy_ah, - .attach_mcast = mthca_attach_mcast, - .detach_mcast = mthca_detach_mcast + .attach_mcast = ibv_cmd_attach_mcast, + .detach_mcast = ibv_cmd_detach_mcast }; static struct ibv_context *mthca_alloc_context(struct ibv_device *ibdev, int cmd_fd) diff --git a/src/mthca.h b/src/mthca.h index 66751f3..9a2e362 100644 --- a/src/mthca.h +++ b/src/mthca.h @@ -372,7 +372,5 @@ int mthca_destroy_ah(struct ibv_ah *ah); int mthca_alloc_av(struct mthca_pd *pd, struct ibv_ah_attr *attr, struct mthca_ah *ah); void mthca_free_av(struct mthca_ah *ah); -int mthca_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); -int mthca_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); #endif /* MTHCA_H */ diff --git a/src/verbs.c b/src/verbs.c index b2ef3ec..f6570c6 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -746,13 +746,3 @@ int mthca_destroy_ah(struct ibv_ah *ah) return 0; } - -int mthca_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) -{ - return ibv_cmd_attach_mcast(qp, gid, lid); -} - -int mthca_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) -{ - return ibv_cmd_detach_mcast(qp, gid, lid); -} -- 1.6.0.4 From jgunthorpe at obsidianresearch.com Mon Jul 20 15:37:24 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 20 Jul 2009 16:37:24 -0600 Subject: [ofa-general] [PATCH mlx4] Remove empty stubs for detach/attach_mcast Message-ID: <20090720223724.GY12693@obsidianresearch.com> Just use ibv_cmd_* directly. Solves const correctness warnings due to changes in libibverbs Signed-off-by: Jason Gunthorpe --- src/mlx4.c | 4 ++-- src/mlx4.h | 2 -- src/verbs.c | 10 ---------- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/mlx4.c b/src/mlx4.c index 34ece39..1295c53 100644 --- a/src/mlx4.c +++ b/src/mlx4.c @@ -94,8 +94,8 @@ static struct ibv_context_ops mlx4_ctx_ops = { .post_recv = mlx4_post_recv, .create_ah = mlx4_create_ah, .destroy_ah = mlx4_destroy_ah, - .attach_mcast = mlx4_attach_mcast, - .detach_mcast = mlx4_detach_mcast + .attach_mcast = ibv_cmd_attach_mcast, + .detach_mcast = ibv_cmd_detach_mcast }; static struct ibv_context *mlx4_alloc_context(struct ibv_device *ibdev, int cmd_fd) diff --git a/src/mlx4.h b/src/mlx4.h index 827a201..0c658cf 100644 --- a/src/mlx4.h +++ b/src/mlx4.h @@ -361,7 +361,5 @@ int mlx4_destroy_ah(struct ibv_ah *ah); int mlx4_alloc_av(struct mlx4_pd *pd, struct ibv_ah_attr *attr, struct mlx4_ah *ah); void mlx4_free_av(struct mlx4_ah *ah); -int mlx4_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); -int mlx4_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); #endif /* MLX4_H */ diff --git a/src/verbs.c b/src/verbs.c index cc179a0..2c19d93 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -651,13 +651,3 @@ int mlx4_destroy_ah(struct ibv_ah *ah) return 0; } - -int mlx4_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) -{ - return ibv_cmd_attach_mcast(qp, gid, lid); -} - -int mlx4_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) -{ - return ibv_cmd_detach_mcast(qp, gid, lid); -} -- 1.6.0.4 From sdake at redhat.com Mon Jul 20 16:12:34 2009 From: sdake at redhat.com (Steven Dake) Date: Mon, 20 Jul 2009 16:12:34 -0700 Subject: [ofa-general] [PATCH] add pkgconfig support to ibverbs library In-Reply-To: <20090720223428.GW12693@obsidianresearch.com> References: <1248113849.2625.216.camel@localhost.localdomain> <20090720184351.GT12693@obsidianresearch.com> <1248116839.2625.240.camel@localhost.localdomain> <20090720223428.GW12693@obsidianresearch.com> Message-ID: <1248131554.2625.265.camel@localhost.localdomain> On Mon, 2009-07-20 at 16:34 -0600, Jason Gunthorpe wrote: > On Mon, Jul 20, 2009 at 12:07:19PM -0700, Steven Dake wrote: > > On Mon, 2009-07-20 at 12:43 -0600, Jason Gunthorpe wrote: > > > On Mon, Jul 20, 2009 at 11:17:29AM -0700, Steven Dake wrote: > > > > The attached patch adds support for pkgconfig to the ibverbs library. > > > > > > > > Cloned from Roland's kernel.org git tree. > > > > > > Erm, am I missing something? Shouldn't your patches include some use > > > of the .pc files for libraries downstream of libibverbs, like > > > librdmacm? > > > > > > > yes librdmacm should use PKG_CHECK_MODULES to determine cflags and > > ldflags and I will submit a patch for this capability. > > Hmm.. Is this actually usefull? It looks to me like even pkg-config > doesn't interact with --prefix in a sane way, so instead of doing: > > CPPFLAGS="-I PREFIX" LDFLAGS="-L PREFIX" ./configure --prefix PREFIX > > I'd now do: > PKG_CONFIG_LIBDIR=PREFIX ./configure --prefix PREFIX > > Is this an improvement worth the mess you went on to describe? > The librdmacm maintainer would have to decide if it is worth it for him to use pkgconfig functionality in his configure scripts. For other downstream projects that link against librdmacm or libibverbs that follow the upstream adoption model of autotools (always use latest autotools, older versions may have bugs), it has many benefits. Example: ./configure --includedir=/tmp/abc make install How does the downstream adopting application know where the include files are located in /tmp/abc? Not all distros put the include files in the same location. There are two alternatives for downstream projects. Have per-distro&version special casing in the configure script to find the libs and headers which is very painful to maintain and support. Option B is to specify the cflags and ldflags required for librdmacm and libibverbs in the configure operation (again requiring the user to be aware of where those files are stored). Neither of these mechanisms help adopting applications of rdmacm and ibverbs "become adopted" because they are then harder to distribute. As a downstream adoptor, solving the "what are the flags" in a scalable easy to use fashion is the main advantage I get with pkgconfig with minimal upstream rdmacm and ibverbs disruption. Regards -steve > Given that this software is completely Linux only it doesn't seem > pkg-config will add much. libtool .la files already take care of the > details for static linking cases, the libraries in the OFA family all > have correct DT_NEEDED's and libtool adds DT_RPATH for the dynamic > case, and -I is all that is needed for gcc. > > I've always regarded the autoconf behavior around --prefix as an > annoying mis-feature.. > > Regards, > Jason From jgunthorpe at obsidianresearch.com Mon Jul 20 17:06:48 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 20 Jul 2009 18:06:48 -0600 Subject: [ofa-general] [PATCH] add pkgconfig support to ibverbs library In-Reply-To: <1248131554.2625.265.camel@localhost.localdomain> References: <1248113849.2625.216.camel@localhost.localdomain> <20090720184351.GT12693@obsidianresearch.com> <1248116839.2625.240.camel@localhost.localdomain> <20090720223428.GW12693@obsidianresearch.com> <1248131554.2625.265.camel@localhost.localdomain> Message-ID: <20090721000648.GZ12693@obsidianresearch.com> On Mon, Jul 20, 2009 at 04:12:34PM -0700, Steven Dake wrote: > The librdmacm maintainer would have to decide if it is worth it for him > to use pkgconfig functionality in his configure scripts. For other > downstream projects that link against librdmacm or libibverbs that > follow the upstream adoption model of autotools (always use latest > autotools, older versions may have bugs), it has many benefits. > > Example: > ./configure --includedir=/tmp/abc > make install > How does the downstream adopting application know where the include > files are located in /tmp/abc? I don't see how that would work even with pkg-config and the PKG_CHECK_MODULES macro. The dumb configure misfeature is that all the options like --includedir that chain off --prefix are the INSTALLATION paths for the thing you are configuring, not the SEARCH paths needed to build. pkg-config's m4 seems to respect that and doesn't add those options to the search path for .pc's > Not all distros put the include files in the same location. There > are two alternatives for downstream projects. No.. distros have to put this stuff in the system search path, under the prefix we've picked so #include works ok, and -libverbs works ok. Anything else is madness, I don't know any distro that would do otherwise - it is contrary to the entire point of a distro... The only case to worry about is when someone installs this stuff outside the system search path, ie your /tmp/abc/ or some-such, in which case, I think you are just trading one obscure environment option (CPPFLAGS/LDFLAGS) for another (PKG_CONFIG_LIBDIR). > As a downstream adoptor, solving the "what are the flags" in a scalable > easy to use fashion is the main advantage I get with pkgconfig with > minimal upstream rdmacm and ibverbs disruption. Some of the gnome software pkg-config was made for have interesting, non-simple requirements that pkg-config handles - but I don't see anything in the OFA packages that matches that stuff. These are all ment to be system libraries that don't have special user compilation requirements or flags when installed in the system search path. Any distro that does otherwise is broken.. Anyhow, sounds like adding the .pc files to the builds isn't a big deal and some minor cases do work better. But I guess I don't see a benefit from using PKG_CHECK_MODULES, and you've outlined enough reasons to avoid it.. BTW, libibcm, libibumad and libibmad should also all get .pc files for consistency. :) Jason From sashak at voltaire.com Mon Jul 20 17:44:20 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Tue, 21 Jul 2009 03:44:20 +0300 Subject: [ofa-general] Re: [PATCH] Clarify the syntax of the hop_weights_file In-Reply-To: <20090720171900.GS12693@obsidianresearch.com> References: <20090720171900.GS12693@obsidianresearch.com> Message-ID: <20090721004420.GH31735@me> On 11:19 Mon 20 Jul , Jason Gunthorpe wrote: > - GUID is a port guid, and is specified in hex with 0x prefix > - Lines with # are comments > - Weights are simplex not duplex. Applied. Thanks. Please next time add your "Signed-off-by:" line. Sasha From sashak at voltaire.com Mon Jul 20 17:48:38 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Tue, 21 Jul 2009 03:48:38 +0300 Subject: [ofa-general] Re: [PATCHv2] opensm/lash: Set minimum VL for LASH to use In-Reply-To: References: <20090706154643.GA17356@comcast.net> <20090720115728.GA22967@me> <20090720162448.GD31735@me> Message-ID: <20090721004838.GI31735@me> On 13:42 Mon 20 Jul , Hal Rosenstock wrote: > > Are you sure about the simplification ? It might leave lash_core flows > as is but I think that the start VL pops up and complicates things > elsewhere. Where for example? For me it looks that you will only need to setup lash->vl_min value as (vl_min - lash_start_vl) at end of discover_network_properties() and update sw->routing_table[n].lane at end of lash cycle - that is all. No? Sasha From sdake at redhat.com Mon Jul 20 19:01:01 2009 From: sdake at redhat.com (Steven Dake) Date: Mon, 20 Jul 2009 19:01:01 -0700 Subject: [ofa-general] [patch] add pkgconfig infrastructure to libibcm Message-ID: <1248141661.2625.267.camel@localhost.localdomain> similar to other patches to add pkgconfig support to libibcm. Regards -steve -------------- next part -------------- A non-text attachment was scrubbed... Name: 0001-Add-pkgconfig-infrastructure-to-libibcm.patch Type: text/x-patch Size: 2701 bytes Desc: not available URL: From sdake at redhat.com Mon Jul 20 19:02:45 2009 From: sdake at redhat.com (Steven Dake) Date: Mon, 20 Jul 2009 19:02:45 -0700 Subject: [ofa-general] [PATCH] add pkgconfig support to ibverbs library In-Reply-To: <20090721000648.GZ12693@obsidianresearch.com> References: <1248113849.2625.216.camel@localhost.localdomain> <20090720184351.GT12693@obsidianresearch.com> <1248116839.2625.240.camel@localhost.localdomain> <20090720223428.GW12693@obsidianresearch.com> <1248131554.2625.265.camel@localhost.localdomain> <20090721000648.GZ12693@obsidianresearch.com> Message-ID: <1248141765.2625.269.camel@localhost.localdomain> On Mon, 2009-07-20 at 18:06 -0600, Jason Gunthorpe wrote: > On Mon, Jul 20, 2009 at 04:12:34PM -0700, Steven Dake wrote: > > The librdmacm maintainer would have to decide if it is worth it for him > > to use pkgconfig functionality in his configure scripts. For other > > downstream projects that link against librdmacm or libibverbs that > > follow the upstream adoption model of autotools (always use latest > > autotools, older versions may have bugs), it has many benefits. > > > > Example: > > ./configure --includedir=/tmp/abc > > make install > > > How does the downstream adopting application know where the include > > files are located in /tmp/abc? > > I don't see how that would work even with pkg-config and the > PKG_CHECK_MODULES macro. The dumb configure misfeature is that all the > options like --includedir that chain off --prefix are the INSTALLATION > paths for the thing you are configuring, not the SEARCH paths needed > to build. > > pkg-config's m4 seems to respect that and doesn't add those options to > the search path for .pc's > > > Not all distros put the include files in the same location. There > > are two alternatives for downstream projects. > > No.. distros have to put this stuff in the system search path, under > the prefix we've picked so #include works ok, and > -libverbs works ok. Anything else is madness, I don't know any distro > that would do otherwise - it is contrary to the entire point of a > distro... > > The only case to worry about is when someone installs this stuff > outside the system search path, ie your /tmp/abc/ or some-such, in > which case, I think you are just trading one obscure environment > option (CPPFLAGS/LDFLAGS) for another (PKG_CONFIG_LIBDIR). > > > As a downstream adoptor, solving the "what are the flags" in a scalable > > easy to use fashion is the main advantage I get with pkgconfig with > > minimal upstream rdmacm and ibverbs disruption. > > Some of the gnome software pkg-config was made for have interesting, > non-simple requirements that pkg-config handles - but I don't see > anything in the OFA packages that matches that stuff. These are all > ment to be system libraries that don't have special user compilation > requirements or flags when installed in the system search path. Any > distro that does otherwise is broken.. > > Anyhow, sounds like adding the .pc files to the builds isn't a big > deal and some minor cases do work better. But I guess I don't see a > benefit from using PKG_CHECK_MODULES, and you've outlined enough > reasons to avoid it.. > > BTW, libibcm, libibumad and libibmad should also all get .pc files for > consistency. :) > have git locations for libibumad and libibmad? Regards -steve > Jason From jgunthorpe at obsidianresearch.com Mon Jul 20 19:13:25 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 20 Jul 2009 20:13:25 -0600 Subject: [ofa-general] [PATCH] add pkgconfig support to ibverbs library In-Reply-To: <1248141765.2625.269.camel@localhost.localdomain> References: <1248113849.2625.216.camel@localhost.localdomain> <20090720184351.GT12693@obsidianresearch.com> <1248116839.2625.240.camel@localhost.localdomain> <20090720223428.GW12693@obsidianresearch.com> <1248131554.2625.265.camel@localhost.localdomain> <20090721000648.GZ12693@obsidianresearch.com> <1248141765.2625.269.camel@localhost.localdomain> Message-ID: <20090721021325.GC20290@obsidianresearch.com> On Mon, Jul 20, 2009 at 07:02:45PM -0700, Steven Dake wrote: > have git locations for libibumad and libibmad? Hmm, these are part of opensm git://git.openfabrics.org/~sashak/management.git Jason From ogerlitz at voltaire.com Mon Jul 20 23:56:12 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Tue, 21 Jul 2009 09:56:12 +0300 Subject: [ofa-general] Re: RDMA_CM support for IPv6 In-Reply-To: <4A649960.7090404@linux.vnet.ibm.com> References: <4A5F66B3.90207@linux.vnet.ibm.com> <4A62E6B9.9030105@mellanox.co.il> <4A649960.7090404@linux.vnet.ibm.com> Message-ID: <4A65668C.8070205@voltaire.com> Pradeep Satyanarayana wrote: > I see it in 2.6.30 > Did you had the chance the test this as well (you can use rping) Or. From vlad at lists.openfabrics.org Tue Jul 21 02:37:20 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Tue, 21 Jul 2009 02:37:20 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090721-0200 daily build status Message-ID: <20090721093720.107BA102054A@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2092: error: 'class_device_attr_ibdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2094: error: 'class_device_attr_port' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2100: error: implicit declaration of function 'class_device_unregister' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -m64 -mno-red-zone -mcmodel=kernel -funit-at-a-time -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2092: error: 'class_device_attr_ibdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2094: error: 'class_device_attr_port' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2100: error: implicit declaration of function 'class_device_unregister' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.27_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1700: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1722: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Os -msoft-float -pipe -mminimal-toc -mtraceback=none -mcall-aixdesc -mtune=power4 -mno-altivec -funit-at-a-time -mstring -Wa,-maltivec -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/include -ggdb -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(sdp_main)" -D"KBUILD_MODNAME=KBUILD_STR(ib_sdp)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/.tmp_sdp_main.o /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c: In function 'sdp_sendmsg': /home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.c:1721: error: implicit declaration of function 'rdtscll' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp/sdp_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/sdp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090721-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From ogerlitz at voltaire.com Tue Jul 21 04:46:58 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Tue, 21 Jul 2009 14:46:58 +0300 Subject: [ofa-general] can an ib-bonding slave work independently? In-Reply-To: <20090716180024.GB5930@sun.com> References: <20090715025215.GA4322@sun.com> <4A5D897F.20708@Voltaire.com> <20090716180024.GB5930@sun.com> Message-ID: <4A65AAB2.4000009@voltaire.com> Isaac Huang wrote: > My understanding, which I'd be happy to find false, was that RDMA cmid couldn't be created and bound to a bonding device. If true, you're wrong, basically, the bonding device is being seen by the rdma cm as a clone of its active slave, e.g see slide 8 of my talk @ www.openfabrics.org/archives/spring2007sonoma/Tuesday%20May%201/Gerlitz%20bonding-sonoma-april-26.ppt > Rdma_resolve_addr over a cmid bound to the slave also failed with RDMA_CM_EVENT_ADDR_ERROR status -ETIMEDOUT > But tcpdump output on 'ib2' did show the ARP request and response: > ... arp who-has 10.1.1.132 tell 10.1.13.49 hardware #32 > ... arp reply 10.1.1.132 is-at 80:00:00:49:fe:80:00:00:00:00:00:10:00:03:ba:00:01:00:fb:8a > The response seemed to have been dropped by ARP for some reason Yes, I believe that these arp replies are being dropped, most likely in the netif_receive_skb --> skb_bond_should_drop flow, or if you are running with pretty old kernel as of the slave having the IFF_NOARP bit set in its flags mask. This explains why arp resolution on the slave fails. Or. From sashak at voltaire.com Tue Jul 21 08:32:28 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Tue, 21 Jul 2009 18:32:28 +0300 Subject: [ofa-general] [PATCH] opensm: find MC group by MGID using fleximap Message-ID: <20090721153228.GJ31735@me> Find MC group by MGID value using fleximap, rather than looping over mlid array. Signed-off-by: Sasha Khapyorsky --- Did this some time ago... Not heavy tested yet. opensm/include/opensm/osm_multicast.h | 5 +++-- opensm/include/opensm/osm_subnet.h | 2 ++ opensm/opensm/osm_mcast_mgr.c | 2 ++ opensm/opensm/osm_sa_mcmember_record.c | 30 ++++++++++++------------------ opensm/opensm/osm_subnet.c | 8 ++++++++ 5 files changed, 27 insertions(+), 20 deletions(-) diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h index a871306..9a47de5 100644 --- a/opensm/include/opensm/osm_multicast.h +++ b/opensm/include/opensm/osm_multicast.h @@ -45,6 +45,7 @@ #include #include +#include #include #include #include @@ -120,7 +121,7 @@ typedef struct osm_mcast_mgr_ctxt { * SYNOPSIS */ typedef struct osm_mgrp { - cl_map_item_t map_item; + cl_fmap_item_t map_item; ib_net16_t mlid; osm_mtree_node_t *p_root; cl_qmap_t mcm_port_tbl; @@ -134,7 +135,7 @@ typedef struct osm_mgrp { /* * FIELDS * map_item -* Map Item for qmap linkage. Must be first element!! +* Map Item for fmap linkage. Must be first element!! * * mlid * The network ordered LID of this Multicast Group (must be diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 59a32ad..c055add 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -46,6 +46,7 @@ #include #include +#include #include #include #include @@ -508,6 +509,7 @@ typedef struct osm_subn { boolean_t first_time_master_sweep; boolean_t coming_out_of_standby; unsigned need_update; + cl_fmap_t mgrp_mgid_tbl; void *mgroups[IB_LID_MCAST_END_HO - IB_LID_MCAST_START_HO + 1]; } osm_subn_t; /* diff --git a/opensm/opensm/osm_mcast_mgr.c b/opensm/opensm/osm_mcast_mgr.c index ea49588..268e1ab 100644 --- a/opensm/opensm/osm_mcast_mgr.c +++ b/opensm/opensm/osm_mcast_mgr.c @@ -1093,6 +1093,8 @@ static ib_api_status_t mcast_mgr_process_mgrp(osm_sm_t * sm, cl_ntoh16(p_mgrp->mlid)); sm->p_subn->mgroups[cl_ntoh16(p_mgrp->mlid) - IB_LID_MCAST_START_HO] = NULL; + cl_fmap_remove_item(&sm->p_subn->mgrp_mgid_tbl, + &p_mgrp->map_item); osm_mgrp_delete(p_mgrp); } diff --git a/opensm/opensm/osm_sa_mcmember_record.c b/opensm/opensm/osm_sa_mcmember_record.c index 5543221..a9e0a3b 100644 --- a/opensm/opensm/osm_sa_mcmember_record.c +++ b/opensm/opensm/osm_sa_mcmember_record.c @@ -148,6 +148,8 @@ static void cleanup_mgrp(IN osm_sa_t * sa, osm_mgrp_t * mgrp) if (cl_is_qmap_empty(&mgrp->mcm_port_tbl) && !mgrp->well_known) { sa->p_subn->mgroups[cl_ntoh16(mgrp->mlid) - IB_LID_MCAST_START_HO] = NULL; + cl_fmap_remove_item(&sa->p_subn->mgrp_mgid_tbl, + &mgrp->map_item); osm_mgrp_delete(mgrp); } } @@ -922,9 +924,14 @@ ib_api_status_t osm_mcmr_rcv_create_new_mgrp(IN osm_sa_t * sa, "Destroying it first\n", cl_ntoh16(mlid)); sa->p_subn->mgroups[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO] = NULL; + cl_fmap_remove_item(&sa->p_subn->mgrp_mgid_tbl, + &p_prev_mgrp->map_item); osm_mgrp_delete(p_prev_mgrp); } + cl_fmap_insert(&sa->p_subn->mgrp_mgid_tbl, + &(*pp_mgrp)->mcmember_rec.mgid, &(*pp_mgrp)->map_item); + sa->p_subn->mgroups[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO] = *pp_mgrp; Exit: @@ -933,18 +940,6 @@ Exit: } /********************************************************************** - *********************************************************************/ -static unsigned match_mgrp_by_mgid(IN osm_mgrp_t * p_mgrp, ib_gid_t * mgid) -{ - /* ignore groups marked for deletion */ - if (p_mgrp->to_be_deleted || - memcmp(&p_mgrp->mcmember_rec.mgid, mgid, sizeof(ib_gid_t))) - return 0; - else - return 1; -} - -/********************************************************************** **********************************************************************/ #define PREFIX_MASK CL_HTON64(0xff10ffff0000ffffULL) #define PREFIX_SIGNATURE CL_HTON64(0xff10601b00000000ULL) @@ -968,7 +963,7 @@ static unsigned match_and_update_ipv6_snm_mgid(ib_gid_t * mgid) osm_mgrp_t *osm_get_mgrp_by_mgid(IN osm_sa_t * sa, IN ib_gid_t * p_mgid) { - int i; + osm_mgrp_t *mg; if (sa->p_subn->opt.consolidate_ipv6_snm_req && match_and_update_ipv6_snm_mgid(p_mgid)) { @@ -979,11 +974,10 @@ osm_mgrp_t *osm_get_mgrp_by_mgid(IN osm_sa_t * sa, IN ib_gid_t * p_mgid) sizeof gid_str)); } - for (i = 0; i <= sa->p_subn->max_mcast_lid_ho - IB_LID_MCAST_START_HO; - i++) - if (sa->p_subn->mgroups[i] && - match_mgrp_by_mgid(sa->p_subn->mgroups[i], p_mgid)) - return sa->p_subn->mgroups[i]; + mg = (osm_mgrp_t *)cl_fmap_get(&sa->p_subn->mgrp_mgid_tbl, p_mgid); + if (mg != (osm_mgrp_t *)cl_fmap_end(&sa->p_subn->mgrp_mgid_tbl) + && !mg->to_be_deleted) + return mg; return NULL; } diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index ec15f8a..e686e3c 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -394,6 +394,11 @@ static const opt_rec_t opt_tbl[] = { /********************************************************************** **********************************************************************/ +static long compar_mgids(const void *m1, const void *m2) +{ + return memcmp(m1, m2, sizeof(ib_gid_t)); +} + void osm_subn_construct(IN osm_subn_t * const p_subn) { memset(p_subn, 0, sizeof(*p_subn)); @@ -407,6 +412,7 @@ void osm_subn_construct(IN osm_subn_t * const p_subn) cl_qlist_init(&p_subn->prefix_routes_list); cl_qmap_init(&p_subn->rtr_guid_tbl); cl_qmap_init(&p_subn->prtn_pkey_tbl); + cl_fmap_init(&p_subn->mgrp_mgid_tbl, compar_mgids); } /********************************************************************** @@ -462,6 +468,8 @@ void osm_subn_destroy(IN osm_subn_t * const p_subn) osm_prtn_delete(&p_prtn); } + cl_fmap_remove_all(&p_subn->mgrp_mgid_tbl); + for (i = 0; i <= p_subn->max_mcast_lid_ho - IB_LID_MCAST_START_HO; i++) { p_mgrp = p_subn->mgroups[i]; -- 1.6.3.3 From Zhen.Liang at Sun.COM Tue Jul 21 08:53:27 2009 From: Zhen.Liang at Sun.COM (Liang Zhen) Date: Tue, 21 Jul 2009 23:53:27 +0800 Subject: [ofa-general] can an ib-bonding slave work independently? In-Reply-To: <4A65AAB2.4000009@voltaire.com> References: <20090715025215.GA4322@sun.com> <4A5D897F.20708@Voltaire.com> <20090716180024.GB5930@sun.com> <4A65AAB2.4000009@voltaire.com> Message-ID: <4A65E477.30107@sun.com> Or Gerlitz wrote: > Isaac Huang wrote: >> My understanding, which I'd be happy to find false, was that RDMA >> cmid couldn't be created and bound to a bonding device. If true, > you're wrong, basically, the bonding device is being seen by the rdma > cm as a clone of its active slave, e.g see slide 8 of my talk @ > www.openfabrics.org/archives/spring2007sonoma/Tuesday%20May%201/Gerlitz%20bonding-sonoma-april-26.ppt > When I dug into code, I found when creating bonding device, alloc_netdev(...ether_setup) will always set net_device::type to ARPHRD_ETHER, that means if binding cmid on bonding device, rdma_copy_addr() will always set rdma_dev_addr::dev_type to RDMA_NODE_RNIC, seems not correct to me... am I wrong? >> Rdma_resolve_addr over a cmid bound to the slave also failed with >> RDMA_CM_EVENT_ADDR_ERROR status -ETIMEDOUT >> But tcpdump output on 'ib2' did show the ARP request and response: >> ... arp who-has 10.1.1.132 tell 10.1.13.49 hardware #32 >> ... arp reply 10.1.1.132 is-at >> 80:00:00:49:fe:80:00:00:00:00:00:10:00:03:ba:00:01:00:fb:8a The >> response seemed to have been dropped by ARP for some reason > Yes, I believe that these arp replies are being dropped, most likely > in the netif_receive_skb --> skb_bond_should_drop flow, or if you are > running with pretty old kernel as of the slave having the IFF_NOARP > bit set in its flags mask. This explains why arp resolution on the > slave fails. > > Or. We actually are using 2.6.18.128.1.6.el5, which is not old. I agree that skb_bond_should_drop is the very likely place dropping reply, customer's bonding is set to active-backup mode, so I think incoming traffic on inactive slave should be dropped, otherwise same packet could be delivered for multiple times... So I think using the inactive slave as standalone interface is not good idea. Thanks Liang > > > > > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general From hnrose at comcast.net Tue Jul 21 11:03:12 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Tue, 21 Jul 2009 14:03:12 -0400 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches Message-ID: <20090721180312.GA12491@comcast.net> Currently, MADs are pipelined to a single switch at a time which effectively serializes these requests due to processing at the SMA. This patch pipelines (stripes) them across the switches first before proceeding with successive blocks. As a result of this striping, multiple switches can process the set and respond concurrently which results in an improvement to the subnet initialization time. This patch also introduces a new config option (max_smps_per_node) which indicates how deep the per node pipeline is (current default is 4). This also has the effect of limiting the number of times that the switch list is traversed. Maybe this embellishment is unnecessary. All unicast routing protocols are updated for this with the exception of file. A similar subsequent change will do this for MFTs. Signed-off-by: Hal Rosenstock --- diff --git a/opensm/include/opensm/osm_base.h b/opensm/include/opensm/osm_base.h index 0537002..617e8a9 100644 --- a/opensm/include/opensm/osm_base.h +++ b/opensm/include/opensm/osm_base.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2006 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * @@ -449,6 +449,18 @@ BEGIN_C_DECLS */ #define OSM_DEFAULT_SMP_MAX_ON_WIRE 4 /***********/ +/****d* OpenSM: Base/OSM_DEFAULT_SMP_MAX_PER_NODE +* NAME +* OSM_DEFAULT_SMP_MAX_PER_NODE +* +* DESCRIPTION +* Specifies the default number of VL15 SMP MADs allowed +* per node for certain attributes. +* +* SYNOPSIS +*/ +#define OSM_DEFAULT_SMP_MAX_PER_NODE 4 +/***********/ /****d* OpenSM: Base/OSM_SM_DEFAULT_QP0_RCV_SIZE * NAME * OSM_SM_DEFAULT_QP0_RCV_SIZE diff --git a/opensm/include/opensm/osm_sm.h b/opensm/include/opensm/osm_sm.h index cc8321d..1776380 100644 --- a/opensm/include/opensm/osm_sm.h +++ b/opensm/include/opensm/osm_sm.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -130,6 +130,7 @@ typedef struct osm_sm { osm_sm_mad_ctrl_t mad_ctrl; osm_lid_mgr_t lid_mgr; osm_ucast_mgr_t ucast_mgr; + boolean_t lfts_updated; cl_disp_reg_handle_t sweep_fail_disp_h; cl_disp_reg_handle_t ni_disp_h; cl_disp_reg_handle_t pi_disp_h; @@ -524,6 +525,45 @@ osm_resp_send(IN osm_sm_t * sm, * *********/ +/****f* OpenSM: SM/osm_sm_set_next_lft_block +* NAME +* osm_sm_set_next_lft_block +* +* DESCRIPTION +* Set the next LFT (LinearForwardingTable) block in the indicated switch. +* +* SYNOPSIS +*/ +void +osm_sm_set_next_lft_block(IN osm_sm_t *p_sm, IN osm_switch_t *p_sw, + IN uint8_t *p_block, IN osm_dr_path_t *p_path, + IN osm_madw_context_t *p_context); +/* +* PARAMETERS +* p_sm +* [in] Pointer to an osm_sm_t object. +* +* p_switch +* [in] Pointer to the switch object. +* +* p_block +* [in] Pointer to the forwarding table block. +* +* p_path +* [in] Pointer to a directed route path object. +* +* p_context +* [in] Mad wrapper context structure to be copied into the wrapper +* context, and thus visible to the recipient of the response. +* +* RETURN VALUES +* None +* +* NOTES +* +* SEE ALSO +*********/ + /****f* OpenSM: SM/osm_sm_mcgrp_join * NAME * osm_sm_mcgrp_join diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 59a32ad..f12afae 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. * @@ -147,6 +147,7 @@ typedef struct osm_subn_opt { uint32_t sweep_interval; uint32_t max_wire_smps; uint32_t transaction_timeout; + uint32_t max_smps_per_node; uint8_t sm_priority; uint8_t lmc; boolean_t lmc_esp0; diff --git a/opensm/include/opensm/osm_switch.h b/opensm/include/opensm/osm_switch.h index 7ce28c5..e12113f 100644 --- a/opensm/include/opensm/osm_switch.h +++ b/opensm/include/opensm/osm_switch.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -102,6 +102,7 @@ typedef struct osm_switch { osm_port_profile_t *p_prof; uint8_t *lft; uint8_t *new_lft; + uint16_t lft_block_id_ho; osm_mcast_tbl_t mcast_tbl; unsigned endport_links; unsigned need_update; diff --git a/opensm/include/opensm/osm_ucast_mgr.h b/opensm/include/opensm/osm_ucast_mgr.h index a040476..fdea49a 100644 --- a/opensm/include/opensm/osm_ucast_mgr.h +++ b/opensm/include/opensm/osm_ucast_mgr.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -233,17 +233,42 @@ osm_ucast_mgr_init(IN osm_ucast_mgr_t * const p_mgr, IN struct osm_sm * sm); * osm_ucast_mgr_destroy *********/ -/****f* OpenSM: Unicast Manager/osm_ucast_mgr_set_fwd_table +/****f* OpenSM: Unicast Manager/osm_ucast_pipeline_tbl * NAME -* osm_ucast_mgr_set_fwd_table +* osm_ucast_pipeline_tbl * * DESCRIPTION -* Setup forwarding table for the switch (from prepared new_lft). +* The osm_ucast_pipeline_tbl function pipelines the LFT +* (LinearForwardingTable) sets across the switches +* (from prepared new_lft). * * SYNOPSIS */ -int osm_ucast_mgr_set_fwd_table(IN osm_ucast_mgr_t * const p_mgr, - IN osm_switch_t * const p_sw); +void osm_ucast_pipeline_tbl(IN osm_ucast_mgr_t * p_mgr); +/* +* PARAMETERS +* p_mgr +* [in] Pointer to an osm_ucast_mgr_t object. +* +* RETURN VALUES +* None. +* +* NOTES +* +* SEE ALSO +*********/ + +/****f* OpenSM: Unicast Manager/osm_ucast_mgr_set_fwd_tbl_top +* NAME +* osm_ucast_mgr_set_fwd_tbl_top +* +* DESCRIPTION +* Setup LinearFDBTop for the switch. +* +* SYNOPSIS +*/ +int osm_ucast_mgr_set_fwd_tbl_top(IN osm_ucast_mgr_t * const p_mgr, + IN osm_switch_t * const p_sw); /* * PARAMETERS * p_mgr diff --git a/opensm/opensm/osm_lin_fwd_rcv.c b/opensm/opensm/osm_lin_fwd_rcv.c index 2edb8d3..cb131b4 100644 --- a/opensm/opensm/osm_lin_fwd_rcv.c +++ b/opensm/opensm/osm_lin_fwd_rcv.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -36,7 +36,7 @@ /* * Abstract: * Implementation of osm_lft_rcv_t. - * This object represents the NodeDescription Receiver object. + * This object represents the Linear Forwarding Table Receiver object. * This object is part of the opensm family of objects. */ @@ -55,6 +55,7 @@ void osm_lft_rcv_process(IN void *context, IN void *data) { osm_sm_t *sm = context; osm_madw_t *p_madw = data; + osm_dr_path_t *p_path; ib_smp_t *p_smp; uint32_t block_num; osm_switch_t *p_sw; @@ -62,6 +63,8 @@ void osm_lft_rcv_process(IN void *context, IN void *data) uint8_t *p_block; ib_net64_t node_guid; ib_api_status_t status; + uint8_t block[IB_SMP_DATA_SIZE]; + osm_madw_context_t mad_context; CL_ASSERT(sm); @@ -94,6 +97,16 @@ void osm_lft_rcv_process(IN void *context, IN void *data) "\n\t\t\t\tSwitch 0x%" PRIx64 "\n", ib_get_err_str(status), cl_ntoh64(node_guid)); } + + p_path = osm_physp_get_dr_path_ptr(osm_node_get_physp_ptr(p_sw->p_node, 0)); + + mad_context.lft_context.node_guid = node_guid; + mad_context.lft_context.set_method = TRUE; + + osm_sm_set_next_lft_block(sm, p_sw, &block[0], p_path, + &mad_context); + + p_sw->lft_block_id_ho++; } CL_PLOCK_RELEASE(sm->p_lock); diff --git a/opensm/opensm/osm_sm.c b/opensm/opensm/osm_sm.c index daa60ff..4e0fd2a 100644 --- a/opensm/opensm/osm_sm.c +++ b/opensm/opensm/osm_sm.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. * @@ -441,6 +441,45 @@ Exit: /********************************************************************** **********************************************************************/ +void osm_sm_set_next_lft_block(IN osm_sm_t *p_sm, IN osm_switch_t *p_sw, + IN uint8_t *p_block, IN osm_dr_path_t *p_path, + IN osm_madw_context_t *context) +{ + ib_api_status_t status; + + for (; + osm_switch_get_lft_block(p_sw, p_sw->lft_block_id_ho, p_block); + p_sw->lft_block_id_ho++) { + if (!p_sw->need_update && !p_sm->p_subn->need_update && + !memcmp(p_block, + p_sw->new_lft + p_sw->lft_block_id_ho * IB_SMP_DATA_SIZE, + IB_SMP_DATA_SIZE)) + continue; + + p_sm->lfts_updated = 1; + + OSM_LOG(p_sm->p_log, OSM_LOG_DEBUG, + "Writing FT block %u to switch 0x%" PRIx64 "\n", + p_sw->lft_block_id_ho, + cl_ntoh64(context->lft_context.node_guid)); + + status = osm_req_set(p_sm, p_path, + p_sw->new_lft + + p_sw->lft_block_id_ho * IB_SMP_DATA_SIZE, + IB_SMP_DATA_SIZE, IB_MAD_ATTR_LIN_FWD_TBL, + cl_hton32(p_sw->lft_block_id_ho), + CL_DISP_MSGID_NONE, context); + + if (status != IB_SUCCESS) + OSM_LOG(p_sm->p_log, OSM_LOG_ERROR, "ERR 2E11: " + "Sending linear fwd. tbl. block failed (%s)\n", + ib_get_err_str(status)); + break; + } +} + +/********************************************************************** + **********************************************************************/ static ib_api_status_t sm_mgrp_process(IN osm_sm_t * p_sm, IN osm_mgrp_t * p_mgrp) { diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index ec15f8a..1964b7f 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. * @@ -295,6 +295,7 @@ static const opt_rec_t opt_tbl[] = { { "m_key_lease_period", OPT_OFFSET(m_key_lease_period), opts_parse_net16, NULL, 1 }, { "sweep_interval", OPT_OFFSET(sweep_interval), opts_parse_uint32, NULL, 1 }, { "max_wire_smps", OPT_OFFSET(max_wire_smps), opts_parse_uint32, NULL, 1 }, + { "max_smps_per_node", OPT_OFFSET(max_smps_per_node), opts_parse_uint32, NULL, 1 }, { "console", OPT_OFFSET(console), opts_parse_charp, NULL, 0 }, { "console_port", OPT_OFFSET(console_port), opts_parse_uint16, NULL, 0 }, { "transaction_timeout", OPT_OFFSET(transaction_timeout), opts_parse_uint32, NULL, 1 }, @@ -671,6 +672,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt) p_opt->m_key_lease_period = 0; p_opt->sweep_interval = OSM_DEFAULT_SWEEP_INTERVAL_SECS; p_opt->max_wire_smps = OSM_DEFAULT_SMP_MAX_ON_WIRE; + p_opt->max_smps_per_node = OSM_DEFAULT_SMP_MAX_PER_NODE; p_opt->console = strdup(OSM_DEFAULT_CONSOLE); p_opt->console_port = OSM_DEFAULT_CONSOLE_PORT; p_opt->transaction_timeout = OSM_DEFAULT_TRANS_TIMEOUT_MILLISEC; @@ -1461,6 +1463,10 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) "max_wire_smps %u\n\n" "# The maximum time in [msec] allowed for a transaction to complete\n" "transaction_timeout %u\n\n" + "# Maximum number of SMPs per node sent in parallel\n" + "# (0 means unlimited)\n" + "# Only applies to certain attributes\n" + "max_smps_per_node %u\n\n" "# Maximal time in [msec] a message can stay in the incoming message queue.\n" "# If there is more than one message in the queue and the last message\n" "# stayed in the queue more than this value, any SA request will be\n" @@ -1470,6 +1476,7 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) "single_thread %s\n\n", p_opts->max_wire_smps, p_opts->transaction_timeout, + p_opts->max_smps_per_node, p_opts->max_msg_fifo_timeout, p_opts->single_thread ? "TRUE" : "FALSE"); diff --git a/opensm/opensm/osm_ucast_cache.c b/opensm/opensm/osm_ucast_cache.c index 216b496..31c930b 100644 --- a/opensm/opensm/osm_ucast_cache.c +++ b/opensm/opensm/osm_ucast_cache.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2008,2009 Mellanox Technologies LTD. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -1085,9 +1085,11 @@ int osm_ucast_cache_process(osm_ucast_mgr_t * p_mgr) memset(p_sw->lft, OSM_NO_PATH, IB_LID_UCAST_END_HO + 1); } - osm_ucast_mgr_set_fwd_table(p_mgr, p_sw); + osm_ucast_mgr_set_fwd_tbl_top(p_mgr, p_sw); } + osm_ucast_pipeline_tbl(p_mgr); + return 0; } diff --git a/opensm/opensm/osm_ucast_file.c b/opensm/opensm/osm_ucast_file.c index 2505c46..099e8ba 100644 --- a/opensm/opensm/osm_ucast_file.c +++ b/opensm/opensm/osm_ucast_file.c @@ -168,8 +168,8 @@ static int do_ucast_file_load(void *context) "routing algorithm\n"); } else if (!strncmp(p, "Unicast lids", 12)) { if (p_sw) - osm_ucast_mgr_set_fwd_table(&p_osm->sm. - ucast_mgr, p_sw); + osm_ucast_mgr_set_fwd_tbl_top(&p_osm->sm. + ucast_mgr, p_sw); q = strstr(p, " guid 0x"); if (!q) { OSM_LOG(&p_osm->log, OSM_LOG_ERROR, @@ -247,7 +247,7 @@ static int do_ucast_file_load(void *context) } if (p_sw) - osm_ucast_mgr_set_fwd_table(&p_osm->sm.ucast_mgr, p_sw); + osm_ucast_mgr_set_fwd_tbl_top(&p_osm->sm.ucast_mgr, p_sw); fclose(file); return 0; diff --git a/opensm/opensm/osm_ucast_ftree.c b/opensm/opensm/osm_ucast_ftree.c index bde6dbd..d65c685 100644 --- a/opensm/opensm/osm_ucast_ftree.c +++ b/opensm/opensm/osm_ucast_ftree.c @@ -2,7 +2,7 @@ * Copyright (c) 2009 Simula Research Laboratory. All rights reserved. * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2007 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -1905,8 +1905,8 @@ static void set_sw_fwd_table(IN cl_map_item_t * const p_map_item, ftree_fabric_t *p_ftree = (ftree_fabric_t *) context; p_sw->p_osm_sw->max_lid_ho = p_ftree->lft_max_lid; - osm_ucast_mgr_set_fwd_table(&p_ftree->p_osm->sm.ucast_mgr, - p_sw->p_osm_sw); + osm_ucast_mgr_set_fwd_tbl_top(&p_ftree->p_osm->sm.ucast_mgr, + p_sw->p_osm_sw); } /*************************************************** @@ -4005,6 +4005,8 @@ static int do_routing(IN void *context) /* for each switch, set its fwd table */ cl_qmap_apply_func(&p_ftree->sw_tbl, set_sw_fwd_table, (void *)p_ftree); + osm_ucast_pipeline_tbl(&p_ftree->p_osm->sm.ucast_mgr); + /* write out hca ordering file */ fabric_dump_hca_ordering(p_ftree); diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 12b5e34..adf5f6c 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2007 Simula Research Laboratory. All rights reserved. * Copyright (c) 2007 Silicon Graphics Inc. All rights reserved. @@ -1045,8 +1045,11 @@ static void populate_fwd_tbls(lash_t * p_lash) physical_egress_port); } } /* for */ - osm_ucast_mgr_set_fwd_table(&p_osm->sm.ucast_mgr, p_sw); + osm_ucast_mgr_set_fwd_tbl_top(&p_osm->sm.ucast_mgr, p_sw); } + + osm_ucast_pipeline_tbl(&p_osm->sm.ucast_mgr); + OSM_LOG_EXIT(p_log); } diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c index 78a7031..86d1c98 100644 --- a/opensm/opensm/osm_ucast_mgr.c +++ b/opensm/opensm/osm_ucast_mgr.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -315,16 +315,14 @@ Exit: /********************************************************************** **********************************************************************/ -int osm_ucast_mgr_set_fwd_table(IN osm_ucast_mgr_t * p_mgr, - IN osm_switch_t * p_sw) +int osm_ucast_mgr_set_fwd_tbl_top(IN osm_ucast_mgr_t * p_mgr, + IN osm_switch_t * p_sw) { osm_node_t *p_node; osm_dr_path_t *p_path; osm_madw_context_t context; ib_api_status_t status; ib_switch_info_t si; - uint16_t block_id_ho = 0; - uint8_t block[IB_SMP_DATA_SIZE]; boolean_t set_swinfo_require = FALSE; uint16_t lin_top; uint8_t life_state; @@ -382,48 +380,8 @@ int osm_ucast_mgr_set_fwd_table(IN osm_ucast_mgr_t * p_mgr, ib_get_err_str(status)); } - /* - Send linear forwarding table blocks to the switch - as long as the switch indicates it has blocks needing - configuration. - */ - - context.lft_context.node_guid = osm_node_get_node_guid(p_node); - context.lft_context.set_method = TRUE; - - if (!p_sw->new_lft) { - /* any routing should provide the new_lft */ - CL_ASSERT(p_mgr->p_subn->opt.use_ucast_cache && - p_mgr->cache_valid && !p_sw->need_update); - goto Exit; - } - - for (block_id_ho = 0; - osm_switch_get_lft_block(p_sw, block_id_ho, block); - block_id_ho++) { - if (!p_sw->need_update && !p_mgr->p_subn->need_update && - !memcmp(block, - p_sw->new_lft + block_id_ho * IB_SMP_DATA_SIZE, - IB_SMP_DATA_SIZE)) - continue; - - OSM_LOG(p_mgr->p_log, OSM_LOG_DEBUG, - "Writing FT block %u\n", block_id_ho); - - status = osm_req_set(p_mgr->sm, p_path, - p_sw->new_lft + - block_id_ho * IB_SMP_DATA_SIZE, - sizeof(block), IB_MAD_ATTR_LIN_FWD_TBL, - cl_hton32(block_id_ho), CL_DISP_MSGID_NONE, - &context); + p_sw->lft_block_id_ho = 0; - if (status != IB_SUCCESS) - OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR, "ERR 3A05: " - "Sending linear fwd. tbl. block failed (%s)\n", - ib_get_err_str(status)); - } - -Exit: OSM_LOG_EXIT(p_mgr->p_log); return 0; } @@ -508,7 +466,7 @@ static void ucast_mgr_process_tbl(IN cl_map_item_t * p_map_item, } } - osm_ucast_mgr_set_fwd_table(p_mgr, p_sw); + osm_ucast_mgr_set_fwd_tbl_top(p_mgr, p_sw); if (p_mgr->p_subn->opt.lmc) free_ports_priv(p_mgr); @@ -516,6 +474,47 @@ static void ucast_mgr_process_tbl(IN cl_map_item_t * p_map_item, OSM_LOG_EXIT(p_mgr->p_log); } +static void ucast_mgr_pipeline_tbl(IN osm_switch_t *p_sw, + IN osm_ucast_mgr_t *p_mgr) +{ + osm_dr_path_t *p_path; + osm_madw_context_t mad_context; + uint8_t block[IB_SMP_DATA_SIZE]; + + OSM_LOG_ENTER(p_mgr->p_log); + + CL_ASSERT(p_sw && p_sw->p_node); + + OSM_LOG(p_mgr->p_log, OSM_LOG_DEBUG, + "Processing switch 0x%" PRIx64 "\n", + cl_ntoh64(osm_node_get_node_guid(p_sw->p_node))); + + /* + Send linear forwarding table blocks to the switch + as long as the switch indicates it has blocks needing + configuration. + */ + if (!p_sw->new_lft) { + /* any routing should provide the new_lft */ + CL_ASSERT(p_mgr->p_subn->opt.use_ucast_cache && + p_mgr->cache_valid && !p_sw->need_update); + goto Exit; + } + + p_path = osm_physp_get_dr_path_ptr(osm_node_get_physp_ptr(p_sw->p_node, 0)); + + mad_context.lft_context.node_guid = osm_node_get_node_guid(p_sw->p_node); + mad_context.lft_context.set_method = TRUE; + + osm_sm_set_next_lft_block(p_mgr->sm, p_sw, &block[0], p_path, + &mad_context); + + p_sw->lft_block_id_ho++; + +Exit: + OSM_LOG_EXIT(p_mgr->p_log); +} + /********************************************************************** **********************************************************************/ static void ucast_mgr_process_neighbors(IN cl_map_item_t * p_map_item, @@ -870,6 +869,28 @@ static void sort_ports_by_switch_load(osm_ucast_mgr_t * m) add_sw_endports_to_order_list(s[i], m); } +void osm_ucast_pipeline_tbl(osm_ucast_mgr_t * p_mgr) +{ + cl_qmap_t *p_sw_tbl; + osm_switch_t *p_sw; + int i; + + for (i = 0; + !p_mgr->p_subn->opt.max_smps_per_node || + i < p_mgr->p_subn->opt.max_smps_per_node; + i++) { + p_mgr->sm->lfts_updated = 0; + p_sw_tbl = &p_mgr->p_subn->sw_guid_tbl; + p_sw = (osm_switch_t *) cl_qmap_head(p_sw_tbl); + while (p_sw != (osm_switch_t *) cl_qmap_end(p_sw_tbl)) { + ucast_mgr_pipeline_tbl(p_sw, p_mgr); + p_sw = (osm_switch_t *) cl_qmap_next(&p_sw->map_item); + } + if (!p_mgr->sm->lfts_updated) + break; + } +} + static int ucast_mgr_build_lfts(osm_ucast_mgr_t * p_mgr) { cl_qlist_init(&p_mgr->port_order_list); @@ -904,6 +925,8 @@ static int ucast_mgr_build_lfts(osm_ucast_mgr_t * p_mgr) cl_qmap_apply_func(&p_mgr->p_subn->sw_guid_tbl, ucast_mgr_process_tbl, p_mgr); + osm_ucast_pipeline_tbl(p_mgr); + cl_qlist_remove_all(&p_mgr->port_order_list); return 0; From jgunthorpe at obsidianresearch.com Tue Jul 21 11:44:46 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Tue, 21 Jul 2009 12:44:46 -0600 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <20090721180312.GA12491@comcast.net> References: <20090721180312.GA12491@comcast.net> Message-ID: <20090721184446.GB12693@obsidianresearch.com> On Tue, Jul 21, 2009 at 02:03:12PM -0400, Hal Rosenstock wrote: > Currently, MADs are pipelined to a single switch at a time which > effectively serializes these requests due to processing at the SMA. > This patch pipelines (stripes) them across the switches first before > proceeding with successive blocks. As a result of this striping, > multiple switches can process the set and respond concurrently > which results in an improvement to the subnet initialization time. Doing this without also using LID routing to the target switch is just going to overload the SMAs in the intermediate switches with too many DR SMPs. The most efficient approach is to program LFTs using a breadth first search of the connectivity graph starting from the SM end port: 1) Within the BFS consider things as layers (number of hops from the SM). All switches in a layer can proceed largely in parallel, within the capability of the SM to capture the MAD replies at least. Advancing to the next layer must wait until the prior layer is fully programmed. 2) At each switch send at most two partial DR MADs - where the MAD is LID routed up to the parent switch in the BFS, and then direct routed one step to the target switch. The two MADs would program the LFT block for the SM LID and for the switch LID. 3) Sent LID routed mads directly to the target switch to fill in the rest of the LFT entries. Step 2 needs to respect the concurrency limit for the parent switch, and #3 needs to respect the concurrency limit for the target switch. As you go out the BFS there are more parent swtiches and target switches available to run in parallel. Eliminating DR hops will significantly improve MAD round trip time and give you more possible parallelism before the SMAs in intermediate switches become overloaded. Overall, optimizing things so that as few DR MADs are used as possible is desirable. The NodeInfo write to set the switch LID can also be put into the above process and then all other switch programming MADs can simply happen via LID routed packets. Jason From hnrose at comcast.net Tue Jul 21 13:51:04 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Tue, 21 Jul 2009 16:51:04 -0400 Subject: [ofa-general] [PATCHv4] opensm/lash: Set minimum VL for LASH to use Message-ID: <20090721205104.GA1525@comcast.net> rather than starting from VL 0 Signed-off-by: Robert Pearson Signed-off-by: Hal Rosenstock --- Changes since v3: Reduce vl_min by start_vl and preserve most code flows Only need to update routing_table.lane Simplification pointed out by Sasha Changes since v2: Restored malloc use (rather than calloc) as pointed out by Sasha Changes since v1: Fixed comparisons with maximum VL Better lash_start_vl option handling Both as pointed out by Sasha diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 59a32ad..da8cc5e 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -221,6 +222,7 @@ typedef struct osm_subn_opt { char *prefix_routes_file; boolean_t consolidate_ipv6_snm_req; struct osm_subn_opt *file_opts; /* used for update */ + uint8_t lash_start_vl; /* starting vl to use in lash */ } osm_subn_opt_t; /* * FIELDS diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in index dbf4b73..c71a79d 100644 --- a/opensm/man/opensm.8.in +++ b/opensm/man/opensm.8.in @@ -1,4 +1,4 @@ -.TH OPENSM 8 "April 22, 2009" "OpenIB" "OpenIB Management" +.TH OPENSM 8 "May 28, 2009" "OpenIB" "OpenIB Management" .SH NAME opensm \- InfiniBand subnet manager and administration (SM/SA) @@ -15,6 +15,7 @@ opensm \- InfiniBand subnet manager and administration (SM/SA) [\-r(eassign_lids)] [\-R | \-\-routing_engine ] [\-\-do_mesh_analysis] +[\-\-lash_start_vl ] [\-A | \-\-ucast_cache] [\-z | \-\-connect_roots] [\-M | \-\-lid_matrix_file ] @@ -147,6 +148,10 @@ This option enables additional analysis for the lash routing engine to precondition switch port assignments in regular cartesian meshes which may reduce the number of SLs required to give a deadlock free routing. .TP +\fB\-\-lash_start_vl\fR +This option sets the starting VL to use for the lash routing algorithm. +Defaults to 0. +.TP \fB\-A\fR, \fB\-\-ucast_cache\fR This option enables unicast routing cache and prevents routing recalculation (which is a heavy task in a large cluster) when diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c index 296d5d5..d682ff5 100644 --- a/opensm/opensm/main.c +++ b/opensm/opensm/main.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009 HNR Consulting. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -181,6 +182,9 @@ static void show_usage(void) " routing engine to precondition switch port assignments\n" " in regular cartesian meshes which may reduce the number\n" " of SLs required to give a deadlock free routing\n\n"); + printf("--lash_start_vl \n" + " Sets the starting VL to use for the lash routing algorithm.\n" + " Defaults to 0.\n"); printf("--connect_roots, -z\n" " This option enforces a routing engine (currently\n" " up/down only) to make connectivity between root switches\n" @@ -601,6 +605,7 @@ int main(int argc, char *argv[]) {"prefix_routes_file", 1, NULL, 3}, {"consolidate_ipv6_snm_req", 0, NULL, 4}, {"do_mesh_analysis", 0, NULL, 5}, + {"lash_start_vl", 1, NULL, 6}, {NULL, 0, NULL, 0} /* Required at the end of the array */ }; @@ -951,6 +956,15 @@ int main(int argc, char *argv[]) case 5: opt.do_mesh_analysis = TRUE; break; + case 6: + temp = strtol(optarg, NULL, 0); + if (temp < 0 || temp >= IB_MAX_NUM_VLS) { + fprintf(stderr, + "ERROR: starting lash vl must be between 0 and 15\n"); + return (-1); + } + opt.lash_start_vl = (uint8_t) temp; + break; case 'h': case '?': case ':': diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index ec15f8a..fda2eb0 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -389,6 +390,7 @@ static const opt_rec_t opt_tbl[] = { { "no_clients_rereg", OPT_OFFSET(no_clients_rereg), opts_parse_boolean, NULL, 1 }, { "prefix_routes_file", OPT_OFFSET(prefix_routes_file), opts_parse_charp, NULL, 0 }, { "consolidate_ipv6_snm_req", OPT_OFFSET(consolidate_ipv6_snm_req), opts_parse_boolean, NULL, 1 }, + { "lash_start_vl", OPT_OFFSET(lash_start_vl), opts_parse_uint8, NULL, 1 }, {0} }; @@ -749,6 +751,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt) p_opt->no_clients_rereg = FALSE; p_opt->prefix_routes_file = strdup(OSM_DEFAULT_PREFIX_ROUTES_FILE); p_opt->consolidate_ipv6_snm_req = FALSE; + p_opt->lash_start_vl = 0; subn_init_qos_options(&p_opt->qos_options, NULL); subn_init_qos_options(&p_opt->qos_ca_options, NULL); subn_init_qos_options(&p_opt->qos_sw0_options, NULL); @@ -1432,6 +1435,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) p_opts->do_mesh_analysis ? "TRUE" : "FALSE"); fprintf(out, + "# Starting VL for LASH algorithm\n" + "lash_start_vl %d\n\n", + p_opts->lash_start_vl); + + fprintf(out, "# SA database file name\nsa_db_file %s\n\n", p_opts->sa_db_file ? p_opts->sa_db_file : null_str); diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 12b5e34..b785551 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -486,6 +486,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) int next_switch2, output_link2; int stop = 0, cycle_found; int cycle_found2; + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; max_filled_lane = 0; min_filled_lane = lanes_needed - 1; @@ -572,8 +573,8 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) virtual_location[dest][src][max_filled_lane] = 0; virtual_location[src][dest][min_filled_lane] = 1; virtual_location[dest][src][min_filled_lane] = 1; - p_lash->switches[src]->routing_table[dest].lane = min_filled_lane; - p_lash->switches[dest]->routing_table[src].lane = min_filled_lane; + p_lash->switches[src]->routing_table[dest].lane = min_filled_lane + start_vl; + p_lash->switches[dest]->routing_table[src].lane = min_filled_lane + start_vl; } if (trials == 0) @@ -804,6 +805,7 @@ static int lash_core(lash_t * p_lash) int cycle_found2 = 0; int status = 0; int *switch_bitmap = NULL; /* Bitmap to check if we have processed this pair */ + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; OSM_LOG_ENTER(p_log); @@ -838,7 +840,9 @@ static int lash_core(lash_t * p_lash) } for (i = 0; i < num_switches; i++) { - for (dest_switch = 0; dest_switch < num_switches; dest_switch++) + for (dest_switch = 0; dest_switch < num_switches; dest_switch++) { + if (lanes_needed > p_lash->vl_min) + goto Error_Not_Enough_Lanes; if (dest_switch != i && switch_bitmap[i * num_switches + dest_switch] == 0) { v_lane = 0; stop = 0; @@ -902,8 +906,8 @@ static int lash_core(lash_t * p_lash) } } - switches[i]->routing_table[dest_switch].lane = v_lane; - switches[dest_switch]->routing_table[i].lane = v_lane; + switches[i]->routing_table[dest_switch].lane = v_lane + start_vl; + switches[dest_switch]->routing_table[i].lane = v_lane + start_vl; if (cycle_found == 1 || cycle_found2 == 1) { if (++lanes_needed > p_lash->vl_min) @@ -926,16 +930,20 @@ static int lash_core(lash_t * p_lash) switch_bitmap[i * num_switches + dest_switch] = 1; switch_bitmap[dest_switch * num_switches + i] = 1; } + } } - OSM_LOG(p_log, OSM_LOG_INFO, - "Lanes needed: %d, Balancing\n", lanes_needed); + if (lanes_needed > p_lash->vl_min) + goto Error_Not_Enough_Lanes; for (i = 0; i < lanes_needed; i++) { OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", i, p_lash->num_mst_in_lane[i]); } + OSM_LOG(p_log, OSM_LOG_INFO, + "Lanes needed: %d, Balancing\n", lanes_needed); + balance_virtual_lanes(p_lash, lanes_needed); for (i = 0; i < lanes_needed; i++) { @@ -948,8 +956,9 @@ static int lash_core(lash_t * p_lash) Error_Not_Enough_Lanes: status = -1; OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4D02: " - "Lane requirements (%d) exceed available lanes (%d)\n", - lanes_needed, p_lash->vl_min); + "Lane requirements (%d) exceed available lanes (%d)" + " with starting lane (%d)\n", + lanes_needed, p_lash->vl_min, start_vl); Exit: if (switch_bitmap) free(switch_bitmap); @@ -1177,10 +1186,11 @@ static int discover_network_properties(lash_t * p_lash) if (vl_min > 15) vl_min = 15; - p_lash->vl_min = vl_min; + p_lash->vl_min = vl_min - p_lash->p_osm->subn.opt.lash_start_vl; OSM_LOG(p_log, OSM_LOG_INFO, - "min operational vl(%d) max_switches(%d)\n", p_lash->vl_min, + "min operational vl(%d) start vl(%d) max_switches(%d)\n", + p_lash->vl_min, p_lash->p_osm->subn.opt.lash_start_vl, p_lash->num_switches); return 0; } @@ -1286,7 +1296,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port, src_id = get_lash_id(p_sw); if (src_id == dst_id) - return OSM_DEFAULT_SL; + return p_osm->subn.opt.lash_start_vl; return (uint8_t) ((switch_t *) p_sw->priv)->routing_table[dst_id].lane; } From liranl at mellanox.co.il Tue Jul 21 13:55:58 2009 From: liranl at mellanox.co.il (Liran Liss) Date: Tue, 21 Jul 2009 23:55:58 +0300 Subject: [ofa-general] RDMAoE path resolution and multicast handling In-Reply-To: <4A5ED08A.2010907@voltaire.com> References: <20090713181310.GA31865@mtls03> <4A5C5332.7020403@voltaire.com> <5D49E7A8952DC44FB38C38FA0D758EAD035ADF95@mtlexch01.mtl.com> <4A5D9357.5020808@voltaire.com> <5D49E7A8952DC44FB38C38FA0D758EAD035AE999@mtlexch01.mtl.com> <4A5ED08A.2010907@voltaire.com> Message-ID: <2ED289D4E09FBD4D92D911E869B97FDD2C4E@mtlexch01.mtl.com> Hi, We accept the recommendations for integrating rdmaoe path resolution and multicast handling within the rdmacm code. The main reasons are: - It is the natural place to abstract transport differences. - rdmaoe uses IP-based addressing, which is what rdmacm is all about. - The sa interface is not widely used as an external interface, so not supporting it won't be a big loss. - Less disruptive to native IB code without code duplication. Thanks for Sean, Robert, Or, and Yossi for their excellent feedback. We will provide a new patch set that reflects this change in a few days. --Liran From weiny2 at llnl.gov Tue Jul 21 13:56:03 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Tue, 21 Jul 2009 13:56:03 -0700 Subject: [ofa-general] [PATCH] Add ib_resolve_gid_via and make portid an out only parameter of ib_resolve_portid_str_via Message-ID: <20090721135603.681fc823.weiny2@llnl.gov> From: Ira Weiny Date: Wed, 22 Apr 2009 17:12:58 -0700 Subject: [PATCH] Add ib_resolve_gid_via and make portid an out only parameter of ib_resolve_portid_str_via Signed-off-by: Ira Weiny --- libibmad/include/infiniband/mad.h | 4 ++++ libibmad/src/libibmad.map | 1 + libibmad/src/resolve.c | 27 +++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 0 deletions(-) diff --git a/libibmad/include/infiniband/mad.h b/libibmad/include/infiniband/mad.h index bdf5158..ee004a9 100644 --- a/libibmad/include/infiniband/mad.h +++ b/libibmad/include/infiniband/mad.h @@ -724,6 +724,7 @@ enum MAD_DEST { IB_DEST_DRPATH, IB_DEST_GUID, IB_DEST_DRSLID, + IB_DEST_GID }; enum MAD_NODE_TYPE { @@ -912,6 +913,9 @@ MAD_EXPORT int ib_resolve_smlid_via(ib_portid_t * sm_id, int timeout, MAD_EXPORT int ib_resolve_guid_via(ib_portid_t * portid, uint64_t * guid, ib_portid_t * sm_id, int timeout, const struct ibmad_port *srcport); +MAD_EXPORT int ib_resolve_gid_via(ib_portid_t * portid, ibmad_gid_t gid, + ib_portid_t * sm_id, int timeout, + const struct ibmad_port *srcport); MAD_EXPORT int ib_resolve_portid_str_via(ib_portid_t * portid, char *addr_str, enum MAD_DEST dest, ib_portid_t * sm_id, const struct ibmad_port *srcport); diff --git a/libibmad/src/libibmad.map b/libibmad/src/libibmad.map index 7b49a4d..1462064 100644 --- a/libibmad/src/libibmad.map +++ b/libibmad/src/libibmad.map @@ -99,6 +99,7 @@ IBMAD_1.3 { ib_path_query_via; ib_resolve_smlid_via; ib_resolve_guid_via; + ib_resolve_gid_via; ib_resolve_portid_str_via; ib_resolve_self_via; mad_field_name; diff --git a/libibmad/src/resolve.c b/libibmad/src/resolve.c index 643c187..05b5602 100644 --- a/libibmad/src/resolve.c +++ b/libibmad/src/resolve.c @@ -70,6 +70,26 @@ int ib_resolve_smlid(ib_portid_t * sm_id, int timeout) return ib_resolve_smlid_via(sm_id, timeout, ibmp); } +int ib_resolve_gid_via(ib_portid_t * portid, ibmad_gid_t gid, + ib_portid_t * sm_id, int timeout, + const struct ibmad_port *srcport) +{ + ib_portid_t sm_portid; + char buf[IB_SA_DATA_SIZE] = { 0 }; + + if (!sm_id) { + sm_id = &sm_portid; + if (ib_resolve_smlid_via(sm_id, timeout, srcport) < 0) + return -1; + } + + if ((portid->lid = + ib_path_query_via(srcport, gid, gid, sm_id, buf)) < 0) + return -1; + + return 0; +} + int ib_resolve_guid_via(ib_portid_t * portid, uint64_t * guid, ib_portid_t * sm_id, int timeout, const struct ibmad_port *srcport) @@ -112,12 +132,15 @@ int ib_resolve_portid_str_via(ib_portid_t * portid, char *addr_str, enum MAD_DEST dest_type, ib_portid_t * sm_id, const struct ibmad_port *srcport) { + ibmad_gid_t gid; uint64_t guid; int lid; char *routepath; ib_portid_t selfportid = { 0 }; int selfport = 0; + memset(portid, 0, sizeof *portid); + switch (dest_type) { case IB_DEST_LID: lid = strtol(addr_str, 0, 0); @@ -152,6 +175,10 @@ int ib_resolve_portid_str_via(ib_portid_t * portid, char *addr_str, return -1; return 0; + case IB_DEST_GID: + if (inet_pton(AF_INET6, addr_str, &gid) <= 0) + return -1; + return ib_resolve_gid_via(portid, gid, sm_id, 0, srcport); default: IBWARN("bad dest_type %d", dest_type); } -- 1.5.4.5 From weiny2 at llnl.gov Tue Jul 21 15:50:00 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Tue, 21 Jul 2009 15:50:00 -0700 Subject: [ofa-general] [PATCH] libibmad: Ensure the proper length is passed into umad_recv on each call Message-ID: <20090721155000.d17df1de.weiny2@llnl.gov> Sasha, I found that once in a while umad_recv returns a trid from a previously timedout request and causes _do_madrpc to loop around. If this occurs the length value has changed from the previous call and results -EINVAL from the kernel. This fixes the problem. Ira From: Ira Weiny Date: Tue, 21 Jul 2009 15:24:49 -0700 Subject: [PATCH] libibmad: Ensure the proper length is passed into umad_recv on each call Signed-off-by: Ira Weiny --- libibmad/src/rpc.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/libibmad/src/rpc.c b/libibmad/src/rpc.c index efea1d3..0b989da 100644 --- a/libibmad/src/rpc.c +++ b/libibmad/src/rpc.c @@ -158,6 +158,7 @@ _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, /* Use same timeout on receive side just in case */ /* send packet is lost somewhere. */ do { + length = len; if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) { IBWARN("recv failed: %m"); return -1; -- 1.5.4.5 From xiaoshuangx at gmail.com Tue Jul 21 19:23:01 2009 From: xiaoshuangx at gmail.com (xiaoshuang xia) Date: Wed, 22 Jul 2009 10:23:01 +0800 Subject: [ofa-general] multiple path in opensm??? Message-ID: Hi, I recently want to implement a routing scheme using opensm, but I face some problem. Help me pls!! THX!! The problem is 1. In the opensm, if the multi-path between src and dst is denoted by the multiple dst-lid. 2. If there are multi-path between the src and dst , what mechanism does client choose the one of them. If this mechanism is implemented by opensm, which file do this job? If choose one prefered path doesn 't implement by opensm, what unit(call it "the choosing unit") finish this job, what data structure does opensm hold to exchange the multi-path information to "the choosing unit" and what file does this exchanging job in opensm. Thank you again!!! Your helps are very important to me!!! -------------- next part -------------- An HTML attachment was scrubbed... URL: From ogerlitz at Voltaire.com Tue Jul 21 22:24:47 2009 From: ogerlitz at Voltaire.com (Or Gerlitz) Date: Wed, 22 Jul 2009 08:24:47 +0300 Subject: [ofa-general] can an ib-bonding slave work independently? In-Reply-To: <4A65E477.30107@sun.com> References: <20090715025215.GA4322@sun.com> <4A5D897F.20708@Voltaire.com> <20090716180024.GB5930@sun.com> <4A65AAB2.4000009@voltaire.com> <4A65E477.30107@sun.com> Message-ID: <4A66A29F.40509@Voltaire.com> Liang Zhen wrote: > When I dug into code, I found when creating bonding device, > alloc_netdev(...ether_setup) will always set net_device::type to > ARPHRD_ETHER, that means if binding cmid on bonding device, > rdma_copy_addr() will always set rdma_dev_addr::dev_type to > RDMA_NODE_RNIC, seems not correct to me... am I wrong? Yes wrong, assuming you have access to Linux box with IB stack up and running you could trivially see this if setting ipoib/bonding and then running one of the librdmacm utils while instructing it to bind to a bond (I recommend you go and do that). Now how it works? when the first ipoib device is added to the bond, bonding senses a mismatch between its ethertype to the bond one and calls bond_setup_by_slave(), this along with bonding automatically setting fail_over_mac to be BOND_FOM_ACTIVE for ipoib device, does the job. Or. From Zhen.Liang at Sun.COM Wed Jul 22 00:33:58 2009 From: Zhen.Liang at Sun.COM (Liang Zhen) Date: Wed, 22 Jul 2009 15:33:58 +0800 Subject: [ofa-general] can an ib-bonding slave work independently? In-Reply-To: <4A66A29F.40509@Voltaire.com> References: <20090715025215.GA4322@sun.com> <4A5D897F.20708@Voltaire.com> <20090716180024.GB5930@sun.com> <4A65AAB2.4000009@voltaire.com> <4A65E477.30107@sun.com> <4A66A29F.40509@Voltaire.com> Message-ID: <4A66C0E6.4090500@sun.com> Or Gerlitz wrote: > Yes wrong, assuming you have access to Linux box with IB stack up and running you > could trivially see this if setting ipoib/bonding and then running one of the > librdmacm utils while instructing it to bind to a bond (I recommend you go and do that). > Now how it works? when the first ipoib device is added to the bond, bonding senses a mismatch between its ethertype to the bond one and calls bond_setup_by_slave(), this along with bonding automatically setting fail_over_mac to be BOND_FOM_ACTIVE for ipoib device, does the job. > Yes, I just noticed that Lustre is in list of supported ULPs of ib-bonding (in your ppt), so I think we can just load Lustre over bonding device insteading of it's slaves, which will definitely simplify our work. Thanks Liang > Or. > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From mashirle at us.ibm.com Wed Jul 22 01:24:14 2009 From: mashirle at us.ibm.com (Shirley Ma) Date: Wed, 22 Jul 2009 08:24:14 -0000 Subject: [ofa-general] Re: Write combining on PPC64 In-Reply-To: <200906281221.52028.jackm@dev.mellanox.co.il> References: <200906281221.52028.jackm@dev.mellanox.co.il> Message-ID: <1246927950.7310.3.camel@localhost.localdomain> Sorry Jack, I was on vacation. You already got the answer, I think. Yes, you can use pgprot_noncached_wc(). BTW, can you please modify the connectX set_4K_mtu parameter to 644 instead of 444 in OFED-1.5? I found for the large message size, if we don't set up 4K MTU size (SM and switch should support 4K MTU too), the performance could drop 5%. Thanks Shirley On Sun, 2009-06-28 at 12:21 +0300, Jack Morgenstein wrote: > Hi Shirley, > > I was reviewing write-combining for the PPC on kernel 2.6.30, and noticed the following > in file arch/powerpc/include/asm/pgtable.h: > > #define _PAGE_CACHE_CTL (_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \ > _PAGE_WRITETHRU) > ... > #define pgprot_noncached_wc(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ > _PAGE_NO_CACHE)) > > It seems to me that this gives the same result as the fix I put in OFED 1.5 for the PPC: > (kernel_patches/fixes/mlx4_0010_add_wc.patch): > > +pgprot_t pgprot_wc(pgprot_t _prot) > +{ > + return __pgprot((pgprot_val(_prot) | _PAGE_NO_CACHE) & > + ~(pgprot_t)_PAGE_GUARDED); > +} > > ( That is, PAGE_NO_CACHE set, and _PAGE_GUARDED cleared). > > Differences are that _PAGE_COHERENT and _PAGE_WRITETHRU are also cleared in pgprot_noncached_wc() > and the patch I put in is only for PPC64, not all PPCs. > > Questions: > 1. Can I use the pgprot_noncached_wc() macro to implement write-combining on all PPCs? (it is defined > thus in arch/powerpc/include/asm/pgtable.h for all PPC platforms) > > 2. Is there any reason that there is no define: > > #define pgprot_writecombine pgprot_noncached_wc > > in file arch/powerpc/include/asm/pgtable.h (so that the general > pgprot_writecombine() call can be used for PPCs as well). > > -Jack From jackm at dev.mellanox.co.il Wed Jul 22 02:02:18 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Wed, 22 Jul 2009 12:02:18 +0300 Subject: [ofa-general] Re: [PATCH ibverbs] Make the gid argument to ibv_attach_mcast and ibv_detach_mcast const In-Reply-To: <20090720185333.GU12693@obsidianresearch.com> References: <20090719052634.GA1149@obsidianresearch.com> <20090720185333.GU12693@obsidianresearch.com> Message-ID: <200907221202.18326.jackm@dev.mellanox.co.il> On Monday 20 July 2009 21:53, Jason Gunthorpe wrote: > I have also patches for mlx4 and mthca to suppress the compiler > warning that results from this patch. ipath is OK as is, and I'm not > sure where the iwarp stuff lives.. > Is this change really necessary? Seems to me that you are creating cross-library dependencies (at least with regard to compiler warnings) -- the user will now be required to change the device libraries if libibverbs is upgraded. Also, if the device libraries get changed, but libibverbs stays the same, I think you will also get warnings if anyone tries to compile user-level apps. Roland? -Jack From sashak at voltaire.com Wed Jul 22 02:22:06 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 22 Jul 2009 12:22:06 +0300 Subject: [ofa-general] Re: [PATCHv4] opensm/lash: Set minimum VL for LASH to use In-Reply-To: <20090721205104.GA1525@comcast.net> References: <20090721205104.GA1525@comcast.net> Message-ID: <20090722092206.GK31735@me> Hi Hal, On 16:51 Tue 21 Jul , Hal Rosenstock wrote: > diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c > index 12b5e34..b785551 100644 > --- a/opensm/opensm/osm_ucast_lash.c > +++ b/opensm/opensm/osm_ucast_lash.c > @@ -486,6 +486,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) > int next_switch2, output_link2; > int stop = 0, cycle_found; > int cycle_found2; > + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; > > max_filled_lane = 0; > min_filled_lane = lanes_needed - 1; > @@ -572,8 +573,8 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) > virtual_location[dest][src][max_filled_lane] = 0; > virtual_location[src][dest][min_filled_lane] = 1; > virtual_location[dest][src][min_filled_lane] = 1; > - p_lash->switches[src]->routing_table[dest].lane = min_filled_lane; > - p_lash->switches[dest]->routing_table[src].lane = min_filled_lane; > + p_lash->switches[src]->routing_table[dest].lane = min_filled_lane + start_vl; > + p_lash->switches[dest]->routing_table[src].lane = min_filled_lane + start_vl; > } > > if (trials == 0) > @@ -804,6 +805,7 @@ static int lash_core(lash_t * p_lash) > int cycle_found2 = 0; > int status = 0; > int *switch_bitmap = NULL; /* Bitmap to check if we have processed this pair */ > + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; > > OSM_LOG_ENTER(p_log); > > @@ -838,7 +840,9 @@ static int lash_core(lash_t * p_lash) > } > > for (i = 0; i < num_switches; i++) { > - for (dest_switch = 0; dest_switch < num_switches; dest_switch++) > + for (dest_switch = 0; dest_switch < num_switches; dest_switch++) { > + if (lanes_needed > p_lash->vl_min) > + goto Error_Not_Enough_Lanes; Hmm, such check is already performed in place where lanes_needed is increased (below). Why do we need another one? > if (dest_switch != i && switch_bitmap[i * num_switches + dest_switch] == 0) { > v_lane = 0; > stop = 0; > @@ -902,8 +906,8 @@ static int lash_core(lash_t * p_lash) > } > } > > - switches[i]->routing_table[dest_switch].lane = v_lane; > - switches[dest_switch]->routing_table[i].lane = v_lane; > + switches[i]->routing_table[dest_switch].lane = v_lane + start_vl; > + switches[dest_switch]->routing_table[i].lane = v_lane + start_vl; > > if (cycle_found == 1 || cycle_found2 == 1) { > if (++lanes_needed > p_lash->vl_min) This one. > @@ -926,16 +930,20 @@ static int lash_core(lash_t * p_lash) > switch_bitmap[i * num_switches + dest_switch] = 1; > switch_bitmap[dest_switch * num_switches + i] = 1; > } > + } > } > > - OSM_LOG(p_log, OSM_LOG_INFO, > - "Lanes needed: %d, Balancing\n", lanes_needed); > + if (lanes_needed > p_lash->vl_min) > + goto Error_Not_Enough_Lanes; The same question, why do we need yet another lanes_needed > vl_min check? > > for (i = 0; i < lanes_needed; i++) { > OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", > i, p_lash->num_mst_in_lane[i]); > } > > + OSM_LOG(p_log, OSM_LOG_INFO, > + "Lanes needed: %d, Balancing\n", lanes_needed); > + > balance_virtual_lanes(p_lash, lanes_needed); > > for (i = 0; i < lanes_needed; i++) { > @@ -948,8 +956,9 @@ static int lash_core(lash_t * p_lash) > Error_Not_Enough_Lanes: > status = -1; > OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4D02: " > - "Lane requirements (%d) exceed available lanes (%d)\n", > - lanes_needed, p_lash->vl_min); > + "Lane requirements (%d) exceed available lanes (%d)" > + " with starting lane (%d)\n", > + lanes_needed, p_lash->vl_min, start_vl); > Exit: > if (switch_bitmap) > free(switch_bitmap); > @@ -1177,10 +1186,11 @@ static int discover_network_properties(lash_t * p_lash) > if (vl_min > 15) > vl_min = 15; > > - p_lash->vl_min = vl_min; > + p_lash->vl_min = vl_min - p_lash->p_osm->subn.opt.lash_start_vl; vl_min =< lash_start_vl could be a legal case and lash->vl_min has unsigned type. You will get an invalid value in this case. I suppose that check should be here with LASH termination if no VLs are available. Sasha From sashak at voltaire.com Wed Jul 22 02:26:47 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 22 Jul 2009 12:26:47 +0300 Subject: [ofa-general] Re: [PATCH] Add ib_resolve_gid_via and make portid an out only parameter of ib_resolve_portid_str_via In-Reply-To: <20090721135603.681fc823.weiny2@llnl.gov> References: <20090721135603.681fc823.weiny2@llnl.gov> Message-ID: <20090722092647.GL31735@me> On 13:56 Tue 21 Jul , Ira Weiny wrote: > > From: Ira Weiny > Date: Wed, 22 Apr 2009 17:12:58 -0700 > Subject: [PATCH] Add ib_resolve_gid_via and make portid an out only parameter of ib_resolve_portid_str_via > > > Signed-off-by: Ira Weiny Applied. Thanks. Sasha From sashak at voltaire.com Wed Jul 22 02:29:20 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 22 Jul 2009 12:29:20 +0300 Subject: [ofa-general] Re: [PATCH] libibmad: Ensure the proper length is passed into umad_recv on each call In-Reply-To: <20090721155000.d17df1de.weiny2@llnl.gov> References: <20090721155000.d17df1de.weiny2@llnl.gov> Message-ID: <20090722092920.GM31735@me> On 15:50 Tue 21 Jul , Ira Weiny wrote: > Sasha, > > I found that once in a while umad_recv returns a trid from a previously > timedout request and causes _do_madrpc to loop around. If this occurs the > length value has changed from the previous call and results -EINVAL from the > kernel. This fixes the problem. > > Ira > > > > From: Ira Weiny > Date: Tue, 21 Jul 2009 15:24:49 -0700 > Subject: [PATCH] libibmad: Ensure the proper length is passed into umad_recv on each call > > > Signed-off-by: Ira Weiny Applied. Thanks. Sasha From vlad at lists.openfabrics.org Wed Jul 22 02:39:30 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Wed, 22 Jul 2009 02:39:30 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090722-0200 daily build status Message-ID: <20090722093930.E2FF310204E1@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Failed: Build failed on i686 with linux-2.6.18 Build failed on i686 with linux-2.6.19 Build failed on i686 with linux-2.6.21.1 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.22 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.20 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.20_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.20_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.20_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.20' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -m64 -mno-red-zone -mcmodel=kernel -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -funit-at-a-time -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -m64 -mno-red-zone -mcmodel=kernel -funit-at-a-time -maccumulate-outgoing-args -DCONFIG_AS_CFI=1 -DCONFIG_AS_CFI_SIGNAL_FRAME=1 -pipe -Wno-sign-compare -fno-asynchronous-unwind-tables -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_x86_64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2092: error: 'class_device_attr_ibdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2094: error: 'class_device_attr_port' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2100: error: implicit declaration of function 'class_device_unregister' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2092: error: 'class_device_attr_ibdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2094: error: 'class_device_attr_port' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.c:2100: error: implicit declaration of function 'class_device_unregister' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.27_x86_64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.27_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ia64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ia64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.23_ia64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.23_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.21.1 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_ia64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.21.1_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.21.1' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.22 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_ia64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.22_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.22' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2092: error: 'class_device_attr_ibdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2094: error: 'class_device_attr_port' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/srp/ib_srp.c:2100: error: implicit declaration of function 'class_device_unregister' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_ia64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -fno-stack-protector -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/srpt/.tmp_ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_ia64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: -DHAVE_WORKING_TEXT_ALIGN -DHAVE_MODEL_SMALL_ATTRIBUTE -DHAVE_SERIALIZE_DIRECTIVE -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -Werror-implicit-function-declaration -Os -pipe -ffixed-r13 -mfixed-range=f12-f15,f32-f127 -falign-functions=32 -frename-registers -fno-optimize-sibling-calls -fomit-frame-pointer -g -fno-stack-protector -Wdeclaration-after-statement -Wno-pointer-sign -Idrivers/infiniband/ulp/srpt -DMODULE -D"KBUILD_STR(s)=#s" -D"KBUILD_BASENAME=KBUILD_STR(ib_srpt)" -D"KBUILD_MODNAME=KBUILD_STR(ib_srpt)" -c -o /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/srpt/.tmp_ib_srpt.o /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/srpt/ib_srpt.c /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/srpt/ib_srpt.c: In function 'srpt_add_one': /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/srpt/ib_srpt.c:2367: error: implicit declaration of function 'dev_set_name' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/srpt/ib_srpt.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_ia64_check/drivers/infiniband/ulp/srpt] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_ia64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.18 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ppc64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.18_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.18' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ppc64 with linux-2.6.19 Log: /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: error: implicit declaration of function 'srp_attach_transport' /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/srp/ib_srp.c:2343: warning: assignment makes pointer from integer without a cast /home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/srp/ib_srp.c:2358: error: implicit declaration of function 'srp_release_transport' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/srp/ib_srp.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ppc64_check/drivers/infiniband/ulp/srp] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ppc64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090722-0200_linux-2.6.19_ppc64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ppc64/linux-2.6.19' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From sashak at voltaire.com Wed Jul 22 03:28:21 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 22 Jul 2009 13:28:21 +0300 Subject: [ofa-general] multiple path in opensm??? In-Reply-To: References: Message-ID: <20090722102821.GN31735@me> On 10:23 Wed 22 Jul , xiaoshuang xia wrote: > > I recently want to implement a routing scheme using opensm, but I face some > problem. Help me pls!! THX!! > > The problem is > > 1. In the opensm, if the multi-path between src and dst is denoted by the > multiple dst-lid. > > 2. If there are multi-path between the src and dst , what mechanism does > client choose the one of them. If this mechanism is implemented by opensm, > which file do this job? If choose one prefered path doesn 't implement by > opensm, what unit(call it "the choosing unit") finish this job, what data > structure does opensm hold to exchange the multi-path information to "the > choosing unit" and what file does this exchanging job in opensm. It is all about SA PathRecord querying, it is processed in osm_sa_path_record.c. Sasha From sashak at voltaire.com Wed Jul 22 03:59:07 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 22 Jul 2009 13:59:07 +0300 Subject: [ofa-general] [PATCH] opensm/osm_sm.c: code consolidation In-Reply-To: <20090721153228.GJ31735@me> References: <20090721153228.GJ31735@me> Message-ID: <20090722105907.GP31735@me> Consolidate lock releasing code in multicast join and leave processors. Signed-off-by: Sasha Khapyorsky --- opensm/opensm/osm_sm.c | 14 ++++---------- 1 files changed, 4 insertions(+), 10 deletions(-) diff --git a/opensm/opensm/osm_sm.c b/opensm/opensm/osm_sm.c index daa60ff..b3ce69a 100644 --- a/opensm/opensm/osm_sm.c +++ b/opensm/opensm/osm_sm.c @@ -487,7 +487,6 @@ ib_api_status_t osm_sm_mcgrp_join(IN osm_sm_t * p_sm, IN const ib_net16_t mlid, CL_PLOCK_EXCL_ACQUIRE(p_sm->p_lock); p_port = osm_get_port_by_guid(p_sm->p_subn, port_guid); if (!p_port) { - CL_PLOCK_RELEASE(p_sm->p_lock); OSM_LOG(p_sm->p_log, OSM_LOG_ERROR, "ERR 2E05: " "No port object for port 0x%016" PRIx64 "\n", cl_ntoh64(port_guid)); @@ -506,7 +505,6 @@ ib_api_status_t osm_sm_mcgrp_join(IN osm_sm_t * p_sm, IN const ib_net16_t mlid, * This can happen since the spinlock is released briefly * before the SA calls this function. */ - CL_PLOCK_RELEASE(p_sm->p_lock); OSM_LOG(p_sm->p_log, OSM_LOG_ERROR, "ERR 2E12: " "MC group with mlid 0x%x doesn't exist or " "port 0x%016" PRIx64 " is not in the group.\n", @@ -523,7 +521,6 @@ ib_api_status_t osm_sm_mcgrp_join(IN osm_sm_t * p_sm, IN const ib_net16_t mlid, p_mcm = (osm_mcm_info_t *) cl_qlist_head(&p_port->mcm_list); while (p_mcm != (osm_mcm_info_t *) cl_qlist_end(&p_port->mcm_list)) { if (p_mcm->mlid == mlid) { - CL_PLOCK_RELEASE(p_sm->p_lock); OSM_LOG(p_sm->p_log, OSM_LOG_DEBUG, "Found mlid object for Port:" "0x%016" PRIx64 " lid:0x%X\n", @@ -535,7 +532,6 @@ ib_api_status_t osm_sm_mcgrp_join(IN osm_sm_t * p_sm, IN const ib_net16_t mlid, status = osm_port_add_mgrp(p_port, mlid); if (status != IB_SUCCESS) { - CL_PLOCK_RELEASE(p_sm->p_lock); OSM_LOG(p_sm->p_log, OSM_LOG_ERROR, "ERR 2E03: " "Unable to associate port 0x%" PRIx64 " to mlid 0x%X\n", cl_ntoh64(osm_port_get_guid(p_port)), @@ -544,10 +540,10 @@ ib_api_status_t osm_sm_mcgrp_join(IN osm_sm_t * p_sm, IN const ib_net16_t mlid, } status = sm_mgrp_process(p_sm, p_mgrp); - CL_PLOCK_RELEASE(p_sm->p_lock); - Exit: + CL_PLOCK_RELEASE(p_sm->p_lock); OSM_LOG_EXIT(p_sm->p_log); + return status; } @@ -573,7 +569,6 @@ ib_api_status_t osm_sm_mcgrp_leave(IN osm_sm_t * p_sm, IN const ib_net16_t mlid, p_port = osm_get_port_by_guid(p_sm->p_subn, port_guid); if (!p_port) { - CL_PLOCK_RELEASE(p_sm->p_lock); OSM_LOG(p_sm->p_log, OSM_LOG_ERROR, "ERR 2E04: " "No port object for port 0x%" PRIx64 "\n", cl_ntoh64(port_guid)); @@ -586,7 +581,6 @@ ib_api_status_t osm_sm_mcgrp_leave(IN osm_sm_t * p_sm, IN const ib_net16_t mlid, */ p_mgrp = osm_get_mgrp_by_mlid(p_sm->p_subn, mlid); if (!p_mgrp) { - CL_PLOCK_RELEASE(p_sm->p_lock); OSM_LOG(p_sm->p_log, OSM_LOG_ERROR, "ERR 2E08: " "No multicast group for MLID 0x%X\n", cl_ntoh16(mlid)); status = IB_INVALID_PARAMETER; @@ -599,10 +593,10 @@ ib_api_status_t osm_sm_mcgrp_leave(IN osm_sm_t * p_sm, IN const ib_net16_t mlid, osm_port_remove_mgrp(p_port, mlid); status = sm_mgrp_process(p_sm, p_mgrp); - CL_PLOCK_RELEASE(p_sm->p_lock); - Exit: + CL_PLOCK_RELEASE(p_sm->p_lock); OSM_LOG_EXIT(p_sm->p_log); + return status; } -- 1.6.3.3 From sashak at voltaire.com Wed Jul 22 04:06:46 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 22 Jul 2009 14:06:46 +0300 Subject: [ofa-general] [PATCH] opensm/osm_mcst_mgr.c: check number of switches only once In-Reply-To: <20090721153228.GJ31735@me> References: <20090721153228.GJ31735@me> Message-ID: <20090722110646.GQ31735@me> Obviously we don't need to re-check a number of switches on a subnet for each multicast group. Signed-off-by: Sasha Khapyorsky --- opensm/opensm/osm_mcast_mgr.c | 25 ++++++++++++++++--------- 1 files changed, 16 insertions(+), 9 deletions(-) diff --git a/opensm/opensm/osm_mcast_mgr.c b/opensm/opensm/osm_mcast_mgr.c index 268e1ab..41e837d 100644 --- a/opensm/opensm/osm_mcast_mgr.c +++ b/opensm/opensm/osm_mcast_mgr.c @@ -1035,15 +1035,6 @@ static ib_api_status_t osm_mcast_mgr_process_tree(osm_sm_t * sm, "Processing multicast group 0x%X\n", cl_ntoh16(mlid)); /* - If there are no switches in the subnet, then we have nothing to do. - */ - if (cl_qmap_count(&sm->p_subn->sw_guid_tbl) == 0) { - OSM_LOG(sm->p_log, OSM_LOG_DEBUG, - "No switches in subnet. Nothing to do\n"); - goto Exit; - } - - /* Clear the multicast tables to start clean, then build the spanning tree which sets the mcast table bits for each port in the group. @@ -1124,6 +1115,13 @@ int osm_mcast_mgr_process(osm_sm_t * sm) */ CL_PLOCK_EXCL_ACQUIRE(sm->p_lock); + /* If there are no switches in the subnet we have nothing to do. */ + if (cl_qmap_count(&sm->p_subn->sw_guid_tbl) == 0) { + OSM_LOG(sm->p_log, OSM_LOG_DEBUG, + "No switches in subnet. Nothing to do\n"); + goto exit; + } + for (i = 0; i <= sm->p_subn->max_mcast_lid_ho - IB_LID_MCAST_START_HO; i++) { /* @@ -1151,6 +1149,7 @@ int osm_mcast_mgr_process(osm_sm_t * sm) free(p); } +exit: CL_PLOCK_RELEASE(sm->p_lock); OSM_LOG_EXIT(sm->p_log); @@ -1177,6 +1176,13 @@ int osm_mcast_mgr_process_mgroups(osm_sm_t * sm) /* we need a lock to make sure the p_mgrp is not change other ways */ CL_PLOCK_EXCL_ACQUIRE(sm->p_lock); + /* If there are no switches in the subnet we have nothing to do. */ + if (cl_qmap_count(&sm->p_subn->sw_guid_tbl) == 0) { + OSM_LOG(sm->p_log, OSM_LOG_DEBUG, + "No switches in subnet. Nothing to do\n"); + goto exit; + } + while (!cl_is_qlist_empty(p_list)) { ctx = (osm_mcast_mgr_ctxt_t *) cl_qlist_remove_head(p_list); @@ -1221,6 +1227,7 @@ int osm_mcast_mgr_process_mgroups(osm_sm_t * sm) osm_dump_mcast_routes(sm->p_subn->p_osm); +exit: CL_PLOCK_RELEASE(sm->p_lock); OSM_LOG_EXIT(sm->p_log); return ret; -- 1.6.3.3 From sashak at voltaire.com Wed Jul 22 05:02:39 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 22 Jul 2009 15:02:39 +0300 Subject: [ofa-general] [PATCH] opensm/osm_macst_mgr.c: merge functions In-Reply-To: <20090722110646.GQ31735@me> References: <20090721153228.GJ31735@me> <20090722110646.GQ31735@me> Message-ID: <20090722120239.GR31735@me> Merge functionally almost equivalent functions: mcast_mgr_process_mgrp() and osm_mcast_mgr_process_tree(). Signed-off-by: Sasha Khapyorsky --- opensm/opensm/osm_mcast_mgr.c | 59 ++++++++++++----------------------------- 1 files changed, 17 insertions(+), 42 deletions(-) diff --git a/opensm/opensm/osm_mcast_mgr.c b/opensm/opensm/osm_mcast_mgr.c index 41e837d..4dbbaa0 100644 --- a/opensm/opensm/osm_mcast_mgr.c +++ b/opensm/opensm/osm_mcast_mgr.c @@ -1019,10 +1019,11 @@ Exit: #endif /********************************************************************** - lock must already be held on entry -**********************************************************************/ -static ib_api_status_t osm_mcast_mgr_process_tree(osm_sm_t * sm, - IN osm_mgrp_t * p_mgrp) + Process the entire group. + NOTE : The lock should be held externally! + **********************************************************************/ +static ib_api_status_t mcast_mgr_process_mgrp(osm_sm_t * sm, + IN osm_mgrp_t * p_mgrp) { ib_api_status_t status = IB_SUCCESS; ib_net16_t mlid; @@ -1041,44 +1042,15 @@ static ib_api_status_t osm_mcast_mgr_process_tree(osm_sm_t * sm, */ mcast_mgr_clear(sm, cl_ntoh16(mlid)); - if (!p_mgrp->full_members) - goto Exit; - - status = mcast_mgr_build_spanning_tree(sm, p_mgrp); - if (status != IB_SUCCESS) { - OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0A17: " - "Unable to create spanning tree (%s)\n", - ib_get_err_str(status)); - goto Exit; - } - -Exit: - OSM_LOG_EXIT(sm->p_log); - return (status); -} - -/********************************************************************** - Process the entire group. - NOTE : The lock should be held externally! - **********************************************************************/ -static ib_api_status_t mcast_mgr_process_mgrp(osm_sm_t * sm, - IN osm_mgrp_t * p_mgrp) -{ - ib_api_status_t status; - - OSM_LOG_ENTER(sm->p_log); - - status = osm_mcast_mgr_process_tree(sm, p_mgrp); - if (status != IB_SUCCESS) { - OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0A19: " - "Unable to create spanning tree (%s)\n", - ib_get_err_str(status)); - goto Exit; - } - p_mgrp->last_tree_id = p_mgrp->last_change_id; - - /* remove MCGRP if it is marked for deletion */ - if (p_mgrp->to_be_deleted) { + if (p_mgrp->full_members) { + status = mcast_mgr_build_spanning_tree(sm, p_mgrp); + if (status != IB_SUCCESS) { + OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0A17: " + "Unable to create spanning tree (%s)\n", + ib_get_err_str(status)); + goto Exit; + } + } else if (p_mgrp->to_be_deleted) { OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "Destroying mgrp with lid:0x%x\n", cl_ntoh16(p_mgrp->mlid)); @@ -1087,8 +1059,11 @@ static ib_api_status_t mcast_mgr_process_mgrp(osm_sm_t * sm, cl_fmap_remove_item(&sm->p_subn->mgrp_mgid_tbl, &p_mgrp->map_item); osm_mgrp_delete(p_mgrp); + goto Exit; } + p_mgrp->last_tree_id = p_mgrp->last_change_id; + Exit: OSM_LOG_EXIT(sm->p_log); return status; -- 1.6.3.3 From ogerlitz at Voltaire.com Wed Jul 22 05:17:25 2009 From: ogerlitz at Voltaire.com (Or Gerlitz) Date: Wed, 22 Jul 2009 15:17:25 +0300 Subject: [ofa-general] can an ib-bonding slave work independently? In-Reply-To: <4A66C0E6.4090500@sun.com> References: <20090715025215.GA4322@sun.com> <4A5D897F.20708@Voltaire.com> <20090716180024.GB5930@sun.com> <4A65AAB2.4000009@voltaire.com> <4A65E477.30107@sun.com> <4A66A29F.40509@Voltaire.com> <4A66C0E6.4090500@sun.com> Message-ID: <4A670355.4020308@Voltaire.com> Liang Zhen wrote: > Yes, I just noticed that Lustre is in list of supported ULPs of ib-bonding (in your ppt) I listed it there simply since Lustre uses the rdma-cm, there's need not be anything special in Lustre to be supported by bonding (ib-bonding is just bonding) Or. From hnrose at comcast.net Wed Jul 22 05:44:49 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Wed, 22 Jul 2009 08:44:49 -0400 Subject: [ofa-general] [PATCHv5] opensm/lash: Set minimum VL for LASH to use Message-ID: <20090722124448.GA24415@comcast.net> rather than starting from VL 0 Signed-off-by: Robert Pearson Signed-off-by: Hal Rosenstock --- Changes since v4: Eliminate redundant insufficient lane checks In discover_network_properties, handle min operational VL insufficient for start VL Pointed out by Sasha Changes since v3: Reduce vl_min by start_vl and preserve most code flows Only need to update routing_table.lane Simplification pointed out by Sasha Changes since v2: Restored malloc use (rather than calloc) as pointed out by Sasha Changes since v1: Fixed comparisons with maximum VL Better lash_start_vl option handling Both as pointed out by Sasha diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 59a32ad..da8cc5e 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -221,6 +222,7 @@ typedef struct osm_subn_opt { char *prefix_routes_file; boolean_t consolidate_ipv6_snm_req; struct osm_subn_opt *file_opts; /* used for update */ + uint8_t lash_start_vl; /* starting vl to use in lash */ } osm_subn_opt_t; /* * FIELDS diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in index dbf4b73..c71a79d 100644 --- a/opensm/man/opensm.8.in +++ b/opensm/man/opensm.8.in @@ -1,4 +1,4 @@ -.TH OPENSM 8 "April 22, 2009" "OpenIB" "OpenIB Management" +.TH OPENSM 8 "May 28, 2009" "OpenIB" "OpenIB Management" .SH NAME opensm \- InfiniBand subnet manager and administration (SM/SA) @@ -15,6 +15,7 @@ opensm \- InfiniBand subnet manager and administration (SM/SA) [\-r(eassign_lids)] [\-R | \-\-routing_engine ] [\-\-do_mesh_analysis] +[\-\-lash_start_vl ] [\-A | \-\-ucast_cache] [\-z | \-\-connect_roots] [\-M | \-\-lid_matrix_file ] @@ -147,6 +148,10 @@ This option enables additional analysis for the lash routing engine to precondition switch port assignments in regular cartesian meshes which may reduce the number of SLs required to give a deadlock free routing. .TP +\fB\-\-lash_start_vl\fR +This option sets the starting VL to use for the lash routing algorithm. +Defaults to 0. +.TP \fB\-A\fR, \fB\-\-ucast_cache\fR This option enables unicast routing cache and prevents routing recalculation (which is a heavy task in a large cluster) when diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c index 296d5d5..d682ff5 100644 --- a/opensm/opensm/main.c +++ b/opensm/opensm/main.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009 HNR Consulting. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -181,6 +182,9 @@ static void show_usage(void) " routing engine to precondition switch port assignments\n" " in regular cartesian meshes which may reduce the number\n" " of SLs required to give a deadlock free routing\n\n"); + printf("--lash_start_vl \n" + " Sets the starting VL to use for the lash routing algorithm.\n" + " Defaults to 0.\n"); printf("--connect_roots, -z\n" " This option enforces a routing engine (currently\n" " up/down only) to make connectivity between root switches\n" @@ -601,6 +605,7 @@ int main(int argc, char *argv[]) {"prefix_routes_file", 1, NULL, 3}, {"consolidate_ipv6_snm_req", 0, NULL, 4}, {"do_mesh_analysis", 0, NULL, 5}, + {"lash_start_vl", 1, NULL, 6}, {NULL, 0, NULL, 0} /* Required at the end of the array */ }; @@ -951,6 +956,15 @@ int main(int argc, char *argv[]) case 5: opt.do_mesh_analysis = TRUE; break; + case 6: + temp = strtol(optarg, NULL, 0); + if (temp < 0 || temp >= IB_MAX_NUM_VLS) { + fprintf(stderr, + "ERROR: starting lash vl must be between 0 and 15\n"); + return (-1); + } + opt.lash_start_vl = (uint8_t) temp; + break; case 'h': case '?': case ':': diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index ec15f8a..fda2eb0 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -3,6 +3,7 @@ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. + * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -389,6 +390,7 @@ static const opt_rec_t opt_tbl[] = { { "no_clients_rereg", OPT_OFFSET(no_clients_rereg), opts_parse_boolean, NULL, 1 }, { "prefix_routes_file", OPT_OFFSET(prefix_routes_file), opts_parse_charp, NULL, 0 }, { "consolidate_ipv6_snm_req", OPT_OFFSET(consolidate_ipv6_snm_req), opts_parse_boolean, NULL, 1 }, + { "lash_start_vl", OPT_OFFSET(lash_start_vl), opts_parse_uint8, NULL, 1 }, {0} }; @@ -749,6 +751,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt) p_opt->no_clients_rereg = FALSE; p_opt->prefix_routes_file = strdup(OSM_DEFAULT_PREFIX_ROUTES_FILE); p_opt->consolidate_ipv6_snm_req = FALSE; + p_opt->lash_start_vl = 0; subn_init_qos_options(&p_opt->qos_options, NULL); subn_init_qos_options(&p_opt->qos_ca_options, NULL); subn_init_qos_options(&p_opt->qos_sw0_options, NULL); @@ -1432,6 +1435,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) p_opts->do_mesh_analysis ? "TRUE" : "FALSE"); fprintf(out, + "# Starting VL for LASH algorithm\n" + "lash_start_vl %d\n\n", + p_opts->lash_start_vl); + + fprintf(out, "# SA database file name\nsa_db_file %s\n\n", p_opts->sa_db_file ? p_opts->sa_db_file : null_str); diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 12b5e34..67471f5 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -486,6 +486,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) int next_switch2, output_link2; int stop = 0, cycle_found; int cycle_found2; + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; max_filled_lane = 0; min_filled_lane = lanes_needed - 1; @@ -572,8 +573,8 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed) virtual_location[dest][src][max_filled_lane] = 0; virtual_location[src][dest][min_filled_lane] = 1; virtual_location[dest][src][min_filled_lane] = 1; - p_lash->switches[src]->routing_table[dest].lane = min_filled_lane; - p_lash->switches[dest]->routing_table[src].lane = min_filled_lane; + p_lash->switches[src]->routing_table[dest].lane = min_filled_lane + start_vl; + p_lash->switches[dest]->routing_table[src].lane = min_filled_lane + start_vl; } if (trials == 0) @@ -804,6 +805,7 @@ static int lash_core(lash_t * p_lash) int cycle_found2 = 0; int status = 0; int *switch_bitmap = NULL; /* Bitmap to check if we have processed this pair */ + unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl; OSM_LOG_ENTER(p_log); @@ -902,8 +904,8 @@ static int lash_core(lash_t * p_lash) } } - switches[i]->routing_table[dest_switch].lane = v_lane; - switches[dest_switch]->routing_table[i].lane = v_lane; + switches[i]->routing_table[dest_switch].lane = v_lane + start_vl; + switches[dest_switch]->routing_table[i].lane = v_lane + start_vl; if (cycle_found == 1 || cycle_found2 == 1) { if (++lanes_needed > p_lash->vl_min) @@ -928,14 +930,14 @@ static int lash_core(lash_t * p_lash) } } - OSM_LOG(p_log, OSM_LOG_INFO, - "Lanes needed: %d, Balancing\n", lanes_needed); - for (i = 0; i < lanes_needed; i++) { OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", i, p_lash->num_mst_in_lane[i]); } + OSM_LOG(p_log, OSM_LOG_INFO, + "Lanes needed: %d, Balancing\n", lanes_needed); + balance_virtual_lanes(p_lash, lanes_needed); for (i = 0; i < lanes_needed; i++) { @@ -948,8 +950,9 @@ static int lash_core(lash_t * p_lash) Error_Not_Enough_Lanes: status = -1; OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4D02: " - "Lane requirements (%d) exceed available lanes (%d)\n", - lanes_needed, p_lash->vl_min); + "Lane requirements (%d) exceed available lanes (%d)" + " with starting lane (%d)\n", + lanes_needed, p_lash->vl_min, start_vl); Exit: if (switch_bitmap) free(switch_bitmap); @@ -1177,10 +1180,18 @@ static int discover_network_properties(lash_t * p_lash) if (vl_min > 15) vl_min = 15; - p_lash->vl_min = vl_min; + if (p_lash->p_osm->subn.opt.lash_start_vl >= vl_min) { + OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4D03: " + "Start VL(%d) too high for min operational vl(%d)\n", + p_lash->p_osm->subn.opt.lash_start_vl, vl_min); + return -1; + } + + p_lash->vl_min = vl_min - p_lash->p_osm->subn.opt.lash_start_vl; OSM_LOG(p_log, OSM_LOG_INFO, - "min operational vl(%d) max_switches(%d)\n", p_lash->vl_min, + "min operational vl(%d) start vl(%d) max_switches(%d)\n", + p_lash->vl_min, p_lash->p_osm->subn.opt.lash_start_vl, p_lash->num_switches); return 0; } @@ -1231,7 +1242,8 @@ static int lash_process(void *context) populate_fwd_tbls(p_lash); Exit: - free_lash_structures(p_lash); + if (p_lash->vl_min) + free_lash_structures(p_lash); OSM_LOG_EXIT(p_log); return return_status; @@ -1286,7 +1298,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port, src_id = get_lash_id(p_sw); if (src_id == dst_id) - return OSM_DEFAULT_SL; + return p_osm->subn.opt.lash_start_vl; return (uint8_t) ((switch_t *) p_sw->priv)->routing_table[dst_id].lane; } From jgunthorpe at obsidianresearch.com Wed Jul 22 08:03:16 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Wed, 22 Jul 2009 09:03:16 -0600 Subject: [ofa-general] Re: [PATCH ibverbs] Make the gid argument to ibv_attach_mcast and ibv_detach_mcast const In-Reply-To: <200907221202.18326.jackm@dev.mellanox.co.il> References: <20090719052634.GA1149@obsidianresearch.com> <20090720185333.GU12693@obsidianresearch.com> <200907221202.18326.jackm@dev.mellanox.co.il> Message-ID: <20090722150316.GE20290@obsidianresearch.com> On Wed, Jul 22, 2009 at 12:02:18PM +0300, Jack Morgenstein wrote: > On Monday 20 July 2009 21:53, Jason Gunthorpe wrote: > > I have also patches for mlx4 and mthca to suppress the compiler > > warning that results from this patch. ipath is OK as is, and I'm not > > sure where the iwarp stuff lives.. > Is this change really necessary? This makes it easier/clearer for user apps - in C++ constness errors are not warnings. > Seems to me that you are creating cross-library dependencies (at > least with regard to compiler warnings) -- the user will now be > required to change the device libraries if libibverbs is upgraded. What? No, not at all. This change is ABI neutral in all the binary interfaces. All that happens is you get harmless compile warnings in the device libraries. The patches for the device libraries are just to remove the harmless warning, it doesn't affect anything. > Also, if the device libraries get changed, but libibverbs stays the > same, I think you will also get warnings if anyone tries to compile > user-level apps. Nobody will get warnings from existing user-level apps. The compiler silently casts from non-const to const. Jason From sashak at voltaire.com Wed Jul 22 08:05:07 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 22 Jul 2009 18:05:07 +0300 Subject: [ofa-general] Re: [PATCHv5] opensm/lash: Set minimum VL for LASH to use In-Reply-To: <20090722124448.GA24415@comcast.net> References: <20090722124448.GA24415@comcast.net> Message-ID: <20090722150507.GS31735@me> On 08:44 Wed 22 Jul , Hal Rosenstock wrote: > > rather than starting from VL 0 > > Signed-off-by: Robert Pearson > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From hnrose at comcast.net Wed Jul 22 08:16:15 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Wed, 22 Jul 2009 11:16:15 -0400 Subject: [ofa-general] [PATCHv2] opensm/osm_mesh.c: Reorder switches for lash Message-ID: <20090722151615.GA24576@comcast.net> This patch changes the order of the switches in the array kept in the lash context from the original order to one in which the switches are presented in 'odometer order'. When the main routine in lash is called, the switches are in an order that is likely based on the order that the switches were originally visited by SM topology discovery which is some sort of tree walk. All of the analysis up to this point is independent of the actual order of the switches, but lash will use that order to enumerate the paths in the fabric and add them to the VL bins. Odometer order means that the switches are labelled s[X0, ..., Xn-1] and ordered s[0, ..., 0], s[0, ..., 1], s[0, ..., Ln-1], s[0, .. 1, 0] etc. The dimensions are also reordered so that the dimension changing the fastest has the largest length, i.e. Ln >= Ln-1 >= ... >= L1. [All this is modulo possible end to end reversal but the basic idea is that the longest axis changes fastest.] Signed-off-by: Robert Pearson Signed-off-by: Hal Rosenstock --- Changes since v1: Made change reentrant FWIW Added more to patch description Added memory allocation failure handling diff --git a/opensm/opensm/osm_mesh.c b/opensm/opensm/osm_mesh.c index 23fad87..dce2ea1 100644 --- a/opensm/opensm/osm_mesh.c +++ b/opensm/opensm/osm_mesh.c @@ -185,6 +185,16 @@ typedef struct _mesh { int dim_order[MAX_DIMENSION]; } mesh_t; +typedef struct sort_ctx { + lash_t *p_lash; + mesh_t *mesh; +} sort_ctx_t; + +typedef struct comp { + int index; + sort_ctx_t *ctx; +} comp_t; + /* * poly_alloc * @@ -1272,6 +1282,87 @@ static int reorder_links(lash_t *p_lash, mesh_t *mesh) } /* + * compare two switches in a sort + */ +static int compare_switch(const void *p1, const void *p2) +{ + int i, j, d; + const comp_t *cp1 = p1, *cp2 = p2; + sort_ctx_t *ctx = cp1->ctx; + switch_t *s1 = ctx->p_lash->switches[cp1->index]; + switch_t *s2 = ctx->p_lash->switches[cp2->index]; + + for (i = 0; i < ctx->mesh->dimension; i++) { + j = ctx->mesh->dim_order[i]; + d = s1->node->coord[j] - s2->node->coord[j]; + + if (d > 0) + return 1; + + if (d < 0) + return -1; + } + + return 0; +} + +/* + * sort_switches - reorder switch array + */ +static void sort_switches(lash_t *p_lash, mesh_t *mesh) +{ + int i, j; + int num_switches = p_lash->num_switches; + sort_ctx_t sort_ctx; + comp_t *index; + int *reverse; + switch_t *s; + switch_t **switches; + + index = malloc(num_switches * sizeof(comp_t)); + reverse = malloc(num_switches * sizeof(int)); + switches = malloc(num_switches * sizeof(switch_t *)); + if (!index || !reverse || !switches) { + OSM_LOG(&p_lash->p_osm->log, OSM_LOG_ERROR, + "Failed memory allocation - switches not sorted!\n"); + goto Exit; + } + + sort_ctx.mesh = mesh; + sort_ctx.p_lash = p_lash; + + for (i = 0; i < num_switches; i++) { + index[i].index = i; + index[i].ctx = &sort_ctx; + } + + qsort(index, num_switches, sizeof(comp_t), compare_switch); + + for (i = 0; i < num_switches; i++) + reverse[index[i].index] = i; + + for (i = 0; i < num_switches; i++) { + s = p_lash->switches[index[i].index]; + switches[i] = s; + s->id = i; + for (j = 0; j < s->node->num_links; j++) + s->node->links[j]->switch_id = + reverse[s->node->links[j]->switch_id]; + } + + for (i = 0; i < num_switches; i++) + p_lash->switches[i] = switches[i]; + +Exit: + if (switches) + free(switches); + if (index) + free(index); + if (reverse) + free(reverse); +} + +/* * osm_mesh_delete - free per mesh resources */ static void mesh_delete(mesh_t *mesh) @@ -1470,6 +1561,8 @@ int osm_do_mesh_analysis(lash_t *p_lash) if (reorder_links(p_lash, mesh)) goto err; + sort_switches(p_lash, mesh); + p = buf; p += sprintf(p, "found "); for (i = 0; i < mesh->dimension; i++) From rdreier at cisco.com Wed Jul 22 09:19:58 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 22 Jul 2009 09:19:58 -0700 Subject: [ofa-general] Re: [PATCH ibverbs] Make the gid argument to ibv_attach_mcast and ibv_detach_mcast const In-Reply-To: <200907221202.18326.jackm@dev.mellanox.co.il> (Jack Morgenstein's message of "Wed, 22 Jul 2009 12:02:18 +0300") References: <20090719052634.GA1149@obsidianresearch.com> <20090720185333.GU12693@obsidianresearch.com> <200907221202.18326.jackm@dev.mellanox.co.il> Message-ID: > Seems to me that you are creating cross-library > dependencies (at least with regard to compiler warnings) -- the user will now be > required to change the device libraries if libibverbs is upgraded. Also, if the > device libraries get changed, but libibverbs stays the same, I think you will also > get warnings if anyone tries to compile user-level apps. This doesn't seem right to me. The change to use ibv_cmd_xxx in driver libraries should not generate warnings with either old or new libibverbs, since the prototypes match in all versions of libibverbs. And in any case device libraries should have no effect on user-level apps at all. Then the change to the user-level verbs to add const should also not add any more warnings -- since passing non-const into a const parameter is fine; the issue is the other way around, which is what Jason's change fixes. From rdreier at cisco.com Wed Jul 22 10:47:23 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 22 Jul 2009 10:47:23 -0700 Subject: [ofa-general] [PATCH/RFC] ummunot: Userspace support for MMU notifications Message-ID: As discussed in and follow-up messages, libraries using RDMA would like to track precisely when application code changes memory mapping via free(), munmap(), etc. Current pure-userspace solutions using malloc hooks and other tricks are not robust, and the feeling among experts is that the issue is unfixable without kernel help. We solve this not by implementing the full API proposed in the email linked above but rather with a simpler and more generic interface, which may be useful in other contexts. Specifically, we implement a new character device driver, ummunot, that creates a /dev/ummunot node. A userspace process can open this node read-only and use the fd as follows: 1. ioctl() to register/unregister an address range to watch in the kernel (cf struct ummunot_register_ioctl in ). 2. read() to retrieve events generated when a mapping in a watched address range is invalidated (cf struct ummunot_event in ). select()/poll()/epoll() and SIGIO are handled for this IO. 3. mmap() one page at offset 0 to map a kernel page that contains a generation counter that is incremented each time an event is generated. This allows userspace to have a fast path that checks that no events have occurred without a system call. Thanks to Jason Gunthorpe for suggestions on the interface design. Also thanks to Jeff Squyres for prototyping support for this in Open MPI, which helped find several bugs during development. Signed-off-by: Roland Dreier --- Andrew -- sending this to you since I don't really know who should merge this sort of thing. Solving the underlying issue here is a pretty big deal to the universe of people that work on MPI implementations (sort of middleware for high-performance computing). I think everyone would be open to better ideas on how to handle this, although I'm pretty happy with how small and self-contained the ummunot implementation ended up being. Anyway, review, suggestions, etc. all gratefully received. Thanks, Roland drivers/char/Kconfig | 12 ++ drivers/char/Makefile | 1 + drivers/char/ummunot.c | 489 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/ummunot.h | 130 +++++++++++++ 4 files changed, 632 insertions(+), 0 deletions(-) create mode 100644 drivers/char/ummunot.c create mode 100644 include/linux/ummunot.h diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 6a06913..8d3dd9d 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -1108,6 +1108,18 @@ config DEVPORT depends on ISA || PCI default y +config UMMUNOT + tristate "Userspace MMU notifications" + select MMU_NOTIFIER + help + The ummunot (userspace MMU notification) driver creates a + character device that can be used by userspace libraries to + get notifications when an application's memory mapping + changed. This is used, for example, by RDMA libraries to + improve the reliability of memory registration caching, since + the kernel's MMU notifications can be used to know precisely + when to shoot down a cached registration. + source "drivers/s390/char/Kconfig" endmenu diff --git a/drivers/char/Makefile b/drivers/char/Makefile index 66f779a..39ff0c3 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -97,6 +97,7 @@ obj-$(CONFIG_NSC_GPIO) += nsc_gpio.o obj-$(CONFIG_CS5535_GPIO) += cs5535_gpio.o obj-$(CONFIG_GPIO_TB0219) += tb0219.o obj-$(CONFIG_TELCLOCK) += tlclk.o +obj-$(CONFIG_UMMUNOT) += ummunot.o obj-$(CONFIG_MWAVE) += mwave/ obj-$(CONFIG_AGP) += agp/ diff --git a/drivers/char/ummunot.c b/drivers/char/ummunot.c new file mode 100644 index 0000000..4a13449 --- /dev/null +++ b/drivers/char/ummunot.c @@ -0,0 +1,489 @@ +/* + * Copyright (c) 2009 Cisco Systems. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version + * 2 as published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +MODULE_AUTHOR("Roland Dreier"); +MODULE_DESCRIPTION("Userspace MMU notifiers"); +MODULE_LICENSE("GPL v2"); + +enum { + UMMUNOT_FLAG_DIRTY = 1, + UMMUNOT_FLAG_HINT = 2, +}; + +struct ummunot_reg { + u64 user_cookie; + unsigned long start; + unsigned long end; + unsigned long hint_start; + unsigned long hint_end; + unsigned long flags; + struct rb_node node; + struct list_head list; +}; + +struct ummunot_file { + struct mmu_notifier mmu_notifier; + struct mm_struct *mm; + struct rb_root reg_tree; + struct list_head dirty_list; + u64 *counter; + spinlock_t lock; + wait_queue_head_t read_wait; + struct fasync_struct *async_queue; + int need_empty; + int used; +}; + +static struct ummunot_file *to_ummunot_file(struct mmu_notifier *mn) +{ + return container_of(mn, struct ummunot_file, mmu_notifier); +} + +static void ummunot_handle_not(struct mmu_notifier *mn, + unsigned long start, unsigned long end) +{ + struct ummunot_file *priv = to_ummunot_file(mn); + struct rb_node *n; + struct ummunot_reg *reg; + unsigned long flags; + int hit = 0; + + spin_lock_irqsave(&priv->lock, flags); + + for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) { + reg = rb_entry(n, struct ummunot_reg, node); + + if (reg->start >= end) + break; + + /* + * Ranges overlap if they're not disjoint; and they're + * disjoint if the end of one is before the start of + * the other one. + */ + if (!(reg->end <= start || end <= reg->start)) { + hit = 1; + + if (!test_and_set_bit(UMMUNOT_FLAG_DIRTY, ®->flags)) + list_add_tail(®->list, &priv->dirty_list); + + if (test_bit(UMMUNOT_FLAG_HINT, ®->flags)) { + clear_bit(UMMUNOT_FLAG_HINT, ®->flags); + } else { + set_bit(UMMUNOT_FLAG_HINT, ®->flags); + reg->hint_start = start; + reg->hint_end = end; + } + } + } + + if (hit) { + ++(*priv->counter); + flush_dcache_page(virt_to_page(priv->counter)); + wake_up_interruptible(&priv->read_wait); + kill_fasync(&priv->async_queue, SIGIO, POLL_IN); + } + + spin_unlock_irqrestore(&priv->lock, flags); +} + +static void ummunot_inval_page(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long addr) +{ + ummunot_handle_not(mn, addr, addr + PAGE_SIZE); +} + +static void ummunot_inval_range_start(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + ummunot_handle_not(mn, start, end); +} + +static const struct mmu_notifier_ops ummunot_mmu_notifier_ops = { + .invalidate_page = ummunot_inval_page, + .invalidate_range_start = ummunot_inval_range_start, +}; + +static int ummunot_open(struct inode *inode, struct file *filp) +{ + struct ummunot_file *priv; + int ret; + + if (filp->f_mode & FMODE_WRITE) + return -EINVAL; + + priv = kmalloc(sizeof *priv, GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->counter = (void *) get_zeroed_page(GFP_KERNEL); + if (!priv->counter) { + ret = -ENOMEM; + goto err; + } + + priv->reg_tree = RB_ROOT; + INIT_LIST_HEAD(&priv->dirty_list); + spin_lock_init(&priv->lock); + init_waitqueue_head(&priv->read_wait); + priv->async_queue = NULL; + priv->need_empty = 0; + priv->used = 0; + + priv->mmu_notifier.ops = &ummunot_mmu_notifier_ops; + /* + * Register notifier last, since notifications can occur as + * soon as we register.... + */ + ret = mmu_notifier_register(&priv->mmu_notifier, current->mm); + if (ret) + goto err_page; + + priv->mm = current->mm; + atomic_inc(&priv->mm->mm_count); + + filp->private_data = priv; + + return 0; + +err_page: + free_page((unsigned long) priv->counter); + +err: + kfree(priv); + return ret; +} + +static int ummunot_close(struct inode *inode, struct file *filp) +{ + struct ummunot_file *priv = filp->private_data; + struct rb_node *n; + struct ummunot_reg *reg; + + mmu_notifier_unregister(&priv->mmu_notifier, priv->mm); + mmdrop(priv->mm); + free_page((unsigned long) priv->counter); + + for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) { + reg = rb_entry(n, struct ummunot_reg, node); + kfree(reg); + } + + kfree(priv); + + return 0; +} + +static bool ummunot_readable(struct ummunot_file *priv) +{ + return priv->need_empty || !list_empty(&priv->dirty_list); +} + +static ssize_t ummunot_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct ummunot_file *priv = filp->private_data; + struct ummunot_reg *reg; + ssize_t ret; + struct ummunot_event *events; + int max; + int n; + + priv->used = 1; + + events = (void *) get_zeroed_page(GFP_KERNEL); + if (!events) { + ret = -ENOMEM; + goto out; + } + + spin_lock_irq(&priv->lock); + + while (!ummunot_readable(priv)) { + spin_unlock_irq(&priv->lock); + + if (filp->f_flags & O_NONBLOCK) { + ret = -EAGAIN; + goto out; + } + + if (wait_event_interruptible(priv->read_wait, + ummunot_readable(priv))) { + ret = -ERESTARTSYS; + goto out; + } + + spin_lock_irq(&priv->lock); + } + + max = min(PAGE_SIZE, count) / sizeof *events; + + for (n = 0; n < max; ++n) { + if (list_empty(&priv->dirty_list)) { + events[n].type = UMMUNOT_EVENT_TYPE_LAST; + events[n].user_cookie_counter = *priv->counter; + ++n; + priv->need_empty = 0; + break; + } + + reg = list_first_entry(&priv->dirty_list, struct ummunot_reg, + list); + + events[n].type = UMMUNOT_EVENT_TYPE_INVAL; + if (test_bit(UMMUNOT_FLAG_HINT, ®->flags)) { + events[n].flags = UMMUNOT_EVENT_FLAG_HINT; + events[n].hint_start = max(reg->start, reg->hint_start); + events[n].hint_end = min(reg->end, reg->hint_end); + } else { + events[n].hint_start = reg->start; + events[n].hint_end = reg->end; + } + events[n].user_cookie_counter = reg->user_cookie; + + list_del(®->list); + reg->flags = 0; + priv->need_empty = 1; + } + + spin_unlock_irq(&priv->lock); + + if (copy_to_user(buf, events, n * sizeof *events)) + ret = -EFAULT; + else + ret = n * sizeof *events; + +out: + free_page((unsigned long) events); + return ret; +} + +static unsigned int ummunot_poll(struct file *filp, struct poll_table_struct *wait) +{ + struct ummunot_file *priv = filp->private_data; + + poll_wait(filp, &priv->read_wait, wait); + + return ummunot_readable(priv) ? (POLLIN | POLLRDNORM) : 0; +} + +static long ummunot_exchange_features(struct ummunot_file *priv, + __u32 __user *arg) +{ + u32 feature_mask; + + if (priv->used) + return -EINVAL; + + priv->used = 1; + + if (get_user(feature_mask, arg)) + return -EFAULT; + + /* No extensions defined at present. */ + feature_mask = 0; + + if (put_user(feature_mask, arg)) + return -EFAULT; + + return 0; +} + +static long ummunot_register_region(struct ummunot_file *priv, + struct ummunot_register_ioctl __user *arg) +{ + struct ummunot_register_ioctl parm; + struct ummunot_reg *reg, *treg; + struct rb_node **n = &priv->reg_tree.rb_node; + struct rb_node *pn; + int ret = 0; + + if (copy_from_user(&parm, arg, sizeof parm)) + return -EFAULT; + + priv->used = 1; + + reg = kmalloc(sizeof *reg, GFP_KERNEL); + if (!reg) + return -ENOMEM; + + reg->user_cookie = parm.user_cookie; + reg->start = parm.start; + reg->end = parm.end; + reg->flags = 0; + + spin_lock_irq(&priv->lock); + + for (pn = rb_first(&priv->reg_tree); pn; pn = rb_next(pn)) { + treg = rb_entry(pn, struct ummunot_reg, node); + + if (treg->user_cookie == parm.user_cookie) { + kfree(reg); + ret = -EINVAL; + goto out; + } + } + + pn = NULL; + while (*n) { + pn = *n; + treg = rb_entry(pn, struct ummunot_reg, node); + + if (reg->start <= treg->start) + n = &pn->rb_left; + else + n = &pn->rb_right; + } + + rb_link_node(®->node, pn, n); + rb_insert_color(®->node, &priv->reg_tree); + +out: + spin_unlock_irq(&priv->lock); + + return ret; +} + +static long ummunot_unregister_region(struct ummunot_file *priv, + __u64 __user *arg) +{ + u64 user_cookie; + struct rb_node *n; + struct ummunot_reg *reg; + int ret = -EINVAL; + + if (get_user(user_cookie, arg)) + return -EFAULT; + + spin_lock_irq(&priv->lock); + + for (n = rb_first(&priv->reg_tree); n; n = rb_next(n)) { + reg = rb_entry(n, struct ummunot_reg, node); + + if (reg->user_cookie == user_cookie) { + rb_erase(n, &priv->reg_tree); + if (test_bit(UMMUNOT_FLAG_DIRTY, ®->flags)) + list_del(®->list); + kfree(reg); + ret = 0; + break; + } + } + + spin_unlock_irq(&priv->lock); + + return ret; +} + +static long ummunot_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + struct ummunot_file *priv = filp->private_data; + void __user *argp = (void __user *) arg; + + switch (cmd) { + case UMMUNOT_EXCHANGE_FEATURES: + return ummunot_exchange_features(priv, argp); + case UMMUNOT_REGISTER_REGION: + return ummunot_register_region(priv, argp); + case UMMUNOT_UNREGISTER_REGION: + return ummunot_unregister_region(priv, argp); + default: + return -ENOIOCTLCMD; + } +} + +static int ummunot_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct ummunot_file *priv = vma->vm_private_data; + + if (vmf->pgoff != 0) + return VM_FAULT_SIGBUS; + + vmf->page = virt_to_page(priv->counter); + get_page(vmf->page); + + return 0; + +} + +static struct vm_operations_struct ummunot_vm_ops = { + .fault = ummunot_fault, +}; + +static int ummunot_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct ummunot_file *priv = filp->private_data; + + if (vma->vm_end - vma->vm_start != PAGE_SIZE || + vma->vm_pgoff != 0) + return -EINVAL; + + vma->vm_ops = &ummunot_vm_ops; + vma->vm_private_data = priv; + + return 0; +} + +static int ummunot_fasync(int fd, struct file *filp, int on) +{ + struct ummunot_file *priv = filp->private_data; + + return fasync_helper(fd, filp, on, &priv->async_queue); +} + +static const struct file_operations ummunot_fops = { + .owner = THIS_MODULE, + .open = ummunot_open, + .release = ummunot_close, + .read = ummunot_read, + .poll = ummunot_poll, + .unlocked_ioctl = ummunot_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ummunot_ioctl, +#endif + .mmap = ummunot_mmap, + .fasync = ummunot_fasync, +}; + +static struct miscdevice ummunot_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "ummunot", + .fops = &ummunot_fops, +}; + +static int __init ummunot_init(void) +{ + return misc_register(&ummunot_misc); +} + +static void __exit ummunot_cleanup(void) +{ + misc_deregister(&ummunot_misc); +} + +module_init(ummunot_init); +module_exit(ummunot_cleanup); diff --git a/include/linux/ummunot.h b/include/linux/ummunot.h new file mode 100644 index 0000000..4640e36 --- /dev/null +++ b/include/linux/ummunot.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2009 Cisco Systems. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenFabrics BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _LINUX_UMMUNOT_H +#define _LINUX_UMMUNOT_H + +#include +#include + +/* + * Ummunot relays MMU notifier events to userspace. A userspace + * process uses it by opening /dev/ummunot, which returns a file + * descriptor. Interest in address ranges is registered using ioctl() + * and MMU notifier events are retrieved using read(), as described in + * more detail below. + */ + +/* + * struct ummunot_register_ioctl describes an address range from start + * to end (including start but not including end) to be monitored. + * user_cookie is an opaque handle that userspace assigns, and which + * is used to unregister. flags and reserved are currently unused and + * should be set to 0 for forward compatibility. + */ +struct ummunot_register_ioctl { + __u64 start; /* in */ + __u64 end; /* in */ + __u64 user_cookie; /* in */ + __u32 flags; /* in */ + __u32 reserved; /* in */ +}; + +#define UMMUNOT_MAGIC 'U' + +/* + * Forward compatibility: Userspace passes in a 32-bit feature mask + * with feature flags set indicating which extensions it wishes to + * use. The kernel will return a feature mask with the bits of + * userspace's mask that the kernel implements; from that point on + * both userspace and the kernel should behave as described by the + * kernel's feature mask. + * + * If userspace does not perform a UMMUNOT_EXCHANGE_FEATURES ioctl, + * then the kernel will use a feature mask of 0. + * + * No feature flags are currently defined, so the kernel will always + * return a feature mask of 0 at present. + */ +#define UMMUNOT_EXCHANGE_FEATURES _IOWR(UMMUNOT_MAGIC, 1, __u32) + +/* + * Register interest in an address range; userspace should pass in a + * struct ummunot_register_ioctl describing the region. + */ +#define UMMUNOT_REGISTER_REGION _IOW(UMMUNOT_MAGIC, 2, \ + struct ummunot_register_ioctl) +/* + * Unregister interest in an address range; userspace should pass in + * the user_cookie value that was used to register the address range. + * No events for the address range will be reported once it is + * unregistered. + */ +#define UMMUNOT_UNREGISTER_REGION _IOW(UMMUNOT_MAGIC, 3, __u64) + +/* + * Invalidation events are returned whenever the kernel changes the + * mapping for a monitored address. These events are retrieved by + * read() on the ummunot file descriptor, which will fill the read() + * buffer with struct ummunot_event. + * + * If type field is INVAL, then user_cookie_counter holds the + * user_cookie for the region being reported; if the HINT flag is set + * then hint_start/hint_end hold the start and end of the mapping that + * was invalidated. (If HINT is not set, then multiple events + * invalidated parts of the registered range and hint_start/hint_end + * and set to the start/end of the whole registered range) + * + * If type is LAST, then the read operation has emptied the list of + * invalidated regions, and user_cookie_counter holds the value of the + * kernel's generation counter when the empty list occurred. The + * other fields are not filled in for this event. + */ + +enum { + UMMUNOT_EVENT_TYPE_INVAL = 0, + UMMUNOT_EVENT_TYPE_LAST = 1, +}; + +enum { + UMMUNOT_EVENT_FLAG_HINT = 1 << 0, +}; + +struct ummunot_event { + __u32 type; + __u32 flags; + __u64 hint_start; + __u64 hint_end; + __u64 user_cookie_counter; +}; + +#endif /* _LINUX_UMMUNOT_H */ -- 1.6.3.3 From davem at systemfabricworks.com Wed Jul 22 10:59:52 2009 From: davem at systemfabricworks.com (David McMillen) Date: Wed, 22 Jul 2009 12:59:52 -0500 Subject: [ofa-general] Running more than 894 processes doing rdma_listen Message-ID: <5e3be0f30907221059j4bd1d89occ97c277bb43f12d@mail.gmail.com> Is there an explicit limit on the number of ports that can be listening using rdma_cm? If so, can you point me to where I can go to adjust this? Here is how I found the limit: for (( pn=19000 ; pn < 19894 ; pn++ )) ; do ./ib_rdma_bw -c -p $pn & done That works just fine. However, do one more: ./ib_rdma_bw -c -p 19894 It prints out CMA: unable to open RDMA device It then doesn't gracefully handle that problem, ending in Program received signal SIGSEGV, Segmentation fault. [Switching to Thread 47695401269920 (LWP 30003)] __ibv_close_device (context=0x0) at src/device.c:154 154 int async_fd = context->async_fd; (gdb) where #0 __ibv_close_device (context=0x0) at src/device.c:154 #1 0x00000034e360184f in ucma_cleanup () at src/cma.c:165 #2 0x00000034e3601a13 in ucma_init () at src/cma.c:257 #3 0x00000034e3602080 in rdma_create_event_channel () at src/cma.c:299 #4 0x0000000000403077 in main (argc=4, argv=0x7fffb739fcc8) at rdma_bw.c:1057 Thanks, Dave -------------- next part -------------- An HTML attachment was scrubbed... URL: From sean.hefty at intel.com Wed Jul 22 11:05:49 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Wed, 22 Jul 2009 11:05:49 -0700 Subject: [ofa-general] RE: Running more than 894 processes doing rdma_listen In-Reply-To: <5e3be0f30907221059j4bd1d89occ97c277bb43f12d@mail.gmail.com> References: <5e3be0f30907221059j4bd1d89occ97c277bb43f12d@mail.gmail.com> Message-ID: <58CAA85864EF4DC8B46021B16E163A51@amr.corp.intel.com> >Is there an explicit limit on the number of ports that can be listening using >rdma_cm? There's no inherent limit built into the code. >It prints out CMA: unable to open RDMA device > >It then doesn't gracefully handle that problem, ending in > >Program received signal SIGSEGV, Segmentation fault. >[Switching to Thread 47695401269920 (LWP 30003)] >__ibv_close_device (context=0x0) at src/device.c:154 >154 int async_fd = context->async_fd; >(gdb) where >#0 __ibv_close_device (context=0x0) at src/device.c:154 >#1 0x00000034e360184f in ucma_cleanup () at src/cma.c:165 >#2 0x00000034e3601a13 in ucma_init () at src/cma.c:257 >#3 0x00000034e3602080 in rdma_create_event_channel () at src/cma.c:299 >#4 0x0000000000403077 in main (argc=4, argv=0x7fffb739fcc8) at rdma_bw.c:1057 Thanks - I see where the bug is for this. - Sean From hal.rosenstock at gmail.com Wed Jul 22 11:19:04 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 22 Jul 2009 14:19:04 -0400 Subject: [ofa-general] [PATCH] opensm/osm_sm.c: code consolidation In-Reply-To: <20090722105907.GP31735@me> References: <20090721153228.GJ31735@me> <20090722105907.GP31735@me> Message-ID: On Wed, Jul 22, 2009 at 6:59 AM, Sasha Khapyorsky wrote: > > Consolidate lock releasing code in multicast join and leave processors. FWIW the way it was released the lock sooner in the error cases at the expense of the extra duplicated code to release the lock. -- Hal From rdreier at cisco.com Wed Jul 22 11:25:40 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 22 Jul 2009 11:25:40 -0700 Subject: [ofa-general] Re: [PATCH mlx4] Remove empty stubs for detach/attach_mcast In-Reply-To: <20090720223724.GY12693@obsidianresearch.com> (Jason Gunthorpe's message of "Mon, 20 Jul 2009 16:37:24 -0600") References: <20090720223724.GY12693@obsidianresearch.com> Message-ID: thanks, applied both the mthca nad mlx4 versions. From rdreier at cisco.com Wed Jul 22 11:30:32 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 22 Jul 2009 11:30:32 -0700 Subject: [ofa-general] [PATCH] add pkgconfig support to ibverbs library In-Reply-To: <1248116839.2625.240.camel@localhost.localdomain> (Steven Dake's message of "Mon, 20 Jul 2009 12:07:19 -0700") References: <1248113849.2625.216.camel@localhost.localdomain> <20090720184351.GT12693@obsidianresearch.com> <1248116839.2625.240.camel@localhost.localdomain> Message-ID: Thanks. I've been meaning to look at pkgconfig support for libibverbs and I guess it might help a bit in some cases ... although really given the fact that there's nothing tricky about libibverbs (in the sense that you just need the include path and library path), are there any real-world cases where this is a win? Now, on the specific patch: > --- a/Makefile.am > +++ b/Makefile.am > @@ -66,6 +66,8 @@ EXTRA_DIST = include/infiniband/driver.h include/infiniband/kern-abi.h \ > src/ibverbs.h examples/pingpong.h \ > src/libibverbs.map libibverbs.spec.in $(man_MANS) > > +SUBDIRS = pkgconfig Is there a way to do this without introducing recursive make? I really like the fact that libibverbs currently builds non-recursively and I'd prefer to keep that property. > +%.pc: ibverbs.pc.in Makefile > + rm -f $@-t $@ > + sed \ > + -e 's#@''PREFIX@#$(exec_prefix)#g' \ > + -e 's#@''LIBDIR@#$(libdir)#g' \ > + -e 's#@''LIBVERSION@#$(VERSION)#g' \ > + -e 's#@''LIB@#'$(*:lib%=%)'#g' \ > + $< > $@-t > + chmod a-w $@-t > + mv $@-t $@ seems strange to be doing this substitution with sed at build time -- isn't making a final file from a .in version exactly what autoconf builds the configure script to do? Can this be done just by adding the .pc file to AC_CONFIG_FILES somehow? - R. From rdreier at cisco.com Wed Jul 22 11:55:53 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 22 Jul 2009 11:55:53 -0700 Subject: [ofa-general] [PATCH] Allow paths to the device specific library to be absolute In-Reply-To: <20090717210817.GF7320@obsidianresearch.com> (Jason Gunthorpe's message of "Fri, 17 Jul 2009 15:08:17 -0600") References: <20090714211653.GA13700@obsidianresearch.com> <20090717210817.GF7320@obsidianresearch.com> Message-ID: > * Missing const's in some function parameters > * Use of 'enum foo' as a type for a bit field. In C/C++ > or'ing enum members results in a type that is the enum > base integral type, not the enum itself. g++ produces > a warning by default. Very annoying :) > > Are you interested in patches for these too? Didn't reply to this before, sorry. But yes I am interested. As far as using enum for bit flags, what is the C++ idiom for that? - R. From hnrose at comcast.net Wed Jul 22 12:00:29 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Wed, 22 Jul 2009 15:00:29 -0400 Subject: [ofa-general] [PATCH] opensm/osm_subnet.c: Format lash_start_vl consistent with other uint8 items Message-ID: <20090722190029.GB565@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index f0dc184..3470b60 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -1444,7 +1444,7 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) fprintf(out, "# Starting VL for LASH algorithm\n" - "lash_start_vl %d\n\n", + "lash_start_vl %u\n\n", p_opts->lash_start_vl); fprintf(out, From hnrose at comcast.net Wed Jul 22 11:59:40 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Wed, 22 Jul 2009 14:59:40 -0400 Subject: [ofa-general] [PATCH] opensm/main.c: Display LASH start VL when specified Message-ID: <20090722185940.GA565@comcast.net> Also, cosmetic formatting change Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c index d682ff5..62242f4 100644 --- a/opensm/opensm/main.c +++ b/opensm/opensm/main.c @@ -910,7 +910,7 @@ int main(int argc, char *argv[]) printf(" I/O Node Guid File: %s\n", opt.io_guid_file); break; case 'H': - opt.max_reverse_hops = atoi(optarg); + opt.max_reverse_hops = atoi(optarg); printf(" Max Reverse Hops: %d\n", opt.max_reverse_hops); break; case 'm': @@ -964,6 +964,7 @@ int main(int argc, char *argv[]) return (-1); } opt.lash_start_vl = (uint8_t) temp; + printf(" LASH starting VL = %d\n", opt.lash_start_vl); break; case 'h': case '?': From hal.rosenstock at gmail.com Wed Jul 22 12:40:50 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 22 Jul 2009 15:40:50 -0400 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <20090721184446.GB12693@obsidianresearch.com> References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> Message-ID: On Tue, Jul 21, 2009 at 2:44 PM, Jason Gunthorpe wrote: > On Tue, Jul 21, 2009 at 02:03:12PM -0400, Hal Rosenstock wrote: > >> Currently, MADs are pipelined to a single switch at a time which >> effectively serializes these requests due to processing at the SMA. >> This patch pipelines (stripes) them across the switches first before >> proceeding with successive blocks. As a result of this striping, >> multiple switches can process the set and respond concurrently >> which results in an improvement to the subnet initialization time. > > Doing this without also using LID routing to the target switch is just > going to overload the SMAs in the intermediate switches with too many > DR SMPs. The "processing" time of LR (LID routing) v. DR forwarding (direct routed) v. set/get of a forwarding table block is implementation dependent. The dominant factor is the block set/get rather than whether it is DR or LR forwarded. The proposed algorithm reduces the potential VL15 overload on intermediate switches relative to the current algorithm for two reasons: the SMPs are spread across the switches first rather than blasting each switch in turn and there is a limit on the number of SMPs per node (not as good but less expensive than the concurrency limit you propose). > The most efficient approach is to program LFTs using a breadth first > search of the connectivity graph starting from the SM end port: A BFS ordered switch list is pretty straightforward. >  1) Within the BFS consider things as layers (number of hops from the >    SM). All switches in a layer can proceed largely in parallel, >    within the capability of the SM to capture the MAD replies at >    least. Advancing to the next layer must wait until the prior >    layer is fully programmed. The premise of this patch is to spread the blocks across the switches first rather than populate an individual switch entirely before proceeding with the next. This is due to the handling of the blocks being significantly more time consuming than any of the forwarding. I think that principle should apply to this algorithm as well. >  2) At each switch send at most two partial DR MADs - where the MAD is >    LID routed up to the parent switch in the BFS, and then direct routed >    one step to the target switch. The two MADs would program the LFT >    block for the SM LID and for the switch LID. Combined LR/DR routing is not a good idea IMO. Some switches don't support this although a requirement. Full DR routing could be used here rather than the combined DR routing although it would be less efficient in terms of forwarding compared with the combined DR (with LR direct to the previous level of switch). >  3) Sent LID routed mads directly to the target switch to fill in the >    rest of the LFT entries. These should be striped across the switches at a given "level". > Step 2 needs to respect the concurrency limit for the parent switch, > and #3 needs to respect the concurrency limit for the target switch. This is the harder part and also more expensive in terms of computation. This limit might also be overly conservative. Also, IMO there would be one configuration item for this limit rather than a per switch configuration. > As you go out the BFS there are more parent swtiches and target > switches available to run in parallel. > Eliminating DR hops will significantly improve MAD round trip time > and give you more possible parallelism before the SMAs in intermediate > switches become overloaded. I can see the MAD round trip time improvement based on the virtual reduction in number of forwarding hops but I don't see the increase in parallelism. SMPs are not flow controlled so I would think the parallelism is the same except that the transaction rate is somewhat higher using LR rather than DR SMPs. > Overall, optimizing things so that as few DR MADs are used as possible > is desirable. The NodeInfo write to set the switch LID can also be put ^^^^^^^^^^ PortInfo > into the above process and then all other switch programming MADs can > simply happen via LID routed packets. Sure; it could be done for many other SMPs but the most important thing in speeding up config time are the LFT/MFT block sets. -- Hal > Jason > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From jgunthorpe at obsidianresearch.com Wed Jul 22 13:48:46 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Wed, 22 Jul 2009 14:48:46 -0600 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> Message-ID: <20090722204846.GF12693@obsidianresearch.com> On Wed, Jul 22, 2009 at 03:40:50PM -0400, Hal Rosenstock wrote: > > Doing this without also using LID routing to the target switch is just > > going to overload the SMAs in the intermediate switches with too many > > DR SMPs. > > The "processing" time of LR (LID routing) v. DR forwarding (direct > routed) v. set/get of a forwarding table block is implementation > dependent. The dominant factor is the block set/get rather than > whether it is DR or LR forwarded. I would be very surprised if any implementation had a significant overhead for the actual set operation compared to the packet handling path. Certainly in our products the incremental cost of a set vs processing a DR is negligible. I expect similar results from any switch. As soon as a SMP goes into the CPU for DR or other processing there is an enormous hit. IIRC when I last looked, it is reasonable to expect a 1us per hop for DR vs a 100ns per hop for LID. If you are 5 switches deep that is 5us vs 500ns!! > The proposed algorithm reduces the potential VL15 overload on > intermediate switches relative to the current algorithm for two > reasons: the SMPs are spread across the switches first rather than > blasting each switch in turn and there is a limit on the number of > SMPs per node (not as good but less expensive than the concurrency > limit you propose). But you overload the switch the SM is connected to with processing N*limit DR SMPs rather than just 'limit' SMPs. That is what concerns me. > > ??1) Within the BFS consider things as layers (number of hops from the > > ?? ??SM). All switches in a layer can proceed largely in parallel, > > ?? ??within the capability of the SM to capture the MAD replies at > > ?? ??least. Advancing to the next layer must wait until the prior > > ?? ??layer is fully programmed. > > The premise of this patch is to spread the blocks across the switches > first rather than populate an individual switch entirely before > proceeding with the next. This is due to the handling of the blocks > being significantly more time consuming than any of the forwarding. I > think that principle should apply to this algorithm as well. As is what I propose, an entire level can be done in parallel within the two concurrency limits. The reason for the BFS is to enable LID routed packets which I view as the biggest gain. You have to progress outward building up paths back to the SM to do this. Randomly programming all switches does not enable LID routing. [Though do note the BFS here is a special kind that traverses the LID routing graph for the SM LID to guarentee the above. A LID route that is not shortest-path will not work with a straight topological BFS] > > ??2) At each switch send at most two partial DR MADs - where the MAD is > > ?? ??LID routed up to the parent switch in the BFS, and then direct routed > > ?? ??one step to the target switch. The two MADs would program the LFT > > ?? ??block for the SM LID and for the switch LID. > > Combined LR/DR routing is not a good idea IMO. Some switches don't support > this although a requirement. Full DR routing could be used here > rather I first implemented an algorithm like this for switches based on Gamla chips, and then for Anafa. If something doesn't support it, it is very uncommon. Thus, a simple global off switch for parallelism seems reasonable if you think there is a worry. Frankly, I'm not very concerned about such significant non-compliance. > than the combined DR routing although it would be less efficient in > terms of forwarding compared with the combined DR (with LR direct to > the previous level of switch). The problem with this is it makes managing the concurrency alot harder, and you hit a bottleneck sooner, since you now have to track every step along the DR path, not just the parent. > > ??3) Sent LID routed mads directly to the target switch to fill in the > > ?? ??rest of the LFT entries. > > These should be striped across the switches at a given "level". Yes, I imagine this algorithm running in parallel for all switches at a level. > > Step 2 needs to respect the concurrency limit for the parent switch, > > and #3 needs to respect the concurrency limit for the target switch. > > This is the harder part and also more expensive in terms of > computation. This limit might also be overly conservative. I don't see how it is more computational, you know the parent switch because you are computing a DR path to it. A simple per-switch counter is all that is needed. > Also, IMO there would be one configuration item for this limit rather > than a per switch configuration. Yes, two configurables would be excellent. Something like 20 for DR and 4 for Get/Set sounds reasonable to me. > > Eliminating DR hops will significantly improve MAD round trip time > > and give you more possible parallelism before the SMAs in intermediate > > switches become overloaded. > > I can see the MAD round trip time improvement based on the virtual > reduction in number of forwarding hops but I don't see the increase in > parallelism. SMPs are not flow controlled so I would think the > parallelism is the same except that the transaction rate is somewhat > higher using LR rather than DR SMPs. In real switch implementations today, the DR path has a much lower forwarding rate than the LID path. The LID path is wire speed, DR is something like 4-20 packets in a burst at best - due to the CPU involvement. If you ask 1000 switches to do a set via DR then the switch the SM is hooked up to will probably start dropping. If you do the same via LID, then no problems. This is not any sort of IBA requirement, just a reflection of the reality of all current implementations. So, LID routing offloads the SMP processing in the switches closest to the SM and lets you push more SMPs through them. You go from, say, a global 20 parallel DR SMP limit at the SM's switch to a 20*N_Switches_At_Level limit, which is much higher. > > into the above process and then all other switch programming MADs can > > simply happen via LID routed packets. > Sure; it could be done for many other SMPs but the most important > thing in speeding up config time are the LFT/MFT block sets. Once you get LID routing setup there is no reason to keep using DR after that point. For instance, just converting MFT programming to use LID only would probably result in noticable gains on big fabrics. Jason From swise at opengridcomputing.com Wed Jul 22 13:51:55 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Wed, 22 Jul 2009 15:51:55 -0500 Subject: [ofa-general] libcxgb3-1.2.3 released Message-ID: <4A677BEB.9060705@opengridcomputing.com> Hello, I'm announcing the release of version 1.2.3 of libcxgb3 which adds support for device ids 0x35 and 0x36. It can be pulled from: http://www.openfabrics.org/downloads/cxgb3/libcxgb3-1.2.3.tar.gz Vlad, please pull this into ofed-1.5. The git tree is: git://www.openfabrics.org/~swise/libcxgb3.git ofed_1_5 Thanks, Steve. From jgunthorpe at obsidianresearch.com Wed Jul 22 14:25:05 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Wed, 22 Jul 2009 15:25:05 -0600 Subject: [ofa-general] [PATCH] Allow paths to the device specific library to be absolute In-Reply-To: References: <20090714211653.GA13700@obsidianresearch.com> <20090717210817.GF7320@obsidianresearch.com> Message-ID: <20090722212505.GG12693@obsidianresearch.com> On Wed, Jul 22, 2009 at 11:55:53AM -0700, Roland Dreier wrote: > Didn't reply to this before, sorry. But yes I am interested. As far as > using enum for bit flags, what is the C++ idiom for that? Well, it isn't just C++, it applies to C too - but gcc isn't as sticky with the errors. Basically, something like this: enum bar {FOO=1, BAR=2}; Creates the distinct type 'enum bar'. The only things in the entire compilation that are of type 'enum bar' is FOO, BAR, and explicit casts to 'enum bar'. The expression (FOO|BAR) implicitly casts FOO and BAR to the enums integer compatible type, does the OR and the resulting type is the enums integer compatible type. So this: enum bar x = FOO|BAR; Is not 'right' at all. G++ gives an error, some C compilers could warn too, but gcc does not.. g++ says: t.cc:3: error: invalid conversion from 'int' to 'bar1' In this platform the integer compatible type for 'enum bar' is 'int', but sadly that can vary.. You can see how this can become a big pain in C++, you have to add enum casts to all sorts of strange places.. :( Basically, when using bit flags in C or C++ there is no advantage to using the enum's type as the flag parameter/member. Static analysis won't catch any errors, ie: enum bar {FOO=1, BAR=2}; enum bar1 {FOO1=1, BAR1=2}; enum bar1 x = FOO|BAR; Results in no warnings as gcc silently casts the integer compatible type back to the enum bar1. So, basically, the message is, don't use the flag enums type as a storage decl, only to manage the namespace. To fix this.. Well.. int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, enum ibv_qp_attr_mask attr_mask); becomes: int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, ENUM_TYPE attr_mask); What I've been mulling over is exactly what ENUM_TYPE should be to have guarenteed ABI compatibility on ppc 32/64, i386 and x86-64. 'int' is the correct answer on x86, x86-64, and ppc32. The basic test program is: # cat t.cc enum bar {FOO=1, BAR=2}; enum bar1 {FOO1=1, BAR1=2}; enum bar1 x = FOO|BAR; # g++ -c -Wall t.cc t.cc:3: error: invalid conversion from 'int' to 'bar1' I'd like to see a result for ia64 and ppc64.. Roland do you have a Debian machine logon? Could you check this on merulo.debian.org (or merkel)? Unfortunately I don't think pescetti is a ppc64 :| I'm not sure where to find a ppc64? (Shirley can you help?) The affected types look to me to be: ibv_device_cap_flags ibv_wc_flags ibv_access_flags ibv_rereg_mr_flags ibv_srq_attr_mask ibv_qp_attr_mask ibv_send_flags What do you think? Regards, Jason From twbowman at gmail.com Wed Jul 22 14:55:42 2009 From: twbowman at gmail.com (Todd Bowman) Date: Wed, 22 Jul 2009 15:55:42 -0600 Subject: [ofa-general] IPoIB connections falling off Message-ID: I need a little direction to help solve an IPoIB issue. Software: OFED 1.3 and 1.4 stacks, running OpenSM Problem: IPoIB connections fail, meaning a node cannot ping all or some of the other IPoIB nodes. IB itself is still up, we can run IB tests with success. So far the only resolution is to restart the IB stack. Size of the cluster seems to be irrelevant. It has happened on clusters from around 64 to 1000s. My first instinct is that some information has been lost from SM/SA which is needed to create an IPoIB connection, but I'm not for sure what that information is or how to verify that it is gone. Thanks in advance, Todd -------------- next part -------------- An HTML attachment was scrubbed... URL: From weiny2 at llnl.gov Wed Jul 22 14:59:30 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Wed, 22 Jul 2009 14:59:30 -0700 Subject: [ofa-general] IPoIB connections falling off In-Reply-To: References: Message-ID: <20090722145930.ca234b60.weiny2@llnl.gov> Check your multicast group membership and forwarding tables on the switches. We have had similar issues and have found that some nodes fail to join the multicast groups for various reasons. Ira On Wed, 22 Jul 2009 15:55:42 -0600 Todd Bowman wrote: > I need a little direction to help solve an IPoIB issue. > Software: OFED 1.3 and 1.4 stacks, running OpenSM > > > Problem: > IPoIB connections fail, meaning a node cannot ping all or some of the other > IPoIB nodes. IB itself is still up, we can run IB tests with success. So > far the only resolution is to restart the IB stack. Size of the cluster > seems to be irrelevant. It has happened on clusters from around 64 to > 1000s. > > > My first instinct is that some information has been lost from SM/SA which is > needed to create an IPoIB connection, but I'm not for sure what that > information is or how to verify that it is gone. > > Thanks in advance, > > Todd > -- Ira Weiny Math Programmer/Computer Scientist Lawrence Livermore National Lab weiny2 at llnl.gov From rdreier at cisco.com Wed Jul 22 15:05:48 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 22 Jul 2009 15:05:48 -0700 Subject: [ofa-general] [PATCH] Allow paths to the device specific library to be absolute In-Reply-To: <20090722212505.GG12693@obsidianresearch.com> (Jason Gunthorpe's message of "Wed, 22 Jul 2009 15:25:05 -0600") References: <20090714211653.GA13700@obsidianresearch.com> <20090717210817.GF7320@obsidianresearch.com> <20090722212505.GG12693@obsidianresearch.com> Message-ID: > # cat t.cc > enum bar {FOO=1, BAR=2}; > enum bar1 {FOO1=1, BAR1=2}; > enum bar1 x = FOO|BAR; > # g++ -c -Wall t.cc > t.cc:3: error: invalid conversion from 'int' to 'bar1' > > I'd like to see a result for ia64 and ppc64.. Roland do you have a > Debian machine logon? Could you check this on merulo.debian.org (or > merkel)? Unfortunately I don't think pescetti is a ppc64 :| I don't have a debian logon, but I do have an ia64 machine: $ uname -m ia64 $ g++ -c -Wall t.cc t.cc:3: error: invalid conversion from ‘int’ to ‘bar1’ and a cross-compiler shows: $ powerpc64-unknown-linux-gnu-g++ -m64 -Wall -c t.cc t.cc:3: error: invalid conversion from 'int' to 'bar1' so int covers the "major" platforms. Of course this is just with the specific g++ version I tried (4.3.3 on ia64, 4.1.1 on ppc64). Could change with g++ version too I assume. > To fix this.. Well.. > > int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, > enum ibv_qp_attr_mask attr_mask); > > becomes: > > int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, > ENUM_TYPE attr_mask); > What I've been mulling over is exactly what ENUM_TYPE should be to > have guarenteed ABI compatibility on ppc 32/64, i386 and x86-64. Yuck... the C standard says The identifiers in an enumerator list are declared as constants that have type int and may appear wherever such are permitted. so using int in the function declaration would be fine, except: Each enumerated type shall be compatible with an integer type. The choice of type is implementation-defined so we might be changing the function signature on some obscure platform without knowing it. I'm not sure if just ppc32/64, x86, ia64 is really good enough -- but on the other hand unbreaking C++ would be nice too. sigh... - R. From hal.rosenstock at gmail.com Wed Jul 22 15:05:56 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 22 Jul 2009 18:05:56 -0400 Subject: [ofa-general] IPoIB connections falling off In-Reply-To: <20090722145930.ca234b60.weiny2@llnl.gov> References: <20090722145930.ca234b60.weiny2@llnl.gov> Message-ID: On Wed, Jul 22, 2009 at 5:59 PM, Ira Weiny wrote: > Check your multicast group membership and forwarding tables on the switches. > > We have had similar issues and have found that some nodes fail to join the multicast groups for various reasons. Also, such failures should be in the opensm log and at least give hint of the issue (e.g. rate, MTU, etc.). -- Hal > > Ira > > On Wed, 22 Jul 2009 15:55:42 -0600 > Todd Bowman wrote: > >> I need a little direction to help solve an IPoIB issue. >> Software: OFED 1.3 and 1.4 stacks, running OpenSM >> >> >> Problem: >> IPoIB connections fail, meaning a node cannot ping all or some of the other >> IPoIB nodes.  IB itself is still up, we can run IB tests with success.  So >> far the only resolution is to restart the IB stack.  Size of the cluster >> seems to be irrelevant.  It has happened on clusters from around 64 to >> 1000s. >> >> >> My first instinct is that some information has been lost from SM/SA which is >> needed to create an IPoIB connection, but I'm not for sure what that >> information is or how to verify that it is gone. >> >> Thanks in advance, >> >> Todd >> > > > -- > Ira Weiny > Math Programmer/Computer Scientist > Lawrence Livermore National Lab > weiny2 at llnl.gov > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From jgunthorpe at obsidianresearch.com Wed Jul 22 15:45:06 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Wed, 22 Jul 2009 16:45:06 -0600 Subject: [ofa-general] [PATCH] Allow paths to the device specific library to be absolute In-Reply-To: References: <20090714211653.GA13700@obsidianresearch.com> <20090717210817.GF7320@obsidianresearch.com> <20090722212505.GG12693@obsidianresearch.com> Message-ID: <20090722224506.GI12693@obsidianresearch.com> On Wed, Jul 22, 2009 at 03:05:48PM -0700, Roland Dreier wrote: > > I'd like to see a result for ia64 and ppc64.. Roland do you have a > > Debian machine logon? Could you check this on merulo.debian.org (or > > merkel)? Unfortunately I don't think pescetti is a ppc64 :| > > I don't have a debian logon, but I do have an ia64 machine: Ah, in the good old days every DD had one of those :) > Of course this is just with the specific g++ version I tried (4.3.3 on > ia64, 4.1.1 on ppc64). Could change with g++ version too I assume. It is ABI defined how this works. Unfortunately the modern ABI is not written down anywhere, AFAIK :( The old i386 SYSV ABI which Linux i386 started with says 'enum' is a 4 byte signed word. (figure 3-1) > > What I've been mulling over is exactly what ENUM_TYPE should be to > > have guarenteed ABI compatibility on ppc 32/64, i386 and x86-64. > > Yuck... the C standard says > > The identifiers in an enumerator list are declared as constants that > have type int and may appear wherever such are permitted. > > so using int in the function declaration would be fine, except: > > Each enumerated type shall be compatible with an integer type. The > choice of type is implementation-defined C99 is a little more descript but not more helpful: The identifiers in an enumerator list are declared as constants that have type int and may appear wherever such are permitted. Each enumerated type shall be compatible with an integer type. The choice of type is implementation-defined, but shall be capable of representing the values of all the members of the enumeration. Basically, it means, the compiler gets to pick. In real life, depending on the constants assigned to the enum members you get different types: t.cc:3: error: invalid conversion from 'int' to 'bar1' t.cc:3: error: invalid conversion from 'unsigned int' to 'bar1' t.cc:3: error: invalid conversion from 'long long int' to 'bar1' t.cc:3: error: invalid conversion from 'long long unsigned int' to 'bar1' > so we might be changing the function signature on some obscure platform > without knowing it. I'm not sure if just ppc32/64, x86, ia64 is really > good enough -- but on the other hand unbreaking C++ would be nice too. Yes, thats right. To be honest, obscure platforms deal with this sort of stuff often enough to be used to it. The user base of the next 3 major platforms [arm, mips, sparc] is very embedded focused and can deal with an ABI breakage pretty easily. That said, at this point finding a platform that didn't use int for these types would be very surprising. [[BTW, as I was looking this up, I found this in C99 under common warnings: A value is given to an object of an enumeration type other than by assignment of an enumeration constant that is a member of that type, or an enumeration variable that has the same type, or the value of a function that returns the same enumeration type (6.7.2.2). donno why gcc doesn't have a warning..]] Jason From hal.rosenstock at gmail.com Wed Jul 22 17:28:25 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 22 Jul 2009 20:28:25 -0400 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <20090722204846.GF12693@obsidianresearch.com> References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> <20090722204846.GF12693@obsidianresearch.com> Message-ID: On Wed, Jul 22, 2009 at 4:48 PM, Jason Gunthorpe wrote: > On Wed, Jul 22, 2009 at 03:40:50PM -0400, Hal Rosenstock wrote: > >> > Doing this without also using LID routing to the target switch is just >> > going to overload the SMAs in the intermediate switches with too many >> > DR SMPs. >> >> The "processing" time of LR (LID routing) v. DR forwarding (direct >> routed) v. set/get of a forwarding table block is implementation >> dependent. The dominant factor is the block set/get rather than >> whether it is DR or LR forwarded. > > I would be very surprised if any implementation had a significant > overhead for the actual set operation compared to the packet handling > path. Certainly in our products the incremental cost of a set vs > processing a DR is negligible. I expect similar results from any > switch. As soon as a SMP goes into the CPU for DR or other processing > there is an enormous hit. > > IIRC when I last looked, it is reasonable to expect a 1us per hop for > DR vs a 100ns per hop for LID. If you are 5 switches deep that is 5us > vs 500ns!! > >> The proposed algorithm reduces the potential VL15 overload on >> intermediate switches relative to the current algorithm for two >> reasons: the SMPs are spread across the switches first rather than >> blasting each switch in turn and there is a limit on the number of >> SMPs per node (not as good but less expensive than the concurrency >> limit you propose). > > But you overload the switch the SM is connected to with processing > N*limit DR SMPs rather than just 'limit' SMPs. That is what concerns > me. As I said, the current algorithm is worse as it sends N*no limit DR SMPs (where no limit means any needed blocks). Not sure that VL15 droppage due to this has been identified. So I think this improves on what's been deployed and seemingly works in OpenSM for quite some time now. >> > ??1) Within the BFS consider things as layers (number of hops from the >> > ?? ??SM). All switches in a layer can proceed largely in parallel, >> > ?? ??within the capability of the SM to capture the MAD replies at >> > ?? ??least. Advancing to the next layer must wait until the prior >> > ?? ??layer is fully programmed. >> >> The premise of this patch is to spread the blocks across the switches >> first rather than populate an individual switch entirely before >> proceeding with the next. This is due to the handling of the blocks >> being significantly more time consuming than any of the forwarding. I >> think that principle should apply to this algorithm as well. > > As is what I propose, an entire level can be done in parallel within > the two concurrency limits. > > The reason for the BFS is to enable LID routed packets which I view as > the biggest gain. You have to progress outward building up paths back > to the SM to do this. > > Randomly programming all switches does not enable LID routing. > > [Though do note the BFS here is a special kind that traverses the >  LID routing graph for the SM LID to guarentee the above. A LID route >  that is not shortest-path will not work with a straight topological >  BFS] > >> > ??2) At each switch send at most two partial DR MADs - where the MAD is >> > ?? ??LID routed up to the parent switch in the BFS, and then direct routed >> > ?? ??one step to the target switch. The two MADs would program the LFT >> > ?? ??block for the SM LID and for the switch LID. >> >> Combined LR/DR routing is not a good idea IMO. Some switches don't support >> this although a requirement. Full DR routing could be used here >> rather > > I first implemented an algorithm like this for switches based on Gamla > chips, and then for Anafa. If something doesn't support it, it is > very uncommon. I'm aware of at least two very different switches where this is the case. > Thus, a simple global off switch for parallelism seems reasonable if > you think there is a worry. Frankly, I'm not very concerned about such > significant non-compliance. There are no compliance tests for this so it's a relatively untested feature (other than ibportstate which uses this). Although you are not concerned, I don't want to orphan these deployed switches and supporting multiple modes complicates matters some. >> than the combined DR routing although it would be less efficient in >> terms of forwarding compared with the combined DR (with LR direct to >> the previous level of switch). > > The problem with this is it makes managing the concurrency alot > harder, and you hit a bottleneck sooner, since you now have to track > every step along the DR path, not just the parent. Understood; that's what I meant when I wrote below that it's harder and more expensive computationally. I think that it's also overly pessimistic so the number might want to be made artificially higher based on experience that these SMPs can be pipelined quite a bit more than this would allow. In short, the current algorithm is most optimisitic/agressive, the proposed one is less so, and yours is most conservative/robust. >> > ??3) Sent LID routed mads directly to the target switch to fill in the >> > ?? ??rest of the LFT entries. >> >> These should be striped across the switches at a given "level". > > Yes, I imagine this algorithm running in parallel for all switches at > a level. > >> > Step 2 needs to respect the concurrency limit for the parent switch, >> > and #3 needs to respect the concurrency limit for the target switch. >> >> This is the harder part and also more expensive in terms of >> computation. This limit might also be overly conservative. > > I don't see how it is more computational, you know the parent switch > because you are computing a DR path to it. A simple per-switch counter > is all that is needed. I was referring to the case where all hops are tracked not just the parent. >> Also, IMO there would be one configuration item for this limit rather >> than a per switch configuration. > > Yes, two configurables would be excellent. Something like 20 for DR > and 4 for Get/Set sounds reasonable to me. > >> > Eliminating DR hops will significantly improve MAD round trip time >> > and give you more possible parallelism before the SMAs in intermediate >> > switches become overloaded. >> >> I can see the MAD round trip time improvement based on the virtual >> reduction in number of forwarding hops but I don't see the increase in >> parallelism. SMPs are not flow controlled so I would think the >> parallelism is the same except that the transaction rate is somewhat >> higher using LR rather than DR SMPs. > > In real switch implementations today, the DR path has a much lower > forwarding rate than the LID path. The LID path is wire speed, DR is > something like 4-20 packets in a burst at best - due to the CPU > involvement. If you ask 1000 switches to do a set via DR then the > switch the SM is hooked up to will probably start dropping. If you do > the same via LID, then no problems. > > This is not any sort of IBA requirement, just a reflection of the > reality of all current implementations. > > So, LID routing offloads the SMP processing in the switches closest to > the SM and lets you push more SMPs through them. You go from, say, a > global 20 parallel DR SMP limit at the SM's switch to a > 20*N_Switches_At_Level limit, which is much higher. > >> > into the above process and then all other switch programming MADs can >> > simply happen via LID routed packets. >> Sure; it could be done for many other SMPs but the most important >> thing in speeding up config time are the LFT/MFT block sets. > > Once you get LID routing setup there is no reason to keep using DR > after that point. For instance, just converting MFT programming to use > LID only would probably result in noticable gains on big fabrics. I was talking more about the LFT gain (which still is dependent on the size of the network (end ports)). MFT gains are less and are based on the number of multicast groups in use and whether or not MLID overloading is being used. -- Hal > Jason > From jgunthorpe at obsidianresearch.com Wed Jul 22 17:51:59 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Wed, 22 Jul 2009 18:51:59 -0600 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> <20090722204846.GF12693@obsidianresearch.com> Message-ID: <20090723005159.GA1230@obsidianresearch.com> On Wed, Jul 22, 2009 at 08:28:25PM -0400, Hal Rosenstock wrote: > > But you overload the switch the SM is connected to with processing > > N*limit DR SMPs rather than just 'limit' SMPs. That is what concerns > > me. > > As I said, the current algorithm is worse as it sends N*no limit DR > SMPs (where no limit means any needed blocks). Not sure that VL15 > droppage due to this has been identified. So I think this improves on > what's been deployed and seemingly works in OpenSM for quite some time > now. Hmm, OK I didn't realize that. I've heard of reports of VL15 droppage in real networks, maybe this is why.. > > I first implemented an algorithm like this for switches based on Gamla > > chips, and then for Anafa. If something doesn't support it, it is > > very uncommon. > > I'm aware of at least two very different switches where this is the case. Well, that's horrible - but again, I personally have a hard time caring if using LID routing gives even a 5% reduction in setup time with compliant devices. I suppose if you really cared it would be asy to black list certain devices. > Understood; that's what I meant when I wrote below that it's harder > and more expensive computationally. I think that it's also overly > pessimistic so the number might want to be made artificially higher > based on experience that these SMPs can be pipelined quite a bit more > than this would allow. That is highly device dependent - some devices have more CPU SMP buffering than others and this affects things greatly. Jason From davem at systemfabricworks.com Wed Jul 22 16:12:23 2009 From: davem at systemfabricworks.com (davem at systemfabricworks.com) Date: Wed, 22 Jul 2009 18:12:23 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries Message-ID: <4A679CD7.mailLHZ11MTGE@systemfabricworks.com> The timeouts in rdma_resolve_addr and rdma_resolve_route actually happen in some fabrics, and there is no retry in rdma_cm so this patch adds process level retries. A combined total of 10 retries for the pair is allowed. Signed-off-by: David A. McMillen --- rdma_bw.c | 16 ++++++++++++++++ rdma_lat.c | 15 +++++++++++++++ 2 files changed, 31 insertions(+), 0 deletions(-) diff --git a/rdma_bw.c b/rdma_bw.c index 2628ac4..737558a 100755 --- a/rdma_bw.c +++ b/rdma_bw.c @@ -131,6 +131,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) char *service; int n; int sockfd = -1; + int n_retries = 10; struct rdma_cm_event *event; struct sockaddr_in sin; struct pingpong_context *ctx = NULL; @@ -152,6 +153,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) sin.sin_addr.s_addr = ((struct sockaddr_in*)res->ai_addr)->sin_addr.s_addr; sin.sin_family = AF_INET; sin.sin_port = htons(data->port); +retry_addr: if (rdma_resolve_addr(data->cm_id, NULL, (struct sockaddr *)&sin, 2000)) { fprintf(stderr, "%d:%s: rdma_resolve_addr failed\n", @@ -162,6 +164,13 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) if (rdma_get_cm_event(data->cm_channel, &event)) goto err2; + + if (event->event == RDMA_CM_EVENT_ADDR_ERROR + && n_retries-- > 0) { + rdma_ack_cm_event(event); + goto retry_addr; + } + if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { fprintf(stderr, "%d:%s: unexpected CM event %d\n", pid, __func__, event->event); @@ -169,6 +178,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) } rdma_ack_cm_event(event); +retry_route: if (rdma_resolve_route(data->cm_id, 2000)) { fprintf(stderr, "%d:%s: rdma_resolve_route failed\n", pid, __func__); @@ -178,6 +188,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) if (rdma_get_cm_event(data->cm_channel, &event)) goto err2; + if (event->event == RDMA_CM_EVENT_ROUTE_ERROR + && n_retries-- > 0) { + rdma_ack_cm_event(event); + goto retry_route; + } + if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { fprintf(stderr, "%d:%s: unexpected CM event %d\n", pid, __func__, event->event); diff --git a/rdma_lat.c b/rdma_lat.c index 3681b35..1f65086 100755 --- a/rdma_lat.c +++ b/rdma_lat.c @@ -207,6 +207,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) char *service; int n; int sockfd = -1; + int n_retries = 10; struct rdma_cm_event *event; struct sockaddr_in sin; struct pingpong_context *ctx = NULL; @@ -228,6 +229,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) sin.sin_addr.s_addr = ((struct sockaddr_in*)res->ai_addr)->sin_addr.s_addr; sin.sin_family = AF_INET; sin.sin_port = htons(data->port); +retry_addr: if (rdma_resolve_addr(data->cm_id, NULL, (struct sockaddr *)&sin, 2000)) { fprintf(stderr, "%d:%s: rdma_resolve_addr failed\n", @@ -238,6 +240,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) if (rdma_get_cm_event(data->cm_channel, &event)) goto err2; + if (event->event == RDMA_CM_EVENT_ADDR_ERROR + && n_retries-- > 0) { + rdma_ack_cm_event(event); + goto retry_addr; + } + if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { fprintf(stderr, "%d:%s: unexpected CM event %d\n", pid, __func__, event->event); @@ -245,6 +253,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) } rdma_ack_cm_event(event); +retry_route: if (rdma_resolve_route(data->cm_id, 2000)) { fprintf(stderr, "%d:%s: rdma_resolve_route failed\n", pid, __func__); @@ -254,6 +263,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) if (rdma_get_cm_event(data->cm_channel, &event)) goto err2; + if (event->event == RDMA_CM_EVENT_ROUTE_ERROR + && n_retries-- > 0) { + rdma_ack_cm_event(event); + goto retry_route; + } + if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { fprintf(stderr, "%d:%s: unexpected CM event %d\n", pid, __func__, event->event); From sashak at voltaire.com Wed Jul 22 21:26:50 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Thu, 23 Jul 2009 07:26:50 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_subnet.c: Format lash_start_vl consistent with other uint8 items In-Reply-To: <20090722190029.GB565@comcast.net> References: <20090722190029.GB565@comcast.net> Message-ID: <20090723042650.GA10899@me> On 15:00 Wed 22 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Wed Jul 22 21:27:08 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Thu, 23 Jul 2009 07:27:08 +0300 Subject: [ofa-general] Re: [PATCH] opensm/main.c: Display LASH start VL when specified In-Reply-To: <20090722185940.GA565@comcast.net> References: <20090722185940.GA565@comcast.net> Message-ID: <20090723042708.GB10899@me> On 14:59 Wed 22 Jul , Hal Rosenstock wrote: > > Also, cosmetic formatting change > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From bart.vanassche at gmail.com Wed Jul 22 23:35:59 2009 From: bart.vanassche at gmail.com (Bart Van Assche) Date: Thu, 23 Jul 2009 08:35:59 +0200 Subject: [ofa-general] Re: 2.6.30.1: possible irq lock inversion dependency detected In-Reply-To: References: Message-ID: On Tue, Jul 14, 2009 at 8:36 PM, Roland Dreier wrote: > >  > With the two patches applied on top of 2.6.31.1 lockdep stopped >  > complaining on my setup. By the way, do you know whether this is a >  > long-standing issue or a result of the changes between 2.6.29 and >  > 2.6.30 ? > > Odd.  The problem I fixed with my second patch was long-standing I > think; the change from .29 to .30 was the addition of lockdep debugging > for timers, so I could see how the first report could pop up only under > .30 (since that involved a timer). > > Oh well, I'll look at cleaning up the second patch and pushing it upstream. Hello Roland, Any idea when you will be able to push this patch upstream ? Or has it already been pushed upstream, and did I miss it ? Thanks, Bart. From vlad at dev.mellanox.co.il Wed Jul 22 23:40:42 2009 From: vlad at dev.mellanox.co.il (Vladimir Sokolovsky) Date: Thu, 23 Jul 2009 09:40:42 +0300 Subject: [ofa-general] Re: libcxgb3-1.2.3 released In-Reply-To: <4A677BEB.9060705@opengridcomputing.com> References: <4A677BEB.9060705@opengridcomputing.com> Message-ID: <4A6805EA.7050300@dev.mellanox.co.il> Steve Wise wrote: > Hello, > > I'm announcing the release of version 1.2.3 of libcxgb3 which adds > support for device ids 0x35 and 0x36. > > It can be pulled from: > > http://www.openfabrics.org/downloads/cxgb3/libcxgb3-1.2.3.tar.gz > > Vlad, please pull this into ofed-1.5. The git tree is: > > git://www.openfabrics.org/~swise/libcxgb3.git ofed_1_5 > > > Thanks, > > Steve. > Done, Regards, Vladimir From jackm at dev.mellanox.co.il Thu Jul 23 01:20:59 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Thu, 23 Jul 2009 11:20:59 +0300 Subject: [ofa-general] Re: [ewg] [Patch mthca backport] Don't use kmalloc > 128k In-Reply-To: <356B6978-3308-4EE9-8C00-00199558BDEA@redhat.com> References: <356B6978-3308-4EE9-8C00-00199558BDEA@redhat.com> Message-ID: <200907231121.00140.jackm@dev.mellanox.co.il> On Thursday 16 July 2009 21:08, Doug Ledford wrote: > On rhel4 and rhel5 machines, the kmalloc implementation does not > automatically forward kmalloc requests > 128kb to __get_free_pages. > Please include this patch in all rhel4 and rhel5 backport directories > so that we do the right thing in the mthca driver on rhel in regards > to kmalloc requests larger than 128k (at least in this code path, > there may be others lurking too, I'll forward additional patches if I > find they are needed). > > commit a7f18a776785aecb5eb9967aef6f0f603b698ba0 Author: Doug Ledford Date: Thu Jul 16 12:47:55 2009 -0400 [mthca] Fix attempts to use kmalloc on overly large allocations Signed-off-by: Doug Ledford ---- Roland, I think that this patch should be taken into the mainstream kernel, rather than just as a backport patch for RHEL. (We can have a similar patch for mlx4). I notice that __get_free_pages(), free_pages(), and get_order() are all in the mainstream kernel. This will fix the 2^20 bits limit on our bitmaps once and for all. If you agree, I will post this patch and one for mlx4 on the general list. Doug posted this patch on the EWG list. Thanks Doug! diff --git a/drivers/infiniband/hw/mthca/mthca_mr.c b/drivers/infiniband/hw/mthca/mthca_mr.c index d606edf..312e18d 100644 --- a/drivers/infiniband/hw/mthca/mthca_mr.c +++ b/drivers/infiniband/hw/mthca/mthca_mr.c @@ -152,8 +152,11 @@ static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order) goto err_out; for (i = 0; i <= buddy->max_order; ++i) { - s = BITS_TO_LONGS(1 << (buddy->max_order - i)); - buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL); + s = BITS_TO_LONGS(1 << (buddy->max_order - i)) * sizeof(long); + if(s > PAGE_SIZE) + buddy->bits[i] = (unsigned long *)__get_free_pages(GFP_KERNEL, get_order(s)); + else + buddy->bits[i] = kmalloc(s, GFP_KERNEL); if (!buddy->bits[i]) goto err_out_free; bitmap_zero(buddy->bits[i], @@ -166,9 +169,13 @@ static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order) return 0; err_out_free: - for (i = 0; i <= buddy->max_order; ++i) - kfree(buddy->bits[i]); - + for (i = 0; i <= buddy->max_order; ++i){ + s = BITS_TO_LONGS(1 << (buddy->max_order - i)) * sizeof(long); + if(s > PAGE_SIZE) + free_pages((unsigned long)buddy->bits[i], get_order(s)); + else + kfree(buddy->bits[i]); + } err_out: kfree(buddy->bits); kfree(buddy->num_free); @@ -178,10 +185,15 @@ err_out: static void mthca_buddy_cleanup(struct mthca_buddy *buddy) { - int i; + int i, s; - for (i = 0; i <= buddy->max_order; ++i) - kfree(buddy->bits[i]); + for (i = 0; i <= buddy->max_order; ++i){ + s = BITS_TO_LONGS(1 << (buddy->max_order - i)) * sizeof(long); + if(s > PAGE_SIZE) + free_pages((unsigned long)buddy->bits[i], get_order(s)); + else + kfree(buddy->bits[i]); + } kfree(buddy->bits); kfree(buddy->num_free); From dotanba at gmail.com Thu Jul 23 01:30:10 2009 From: dotanba at gmail.com (Dotan Barak) Date: Thu, 23 Jul 2009 11:30:10 +0300 Subject: [ofa-general] RE: Running more than 894 processes doing rdma_listen In-Reply-To: <58CAA85864EF4DC8B46021B16E163A51@amr.corp.intel.com> References: <5e3be0f30907221059j4bd1d89occ97c277bb43f12d@mail.gmail.com> <58CAA85864EF4DC8B46021B16E163A51@amr.corp.intel.com> Message-ID: <2f3bf9a60907230130o79a151ceq8b9ed0477dbd268@mail.gmail.com> On Wed, Jul 22, 2009 at 9:05 PM, Sean Hefty wrote: >>Is there an explicit limit on the number of ports that can be listening using >>rdma_cm? > > There's no inherent limit built into the code. > Maybe this limitation is being caused by the ulimit ("ulimit -n" produced 1024 for my system). Try changing this value. Dotan From xiaoshuangx at gmail.com Thu Jul 23 02:27:03 2009 From: xiaoshuangx at gmail.com (xiaoshuang xia) Date: Thu, 23 Jul 2009 17:27:03 +0800 Subject: Fwd: [ofa-general] multiple path in opensm??? In-Reply-To: References: <20090722102821.GN31735@me> Message-ID: ---------- Forwarded message ---------- From: xiaoshuang xia Date: Thu, Jul 23, 2009 at 5:26 PM Subject: Re: [ofa-general] multiple path in opensm??? To: Sasha Khapyorsky Hi, Sasha I have read osm_sa_path_record.c carefully, but I think this file only return a list of appropriate path record for the user. I want to know either opensm do the choosing job of this list or another package do this job. Xiaoshuang On Wed, Jul 22, 2009 at 6:28 PM, Sasha Khapyorsky wrote: > On 10:23 Wed 22 Jul , xiaoshuang xia wrote: > > > > I recently want to implement a routing scheme using opensm, but I face > some > > problem. Help me pls!! THX!! > > > > The problem is > > > > 1. In the opensm, if the multi-path between src and dst is denoted by the > > multiple dst-lid. > > > > 2. If there are multi-path between the src and dst , what mechanism does > > client choose the one of them. If this mechanism is implemented by > opensm, > > which file do this job? If choose one prefered path doesn 't implement by > > opensm, what unit(call it "the choosing unit") finish this job, what data > > structure does opensm hold to exchange the multi-path information to "the > > choosing unit" and what file does this exchanging job in opensm. > > It is all about SA PathRecord querying, it is processed in > osm_sa_path_record.c. > > Sasha > -------------- next part -------------- An HTML attachment was scrubbed... URL: From dorfman.eli at gmail.com Thu Jul 23 02:36:51 2009 From: dorfman.eli at gmail.com (Eli Dorfman (Voltaire)) Date: Thu, 23 Jul 2009 12:36:51 +0300 Subject: [ofa-general] How to choose a path through many of them? In-Reply-To: <91fe68d50907191809o58f4fd3did3580af7d67b7232@mail.gmail.com> References: <91fe68d50907191809o58f4fd3did3580af7d67b7232@mail.gmail.com> Message-ID: <4A682F33.6020209@gmail.com> Jordan wrote: > When LMC > 0 , there are 2^LMC LIDs assigned to a port. When choosing > different LID, there maybe different paths to get to this port.(all > these paths are valid) > In other words, different LIDs represent different paths. But which path > should I choose to get to the destination? It seems that OpenSM > does not provide such algorithm . Since all the paths are valid you can choose any of them. Use MultiPathRecord attribute to get all available paths from src to dst. Eli > > Does OFED provide such algorithm to manage which path should to choose ? > > > ------------------------------------------------------------------------ > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From vlad at lists.openfabrics.org Thu Jul 23 02:55:06 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Thu, 23 Jul 2009 02:55:06 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090723-0200 daily build status Message-ID: <20090723095506.36117E30151@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Passed on i686 with linux-2.6.18 Passed on i686 with linux-2.6.19 Passed on i686 with linux-2.6.21.1 Passed on i686 with linux-2.6.22 Passed on x86_64 with linux-2.6.18 Passed on x86_64 with linux-2.6.19 Passed on x86_64 with linux-2.6.21.1 Passed on x86_64 with linux-2.6.20 Passed on x86_64 with linux-2.6.22 Passed on ia64 with linux-2.6.19 Passed on ia64 with linux-2.6.18 Passed on ia64 with linux-2.6.21.1 Passed on ia64 with linux-2.6.22 Passed on ppc64 with linux-2.6.18 Passed on ppc64 with linux-2.6.19 Failed: Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.23_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.23_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.23_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From hal.rosenstock at gmail.com Thu Jul 23 04:31:09 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Thu, 23 Jul 2009 07:31:09 -0400 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <20090723005159.GA1230@obsidianresearch.com> References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> <20090722204846.GF12693@obsidianresearch.com> <20090723005159.GA1230@obsidianresearch.com> Message-ID: On Wed, Jul 22, 2009 at 8:51 PM, Jason Gunthorpe wrote: > On Wed, Jul 22, 2009 at 08:28:25PM -0400, Hal Rosenstock wrote: > >> > But you overload the switch the SM is connected to with processing >> > N*limit DR SMPs rather than just 'limit' SMPs. That is what concerns >> > me. >> >> As I said, the current algorithm is worse as it sends N*no limit DR >> SMPs (where no limit means any needed blocks). Not sure that VL15 >> droppage due to this has been identified. So I think this improves on >> what's been deployed and seemingly works in OpenSM for quite some time >> now. > > Hmm, OK I didn't realize that. > > I've heard of reports of VL15 droppage in real networks, maybe this is why.. > >> > I first implemented an algorithm like this for switches based on Gamla >> > chips, and then for Anafa. If something doesn't support it, it is >> > very uncommon. >> >> I'm aware of at least two very different switches where this is the case. > > Well, that's horrible - but again, I personally have a hard time > caring if using LID routing gives even a 5% reduction in setup time > with compliant devices. > > I suppose if you really cared it would be asy to black list certain > devices. > >> Understood; that's what I meant when I wrote below that it's harder >> and more expensive computationally. I think that it's also overly >> pessimistic so the number might want to be made artificially higher >> based on experience that these SMPs can be pipelined quite a bit more >> than this would allow. > > That is highly device dependent - some devices have more CPU SMP > buffering than others and this affects things greatly. The reason I said that this was overly pessismistic is that the count at the parent switch deals on a transaction basis rather than in flight as it passes that switch. So to compensate the default(s) would need careful selection (and be made artificially higher to gain back performance lost by cranking this down too tightly. -- Hal > > Jason > From dledford at redhat.com Thu Jul 23 04:53:49 2009 From: dledford at redhat.com (Doug Ledford) Date: Thu, 23 Jul 2009 07:53:49 -0400 Subject: [ofa-general] Re: [ewg] [Patch mthca backport] Don't use kmalloc > 128k In-Reply-To: <200907231121.00140.jackm@dev.mellanox.co.il> References: <356B6978-3308-4EE9-8C00-00199558BDEA@redhat.com> <200907231121.00140.jackm@dev.mellanox.co.il> Message-ID: On Jul 23, 2009, at 4:20 AM, Jack Morgenstein wrote: > On Thursday 16 July 2009 21:08, Doug Ledford wrote: >> On rhel4 and rhel5 machines, the kmalloc implementation does not >> automatically forward kmalloc requests > 128kb to __get_free_pages. >> Please include this patch in all rhel4 and rhel5 backport directories >> so that we do the right thing in the mthca driver on rhel in regards >> to kmalloc requests larger than 128k (at least in this code path, >> there may be others lurking too, I'll forward additional patches if I >> find they are needed). >> >> > commit a7f18a776785aecb5eb9967aef6f0f603b698ba0 > Author: Doug Ledford > Date: Thu Jul 16 12:47:55 2009 -0400 > > [mthca] Fix attempts to use kmalloc on overly large allocations > > Signed-off-by: Doug Ledford This needs a correct signed-off-by: line. Mine got added when I put it in my local git tree, but the original patch came from Red Hat's bugzilla, bug #508902, author David Jeffery > ---- > > Roland, > I think that this patch should be taken into the mainstream kernel, > rather > than just as a backport patch for RHEL. (We can have a similar > patch for mlx4). > I notice that __get_free_pages(), free_pages(), and get_order() are > all in the > mainstream kernel. > > This will fix the 2^20 bits limit on our bitmaps once and for all. > If you agree, I will post this patch and one for mlx4 on the general > list. > > Doug posted this patch on the EWG list. > > Thanks Doug! > > diff --git a/drivers/infiniband/hw/mthca/mthca_mr.c b/drivers/ > infiniband/hw/mthca/mthca_mr.c > index d606edf..312e18d 100644 > --- a/drivers/infiniband/hw/mthca/mthca_mr.c > +++ b/drivers/infiniband/hw/mthca/mthca_mr.c > @@ -152,8 +152,11 @@ static int mthca_buddy_init(struct mthca_buddy > *buddy, int max_order) > goto err_out; > > for (i = 0; i <= buddy->max_order; ++i) { > - s = BITS_TO_LONGS(1 << (buddy->max_order - i)); > - buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL); > + s = BITS_TO_LONGS(1 << (buddy->max_order - i)) * sizeof(long); > + if(s > PAGE_SIZE) > + buddy->bits[i] = (unsigned long *)__get_free_pages(GFP_KERNEL, > get_order(s)); > + else > + buddy->bits[i] = kmalloc(s, GFP_KERNEL); > if (!buddy->bits[i]) > goto err_out_free; > bitmap_zero(buddy->bits[i], > @@ -166,9 +169,13 @@ static int mthca_buddy_init(struct mthca_buddy > *buddy, int max_order) > return 0; > > err_out_free: > - for (i = 0; i <= buddy->max_order; ++i) > - kfree(buddy->bits[i]); > - > + for (i = 0; i <= buddy->max_order; ++i){ > + s = BITS_TO_LONGS(1 << (buddy->max_order - i)) * sizeof(long); > + if(s > PAGE_SIZE) > + free_pages((unsigned long)buddy->bits[i], get_order(s)); > + else > + kfree(buddy->bits[i]); > + } > err_out: > kfree(buddy->bits); > kfree(buddy->num_free); > @@ -178,10 +185,15 @@ err_out: > > static void mthca_buddy_cleanup(struct mthca_buddy *buddy) > { > - int i; > + int i, s; > > - for (i = 0; i <= buddy->max_order; ++i) > - kfree(buddy->bits[i]); > + for (i = 0; i <= buddy->max_order; ++i){ > + s = BITS_TO_LONGS(1 << (buddy->max_order - i)) * sizeof(long); > + if(s > PAGE_SIZE) > + free_pages((unsigned long)buddy->bits[i], get_order(s)); > + else > + kfree(buddy->bits[i]); > + } > > kfree(buddy->bits); > kfree(buddy->num_free); -- Doug Ledford GPG KeyID: CFBFF194 http://people.redhat.com/dledford InfiniBand Specific RPMS http://people.redhat.com/dledford/Infiniband -------------- next part -------------- A non-text attachment was scrubbed... Name: PGP.sig Type: application/pgp-signature Size: 203 bytes Desc: This is a digitally signed message part URL: From kliteyn at dev.mellanox.co.il Thu Jul 23 05:53:07 2009 From: kliteyn at dev.mellanox.co.il (Yevgeny Kliteynik) Date: Thu, 23 Jul 2009 15:53:07 +0300 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <20090722204846.GF12693@obsidianresearch.com> References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> <20090722204846.GF12693@obsidianresearch.com> Message-ID: <4A685D33.2050803@dev.mellanox.co.il> Jason, Jason Gunthorpe wrote: > On Wed, Jul 22, 2009 at 03:40:50PM -0400, Hal Rosenstock wrote: > >>> Doing this without also using LID routing to the target switch is just >>> going to overload the SMAs in the intermediate switches with too many >>> DR SMPs. >> The "processing" time of LR (LID routing) v. DR forwarding (direct >> routed) v. set/get of a forwarding table block is implementation >> dependent. The dominant factor is the block set/get rather than >> whether it is DR or LR forwarded. > > I would be very surprised if any implementation had a significant > overhead for the actual set operation compared to the packet handling > path. Certainly in our products the incremental cost of a set vs > processing a DR is negligible. I expect similar results from any > switch. As soon as a SMP goes into the CPU for DR or other processing > there is an enormous hit. Whether the DR packets forwarding is done in HW, FW or SW is implementation dependent. I agree with you that if the same entity is processing LFT block set and DR MAD forwarding, then the difference between these two operations would not be very big (having said that, I'm not sure it's negligible - again, it's implementation dependent). I don't know about all the IB switches, but in InfiniScale IV (and any InfiniScaleIV-based switches out there) DR packets forwarding is done in HW, so its significantly faster than processing LFT block by the SMA. Same goes for any future IB switch that will improve handling of DR MADs forwarding. -- Yevgeny > IIRC when I last looked, it is reasonable to expect a 1us per hop for > DR vs a 100ns per hop for LID. If you are 5 switches deep that is 5us > vs 500ns!! > >> The proposed algorithm reduces the potential VL15 overload on >> intermediate switches relative to the current algorithm for two >> reasons: the SMPs are spread across the switches first rather than >> blasting each switch in turn and there is a limit on the number of >> SMPs per node (not as good but less expensive than the concurrency >> limit you propose). > > But you overload the switch the SM is connected to with processing > N*limit DR SMPs rather than just 'limit' SMPs. That is what concerns > me. > >>> ??1) Within the BFS consider things as layers (number of hops from the >>> ?? ??SM). All switches in a layer can proceed largely in parallel, >>> ?? ??within the capability of the SM to capture the MAD replies at >>> ?? ??least. Advancing to the next layer must wait until the prior >>> ?? ??layer is fully programmed. >> The premise of this patch is to spread the blocks across the switches >> first rather than populate an individual switch entirely before >> proceeding with the next. This is due to the handling of the blocks >> being significantly more time consuming than any of the forwarding. I >> think that principle should apply to this algorithm as well. > > As is what I propose, an entire level can be done in parallel within > the two concurrency limits. > > The reason for the BFS is to enable LID routed packets which I view as > the biggest gain. You have to progress outward building up paths back > to the SM to do this. > > Randomly programming all switches does not enable LID routing. > > [Though do note the BFS here is a special kind that traverses the > LID routing graph for the SM LID to guarentee the above. A LID route > that is not shortest-path will not work with a straight topological > BFS] > >>> ??2) At each switch send at most two partial DR MADs - where the MAD is >>> ?? ??LID routed up to the parent switch in the BFS, and then direct routed >>> ?? ??one step to the target switch. The two MADs would program the LFT >>> ?? ??block for the SM LID and for the switch LID. >> Combined LR/DR routing is not a good idea IMO. Some switches don't support >> this although a requirement. Full DR routing could be used here >> rather > > I first implemented an algorithm like this for switches based on Gamla > chips, and then for Anafa. If something doesn't support it, it is > very uncommon. > > Thus, a simple global off switch for parallelism seems reasonable if > you think there is a worry. Frankly, I'm not very concerned about such > significant non-compliance. > >> than the combined DR routing although it would be less efficient in >> terms of forwarding compared with the combined DR (with LR direct to >> the previous level of switch). > > The problem with this is it makes managing the concurrency alot > harder, and you hit a bottleneck sooner, since you now have to track > every step along the DR path, not just the parent. > >>> ??3) Sent LID routed mads directly to the target switch to fill in the >>> ?? ??rest of the LFT entries. >> These should be striped across the switches at a given "level". > > Yes, I imagine this algorithm running in parallel for all switches at > a level. > >>> Step 2 needs to respect the concurrency limit for the parent switch, >>> and #3 needs to respect the concurrency limit for the target switch. >> This is the harder part and also more expensive in terms of >> computation. This limit might also be overly conservative. > > I don't see how it is more computational, you know the parent switch > because you are computing a DR path to it. A simple per-switch counter > is all that is needed. > >> Also, IMO there would be one configuration item for this limit rather >> than a per switch configuration. > > Yes, two configurables would be excellent. Something like 20 for DR > and 4 for Get/Set sounds reasonable to me. > >>> Eliminating DR hops will significantly improve MAD round trip time >>> and give you more possible parallelism before the SMAs in intermediate >>> switches become overloaded. >> I can see the MAD round trip time improvement based on the virtual >> reduction in number of forwarding hops but I don't see the increase in >> parallelism. SMPs are not flow controlled so I would think the >> parallelism is the same except that the transaction rate is somewhat >> higher using LR rather than DR SMPs. > > In real switch implementations today, the DR path has a much lower > forwarding rate than the LID path. The LID path is wire speed, DR is > something like 4-20 packets in a burst at best - due to the CPU > involvement. If you ask 1000 switches to do a set via DR then the > switch the SM is hooked up to will probably start dropping. If you do > the same via LID, then no problems. > > This is not any sort of IBA requirement, just a reflection of the > reality of all current implementations. > > So, LID routing offloads the SMP processing in the switches closest to > the SM and lets you push more SMPs through them. You go from, say, a > global 20 parallel DR SMP limit at the SM's switch to a > 20*N_Switches_At_Level limit, which is much higher. > >>> into the above process and then all other switch programming MADs can >>> simply happen via LID routed packets. >> Sure; it could be done for many other SMPs but the most important >> thing in speeding up config time are the LFT/MFT block sets. > > Once you get LID routing setup there is no reason to keep using DR > after that point. For instance, just converting MFT programming to use > LID only would probably result in noticable gains on big fabrics. > > Jason > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From hnrose at comcast.net Thu Jul 23 06:05:29 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Thu, 23 Jul 2009 09:05:29 -0400 Subject: [ofa-general] [PATCH] opensm: Add ability to configure SMSL Message-ID: <20090723130529.GA27568@comcast.net> Override default SM's SL to use in cases where LASH is not used. Signed-off-by: Robert Pearson Signed-off-by: Hal Rosenstock --- diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 29d3b73..6c20de8 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -224,6 +224,7 @@ typedef struct osm_subn_opt { boolean_t consolidate_ipv6_snm_req; struct osm_subn_opt *file_opts; /* used for update */ uint8_t lash_start_vl; /* starting vl to use in lash */ + uint8_t sm_sl; /* which SL to use for SM/SA communication */ } osm_subn_opt_t; /* * FIELDS diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in index c71a79d..b23a973 100644 --- a/opensm/man/opensm.8.in +++ b/opensm/man/opensm.8.in @@ -12,6 +12,7 @@ opensm \- InfiniBand subnet manager and administration (SM/SA) [\-l(mc) ] [\-p(riority) ] [\-smkey ] +[\-\-sm_sl ] [\-r(eassign_lids)] [\-R | \-\-routing_engine ] [\-\-do_mesh_analysis] @@ -129,6 +130,10 @@ Note that OpenSM version 3.2.1 and below used the default value '1' in a host byte order, it is fixed now but you may need this option to interoperate with old OpenSM running on a little endian machine. .TP +\fB\-\-sm_sl\fR +This option sets the SL to use for communication with the SM/SA. +Defaults to 0. +.TP \fB\-r\fR, \fB\-\-reassign_lids\fR This option causes OpenSM to reassign LIDs to all end nodes. Specifying -r on a running subnet diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c index 62242f4..bf8e5c6 100644 --- a/opensm/opensm/main.c +++ b/opensm/opensm/main.c @@ -185,6 +185,8 @@ static void show_usage(void) printf("--lash_start_vl \n" " Sets the starting VL to use for the lash routing algorithm.\n" " Defaults to 0.\n"); + printf("--sm_sl \n" + " Sets the SL to use to communicate with the SM/SA. Defaults to 0.\n\n"); printf("--connect_roots, -z\n" " This option enforces a routing engine (currently\n" " up/down only) to make connectivity between root switches\n" @@ -606,6 +608,7 @@ int main(int argc, char *argv[]) {"consolidate_ipv6_snm_req", 0, NULL, 4}, {"do_mesh_analysis", 0, NULL, 5}, {"lash_start_vl", 1, NULL, 6}, + {"sm_sl", 1, NULL, 7}, {NULL, 0, NULL, 0} /* Required at the end of the array */ }; @@ -966,6 +969,15 @@ int main(int argc, char *argv[]) opt.lash_start_vl = (uint8_t) temp; printf(" LASH starting VL = %d\n", opt.lash_start_vl); break; + case 7: + temp = strtol(optarg, NULL, 0); + if (temp < 0 || temp > 15) { + fprintf(stderr, + "ERROR: SM's SL must be between 0 and 15\n"); + return (-1); + } + opt.sm_sl = (uint8_t) temp; + break; case 'h': case '?': case ':': diff --git a/opensm/opensm/osm_link_mgr.c b/opensm/opensm/osm_link_mgr.c index 285096c..c9bdfee 100644 --- a/opensm/opensm/osm_link_mgr.c +++ b/opensm/opensm/osm_link_mgr.c @@ -71,7 +71,7 @@ static uint8_t link_mgr_get_smsl(IN osm_sm_t * sm, IN osm_physp_t * p_physp) if (p_osm->routing_engine_used != OSM_ROUTING_ENGINE_TYPE_LASH) { /* Use default SL if lash routing is not used */ OSM_LOG_EXIT(sm->p_log); - return (OSM_DEFAULT_SL); + return (sm->p_subn->opt.sm_sl); } /* Find osm_port of the SM itself = dest_port */ diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index 3470b60..0d11811 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -391,6 +391,7 @@ static const opt_rec_t opt_tbl[] = { { "prefix_routes_file", OPT_OFFSET(prefix_routes_file), opts_parse_charp, NULL, 0 }, { "consolidate_ipv6_snm_req", OPT_OFFSET(consolidate_ipv6_snm_req), opts_parse_boolean, NULL, 1 }, { "lash_start_vl", OPT_OFFSET(lash_start_vl), opts_parse_uint8, NULL, 1 }, + { "sm_sl", OPT_OFFSET(sm_sl), opts_parse_uint8, NULL, 1 }, {0} }; @@ -760,6 +761,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt) p_opt->prefix_routes_file = strdup(OSM_DEFAULT_PREFIX_ROUTES_FILE); p_opt->consolidate_ipv6_snm_req = FALSE; p_opt->lash_start_vl = 0; + p_opt->sm_sl = OSM_DEFAULT_SL; subn_init_qos_options(&p_opt->qos_options, NULL); subn_init_qos_options(&p_opt->qos_ca_options, NULL); subn_init_qos_options(&p_opt->qos_sw0_options, NULL); @@ -1277,6 +1279,8 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) "# enhanced switch port 0. If TRUE, LMC value for subnet is used for\n" "# ESP0. Otherwise, LMC value for ESP0s is 0.\n" "lmc_esp0 %s\n\n" + "# sm_sl determines SMSL used for SM/SA communication\n" + "sm_sl %u\n\n" "# The code of maximal time a packet can live in a switch\n" "# The actual time is 4.096usec * 2^\n" "# The value 0x14 disables this mechanism\n" @@ -1326,6 +1330,7 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) cl_ntoh64(p_opts->subnet_prefix), p_opts->lmc, p_opts->lmc_esp0 ? "TRUE" : "FALSE", + p_opts->sm_sl, p_opts->packet_life_time, p_opts->vl_stall_count, p_opts->leaf_vl_stall_count, From jackm at dev.mellanox.co.il Thu Jul 23 06:30:59 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Thu, 23 Jul 2009 16:30:59 +0300 Subject: [ofa-general] ofa_1_5_kernel 20090723-0200 daily build status In-Reply-To: <20090723095506.36117E30151@openfabrics.org> References: <20090723095506.36117E30151@openfabrics.org> Message-ID: <200907231630.59696.jackm@dev.mellanox.co.il> Andy, This snippet is from the EWG list, regarding the daily build of OFED 1.5 (which is based on kernel 2.6.30). Note the failure below (when compiling on kernel 2.6.26). Please note that rds will fail in ALL backports (i.e, kernel 2.6.29 and earlier), because the 'DECLARE_PER_CPU_SHARED_ALIGNED' macro is new for kernel 2.6.30. You need to arrange backports here. -Jack On Thursday 23 July 2009 12:55, Vladimir Sokolovsky (Mellanox) wrote: > Build failed on x86_64 with linux-2.6.26 > Log: > In file included from /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: > /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' > /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class > /home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' > make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 > make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 > make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090723-0200_linux-2.6.26_x86_64_check] Error 2 > make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' > make: *** [kernel] Error 2 > From hal.rosenstock at gmail.com Thu Jul 23 07:26:12 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Thu, 23 Jul 2009 10:26:12 -0400 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <20090723005159.GA1230@obsidianresearch.com> References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> <20090722204846.GF12693@obsidianresearch.com> <20090723005159.GA1230@obsidianresearch.com> Message-ID: On Wed, Jul 22, 2009 at 8:51 PM, Jason Gunthorpe wrote: > On Wed, Jul 22, 2009 at 08:28:25PM -0400, Hal Rosenstock wrote: > >> > But you overload the switch the SM is connected to with processing >> > N*limit DR SMPs rather than just 'limit' SMPs. That is what concerns >> > me. >> >> As I said, the current algorithm is worse as it sends N*no limit DR >> SMPs (where no limit means any needed blocks). Not sure that VL15 >> droppage due to this has been identified. So I think this improves on >> what's been deployed and seemingly works in OpenSM for quite some time >> now. > > Hmm, OK I didn't realize that. > > I've heard of reports of VL15 droppage in real networks, maybe this is why.. Could be; another possible source is any tools/apps which use VL15 like concurrent diag use (ibnetdiscover, ibdiagnet, many other infiniband-diags, ....). Point is that these subnets seem to recover/still work so the retry must be working. -- Hal From jgunthorpe at obsidianresearch.com Thu Jul 23 08:19:39 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Thu, 23 Jul 2009 09:19:39 -0600 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <4A685D33.2050803@dev.mellanox.co.il> References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> <20090722204846.GF12693@obsidianresearch.com> <4A685D33.2050803@dev.mellanox.co.il> Message-ID: <20090723151939.GB1230@obsidianresearch.com> On Thu, Jul 23, 2009 at 03:53:07PM +0300, Yevgeny Kliteynik wrote: > >I would be very surprised if any implementation had a significant > >overhead for the actual set operation compared to the packet handling > >path. Certainly in our products the incremental cost of a set vs > >processing a DR is negligible. I expect similar results from any > >switch. As soon as a SMP goes into the CPU for DR or other processing > >there is an enormous hit. > > Whether the DR packets forwarding is done in HW, FW or SW is > implementation dependent. I agree with you that if the same > entity is processing LFT block set and DR MAD forwarding, then > the difference between these two operations would not be very > big (having said that, I'm not sure it's negligible - again, > it's implementation dependent). > > I don't know about all the IB switches, but in InfiniScale IV > (and any InfiniScaleIV-based switches out there) DR packets > forwarding is done in HW, so its significantly faster than > processing LFT block by the SMA. Yes, this is sort of what worries me.. Tuning/testing for these kind of switches can horribly break on other switches that have low limits on DR SMP processing. Jason From jgunthorpe at obsidianresearch.com Thu Jul 23 09:02:29 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Thu, 23 Jul 2009 10:02:29 -0600 Subject: [ofa-general] [PATCH ibverbs] Do not use enum object types for bitfields Message-ID: <20090723160229.GA22400@obsidianresearch.com> Arithmetic operations on enum members does not result in the enum type. So all flag enums result in compile errors from g++ when they are or'd togeth. Replace all flag enum objects with 'int'. 'int' was selected to preserve the ABI and validated on i386/x86-64/ppc32/ppc64/ia64 Signed-off-by: Jason Gunthorpe --- include/infiniband/driver.h | 8 ++++---- include/infiniband/verbs.h | 28 ++++++++++++++-------------- src/cmd.c | 10 +++++----- src/compat-1_0.c | 18 +++++++++--------- src/verbs.c | 8 ++++---- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h index e54e0e3..9a81416 100644 --- a/include/infiniband/driver.h +++ b/include/infiniband/driver.h @@ -77,7 +77,7 @@ int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd, int ibv_cmd_dealloc_pd(struct ibv_pd *pd); #define IBV_CMD_REG_MR_HAS_RESP_PARAMS int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, - uint64_t hca_va, enum ibv_access_flags access, + uint64_t hca_va, int access, struct ibv_mr *mr, struct ibv_reg_mr *cmd, size_t cmd_size, struct ibv_reg_mr_resp *resp, size_t resp_size); @@ -101,7 +101,7 @@ int ibv_cmd_create_srq(struct ibv_pd *pd, struct ibv_create_srq_resp *resp, size_t resp_size); int ibv_cmd_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask, + int srq_attr_mask, struct ibv_modify_srq *cmd, size_t cmd_size); int ibv_cmd_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, @@ -113,11 +113,11 @@ int ibv_cmd_create_qp(struct ibv_pd *pd, struct ibv_create_qp *cmd, size_t cmd_size, struct ibv_create_qp_resp *resp, size_t resp_size); int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *qp_init_attr, struct ibv_query_qp *cmd, size_t cmd_size); int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_modify_qp *cmd, size_t cmd_size); int ibv_cmd_destroy_qp(struct ibv_qp *qp); int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h index 226d85e..0f1cb2e 100644 --- a/include/infiniband/verbs.h +++ b/include/infiniband/verbs.h @@ -269,7 +269,7 @@ struct ibv_wc { uint32_t imm_data; /* in network byte order */ uint32_t qp_num; uint32_t src_qp; - enum ibv_wc_flags wc_flags; + int wc_flags; uint16_t pkey_index; uint16_t slid; uint8_t sl; @@ -508,7 +508,7 @@ struct ibv_send_wr { struct ibv_sge *sg_list; int num_sge; enum ibv_wr_opcode opcode; - enum ibv_send_flags send_flags; + int send_flags; uint32_t imm_data; /* in network byte order */ union { struct { @@ -541,8 +541,8 @@ struct ibv_mw_bind { struct ibv_mr *mr; void *addr; size_t length; - enum ibv_send_flags send_flags; - enum ibv_access_flags mw_access_flags; + int send_flags; + int mw_access_flags; }; struct ibv_srq { @@ -633,12 +633,12 @@ struct ibv_context_ops { struct ibv_pd * (*alloc_pd)(struct ibv_context *context); int (*dealloc_pd)(struct ibv_pd *pd); struct ibv_mr * (*reg_mr)(struct ibv_pd *pd, void *addr, size_t length, - enum ibv_access_flags access); + int access); struct ibv_mr * (*rereg_mr)(struct ibv_mr *mr, - enum ibv_rereg_mr_flags flags, + int flags, struct ibv_pd *pd, void *addr, size_t length, - enum ibv_access_flags access); + int access); int (*dereg_mr)(struct ibv_mr *mr); struct ibv_mw * (*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type); int (*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw, @@ -656,7 +656,7 @@ struct ibv_context_ops { struct ibv_srq_init_attr *srq_init_attr); int (*modify_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask); + int srq_attr_mask); int (*query_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); int (*destroy_srq)(struct ibv_srq *srq); @@ -665,10 +665,10 @@ struct ibv_context_ops { struct ibv_recv_wr **bad_recv_wr); struct ibv_qp * (*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr); int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask); + int attr_mask); int (*destroy_qp)(struct ibv_qp *qp); int (*post_send)(struct ibv_qp *qp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr); @@ -793,7 +793,7 @@ int ibv_dealloc_pd(struct ibv_pd *pd); * ibv_reg_mr - Register a memory region */ struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr, - size_t length, enum ibv_access_flags access); + size_t length, int access); /** * ibv_dereg_mr - Deregister a memory region @@ -926,7 +926,7 @@ struct ibv_srq *ibv_create_srq(struct ibv_pd *pd, */ int ibv_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask); + int srq_attr_mask); /** * ibv_query_srq - Returns the attribute list and current values for the @@ -966,7 +966,7 @@ struct ibv_qp *ibv_create_qp(struct ibv_pd *pd, * ibv_modify_qp - Modify a queue pair. */ int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask); + int attr_mask); /** * ibv_query_qp - Returns the attribute list and current values for the @@ -980,7 +980,7 @@ int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, * selected attributes. */ int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr); /** diff --git a/src/cmd.c b/src/cmd.c index c2e76cf..cbd5288 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -231,7 +231,7 @@ int ibv_cmd_dealloc_pd(struct ibv_pd *pd) } int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, - uint64_t hca_va, enum ibv_access_flags access, + uint64_t hca_va, int access, struct ibv_mr *mr, struct ibv_reg_mr *cmd, size_t cmd_size, struct ibv_reg_mr_resp *resp, size_t resp_size) @@ -485,7 +485,7 @@ int ibv_cmd_create_srq(struct ibv_pd *pd, static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask, + int srq_attr_mask, struct ibv_modify_srq *new_cmd, size_t new_cmd_size) { @@ -513,7 +513,7 @@ static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq, int ibv_cmd_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask, + int srq_attr_mask, struct ibv_modify_srq *cmd, size_t cmd_size) { if (abi_ver == 3) @@ -651,7 +651,7 @@ int ibv_cmd_create_qp(struct ibv_pd *pd, } int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr, struct ibv_query_qp *cmd, size_t cmd_size) { @@ -733,7 +733,7 @@ int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, } int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_modify_qp *cmd, size_t cmd_size) { IBV_INIT_CMD(cmd, cmd_size, MODIFY_QP); diff --git a/src/compat-1_0.c b/src/compat-1_0.c index 459ade9..3f5ff35 100644 --- a/src/compat-1_0.c +++ b/src/compat-1_0.c @@ -88,7 +88,7 @@ struct ibv_send_wr_1_0 { struct ibv_sge *sg_list; int num_sge; enum ibv_wr_opcode opcode; - enum ibv_send_flags send_flags; + int send_flags; uint32_t imm_data; /* in network byte order */ union { struct { @@ -172,7 +172,7 @@ struct ibv_context_ops_1_0 { struct ibv_pd * (*alloc_pd)(struct ibv_context *context); int (*dealloc_pd)(struct ibv_pd *pd); struct ibv_mr * (*reg_mr)(struct ibv_pd *pd, void *addr, size_t length, - enum ibv_access_flags access); + int access); int (*dereg_mr)(struct ibv_mr *mr); struct ibv_cq * (*create_cq)(struct ibv_context *context, int cqe, struct ibv_comp_channel *channel, @@ -188,7 +188,7 @@ struct ibv_context_ops_1_0 { struct ibv_srq_init_attr *srq_init_attr); int (*modify_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask); + int srq_attr_mask); int (*query_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); int (*destroy_srq)(struct ibv_srq *srq); @@ -197,10 +197,10 @@ struct ibv_context_ops_1_0 { struct ibv_recv_wr_1_0 **bad_recv_wr); struct ibv_qp * (*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr); int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask); + int attr_mask); int (*destroy_qp)(struct ibv_qp *qp); int (*post_send)(struct ibv_qp_1_0 *qp, struct ibv_send_wr_1_0 *wr, @@ -596,7 +596,7 @@ int __ibv_dealloc_pd_1_0(struct ibv_pd_1_0 *pd) symver(__ibv_dealloc_pd_1_0, ibv_dealloc_pd, IBVERBS_1.0); struct ibv_mr_1_0 *__ibv_reg_mr_1_0(struct ibv_pd_1_0 *pd, void *addr, - size_t length, enum ibv_access_flags access) + size_t length, int access) { struct ibv_mr *real_mr; struct ibv_mr_1_0 *mr; @@ -736,7 +736,7 @@ symver(__ibv_create_srq_1_0, ibv_create_srq, IBVERBS_1.0); int __ibv_modify_srq_1_0(struct ibv_srq_1_0 *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask) + int srq_attr_mask) { return ibv_modify_srq(srq->real_srq, srq_attr, srq_attr_mask); } @@ -806,7 +806,7 @@ struct ibv_qp_1_0 *__ibv_create_qp_1_0(struct ibv_pd_1_0 *pd, symver(__ibv_create_qp_1_0, ibv_create_qp, IBVERBS_1.0); int __ibv_query_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr_1_0 *init_attr) { struct ibv_qp_init_attr real_init_attr; @@ -829,7 +829,7 @@ int __ibv_query_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr, symver(__ibv_query_qp_1_0, ibv_query_qp, IBVERBS_1.0); int __ibv_modify_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask) + int attr_mask) { return ibv_modify_qp(qp->real_qp, attr, attr_mask); } diff --git a/src/verbs.c b/src/verbs.c index 477e412..ba3c0a4 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -155,7 +155,7 @@ int __ibv_dealloc_pd(struct ibv_pd *pd) default_symver(__ibv_dealloc_pd, ibv_dealloc_pd); struct ibv_mr *__ibv_reg_mr(struct ibv_pd *pd, void *addr, - size_t length, enum ibv_access_flags access) + size_t length, int access) { struct ibv_mr *mr; @@ -377,7 +377,7 @@ default_symver(__ibv_create_srq, ibv_create_srq); int __ibv_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask) + int srq_attr_mask) { return srq->context->ops.modify_srq(srq, srq_attr, srq_attr_mask); } @@ -419,7 +419,7 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd, default_symver(__ibv_create_qp, ibv_create_qp); int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr) { int ret; @@ -436,7 +436,7 @@ int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, default_symver(__ibv_query_qp, ibv_query_qp); int __ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask) + int attr_mask) { int ret; -- 1.5.4.2 From jgunthorpe at obsidianresearch.com Thu Jul 23 09:03:12 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Thu, 23 Jul 2009 10:03:12 -0600 Subject: [ofa-general] [PATCH mlx4] Update function prototypes to match ibverbs Message-ID: <20090723160312.GB22400@obsidianresearch.com> Replace certain enums with int. Signed-off-by: Jason Gunthorpe --- src/mlx4.h | 8 ++++---- src/verbs.c | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mlx4.h b/src/mlx4.h index 0c658cf..4445998 100644 --- a/src/mlx4.h +++ b/src/mlx4.h @@ -302,7 +302,7 @@ struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context); int mlx4_free_pd(struct ibv_pd *pd); struct ibv_mr *mlx4_reg_mr(struct ibv_pd *pd, void *addr, - size_t length, enum ibv_access_flags access); + size_t length, int access); int mlx4_dereg_mr(struct ibv_mr *mr); struct ibv_cq *mlx4_create_cq(struct ibv_context *context, int cqe, @@ -323,7 +323,7 @@ struct ibv_srq *mlx4_create_srq(struct ibv_pd *pd, struct ibv_srq_init_attr *attr); int mlx4_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, - enum ibv_srq_attr_mask mask); + int mask); int mlx4_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr); int mlx4_destroy_srq(struct ibv_srq *srq); @@ -336,10 +336,10 @@ int mlx4_post_srq_recv(struct ibv_srq *ibsrq, struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); int mlx4_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr); int mlx4_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask); + int attr_mask); int mlx4_destroy_qp(struct ibv_qp *qp); void mlx4_init_qp_indices(struct mlx4_qp *qp); void mlx4_qp_init_sq_ownership(struct mlx4_qp *qp); diff --git a/src/verbs.c b/src/verbs.c index 2c19d93..1ac1362 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -108,7 +108,7 @@ int mlx4_free_pd(struct ibv_pd *pd) } struct ibv_mr *mlx4_reg_mr(struct ibv_pd *pd, void *addr, size_t length, - enum ibv_access_flags access) + int access) { struct ibv_mr *mr; struct ibv_reg_mr cmd; @@ -353,7 +353,7 @@ err: int mlx4_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, - enum ibv_srq_attr_mask attr_mask) + int attr_mask) { struct ibv_modify_srq cmd; @@ -497,7 +497,7 @@ err: } int mlx4_query_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr) { struct ibv_query_qp cmd; @@ -518,7 +518,7 @@ int mlx4_query_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, } int mlx4_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask) + int attr_mask) { struct ibv_modify_qp cmd; int ret; -- 1.5.4.2 From jgunthorpe at obsidianresearch.com Thu Jul 23 09:04:29 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Thu, 23 Jul 2009 10:04:29 -0600 Subject: [ofa-general] [PATCH mthca] Update function prototypes to match ibverbs Message-ID: <20090723160429.GC22400@obsidianresearch.com> Replace certain enums with int. Signed-off-by: Jason Gunthorpe --- src/mthca.h | 8 ++++---- src/verbs.c | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/mthca.h b/src/mthca.h index 9a2e362..bd1e7a2 100644 --- a/src/mthca.h +++ b/src/mthca.h @@ -309,7 +309,7 @@ struct ibv_pd *mthca_alloc_pd(struct ibv_context *context); int mthca_free_pd(struct ibv_pd *pd); struct ibv_mr *mthca_reg_mr(struct ibv_pd *pd, void *addr, - size_t length, enum ibv_access_flags access); + size_t length, int access); int mthca_dereg_mr(struct ibv_mr *mr); struct ibv_cq *mthca_create_cq(struct ibv_context *context, int cqe, @@ -330,7 +330,7 @@ struct ibv_srq *mthca_create_srq(struct ibv_pd *pd, struct ibv_srq_init_attr *attr); int mthca_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, - enum ibv_srq_attr_mask mask); + int mask); int mthca_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr); int mthca_destroy_srq(struct ibv_srq *srq); @@ -346,10 +346,10 @@ int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq, struct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); int mthca_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr); int mthca_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask); + int attr_mask); int mthca_destroy_qp(struct ibv_qp *qp); void mthca_init_qp_indices(struct mthca_qp *qp); int mthca_tavor_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, diff --git a/src/verbs.c b/src/verbs.c index f6570c6..b6782c9 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -117,7 +117,7 @@ int mthca_free_pd(struct ibv_pd *pd) static struct ibv_mr *__mthca_reg_mr(struct ibv_pd *pd, void *addr, size_t length, uint64_t hca_va, - enum ibv_access_flags access, + int access, int dma_sync) { struct ibv_mr *mr; @@ -157,7 +157,7 @@ static struct ibv_mr *__mthca_reg_mr(struct ibv_pd *pd, void *addr, } struct ibv_mr *mthca_reg_mr(struct ibv_pd *pd, void *addr, - size_t length, enum ibv_access_flags access) + size_t length, int access) { return __mthca_reg_mr(pd, addr, length, (uintptr_t) addr, access, 0); } @@ -468,7 +468,7 @@ err: int mthca_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, - enum ibv_srq_attr_mask attr_mask) + int attr_mask) { struct ibv_modify_srq cmd; @@ -618,7 +618,7 @@ err: } int mthca_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr) { struct ibv_query_qp cmd; @@ -627,7 +627,7 @@ int mthca_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, } int mthca_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask) + int attr_mask) { struct ibv_modify_qp cmd; int ret; -- 1.5.4.2 From tziporet at mellanox.co.il Thu Jul 23 11:17:05 2009 From: tziporet at mellanox.co.il (Tziporet Koren) Date: Thu, 23 Jul 2009 21:17:05 +0300 Subject: [ofa-general] OFED 1.5 alpha release is available References: <5D49E7A8952DC44FB38C38FA0D758EAD02C12252@mtlexch01.mtl.com> <5D49E7A8952DC44FB38C38FA0D758EAD02D5F39D@mtlexch01.mtl.com> Message-ID: <2ED289D4E09FBD4D92D911E869B97FDD29F156@mtlexch01.mtl.com> OFED 1.5-alpha4 is available Notes: The tarball is available on: http://www.openfabrics.org/downloads/OFED/ofed-1.5/OFED-1.5-alpha4.tgz To get BUILD_ID run ofed_info Please report any issues in bugzilla https://bugs.openfabrics.org/ for OFED 1.5 Vladimir & Tziporet ======================================================================== Release information: ------------------------------ Linux Operating Systems: o Linux Operating Systems: - RedHat EL4 up6: 2.6.9-67.ELsmp - RedHat EL4 up7: 2.6.9-78.ELsmp - RedHat EL4 up8: 2.6.9-89.ELsmp - RedHat EL5 up2: 2.6.18-92.el5 - RedHat EL5 up3: 2.6.18-128.el5 - SLES10 SP2: 2.6.16.60-0.21-smp - SLES11: 2.6.27.19-5-default - OpenSuSE 10.3: 2.6.22.5-31 * - OEL 4 up7 2.6.9-78.ELsmp - OEL 5 up2 2.6.18-92.el5 - CentOS5.2 2.6.18-92.el5 - CentOS5.3 2.6.18-128.el5 - kernel.org: 2.6.29 and 2.6.30 * Minimal QA for these versions Systems: * x86_64 * x86 * ia64 * ppc64 Changes from OFED-1.4.1 ======================== 1 General changes o Kernel code based on 2.6.30 2 SDP o Performance improvements 3 uDAPL o New library 4 Management o OpenSM - Mesh Analysis for LASH routing algorithm. - Reloadable OpenSM configuration (preliminary implemented) - Routing paths sorted balancing (for UpDown and MinHops) - Weighted Lid Matrices calculation (for UpDown, MinHop and DOR). - I/O nodes connectivity (for FatTree). 5 MPI: - For now same versions as in OFED 1.4.1 Tasks that should be completed for the beta =========================================== 1. Complete Backports for all kernel modules 2. Move to new libraries package scheme (as we agreed in Sonoma) 3. SDP Zero Copy 4. Stability, stability, Stability ... From rdreier at cisco.com Thu Jul 23 12:04:32 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 23 Jul 2009 12:04:32 -0700 Subject: [ofa-general] Re: 2.6.30.1: possible irq lock inversion dependency detected In-Reply-To: (Bart Van Assche's message of "Thu, 23 Jul 2009 08:35:59 +0200") References: Message-ID: > Any idea when you will be able to push this patch upstream ? Or has it > already been pushed upstream, and did I miss it ? Probably for 2.6.32... thanks for the reminder, I need a chance to clean it up for merging. From rdreier at cisco.com Thu Jul 23 12:06:41 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 23 Jul 2009 12:06:41 -0700 Subject: [ofa-general] Re: [ewg] [Patch mthca backport] Don't use kmalloc > 128k In-Reply-To: <200907231121.00140.jackm@dev.mellanox.co.il> (Jack Morgenstein's message of "Thu, 23 Jul 2009 11:20:59 +0300") References: <356B6978-3308-4EE9-8C00-00199558BDEA@redhat.com> <200907231121.00140.jackm@dev.mellanox.co.il> Message-ID: > This will fix the 2^20 bits limit on our bitmaps once and for all. Not really... since getting > 128KB of contiguous memory is likely to fail anyway. And I don't think the upstream kernel has that limit on kmalloc size either (at least with SLUB, not sure about SLAB). Really the long-term fix is to handle non-contiguous memory in the bitmap allocator. maybe using vmalloc(), although I always hate big allocations with vmalloc too. From rdreier at cisco.com Thu Jul 23 14:09:11 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 23 Jul 2009 14:09:11 -0700 Subject: [ofa-general] [PATCH ibverbs] Do not use enum object types for bitfields In-Reply-To: <20090723160229.GA22400@obsidianresearch.com> (Jason Gunthorpe's message of "Thu, 23 Jul 2009 10:02:29 -0600") References: <20090723160229.GA22400@obsidianresearch.com> Message-ID: Thanks... I guess we can take a stab at doing this and see if anyone breaks because of it. I do notice that this patch doesn't update the man pages and therefore leaves them out of sync with the actual code. And that makes me notice that by going from "enum ibv_foo foo" to "int foo" we lose the nice self-documenting property of the API that it tells you what set of flags are supposed to be used there. I guess we should make sure that at least all the interfaces in verbs.h used by applications (as opposed to driver stuff in driver.h) document what enum contains the flags that go into each parameter. - R. From rdreier at cisco.com Thu Jul 23 14:17:17 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 23 Jul 2009 14:17:17 -0700 Subject: [ofa-general] [PATCH] Allow paths to the device specific library to be absolute In-Reply-To: <20090722224506.GI12693@obsidianresearch.com> (Jason Gunthorpe's message of "Wed, 22 Jul 2009 16:45:06 -0600") References: <20090714211653.GA13700@obsidianresearch.com> <20090717210817.GF7320@obsidianresearch.com> <20090722212505.GG12693@obsidianresearch.com> <20090722224506.GI12693@obsidianresearch.com> Message-ID: > > I don't have a debian logon, but I do have an ia64 machine: > Ah, in the good old days every DD had one of those :) BTW I'm not a dd -- just have a sponsor for my packages. Maybe someday I'll try to become a dd but for now I'm too lazy. > The identifiers in an enumerator list are declared as constants that > have type int and may appear wherever such are permitted. > > Each enumerated type shall be compatible with an integer type. The > choice of type is implementation-defined, but shall be capable of > representing the values of all the members of the enumeration. thinking about this, I confused myself. Given enum foo { FOO, BAR }; and given that the standard quoted above says FOO and BAR have type int, why is it valid to do enum foo x = FOO; but not enum foo y = FOO | BAR; ? It seems FOO and (FOO | BAR) would both have type int... From mjp at mit.edu Thu Jul 23 14:27:23 2009 From: mjp at mit.edu (Mark J. Pearrow) Date: Thu, 23 Jul 2009 17:27:23 -0400 Subject: [ofa-general] Infiniband and Ubuntu Message-ID: <7AD18005-56B0-4598-8544-ABB05EC0E3E2@mit.edu> Hi all, The short question is "is there a howto for getting Infiniband and Ubuntu working?" I have a cluster of Dell PowerEdge servers that are running Ubuntu 9.04. I have Mellanox HCAs installed in them. My eventual goal is to get GlusterFS up and running across them with IB as the backplane instead of GigE. I've been following an older howto here: http://www.lucubration.com/open-source-projects/ubuntu-infiniband-ipoib.html However, this is somewhat dated. I've managed to build most of the RPMs using the install.pl included with OFED-1.5, but it has been a hack-and-slash effort; and it still fails before completing. I've looked through this mailing list's archives and through Google, of course, for a couple weeks now, but would appreciate any guidance if you've been successful in getting an Ubuntu system working with IB. Many thanks, and apologies for the extremely newbie-ish question. mjp From jgunthorpe at obsidianresearch.com Thu Jul 23 14:29:28 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Thu, 23 Jul 2009 15:29:28 -0600 Subject: [ofa-general] [PATCH ibverbs] Do not use enum object types for bitfields In-Reply-To: References: <20090723160229.GA22400@obsidianresearch.com> Message-ID: <20090723212928.GK12693@obsidianresearch.com> On Thu, Jul 23, 2009 at 02:09:11PM -0700, Roland Dreier wrote: > Thanks... I guess we can take a stab at doing this and see if anyone > breaks because of it. > > I do notice that this patch doesn't update the man pages and therefore > leaves them out of sync with the actual code. And that makes me notice > that by going from "enum ibv_foo foo" to "int foo" we lose the nice > self-documenting property of the API that it tells you what set of > flags Oops, I forgot to do those, I'll re-send the patch. Well, I'll try, my nroff is terrible.. > are supposed to be used there. I guess we should make sure that at > least all the interfaces in verbs.h used by applications (as opposed to > driver stuff in driver.h) document what enum contains the flags that go > into each parameter. A good man page should list and describe each of the constants that are valid, a reference to the enum is not necessary. I will try and look at this too.. Jason From hal.rosenstock at gmail.com Thu Jul 23 14:43:38 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Thu, 23 Jul 2009 17:43:38 -0400 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <20090723151939.GB1230@obsidianresearch.com> References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> <20090722204846.GF12693@obsidianresearch.com> <4A685D33.2050803@dev.mellanox.co.il> <20090723151939.GB1230@obsidianresearch.com> Message-ID: On Thu, Jul 23, 2009 at 11:19 AM, Jason Gunthorpe wrote: > On Thu, Jul 23, 2009 at 03:53:07PM +0300, Yevgeny Kliteynik wrote: >> >I would be very surprised if any implementation had a significant >> >overhead for the actual set operation compared to the packet handling >> >path. Certainly in our products the incremental cost of a set vs >> >processing a DR is negligible. I expect similar results from any >> >switch. As soon as a SMP goes into the CPU for DR or other processing >> >there is an enormous hit. >> >> Whether the DR packets forwarding is done in HW, FW or SW is >> implementation dependent. I agree with you that if the same >> entity is processing LFT block set and DR MAD forwarding, then >> the difference between these two operations would not be very >> big (having said that, I'm not sure it's negligible - again, >> it's implementation dependent). >> >> I don't know about all the IB switches, but in InfiniScale IV >> (and any InfiniScaleIV-based switches out there) DR packets >> forwarding is done in HW, so its significantly faster than >> processing LFT block by the SMA. > > Yes, this is sort of what worries me.. Tuning/testing for these kind > of switches can horribly break on other switches that have low limits > on DR SMP processing. The proposed algorithm is less aggressive in terms of sending DR SMPs than the current one which seems to work well enough based on field experience. If the concern with this approach is breaking things over the current approach, I suppose a config variable could be added for algorithm selection with the default being the current one. I had decided against this originally but perhaps that would make people feel more comfortable with the proposed algorithm in the patch submitted. -- Hal > Jason > From donald.e.wood at intel.com Thu Jul 23 15:00:31 2009 From: donald.e.wood at intel.com (Don Wood) Date: Thu, 23 Jul 2009 17:00:31 -0500 Subject: [ofa-general] [PATCH] RDMA/nes: Update refcnt during disconnect Message-ID: <20090723220031.GA7432@dewood-MOBL> During termination, it is possible for the refcnt to go to zero while the worker thread is posting events upward. This fix increments the refcnt before the request is passed to the worker thread. The thread decrements the refcnt when the request is completed. Signed-off-by: Don Wood --- drivers/infiniband/hw/nes/nes_cm.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c index 114b802..fe08eb5 100644 --- a/drivers/infiniband/hw/nes/nes_cm.c +++ b/drivers/infiniband/hw/nes/nes_cm.c @@ -2456,6 +2456,7 @@ int nes_cm_disconn(struct nes_qp *nesqp) if (nesqp->disconn_pending == 0) { nesqp->disconn_pending++; spin_unlock_irqrestore(&nesqp->lock, flags); + nes_add_ref(&nesqp->ibqp); /* init our disconnect work element, to */ INIT_WORK(&nesqp->disconn_work, nes_disconnect_worker); @@ -2477,6 +2478,7 @@ static void nes_disconnect_worker(struct work_struct *work) nes_debug(NES_DBG_CM, "processing AEQE id 0x%04X for QP%u.\n", nesqp->last_aeq, nesqp->hwqp.qp_id); nes_cm_disconn_true(nesqp); + nes_rem_ref(&nesqp->ibqp); } -- 1.6.0 From donald.e.wood at intel.com Thu Jul 23 15:00:34 2009 From: donald.e.wood at intel.com (Don Wood) Date: Thu, 23 Jul 2009 17:00:34 -0500 Subject: [ofa-general] [PATCH] RDMA/nes: Allocate work item for disconnect event handling Message-ID: <20090723220034.GA5356@dewood-MOBL> The code currently has a work structure in the qp. This requires a lock and a pending flag to ensure there is never more than one request active. When two events happen quickly (such as FIN and LLP CLOSE), it causes unnecessary timeouts since the second one is dropped. This fix allocates memory for the work request so the second one can be queued. A lock is removed since it is no longer needed. Signed-off-by: Don Wood --- drivers/infiniband/hw/nes/nes_cm.c | 26 +++++++++++--------------- drivers/infiniband/hw/nes/nes_verbs.h | 7 +++++-- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c index fe08eb5..3129ab1 100644 --- a/drivers/infiniband/hw/nes/nes_cm.c +++ b/drivers/infiniband/hw/nes/nes_cm.c @@ -2450,20 +2450,16 @@ static int nes_cm_init_tsa_conn(struct nes_qp *nesqp, struct nes_cm_node *cm_nod */ int nes_cm_disconn(struct nes_qp *nesqp) { - unsigned long flags; + struct disconn_work *work; - spin_lock_irqsave(&nesqp->lock, flags); - if (nesqp->disconn_pending == 0) { - nesqp->disconn_pending++; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_add_ref(&nesqp->ibqp); - /* init our disconnect work element, to */ - INIT_WORK(&nesqp->disconn_work, nes_disconnect_worker); - - queue_work(g_cm_core->disconn_wq, &nesqp->disconn_work); - } else - spin_unlock_irqrestore(&nesqp->lock, flags); + work = kzalloc(sizeof *work, GFP_ATOMIC); + if (!work) + return -ENOMEM; /* Timer will clean up */ + nes_add_ref(&nesqp->ibqp); + work->nesqp = nesqp; + INIT_WORK(&work->work, nes_disconnect_worker); + queue_work(g_cm_core->disconn_wq, &work->work); return 0; } @@ -2473,8 +2469,10 @@ int nes_cm_disconn(struct nes_qp *nesqp) */ static void nes_disconnect_worker(struct work_struct *work) { - struct nes_qp *nesqp = container_of(work, struct nes_qp, disconn_work); + struct disconn_work *dwork = container_of(work, struct disconn_work, work); + struct nes_qp *nesqp = dwork->nesqp; + kfree(work); nes_debug(NES_DBG_CM, "processing AEQE id 0x%04X for QP%u.\n", nesqp->last_aeq, nesqp->hwqp.qp_id); nes_cm_disconn_true(nesqp); @@ -2557,7 +2555,6 @@ static int nes_cm_disconn_true(struct nes_qp *nesqp) spin_lock_irqsave(&nesqp->lock, flags); } - nesqp->disconn_pending = 0; /* There might have been another AE while the lock was released */ original_hw_tcp_state = nesqp->hw_tcp_state; original_ibqp_state = nesqp->ibqp_state; @@ -2610,7 +2607,6 @@ static int nes_cm_disconn_true(struct nes_qp *nesqp) } } } else { - nesqp->disconn_pending = 0; spin_unlock_irqrestore(&nesqp->lock, flags); } diff --git a/drivers/infiniband/hw/nes/nes_verbs.h b/drivers/infiniband/hw/nes/nes_verbs.h index 41c07f2..7df34fe 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.h +++ b/drivers/infiniband/hw/nes/nes_verbs.h @@ -119,6 +119,11 @@ struct nes_wq { spinlock_t lock; }; +struct disconn_work { + struct work_struct work; + struct nes_qp *nesqp; +}; + struct iw_cm_id; struct ietf_mpa_frame; @@ -127,7 +132,6 @@ struct nes_qp { void *allocated_buffer; struct iw_cm_id *cm_id; struct workqueue_struct *wq; - struct work_struct disconn_work; struct nes_cq *nesscq; struct nes_cq *nesrcq; struct nes_pd *nespd; @@ -165,7 +169,6 @@ struct nes_qp { u8 hw_iwarp_state; u8 flush_issued; u8 hw_tcp_state; - u8 disconn_pending; u8 destroyed; }; #endif /* NES_VERBS_H */ -- 1.6.0 From donald.e.wood at intel.com Thu Jul 23 15:00:36 2009 From: donald.e.wood at intel.com (Don Wood) Date: Thu, 23 Jul 2009 17:00:36 -0500 Subject: [ofa-general] [PATCH] Change memory allocation for cqp request to GFP_ATOMIC Message-ID: <20090723220036.GA4680@dewood-MOBL> The routine to allocate a cqp request is not called from process context code. Since it is not ok to sleep, it needs to be GFP_ATOMIC not GFP_KERNEL. Signed-off-by: Don Wood --- drivers/infiniband/hw/nes/nes_utils.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_utils.c b/drivers/infiniband/hw/nes/nes_utils.c index a282031..b34072b 100644 --- a/drivers/infiniband/hw/nes/nes_utils.c +++ b/drivers/infiniband/hw/nes/nes_utils.c @@ -548,7 +548,7 @@ struct nes_cqp_request *nes_get_cqp_request(struct nes_device *nesdev) spin_unlock_irqrestore(&nesdev->cqp.lock, flags); } if (cqp_request == NULL) { - cqp_request = kzalloc(sizeof(struct nes_cqp_request), GFP_KERNEL); + cqp_request = kzalloc(sizeof(struct nes_cqp_request), GFP_ATOMIC); if (cqp_request) { cqp_request->dynamic = 1; INIT_LIST_HEAD(&cqp_request->list); -- 1.6.0 From donald.e.wood at intel.com Thu Jul 23 15:00:38 2009 From: donald.e.wood at intel.com (Don Wood) Date: Thu, 23 Jul 2009 17:00:38 -0500 Subject: [ofa-general] [PATCH] RDMA/nes: Clean out cq completions when qp is destroyed Message-ID: <20090723220038.GA5376@dewood-MOBL> When a qp is destroyed, unprocessed cq entries could still reference the qp. This change zeroes the context value at qp destroy time. By skipping over cqe's with a zero context, poll_cq no longer processes a cqe for a destroyed qp. Signed-off-by: Don Wood --- drivers/infiniband/hw/nes/nes_verbs.c | 119 ++++++++++++++++++++++----------- 1 files changed, 80 insertions(+), 39 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index 21e0fd3..c6b5873 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -1506,12 +1506,45 @@ static struct ib_qp *nes_create_qp(struct ib_pd *ibpd, /** + * nes_clean_cq + */ +static void nes_clean_cq(struct nes_qp *nesqp, struct nes_cq *nescq) +{ + u32 cq_head; + u32 lo; + u32 hi; + u64 u64temp; + unsigned long flags = 0; + + spin_lock_irqsave(&nescq->lock, flags); + + cq_head = nescq->hw_cq.cq_head; + while (le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) { + rmb(); + lo = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); + hi = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]); + u64temp = (((u64)hi) << 32) | ((u64)lo); + u64temp &= ~(NES_SW_CONTEXT_ALIGN-1); + if (u64temp == (u64)(unsigned long)nesqp) { + /* Zero the context value so cqe will be ignored */ + nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = 0; + nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX] = 0; + } + + if (++cq_head >= nescq->hw_cq.cq_size) + cq_head = 0; + } + + spin_unlock_irqrestore(&nescq->lock, flags); +} + + +/** * nes_destroy_qp */ static int nes_destroy_qp(struct ib_qp *ibqp) { struct nes_qp *nesqp = to_nesqp(ibqp); - /* struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); */ struct nes_ucontext *nes_ucontext; struct ib_qp_attr attr; struct iw_cm_id *cm_id; @@ -1548,7 +1581,6 @@ static int nes_destroy_qp(struct ib_qp *ibqp) nes_debug(NES_DBG_QP, "OFA CM event_handler returned, ret=%d\n", ret); } - if (nesqp->user_mode) { if ((ibqp->uobject)&&(ibqp->uobject->context)) { nes_ucontext = to_nesucontext(ibqp->uobject->context); @@ -1560,6 +1592,13 @@ static int nes_destroy_qp(struct ib_qp *ibqp) } if (nesqp->pbl_pbase) kunmap(nesqp->page); + } else { + /* Clean any pending completions from the cq(s) */ + if (nesqp->nesscq) + nes_clean_cq(nesqp, nesqp->nesscq); + + if ((nesqp->nesrcq) && (nesqp->nesrcq != nesqp->nesscq)) + nes_clean_cq(nesqp, nesqp->nesrcq); } nes_rem_ref(&nesqp->ibqp); @@ -3547,7 +3586,6 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) { u64 u64temp; u64 wrid; - /* u64 u64temp; */ unsigned long flags = 0; struct nes_vnic *nesvnic = to_nesvnic(ibcq->device); struct nes_device *nesdev = nesvnic->nesdev; @@ -3560,7 +3598,6 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) u32 cqe_count = 0; u32 wqe_index; u32 u32temp; - /* u32 counter; */ nes_debug(NES_DBG_CQ, "\n"); @@ -3570,24 +3607,27 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) cq_size = nescq->hw_cq.cq_size; while (cqe_count < num_entries) { - if (le32_to_cpu(nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & - NES_CQE_VALID) { - /* - * Make sure we read CQ entry contents *after* - * we've checked the valid bit. - */ - rmb(); - - cqe = nescq->hw_cq.cq_vbase[head]; - nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; - u32temp = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); - wqe_index = u32temp & - (nesdev->nesadapter->max_qp_wr - 1); - u32temp &= ~(NES_SW_CONTEXT_ALIGN-1); - /* parse CQE, get completion context from WQE (either rq or sq */ - u64temp = (((u64)(le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) | - ((u64)u32temp); - nesqp = *((struct nes_qp **)&u64temp); + if ((le32_to_cpu(nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & + NES_CQE_VALID) == 0) + break; + + /* + * Make sure we read CQ entry contents *after* + * we've checked the valid bit. + */ + rmb(); + + cqe = nescq->hw_cq.cq_vbase[head]; + nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; + u32temp = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); + wqe_index = u32temp & (nesdev->nesadapter->max_qp_wr - 1); + u32temp &= ~(NES_SW_CONTEXT_ALIGN-1); + /* parse CQE, get completion context from WQE (either rq or sq) */ + u64temp = (((u64)(le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) | + ((u64)u32temp); + + if (u64temp) { + nesqp = (struct nes_qp *)(unsigned long)u64temp; memset(entry, 0, sizeof *entry); if (cqe.cqe_words[NES_CQE_ERROR_CODE_IDX] == 0) { entry->status = IB_WC_SUCCESS; @@ -3601,7 +3641,7 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) if (le32_to_cpu(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) { if (nesqp->skip_lsmm) { nesqp->skip_lsmm = 0; - wq_tail = nesqp->hwqp.sq_tail++; + nesqp->hwqp.sq_tail++; } /* Working on a SQ Completion*/ @@ -3643,24 +3683,25 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wq_tail].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); entry->opcode = IB_WC_RECV; } - entry->wr_id = wrid; - if (++head >= cq_size) - head = 0; - cqe_count++; - nescq->polled_completions++; - if ((nescq->polled_completions > (cq_size / 2)) || - (nescq->polled_completions == 255)) { - nes_debug(NES_DBG_CQ, "CQ%u Issuing CQE Allocate since more than half of cqes" - " are pending %u of %u.\n", - nescq->hw_cq.cq_number, nescq->polled_completions, cq_size); - nes_write32(nesdev->regs+NES_CQE_ALLOC, - nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); - nescq->polled_completions = 0; - } + entry->wr_id = wrid; entry++; - } else - break; + cqe_count++; + } + + if (++head >= cq_size) + head = 0; + nescq->polled_completions++; + + if ((nescq->polled_completions > (cq_size / 2)) || + (nescq->polled_completions == 255)) { + nes_debug(NES_DBG_CQ, "CQ%u Issuing CQE Allocate since more than half of cqes" + " are pending %u of %u.\n", + nescq->hw_cq.cq_number, nescq->polled_completions, cq_size); + nes_write32(nesdev->regs+NES_CQE_ALLOC, + nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); + nescq->polled_completions = 0; + } } if (nescq->polled_completions) { -- 1.6.0 From donald.e.wood at intel.com Thu Jul 23 15:00:40 2009 From: donald.e.wood at intel.com (Don Wood) Date: Thu, 23 Jul 2009 17:00:40 -0500 Subject: [ofa-general] [PATCH] RDMA/nes: Add to cq error handling Message-ID: <20090723220040.GA4748@dewood-MOBL> A cq error is not being handled correctly. Put in the the upcall for a cq error. Signed-off-by: Don Wood --- drivers/infiniband/hw/nes/nes_hw.c | 12 ++++++++++++ 1 files changed, 12 insertions(+), 0 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c index 4a84d02..2a0c5a1 100644 --- a/drivers/infiniband/hw/nes/nes_hw.c +++ b/drivers/infiniband/hw/nes/nes_hw.c @@ -2913,6 +2913,8 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, u64 aeqe_context = 0; unsigned long flags; struct nes_qp *nesqp; + struct nes_hw_cq *hw_cq; + struct nes_cq *nescq; int resource_allocated; /* struct iw_cm_id *cm_id; */ struct nes_adapter *nesadapter = nesdev->nesadapter; @@ -3153,6 +3155,16 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, if (resource_allocated) { printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n", __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])); + hw_cq = (struct nes_hw_cq *)(unsigned long)context; + if (hw_cq) { + nescq = container_of(hw_cq, struct nes_cq, hw_cq); + if (nescq->ibcq.event_handler) { + ibevent.device = nescq->ibcq.device; + ibevent.event = IB_EVENT_CQ_ERR; + ibevent.element.cq = &nescq->ibcq; + nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context); + } + } } break; case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: -- 1.6.0 From donald.e.wood at intel.com Thu Jul 23 15:00:42 2009 From: donald.e.wood at intel.com (Don Wood) Date: Thu, 23 Jul 2009 17:00:42 -0500 Subject: [ofa-general] [PATCH 1/5] RDMA/nes: Implement Terminate Packet Message-ID: <20090723220042.GA2184@dewood-MOBL> Implement the sending and receiving of Terminate packets. Signed-off-by: Don Wood --- drivers/infiniband/hw/nes/nes.h | 2 +- drivers/infiniband/hw/nes/nes_cm.h | 2 - drivers/infiniband/hw/nes/nes_hw.c | 701 ++++++++++++++++++++++----------- drivers/infiniband/hw/nes/nes_hw.h | 91 +++++ drivers/infiniband/hw/nes/nes_utils.c | 3 + drivers/infiniband/hw/nes/nes_verbs.c | 26 ++- drivers/infiniband/hw/nes/nes_verbs.h | 7 + 7 files changed, 607 insertions(+), 225 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes.h b/drivers/infiniband/hw/nes/nes.h index bf1720f..bcc6abc 100644 --- a/drivers/infiniband/hw/nes/nes.h +++ b/drivers/infiniband/hw/nes/nes.h @@ -523,7 +523,7 @@ int nes_cm_disconn(struct nes_qp *); void nes_cm_disconn_worker(void *); /* nes_verbs.c */ -int nes_hw_modify_qp(struct nes_device *, struct nes_qp *, u32, u32); +int nes_hw_modify_qp(struct nes_device *, struct nes_qp *, u32, u32, u32); int nes_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *); struct nes_ib_device *nes_init_ofa_device(struct net_device *); void nes_destroy_ofa_device(struct nes_ib_device *); diff --git a/drivers/infiniband/hw/nes/nes_cm.h b/drivers/infiniband/hw/nes/nes_cm.h index 8b7e7c0..90e8e4d 100644 --- a/drivers/infiniband/hw/nes/nes_cm.h +++ b/drivers/infiniband/hw/nes/nes_cm.h @@ -410,8 +410,6 @@ struct nes_cm_ops { int schedule_nes_timer(struct nes_cm_node *, struct sk_buff *, enum nes_timer_type, int, int); -int nes_cm_disconn(struct nes_qp *); - int nes_accept(struct iw_cm_id *, struct iw_cm_conn_param *); int nes_reject(struct iw_cm_id *, const void *, u8); int nes_connect(struct iw_cm_id *, struct iw_cm_conn_param *); diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c index 2a0c5a1..d382dda 100644 --- a/drivers/infiniband/hw/nes/nes_hw.c +++ b/drivers/infiniband/hw/nes/nes_hw.c @@ -74,6 +74,8 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, static void process_critical_error(struct nes_device *nesdev); static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number); static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode); +static void nes_terminate_timeout(unsigned long context); +static void nes_terminate_start_timer(struct nes_qp *nesqp); #ifdef CONFIG_INFINIBAND_NES_DEBUG static unsigned char *nes_iwarp_state_str[] = { @@ -2903,6 +2905,383 @@ static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq) } +static u8 *locate_mpa(u8 *pkt, u32 aeq_info) +{ + u16 pkt_len; + + if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) { + /* skip over ethernet header */ + pkt_len = be16_to_cpu(*(u16 *)(pkt + ETH_HLEN - 2)); + pkt += ETH_HLEN; + + /* Skip over IP and TCP headers */ + pkt += 4 * (pkt[0] & 0x0f); + pkt += 4 * ((pkt[12] >> 4) & 0x0f); + } + return pkt; +} + +/* Determine if incoming error pkt is rdma layer */ +static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info) +{ + u8 *pkt; + u16 *mpa; + u32 opcode = 0xffffffff; + + if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) { + pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET; + mpa = (u16 *)locate_mpa(pkt, aeq_info); + opcode = be16_to_cpu(mpa[1]) & 0xf; + } + + return opcode; +} + +/* Build iWARP terminate header */ +static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info) +{ + u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET; + u16 ddp_seg_len; + int copy_len = 0; + u8 is_tagged = 0; + struct nes_terminate_hdr *termhdr; + + termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase; + memset(termhdr, 0, 64); + + if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) { + + /* Use data from offending packet to fill in ddp & rdma hdrs */ + pkt = locate_mpa(pkt, aeq_info); + ddp_seg_len = be16_to_cpu(*(u16 *)pkt); + if (ddp_seg_len) { + copy_len = 2; + termhdr->hdrct = DDP_LEN_FLAG; + if (pkt[2] & 0x80) { + is_tagged = 1; + if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) { + copy_len += TERM_DDP_LEN_TAGGED; + termhdr->hdrct |= DDP_HDR_FLAG; + } + } else { + if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) { + copy_len += TERM_DDP_LEN_UNTAGGED; + termhdr->hdrct |= DDP_HDR_FLAG; + } + + if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) { + if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) { + copy_len += TERM_RDMA_LEN; + termhdr->hdrct |= RDMA_HDR_FLAG; + } + } + } + } + } + + switch (async_event_id) { + case NES_AEQE_AEID_AMP_UNALLOCATED_STAG: + switch (iwarp_opcode(nesqp, aeq_info)) { + case IWARP_OPCODE_WRITE: + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; + termhdr->error_code = DDP_TAGGED_INV_STAG; + break; + default: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; + termhdr->error_code = RDMAP_INV_STAG; + } + break; + case NES_AEQE_AEID_AMP_INVALID_STAG: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; + termhdr->error_code = RDMAP_INV_STAG; + break; + case NES_AEQE_AEID_AMP_BAD_QP: + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; + termhdr->error_code = DDP_UNTAGGED_INV_QN; + break; + case NES_AEQE_AEID_AMP_BAD_STAG_KEY: + case NES_AEQE_AEID_AMP_BAD_STAG_INDEX: + switch (iwarp_opcode(nesqp, aeq_info)) { + case IWARP_OPCODE_SEND_INV: + case IWARP_OPCODE_SEND_SE_INV: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; + termhdr->error_code = RDMAP_CANT_INV_STAG; + break; + default: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; + termhdr->error_code = RDMAP_INV_STAG; + } + break; + case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION: + if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) { + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; + termhdr->error_code = DDP_TAGGED_BOUNDS; + } else { + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; + termhdr->error_code = RDMAP_INV_BOUNDS; + } + break; + case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION: + case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: + case NES_AEQE_AEID_PRIV_OPERATION_DENIED: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; + termhdr->error_code = RDMAP_ACCESS; + break; + case NES_AEQE_AEID_AMP_TO_WRAP: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; + termhdr->error_code = RDMAP_TO_WRAP; + break; + case NES_AEQE_AEID_AMP_BAD_PD: + switch (iwarp_opcode(nesqp, aeq_info)) { + case IWARP_OPCODE_WRITE: + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; + termhdr->error_code = DDP_TAGGED_UNASSOC_STAG; + break; + case IWARP_OPCODE_SEND_INV: + case IWARP_OPCODE_SEND_SE_INV: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; + termhdr->error_code = RDMAP_CANT_INV_STAG; + break; + default: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; + termhdr->error_code = RDMAP_UNASSOC_STAG; + } + break; + case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH: + termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP; + termhdr->error_code = MPA_MARKER; + break; + case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR: + termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP; + termhdr->error_code = MPA_CRC; + break; + case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE: + case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL: + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC; + termhdr->error_code = DDP_CATASTROPHIC_LOCAL; + break; + case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC: + case NES_AEQE_AEID_DDP_NO_L_BIT: + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC; + termhdr->error_code = DDP_CATASTROPHIC_LOCAL; + break; + case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN: + case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID: + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; + termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE; + break; + case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; + termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG; + break; + case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION: + if (is_tagged) { + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; + termhdr->error_code = DDP_TAGGED_INV_DDP_VER; + } else { + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; + termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER; + } + break; + case NES_AEQE_AEID_DDP_UBE_INVALID_MO: + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; + termhdr->error_code = DDP_UNTAGGED_INV_MO; + break; + case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; + termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF; + break; + case NES_AEQE_AEID_DDP_UBE_INVALID_QN: + termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; + termhdr->error_code = DDP_UNTAGGED_INV_QN; + break; + case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; + termhdr->error_code = RDMAP_INV_RDMAP_VER; + break; + case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; + termhdr->error_code = RDMAP_UNEXPECTED_OP; + break; + default: + termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; + termhdr->error_code = RDMAP_UNSPECIFIED; + break; + } + + if (copy_len) + memcpy(termhdr + 1, pkt, copy_len); + + return sizeof(struct nes_terminate_hdr) + copy_len; +} + +static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp, + struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype) +{ + u64 context; + unsigned long flags; + u32 aeq_info; + u16 async_event_id; + u8 tcp_state; + u8 iwarp_state; + u32 termlen = 0; + u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE | + NES_CQP_QP_TERM_DONT_SEND_FIN; + struct nes_adapter *nesadapter = nesdev->nesadapter; + + if (nesqp->term_flags & NES_TERM_SENT) + return; /* Sanity check */ + + aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); + tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT; + iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT; + async_event_id = (u16)aeq_info; + + context = (unsigned long)nesadapter->qp_table[le32_to_cpu( + aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN]; + if (!context) { + WARN_ON(!context); + return; + } + + nesqp = (struct nes_qp *)(unsigned long)context; + spin_lock_irqsave(&nesqp->lock, flags); + nesqp->hw_iwarp_state = iwarp_state; + nesqp->hw_tcp_state = tcp_state; + nesqp->last_aeq = async_event_id; + nesqp->terminate_eventtype = eventtype; + spin_unlock_irqrestore(&nesqp->lock, flags); + + if (nesadapter->send_term_ok) + termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info); + else + mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG; + + nes_terminate_start_timer(nesqp); + nesqp->term_flags |= NES_TERM_SENT; + nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0); +} + +static void nes_terminate_send_fin(struct nes_device *nesdev, + struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe) +{ + u32 aeq_info; + u16 async_event_id; + u8 tcp_state; + u8 iwarp_state; + unsigned long flags; + + aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); + tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT; + iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT; + async_event_id = (u16)aeq_info; + + spin_lock_irqsave(&nesqp->lock, flags); + nesqp->hw_iwarp_state = iwarp_state; + nesqp->hw_tcp_state = tcp_state; + nesqp->last_aeq = async_event_id; + spin_unlock_irqrestore(&nesqp->lock, flags); + + /* Send the fin only */ + nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE | + NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0); +} + +/* Cleanup after a terminate sent or received */ +static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred) +{ + u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; + unsigned long flags; + struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device); + struct nes_device *nesdev = nesvnic->nesdev; + u8 first_time = 0; + + spin_lock_irqsave(&nesqp->lock, flags); + if (nesqp->hte_added) { + nesqp->hte_added = 0; + next_iwarp_state |= NES_CQP_QP_DEL_HTE; + } + + first_time = (nesqp->term_flags & NES_TERM_DONE) == 0; + nesqp->term_flags |= NES_TERM_DONE; + spin_unlock_irqrestore(&nesqp->lock, flags); + + /* Make sure we go through this only once */ + if (first_time) { + if (timeout_occurred == 0) + del_timer(&nesqp->terminate_timer); + else + next_iwarp_state |= NES_CQP_QP_RESET; + + nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0); + nes_cm_disconn(nesqp); + } +} + +static void nes_terminate_received(struct nes_device *nesdev, + struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe) +{ + u32 aeq_info; + u8 *pkt; + u32 *mpa; + u8 ddp_ctl; + u8 rdma_ctl; + u16 aeq_id = 0; + + aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); + if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) { + /* Terminate is not a performance path so the silicon */ + /* did not validate the frame - do it now */ + pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET; + mpa = (u32 *)locate_mpa(pkt, aeq_info); + ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff; + rdma_ctl = be32_to_cpu(mpa[0]) & 0xff; + if ((ddp_ctl & 0xc0) != 0x40) + aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC; + else if ((ddp_ctl & 0x03) != 1) + aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION; + else if (be32_to_cpu(mpa[2]) != 2) + aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN; + else if (be32_to_cpu(mpa[3]) != 1) + aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN; + else if (be32_to_cpu(mpa[4]) != 0) + aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO; + else if ((rdma_ctl & 0xc0) != 0x40) + aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION; + + if (aeq_id) { + /* Bad terminate recvd - send back a terminate */ + aeq_info = (aeq_info & 0xffff0000) | aeq_id; + aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info); + nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL); + return; + } + } + + nesqp->term_flags |= NES_TERM_RCVD; + nesqp->terminate_eventtype = IB_EVENT_QP_FATAL; + nes_terminate_start_timer(nesqp); + nes_terminate_send_fin(nesdev, nesqp, aeqe); +} + +/* Timeout routine in case terminate fails to complete */ +static void nes_terminate_timeout(unsigned long context) +{ + struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context; + + nes_terminate_done(nesqp, 1); +} + +/* Set a timer in case hw cannot complete the terminate sequence */ +static void nes_terminate_start_timer(struct nes_qp *nesqp) +{ + init_timer(&nesqp->terminate_timer); + nesqp->terminate_timer.function = nes_terminate_timeout; + nesqp->terminate_timer.expires = jiffies + HZ; + nesqp->terminate_timer.data = (unsigned long)nesqp; + add_timer(&nesqp->terminate_timer); +} + /** * nes_process_iwarp_aeqe */ @@ -2910,30 +3289,27 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, struct nes_hw_aeqe *aeqe) { u64 context; - u64 aeqe_context = 0; unsigned long flags; struct nes_qp *nesqp; struct nes_hw_cq *hw_cq; struct nes_cq *nescq; int resource_allocated; - /* struct iw_cm_id *cm_id; */ struct nes_adapter *nesadapter = nesdev->nesadapter; - struct ib_event ibevent; - /* struct iw_cm_event cm_event; */ u32 aeq_info; u32 next_iwarp_state = 0; u16 async_event_id; u8 tcp_state; u8 iwarp_state; + int must_disconn = 1; + int must_terminate = 0; + struct ib_event ibevent; nes_debug(NES_DBG_AEQ, "\n"); aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) { + if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) { context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]); context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32; } else { - aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]); - aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32; context = (unsigned long)nesadapter->qp_table[le32_to_cpu( aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN]; BUG_ON(!context); @@ -2950,7 +3326,11 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, switch (async_event_id) { case NES_AEQE_AEID_LLP_FIN_RECEIVED: - nesqp = *((struct nes_qp **)&context); + nesqp = (struct nes_qp *)(unsigned long)context; + + if (nesqp->term_flags) + return; /* Ignore it, wait for close complete */ + if (atomic_inc_return(&nesqp->close_timer_started) == 1) { nesqp->cm_id->add_ref(nesqp->cm_id); schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp, @@ -2961,18 +3341,24 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), async_event_id, nesqp->last_aeq, tcp_state); } + if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) || (nesqp->ibqp_state != IB_QPS_RTS)) { /* FIN Received but tcp state or IB state moved on, should expect a close complete */ return; } + case NES_AEQE_AEID_LLP_CLOSE_COMPLETE: + nesqp = (struct nes_qp *)(unsigned long)context; + if (nesqp->term_flags) { + nes_terminate_done(nesqp, 0); + return; + } + case NES_AEQE_AEID_LLP_CONNECTION_RESET: - case NES_AEQE_AEID_TERMINATE_SENT: - case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE: case NES_AEQE_AEID_RESET_SENT: - nesqp = *((struct nes_qp **)&context); + nesqp = (struct nes_qp *)(unsigned long)context; if (async_event_id == NES_AEQE_AEID_RESET_SENT) { tcp_state = NES_AEQE_TCP_STATE_CLOSED; } @@ -2984,12 +3370,7 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) || (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) { nesqp->hte_added = 0; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n", - nesqp->hwqp.qp_id); - nes_hw_modify_qp(nesdev, nesqp, - NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0); - spin_lock_irqsave(&nesqp->lock, flags); + next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE; } if ((nesqp->ibqp_state == IB_QPS_RTS) && @@ -3001,151 +3382,106 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING; break; case NES_AEQE_IWARP_STATE_TERMINATE: - next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE; - if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) { - next_iwarp_state |= 0x02000000; - nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED; - } + must_disconn = 0; /* terminate path takes care of disconn */ + if (nesqp->term_flags == 0) + must_terminate = 1; break; - default: - next_iwarp_state = 0; - } - spin_unlock_irqrestore(&nesqp->lock, flags); - if (next_iwarp_state) { - nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X," - " also added another reference\n", - nesqp->hwqp.qp_id, next_iwarp_state); - nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0); } - nes_cm_disconn(nesqp); } else { if (async_event_id == NES_AEQE_AEID_LLP_FIN_RECEIVED) { /* FIN Received but ib state not RTS, close complete will be on its way */ - spin_unlock_irqrestore(&nesqp->lock, flags); - return; - } - spin_unlock_irqrestore(&nesqp->lock, flags); - if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) { - next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000; - nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED; - nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X," - " also added another reference\n", - nesqp->hwqp.qp_id, next_iwarp_state); - nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0); + must_disconn = 0; } - nes_cm_disconn(nesqp); } - break; - case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED: - nesqp = *((struct nes_qp **)&context); - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; spin_unlock_irqrestore(&nesqp->lock, flags); - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED" - " event on QP%u \n Q2 Data:\n", - nesqp->hwqp.qp_id); - if (nesqp->ibqp.event_handler) { - ibevent.device = nesqp->ibqp.device; - ibevent.element.qp = &nesqp->ibqp; - ibevent.event = IB_EVENT_QP_FATAL; - nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); - } - if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) || - ((nesqp->ibqp_state == IB_QPS_RTS)&& - (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) { + + if (must_terminate) + nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL); + else if (must_disconn) { + if (next_iwarp_state) { + nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X\n" + nesqp->hwqp.qp_id, next_iwarp_state); + nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0); + } nes_cm_disconn(nesqp); - } else { - nesqp->in_disconnect = 0; - wake_up(&nesqp->kick_waitq); } break; - case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES: - nesqp = *((struct nes_qp **)&context); - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR; - nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED; - nesqp->last_aeq = async_event_id; - if (nesqp->cm_id) { - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES" - " event on QP%u, remote IP = 0x%08X \n", - nesqp->hwqp.qp_id, - ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr)); - } else { - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES" - " event on QP%u \n", - nesqp->hwqp.qp_id); - } - spin_unlock_irqrestore(&nesqp->lock, flags); - next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET; - nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0); - if (nesqp->ibqp.event_handler) { - ibevent.device = nesqp->ibqp.device; - ibevent.element.qp = &nesqp->ibqp; - ibevent.event = IB_EVENT_QP_FATAL; - nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); - } + + case NES_AEQE_AEID_TERMINATE_SENT: + nesqp = (struct nes_qp *)(unsigned long)context; + nes_terminate_send_fin(nesdev, nesqp, aeqe); break; - case NES_AEQE_AEID_AMP_BAD_STAG_INDEX: - if (NES_AEQE_INBOUND_RDMA&aeq_info) { - nesqp = nesadapter->qp_table[le32_to_cpu( - aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN]; - } else { - /* TODO: get the actual WQE and mask off wqe index */ - context &= ~((u64)511); - nesqp = *((struct nes_qp **)&context); - } - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n", - nesqp->hwqp.qp_id); - if (nesqp->ibqp.event_handler) { - ibevent.device = nesqp->ibqp.device; - ibevent.element.qp = &nesqp->ibqp; - ibevent.event = IB_EVENT_QP_ACCESS_ERR; - nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); - } + + case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED: + nesqp = (struct nes_qp *)(unsigned long)context; + nes_terminate_received(nesdev, nesqp, aeqe); break; + + case NES_AEQE_AEID_AMP_BAD_STAG_KEY: + case NES_AEQE_AEID_AMP_BAD_STAG_INDEX: case NES_AEQE_AEID_AMP_UNALLOCATED_STAG: - nesqp = *((struct nes_qp **)&context); - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n", - nesqp->hwqp.qp_id); - if (nesqp->ibqp.event_handler) { - ibevent.device = nesqp->ibqp.device; - ibevent.element.qp = &nesqp->ibqp; - ibevent.event = IB_EVENT_QP_ACCESS_ERR; - nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); - } - break; + case NES_AEQE_AEID_AMP_INVALID_STAG: + case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION: + case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: case NES_AEQE_AEID_PRIV_OPERATION_DENIED: - nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words - [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN]; - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u," - " nesqp = %p, AE reported %p\n", - nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context)); - if (nesqp->ibqp.event_handler) { - ibevent.device = nesqp->ibqp.device; - ibevent.element.qp = &nesqp->ibqp; - ibevent.event = IB_EVENT_QP_ACCESS_ERR; - nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); + case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: + case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION: + case NES_AEQE_AEID_AMP_TO_WRAP: + nesqp = (struct nes_qp *)(unsigned long)context; + nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR); + break; + + case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE: + case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL: + case NES_AEQE_AEID_DDP_UBE_INVALID_MO: + case NES_AEQE_AEID_DDP_UBE_INVALID_QN: + nesqp = (struct nes_qp *)(unsigned long)context; + if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) { + aeq_info &= 0xffff0000; + aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE; + aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info); } + + case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE: + case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES: + case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: + case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR: + case NES_AEQE_AEID_AMP_BAD_QP: + case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH: + case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC: + case NES_AEQE_AEID_DDP_NO_L_BIT: + case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN: + case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID: + case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION: + case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION: + case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE: + case NES_AEQE_AEID_AMP_BAD_PD: + case NES_AEQE_AEID_AMP_FASTREG_SHARED: + case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG: + case NES_AEQE_AEID_AMP_FASTREG_MW_STAG: + case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS: + case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW: + case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH: + case NES_AEQE_AEID_AMP_INVALIDATE_SHARED: + case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS: + case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG: + case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG: + case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG: + case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG: + case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS: + case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS: + case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT: + case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED: + case NES_AEQE_AEID_BAD_CLOSE: + case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO: + case NES_AEQE_AEID_STAG_ZERO_INVALID: + case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST: + case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP: + nesqp = (struct nes_qp *)(unsigned long)context; + nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL); break; + case NES_AEQE_AEID_CQ_OPERATION_ERROR: context <<= 1; nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n", @@ -3167,81 +3503,7 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, } } break; - case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: - nesqp = nesadapter->qp_table[le32_to_cpu( - aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN]; - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG" - "_FOR_AVAILABLE_BUFFER event on QP%u\n", - nesqp->hwqp.qp_id); - if (nesqp->ibqp.event_handler) { - ibevent.device = nesqp->ibqp.device; - ibevent.element.qp = &nesqp->ibqp; - ibevent.event = IB_EVENT_QP_ACCESS_ERR; - nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); - } - /* tell cm to disconnect, cm will queue work to thread */ - nes_cm_disconn(nesqp); - break; - case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: - nesqp = *((struct nes_qp **)&context); - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN" - "_NO_BUFFER_AVAILABLE event on QP%u\n", - nesqp->hwqp.qp_id); - if (nesqp->ibqp.event_handler) { - ibevent.device = nesqp->ibqp.device; - ibevent.element.qp = &nesqp->ibqp; - ibevent.event = IB_EVENT_QP_FATAL; - nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); - } - /* tell cm to disconnect, cm will queue work to thread */ - nes_cm_disconn(nesqp); - break; - case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR: - nesqp = *((struct nes_qp **)&context); - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR" - " event on QP%u \n Q2 Data:\n", - nesqp->hwqp.qp_id); - if (nesqp->ibqp.event_handler) { - ibevent.device = nesqp->ibqp.device; - ibevent.element.qp = &nesqp->ibqp; - ibevent.event = IB_EVENT_QP_FATAL; - nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); - } - /* tell cm to disconnect, cm will queue work to thread */ - nes_cm_disconn(nesqp); - break; - /* TODO: additional AEs need to be here */ - case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION: - nesqp = *((struct nes_qp **)&context); - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - if (nesqp->ibqp.event_handler) { - ibevent.device = nesqp->ibqp.device; - ibevent.element.qp = &nesqp->ibqp; - ibevent.event = IB_EVENT_QP_ACCESS_ERR; - nesqp->ibqp.event_handler(&ibevent, - nesqp->ibqp.qp_context); - } - nes_cm_disconn(nesqp); - break; + default: nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n", async_event_id); @@ -3250,7 +3512,6 @@ static void nes_process_iwarp_aeqe(struct nes_device *nesdev, } - /** * nes_iwarp_ce_handler */ diff --git a/drivers/infiniband/hw/nes/nes_hw.h b/drivers/infiniband/hw/nes/nes_hw.h index c3654c6..4a0bfcd 100644 --- a/drivers/infiniband/hw/nes/nes_hw.h +++ b/drivers/infiniband/hw/nes/nes_hw.h @@ -241,6 +241,7 @@ enum nes_cqp_stag_wqeword_idx { }; #define NES_CQP_OP_IWARP_STATE_SHIFT 28 +#define NES_CQP_OP_TERMLEN_SHIFT 28 enum nes_cqp_qp_bits { NES_CQP_QP_ARP_VALID = (1<<8), @@ -265,6 +266,8 @@ enum nes_cqp_qp_bits { NES_CQP_QP_IWARP_STATE_TERMINATE = (5< 21)) || ((major_ver > 2) && (major_ver != 255))) { nesadapter->virtwq = 1; } + if (((major_ver == 3) && (minor_ver >= 16)) || (major_ver > 3)) + nesadapter->send_term_ok = 1; + nesadapter->firmware_version = (((u32)(u8)(eeprom_data>>8)) << 16) + (u32)((u8)eeprom_data); diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index c6b5873..36666ac 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -2923,7 +2923,7 @@ static int nes_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, * nes_hw_modify_qp */ int nes_hw_modify_qp(struct nes_device *nesdev, struct nes_qp *nesqp, - u32 next_iwarp_state, u32 wait_completion) + u32 next_iwarp_state, u32 termlen, u32 wait_completion) { struct nes_hw_cqp_wqe *cqp_wqe; /* struct iw_cm_id *cm_id = nesqp->cm_id; */ @@ -2955,6 +2955,13 @@ int nes_hw_modify_qp(struct nes_device *nesdev, struct nes_qp *nesqp, set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, (u64)nesqp->nesqp_context_pbase); + /* If sending a terminate message, fill in the length (in words) */ + if (((next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK) == NES_CQP_QP_IWARP_STATE_TERMINATE) && + !(next_iwarp_state & NES_CQP_QP_TERM_DONT_SEND_TERM_MSG)) { + termlen = ((termlen + 3) >> 2) << NES_CQP_OP_TERMLEN_SHIFT; + set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_NEW_MSS_IDX, termlen); + } + atomic_set(&cqp_request->refcount, 2); nes_post_cqp_request(nesdev, cqp_request); @@ -3125,6 +3132,9 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, } nes_debug(NES_DBG_MOD_QP, "QP%u: new state = error\n", nesqp->hwqp.qp_id); + if (nesqp->term_flags) + del_timer(&nesqp->terminate_timer); + next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */ if (nesqp->hte_added) { @@ -3202,7 +3212,7 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (issue_modify_qp) { nes_debug(NES_DBG_MOD_QP, "call nes_hw_modify_qp\n"); - ret = nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 1); + ret = nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 1); if (ret) nes_debug(NES_DBG_MOD_QP, "nes_hw_modify_qp (next_iwarp_state = 0x%08X)" " failed for QP%u.\n", @@ -3367,6 +3377,12 @@ static int nes_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr, head = nesqp->hwqp.sq_head; while (ib_wr) { + /* Check for QP error */ + if (nesqp->term_flags) { + err = -EINVAL; + break; + } + /* Check for SQ overflow */ if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) { err = -EINVAL; @@ -3523,6 +3539,12 @@ static int nes_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr, head = nesqp->hwqp.rq_head; while (ib_wr) { + /* Check for QP error */ + if (nesqp->term_flags) { + err = -EINVAL; + break; + } + if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { err = -EINVAL; break; diff --git a/drivers/infiniband/hw/nes/nes_verbs.h b/drivers/infiniband/hw/nes/nes_verbs.h index 7df34fe..d92b1ef 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.h +++ b/drivers/infiniband/hw/nes/nes_verbs.h @@ -40,6 +40,10 @@ struct nes_device; #define NES_MAX_USER_DB_REGIONS 4096 #define NES_MAX_USER_WQ_REGIONS 4096 +#define NES_TERM_SENT 0x01 +#define NES_TERM_RCVD 0x02 +#define NES_TERM_DONE 0x04 + struct nes_ucontext { struct ib_ucontext ibucontext; struct nes_device *nesdev; @@ -159,6 +163,8 @@ struct nes_qp { void *pbl_vbase; dma_addr_t pbl_pbase; struct page *page; + struct timer_list terminate_timer; + enum ib_event_type terminate_eventtype; wait_queue_head_t kick_waitq; u16 in_disconnect; u16 private_data_len; @@ -169,6 +175,7 @@ struct nes_qp { u8 hw_iwarp_state; u8 flush_issued; u8 hw_tcp_state; + u8 term_flags; u8 destroyed; }; #endif /* NES_VERBS_H */ -- 1.6.0 From donald.e.wood at intel.com Thu Jul 23 15:00:45 2009 From: donald.e.wood at intel.com (Don Wood) Date: Thu, 23 Jul 2009 17:00:45 -0500 Subject: [ofa-general] [PATCH 2/5] RDMA/nes: Use flush mechanism to set status for wqe in error Message-ID: <20090723220045.GA6480@dewood-MOBL> When an asynchronous event occurs that requires a terminate, it is sometimes possible to identify the wqe in error. This change uses flush to get this information to the poll routine. The flush operation puts the status into the cqe. If this information is not available, it continues to use the more generic flush code as before. Signed-off-by: Don Wood --- drivers/infiniband/hw/nes/nes_hw.c | 54 +++++++++++++++++++++++++++++++++ drivers/infiniband/hw/nes/nes_hw.h | 12 +++++++ drivers/infiniband/hw/nes/nes_verbs.h | 2 + 3 files changed, 68 insertions(+), 0 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c index d382dda..933e009 100644 --- a/drivers/infiniband/hw/nes/nes_hw.c +++ b/drivers/infiniband/hw/nes/nes_hw.c @@ -2944,6 +2944,7 @@ static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 u16 ddp_seg_len; int copy_len = 0; u8 is_tagged = 0; + u8 flush_code = 0; struct nes_terminate_hdr *termhdr; termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase; @@ -2983,19 +2984,23 @@ static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG: switch (iwarp_opcode(nesqp, aeq_info)) { case IWARP_OPCODE_WRITE: + flush_code = IB_WC_LOC_PROT_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; termhdr->error_code = DDP_TAGGED_INV_STAG; break; default: + flush_code = IB_WC_REM_ACCESS_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; termhdr->error_code = RDMAP_INV_STAG; } break; case NES_AEQE_AEID_AMP_INVALID_STAG: + flush_code = IB_WC_REM_ACCESS_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; termhdr->error_code = RDMAP_INV_STAG; break; case NES_AEQE_AEID_AMP_BAD_QP: + flush_code = IB_WC_LOC_QP_OP_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; termhdr->error_code = DDP_UNTAGGED_INV_QN; break; @@ -3004,19 +3009,23 @@ static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 switch (iwarp_opcode(nesqp, aeq_info)) { case IWARP_OPCODE_SEND_INV: case IWARP_OPCODE_SEND_SE_INV: + flush_code = IB_WC_REM_OP_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; termhdr->error_code = RDMAP_CANT_INV_STAG; break; default: + flush_code = IB_WC_REM_ACCESS_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; termhdr->error_code = RDMAP_INV_STAG; } break; case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION: if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) { + flush_code = IB_WC_LOC_PROT_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; termhdr->error_code = DDP_TAGGED_BOUNDS; } else { + flush_code = IB_WC_REM_ACCESS_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; termhdr->error_code = RDMAP_INV_BOUNDS; } @@ -3024,57 +3033,69 @@ static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION: case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: case NES_AEQE_AEID_PRIV_OPERATION_DENIED: + flush_code = IB_WC_REM_ACCESS_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; termhdr->error_code = RDMAP_ACCESS; break; case NES_AEQE_AEID_AMP_TO_WRAP: + flush_code = IB_WC_REM_ACCESS_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; termhdr->error_code = RDMAP_TO_WRAP; break; case NES_AEQE_AEID_AMP_BAD_PD: switch (iwarp_opcode(nesqp, aeq_info)) { case IWARP_OPCODE_WRITE: + flush_code = IB_WC_LOC_PROT_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; termhdr->error_code = DDP_TAGGED_UNASSOC_STAG; break; case IWARP_OPCODE_SEND_INV: case IWARP_OPCODE_SEND_SE_INV: + flush_code = IB_WC_REM_ACCESS_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; termhdr->error_code = RDMAP_CANT_INV_STAG; break; default: + flush_code = IB_WC_REM_ACCESS_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; termhdr->error_code = RDMAP_UNASSOC_STAG; } break; case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH: + flush_code = IB_WC_LOC_LEN_ERR; termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP; termhdr->error_code = MPA_MARKER; break; case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR: + flush_code = IB_WC_GENERAL_ERR; termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP; termhdr->error_code = MPA_CRC; break; case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE: case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL: + flush_code = IB_WC_LOC_LEN_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC; termhdr->error_code = DDP_CATASTROPHIC_LOCAL; break; case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC: case NES_AEQE_AEID_DDP_NO_L_BIT: + flush_code = IB_WC_FATAL_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC; termhdr->error_code = DDP_CATASTROPHIC_LOCAL; break; case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN: case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID: + flush_code = IB_WC_GENERAL_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE; break; case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: + flush_code = IB_WC_LOC_LEN_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG; break; case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION: + flush_code = IB_WC_GENERAL_ERR; if (is_tagged) { termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; termhdr->error_code = DDP_TAGGED_INV_DDP_VER; @@ -3084,26 +3105,32 @@ static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 } break; case NES_AEQE_AEID_DDP_UBE_INVALID_MO: + flush_code = IB_WC_GENERAL_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; termhdr->error_code = DDP_UNTAGGED_INV_MO; break; case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: + flush_code = IB_WC_REM_OP_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF; break; case NES_AEQE_AEID_DDP_UBE_INVALID_QN: + flush_code = IB_WC_GENERAL_ERR; termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; termhdr->error_code = DDP_UNTAGGED_INV_QN; break; case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION: + flush_code = IB_WC_GENERAL_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; termhdr->error_code = RDMAP_INV_RDMAP_VER; break; case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE: + flush_code = IB_WC_LOC_QP_OP_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; termhdr->error_code = RDMAP_UNEXPECTED_OP; break; default: + flush_code = IB_WC_FATAL_ERR; termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; termhdr->error_code = RDMAP_UNSPECIFIED; break; @@ -3112,6 +3139,13 @@ static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 if (copy_len) memcpy(termhdr + 1, pkt, copy_len); + if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) { + if (aeq_info & NES_AEQE_SQ) + nesqp->term_sq_flush_code = flush_code; + else + nesqp->term_rq_flush_code = flush_code; + } + return sizeof(struct nes_terminate_hdr) + copy_len; } @@ -3646,6 +3680,8 @@ void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp, { struct nes_cqp_request *cqp_request; struct nes_hw_cqp_wqe *cqp_wqe; + u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH; + u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH; int ret; cqp_request = nes_get_cqp_request(nesdev); @@ -3662,6 +3698,24 @@ void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp, cqp_wqe = &cqp_request->cqp_wqe; nes_fill_init_cqp_wqe(cqp_wqe, nesdev); + /* If wqe in error was identified, set code to be put into cqe */ + if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) { + which_wq |= NES_CQP_FLUSH_MAJ_MIN; + sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code; + nesqp->term_sq_flush_code = 0; + } + + if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) { + which_wq |= NES_CQP_FLUSH_MAJ_MIN; + rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code; + nesqp->term_rq_flush_code = 0; + } + + if (which_wq & NES_CQP_FLUSH_MAJ_MIN) { + cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code); + cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code); + } + cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq); cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id); diff --git a/drivers/infiniband/hw/nes/nes_hw.h b/drivers/infiniband/hw/nes/nes_hw.h index 4a0bfcd..f28a41b 100644 --- a/drivers/infiniband/hw/nes/nes_hw.h +++ b/drivers/infiniband/hw/nes/nes_hw.h @@ -274,6 +274,8 @@ enum nes_cqp_qp_bits { enum nes_cqp_qp_wqe_word_idx { NES_CQP_QP_WQE_CONTEXT_LOW_IDX = 6, NES_CQP_QP_WQE_CONTEXT_HIGH_IDX = 7, + NES_CQP_QP_WQE_FLUSH_SQ_CODE = 8, + NES_CQP_QP_WQE_FLUSH_RQ_CODE = 9, NES_CQP_QP_WQE_NEW_MSS_IDX = 15, }; @@ -364,6 +366,7 @@ enum nes_cqp_arp_bits { enum nes_cqp_flush_bits { NES_CQP_FLUSH_SQ = (1<<30), NES_CQP_FLUSH_RQ = (1<<31), + NES_CQP_FLUSH_MAJ_MIN = (1<<28), }; enum nes_cqe_opcode_bits { @@ -757,6 +760,15 @@ enum nes_iwarp_sq_wqe_bits { NES_IWARP_SQ_OP_NOP = 12, }; +enum nes_iwarp_cqe_major_code { + NES_IWARP_CQE_MAJOR_FLUSH = 1, + NES_IWARP_CQE_MAJOR_DRV = 0x8000 +}; + +enum nes_iwarp_cqe_minor_code { + NES_IWARP_CQE_MINOR_FLUSH = 1 +}; + #define NES_EEPROM_READ_REQUEST (1<<16) #define NES_MAC_ADDR_VALID (1<<20) diff --git a/drivers/infiniband/hw/nes/nes_verbs.h b/drivers/infiniband/hw/nes/nes_verbs.h index d92b1ef..89822d7 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.h +++ b/drivers/infiniband/hw/nes/nes_verbs.h @@ -168,6 +168,8 @@ struct nes_qp { wait_queue_head_t kick_waitq; u16 in_disconnect; u16 private_data_len; + u16 term_sq_flush_code; + u16 term_rq_flush_code; u8 active_conn; u8 skip_lsmm; u8 user_mode; -- 1.6.0 From donald.e.wood at intel.com Thu Jul 23 15:00:47 2009 From: donald.e.wood at intel.com (Don Wood) Date: Thu, 23 Jul 2009 17:00:47 -0500 Subject: [ofa-general] [PATCH 3/5] RDMA/nes: Make poll_cq return correct number of wqes during flush Message-ID: <20090723220047.GA4860@dewood-MOBL> When a flush request is given to the hw, it will place one cqe marked as flushed (unless there is nothing to flush). An application that is waiting for all wqe's to complete will be left hanging. This modifies poll_cq to return the correct number of flushes for the pending elements on the wq. Signed-off-by: Don Wood --- drivers/infiniband/hw/nes/nes_verbs.c | 61 +++++++++++++++++++++----------- 1 files changed, 40 insertions(+), 21 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index 36666ac..ad3c891 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -3615,11 +3615,13 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) struct nes_qp *nesqp; struct nes_hw_cqe cqe; u32 head; - u32 wq_tail; + u32 wq_tail = 0; u32 cq_size; u32 cqe_count = 0; u32 wqe_index; u32 u32temp; + u32 move_cq_head = 1; + u32 err_code; nes_debug(NES_DBG_CQ, "\n"); @@ -3640,7 +3642,6 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) rmb(); cqe = nescq->hw_cq.cq_vbase[head]; - nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; u32temp = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); wqe_index = u32temp & (nesdev->nesadapter->max_qp_wr - 1); u32temp &= ~(NES_SW_CONTEXT_ALIGN-1); @@ -3667,16 +3668,14 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) } /* Working on a SQ Completion*/ - wq_tail = wqe_index; - nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1); - wrid = (((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wq_tail]. + wrid = (((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index]. wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) | - ((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wq_tail]. + ((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index]. wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX]))); - entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wq_tail]. + entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]); - switch (le32_to_cpu(nesqp->hwqp.sq_vbase[wq_tail]. + switch (le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) { case NES_IWARP_SQ_OP_RDMAW: nes_debug(NES_DBG_CQ, "Operation = RDMA WRITE.\n"); @@ -3685,7 +3684,7 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) case NES_IWARP_SQ_OP_RDMAR: nes_debug(NES_DBG_CQ, "Operation = RDMA READ.\n"); entry->opcode = IB_WC_RDMA_READ; - entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wq_tail]. + entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]); break; case NES_IWARP_SQ_OP_SENDINV: @@ -3696,14 +3695,24 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) entry->opcode = IB_WC_SEND; break; } + + nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1); + if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head)) { + move_cq_head = 0; + wq_tail = nesqp->hwqp.sq_tail; + } } else { /* Working on a RQ Completion*/ - wq_tail = wqe_index; - nesqp->hwqp.rq_tail = (wqe_index+1)&(nesqp->hwqp.rq_size - 1); entry->byte_len = le32_to_cpu(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]); - wrid = ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wq_tail].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX]))) | - ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wq_tail].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); + wrid = ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX]))) | + ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); entry->opcode = IB_WC_RECV; + + nesqp->hwqp.rq_tail = (wqe_index+1)&(nesqp->hwqp.rq_size - 1); + if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head)) { + move_cq_head = 0; + wq_tail = nesqp->hwqp.rq_tail; + } } entry->wr_id = wrid; @@ -3711,18 +3720,28 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) cqe_count++; } - if (++head >= cq_size) - head = 0; - nescq->polled_completions++; + if (move_cq_head) { + nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; + if (++head >= cq_size) + head = 0; + nescq->polled_completions++; - if ((nescq->polled_completions > (cq_size / 2)) || - (nescq->polled_completions == 255)) { - nes_debug(NES_DBG_CQ, "CQ%u Issuing CQE Allocate since more than half of cqes" + if ((nescq->polled_completions > (cq_size / 2)) || + (nescq->polled_completions == 255)) { + nes_debug(NES_DBG_CQ, "CQ%u Issuing CQE Allocate since more than half of cqes" " are pending %u of %u.\n", nescq->hw_cq.cq_number, nescq->polled_completions, cq_size); - nes_write32(nesdev->regs+NES_CQE_ALLOC, + nes_write32(nesdev->regs+NES_CQE_ALLOC, nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); - nescq->polled_completions = 0; + nescq->polled_completions = 0; + } + } else { + /* Update the wqe index and set status to flush */ + wqe_index = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); + wqe_index = (wqe_index & (~(nesdev->nesadapter->max_qp_wr - 1))) | wq_tail; + nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = + cpu_to_le32(wqe_index); + move_cq_head = 1; /* ready for next pass */ } } -- 1.6.0 From donald.e.wood at intel.com Thu Jul 23 15:00:49 2009 From: donald.e.wood at intel.com (Don Wood) Date: Thu, 23 Jul 2009 17:00:49 -0500 Subject: [ofa-general] [PATCH 4/5] RDMA/nes: Use the flush code to fill in cqe error Message-ID: <20090723220049.GA7844@dewood-MOBL> Use the flush status to fill in cqe status when a specific error has been identified. Subsequent flushed completions still use the flushed value. Signed-off-by: Don Wood --- drivers/infiniband/hw/nes/nes_verbs.c | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index ad3c891..993c1d4 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -3655,7 +3655,16 @@ static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) if (cqe.cqe_words[NES_CQE_ERROR_CODE_IDX] == 0) { entry->status = IB_WC_SUCCESS; } else { - entry->status = IB_WC_WR_FLUSH_ERR; + err_code = le32_to_cpu(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]); + if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16)) { + entry->status = err_code & 0x0000ffff; + + /* The rest of the cqe's will be marked as flushed */ + nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX] = + cpu_to_le32((NES_IWARP_CQE_MAJOR_FLUSH << 16) | + NES_IWARP_CQE_MINOR_FLUSH); + } else + entry->status = IB_WC_WR_FLUSH_ERR; } entry->qp = &nesqp->ibqp; -- 1.6.0 From donald.e.wood at intel.com Thu Jul 23 15:00:51 2009 From: donald.e.wood at intel.com (Don Wood) Date: Thu, 23 Jul 2009 17:00:51 -0500 Subject: [ofa-general] [PATCH 5/5] RDMA/nes: Rework the disconn routine for terminate and flushing Message-ID: <20090723220051.GA5304@dewood-MOBL> The disconn routine has been reworked to acoomodate the terminate and flushing changes. The routine has been reorganized to make all the decisions at the start then it performs all the required operations. This simplified the lock handling and is easier to follow. Signed-off-by: Don Wood --- drivers/infiniband/hw/nes/nes_cm.c | 102 ++++++++++++++++++----------------- 1 files changed, 52 insertions(+), 50 deletions(-) diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c index 3129ab1..1d16894 100644 --- a/drivers/infiniband/hw/nes/nes_cm.c +++ b/drivers/infiniband/hw/nes/nes_cm.c @@ -2493,7 +2493,12 @@ static int nes_cm_disconn_true(struct nes_qp *nesqp) u16 last_ae; u8 original_hw_tcp_state; u8 original_ibqp_state; - u8 issued_disconnect_reset = 0; + enum iw_cm_event_type disconn_status = IW_CM_EVENT_STATUS_OK; + int issue_disconn = 0; + int issue_close = 0; + int issue_flush = 0; + u32 flush_q = NES_CQP_FLUSH_RQ; + struct ib_event ibevent; if (!nesqp) { nes_debug(NES_DBG_CM, "disconnect_worker nesqp is NULL\n"); @@ -2517,24 +2522,55 @@ static int nes_cm_disconn_true(struct nes_qp *nesqp) original_ibqp_state = nesqp->ibqp_state; last_ae = nesqp->last_aeq; + if (nesqp->term_flags) { + issue_disconn = 1; + issue_close = 1; + nesqp->cm_id = NULL; + if (nesqp->flush_issued == 0) { + nesqp->flush_issued = 1; + issue_flush = 1; + } + } else if ((original_hw_tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) || + ((original_ibqp_state == IB_QPS_RTS) && + (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET))) { + issue_disconn = 1; + if (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET) + disconn_status = IW_CM_EVENT_STATUS_RESET; + } + + if (((original_hw_tcp_state == NES_AEQE_TCP_STATE_CLOSED) || + (original_hw_tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT) || + (last_ae == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) || + (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET))) { + issue_close = 1; + nesqp->cm_id = NULL; + if (nesqp->flush_issued == 0) { + nesqp->flush_issued = 1; + issue_flush = 1; + } + } + + spin_unlock_irqrestore(&nesqp->lock, flags); - nes_debug(NES_DBG_CM, "set ibqp_state=%u\n", nesqp->ibqp_state); + if ((issue_flush) && (nesqp->destroyed == 0)) { + /* Flush the queue(s) */ + if (nesqp->hw_iwarp_state >= NES_AEQE_IWARP_STATE_TERMINATE) + flush_q |= NES_CQP_FLUSH_SQ; + flush_wqes(nesvnic->nesdev, nesqp, flush_q, 1); - if ((nesqp->cm_id) && (cm_id->event_handler)) { - if ((original_hw_tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) || - ((original_ibqp_state == IB_QPS_RTS) && - (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET))) { + if (nesqp->term_flags) { + ibevent.device = nesqp->ibqp.device; + ibevent.event = nesqp->terminate_eventtype; + ibevent.element.qp = &nesqp->ibqp; + nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); + } + } + + if ((cm_id) && (cm_id->event_handler)) { + if (issue_disconn) { atomic_inc(&cm_disconnects); cm_event.event = IW_CM_EVENT_DISCONNECT; - if (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET) { - cm_event.status = IW_CM_EVENT_STATUS_RESET; - nes_debug(NES_DBG_CM, "Generating a CM " - "Disconnect Event (status reset) for " - "QP%u, cm_id = %p. \n", - nesqp->hwqp.qp_id, cm_id); - } else - cm_event.status = IW_CM_EVENT_STATUS_OK; - + cm_event.status = disconn_status; cm_event.local_addr = cm_id->local_addr; cm_event.remote_addr = cm_id->remote_addr; cm_event.private_data = NULL; @@ -2547,28 +2583,14 @@ static int nes_cm_disconn_true(struct nes_qp *nesqp) nesqp->hwqp.sq_tail, cm_id, atomic_read(&nesqp->refcount)); - spin_unlock_irqrestore(&nesqp->lock, flags); ret = cm_id->event_handler(cm_id, &cm_event); if (ret) nes_debug(NES_DBG_CM, "OFA CM event_handler " "returned, ret=%d\n", ret); - spin_lock_irqsave(&nesqp->lock, flags); } - /* There might have been another AE while the lock was released */ - original_hw_tcp_state = nesqp->hw_tcp_state; - original_ibqp_state = nesqp->ibqp_state; - last_ae = nesqp->last_aeq; - - if ((issued_disconnect_reset == 0) && (nesqp->cm_id) && - ((original_hw_tcp_state == NES_AEQE_TCP_STATE_CLOSED) || - (original_hw_tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT) || - (last_ae == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) || - (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET))) { + if (issue_close) { atomic_inc(&cm_closes); - nesqp->cm_id = NULL; - nesqp->in_disconnect = 0; - spin_unlock_irqrestore(&nesqp->lock, flags); nes_disconnect(nesqp, 1); cm_id->provider_data = nesqp; @@ -2587,27 +2609,7 @@ static int nes_cm_disconn_true(struct nes_qp *nesqp) } cm_id->rem_ref(cm_id); - - spin_lock_irqsave(&nesqp->lock, flags); - if (nesqp->flush_issued == 0) { - nesqp->flush_issued = 1; - spin_unlock_irqrestore(&nesqp->lock, flags); - flush_wqes(nesvnic->nesdev, nesqp, - NES_CQP_FLUSH_RQ, 1); - } else - spin_unlock_irqrestore(&nesqp->lock, flags); - } else { - cm_id = nesqp->cm_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - /* check to see if the inbound reset beat the outbound reset */ - if ((!cm_id) && (last_ae==NES_AEQE_AEID_RESET_SENT)) { - nes_debug(NES_DBG_CM, "QP%u: Decing refcount " - "due to inbound reset beating the " - "outbound reset.\n", nesqp->hwqp.qp_id); - } } - } else { - spin_unlock_irqrestore(&nesqp->lock, flags); } return 0; -- 1.6.0 From jgunthorpe at obsidianresearch.com Thu Jul 23 15:12:54 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Thu, 23 Jul 2009 16:12:54 -0600 Subject: [ofa-general] [PATCH] Allow paths to the device specific library to be absolute In-Reply-To: References: <20090714211653.GA13700@obsidianresearch.com> <20090717210817.GF7320@obsidianresearch.com> <20090722212505.GG12693@obsidianresearch.com> <20090722224506.GI12693@obsidianresearch.com> Message-ID: <20090723221254.GL12693@obsidianresearch.com> On Thu, Jul 23, 2009 at 02:17:17PM -0700, Roland Dreier wrote: > > > > I don't have a debian logon, but I do have an ia64 machine: > > > Ah, in the good old days every DD had one of those :) > > BTW I'm not a dd -- just have a sponsor for my packages. > Maybe someday I'll try to become a dd but for now I'm too lazy. Ah! Yes, that all changed some years ago. I got my DD-ship over an IRC conversation, these days you have to jump through hoops! Unfortunately I haven't used it in 8 years and the password is lost to me. > > The identifiers in an enumerator list are declared as constants that > > have type int and may appear wherever such are permitted. > > > > Each enumerated type shall be compatible with an integer type. The > > choice of type is implementation-defined, but shall be capable of > > representing the values of all the members of the enumeration. > > thinking about this, I confused myself. Given > > enum foo { FOO, BAR }; > > and given that the standard quoted above says FOO and BAR have type int, > why is it valid to do Well, this is messy. C++ and C are different. C++ strongly considers the enum constant to be of the enum type, and implicit casting from int to enum is not allowed. In C99 this is only weakly true (due to the legacy), and it is up to the compiler to optionally produce a warning if arbitary ints are cast to the enum (the other text I quoted from Annex I, Common Warnings) Jason From jgunthorpe at obsidianresearch.com Thu Jul 23 15:14:24 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Thu, 23 Jul 2009 16:14:24 -0600 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> <20090722204846.GF12693@obsidianresearch.com> <4A685D33.2050803@dev.mellanox.co.il> <20090723151939.GB1230@obsidianresearch.com> Message-ID: <20090723221424.GM12693@obsidianresearch.com> On Thu, Jul 23, 2009 at 05:43:38PM -0400, Hal Rosenstock wrote: > The proposed algorithm is less aggressive in terms of sending DR SMPs > than the current one which seems to work well enough based on field > experience. If the concern with this approach is breaking things over > the current approach, I suppose a config variable could be added for No, it just wasn't clear to me what the substantive difference was over the current approach. Reading your email made it seem like it was more agressive not less. Anyhow, I had no particularly strong objection just an observation that this could be done much better. Jason From hal.rosenstock at gmail.com Thu Jul 23 15:19:02 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Thu, 23 Jul 2009 18:19:02 -0400 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <20090723221424.GM12693@obsidianresearch.com> References: <20090721180312.GA12491@comcast.net> <20090721184446.GB12693@obsidianresearch.com> <20090722204846.GF12693@obsidianresearch.com> <4A685D33.2050803@dev.mellanox.co.il> <20090723151939.GB1230@obsidianresearch.com> <20090723221424.GM12693@obsidianresearch.com> Message-ID: On Thu, Jul 23, 2009 at 6:14 PM, Jason Gunthorpe wrote: > On Thu, Jul 23, 2009 at 05:43:38PM -0400, Hal Rosenstock wrote: > >> The proposed algorithm is less aggressive in terms of sending DR SMPs >> than the current one which seems to work well enough based on field >> experience. If the concern with this approach is breaking things over >> the current approach, I suppose a config variable could be added for > > No, it just wasn't clear to me what the substantive difference was > over the current approach. Reading your email made it seem like it was > more agressive not less. Sorry for the confusion. The proposed algorithm is less aggressive as it limits things based on the max_smps_per_node which defaults to 4 whereas the current algorithm in the tree will send all blocks. > Anyhow, I had no particularly strong objection Glad to hear this. > just an observation that this could be done much better. > > Jason -- Hal From jgunthorpe at obsidianresearch.com Thu Jul 23 15:44:10 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Thu, 23 Jul 2009 16:44:10 -0600 Subject: [ofa-general] Infiniband and Ubuntu In-Reply-To: <7AD18005-56B0-4598-8544-ABB05EC0E3E2@mit.edu> References: <7AD18005-56B0-4598-8544-ABB05EC0E3E2@mit.edu> Message-ID: <20090723224410.GN12693@obsidianresearch.com> On Thu, Jul 23, 2009 at 05:27:23PM -0400, Mark J. Pearrow wrote: > The short question is "is there a howto for getting Infiniband and Ubuntu > working?" I've done this and other similar things, it isn't too hard.. I like to use 2.6.30 these days for ConnectX, this is the base kernel for OFED 1.5 and has a few additional fixes over .28 that comes with Jaunty, but you can do that after getting the user space setup. I don't think there is anything fatal in .28.. When using stock kernels it is apparently best not to use OFED and its patches.. You are going to want to find the following tar files: infiniband-diags-1.5.1.tar.gz libibmad-1.3.1.tar.gz libmlx4-1.0.1.tar.gz libibcm-1.0.4.tar.gz libibumad-1.3.1.tar.gz librdmacm-1.0.8.tar.gz libibverbs-1.1.2.tar.gz opensm-3.3.1.tar.gz I usually just grab them from git rather that fussing: git://git.openfabrics.org/~shefty/libibcm.git git://git.kernel.org/pub/scm/libs/infiniband/libibverbs.git git://git.kernel.org/pub/scm/libs/infiniband/libmlx4.git git://git.openfabrics.org/~shefty/librdmacm.git git://git.openfabrics.org/~sashak/management.git [umad, mad, opensm and diags] I like to install the entire set in an opt directory. Do the following repeatedly: cd XXXX ./autogen.sh LDFLAGS=-L/opt/ofa-1.5/lib CPPFLAGS=-I/opt/ofa-1.5/include ./configure --prefix=/opt/ofa-1.5 make make install In this order: libumad libibmad opensm libibverbs (*) libmlx4 (*) libibcm librdmacm infiniband-diags The ones with a * are already available through ubuntu, but mlx4 should be updated to 1.0.1. You can use 'debian/rules binary' instead of the configure/make/make install sequence to produce a new binary .deb for these two. Once complete you'll have a /opt/ofa-1.5/ full of all the tools and libraries you need. Since the stock Jaunty kernel already includes the kernel side and pci driver autoloading does most of the work all you need is to modprobe a few additional things: ib_ucm ib_ipoib rdma_ucm Thats basically it for the core IB. Ah, one trick, depending on various factors you may not get rpath support out of libtool. I forget how to fix this up - You really want to build all of the above using rpath.. But barring that, you can do 'export LD_LIBRARY_PATH=/opt/ofa-1.5/lib' at the very start to make it all work out. The biggest problem was the ubuntu kernel version. You want at least 2.6.30 for ConnectX cards. Jaunty is only .28.. Oh, do the following to setup the ofa-1.5 directory to make it nice and tidy: mkdir -p /opt/ofa-1.5/share/man (cd /opt/ofa-1.5/ && ln -s share/man man) [Why the variance amongst the packages I have not tracked down] There is probably also a way to do the above, but use the included .spec files to make rpms and alien those to .debs but that has always been too much trouble for me. I'm sure lots of people would be grateful to have Debian packaging :) Jason From swise at opengridcomputing.com Thu Jul 23 18:29:16 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Thu, 23 Jul 2009 20:29:16 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <4A679CD7.mailLHZ11MTGE@systemfabricworks.com> References: <4A679CD7.mailLHZ11MTGE@systemfabricworks.com> Message-ID: <4A690E6C.4000001@opengridcomputing.com> Can't you just up the value passed into rdma_resolve_addr()? Currently this code passes in 2000 (ms). Did you try changing this to say 20000? davem at systemfabricworks.com wrote: > The timeouts in rdma_resolve_addr and rdma_resolve_route actually happen in > some fabrics, and there is no retry in rdma_cm so this patch adds process > level retries. A combined total of 10 retries for the pair is allowed. > > Signed-off-by: David A. McMillen > --- > rdma_bw.c | 16 ++++++++++++++++ > rdma_lat.c | 15 +++++++++++++++ > 2 files changed, 31 insertions(+), 0 deletions(-) > > diff --git a/rdma_bw.c b/rdma_bw.c > index 2628ac4..737558a 100755 > --- a/rdma_bw.c > +++ b/rdma_bw.c > @@ -131,6 +131,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) > char *service; > int n; > int sockfd = -1; > + int n_retries = 10; > struct rdma_cm_event *event; > struct sockaddr_in sin; > struct pingpong_context *ctx = NULL; > @@ -152,6 +153,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) > sin.sin_addr.s_addr = ((struct sockaddr_in*)res->ai_addr)->sin_addr.s_addr; > sin.sin_family = AF_INET; > sin.sin_port = htons(data->port); > +retry_addr: > if (rdma_resolve_addr(data->cm_id, NULL, > (struct sockaddr *)&sin, 2000)) { > fprintf(stderr, "%d:%s: rdma_resolve_addr failed\n", > @@ -162,6 +164,13 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) > if (rdma_get_cm_event(data->cm_channel, &event)) > goto err2; > > + > + if (event->event == RDMA_CM_EVENT_ADDR_ERROR > + && n_retries-- > 0) { > + rdma_ack_cm_event(event); > + goto retry_addr; > + } > + > if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { > fprintf(stderr, "%d:%s: unexpected CM event %d\n", > pid, __func__, event->event); > @@ -169,6 +178,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) > } > rdma_ack_cm_event(event); > > +retry_route: > if (rdma_resolve_route(data->cm_id, 2000)) { > fprintf(stderr, "%d:%s: rdma_resolve_route failed\n", > pid, __func__); > @@ -178,6 +188,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) > if (rdma_get_cm_event(data->cm_channel, &event)) > goto err2; > > + if (event->event == RDMA_CM_EVENT_ROUTE_ERROR > + && n_retries-- > 0) { > + rdma_ack_cm_event(event); > + goto retry_route; > + } > + > if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { > fprintf(stderr, "%d:%s: unexpected CM event %d\n", > pid, __func__, event->event); > diff --git a/rdma_lat.c b/rdma_lat.c > index 3681b35..1f65086 100755 > --- a/rdma_lat.c > +++ b/rdma_lat.c > @@ -207,6 +207,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) > char *service; > int n; > int sockfd = -1; > + int n_retries = 10; > struct rdma_cm_event *event; > struct sockaddr_in sin; > struct pingpong_context *ctx = NULL; > @@ -228,6 +229,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) > sin.sin_addr.s_addr = ((struct sockaddr_in*)res->ai_addr)->sin_addr.s_addr; > sin.sin_family = AF_INET; > sin.sin_port = htons(data->port); > +retry_addr: > if (rdma_resolve_addr(data->cm_id, NULL, > (struct sockaddr *)&sin, 2000)) { > fprintf(stderr, "%d:%s: rdma_resolve_addr failed\n", > @@ -238,6 +240,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) > if (rdma_get_cm_event(data->cm_channel, &event)) > goto err2; > > + if (event->event == RDMA_CM_EVENT_ADDR_ERROR > + && n_retries-- > 0) { > + rdma_ack_cm_event(event); > + goto retry_addr; > + } > + > if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { > fprintf(stderr, "%d:%s: unexpected CM event %d\n", > pid, __func__, event->event); > @@ -245,6 +253,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) > } > rdma_ack_cm_event(event); > > +retry_route: > if (rdma_resolve_route(data->cm_id, 2000)) { > fprintf(stderr, "%d:%s: rdma_resolve_route failed\n", > pid, __func__); > @@ -254,6 +263,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) > if (rdma_get_cm_event(data->cm_channel, &event)) > goto err2; > > + if (event->event == RDMA_CM_EVENT_ROUTE_ERROR > + && n_retries-- > 0) { > + rdma_ack_cm_event(event); > + goto retry_route; > + } > + > if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { > fprintf(stderr, "%d:%s: unexpected CM event %d\n", > pid, __func__, event->event); > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From davem at systemfabricworks.com Thu Jul 23 19:42:09 2009 From: davem at systemfabricworks.com (David McMillen) Date: Thu, 23 Jul 2009 21:42:09 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <4A690E6C.4000001@opengridcomputing.com> References: <4A679CD7.mailLHZ11MTGE@systemfabricworks.com> <4A690E6C.4000001@opengridcomputing.com> Message-ID: <5e3be0f30907231942w5d2d8146kc2bccfa171b3a282@mail.gmail.com> On Thu, Jul 23, 2009 at 8:29 PM, Steve Wise wrote: > Can't you just up the value passed into rdma_resolve_addr()? Currently > this code passes in 2000 (ms). Did you try changing this to say 20000? I didn't try that. Timeouts on rdma_resolve_addr are much more rare than on rdma_resolve_route, so test cases are harder to come by. I did want to offer a solution that seemed to work. I have not looked at every code path for every possible subsystem that rdma_cm will use. I don't even have a good reason to know that any particular timeout value is appropriate. It would be nice if there was some way to get that information for a particular instance of an rdma_cm_id. The same goes for the retry mechanism - is it worthwhile to retry, and how many times is enough? The values in this patch happen to work for the Infiniband fabrics I use, but my experience is limited. Are you saying that one rdma_resolve_addr with a 20,000 ms timeout is as good (or maybe even better) than 10 repeats of failed calls using 2,000 ms timeouts? If that is true, and always will be for any fabric rdma_cm uses, then it seems obvious that we should just change the timeout and not do the retry. Thanks for thinking about this problem. Dave > > > davem at systemfabricworks.com wrote: > >> The timeouts in rdma_resolve_addr and rdma_resolve_route actually happen >> in >> some fabrics, and there is no retry in rdma_cm so this patch adds process >> level retries. A combined total of 10 retries for the pair is allowed. >> >> Signed-off-by: David A. McMillen >> --- >> rdma_bw.c | 16 ++++++++++++++++ >> rdma_lat.c | 15 +++++++++++++++ >> 2 files changed, 31 insertions(+), 0 deletions(-) >> >> diff --git a/rdma_bw.c b/rdma_bw.c >> index 2628ac4..737558a 100755 >> --- a/rdma_bw.c >> +++ b/rdma_bw.c >> @@ -131,6 +131,7 @@ static struct pingpong_context >> *pp_client_connect(struct pp_data *data) >> char *service; >> int n; >> int sockfd = -1; >> + int n_retries = 10; >> struct rdma_cm_event *event; >> struct sockaddr_in sin; >> struct pingpong_context *ctx = NULL; >> @@ -152,6 +153,7 @@ static struct pingpong_context >> *pp_client_connect(struct pp_data *data) >> sin.sin_addr.s_addr = ((struct >> sockaddr_in*)res->ai_addr)->sin_addr.s_addr; >> sin.sin_family = AF_INET; >> sin.sin_port = htons(data->port); >> +retry_addr: >> if (rdma_resolve_addr(data->cm_id, NULL, >> (struct sockaddr *)&sin, 2000)) { >> fprintf(stderr, "%d:%s: rdma_resolve_addr >> failed\n", >> @@ -162,6 +164,13 @@ static struct pingpong_context >> *pp_client_connect(struct pp_data *data) >> if (rdma_get_cm_event(data->cm_channel, &event)) >> goto err2; >> + >> + if (event->event == RDMA_CM_EVENT_ADDR_ERROR >> + && n_retries-- > 0) { >> + rdma_ack_cm_event(event); >> + goto retry_addr; >> + } >> + >> if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { >> fprintf(stderr, "%d:%s: unexpected CM event %d\n", >> pid, __func__, event->event); >> @@ -169,6 +178,7 @@ static struct pingpong_context >> *pp_client_connect(struct pp_data *data) >> } >> rdma_ack_cm_event(event); >> >> +retry_route: >> if (rdma_resolve_route(data->cm_id, 2000)) { >> fprintf(stderr, "%d:%s: rdma_resolve_route >> failed\n", pid, __func__); >> @@ -178,6 +188,12 @@ static struct pingpong_context >> *pp_client_connect(struct pp_data *data) >> if (rdma_get_cm_event(data->cm_channel, &event)) >> goto err2; >> + if (event->event == RDMA_CM_EVENT_ROUTE_ERROR >> + && n_retries-- > 0) { >> + rdma_ack_cm_event(event); >> + goto retry_route; >> + } >> + >> if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { >> fprintf(stderr, "%d:%s: unexpected CM event %d\n", >> pid, __func__, event->event); >> diff --git a/rdma_lat.c b/rdma_lat.c >> index 3681b35..1f65086 100755 >> --- a/rdma_lat.c >> +++ b/rdma_lat.c >> @@ -207,6 +207,7 @@ static struct pingpong_context >> *pp_client_connect(struct pp_data *data) >> char *service; >> int n; >> int sockfd = -1; >> + int n_retries = 10; >> struct rdma_cm_event *event; >> struct sockaddr_in sin; >> struct pingpong_context *ctx = NULL; >> @@ -228,6 +229,7 @@ static struct pingpong_context >> *pp_client_connect(struct pp_data *data) >> sin.sin_addr.s_addr = ((struct >> sockaddr_in*)res->ai_addr)->sin_addr.s_addr; >> sin.sin_family = AF_INET; >> sin.sin_port = htons(data->port); >> +retry_addr: >> if (rdma_resolve_addr(data->cm_id, NULL, >> (struct sockaddr *)&sin, 2000)) { >> fprintf(stderr, "%d:%s: rdma_resolve_addr >> failed\n", >> @@ -238,6 +240,12 @@ static struct pingpong_context >> *pp_client_connect(struct pp_data *data) >> if (rdma_get_cm_event(data->cm_channel, &event)) >> goto err2; >> + if (event->event == RDMA_CM_EVENT_ADDR_ERROR >> + && n_retries-- > 0) { >> + rdma_ack_cm_event(event); >> + goto retry_addr; >> + } >> + >> if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { >> fprintf(stderr, "%d:%s: unexpected CM event %d\n", >> pid, __func__, event->event); >> @@ -245,6 +253,7 @@ static struct pingpong_context >> *pp_client_connect(struct pp_data *data) >> } >> rdma_ack_cm_event(event); >> >> +retry_route: >> if (rdma_resolve_route(data->cm_id, 2000)) { >> fprintf(stderr, "%d:%s: rdma_resolve_route >> failed\n", pid, __func__); >> @@ -254,6 +263,12 @@ static struct pingpong_context >> *pp_client_connect(struct pp_data *data) >> if (rdma_get_cm_event(data->cm_channel, &event)) >> goto err2; >> + if (event->event == RDMA_CM_EVENT_ROUTE_ERROR >> + && n_retries-- > 0) { >> + rdma_ack_cm_event(event); >> + goto retry_route; >> + } >> + >> if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { >> fprintf(stderr, "%d:%s: unexpected CM event %d\n", >> pid, __func__, event->event); >> _______________________________________________ >> general mailing list >> general at lists.openfabrics.org >> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >> >> To unsubscribe, please visit >> http://openib.org/mailman/listinfo/openib-general >> >> > -------------- next part -------------- An HTML attachment was scrubbed... URL: From swise at opengridcomputing.com Thu Jul 23 19:50:21 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Thu, 23 Jul 2009 21:50:21 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <5e3be0f30907231942w5d2d8146kc2bccfa171b3a282@mail.gmail.com> References: <4A679CD7.mailLHZ11MTGE@systemfabricworks.com> <4A690E6C.4000001@opengridcomputing.com> <5e3be0f30907231942w5d2d8146kc2bccfa171b3a282@mail.gmail.com> Message-ID: <4A69216D.6070601@opengridcomputing.com> David McMillen wrote: > > > On Thu, Jul 23, 2009 at 8:29 PM, Steve Wise > > wrote: > > Can't you just up the value passed into rdma_resolve_addr()? > Currently this code passes in 2000 (ms). Did you try changing > this to say 20000? > > > I didn't try that. Timeouts on rdma_resolve_addr are much more rare > than on rdma_resolve_route, so test cases are harder to come by. I > did want to offer a solution that seemed to work. > > I have not looked at every code path for every possible subsystem that > rdma_cm will use. I don't even have a good reason to know that any > particular timeout value is appropriate. It would be nice if there > was some way to get that information for a particular instance of an > rdma_cm_id. The same goes for the retry mechanism - is it worthwhile > to retry, and how many times is enough? The values in this patch > happen to work for the Infiniband fabrics I use, but my experience is > limited. > > Are you saying that one rdma_resolve_addr with a 20,000 ms timeout is > as good (or maybe even better) than 10 repeats of failed calls using > 2,000 ms timeouts? If that is true, and always will be for any fabric > rdma_cm uses, then it seems obvious that we should just change the > timeout and not do the retry. I think so. But if you test it on your setup, that would be best... Stevo From davem at systemfabricworks.com Thu Jul 23 20:11:42 2009 From: davem at systemfabricworks.com (davem at systemfabricworks.com) Date: Thu, 23 Jul 2009 22:11:42 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries Message-ID: <4A69266E.mail4JH11RL5U@systemfabricworks.com> Here is version 2 of the patch. Based on observations of tests, I believe Steve Wise's comments are reasonable, so I removed the rdma_resolve_addr retry and simply changed the timeout value. Feel free to use whichever one of these patches you like best. However, I urge you to apply one of these, since the programs fail in a busy large fabric. Dave The timeouts in rdma_resolve_addr and rdma_resolve_route actually happen in some fabrics. Steve Wise pointed out that increasing the rdma_resolve_addr timeout value should be as good as doing timeouts, so this patch changes that from 2 seconds to 20 seconds. As far as getting path records, there is no retry in rdma_cm so this patch adds 10 process level retries to the rdma_resolve_route call. Signed-off-by: David A. McMillen --- rdma_bw.c | 10 +++++++++- rdma_lat.c | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/rdma_bw.c b/rdma_bw.c index 2628ac4..84ccf94 100755 --- a/rdma_bw.c +++ b/rdma_bw.c @@ -131,6 +131,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) char *service; int n; int sockfd = -1; + int n_retries = 10; struct rdma_cm_event *event; struct sockaddr_in sin; struct pingpong_context *ctx = NULL; @@ -153,7 +154,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) sin.sin_family = AF_INET; sin.sin_port = htons(data->port); if (rdma_resolve_addr(data->cm_id, NULL, - (struct sockaddr *)&sin, 2000)) { + (struct sockaddr *)&sin, 20000)) { fprintf(stderr, "%d:%s: rdma_resolve_addr failed\n", pid, __func__ ); goto err2; @@ -169,6 +170,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) } rdma_ack_cm_event(event); +retry_route: if (rdma_resolve_route(data->cm_id, 2000)) { fprintf(stderr, "%d:%s: rdma_resolve_route failed\n", pid, __func__); @@ -178,6 +180,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) if (rdma_get_cm_event(data->cm_channel, &event)) goto err2; + if (event->event == RDMA_CM_EVENT_ROUTE_ERROR + && n_retries-- > 0) { + rdma_ack_cm_event(event); + goto retry_route; + } + if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { fprintf(stderr, "%d:%s: unexpected CM event %d\n", pid, __func__, event->event); diff --git a/rdma_lat.c b/rdma_lat.c index 3681b35..818e924 100755 --- a/rdma_lat.c +++ b/rdma_lat.c @@ -207,6 +207,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) char *service; int n; int sockfd = -1; + int n_retries = 10; struct rdma_cm_event *event; struct sockaddr_in sin; struct pingpong_context *ctx = NULL; @@ -229,7 +230,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) sin.sin_family = AF_INET; sin.sin_port = htons(data->port); if (rdma_resolve_addr(data->cm_id, NULL, - (struct sockaddr *)&sin, 2000)) { + (struct sockaddr *)&sin, 20000)) { fprintf(stderr, "%d:%s: rdma_resolve_addr failed\n", pid, __func__ ); goto err2; @@ -245,6 +246,7 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) } rdma_ack_cm_event(event); +retry_route: if (rdma_resolve_route(data->cm_id, 2000)) { fprintf(stderr, "%d:%s: rdma_resolve_route failed\n", pid, __func__); @@ -254,6 +256,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) if (rdma_get_cm_event(data->cm_channel, &event)) goto err2; + if (event->event == RDMA_CM_EVENT_ROUTE_ERROR + && n_retries-- > 0) { + rdma_ack_cm_event(event); + goto retry_route; + } + if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { fprintf(stderr, "%d:%s: unexpected CM event %d\n", pid, __func__, event->event); From gmpc at sanger.ac.uk Fri Jul 24 01:34:11 2009 From: gmpc at sanger.ac.uk (Guy Coates) Date: Fri, 24 Jul 2009 09:34:11 +0100 Subject: [ofa-general] Infiniband and Ubuntu In-Reply-To: <20090723224410.GN12693@obsidianresearch.com> References: <7AD18005-56B0-4598-8544-ABB05EC0E3E2@mit.edu> <20090723224410.GN12693@obsidianresearch.com> Message-ID: <4A697203.2000205@sanger.ac.uk> You might want to try the debs: http://pkg-ofed.alioth.debian.org/ There were built on debian/lenny, but I have had some reports of success on Ubuntu. I suspect rebuilding the debs from source will work if the lenny binaries do not work out-of-the-box. Cheers, Guy -- Dr Guy Coates, Informatics System Group The Wellcome Trust Sanger Institute, Hinxton, Cambridge, CB10 1HH, UK Tel: +44 (0)1223 834244 ex 6925 Fax: +44 (0)1223 496802 -- The Wellcome Trust Sanger Institute is operated by Genome Research Limited, a charity registered in England with number 1021457 and a company registered in England with number 2742969, whose registered office is 215 Euston Road, London, NW1 2BE. From vlad at lists.openfabrics.org Fri Jul 24 02:53:44 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Fri, 24 Jul 2009 02:53:44 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090724-0200 daily build status Message-ID: <20090724095344.F3989E61CB1@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Passed on i686 with linux-2.6.19 Passed on i686 with linux-2.6.18 Passed on i686 with linux-2.6.21.1 Passed on i686 with linux-2.6.22 Passed on x86_64 with linux-2.6.18 Passed on x86_64 with linux-2.6.20 Passed on x86_64 with linux-2.6.19 Passed on x86_64 with linux-2.6.21.1 Passed on x86_64 with linux-2.6.22 Passed on ia64 with linux-2.6.18 Passed on ia64 with linux-2.6.19 Passed on ia64 with linux-2.6.21.1 Passed on ia64 with linux-2.6.22 Passed on ppc64 with linux-2.6.18 Passed on ppc64 with linux-2.6.19 Failed: Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.23_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.23_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.23_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090724-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From sashak at voltaire.com Fri Jul 24 03:41:16 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Fri, 24 Jul 2009 13:41:16 +0300 Subject: [ofa-general] [PATCH] opensm/osm_sm.c: code consolidation In-Reply-To: References: <20090721153228.GJ31735@me> <20090722105907.GP31735@me> Message-ID: <20090724104116.GE10899@me> On 14:19 Wed 22 Jul , Hal Rosenstock wrote: > On Wed, Jul 22, 2009 at 6:59 AM, Sasha Khapyorsky wrote: > > > > Consolidate lock releasing code in multicast join and leave processors. > > FWIW the way it was released the lock sooner in the error cases at the > expense of the extra duplicated code to release the lock. Some of those error cases represent an invalid usage case (and actually the checks could be removed) and rest have a low probability. Sasha From roel.kluin at gmail.com Fri Jul 24 04:30:53 2009 From: roel.kluin at gmail.com (Roel Kluin) Date: Fri, 24 Jul 2009 13:30:53 +0200 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds Message-ID: <4A699B6D.6060309@gmail.com> Ensure index stays within smp->return_path[] and ->initial_path[]. Signed-off-by: Roel Kluin --- This was observed using Parfait (http://research.sun.com/projects/parfait/) It appears that hop_ptr may be able to range up to 255, potentially writing outside the buffer. diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c index 8723675..8303b80 100644 --- a/drivers/infiniband/core/smi.c +++ b/drivers/infiniband/core/smi.c @@ -140,7 +140,8 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, u8 node_type, /* C14-9:2 -- intermediate hop */ if (hop_ptr && hop_ptr < hop_cnt) { - if (node_type != RDMA_NODE_IB_SWITCH) + if (node_type != RDMA_NODE_IB_SWITCH || + hop_ptr >= IB_SMP_MAX_PATH_HOPS - 1) return IB_SMI_DISCARD; smp->return_path[hop_ptr] = port_num; From hal.rosenstock at gmail.com Fri Jul 24 04:53:40 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Fri, 24 Jul 2009 07:53:40 -0400 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: <4A699B6D.6060309@gmail.com> References: <4A699B6D.6060309@gmail.com> Message-ID: On Fri, Jul 24, 2009 at 7:30 AM, Roel Kluin wrote: > Ensure index stays within smp->return_path[] and ->initial_path[]. > > Signed-off-by: Roel Kluin > --- > This was observed using Parfait (http://research.sun.com/projects/parfait/) > > It appears that hop_ptr may be able to range up to 255, potentially writing > outside the buffer. > > diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c > index 8723675..8303b80 100644 > --- a/drivers/infiniband/core/smi.c > +++ b/drivers/infiniband/core/smi.c > @@ -140,7 +140,8 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, u8 node_type, > >                /* C14-9:2 -- intermediate hop */ >                if (hop_ptr && hop_ptr < hop_cnt) { > -                       if (node_type != RDMA_NODE_IB_SWITCH) > +                       if (node_type != RDMA_NODE_IB_SWITCH || > +                                       hop_ptr >= IB_SMP_MAX_PATH_HOPS - 1) Yes, nice catch. This appears correct but I think the issue of hop path/count validation is slightly larger. In that routine (smi_handle_dr_smp_recv), I would think that if either hop_ptr or hop_cnt are >= IB_SMP_MAX_PATH_HOPS then IB_SMI_DISCARD should be returned. If that validation is done at the top of that routine, then the hop_ptr < hop_cnt check in this C14-9:2 case eliminates the need to special case this here. Make sense ? -- Hal >                                return IB_SMI_DISCARD; > >                        smp->return_path[hop_ptr] = port_num; > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From hnrose at comcast.net Fri Jul 24 07:14:51 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Fri, 24 Jul 2009 10:14:51 -0400 Subject: [ofa-general] [PATCH] opensm/osm_ucast_lash.c: Minor cleanups Message-ID: <20090724141451.GA18684@comcast.net> Eliminate some unneeded initializations Eliminate #if 0'd code Cosmetic formatting changes Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 67471f5..7133e25 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -123,8 +123,7 @@ static void connect_switches(lash_t * p_lash, int sw1, int sw2, int phy_port_1) node->num_links++; OSM_LOG(p_log, OSM_LOG_VERBOSE, - "LASH connect: %d, %d, %d\n", sw1, sw2, - phy_port_1); + "LASH connect: %d, %d, %d\n", sw1, sw2, phy_port_1); } static osm_switch_t *get_osm_switch_from_port(const osm_port_t * port) @@ -137,19 +136,6 @@ static osm_switch_t *get_osm_switch_from_port(const osm_port_t * port) return NULL; } -#if 0 -static int randint(int high) -{ - int r; - - if (high == 0) - return 0; - r = rand(); - high++; - return (r % high); -} -#endif - static int cycle_exists(cdg_vertex_t * start, cdg_vertex_t * current, cdg_vertex_t * prev, int visit_num) { @@ -277,7 +263,7 @@ inline static void dequeue(cl_list_t * bfsq, switch_t ** sw) static int get_phys_connection(switch_t *sw, int switch_to) { - unsigned int i = 0; + unsigned int i; for (i = 0; i < sw->node->num_links; i++) if (sw->node->links[i]->switch_id == switch_to) @@ -327,10 +313,9 @@ static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, while (i_dest != NULL) { if (sw->routing_table[i_dest->switch_id].out_link == - NONE) { + NONE) sw->routing_table[i_dest->switch_id].out_link = get_phys_connection(sw, next_switch); - } prev = i_dest; i_dest = i_dest->next; @@ -739,9 +724,8 @@ static int init_lash_structures(lash_t * p_lash) if (p_lash->cdg_vertex_matrix[i][j] == NULL) goto Exit_Mem_Error; - for (k = 0; k < num_switches; k++) { + for (k = 0; k < num_switches; k++) p_lash->cdg_vertex_matrix[i][j][k] = NULL; - } } } @@ -767,9 +751,8 @@ static int init_lash_structures(lash_t * p_lash) (int *)malloc(vl_min * sizeof(int *)); if (p_lash->virtual_location[i][j] == NULL) goto Exit_Mem_Error; - for (k = 0; k < vl_min; k++) { + for (k = 0; k < vl_min; k++) p_lash->virtual_location[i][j][k] = 0; - } } } @@ -930,20 +913,18 @@ static int lash_core(lash_t * p_lash) } } - for (i = 0; i < lanes_needed; i++) { + for (i = 0; i < lanes_needed; i++) OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", i, p_lash->num_mst_in_lane[i]); - } OSM_LOG(p_log, OSM_LOG_INFO, "Lanes needed: %d, Balancing\n", lanes_needed); balance_virtual_lanes(p_lash, lanes_needed); - for (i = 0; i < lanes_needed; i++) { + for (i = 0; i < lanes_needed; i++) OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n", i, p_lash->num_mst_in_lane[i]); - } goto Exit; @@ -1131,7 +1112,7 @@ static void lash_cleanup(lash_t * p_lash) static int discover_network_properties(lash_t * p_lash) { - int i = 0, id = 0; + int i, id = 0; uint8_t vl_min; osm_subn_t *p_subn = &p_lash->p_osm->subn; osm_switch_t *p_next_sw, *p_sw; From roel.kluin at gmail.com Fri Jul 24 08:45:32 2009 From: roel.kluin at gmail.com (Roel Kluin) Date: Fri, 24 Jul 2009 17:45:32 +0200 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> Message-ID: <4A69D71C.4090400@gmail.com> Ensure index stays within smp->return_path[] and ->initial_path[]. A hop_cnt or hop_ptr greater or equal to IB_SMP_MAX_PATH_HOPS is invalid. Signed-off-by: Roel Kluin --- Observed using Parfait, http://research.sun.com/projects/parfait/ Oops, restoring CCs, this was the discussion: >>> Yes, nice catch. This appears correct but I think the issue of hop >>> path/count validation is slightly larger. In that routine >>> (smi_handle_dr_smp_recv), I would think that if either hop_ptr or >>> hop_cnt are >= IB_SMP_MAX_PATH_HOPS then IB_SMI_DISCARD should be >>> returned. If that validation is done at the top of that routine, then >>> the hop_ptr < hop_cnt check in this C14-9:2 case eliminates the need >>> to special case this here. Make sense ? >> Some, but there may be more to be considered: >> >> * In C14-13:1 and C14-13:2 a decrement occurs before accessing the >> smp->return_path[] element. Therefore an initial smp->hop_ptr of >> SMP_MAX_PATH_HOPS could be valid (not sure). But implimenting >> the test at the top of that routine will cause a return of IB_SMI_DISCARD. > > Is your point that if hop_ptr of MAX_PATH_HOPS is valid there, then > returning IB_SMI_DISCARD is incorrect ? The point was that I wasn't sure. > When this routine is called, hop_ptr and hop_cnt are as received from > the IB wire. > > In C14-13:2, hop_ptr <= hop_cnt and hop_cnt of MAX_PATH_HOPS is > invalid so this is safe. > > In C14-13:1, hop_ptr == hop_cnt + 1 does appear to allow hop_ptr to be > MAX_PATH_HOPS there. > > So I now think your check is needed as well as adding a validation of hop_cnt > at the top of the routine as valid values are 0 to 63 as indicated in C14-6. So I see... > The good news is this is all theoretical as no one has come close to > building a depth 64 IB subnet. > > -- Hal Thanks for your review. I hope this was as you intended? diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c index 8723675..9d66c42 100644 --- a/drivers/infiniband/core/smi.c +++ b/drivers/infiniband/core/smi.c @@ -132,6 +132,9 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, u8 node_type, hop_ptr = smp->hop_ptr; hop_cnt = smp->hop_cnt; + if (hop_ptr >= IB_SMP_MAX_PATH_HOPS || hop_cnt >= IB_SMP_MAX_PATH_HOPS) + return IB_SMI_DISCARD; + /* See section 14.2.2.2, Vol 1 IB spec */ if (!ib_get_smp_direction(smp)) { /* C14-9:1 -- sender should have incremented hop_ptr */ @@ -140,7 +143,8 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, u8 node_type, /* C14-9:2 -- intermediate hop */ if (hop_ptr && hop_ptr < hop_cnt) { - if (node_type != RDMA_NODE_IB_SWITCH) + if (node_type != RDMA_NODE_IB_SWITCH || + hop_ptr + 1 >= IB_SMP_MAX_PATH_HOPS) return IB_SMI_DISCARD; smp->return_path[hop_ptr] = port_num; From hal.rosenstock at gmail.com Fri Jul 24 08:47:39 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Fri, 24 Jul 2009 11:47:39 -0400 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: <4A69D71C.4090400@gmail.com> References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> Message-ID: On Fri, Jul 24, 2009 at 11:45 AM, Roel Kluin wrote: > Ensure index stays within smp->return_path[] and ->initial_path[]. > A hop_cnt or hop_ptr greater or equal to IB_SMP_MAX_PATH_HOPS is invalid. > > Signed-off-by: Roel Kluin > --- > Observed using Parfait, http://research.sun.com/projects/parfait/ > > Oops, restoring CCs, this was the discussion: > >>>> Yes, nice catch. This appears correct but I think the issue of hop >>>> path/count validation is slightly larger. In that routine >>>> (smi_handle_dr_smp_recv), I would think that if either hop_ptr or >>>> hop_cnt are >= IB_SMP_MAX_PATH_HOPS then IB_SMI_DISCARD should be >>>> returned. If that validation is done at the top of that routine, then >>>> the hop_ptr < hop_cnt check in this C14-9:2 case eliminates the need >>>> to special case this here. Make sense ? >>> Some, but there may be more to be considered: >>> >>> * In C14-13:1 and C14-13:2 a decrement occurs before accessing the >>> smp->return_path[] element. Therefore an initial smp->hop_ptr of >>> SMP_MAX_PATH_HOPS could be valid (not sure). But implimenting >>> the test at the top of that routine will cause a return of IB_SMI_DISCARD. >> >> Is your point that if hop_ptr of MAX_PATH_HOPS is valid there, then >> returning IB_SMI_DISCARD is incorrect ? > > The point was that I wasn't sure. > >> When this routine is called, hop_ptr and hop_cnt are as received from >> the IB wire. >> >> In C14-13:2, hop_ptr <= hop_cnt and hop_cnt of MAX_PATH_HOPS is >> invalid so this is safe. >> >> In C14-13:1, hop_ptr == hop_cnt + 1 does appear to allow hop_ptr to be >> MAX_PATH_HOPS there. >> >> So I now think your check is needed as well as adding a validation of hop_cnt >> at the top of the routine as valid values are 0 to 63 as indicated in C14-6. > > So I see... > >> The good news is this is all theoretical as no one has come close to >> building a depth 64 IB subnet. >> >> -- Hal > > Thanks for your review. I hope this was as you intended? Close; see below. > diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c > index 8723675..9d66c42 100644 > --- a/drivers/infiniband/core/smi.c > +++ b/drivers/infiniband/core/smi.c > @@ -132,6 +132,9 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, u8 node_type, >        hop_ptr = smp->hop_ptr; >        hop_cnt = smp->hop_cnt; > > +       if (hop_ptr >= IB_SMP_MAX_PATH_HOPS || hop_cnt >= IB_SMP_MAX_PATH_HOPS) Based on the spec limiting hop pointer to 255 and not 63, I think the above should just be a check on hop count and not hop pointer: if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) -- Hal > +               return IB_SMI_DISCARD; > + >        /* See section 14.2.2.2, Vol 1 IB spec */ >        if (!ib_get_smp_direction(smp)) { >                /* C14-9:1 -- sender should have incremented hop_ptr */ > @@ -140,7 +143,8 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, u8 node_type, > >                /* C14-9:2 -- intermediate hop */ >                if (hop_ptr && hop_ptr < hop_cnt) { > -                       if (node_type != RDMA_NODE_IB_SWITCH) > +                       if (node_type != RDMA_NODE_IB_SWITCH || > +                                       hop_ptr + 1 >= IB_SMP_MAX_PATH_HOPS) >                                return IB_SMI_DISCARD; > >                        smp->return_path[hop_ptr] = port_num; > From sean.hefty at intel.com Fri Jul 24 08:52:32 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Fri, 24 Jul 2009 08:52:32 -0700 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <4A69266E.mail4JH11RL5U@systemfabricworks.com> References: <4A69266E.mail4JH11RL5U@systemfabricworks.com> Message-ID: <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> >Here is version 2 of the patch. Based on observations of tests, I believe >Steve Wise's comments are reasonable, so I removed the rdma_resolve_addr >retry and simply changed the timeout value. Feel free to use whichever one >of these patches you like best. However, I urge you to apply one of these, >since the programs fail in a busy large fabric. Why not just make the retry and timeout values command line parameters and allow adjusting both? - Sean From swise at opengridcomputing.com Fri Jul 24 09:09:09 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Fri, 24 Jul 2009 11:09:09 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> References: <4A69266E.mail4JH11RL5U@systemfabricworks.com> <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> Message-ID: <4A69DCA5.9040705@opengridcomputing.com> Sean Hefty wrote: >> Here is version 2 of the patch. Based on observations of tests, I believe >> Steve Wise's comments are reasonable, so I removed the rdma_resolve_addr >> retry and simply changed the timeout value. Feel free to use whichever one >> of these patches you like best. However, I urge you to apply one of these, >> since the programs fail in a busy large fabric. >> > > Why not just make the retry and timeout values command line parameters and allow > adjusting both? > > - Sean > > I'm not sure we need the retry. But making the timeout a parameter is good idea. And fix all the rdmacm programs in perftest (I think there are two: bw and lat). Steve. From sean.hefty at intel.com Fri Jul 24 09:11:07 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Fri, 24 Jul 2009 09:11:07 -0700 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <4A69DCA5.9040705@opengridcomputing.com> References: <4A69266E.mail4JH11RL5U@systemfabricworks.com> <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> <4A69DCA5.9040705@opengridcomputing.com> Message-ID: >I'm not sure we need the retry. On IB, resolve route is done using unreliable datagram with no lower level timeout or retry. - Sean From swise at opengridcomputing.com Fri Jul 24 09:16:50 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Fri, 24 Jul 2009 11:16:50 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: References: <4A69266E.mail4JH11RL5U@systemfabricworks.com> <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> <4A69DCA5.9040705@opengridcomputing.com> Message-ID: <4A69DE72.4020901@opengridcomputing.com> Sean Hefty wrote: >> I'm not sure we need the retry. >> > > On IB, resolve route is done using unreliable datagram with no lower level > timeout or retry. > Ah. Seems like the retry functionality is perhaps useful enough then to add it into the rdma-cm itself? IE have a retry count param to rdma_resolve_route() maybe? Just thinking aloud... Stevo From davem at systemfabricworks.com Fri Jul 24 11:01:24 2009 From: davem at systemfabricworks.com (davem at systemfabricworks.com) Date: Fri, 24 Jul 2009 13:01:24 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries Message-ID: <4A69F6F4.mailA5W1C7NLW@systemfabricworks.com> The overloaded argument 'ptr' was used in a manner that generated compiler warnings about possible uninitialized use. This localizes each use to the block that uses it and removes the warning. Signed-off-by: David A. McMillen --- rdma_bw.c | 7 +++---- rdma_lat.c | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/rdma_bw.c b/rdma_bw.c index 2628ac4..18c4244 100755 --- a/rdma_bw.c +++ b/rdma_bw.c @@ -510,8 +510,6 @@ err: static struct pingpong_context *pp_init_ctx(void *ptr, struct pp_data *data) { struct pingpong_context *ctx; - struct ibv_device *ib_dev; - struct rdma_cm_id *cm_id; ctx = malloc(sizeof *ctx); if (!ctx) @@ -530,7 +528,7 @@ static struct pingpong_context *pp_init_ctx(void *ptr, struct pp_data *data) memset(ctx->buf, 0, ctx->size * 2); if (data->use_cma) { - cm_id = (struct rdma_cm_id *)ptr; + struct rdma_cm_id *cm_id = (struct rdma_cm_id *)ptr; ctx->context = cm_id->verbs; if (!ctx->context) { fprintf(stderr, "%d:%s: Unbound cm_id!!\n", pid, @@ -539,7 +537,7 @@ static struct pingpong_context *pp_init_ctx(void *ptr, struct pp_data *data) } } else { - ib_dev = (struct ibv_device *)ptr; + struct ibv_device *ib_dev = (struct ibv_device *)ptr; ctx->context = ibv_open_device(ib_dev); if (!ctx->context) { fprintf(stderr, "%d:%s: Couldn't get context for %s\n", @@ -603,6 +601,7 @@ static struct pingpong_context *pp_init_ctx(void *ptr, struct pp_data *data) }; if (data->use_cma) { + struct rdma_cm_id *cm_id = (struct rdma_cm_id *)ptr; if (rdma_create_qp(cm_id, ctx->pd, &attr)) { fprintf(stderr, "%d:%s: Couldn't create QP\n", pid, __func__); return NULL; diff --git a/rdma_lat.c b/rdma_lat.c index 3681b35..0c45af8 100755 --- a/rdma_lat.c +++ b/rdma_lat.c @@ -523,8 +523,6 @@ static int pp_server_exch_dest(struct pp_data *data) static struct pingpong_context *pp_init_ctx(void *ptr, struct pp_data *data) { struct pingpong_context *ctx; - struct ibv_device *ib_dev; - struct rdma_cm_id *cm_id; ctx = malloc(sizeof *ctx); if (!ctx) @@ -547,7 +545,7 @@ static struct pingpong_context *pp_init_ctx(void *ptr, struct pp_data *data) if (data->use_cma) { - cm_id = (struct rdma_cm_id *)ptr; + struct rdma_cm_id *cm_id = (struct rdma_cm_id *)ptr; ctx->context = cm_id->verbs; if (!ctx->context) { fprintf(stderr, "%d:%s: Unbound cm_id!!\n", pid, @@ -556,7 +554,7 @@ static struct pingpong_context *pp_init_ctx(void *ptr, struct pp_data *data) } } else { - ib_dev = (struct ibv_device *)ptr; + struct ibv_device *ib_dev = (struct ibv_device *)ptr; ctx->context = ibv_open_device(ib_dev); if (!ctx->context) { fprintf(stderr, "%d:%s: Couldn't get context for %s\n", @@ -612,6 +610,7 @@ static struct pingpong_context *pp_init_ctx(void *ptr, struct pp_data *data) }; if (data->use_cma) { + struct rdma_cm_id *cm_id = (struct rdma_cm_id *)ptr; if (rdma_create_qp(cm_id, ctx->pd, &attr)) { fprintf(stderr, "%d:%s: Couldn't create QP\n", pid, __func__); return NULL; From davem at systemfabricworks.com Fri Jul 24 11:01:58 2009 From: davem at systemfabricworks.com (davem at systemfabricworks.com) Date: Fri, 24 Jul 2009 13:01:58 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries Message-ID: <4A69F716.mailA641PWNRQ@systemfabricworks.com> Here is version 3 of the patch. Between Steve and Sean's comments, it seems there is no universally accepted answer, which is why it would be nice if the underlying system could provide good defaults for the user mode programs. However, that isn't here yet, and I am not prepared to try to create such a thing, so I have redone this patch to allow command line specification of the timeout values and retry counts. The timeout values are the same as the original code, and the retry counts are both set to 10. Dave The timeouts in rdma_resolve_addr and rdma_resolve_route actually happen in some fabrics. This adds command line options to set the number of retries for each of the calls, with a default of 10. Since there may be cases where larger timeouts are desired, probably along with fewer retries, this patch also adds the ability to specify the timeout values on the command line. If none of the command line options are chosen, it will now do the retries and not fail in the larger and busier fabrics. Signed-off-by: David A. McMillen --- rdma_bw.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++- rdma_lat.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 4 deletions(-) diff --git a/rdma_bw.c b/rdma_bw.c index 2628ac4..14ff80b 100755 --- a/rdma_bw.c +++ b/rdma_bw.c @@ -61,6 +61,10 @@ #define PINGPONG_RDMA_WRID 3 static int sl = 0; +static int addr_timeout = 2000; +static int addr_retries = 10; +static int route_timeout = 2000; +static int route_retries = 10; static int page_size; static pid_t pid; @@ -152,8 +156,9 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) sin.sin_addr.s_addr = ((struct sockaddr_in*)res->ai_addr)->sin_addr.s_addr; sin.sin_family = AF_INET; sin.sin_port = htons(data->port); +retry_addr: if (rdma_resolve_addr(data->cm_id, NULL, - (struct sockaddr *)&sin, 2000)) { + (struct sockaddr *)&sin, addr_retries)) { fprintf(stderr, "%d:%s: rdma_resolve_addr failed\n", pid, __func__ ); goto err2; @@ -162,6 +167,13 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) if (rdma_get_cm_event(data->cm_channel, &event)) goto err2; + + if (event->event == RDMA_CM_EVENT_ADDR_ERROR + && addr_retries-- > 0) { + rdma_ack_cm_event(event); + goto retry_addr; + } + if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { fprintf(stderr, "%d:%s: unexpected CM event %d\n", pid, __func__, event->event); @@ -169,7 +181,8 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) } rdma_ack_cm_event(event); - if (rdma_resolve_route(data->cm_id, 2000)) { +retry_route: + if (rdma_resolve_route(data->cm_id, route_timeout)) { fprintf(stderr, "%d:%s: rdma_resolve_route failed\n", pid, __func__); goto err2; @@ -178,6 +191,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) if (rdma_get_cm_event(data->cm_channel, &event)) goto err2; + if (event->event == RDMA_CM_EVENT_ROUTE_ERROR + && route_retries-- > 0) { + rdma_ack_cm_event(event); + goto retry_route; + } + if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { fprintf(stderr, "%d:%s: unexpected CM event %d\n", pid, __func__, event->event); @@ -863,6 +882,10 @@ static void usage(const char *argv0) printf(" -S, --sl= SL (default 0)\n"); printf(" -b, --bidirectional measure bidirectional bandwidth (default unidirectional)\n"); printf(" -c, --cma use RDMA CM\n"); + printf(" --addr-timeout= RDMA CM resolve_addr timeout ms (default 2000)\n"); + printf(" --addr-retries= RDMA CM resolve_addr retry count (default 10)\n"); + printf(" --route-timeout= RDMA CM resolve_route timeout ms (default 2000)\n"); + printf(" --route-retries= RDMA CM resolve_route retry count (default 10)\n"); } static void print_report(unsigned int iters, unsigned size, int duplex, @@ -949,6 +972,10 @@ int main(int argc, char *argv[]) { .name = "sl", .has_arg = 1, .val = 'S' }, { .name = "bidirectional", .has_arg = 0, .val = 'b' }, { .name = "cma", .has_arg = 0, .val = 'c' }, + { .name = "addr-timeout", .has_arg = 1, .val = 1 }, + { .name = "addr-retries", .has_arg = 1, .val = 2 }, + { .name = "route-timeout", .has_arg = 1, .val = 3 }, + { .name = "route-retries", .has_arg = 1, .val = 4 }, { 0 } }; @@ -1011,6 +1038,27 @@ int main(int argc, char *argv[]) case 'c': data.use_cma = 1; break; + + case 1: + addr_timeout = strtol(optarg, NULL, 0); + if (addr_timeout <= 0) { usage(argv[0]); return 1; } + break; + + case 2: + addr_retries = strtol(optarg, NULL, 0); + if (addr_retries < 0) { usage(argv[0]); return 1; } + break; + + case 3: + route_timeout = strtol(optarg, NULL, 0); + if (route_timeout <= 0) { usage(argv[0]); return 1; } + break; + + case 4: + route_retries = strtol(optarg, NULL, 0); + if (route_retries < 0) { usage(argv[0]); return 1; } + break; + default: usage(argv[0]); return 1; diff --git a/rdma_lat.c b/rdma_lat.c index 3681b35..cb5a6e4 100755 --- a/rdma_lat.c +++ b/rdma_lat.c @@ -63,6 +63,10 @@ static int inline_size = MAX_INLINE; static int sl = 0; +static int addr_timeout = 2000; +static int addr_retries = 10; +static int route_timeout = 2000; +static int route_retries = 10; static int page_size; static pid_t pid; @@ -228,8 +232,9 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) sin.sin_addr.s_addr = ((struct sockaddr_in*)res->ai_addr)->sin_addr.s_addr; sin.sin_family = AF_INET; sin.sin_port = htons(data->port); +retry_addr: if (rdma_resolve_addr(data->cm_id, NULL, - (struct sockaddr *)&sin, 2000)) { + (struct sockaddr *)&sin, addr_timeout)) { fprintf(stderr, "%d:%s: rdma_resolve_addr failed\n", pid, __func__ ); goto err2; @@ -238,6 +243,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) if (rdma_get_cm_event(data->cm_channel, &event)) goto err2; + if (event->event == RDMA_CM_EVENT_ADDR_ERROR + && addr_retries-- > 0) { + rdma_ack_cm_event(event); + goto retry_addr; + } + if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { fprintf(stderr, "%d:%s: unexpected CM event %d\n", pid, __func__, event->event); @@ -245,7 +256,8 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) } rdma_ack_cm_event(event); - if (rdma_resolve_route(data->cm_id, 2000)) { +retry_route: + if (rdma_resolve_route(data->cm_id, route_timeout)) { fprintf(stderr, "%d:%s: rdma_resolve_route failed\n", pid, __func__); goto err2; @@ -254,6 +266,12 @@ static struct pingpong_context *pp_client_connect(struct pp_data *data) if (rdma_get_cm_event(data->cm_channel, &event)) goto err2; + if (event->event == RDMA_CM_EVENT_ROUTE_ERROR + && route_retries-- > 0) { + rdma_ack_cm_event(event); + goto retry_route; + } + if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { fprintf(stderr, "%d:%s: unexpected CM event %d\n", pid, __func__, event->event); @@ -929,6 +947,10 @@ static void usage(const char *argv0) printf(" -H, --report-histogram print out all results (default print summary only)\n"); printf(" -U, --report-unsorted (implies -H) print out unsorted results (default sorted)\n"); printf(" -c, --cma Use the RDMA CMA to setup the RDMA connection\n"); + printf(" --addr-timeout= RDMA CM resolve_addr timeout ms (default 2000)\n"); + printf(" --addr-retries= RDMA CM resolve_addr retry count (default 10)\n"); + printf(" --route-timeout= RDMA CM resolve_route timeout ms (default 2000)\n"); + printf(" --route-retries= RDMA CM resolve_route retry count (default 10)\n"); } /* @@ -1052,6 +1074,10 @@ int main(int argc, char *argv[]) { .name = "report-histogram",.has_arg = 0, .val = 'H' }, { .name = "report-unsorted",.has_arg = 0, .val = 'U' }, { .name = "cma", .has_arg = 0, .val = 'c' }, + { .name = "addr-timeout", .has_arg = 1, .val = 1 }, + { .name = "addr-retries", .has_arg = 1, .val = 2 }, + { .name = "route-timeout", .has_arg = 1, .val = 3 }, + { .name = "route-retries", .has_arg = 1, .val = 4 }, { 0 } }; @@ -1123,6 +1149,38 @@ int main(int argc, char *argv[]) data.use_cma = 1; break; + case 1: + addr_timeout = strtol(optarg, NULL, 0); + if (addr_timeout <= 0) { + usage(argv[0]); + return 7; + } + break; + + case 2: + addr_retries = strtol(optarg, NULL, 0); + if (addr_retries < 0) { + usage(argv[0]); + return 7; + } + break; + + case 3: + route_timeout = strtol(optarg, NULL, 0); + if (route_timeout <= 0) { + usage(argv[0]); + return 7; + } + break; + + case 4: + route_retries = strtol(optarg, NULL, 0); + if (route_retries < 0) { + usage(argv[0]); + return 7; + } + break; + default: usage(argv[0]); return 7; From hnrose at comcast.net Fri Jul 24 16:03:03 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Fri, 24 Jul 2009 19:03:03 -0400 Subject: [ofa-general] [PATCH] opensm/osm_mesh: Fix use after free problem in osm_mesh_node_delete Message-ID: <20090724230303.GA28379@comcast.net> When osm_mesh_node_delete is called, osm_switch_delete may already have been called so sw->p_sw is no longer valid to be used although it was being used to obtain num_ports. Fix this by saving num_ports in mesh node structure on create and use this on delete. Signed-off-by: Hal Rosenstock --- diff --git a/opensm/include/opensm/osm_mesh.h b/opensm/include/opensm/osm_mesh.h index 173fa86..3dfb047 100644 --- a/opensm/include/opensm/osm_mesh.h +++ b/opensm/include/opensm/osm_mesh.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2088 System Fabric Works, Inc. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -65,6 +66,7 @@ typedef struct _mesh_node { int dimension; /* apparent dimension of mesh around node */ int temp; /* temporary holder for distance info */ int type; /* index of node type in mesh_info array */ + unsigned num_ports; unsigned int num_links; /* number of 'links' to adjacent switches */ link_t *links[0]; /* per link information */ } mesh_node_t; diff --git a/opensm/opensm/osm_mesh.c b/opensm/opensm/osm_mesh.c index 23fad87..f78d834 100644 --- a/opensm/opensm/osm_mesh.c +++ b/opensm/opensm/osm_mesh.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2008,2009 System Fabric Works, Inc. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -1323,7 +1324,7 @@ void osm_mesh_node_delete(lash_t *p_lash, switch_t *sw) osm_log_t *p_log = &p_lash->p_osm->log; int i; mesh_node_t *node = sw->node; - unsigned num_ports = sw->p_sw->num_ports; + unsigned num_ports = node->num_ports; OSM_LOG_ENTER(p_log); @@ -1383,6 +1384,8 @@ int osm_mesh_node_create(lash_t *p_lash, switch_t *sw) node->links[i]->switch_id = NONE; } + node->num_ports = num_ports; + OSM_LOG_EXIT(p_log); return 0; From hnrose at comcast.net Fri Jul 24 16:04:47 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Fri, 24 Jul 2009 19:04:47 -0400 Subject: [ofa-general] [PATCH] opensm/osm_switch.c: In osm_switch_delete, set priv to NULL Message-ID: <20090724230447.GB28379@comcast.net> This can help with finding use after free issues if memory not reused updn and lash rely on this priv pointer and may have stale osm_switch_t pointers Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_switch.c b/opensm/opensm/osm_switch.c index ce1ca63..e23b32f 100644 --- a/opensm/opensm/osm_switch.c +++ b/opensm/opensm/osm_switch.c @@ -94,6 +94,7 @@ void osm_switch_delete(IN OUT osm_switch_t ** const pp_sw) free(p_sw->hops[i]); free(p_sw->hops); } + p_sw->priv = NULL; free(*pp_sw); *pp_sw = NULL; } From davem at systemfabricworks.com Sat Jul 25 02:04:15 2009 From: davem at systemfabricworks.com (David McMillen) Date: Sat, 25 Jul 2009 04:04:15 -0500 Subject: [ofa-general] RE: Running more than 894 processes doing rdma_listen In-Reply-To: <2f3bf9a60907230130o79a151ceq8b9ed0477dbd268@mail.gmail.com> References: <5e3be0f30907221059j4bd1d89occ97c277bb43f12d@mail.gmail.com> <58CAA85864EF4DC8B46021B16E163A51@amr.corp.intel.com> <2f3bf9a60907230130o79a151ceq8b9ed0477dbd268@mail.gmail.com> Message-ID: <5e3be0f30907250204u35e4e99fnf9c27795424c1b25@mail.gmail.com> On Thu, Jul 23, 2009 at 3:30 AM, Dotan Barak wrote: > On Wed, Jul 22, 2009 at 9:05 PM, Sean Hefty wrote: > >>Is there an explicit limit on the number of ports that can be listening > using > >>rdma_cm? > > > > There's no inherent limit built into the code. > > > Maybe this limitation is being caused by the ulimit > ("ulimit -n" produced 1024 for my system). > > Try changing this value. > I first had problems with a system where ulimit -n showed 65536 I get the same 894 maximum when ulimit -n is set to 512 or 1024. This is not a per-process limitation, as the first 894 processes don't have any problems. There are no problems starting the required number of processes. This is some kind of limit imposed by the way rdma_create_event_channel works. Dave -------------- next part -------------- An HTML attachment was scrubbed... URL: From vlad at lists.openfabrics.org Sat Jul 25 02:54:56 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Sat, 25 Jul 2009 02:54:56 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090725-0200 daily build status Message-ID: <20090725095456.79948E2817B@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Passed on i686 with linux-2.6.18 Passed on i686 with linux-2.6.21.1 Passed on i686 with linux-2.6.19 Passed on i686 with linux-2.6.22 Passed on x86_64 with linux-2.6.18 Passed on x86_64 with linux-2.6.21.1 Passed on x86_64 with linux-2.6.19 Passed on x86_64 with linux-2.6.20 Passed on x86_64 with linux-2.6.22 Passed on ia64 with linux-2.6.18 Passed on ia64 with linux-2.6.19 Passed on ia64 with linux-2.6.21.1 Passed on ia64 with linux-2.6.22 Passed on ppc64 with linux-2.6.18 Passed on ppc64 with linux-2.6.19 Failed: Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.23_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.23_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.23_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090725-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From monis at Voltaire.COM Sat Jul 25 23:53:58 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Sun, 26 Jul 2009 09:53:58 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A62D1E6.30800@Voltaire.COM> References: <4A521784.5090304@Voltaire.COM> <4A5DF1C0.2060701@Voltaire.COM> <20090717211521.GG7320@obsidianresearch.com> <4A62D1E6.30800@Voltaire.COM> Message-ID: <4A6BFD86.7040808@Voltaire.COM> Roland, With Jason's example of corrupting mcast list of IPoIB from userspace, what do you think about the necessity of the patch? Moni Shoua wrote: >>> Is there any way userspace can inject a bogus multicast address? >> Can you do it with netlink? >> >> ip maddr add address ... dev ib0 >> >> Jason > > Thanks Jason. This is an example that shows it can be done from userspace. > Roland, I think that the output below answers your question. > > > Before ip maddr show > ------------------------------- > linux:/etc/sysconfig/network # ip m s dev ib0 > 9: ib0 > link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:00:00:00:00 > link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 > link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 > inet 224.0.0.1 > inet6 ff02::1:ff96:ca05 > inet6 ff02::1 > > linux:/etc/sysconfig/network # ip maddr add 33:33:00:00:00:01 dev ib0 > linux:/etc/sysconfig/network # ip m s dev ib0 > > Before ip maddr show > ------------------------------- > 9: ib0 > link 33:33:00:00:00:01:00:00:00:00:00:00:00:00:00:00:00:00:00:00 static > link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:00:00:00:00 > link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 > link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 > inet 224.0.0.1 > inet6 ff02::1:ff96:ca05 > inet6 ff02::1 > >>From dmesg (22 means EINVAL) > ------------------------------- > ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 > ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 > ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 > ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 > ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 > ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 > ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 > ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From ogerlitz at voltaire.com Sat Jul 25 23:55:07 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Sun, 26 Jul 2009 09:55:07 +0300 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: References: <4A69266E.mail4JH11RL5U@systemfabricworks.com> <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> <4A69DCA5.9040705@opengridcomputing.com> Message-ID: <4A6BFDCB.3050906@voltaire.com> Sean Hefty wrote: > On IB, resolve route is done using unreliable datagram with no lower level timeout or retry. Hi Sean, I wasn't sure to follow on your comment, since the sa_query kernel code does deliver the resolve_route timeout param down to the mad code, and with some enhancement it could also program the retries field of the ib_send_mad_buf structure, this way both the timeout (as today) and the retries would be done in the lowest layer (mad). Or. From monis at Voltaire.COM Sat Jul 25 23:55:57 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Sun, 26 Jul 2009 09:55:57 +0300 Subject: [ofa-general] [PATCH] IB/IPoIB: Don't let a bad muticast address in the join list stop subsequent joins In-Reply-To: <4A6BFD86.7040808@Voltaire.COM> References: <4A521784.5090304@Voltaire.COM> <4A5DF1C0.2060701@Voltaire.COM> <20090717211521.GG7320@obsidianresearch.com> <4A62D1E6.30800@Voltaire.COM> <4A6BFD86.7040808@Voltaire.COM> Message-ID: <4A6BFDFD.6080503@Voltaire.COM> Moni Shoua wrote: Sorry, previous message was sent by mistake to myself with others on CC > Roland, > With Jason's example of corrupting mcast list of IPoIB from userspace, what do you think about the necessity of the patch? > > Moni Shoua wrote: >>>> Is there any way userspace can inject a bogus multicast address? >>> Can you do it with netlink? >>> >>> ip maddr add address ... dev ib0 >>> >>> Jason >> Thanks Jason. This is an example that shows it can be done from userspace. >> Roland, I think that the output below answers your question. >> >> >> Before ip maddr show >> ------------------------------- >> linux:/etc/sysconfig/network # ip m s dev ib0 >> 9: ib0 >> link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:00:00:00:00 >> link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 >> link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 >> inet 224.0.0.1 >> inet6 ff02::1:ff96:ca05 >> inet6 ff02::1 >> >> linux:/etc/sysconfig/network # ip maddr add 33:33:00:00:00:01 dev ib0 >> linux:/etc/sysconfig/network # ip m s dev ib0 >> >> Before ip maddr show >> ------------------------------- >> 9: ib0 >> link 33:33:00:00:00:01:00:00:00:00:00:00:00:00:00:00:00:00:00:00 static >> link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:01:00:00:00:00 >> link 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 >> link 00:ff:ff:ff:ff:12:60:1b:ff:ff:00:00:00:00:00:00:00:00:00:00 >> inet 224.0.0.1 >> inet6 ff02::1:ff96:ca05 >> inet6 ff02::1 >> >> >From dmesg (22 means EINVAL) >> ------------------------------- >> ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 >> ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 >> ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 >> ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 >> ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 >> ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 >> ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 >> ib0: multicast join failed for 0001:0000:0000:0000:0000:0000:0000:0000, status -22 >> _______________________________________________ >> general mailing list >> general at lists.openfabrics.org >> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general >> >> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general >> > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From ogerlitz at voltaire.com Sun Jul 26 00:06:25 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Sun, 26 Jul 2009 10:06:25 +0300 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <4A69DE72.4020901@opengridcomputing.com> References: <4A69266E.mail4JH11RL5U@systemfabricworks.com> <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> <4A69DCA5.9040705@opengridcomputing.com> <4A69DE72.4020901@opengridcomputing.com> Message-ID: <4A6C0071.3050305@voltaire.com> Steve Wise wrote: > Seems like the retry functionality is perhaps useful enough then to > add it into the rdma-cm itself? IE have a retry count param to > rdma_resolve_route() maybe? Just thinking aloud... Steve, Sean As I just wrote, with IB, there's a lower level mechanism (mad) to enforce timeout and implement retries for route lookup. Since with iWARP route resolution is no-op, its seems we are done in that respect. As for address resolution, the kernel addr module implements a timeout mechanism. I wonder if one of the neighbour subsystem sysctls/params, could be used to implement retries, looking on arp(7) I came a cross the retrans_time sysctl which stands for "The number of jiffies to delay before retransmitting a request. Defaults to 1 second", but I wasn't sure if it stands for itself or has to be used in conjunction with other sysctls. Basically, when an arp is a byproduct of the IP stack wanting to send a packet, there's a rich state-machine of retries, etc, I assume it should/could be usable for the RDMA stack. Or. From vlad at lists.openfabrics.org Sun Jul 26 02:54:56 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Sun, 26 Jul 2009 02:54:56 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090726-0200 daily build status Message-ID: <20090726095457.1C6C5E28493@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Passed on i686 with linux-2.6.18 Passed on i686 with linux-2.6.21.1 Passed on i686 with linux-2.6.19 Passed on i686 with linux-2.6.22 Passed on x86_64 with linux-2.6.18 Passed on x86_64 with linux-2.6.21.1 Passed on x86_64 with linux-2.6.19 Passed on x86_64 with linux-2.6.20 Passed on x86_64 with linux-2.6.22 Passed on ia64 with linux-2.6.18 Passed on ia64 with linux-2.6.19 Passed on ia64 with linux-2.6.21.1 Passed on ia64 with linux-2.6.22 Passed on ppc64 with linux-2.6.18 Passed on ppc64 with linux-2.6.19 Failed: Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.23_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.23_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.23_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090726-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From davem at systemfabricworks.com Sun Jul 26 03:32:37 2009 From: davem at systemfabricworks.com (David McMillen) Date: Sun, 26 Jul 2009 05:32:37 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <4A6C0071.3050305@voltaire.com> References: <4A69266E.mail4JH11RL5U@systemfabricworks.com> <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> <4A69DCA5.9040705@opengridcomputing.com> <4A69DE72.4020901@opengridcomputing.com> <4A6C0071.3050305@voltaire.com> Message-ID: <5e3be0f30907260332w3c7097f2g5ff61b2b382f368a@mail.gmail.com> I am pleased to see the discussions this has raised about possible changes to the underlying kernel services. However, this user-mode patch does address a real problem in the stack as it stands, and I'd like to see agreement that it should be applied. The last (V3) version of the patch makes the default action behave fairly well, and adds command line switches that allow complete control over the timeouts/retries. I don't know enough about the kernel level code to make comments on the best way to improve it, but I would like to make an observation from the user level. There is no way for the user level to have an informed decision about the proper value for timeouts or retry counts. If you are writing some code (MPI comes to mind) that you know will impose a difficult load and therefore requires special tuning, it actually impacts all other user level code running on the fabric. Nothing allows that special tuning to be known or shared. I think those kinds of changes really need to be taken over by system-wide parameters in the kernel, so much so that perhaps some consideration should be given to ignoring the timeout parameters on rdma_resolve_addr and rdma_resolve_route in favor of kernel parameters. Dave -------------- next part -------------- An HTML attachment was scrubbed... URL: From monis at Voltaire.COM Sun Jul 26 06:01:40 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Sun, 26 Jul 2009 16:01:40 +0300 Subject: [ofa-general] IPoIB connections falling off In-Reply-To: <20090722145930.ca234b60.weiny2@llnl.gov> References: <20090722145930.ca234b60.weiny2@llnl.gov> Message-ID: <4A6C53B4.8090404@Voltaire.COM> Ira Weiny wrote: > Check your multicast group membership and forwarding tables on the switches. > > We have had similar issues and have found that some nodes fail to join the multicast groups for various reasons. > > Ira I think that a good start would be to send the following 1. kernel neigh table ip neigh show 2. ipoib path cache cat /sys/kernel/debug/ipoib/ib0_path (you probably need to mount debugfs first with 'mount -t debugfs none /sys/kernel/debug') 2. ipoib mcast joins cat /sys/kernel/debug/ipoib/ib0_mcg From slavas at Voltaire.COM Mon Jul 27 01:24:56 2009 From: slavas at Voltaire.COM (Slava Strebkov) Date: Mon, 27 Jul 2009 11:24:56 +0300 Subject: [ofa-general] [PATCH 1/3 v3]opensm: Patch defines new infrastructure and functions for mgid-to-mlid storage Message-ID: <4A6D6458.7040705@Voltaire.COM> >From 3885d8fc7edab1a74f8530b1fec6cde5601eaeef Mon Sep 17 00:00:00 2001 From: Slava Strebkov Date: Mon, 27 Jul 2009 11:09:44 +0300 Patch defines new infrastructure and functions for implementation of storage mgroups into mlid holder. Signed-off-by: Slava Strebkov --- opensm/include/opensm/osm_multicast.h | 270 ++++++++++++++++++++++++++++++++- opensm/opensm/osm_multicast.c | 175 +++++++++++++++++++++- 2 files changed, 443 insertions(+), 2 deletions(-) diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h index 9a47de5..69ea06b 100644 --- a/opensm/include/opensm/osm_multicast.h +++ b/opensm/include/opensm/osm_multicast.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * @@ -107,6 +107,72 @@ typedef struct osm_mcast_mgr_ctxt { * * SEE ALSO *********/ +/****s* OpenSM: Multicast Group Holder/osm_mgrp_holder_t +* NAME +* osm_mgrp_holder_t +* +* DESCRIPTION +* Holder for mgroups. +* +* The osm_mgrp_t object should be treated as opaque and should +* be manipulated only through the provided functions. +* +* SYNOPSIS +*/ + +typedef struct osm_mgrp_holder { + cl_qmap_t mgrp_port_map; + cl_qlist_t mgrp_list; + osm_mtree_node_t *p_root; + ib_net16_t mlid; +} osm_mgrp_holder_t; + +/* +* FIELDS +* mgrp_list +* Map for mgroups. Must be first element!! +* +* mgrp_port_map +* Map of all ports joined same mlid +* +* mgrp_list +* List of mgroups having same mlid +* +* p_root +* Pointer to the root "tree node" in the single spanning tree +* for this multicast group holder.The nodes of the tree represent +* switches. Member ports are not represented in the tree. +* +* mlid +* mlid of current group holder +*/ + /****s* OpenSM: Multicast group Port /osm_mgrp_port _t +* NAME +* osm_mgrp_port _t +* +* DESCRIPTION +* Holder for pointers to mgroups and port guid. +* +* +* SYNOPSIS +*/ +typedef struct _osm_mgrp_port { + cl_map_item_t guid_item; + cl_qlist_t mgroups; + ib_net64_t port_guid; +} osm_mgrp_port_t; +/* +* FIELDS +* guid_item +* Map for ports. Must be first element +* +* mgroups +* List of mgroups opened by this port. +* +* portguid +* guid of port representing current structure +*/ + /****s* OpenSM: Multicast Group/osm_mgrp_t * NAME @@ -122,6 +188,8 @@ typedef struct osm_mcast_mgr_ctxt { */ typedef struct osm_mgrp { cl_fmap_item_t map_item; + cl_list_item_t mlid_item; + cl_list_item_t port_item; ib_net16_t mlid; osm_mtree_node_t *p_root; cl_qmap_t mcm_port_tbl; @@ -137,6 +205,12 @@ typedef struct osm_mgrp { * map_item * Map Item for fmap linkage. Must be first element!! * +* mlid_item +* List item for groups with same MLID +* +* port_item +* List item for groups opened on same port +* * mlid * The network ordered LID of this Multicast Group (must be * >= 0xC000). @@ -489,6 +563,200 @@ osm_mgrp_apply_func(const osm_mgrp_t * const p_mgrp, * SEE ALSO * Multicast Group *********/ +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_new +* NAME +* osm_mgrp_holder_new +* +* DESCRIPTION +* Allocates and initializes a Multicast Group Holder for use. +* +* SYNOPSIS +*/ +osm_mgrp_holder_t *osm_mgrp_holder_new(IN osm_subn_t * p_subn, + IN ib_net16_t mlid); +/* +* PARAMETERS +* p_subn +* (in) pointer to osm_subnet +* mlid +* [in] Multicast LID for this multicast group holder. +* +* RETURN VALUES +* pointer to initialized osm_mgrp_holder_t +* or NULL, if unsuccessful +* +* SEE ALSO +* Multicast Group Holder, osm_mgrp_holder_delete +*********/ +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_delete +* NAME +* osm_mgrp_holder_delete +* +* DESCRIPTION +* Removes entry from array of holders +* Removes port from mgroup port list +* +* SYNOPSIS +*/ +void osm_mgrp_holder_delete(IN osm_subn_t * p_subn, IN ib_net16_t mlid); + +/* +* PARAMETERS +* +* p_subn +* [in] Pointer to osm_subnet +* +* mlid +* [in] holder's mlid +* +* RETURN VALUES +* None. +* +* NOTES +* +* SEE ALSO +* +*********/ +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_add_mgrp_port +* NAME +* osm_mgrp_holder_port_add_mgrp +* +* DESCRIPTION +* Allocates osm_mgrp_port_t for new port joined to mgroup with mlid of this holder, +* (or / and) adds mgroup to mgroup map of existed osm_mgrp_port_t object. +* +* SYNOPSIS +*/ +ib_api_status_t osm_mgrp_holder_port_add_mgrp(IN osm_mgrp_holder_t * + p_mgrp_holder, + IN osm_mgrp_t * p_mgrp, + IN ib_net64_t port_guid); + +/* +* PARAMETERS +* p_mgrp_holder +* (in) pointer to osm_mgrp_holder_t +* p_mgrp +* (in) pointer to osm_mgrp_t +* +* RETURN VALUES +* IB_SUCCESS or +* IB_INSUFFICIENT_MEMORY +* +* SEE ALSO +* Multicast Group Holder, osm_mgrp_holder_delete_mgrp_port +*********/ +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_delete_mgrp_port +* NAME +* osm_mgrp_holder_port_delete_mgrp +* +* DESCRIPTION +* Deletes osm_mgrp_port_t for specified port +* +* SYNOPSIS +*/ +void osm_mgrp_holder_port_delete_mgrp(IN osm_mgrp_holder_t * p_mgrp_holder, + IN osm_mgrp_t * p_mgrp, + IN ib_net64_t port_guid); + +/* +* PARAMETERS +* p_mgrp_holder +* [in] Pointer to an osm_mgrp_holder_t object. +* +* p_mgrp +* (in) Pointer to osm_mgrp_t object +* +* port_guid +* [in] Port guid of the departing port. +* +* RETURN VALUES +* None. +* +* NOTES +* +* SEE ALSO + Multicast Group Holder,osm_holder_add_mgrp_port +*********/ +/****f* OpenSM: Multicast Group Holder /osm_mgrp_holder_remove_port +* NAME +* osm_mgrp_holder_remove_port +* +* DESCRIPTION +* Removes osm_mgrp_port_t from mgrp_port_map of holder +* Removes port from mgroup port list +* +* SYNOPSIS +*/ +void osm_mgrp_holder_remove_port(IN osm_subn_t * const p_subn, + IN osm_log_t * const p_log, + IN osm_mgrp_holder_t * const p_mgrp_holder, + IN const ib_net64_t port_guid); + +/* +* PARAMETERS +* +* p_subn +* [in] Pointer to the subnet object +* +* p_log +* [in] The log object pointer +* +* p_mgrp_holder +* [in] Pointer to an osm_mgrp_holder_t object. +* +* port_guid +* [in] Port guid of the departing port. +* +* RETURN VALUES +* None. +* +* NOTES +* +* SEE ALSO +* +*********/ +/****f* OpenSM: Subnet/osm_get_mgrp_by_mlid +* NAME +* osm_get_mgrp_by_mlid +* +* DESCRIPTION +* The looks for the given multicast group in the subnet table by mlid. +* NOTE: this code is not thread safe. Need to grab the lock before +* calling it. +* +* SYNOPSIS +*/ +static inline struct osm_mgrp_holder *osm_get_mgrp_holder_by_mlid(osm_subn_t const + *p_subn, + ib_net16_t + mlid) +{ + return p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO]; +} + +/* +* PARAMETERS +* p_subn +* [in] Pointer to an osm_subn_t object +* +* mlid +* [in] The multicast group mlid in network order +* +* RETURN VALUES +* The multicast group structure pointer if found. NULL otherwise. +*********/ +static inline ib_net16_t osm_mgrp_holder_get_mlid(IN osm_mgrp_holder_t * + const p_mgrp_holder) +{ + return (p_mgrp_holder->mlid); +} + +static inline boolean_t osm_mgrp_holder_is_empty(IN const osm_mgrp_holder_t * + const p_mgrp_holder) +{ + return (cl_qmap_count(&p_mgrp_holder->mgrp_port_map) == 0); +} END_C_DECLS #endif /* _OSM_MULTICAST_H_ */ diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c index d2733c4..2106a63 100644 --- a/opensm/opensm/osm_multicast.c +++ b/opensm/opensm/osm_multicast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. + * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * @@ -298,3 +298,176 @@ void osm_mgrp_apply_func(const osm_mgrp_t * p_mgrp, osm_mgrp_func_t p_func, if (p_mtn) mgrp_apply_func_sub(p_mgrp, p_mtn, p_func, context); } + +/********************************************************************** + **********************************************************************/ +static osm_mgrp_port_t *osm_mgrp_port_new(ib_net64_t port_guid) +{ + osm_mgrp_port_t *p_mgrp_port = + (osm_mgrp_port_t *) malloc(sizeof(osm_mgrp_port_t)); + if (!p_mgrp_port) { + return NULL; + } + memset(p_mgrp_port, 0, sizeof(*p_mgrp_port)); + p_mgrp_port->port_guid = port_guid; + cl_qlist_init(&p_mgrp_port->mgroups); + return p_mgrp_port; +} + +/********************************************************************** + **********************************************************************/ +osm_mgrp_holder_t *osm_mgrp_holder_new(IN osm_subn_t * p_subn, + ib_net16_t mlid) +{ + osm_mgrp_holder_t *p_mgrp_holder; + p_mgrp_holder = + p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO] = + (osm_mgrp_holder_t *) malloc(sizeof(*p_mgrp_holder)); + if (!p_mgrp_holder) + return NULL; + + memset(p_mgrp_holder, 0, sizeof(*p_mgrp_holder)); + p_mgrp_holder->mlid = mlid; + cl_qmap_init(&p_mgrp_holder->mgrp_port_map); + cl_qlist_init(&p_mgrp_holder->mgrp_list); + return p_mgrp_holder; +} + +/********************************************************************** + **********************************************************************/ +void osm_mgrp_holder_delete(IN osm_subn_t *p_subn, osm_mgrp_t *p_mgrp) +{ + osm_mgrp_port_t *p_osm_mgr_port; + cl_map_item_t *p_item; + + osm_mgrp_holder_t *p_mgrp_holder = + p_subn->mgroup_holders[cl_ntoh16(p_mgrp->mlid) - IB_LID_MCAST_START_HO]; + p_item = cl_qmap_head(&p_mgrp_holder->mgrp_port_map); + /* Delete ports shared same MLID */ + while (p_item != cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { + p_osm_mgr_port = (osm_mgrp_port_t *) p_item; + cl_qlist_remove_all(&p_osm_mgr_port->mgroups); + cl_qmap_remove_item(&p_mgrp_holder->mgrp_port_map, p_item); + p_item = cl_qmap_head(&p_mgrp_holder->mgrp_port_map); + free(p_osm_mgr_port); + } + /* Remove mgrp from this MLID */ + cl_qlist_remove_item(&p_mgrp_holder->mgrp_list,&p_mgrp->mlid_item); + /* Destroy the mtree_node structure */ + osm_mtree_destroy(p_mgrp_holder->p_root); + p_subn->mgroup_holders[cl_ntoh16(mlid) - IB_LID_MCAST_START_HO] = NULL; + free(p_mgrp_holder); +} + +/********************************************************************** + **********************************************************************/ +void osm_mgrp_holder_remove_port(osm_subn_t * subn, osm_log_t * p_log, + osm_mgrp_holder_t * p_mgrp_holder, + ib_net64_t port_guid) +{ + osm_mgrp_t *p_mgrp; + cl_list_item_t *p_item; + + OSM_LOG_ENTER(p_log); + + osm_mgrp_port_t *p_mgrp_port = (osm_mgrp_port_t *) + cl_qmap_remove(&p_mgrp_holder->mgrp_port_map, port_guid); + if (p_mgrp_port != + (osm_mgrp_port_t *) cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { + char gid_str[INET6_ADDRSTRLEN]; + OSM_LOG(p_log, OSM_LOG_DEBUG, + "port 0x%" PRIx64 " removed from mlid 0x%X\n", + port_guid, cl_ntoh16(p_mgrp_holder->mlid)); + while ((p_item = + cl_qlist_remove_head(&p_mgrp_port->mgroups)) != + cl_qlist_end(&p_mgrp_port->mgroups)) { + p_mgrp = (osm_mgrp_t *) + PARENT_STRUCT(p_item, osm_mgrp_t,port_item); + OSM_LOG(p_log, OSM_LOG_DEBUG, + "removing mgrp mgid %s from port 0x%" PRIx64"\n", + inet_ntop(AF_INET6,p_mgrp->mcmember_rec.mgid.raw, + gid_str, sizeof(gid_str)), + cl_ntoh64(port_guid)); + osm_mgrp_delete_port(subn, p_log, p_mgrp, port_guid); + } + free(p_mgrp_port); + } + OSM_LOG_EXIT(p_log); +} + +/********************************************************************** + **********************************************************************/ +void osm_mgrp_holder_add_mgrp(osm_mgrp_holder_t * p_mgrp_holder, + osm_mgrp_t * p_mgrp) +{ + char gid_str[INET6_ADDRSTRLEN]; + + OSM_LOG_ENTER(p_log); + cl_qlist_insert_tail(&p_mgrp_holder->mgrp_list, &p_mgrp->mlid_item); + OSM_LOG(p_log, OSM_LOG_DEBUG, + "mgrp with MGID:%s added to holder with mlid = 0x%X\n", + inet_ntop(AF_INET6, p_mgrp->mcmember_rec.mgid.raw, gid_str, + sizeof(gid_str)), cl_ntoh16(p_mgrp_holder->mlid)); + p_mgrp_holder->last_change_id++; + OSM_LOG_EXIT(p_log); +} + +/********************************************************************** + **********************************************************************/ +void osm_mgrp_holder_delete_mgrp(osm_mgrp_holder_t * p_mgrp_holder, + osm_mgrp_t * p_mgrp) +{ + p_mgrp->to_be_deleted = 1; + if (0 == cl_qlist_count(&p_mgrp_holder->mgrp_list)) { + /* No more mgroups on this mlid */ + p_mgrp_holder->to_be_deleted = 1; + p_mgrp_holder->last_tree_id = 0; + p_mgrp_holder->last_change_id = 0; + } +} + +/********************************************************************** + **********************************************************************/ +ib_api_status_t osm_mgrp_holder_port_add_mgrp(osm_mgrp_holder_t * p_mgrp_holder, + osm_mgrp_t * p_mgrp, + ib_net64_t port_guid) +{ + osm_mgrp_port_t *p_mgrp_port = (osm_mgrp_port_t *) + cl_qmap_get(&p_mgrp_holder->mgrp_port_map, port_guid); + if (p_mgrp_port == + (osm_mgrp_port_t *) cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { + /* new port to mlid */ + p_mgrp_port = osm_mgrp_port_new(port_guid); + if (!p_mgrp_port) { + return IB_INSUFFICIENT_MEMORY; + } + cl_qmap_insert(&p_mgrp_holder->mgrp_port_map, + p_mgrp_port->port_guid, &p_mgrp_port->guid_item); + cl_qlist_insert_tail(&p_mgrp_port->mgroups, &p_mgrp->port_item); + } else { + cl_qlist_insert_tail(&p_mgrp_port->mgroups, &p_mgrp->port_item); + } + return IB_SUCCESS; +} + +/********************************************************************** + **********************************************************************/ +void osm_mgrp_holder_port_delete_mgrp(osm_mgrp_holder_t * p_mgrp_holder, + osm_mgrp_t * p_mgrp, + ib_net64_t port_guid) +{ + osm_mgrp_port_t *p_mgrp_port = (osm_mgrp_port_t *) + cl_qmap_get(&p_mgrp_holder->mgrp_port_map, port_guid); + if (p_mgrp_port != + (osm_mgrp_port_t *) cl_qmap_end(&p_mgrp_holder->mgrp_port_map)) { + cl_qlist_remove_item(&p_mgrp_port->mgroups, &p_mgrp->port_item); + if (0 == cl_qlist_count(&p_mgrp_port->mgroups)) { + /* No mgroups registered on this port for current mlid */ + cl_qmap_remove_item(&p_mgrp_holder->mgrp_port_map, + &p_mgrp_port->guid_item); + free(p_mgrp_port); + } + p_mgrp_holder->last_change_id++; + } +} + -- 1.5.5 From sebastien.dugue at bull.net Mon Jul 27 02:02:33 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Mon, 27 Jul 2009 11:02:33 +0200 Subject: [ofa-general] [PATCH] V3 libmlx4 - Optimize memory allocation of buffers Message-ID: <20090727110233.437ac165@frecb007965> Hi, here is a respin of the buffers allocation optimization patch. Changes V2 -> V3: ---------------- - Allocate all aligned buffers with mmap(), not only QP buffers. Changes V1 -> V2: ---------------- - Use mmap whatever the page size, not only with 64K pages. Buffers are allocated with mthca_alloc_buf(), which rounds the buffers size to the page size and then allocates page aligned memory using posix_memalign(). However, this allocation is quite wasteful on architectures using 64K pages (ia64 for example) because we then hit glibc's MMAP_THRESHOLD malloc parameter and chunks are allocated using mmap. thus we end up allocating: (requested size rounded to the page size) + (page size) + (malloc overhead) rounded internally to the page size. So for example, if we request a buffer of page_size bytes, we end up consuming 3 pages. In short, for each buffer we allocate, there is an overhead of 2 pages. This is quite visible on large clusters especially where the number of QP can reach several thousands. This patch replaces the call to posix_memalign() in mthca_alloc_buf() with a direct call to mmap(). Signed-off-by: Sebastien Dugue --- src/buf.c | 21 +++++++++++++-------- 1 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/buf.c b/src/buf.c index 0e5f9b6..529affc 100644 --- a/src/buf.c +++ b/src/buf.c @@ -35,6 +35,8 @@ #endif /* HAVE_CONFIG_H */ #include +#include +#include #include "mlx4.h" @@ -61,16 +63,19 @@ int mlx4_alloc_buf(struct mlx4_buf *buf, size_t size, int page_size) { int ret; - ret = posix_memalign(&buf->buf, page_size, align(size, page_size)); - if (ret) - return ret; + /* Use mmap directly to allocate an aligned buffer */ + buf->buf = mmap(0 ,align(size, page_size) , PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + if (buf->buf == MAP_FAILED) + return errno; ret = ibv_dontfork_range(buf->buf, size); - if (ret) - free(buf->buf); - if (!ret) - buf->length = size; + if (ret) + munmap(buf->buf, align(size, page_size)); + else + buf->length = align(size, page_size); return ret; } @@ -78,5 +83,5 @@ int mlx4_alloc_buf(struct mlx4_buf *buf, size_t size, int page_size) void mlx4_free_buf(struct mlx4_buf *buf) { ibv_dofork_range(buf->buf, buf->length); - free(buf->buf); + munmap(buf->buf, buf->length); } -- 1.6.3.1 From sebastien.dugue at bull.net Mon Jul 27 02:02:31 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Mon, 27 Jul 2009 11:02:31 +0200 Subject: [ofa-general] [PATCH] V3 libmthca - Optimize memory allocation of buffers Message-ID: <20090727110231.7275c15b@frecb007965> Hi, here is a respin of the buffers allocation optimization patch. Changes V2 -> V3: ---------------- - Allocate all aligned buffers with mmap(), not only QP buffers. Changes V1 -> V2: ---------------- - Use mmap whatever the page size, not only with 64K pages. Buffers are allocated with mthca_alloc_buf(), which rounds the buffers size to the page size and then allocates page aligned memory using posix_memalign(). However, this allocation is quite wasteful on architectures using 64K pages (ia64 for example) because we then hit glibc's MMAP_THRESHOLD malloc parameter and chunks are allocated using mmap. thus we end up allocating: (requested size rounded to the page size) + (page size) + (malloc overhead) rounded internally to the page size. So for example, if we request a buffer of page_size bytes, we end up consuming 3 pages. In short, for each buffer we allocate, there is an overhead of 2 pages. This is quite visible on large clusters especially where the number of QP can reach several thousands. This patch replaces the call to posix_memalign() in mthca_alloc_buf() with a direct call to mmap(). Signed-off-by: Sebastien Dugue --- src/buf.c | 21 +++++++++++++-------- 1 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/buf.c b/src/buf.c index 6c1be4f..985c1f7 100644 --- a/src/buf.c +++ b/src/buf.c @@ -35,6 +35,8 @@ #endif /* HAVE_CONFIG_H */ #include +#include +#include #include "mthca.h" @@ -61,16 +63,19 @@ int mthca_alloc_buf(struct mthca_buf *buf, size_t size, int page_size) { int ret; - ret = posix_memalign(&buf->buf, page_size, align(size, page_size)); - if (ret) - return ret; + /* Use mmap directly to allocate an aligned buffer */ + buf->buf = mmap(0 ,align(size, page_size) , PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + if (buf->buf == MAP_FAILED) + return errno; ret = ibv_dontfork_range(buf->buf, size); - if (ret) - free(buf->buf); - if (!ret) - buf->length = size; + if (ret) + munmap(buf->buf, align(size, page_size)); + else + buf->length = align(size, page_size); return ret; } @@ -78,5 +83,5 @@ int mthca_alloc_buf(struct mthca_buf *buf, size_t size, int page_size) void mthca_free_buf(struct mthca_buf *buf) { ibv_dofork_range(buf->buf, buf->length); - free(buf->buf); + munmap(buf->buf, buf->length); } -- 1.6.3.1 From sebastien.dugue at bull.net Mon Jul 27 02:08:06 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Mon, 27 Jul 2009 11:08:06 +0200 Subject: [ofa-general] [PATCH RESEND] mstflint - Fix redirection to /dev/null in hca_self_test.ofed Message-ID: <20090727110806.10f7c4be@frecb007965> Redirect 'rpm -qa' stderr to /dev/null instead of null. Signed-Off-By: Sebastien Dugue --- hca_self_test.ofed | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/hca_self_test.ofed b/hca_self_test.ofed index c7c5492..4f29080 100755 --- a/hca_self_test.ofed +++ b/hca_self_test.ofed @@ -168,7 +168,7 @@ else fi if [ $RPM_KER_VER -ne 0 ]; then - RPM_CUR_BOOTED_KER=`rpm -qa 2> null| grep kernel-ib | grep $(echo $BOOTED_KER | sed s/-/_/) | wc -l` + RPM_CUR_BOOTED_KER=`rpm -qa 2> /dev/null| grep kernel-ib | grep $(echo $BOOTED_KER | sed s/-/_/) | wc -l` if [ $RPM_CUR_BOOTED_KER -eq 0 ]; then echo -e "Host Driver RPM Check .................. ${red}FAIL" tput sgr0 -- 1.6.3.1 From sebastien.dugue at bull.net Mon Jul 27 02:11:13 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Mon, 27 Jul 2009 11:11:13 +0200 Subject: [ofa-general] [PATCH RESEND] perftest - Fix proc_get_cpu_mhz() on IA64 Message-ID: <20090727111113.14b90dc8@frecb007965> On IA64, proc_get_cpu_mhz() must use the ITC frequency rather than the CPU frequency. Signed-off-by: Sebastien Dugue --- get_clock.c | 18 +++++++++++++----- 1 files changed, 13 insertions(+), 5 deletions(-) diff --git a/get_clock.c b/get_clock.c index 0acb074..cc86452 100755 --- a/get_clock.c +++ b/get_clock.c @@ -144,12 +144,20 @@ static double proc_get_cpu_mhz(int no_cpu_freq_fail) while(fgets(buf, sizeof(buf), f)) { double m; int rc; + +#if defined (__ia64__) + /* Use the ITC frequency on IA64 */ + rc = sscanf(buf, "itc MHz : %lf", &m); +#elif defined (__PPC__) || defined (__PPC64__) + /* PPC has a different format as well */ + rc = sscanf(buf, "clock : %lf", &m); +#else rc = sscanf(buf, "cpu MHz : %lf", &m); - if (rc != 1) { /* PPC has a different format */ - rc = sscanf(buf, "clock : %lf", &m); - if (rc != 1) - continue; - } +#endif + + if (rc != 1) + continue; + if (mhz == 0.0) { mhz = m; continue; -- 1.6.3.1 From vlad at lists.openfabrics.org Mon Jul 27 02:56:12 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Mon, 27 Jul 2009 02:56:12 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090727-0200 daily build status Message-ID: <20090727095613.2593D10201B5@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Passed on i686 with linux-2.6.19 Passed on i686 with linux-2.6.18 Passed on i686 with linux-2.6.21.1 Passed on i686 with linux-2.6.22 Passed on x86_64 with linux-2.6.18 Passed on x86_64 with linux-2.6.21.1 Passed on x86_64 with linux-2.6.19 Passed on x86_64 with linux-2.6.20 Passed on x86_64 with linux-2.6.22 Passed on ia64 with linux-2.6.18 Passed on ia64 with linux-2.6.19 Passed on ia64 with linux-2.6.21.1 Passed on ia64 with linux-2.6.22 Passed on ppc64 with linux-2.6.18 Passed on ppc64 with linux-2.6.19 Failed: Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.23_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.23_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.23_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090727-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From sebastien.dugue at bull.net Mon Jul 27 04:37:14 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Mon, 27 Jul 2009 13:37:14 +0200 Subject: [ofa-general] [PATCH] ibsim - Fix umad2sim build with glibc >= 2.10 Message-ID: <20090727133715.40e25f45@frecb007965> Hi Sasha, Starting with glibc 2.10 scandir() was made POSIX compliant which changed the prototype if the comparison function from: int (*compar) (const void *, const void *) to: int (*compar) (const struct dirent **, const struct dirent **) This patch detects glibc's version to use the correct prototype. Signed-off-by: Sebastien Dugue --- umad2sim/umad2sim.c | 13 +++++++++++++ 1 files changed, 13 insertions(+), 0 deletions(-) diff --git a/umad2sim/umad2sim.c b/umad2sim/umad2sim.c index 6b10f28..55440ec 100644 --- a/umad2sim/umad2sim.c +++ b/umad2sim/umad2sim.c @@ -94,9 +94,16 @@ static ssize_t(*real_write) (int fd, const void *buf, size_t count); static int (*real_poll) (struct pollfd * pfds, nfds_t nfds, int timeout); static int (*real_ioctl) (int d, int request, ...); static DIR *(*real_opendir) (const char *dir); +#if __GLIBC_PREREQ(2,10) +static int (*real_scandir) (const char *dir, struct dirent *** namelist, + int (*filter) (const struct dirent *), + int (*compar) (const struct dirent **, + const struct dirent **)); +#else static int (*real_scandir) (const char *dir, struct dirent *** namelist, int (*filter) (const struct dirent *), int (*compar) (const void *, const void *)); +#endif static char sysfs_infiniband_dir[] = SYS_INFINIBAND; static char sysfs_infiniband_mad_dir[] = IB_UMAD_ABI_DIR; @@ -694,9 +701,15 @@ DIR *opendir(const char *path) return real_opendir(path); } +#if __GLIBC_PREREQ(2,10) +int scandir(const char *path, struct dirent ***namelist, + int (*filter) (const struct dirent *), + int (*compar) (const struct dirent **, const struct dirent **)) +#else int scandir(const char *path, struct dirent ***namelist, int (*filter) (const struct dirent *), int (*compar) (const void *, const void *)) +#endif { char new_path[4096]; -- 1.6.3.1 From sebastien.dugue at bull.net Mon Jul 27 04:38:01 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Mon, 27 Jul 2009 13:38:01 +0200 Subject: [ofa-general] [PATCH] ibsim: Drop vendor MADs and vendor specific SM MADs Message-ID: <20090727133801.38451235@frecb007965> This is useful for testing applications that send those kind of MADS. Signed-off-by: Sebastien Dugue --- ibsim/sim_mad.c | 19 +++++++++++++++++++ 1 files changed, 19 insertions(+), 0 deletions(-) diff --git a/ibsim/sim_mad.c b/ibsim/sim_mad.c index 61d4866..b79a5e7 100644 --- a/ibsim/sim_mad.c +++ b/ibsim/sim_mad.c @@ -1170,6 +1170,25 @@ int process_packet(Client * cl, void *p, int size, Client ** dcl) if ((response = decode_sim_MAD(cl, r, &rpc, &path, data)) < 0) return -1; + /* Drop vendor MADs */ + if (((rpc.mgtclass >= IB_VENDOR_RANGE1_START_CLASS) && + (rpc.mgtclass <= IB_VENDOR_RANGE1_END_CLASS)) || + ((rpc.mgtclass >= IB_VENDOR_RANGE2_START_CLASS) && + (rpc.mgtclass <= IB_VENDOR_RANGE2_END_CLASS))) { + IBWARN("ignoring vendor MAD: class 0x%x, attr 0x%x", + rpc.mgtclass, rpc.attr.id); + goto _dropped; + } + + /* Drop vendor specific SM MADs */ + if (((rpc.mgtclass == IB_SMI_CLASS) || + (rpc.mgtclass == IB_SMI_DIRECT_CLASS)) && + (rpc.attr.id >= 0xff00)) { + IBWARN("ignoring vendor specific SM MAD: attr 0x%x", + rpc.attr.id); + goto _dropped; + } + if (rpc.method == 0x7) { IBWARN("lid %u got trap repress - dropping", ntohs(r->dlid)); *dcl = 0; -- 1.6.3.1 From sebastien.dugue at bull.net Mon Jul 27 04:38:35 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Mon, 27 Jul 2009 13:38:35 +0200 Subject: [ofa-general] [PATCH] ibsim: Allow multi-threaded use of ibsim Message-ID: <20090727133835.4883cee4@frecb007965> Right now, the socket name used between the client and ibsim is appended with the client's pid. However this does not work when several threads with the same pid want to register their clients. instead of the pid, use a combination of the thread pid along with a connection number incremented for each new client. Also, attach the agents to the file instance opened on the device rather than on the device itself. Signed-off-by: Sebastien Dugue --- umad2sim/sim_client.c | 11 ++- umad2sim/umad2sim.c | 171 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 140 insertions(+), 42 deletions(-) diff --git a/umad2sim/sim_client.c b/umad2sim/sim_client.c index eb42a7c..22a50a2 100644 --- a/umad2sim/sim_client.c +++ b/umad2sim/sim_client.c @@ -209,6 +209,8 @@ static int sim_init(struct sim_client *sc, char *nodeid) char *connect_port; char *connect_host; unsigned short port; + static int conn_num = 0; + int conn_id; connect_port = getenv("IBSIM_SERVER_PORT"); connect_host = getenv("IBSIM_SERVER_NAME"); @@ -228,7 +230,10 @@ static int sim_init(struct sim_client *sc, char *nodeid) if ((ctlfd = socket(remote_mode ? PF_INET : PF_LOCAL, SOCK_DGRAM, 0)) < 0) IBPANIC("can't get socket (ctlfd)"); - size = make_name(&name, NULL, 0, "%s:ctl%d", socket_basename, pid); + conn_id = ((conn_num & 0xff) << 24) | (pid & 0xffffff); + conn_num++; + + size = make_name(&name, NULL, 0, "%s:ctl%d", socket_basename, conn_id); if (bind(ctlfd, (struct sockaddr *)&name, size) < 0) IBPANIC("can't bind ctl socket"); @@ -243,7 +248,7 @@ static int sim_init(struct sim_client *sc, char *nodeid) sc->fd_ctl = ctlfd; - size = make_name(&name, NULL, 0, "%s:in%d", socket_basename, pid); + size = make_name(&name, NULL, 0, "%s:in%d", socket_basename, conn_id); if (bind(fd, (struct sockaddr *)&name, size) < 0) IBPANIC("can't bind input socket"); @@ -254,7 +259,7 @@ static int sim_init(struct sim_client *sc, char *nodeid) IBPANIC("can't read data from bound socket"); port = ntohs(name.name_i.sin_port); - sc->clientid = sim_connect(sc, remote_mode ? port : pid, 0, nodeid); + sc->clientid = sim_connect(sc, remote_mode ? port : conn_id, 0, nodeid); if (sc->clientid < 0) IBPANIC("connect failed"); diff --git a/umad2sim/umad2sim.c b/umad2sim/umad2sim.c index 55440ec..71f7c67 100644 --- a/umad2sim/umad2sim.c +++ b/umad2sim/umad2sim.c @@ -81,12 +81,17 @@ struct umad2sim_dev { char name[32]; uint8_t port; struct sim_client sim_client; - unsigned agent_idx[256]; - struct ib_user_mad_reg_req agents[32]; char umad_path[256]; char issm_path[256]; }; +struct umad2sim_devfile { + struct umad2sim_dev *dev; + struct sim_client sim_client; + struct ib_user_mad_reg_req agents[32]; + unsigned agent_idx[256]; +}; + static int (*real_open) (const char *path, int flags, ...); static int (*real_close) (int fd); static ssize_t(*real_read) (int fd, void *buf, size_t count); @@ -113,6 +118,7 @@ static char umad2sim_sysfs_prefix[32]; static unsigned umad2sim_initialized; static struct umad2sim_dev *devices[32]; +static struct umad2sim_devfile *devfiles[1024]; /* * sysfs stuff @@ -378,9 +384,69 @@ static int dev_sysfs_create(struct umad2sim_dev *dev) * umad2sim device * */ +static int umad2sim_open(struct umad2sim_dev *dev) +{ + int i; + int fd; + struct umad2sim_devfile *file; + + /* Find unused fd */ + for (fd = 0; fd < 1024; fd++) + if (devfiles[fd] == NULL) + break; + + if (fd == 1024) { + ERROR("umad2sim_open: No more available files\n"); + errno = EMFILE; + return -1; + } -static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) + if ((file = calloc(1, sizeof(struct umad2sim_devfile))) == NULL) { + ERROR("umad2sim_open: Out of memory\n"); + errno = ENOMEM; + return -1; + } + + file->dev = dev; + + for (i = 0; i < arrsize(file->agents); i++) + file->agents[i].id = (uint32_t)(-1); + + for (i = 0; i < arrsize(file->agent_idx); i++) + file->agent_idx[i] = (unsigned)(-1); + + if (sim_client_init(&file->sim_client) < 0) { + free(file); + return -1; + } + + devfiles[fd] = file; + + return 1024 + fd; +} + +static int umad2sim_close(int fd) { + struct umad2sim_devfile *file; + + if ((file = devfiles[fd - 1024]) == NULL) { + errno = EBADF; + ERROR("umad2sim_close: no such file\n"); + return -1; + } + + sim_client_exit(&file->sim_client); + + free(file); + + devfiles[fd - 1024] = NULL; + + return 0; +} + +static ssize_t umad2sim_read(int fd, void *buf, size_t count) +{ + struct umad2sim_devfile *file; struct sim_request req; ib_user_mad_t *umad = (ib_user_mad_t *) buf; unsigned mgmt_class; @@ -388,7 +454,13 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) DEBUG("umad2sim_read: %zu...\n", count); - cnt = real_read(dev->sim_client.fd_pktin, &req, sizeof(req)); + if ((file = devfiles[fd - 1024]) == NULL) { + errno = EBADF; + ERROR("umad2sim_read: no such file\n"); + return -1; + } + + cnt = real_read(file->sim_client.fd_pktin, &req, sizeof(req)); DEBUG("umad2sim_read: got %d...\n", cnt); if (cnt < sizeof(req)) { ERROR("umad2sim_read: partial request - skip.\n"); @@ -406,7 +478,7 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) mad_get_field(req.mad, 0, IB_MAD_ATTRID_F), mad_get_field(req.mad, 0, IB_MAD_ATTRMOD_F)); - if (mgmt_class >= arrsize(dev->agent_idx)) { + if (mgmt_class >= arrsize(file->agent_idx)) { ERROR("bad mgmt_class 0x%x\n", mgmt_class); mgmt_class = 0; } @@ -415,7 +487,7 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) uint64_t trid = mad_get_field64(req.mad, 0, IB_MAD_TRID_F); umad->agent_id = (trid >> 32) & 0xffff; } else - umad->agent_id = dev->agent_idx[mgmt_class]; + umad->agent_id = file->agent_idx[mgmt_class]; umad->status = ntohl(req.status); umad->timeout_ms = 0; @@ -437,9 +509,9 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) return umad->length; } -static ssize_t umad2sim_write(struct umad2sim_dev *dev, - const void *buf, size_t count) +static ssize_t umad2sim_write(int fd, const void *buf, size_t count) { + struct umad2sim_devfile *file; struct sim_request req; ib_user_mad_t *umad = (ib_user_mad_t *) buf; int cnt; @@ -457,12 +529,18 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, DEBUG("umad2sim_write: %zu...\n", count); + if ((file = devfiles[fd - 1024]) == NULL) { + errno = EBADF; + ERROR("umad2sim_write: no such file\n"); + return -1; + } + DEBUG("umad2sim_write: umad: agent_id=%u, retries=%u, " "agent.class=%x, agent.qpn=%u, " "addr.qpn=%u, addr.lid=%u\n", umad->agent_id, umad->retries, - dev->agents[umad->agent_id].mgmt_class, - dev->agents[umad->agent_id].qpn, + file->agents[umad->agent_id].mgmt_class, + file->agents[umad->agent_id].qpn, htonl(umad->addr.qpn), htons(umad->addr.lid)); DEBUG("umad2sim_write: mad: method=%x, response=%x, mgmtclass=%x, " "attrid=%x, attrmod=%x\n", @@ -476,7 +554,7 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, req.slid = req.dlid == 0xffff ? 0xffff : 0; /* 0 - means auto (supported by ibsim) */ ; req.dqp = umad->addr.qpn; - req.sqp = htonl(dev->agents[umad->agent_id].qpn); + req.sqp = htonl(file->agents[umad->agent_id].qpn); req.status = 0; cnt = count - umad_size(); @@ -492,7 +570,7 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, mad_set_field64(req.mad, 0, IB_MAD_TRID_F, trid); } - cnt = write(dev->sim_client.fd_pktout, (void *)&req, sizeof(req)); + cnt = write(file->sim_client.fd_pktout, (void *)&req, sizeof(req)); if (cnt < 0) { ERROR("umad2sim_write: cannot write\n"); return -1; @@ -503,19 +581,21 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, return count; } -static int register_agent(struct umad2sim_dev *dev, +static int register_agent(struct umad2sim_devfile *file, struct ib_user_mad_reg_req *req) { - unsigned i; + unsigned i; + DEBUG("register_agent: id = %u, qpn = %u, mgmt_class = %u," " mgmt_class_version = %u, rmpp_version = %u\n", req->id, req->qpn, req->mgmt_class, req->mgmt_class_version, req->rmpp_version); - for (i = 0; i < arrsize(dev->agents); i++) - if (dev->agents[i].id == (uint32_t)(-1)) { + + for (i = 0; i < arrsize(file->agents); i++) + if (file->agents[i].id == (uint32_t)(-1)) { req->id = i; - dev->agents[i] = *req; - dev->agent_idx[req->mgmt_class] = i; + file->agents[i] = *req; + file->agent_idx[req->mgmt_class] = i; DEBUG("agent registered: %d\n", i); return 0; } @@ -523,28 +603,36 @@ static int register_agent(struct umad2sim_dev *dev, return -1; } -static int unregister_agent(struct umad2sim_dev *dev, unsigned id) +static int unregister_agent(struct umad2sim_devfile *file, unsigned id) { unsigned mgmt_class; - if (id >= arrsize(dev->agents)) { + if (id >= arrsize(file->agents)) { errno = EINVAL; return -1; } - mgmt_class = dev->agents[id].mgmt_class; - dev->agents[id].id = (uint32_t)(-1); - dev->agent_idx[mgmt_class] = -1; + mgmt_class = file->agents[id].mgmt_class; + file->agents[id].id = (uint32_t)(-1); + file->agent_idx[mgmt_class] = -1; return 0; } -static int umad2sim_ioctl(struct umad2sim_dev *dev, unsigned long request, - void *arg) +static int umad2sim_ioctl(int fd, unsigned long request, void *arg) { + struct umad2sim_devfile *file; + DEBUG("umad2sim_ioctl: %lu, %p...\n", request, arg); + + if ((file = devfiles[fd - 1024]) == NULL) { + errno = EBADF; + ERROR("umad2sim_ioctl: no such file\n"); + return -1; + } + switch (request) { case IB_USER_MAD_REGISTER_AGENT: - return register_agent(dev, arg); + return register_agent(file, arg); case IB_USER_MAD_UNREGISTER_AGENT: - return unregister_agent(dev, *((unsigned *)arg)); + return unregister_agent(file, *((unsigned *)arg)); case IB_USER_MAD_ENABLE_PKEY: return 0; default: @@ -556,7 +644,6 @@ static int umad2sim_ioctl(struct umad2sim_dev *dev, unsigned long request, static struct umad2sim_dev *umad2sim_dev_create(unsigned num, const char *name) { struct umad2sim_dev *dev; - unsigned i; DEBUG("umad2sim_dev_create: %s...\n", name); @@ -573,10 +660,6 @@ static struct umad2sim_dev *umad2sim_dev_create(unsigned num, const char *name) dev->port = mad_get_field(&dev->sim_client.portinfo, 0, IB_PORT_LOCAL_PORT_F); - for (i = 0; i < arrsize(dev->agents); i++) - dev->agents[i].id = (uint32_t)(-1); - for (i = 0; i < arrsize(dev->agent_idx); i++) - dev->agent_idx[i] = (unsigned)(-1); dev_sysfs_create(dev); @@ -637,6 +720,11 @@ static void umad2sim_cleanup(void) char path[1024]; unsigned i; DEBUG("umad2sim_cleanup...\n"); + + for (i = 0; i < 1024; i++) + if (devfiles[i]) + free(devfiles[i]); + for (i = 0; i < arrsize(devices); i++) if (devices[i]) { umad2sim_dev_delete(devices[i]); @@ -756,7 +844,7 @@ int open(const char *path, int flags, ...) if (!(dev = devices[i])) continue; if (!strncmp(path, dev->umad_path, sizeof(dev->umad_path))) { - return 1024 + i; + return umad2sim_open(dev); } if (!strncmp(path, dev->issm_path, sizeof(dev->issm_path))) { sim_client_set_sm(&dev->sim_client, 1); @@ -779,7 +867,7 @@ int close(int fd) sim_client_set_sm(&dev->sim_client, 0); return 0; } else if (fd >= 1024) { - return 0; + return umad2sim_close(fd); } else return real_close(fd); } @@ -791,7 +879,7 @@ ssize_t read(int fd, void *buf, size_t count) if (fd >= 2048) return -1; else if (fd >= 1024) - return umad2sim_read(devices[fd - 1024], buf, count); + return umad2sim_read(fd, buf, count); else return real_read(fd, buf, count); } @@ -803,7 +891,7 @@ ssize_t write(int fd, const void *buf, size_t count) if (fd >= 2048) return -1; else if (fd >= 1024) - return umad2sim_write(devices[fd - 1024], buf, count); + return umad2sim_write(fd, buf, count); else return real_write(fd, buf, count); } @@ -821,7 +909,7 @@ int ioctl(int fd, unsigned long request, ...) if (fd >= 2048) return -1; else if (fd >= 1024) - return umad2sim_ioctl(devices[fd - 1024], request, arg); + return umad2sim_ioctl(fd, request, arg); else return real_ioctl(fd, request, arg); } @@ -836,9 +924,14 @@ int poll(struct pollfd *pfds, nfds_t nfds, int timeout) for (i = 0; i < nfds; i++) { if (pfds[i].fd >= 1024 && pfds[i].fd < 2048) { - struct umad2sim_dev *dev = devices[pfds[i].fd - 1024]; + struct umad2sim_devfile *file; + + if ((file = devfiles[pfds[i].fd - 1024]) == NULL) { + errno = EBADF; + return -1; + } saved_fds[i] = pfds[i].fd; - pfds[i].fd = dev->sim_client.fd_pktin; + pfds[i].fd = file->sim_client.fd_pktin; } else saved_fds[i] = 0; } -- 1.6.3.1 From twbowman at gmail.com Mon Jul 27 06:26:09 2009 From: twbowman at gmail.com (Todd Bowman) Date: Mon, 27 Jul 2009 07:26:09 -0600 Subject: [ofa-general] IPoIB connections falling off In-Reply-To: <4A6C53B4.8090404@Voltaire.COM> References: <20090722145930.ca234b60.weiny2@llnl.gov> <4A6C53B4.8090404@Voltaire.COM> Message-ID: I haven't been able to replicate this problem, but when it occurs again I will check these out and send reply. Thanks, Todd On Sun, Jul 26, 2009 at 7:01 AM, Moni Shoua wrote: > Ira Weiny wrote: > > Check your multicast group membership and forwarding tables on the > switches. > > > > We have had similar issues and have found that some nodes fail to join > the multicast groups for various reasons. > > > > Ira > I think that a good start would be to send the following > 1. kernel neigh table > ip neigh show > > 2. ipoib path cache > cat /sys/kernel/debug/ipoib/ib0_path > (you probably need to mount debugfs first with 'mount -t debugfs none > /sys/kernel/debug') > > 2. ipoib mcast joins > cat /sys/kernel/debug/ipoib/ib0_mcg > -------------- next part -------------- An HTML attachment was scrubbed... URL: From ogerlitz at voltaire.com Mon Jul 27 07:01:42 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Mon, 27 Jul 2009 17:01:42 +0300 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <5e3be0f30907260332w3c7097f2g5ff61b2b382f368a@mail.gmail.com> References: <4A69266E.mail4JH11RL5U@systemfabricworks.com> <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> <4A69DCA5.9040705@opengridcomputing.com> <4A69DE72.4020901@opengridcomputing.com> <4A6C0071.3050305@voltaire.com> <5e3be0f30907260332w3c7097f2g5ff61b2b382f368a@mail.gmail.com> Message-ID: <4A6DB346.6000500@voltaire.com> David McMillen wrote: > > I am pleased to see the discussions this has raised about possible > changes to the underlying kernel services. However, this user-mode > patch does address a real problem in the stack as it stands, and I'd > like to see agreement that it should be applied Generally speaking, since you only changed a synthetic program, I don't have any issue with your patch, except for maybe creating possible confusion among people that can use this code as reference for their apps, see more below > I don't know enough about the kernel level code to make comments on > the best way to improve it, but I would like to make an observation > from the user level. There is no way for the user level to have an > informed decision about the proper value for timeouts or retry counts ... The scalability issue is in the air for couple of years now, somehow it came into of many people being sure the problem is SA scalability, where personally, I am not sure this is the case. Also, in the past have tried to set up time for low level technical talking on the matter in ofa meeting, but it was almost always washed a way or given very tiny time slot since more important issues such as why ofed is the greatest thing on earth and what was the content of its last version and what will be the content of its next version, etc, etc. So what happens is that from time to time this or that related issue comes to the list, we have a thread on that and things are left in the air. Recently Sean commented that he works on something http://lists.openfabrics.org/pipermail/ewg/2009-July/013618.html also at May we had a related thread "How to establish IB communcation more effectively?" @ http://lists.openfabrics.org/pipermail/general/2009-May/thread.html#59574 etc etc Or. From dledford at redhat.com Mon Jul 27 07:41:53 2009 From: dledford at redhat.com (Doug Ledford) Date: Mon, 27 Jul 2009 10:41:53 -0400 Subject: [ofa-general] Re: [ewg] [Patch mthca backport] Don't use kmalloc > 128k In-Reply-To: References: <356B6978-3308-4EE9-8C00-00199558BDEA@redhat.com> <200907231121.00140.jackm@dev.mellanox.co.il> Message-ID: On Jul 23, 2009, at 3:06 PM, Roland Dreier wrote: >> This will fix the 2^20 bits limit on our bitmaps once and for all. > > Not really... since getting > 128KB of contiguous memory is likely to > fail anyway. That depends. If you mean at bootup when you are first loading the module, no. You only need large allocations on large memory boxes, and fragmentation won't have happened yet. So it's a perfectly reliable mechanism then. If you are talking about unloading and reloading the module on a busy system, then yes, it could fail then. However, I would argue that if you get a module load failure on reload, then you could always just reboot (keeping in mind that really users shouldn't *need* to ever reload the module anyway, and anything that makes them reload the module is probably a bug, I'm perfectly happy with saying the bug requires a reboot instead of a module reload...that might even provide extra incentive to fix the bug). > And I don't think the upstream kernel has that limit on kmalloc size > either (at least with SLUB, not sure about SLAB). This patch was actually written as an emulation of the upstream SLUB behavior, which is exactly the same thing: on large allocations forward to __g_f_p(). See include/linux/slub_def.h's definition of kmalloc_large and kmalloc. -- Doug Ledford GPG KeyID: CFBFF194 http://people.redhat.com/dledford InfiniBand Specific RPMS http://people.redhat.com/dledford/Infiniband -------------- next part -------------- A non-text attachment was scrubbed... Name: PGP.sig Type: application/pgp-signature Size: 203 bytes Desc: This is a digitally signed message part URL: From sashak at voltaire.com Mon Jul 27 08:20:36 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 27 Jul 2009 18:20:36 +0300 Subject: [ofa-general] Re: [PATCH] ibsim: Drop vendor MADs and vendor specific SM MADs In-Reply-To: <20090727133801.38451235@frecb007965> References: <20090727133801.38451235@frecb007965> Message-ID: <20090727152036.GB18150@me> On 13:38 Mon 27 Jul , sebastien dugue wrote: > > This is useful for testing applications that send those kind of MADS. How this is useful in such test? I see that this just drops every sort of such MADs without letting chance to client to process it. No? Sasha > > Signed-off-by: Sebastien Dugue > --- > ibsim/sim_mad.c | 19 +++++++++++++++++++ > 1 files changed, 19 insertions(+), 0 deletions(-) > > diff --git a/ibsim/sim_mad.c b/ibsim/sim_mad.c > index 61d4866..b79a5e7 100644 > --- a/ibsim/sim_mad.c > +++ b/ibsim/sim_mad.c > @@ -1170,6 +1170,25 @@ int process_packet(Client * cl, void *p, int size, Client ** dcl) > if ((response = decode_sim_MAD(cl, r, &rpc, &path, data)) < 0) > return -1; > > + /* Drop vendor MADs */ > + if (((rpc.mgtclass >= IB_VENDOR_RANGE1_START_CLASS) && > + (rpc.mgtclass <= IB_VENDOR_RANGE1_END_CLASS)) || > + ((rpc.mgtclass >= IB_VENDOR_RANGE2_START_CLASS) && > + (rpc.mgtclass <= IB_VENDOR_RANGE2_END_CLASS))) { > + IBWARN("ignoring vendor MAD: class 0x%x, attr 0x%x", > + rpc.mgtclass, rpc.attr.id); > + goto _dropped; > + } > + > + /* Drop vendor specific SM MADs */ > + if (((rpc.mgtclass == IB_SMI_CLASS) || > + (rpc.mgtclass == IB_SMI_DIRECT_CLASS)) && > + (rpc.attr.id >= 0xff00)) { > + IBWARN("ignoring vendor specific SM MAD: attr 0x%x", > + rpc.attr.id); > + goto _dropped; > + } > + > if (rpc.method == 0x7) { > IBWARN("lid %u got trap repress - dropping", ntohs(r->dlid)); > *dcl = 0; > -- > 1.6.3.1 > From sashak at voltaire.com Mon Jul 27 08:31:41 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 27 Jul 2009 18:31:41 +0300 Subject: [ofa-general] Re: [PATCH] ibsim - Fix umad2sim build with glibc >= 2.10 In-Reply-To: <20090727133715.40e25f45@frecb007965> References: <20090727133715.40e25f45@frecb007965> Message-ID: <20090727153141.GC18150@me> On 13:37 Mon 27 Jul , sebastien dugue wrote: > > Hi Sasha, > > Starting with glibc 2.10 scandir() was made POSIX compliant which changed > the prototype if the comparison function from: > > int (*compar) (const void *, const void *) > > to: > > int (*compar) (const struct dirent **, const struct dirent **) > > This patch detects glibc's version to use the correct prototype. > > Signed-off-by: Sebastien Dugue Applied. Thanks. Sasha From sashak at voltaire.com Mon Jul 27 08:47:22 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 27 Jul 2009 18:47:22 +0300 Subject: [ofa-general] Re: [PATCH] ibsim: Allow multi-threaded use of ibsim In-Reply-To: <20090727133835.4883cee4@frecb007965> References: <20090727133835.4883cee4@frecb007965> Message-ID: <20090727154722.GD18150@me> On 13:38 Mon 27 Jul , sebastien dugue wrote: > > Right now, the socket name used between the client and ibsim is appended > with the client's pid. However this does not work when several threads with > the same pid want to register their clients. And strictly speaking umad2sim by itself is not thread safe. So enabling a multi-threaded clients you need to keep this in a mind, may be to add some notes about this to README. > instead of the pid, use a combination of the thread pid along with a > connection number incremented for each new client. > > Also, attach the agents to the file instance opened on the device rather > than on the device itself. Would be nice to have this as separate patch. > > Signed-off-by: Sebastien Dugue > --- > umad2sim/sim_client.c | 11 ++- > umad2sim/umad2sim.c | 171 +++++++++++++++++++++++++++++++++++++----------- > 2 files changed, 140 insertions(+), 42 deletions(-) > > diff --git a/umad2sim/sim_client.c b/umad2sim/sim_client.c > index eb42a7c..22a50a2 100644 > --- a/umad2sim/sim_client.c > +++ b/umad2sim/sim_client.c > @@ -209,6 +209,8 @@ static int sim_init(struct sim_client *sc, char *nodeid) > char *connect_port; > char *connect_host; > unsigned short port; > + static int conn_num = 0; > + int conn_id; > > connect_port = getenv("IBSIM_SERVER_PORT"); > connect_host = getenv("IBSIM_SERVER_NAME"); > @@ -228,7 +230,10 @@ static int sim_init(struct sim_client *sc, char *nodeid) > if ((ctlfd = socket(remote_mode ? PF_INET : PF_LOCAL, SOCK_DGRAM, 0)) < 0) > IBPANIC("can't get socket (ctlfd)"); > > - size = make_name(&name, NULL, 0, "%s:ctl%d", socket_basename, pid); > + conn_id = ((conn_num & 0xff) << 24) | (pid & 0xffffff); > + conn_num++; BTW why to not use pid/tid combination? This would eliminate the needs in yet another static (and thread unsafe) variable. > + > + size = make_name(&name, NULL, 0, "%s:ctl%d", socket_basename, conn_id); > > if (bind(ctlfd, (struct sockaddr *)&name, size) < 0) > IBPANIC("can't bind ctl socket"); > @@ -243,7 +248,7 @@ static int sim_init(struct sim_client *sc, char *nodeid) > > sc->fd_ctl = ctlfd; > > - size = make_name(&name, NULL, 0, "%s:in%d", socket_basename, pid); > + size = make_name(&name, NULL, 0, "%s:in%d", socket_basename, conn_id); > > if (bind(fd, (struct sockaddr *)&name, size) < 0) > IBPANIC("can't bind input socket"); > @@ -254,7 +259,7 @@ static int sim_init(struct sim_client *sc, char *nodeid) > IBPANIC("can't read data from bound socket"); > port = ntohs(name.name_i.sin_port); > > - sc->clientid = sim_connect(sc, remote_mode ? port : pid, 0, nodeid); > + sc->clientid = sim_connect(sc, remote_mode ? port : conn_id, 0, nodeid); > if (sc->clientid < 0) > IBPANIC("connect failed"); > > diff --git a/umad2sim/umad2sim.c b/umad2sim/umad2sim.c > index 55440ec..71f7c67 100644 > --- a/umad2sim/umad2sim.c > +++ b/umad2sim/umad2sim.c > @@ -81,12 +81,17 @@ struct umad2sim_dev { > char name[32]; > uint8_t port; > struct sim_client sim_client; Why do we need sim_client field here if it is going to umad2sim_devfile structure? > - unsigned agent_idx[256]; > - struct ib_user_mad_reg_req agents[32]; > char umad_path[256]; > char issm_path[256]; > }; > > +struct umad2sim_devfile { > + struct umad2sim_dev *dev; > + struct sim_client sim_client; > + struct ib_user_mad_reg_req agents[32]; > + unsigned agent_idx[256]; > +}; > + > static int (*real_open) (const char *path, int flags, ...); > static int (*real_close) (int fd); > static ssize_t(*real_read) (int fd, void *buf, size_t count); > @@ -113,6 +118,7 @@ static char umad2sim_sysfs_prefix[32]; > > static unsigned umad2sim_initialized; > static struct umad2sim_dev *devices[32]; > +static struct umad2sim_devfile *devfiles[1024]; > > /* > * sysfs stuff > @@ -378,9 +384,69 @@ static int dev_sysfs_create(struct umad2sim_dev *dev) > * umad2sim device > * > */ > +static int umad2sim_open(struct umad2sim_dev *dev) > +{ > + int i; > + int fd; > + struct umad2sim_devfile *file; > + > + /* Find unused fd */ > + for (fd = 0; fd < 1024; fd++) arrsize() is more suitable here (and in another similar places). > + if (devfiles[fd] == NULL) > + break; > + > + if (fd == 1024) { > + ERROR("umad2sim_open: No more available files\n"); > + errno = EMFILE; > + return -1; > + } > > -static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) > + if ((file = calloc(1, sizeof(struct umad2sim_devfile))) == NULL) { > + ERROR("umad2sim_open: Out of memory\n"); > + errno = ENOMEM; > + return -1; > + } > + > + file->dev = dev; > + > + for (i = 0; i < arrsize(file->agents); i++) > + file->agents[i].id = (uint32_t)(-1); > + > + for (i = 0; i < arrsize(file->agent_idx); i++) > + file->agent_idx[i] = (unsigned)(-1); > + > + if (sim_client_init(&file->sim_client) < 0) { > + free(file); > + return -1; > + } > + > + devfiles[fd] = file; > + > + return 1024 + fd; > +} > + > +static int umad2sim_close(int fd) > { > + struct umad2sim_devfile *file; > + > + if ((file = devfiles[fd - 1024]) == NULL) { > + errno = EBADF; > + ERROR("umad2sim_close: no such file\n"); > + return -1; > + } > + > + sim_client_exit(&file->sim_client); > + > + free(file); > + > + devfiles[fd - 1024] = NULL; > + > + return 0; > +} > + > +static ssize_t umad2sim_read(int fd, void *buf, size_t count) > +{ > + struct umad2sim_devfile *file; > struct sim_request req; > ib_user_mad_t *umad = (ib_user_mad_t *) buf; > unsigned mgmt_class; > @@ -388,7 +454,13 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) > > DEBUG("umad2sim_read: %zu...\n", count); > > - cnt = real_read(dev->sim_client.fd_pktin, &req, sizeof(req)); > + if ((file = devfiles[fd - 1024]) == NULL) { > + errno = EBADF; > + ERROR("umad2sim_read: no such file\n"); > + return -1; > + } > + > + cnt = real_read(file->sim_client.fd_pktin, &req, sizeof(req)); > DEBUG("umad2sim_read: got %d...\n", cnt); > if (cnt < sizeof(req)) { > ERROR("umad2sim_read: partial request - skip.\n"); > @@ -406,7 +478,7 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) > mad_get_field(req.mad, 0, IB_MAD_ATTRID_F), > mad_get_field(req.mad, 0, IB_MAD_ATTRMOD_F)); > > - if (mgmt_class >= arrsize(dev->agent_idx)) { > + if (mgmt_class >= arrsize(file->agent_idx)) { > ERROR("bad mgmt_class 0x%x\n", mgmt_class); > mgmt_class = 0; > } > @@ -415,7 +487,7 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) > uint64_t trid = mad_get_field64(req.mad, 0, IB_MAD_TRID_F); > umad->agent_id = (trid >> 32) & 0xffff; > } else > - umad->agent_id = dev->agent_idx[mgmt_class]; > + umad->agent_id = file->agent_idx[mgmt_class]; > > umad->status = ntohl(req.status); > umad->timeout_ms = 0; > @@ -437,9 +509,9 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) > return umad->length; > } > > -static ssize_t umad2sim_write(struct umad2sim_dev *dev, > - const void *buf, size_t count) > +static ssize_t umad2sim_write(int fd, const void *buf, size_t count) > { > + struct umad2sim_devfile *file; > struct sim_request req; > ib_user_mad_t *umad = (ib_user_mad_t *) buf; > int cnt; > @@ -457,12 +529,18 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, > > DEBUG("umad2sim_write: %zu...\n", count); > > + if ((file = devfiles[fd - 1024]) == NULL) { > + errno = EBADF; > + ERROR("umad2sim_write: no such file\n"); > + return -1; > + } > + > DEBUG("umad2sim_write: umad: agent_id=%u, retries=%u, " > "agent.class=%x, agent.qpn=%u, " > "addr.qpn=%u, addr.lid=%u\n", > umad->agent_id, umad->retries, > - dev->agents[umad->agent_id].mgmt_class, > - dev->agents[umad->agent_id].qpn, > + file->agents[umad->agent_id].mgmt_class, > + file->agents[umad->agent_id].qpn, > htonl(umad->addr.qpn), htons(umad->addr.lid)); > DEBUG("umad2sim_write: mad: method=%x, response=%x, mgmtclass=%x, " > "attrid=%x, attrmod=%x\n", > @@ -476,7 +554,7 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, > req.slid = req.dlid == 0xffff ? 0xffff : 0; /* 0 - means auto > (supported by ibsim) */ ; > req.dqp = umad->addr.qpn; > - req.sqp = htonl(dev->agents[umad->agent_id].qpn); > + req.sqp = htonl(file->agents[umad->agent_id].qpn); > req.status = 0; > > cnt = count - umad_size(); > @@ -492,7 +570,7 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, > mad_set_field64(req.mad, 0, IB_MAD_TRID_F, trid); > } > > - cnt = write(dev->sim_client.fd_pktout, (void *)&req, sizeof(req)); > + cnt = write(file->sim_client.fd_pktout, (void *)&req, sizeof(req)); > if (cnt < 0) { > ERROR("umad2sim_write: cannot write\n"); > return -1; > @@ -503,19 +581,21 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, > return count; > } > > -static int register_agent(struct umad2sim_dev *dev, > +static int register_agent(struct umad2sim_devfile *file, > struct ib_user_mad_reg_req *req) > { > - unsigned i; > + unsigned i; > + > DEBUG("register_agent: id = %u, qpn = %u, mgmt_class = %u," > " mgmt_class_version = %u, rmpp_version = %u\n", > req->id, req->qpn, req->mgmt_class, req->mgmt_class_version, > req->rmpp_version); > - for (i = 0; i < arrsize(dev->agents); i++) > - if (dev->agents[i].id == (uint32_t)(-1)) { > + > + for (i = 0; i < arrsize(file->agents); i++) > + if (file->agents[i].id == (uint32_t)(-1)) { > req->id = i; > - dev->agents[i] = *req; > - dev->agent_idx[req->mgmt_class] = i; > + file->agents[i] = *req; > + file->agent_idx[req->mgmt_class] = i; > DEBUG("agent registered: %d\n", i); > return 0; > } > @@ -523,28 +603,36 @@ static int register_agent(struct umad2sim_dev *dev, > return -1; > } > > -static int unregister_agent(struct umad2sim_dev *dev, unsigned id) > +static int unregister_agent(struct umad2sim_devfile *file, unsigned id) > { > unsigned mgmt_class; > - if (id >= arrsize(dev->agents)) { > + if (id >= arrsize(file->agents)) { > errno = EINVAL; > return -1; > } > - mgmt_class = dev->agents[id].mgmt_class; > - dev->agents[id].id = (uint32_t)(-1); > - dev->agent_idx[mgmt_class] = -1; > + mgmt_class = file->agents[id].mgmt_class; > + file->agents[id].id = (uint32_t)(-1); > + file->agent_idx[mgmt_class] = -1; > return 0; > } > > -static int umad2sim_ioctl(struct umad2sim_dev *dev, unsigned long request, > - void *arg) > +static int umad2sim_ioctl(int fd, unsigned long request, void *arg) > { > + struct umad2sim_devfile *file; > + > DEBUG("umad2sim_ioctl: %lu, %p...\n", request, arg); > + > + if ((file = devfiles[fd - 1024]) == NULL) { > + errno = EBADF; > + ERROR("umad2sim_ioctl: no such file\n"); > + return -1; > + } > + > switch (request) { > case IB_USER_MAD_REGISTER_AGENT: > - return register_agent(dev, arg); > + return register_agent(file, arg); > case IB_USER_MAD_UNREGISTER_AGENT: > - return unregister_agent(dev, *((unsigned *)arg)); > + return unregister_agent(file, *((unsigned *)arg)); > case IB_USER_MAD_ENABLE_PKEY: > return 0; > default: > @@ -556,7 +644,6 @@ static int umad2sim_ioctl(struct umad2sim_dev *dev, unsigned long request, > static struct umad2sim_dev *umad2sim_dev_create(unsigned num, const char *name) > { > struct umad2sim_dev *dev; > - unsigned i; > > DEBUG("umad2sim_dev_create: %s...\n", name); > > @@ -573,10 +660,6 @@ static struct umad2sim_dev *umad2sim_dev_create(unsigned num, const char *name) > > dev->port = mad_get_field(&dev->sim_client.portinfo, 0, > IB_PORT_LOCAL_PORT_F); > - for (i = 0; i < arrsize(dev->agents); i++) > - dev->agents[i].id = (uint32_t)(-1); > - for (i = 0; i < arrsize(dev->agent_idx); i++) > - dev->agent_idx[i] = (unsigned)(-1); > > dev_sysfs_create(dev); > > @@ -637,6 +720,11 @@ static void umad2sim_cleanup(void) > char path[1024]; > unsigned i; > DEBUG("umad2sim_cleanup...\n"); > + > + for (i = 0; i < 1024; i++) > + if (devfiles[i]) > + free(devfiles[i]); And what about sim_client disconnection? umad2sim_dev_delete() did this. Sasha > + > for (i = 0; i < arrsize(devices); i++) > if (devices[i]) { > umad2sim_dev_delete(devices[i]); > @@ -756,7 +844,7 @@ int open(const char *path, int flags, ...) > if (!(dev = devices[i])) > continue; > if (!strncmp(path, dev->umad_path, sizeof(dev->umad_path))) { > - return 1024 + i; > + return umad2sim_open(dev); > } > if (!strncmp(path, dev->issm_path, sizeof(dev->issm_path))) { > sim_client_set_sm(&dev->sim_client, 1); > @@ -779,7 +867,7 @@ int close(int fd) > sim_client_set_sm(&dev->sim_client, 0); > return 0; > } else if (fd >= 1024) { > - return 0; > + return umad2sim_close(fd); > } else > return real_close(fd); > } > @@ -791,7 +879,7 @@ ssize_t read(int fd, void *buf, size_t count) > if (fd >= 2048) > return -1; > else if (fd >= 1024) > - return umad2sim_read(devices[fd - 1024], buf, count); > + return umad2sim_read(fd, buf, count); > else > return real_read(fd, buf, count); > } > @@ -803,7 +891,7 @@ ssize_t write(int fd, const void *buf, size_t count) > if (fd >= 2048) > return -1; > else if (fd >= 1024) > - return umad2sim_write(devices[fd - 1024], buf, count); > + return umad2sim_write(fd, buf, count); > else > return real_write(fd, buf, count); > } > @@ -821,7 +909,7 @@ int ioctl(int fd, unsigned long request, ...) > if (fd >= 2048) > return -1; > else if (fd >= 1024) > - return umad2sim_ioctl(devices[fd - 1024], request, arg); > + return umad2sim_ioctl(fd, request, arg); > else > return real_ioctl(fd, request, arg); > } > @@ -836,9 +924,14 @@ int poll(struct pollfd *pfds, nfds_t nfds, int timeout) > > for (i = 0; i < nfds; i++) { > if (pfds[i].fd >= 1024 && pfds[i].fd < 2048) { > - struct umad2sim_dev *dev = devices[pfds[i].fd - 1024]; > + struct umad2sim_devfile *file; > + > + if ((file = devfiles[pfds[i].fd - 1024]) == NULL) { > + errno = EBADF; > + return -1; > + } > saved_fds[i] = pfds[i].fd; > - pfds[i].fd = dev->sim_client.fd_pktin; > + pfds[i].fd = file->sim_client.fd_pktin; > } else > saved_fds[i] = 0; > } > -- > 1.6.3.1 > From nashwath at gmail.com Mon Jul 27 08:59:20 2009 From: nashwath at gmail.com (Ashwath Narasimhan) Date: Mon, 27 Jul 2009 11:59:20 -0400 Subject: [ofa-general] Configuration of HCA ports- sender/receiver Message-ID: Hello, I am an infiniband newbie and I wish to configure one of the ports of the Infiniband HCA for sending IP (IPoIB) traffic and the other port of the HCA for receiving IP(IPoIB) traffic. How should I go about this? Is there any utility function/configuration file that I need to alter? Or should I jump into the source code of the stack to change the port number/id? I am using Mellanox Infinihost III cards and I am able to run opensm and few other test cases correctly. Thank you for your time. regards, Ashwath -------------- next part -------------- An HTML attachment was scrubbed... URL: From sashak at voltaire.com Mon Jul 27 09:32:28 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 27 Jul 2009 19:32:28 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_switch.c: In osm_switch_delete, set priv to NULL In-Reply-To: <20090724230447.GB28379@comcast.net> References: <20090724230447.GB28379@comcast.net> Message-ID: <20090727163228.GE18150@me> Hi Hal, On 19:04 Fri 24 Jul , Hal Rosenstock wrote: > > This can help with finding use after free issues if memory not reused > updn and lash rely on this priv pointer and may have stale > osm_switch_t pointers As far as I remember LASH and UPDN should never be in this state. If you are able to see such cases this indicates some serious bug (probably race between SA PR processor and drop manager if it happens with LASH) and actually is a good environment for finding and fixing. The proposed patch is just hide a *real* problem (unless I'm wrong above and the case is legal). Sasha > > Signed-off-by: Hal Rosenstock > --- > diff --git a/opensm/opensm/osm_switch.c b/opensm/opensm/osm_switch.c > index ce1ca63..e23b32f 100644 > --- a/opensm/opensm/osm_switch.c > +++ b/opensm/opensm/osm_switch.c > @@ -94,6 +94,7 @@ void osm_switch_delete(IN OUT osm_switch_t ** const pp_sw) > free(p_sw->hops[i]); > free(p_sw->hops); > } > + p_sw->priv = NULL; > free(*pp_sw); > *pp_sw = NULL; > } > From sashak at voltaire.com Mon Jul 27 09:42:13 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 27 Jul 2009 19:42:13 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Minor cleanups In-Reply-To: <20090724141451.GA18684@comcast.net> References: <20090724141451.GA18684@comcast.net> Message-ID: <20090727164213.GG18150@me> On 10:14 Fri 24 Jul , Hal Rosenstock wrote: > > Eliminate some unneeded initializations > Eliminate #if 0'd code > Cosmetic formatting changes > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From hal.rosenstock at gmail.com Mon Jul 27 09:41:42 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 27 Jul 2009 12:41:42 -0400 Subject: [ofa-general] Re: [PATCH] opensm/osm_switch.c: In osm_switch_delete, set priv to NULL In-Reply-To: <20090727163228.GE18150@me> References: <20090724230447.GB28379@comcast.net> <20090727163228.GE18150@me> Message-ID: Hi Sasha, On Mon, Jul 27, 2009 at 12:32 PM, Sasha Khapyorsky wrote: > Hi Hal, > > On 19:04 Fri 24 Jul     , Hal Rosenstock wrote: >> >> This can help with finding use after free issues if memory not reused >> updn and lash rely on this priv pointer and may have stale >> osm_switch_t pointers > > As far as I remember LASH and UPDN should never be in this state. If you > are able to see such cases this indicates some serious bug (probably > race between SA PR processor and drop manager if it happens with LASH) > and actually is a good environment for finding and fixing. > > The proposed patch is just hide a *real* problem (unless I'm wrong > above and the case is legal). The intent of the patch is to cause a NULL ptr fault rather than using memory not currently owned. The latter can have unpredictable results. -- Hal > Sasha > >> >> Signed-off-by: Hal Rosenstock >> --- >> diff --git a/opensm/opensm/osm_switch.c b/opensm/opensm/osm_switch.c >> index ce1ca63..e23b32f 100644 >> --- a/opensm/opensm/osm_switch.c >> +++ b/opensm/opensm/osm_switch.c >> @@ -94,6 +94,7 @@ void osm_switch_delete(IN OUT osm_switch_t ** const pp_sw) >>                               free(p_sw->hops[i]); >>               free(p_sw->hops); >>       } >> +     p_sw->priv = NULL; >>       free(*pp_sw); >>       *pp_sw = NULL; >>  } >> > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From sashak at voltaire.com Mon Jul 27 09:58:33 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 27 Jul 2009 19:58:33 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_switch.c: In osm_switch_delete, set priv to NULL In-Reply-To: References: <20090724230447.GB28379@comcast.net> <20090727163228.GE18150@me> Message-ID: <20090727165833.GH18150@me> On 12:41 Mon 27 Jul , Hal Rosenstock wrote: > > The intent of the patch is to cause a NULL ptr fault rather than using > memory not currently owned. The latter can have unpredictable results. This could be nice for debugging, but the case itself is illegal and should not happen in a main stream (unless there are bugs which should be fixed). BTW LASH checks this pointer for NULL in several places, so this patch will effectively hide a potential bugs. Sasha From sashak at voltaire.com Mon Jul 27 10:06:27 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 27 Jul 2009 20:06:27 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_mesh: Fix use after free problem in osm_mesh_node_delete In-Reply-To: <20090724230303.GA28379@comcast.net> References: <20090724230303.GA28379@comcast.net> Message-ID: <20090727170627.GI18150@me> On 19:03 Fri 24 Jul , Hal Rosenstock wrote: > > When osm_mesh_node_delete is called, osm_switch_delete may already have > been called so sw->p_sw is no longer valid to be used although it was > being used to obtain num_ports. And this looks like a bug in mesh code. Why does mesh need mesh node structure after routing was calculated? LASH needs its switch info in order to provide proper SL value to SA queries, but mesh info is not needed then. I think that proper way to fix this is to free mesh nodes when it is not needed anymore and not on a next LASH cycle start (like LASH switches). Sasha From rdreier at cisco.com Mon Jul 27 10:10:29 2009 From: rdreier at cisco.com (Roland Dreier) Date: Mon, 27 Jul 2009 10:10:29 -0700 Subject: [ofa-general] Re: [ewg] [Patch mthca backport] Don't use kmalloc > 128k In-Reply-To: (Doug Ledford's message of "Mon, 27 Jul 2009 10:41:53 -0400") References: <356B6978-3308-4EE9-8C00-00199558BDEA@redhat.com> <200907231121.00140.jackm@dev.mellanox.co.il> Message-ID: > > And I don't think the upstream kernel has that limit on kmalloc size > > either (at least with SLUB, not sure about SLAB). > > This patch was actually written as an emulation of the upstream SLUB > behavior, which is exactly the same thing: on large allocations > forward to __g_f_p(). See include/linux/slub_def.h's definition of > kmalloc_large and kmalloc. Right. But does upstream SLAB also pass through to the page allocator the same as SLUB? How about SLQB? - R. From yosefe at voltaire.com Mon Jul 27 10:11:42 2009 From: yosefe at voltaire.com (Yossi Etigin) Date: Mon, 27 Jul 2009 20:11:42 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes Message-ID: <4A6DDFCE.9060009@voltaire.com> If the LID of an ipoib neighbour changes without a SM event on the local node, IPoIB will keep caching the invalid path until the device is flushed. The patch below will remove the path for every incoming ARP packet where the sender hardware address does not match the cached lid. It works because the IP stack has a periodical arp refresh which will eventually cause the remote node to send an ARP reply. This is better that a periodic refresh mechanism, because it will not overwhelm the SM. Signed-off-by: Yossi Etigin --- Index: b/drivers/infiniband/ulp/ipoib/ipoib.h =================================================================== --- a/drivers/infiniband/ulp/ipoib/ipoib.h 2009-07-27 19:44:40.000000000 +0300 +++ b/drivers/infiniband/ulp/ipoib/ipoib.h 2009-07-27 20:01:03.000000000 +0300 @@ -444,6 +444,7 @@ void ipoib_reap_ah(struct work_struct *w void ipoib_mark_paths_invalid(struct net_device *dev); void ipoib_flush_paths(struct net_device *dev); +void ipoib_path_refresh(struct net_device *dev, void *gid, u16 lid); struct ipoib_dev_priv *ipoib_intf_alloc(const char *format); int ipoib_ib_dev_init(struct net_device *dev, struct ib_device *ca, int port); Index: b/drivers/infiniband/ulp/ipoib/ipoib_cm.c =================================================================== --- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c 2009-07-27 19:44:40.000000000 +0300 +++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c 2009-07-27 19:44:45.000000000 +0300 @@ -36,6 +36,7 @@ #include #include #include +#include #include "ipoib.h" @@ -662,6 +663,10 @@ copied: skb_reset_mac_header(skb); skb_pull(skb, IPOIB_ENCAP_LEN); + if (skb->protocol == htons(ETH_P_ARP)) + ipoib_path_refresh(dev, skb->data + sizeof(struct arphdr) + 4, + wc->slid); + dev->last_rx = jiffies; ++dev->stats.rx_packets; dev->stats.rx_bytes += skb->len; Index: b/drivers/infiniband/ulp/ipoib/ipoib_ib.c =================================================================== --- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2009-07-27 19:44:40.000000000 +0300 +++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2009-07-27 19:44:45.000000000 +0300 @@ -38,6 +38,7 @@ #include #include +#include #include "ipoib.h" @@ -276,6 +277,10 @@ static void ipoib_ib_handle_rx_wc(struct skb_reset_mac_header(skb); skb_pull(skb, IPOIB_ENCAP_LEN); + if (skb->protocol == htons(ETH_P_ARP)) + ipoib_path_refresh(dev, skb->data + sizeof(struct arphdr) + 4, + wc->slid); + dev->last_rx = jiffies; ++dev->stats.rx_packets; dev->stats.rx_bytes += skb->len; Index: b/drivers/infiniband/ulp/ipoib/ipoib_main.c =================================================================== --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c 2009-07-27 19:44:40.000000000 +0300 +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c 2009-07-27 19:44:45.000000000 +0300 @@ -399,6 +399,35 @@ void ipoib_flush_paths(struct net_device netif_tx_unlock_bh(dev); } +void ipoib_path_refresh(struct net_device *dev, void *gid, u16 lid) +{ + struct ipoib_dev_priv *priv = netdev_priv(dev); + unsigned long flags; + struct ipoib_path *path; + + netif_tx_lock_bh(dev); + spin_lock_irqsave(&priv->lock, flags); + + path = __path_find(dev, gid); + if (!path || path->query || !path->ah || + be16_to_cpu(path->pathrec.dlid) == lid) { + spin_unlock_irqrestore(&priv->lock, flags); + netif_tx_unlock_bh(dev); + return; + } + + ipoib_dbg(priv, "Path %pI6: LID changed from 0x%04x to 0x%04x\n", + path->pathrec.dgid.raw, be16_to_cpu(path->pathrec.dlid), lid); + + list_del(&path->list); + rb_erase(&path->rb_node, &priv->path_tree); + + spin_unlock_irqrestore(&priv->lock, flags); + netif_tx_unlock_bh(dev); + + path_free(dev, path); +} + static void path_rec_completion(int status, struct ib_sa_path_rec *pathrec, void *path_ptr) From chu11 at llnl.gov Mon Jul 27 10:12:03 2009 From: chu11 at llnl.gov (Al Chu) Date: Mon, 27 Jul 2009 10:12:03 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [0/6] fix libibnetdisc API consistency and bugs Message-ID: <1248714723.16723.322.camel@auk31.llnl.gov> Hey Sasha, This is a redo of my previous patch series. Ira or myself will instead write 1 big patch later on to make a lot of the structs more public. These are the patches to fix bugs and/or make things more consistent for what's already there. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory From chu11 at llnl.gov Mon Jul 27 10:12:32 2009 From: chu11 at llnl.gov (Al Chu) Date: Mon, 27 Jul 2009 10:12:32 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [1/6] fix libibnetdisc API consistency and bugs Message-ID: <1248714752.16723.324.camel@auk31.llnl.gov> In libibnetdisc, do not automatically output messages to stderr/stdout for warnings or non-fatal errors. Use IBND_DEBUG instead of IBPANIC or IBWARN to allow users of the lib to decide output conditions. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- From: Albert Chu Date: Thu, 23 Jul 2009 11:56:53 -0700 Subject: [PATCH] In libibnetdisc, do not automatically output messages to stderr/stdout for warnings or non-fatal errors. Use IBND_DEBUG instead of IBPANIC or IBWARN to allow users of the lib to decide output conditions. Signed-off-by: Albert Chu --- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index b640bc1..4b90744 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -464,8 +464,8 @@ get_remote_node(struct ibnd_fabric *fabric, struct ibnd_node *node, struct ibnd_ return -1; if (query_node(fabric, &node_buf, &port_buf, path)) { - IBWARN("NodeInfo on %s failed, skipping port", - portid2str(path)); + IBND_DEBUG("NodeInfo on %s failed, skipping port", + portid2str(path)); path->drpath.cnt--; /* restore path */ return -1; } @@ -507,15 +507,15 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, int max_hops = MAXHOPS-1; /* default find everything */ if (!ibmad_port) { - IBPANIC("ibmad_port must be specified to " - "ibnd_discover_fabric\n"); + IBND_DEBUG("ibmad_port must be specified to " + "ibnd_discover_fabric\n"); return (NULL); } if (mad_rpc_class_agent(ibmad_port, IB_SMI_CLASS) == -1 || mad_rpc_class_agent(ibmad_port, IB_SMI_DIRECT_CLASS) == -1) { - IBPANIC("ibmad_port must be opened with " - "IB_SMI_CLASS && IB_SMI_DIRECT_CLASS\n"); + IBND_DEBUG("ibmad_port must be opened with " + "IB_SMI_CLASS && IB_SMI_DIRECT_CLASS\n"); return (NULL); } @@ -545,7 +545,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, memset(&port_buf, 0, sizeof(port_buf)); if (query_node(fabric, &node_buf, &port_buf, from)) { - IBWARN("can't reach node %s\n", portid2str(from)); + IBND_DEBUG("can't reach node %s\n", portid2str(from)); goto error; } @@ -579,7 +579,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, continue; if (get_port_info(fabric, &port_buf, i, path)) { - IBWARN("can't reach node %s port %d", portid2str(path), i); + IBND_DEBUG("can't reach node %s port %d", portid2str(path), i); continue; } -- 1.5.4.5 From chu11 at llnl.gov Mon Jul 27 10:12:36 2009 From: chu11 at llnl.gov (Al Chu) Date: Mon, 27 Jul 2009 10:12:36 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [2/6] fix libibnetdisc API consistency and bugs Message-ID: <1248714756.16723.325.camel@auk31.llnl.gov> Use IBPANIC consistently in libibnetdisc, in particular, since IBPANIC calls exit, there's no use in returning a value after an error. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- From: Albert Chu Date: Thu, 23 Jul 2009 11:57:35 -0700 Subject: [PATCH] Use IBPANIC consistently in libibnetdisc, in particular, since IBPANIC calls exit, there's no use in returning a value after an error. Signed-off-by: Albert Chu --- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 8 ++------ 1 files changed, 2 insertions(+), 6 deletions(-) diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index 4b90744..1f041ac 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -369,10 +369,8 @@ create_node(struct ibnd_fabric *fabric, struct ibnd_node *temp, ib_portid_t *pat struct ibnd_node *node; node = malloc(sizeof(*node)); - if (!node) { + if (!node) IBPANIC("OOM: node creation failed\n"); - return NULL; - } memcpy(node, temp, sizeof(*node)); node->node.dist = dist; @@ -530,10 +528,8 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, fabric = malloc(sizeof(*fabric)); - if (!fabric) { + if (!fabric) IBPANIC("OOM: failed to malloc ibnd_fabric_t\n"); - return (NULL); - } memset(fabric, 0, sizeof(*fabric)); -- 1.5.4.5 From chu11 at llnl.gov Mon Jul 27 10:12:41 2009 From: chu11 at llnl.gov (Al Chu) Date: Mon, 27 Jul 2009 10:12:41 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [3/6] fix libibnetdisc API consistency and bugs Message-ID: <1248714761.16723.326.camel@auk31.llnl.gov> Fix potential memleak in ibnd_discover_fabric error path. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- From: Albert Chu Date: Thu, 23 Jul 2009 11:58:45 -0700 Subject: [PATCH] Fix potential memleak in ibnd_discover_fabric error path. Signed-off-by: Albert Chu --- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index 1f041ac..a942ef4 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -602,7 +602,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, return ((ibnd_fabric_t *)fabric); error: - free(fabric); + ibnd_destroy_fabric(fabric); return (NULL); } -- 1.5.4.5 From chu11 at llnl.gov Mon Jul 27 10:12:51 2009 From: chu11 at llnl.gov (Al Chu) Date: Mon, 27 Jul 2009 10:12:51 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [4/6] fix libibnetdisc API consistency and bugs Message-ID: <1248714771.16723.327.camel@auk31.llnl.gov> Make api more consistent and make struct ibnd_fabric a struct that represents just fabric data by removing the ibmad_port and making it a function paramete in appropriate functions. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- From: Albert Chu Date: Thu, 23 Jul 2009 14:14:57 -0700 Subject: [PATCH] Make api more consistent and make struct ibnd_fabric a struct that represents just fabric data by removing the ibmad_port and making it a function paramete in appropriate functions. Signed-off-by: Albert Chu --- .../libibnetdisc/include/infiniband/ibnetdisc.h | 4 +- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 93 +++++++++++--------- 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h index 5f07805..f65690c 100644 --- a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h +++ b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h @@ -121,7 +121,6 @@ typedef struct chassis { * Main fabric object which is returned and represents the data discovered */ typedef struct ib_fabric { - struct ibmad_port *ibmad_port; /* the node the discover was initiated from * "from" parameter in ibnd_discover_fabric * or by default the node you ar running on @@ -160,7 +159,8 @@ MAD_EXPORT void ibnd_destroy_fabric(ibnd_fabric_t *fabric); */ MAD_EXPORT ibnd_node_t *ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid); MAD_EXPORT ibnd_node_t *ibnd_find_node_dr(ibnd_fabric_t *fabric, char *dr_str); -MAD_EXPORT ibnd_node_t *ibnd_update_node(ibnd_fabric_t *fabric, ibnd_node_t *node); +MAD_EXPORT ibnd_node_t *ibnd_update_node(struct ibmad_port *ibmad_port, + ibnd_fabric_t *fabric, ibnd_node_t *node); typedef void (*ibnd_iter_node_func_t)(ibnd_node_t *node, void *user_data); MAD_EXPORT void ibnd_iter_nodes(ibnd_fabric_t *fabric, diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index a942ef4..5c5814d 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -69,8 +69,8 @@ decode_port_info(ibnd_port_t *port) } static int -get_port_info(struct ibnd_fabric *fabric, struct ibnd_port *port, - int portnum, ib_portid_t *portid) +get_port_info(struct ibmad_port *ibmad_port, struct ibnd_fabric *fabric, + struct ibnd_port *port, int portnum, ib_portid_t *portid) { char width[64], speed[64]; int iwidth; @@ -81,7 +81,7 @@ get_port_info(struct ibnd_fabric *fabric, struct ibnd_port *port, ispeed = mad_get_field(port->port.info, 0, IB_PORT_LINK_SPEED_ACTIVE_F); if (!smp_query_via(port->port.info, portid, IB_ATTR_PORT_INFO, portnum, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) return -1; decode_port_info(&(port->port)); @@ -99,10 +99,11 @@ get_port_info(struct ibnd_fabric *fabric, struct ibnd_port *port, * Returns -1 if error. */ static int -query_node_info(struct ibnd_fabric *fabric, struct ibnd_node *node, ib_portid_t *portid) +query_node_info(struct ibmad_port *ibmad_port, struct ibnd_fabric *fabric, + struct ibnd_node *node, ib_portid_t *portid) { if (!smp_query_via(&(node->node.info), portid, IB_ATTR_NODE_INFO, 0, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) return -1; /* decode just a couple of fields for quicker reference. */ @@ -118,25 +119,25 @@ query_node_info(struct ibnd_fabric *fabric, struct ibnd_node *node, ib_portid_t * Returns 0 if non switch node is found, 1 if switch is found, -1 if error. */ static int -query_node(struct ibnd_fabric *fabric, struct ibnd_node *inode, - struct ibnd_port *iport, ib_portid_t *portid) +query_node(struct ibmad_port *ibmad_port, struct ibnd_fabric *fabric, + struct ibnd_node *inode, struct ibnd_port *iport, ib_portid_t *portid) { ibnd_node_t *node = &(inode->node); ibnd_port_t *port = &(iport->port); void *nd = inode->node.nodedesc; - if (query_node_info(fabric, inode, portid)) + if (query_node_info(ibmad_port, fabric, inode, portid)) return -1; port->portnum = mad_get_field(node->info, 0, IB_NODE_LOCAL_PORT_F); port->guid = mad_get_field64(node->info, 0, IB_NODE_PORT_GUID_F); if (!smp_query_via(nd, portid, IB_ATTR_NODE_DESC, 0, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) return -1; if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, 0, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) return -1; decode_port_info(port); @@ -148,7 +149,7 @@ query_node(struct ibnd_fabric *fabric, struct ibnd_node *inode, /* after we have the sma information find out the real PortInfo for this port */ if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, port->portnum, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) return -1; decode_port_info(port); @@ -156,7 +157,7 @@ query_node(struct ibnd_fabric *fabric, struct ibnd_node *inode, port->lmc = (uint8_t) node->smalmc; if (!smp_query_via(node->switchinfo, portid, IB_ATTR_SWITCH_INFO, 0, timeout_ms, - fabric->fabric.ibmad_port)) + ibmad_port)) node->smaenhsp0 = 0; /* assume base SP0 */ else mad_decode_field(node->switchinfo, IB_SW_ENHANCED_PORT0_F, &node->smaenhsp0); @@ -177,7 +178,8 @@ add_port_to_dpath(ib_dr_path_t *path, int nextport) } static int -extend_dpath(struct ibnd_fabric *f, ib_portid_t *portid, int nextport) +extend_dpath(struct ibmad_port *ibmad_port, struct ibnd_fabric *f, + ib_portid_t *portid, int nextport) { int rc = 0; @@ -185,7 +187,7 @@ extend_dpath(struct ibnd_fabric *f, ib_portid_t *portid, int nextport) /* If we were LID routed we need to set up the drslid */ if (!f->selfportid.lid) if (ib_resolve_self_via(&f->selfportid, NULL, NULL, - f->fabric.ibmad_port) < 0) + ibmad_port) < 0) return -1; portid->drpath.drslid = (uint16_t) f->selfportid.lid; @@ -244,8 +246,25 @@ ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid) return NULL; } +static int +_check_ibmad_port(struct ibmad_port *ibmad_port) +{ + if (!ibmad_port) { + IBND_DEBUG("ibmad_port must be specified\n"); + return (-1); + } + if (mad_rpc_class_agent(ibmad_port, IB_SMI_CLASS) == -1 + || + mad_rpc_class_agent(ibmad_port, IB_SMI_DIRECT_CLASS) == -1) { + IBND_DEBUG("ibmad_port must be opened with " + "IB_SMI_CLASS && IB_SMI_DIRECT_CLASS\n"); + return (-1); + } + return (0); +} + ibnd_node_t * -ibnd_update_node(ibnd_fabric_t *fabric, ibnd_node_t *node) +ibnd_update_node(struct ibmad_port *ibmad_port, ibnd_fabric_t *fabric, ibnd_node_t *node) { char portinfo_port0[IB_SMP_DATA_SIZE]; void *nd = node->nodedesc; @@ -253,30 +272,34 @@ ibnd_update_node(ibnd_fabric_t *fabric, ibnd_node_t *node) struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric); struct ibnd_node *n = CONV_NODE_INTERNAL(node); - if (query_node_info(f, n, &(n->node.path_portid))) + if (_check_ibmad_port(ibmad_port) < 0) + return (NULL); + + if (query_node_info(ibmad_port, f, n, &(n->node.path_portid))) return (NULL); if (!smp_query_via(nd, &(n->node.path_portid), IB_ATTR_NODE_DESC, 0, timeout_ms, - f->fabric.ibmad_port)) + ibmad_port)) return (NULL); /* update all the port info's */ for (p = 1; p >= n->node.numports; p++) { - get_port_info(f, CONV_PORT_INTERNAL(n->node.ports[p]), p, &(n->node.path_portid)); + get_port_info(ibmad_port, f, CONV_PORT_INTERNAL(n->node.ports[p]), + p, &(n->node.path_portid)); } if (n->node.type != IB_NODE_SWITCH) goto done; if (!smp_query_via(portinfo_port0, &(n->node.path_portid), IB_ATTR_PORT_INFO, 0, timeout_ms, - f->fabric.ibmad_port)) + ibmad_port)) return (NULL); n->node.smalid = mad_get_field(portinfo_port0, 0, IB_PORT_LID_F); n->node.smalmc = mad_get_field(portinfo_port0, 0, IB_PORT_LMC_F); if (!smp_query_via(node->switchinfo, &(n->node.path_portid), IB_ATTR_SWITCH_INFO, 0, timeout_ms, - f->fabric.ibmad_port)) + ibmad_port)) node->smaenhsp0 = 0; /* assume base SP0 */ else mad_decode_field(node->switchinfo, IB_SW_ENHANCED_PORT0_F, &n->node.smaenhsp0); @@ -441,7 +464,8 @@ link_ports(struct ibnd_node *node, struct ibnd_port *port, } static int -get_remote_node(struct ibnd_fabric *fabric, struct ibnd_node *node, struct ibnd_port *port, ib_portid_t *path, +get_remote_node(struct ibmad_port *ibmad_port, struct ibnd_fabric *fabric, + struct ibnd_node *node, struct ibnd_port *port, ib_portid_t *path, int portnum, int dist) { struct ibnd_node node_buf; @@ -458,10 +482,10 @@ get_remote_node(struct ibnd_fabric *fabric, struct ibnd_node *node, struct ibnd_ != IB_PORT_PHYS_STATE_LINKUP) return -1; - if (extend_dpath(fabric, path, portnum) < 0) + if (extend_dpath(ibmad_port, fabric, path, portnum) < 0) return -1; - if (query_node(fabric, &node_buf, &port_buf, path)) { + if (query_node(ibmad_port, fabric, &node_buf, &port_buf, path)) { IBND_DEBUG("NodeInfo on %s failed, skipping port", portid2str(path)); path->drpath.cnt--; /* restore path */ @@ -504,18 +528,8 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, ib_portid_t *path; int max_hops = MAXHOPS-1; /* default find everything */ - if (!ibmad_port) { - IBND_DEBUG("ibmad_port must be specified to " - "ibnd_discover_fabric\n"); - return (NULL); - } - if (mad_rpc_class_agent(ibmad_port, IB_SMI_CLASS) == -1 - || - mad_rpc_class_agent(ibmad_port, IB_SMI_DIRECT_CLASS) == -1) { - IBND_DEBUG("ibmad_port must be opened with " - "IB_SMI_CLASS && IB_SMI_DIRECT_CLASS\n"); + if (_check_ibmad_port(ibmad_port) < 0) return (NULL); - } /* if not everything how much? */ if (hops >= 0) { @@ -533,14 +547,12 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, memset(fabric, 0, sizeof(*fabric)); - fabric->fabric.ibmad_port = ibmad_port; - IBND_DEBUG("from %s\n", portid2str(from)); memset(&node_buf, 0, sizeof(node_buf)); memset(&port_buf, 0, sizeof(port_buf)); - if (query_node(fabric, &node_buf, &port_buf, from)) { + if (query_node(ibmad_port, fabric, &node_buf, &port_buf, from)) { IBND_DEBUG("can't reach node %s\n", portid2str(from)); goto error; } @@ -555,7 +567,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, if (!port) IBPANIC("out of memory"); - if(get_remote_node(fabric, node, port, from, + if(get_remote_node(ibmad_port, fabric, node, port, from, mad_get_field(node->node.info, 0, IB_NODE_LOCAL_PORT_F), 0) < 0) return ((ibnd_fabric_t *)fabric); @@ -574,7 +586,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, IB_NODE_LOCAL_PORT_F)) continue; - if (get_port_info(fabric, &port_buf, i, path)) { + if (get_port_info(ibmad_port, fabric, &port_buf, i, path)) { IBND_DEBUG("can't reach node %s port %d", portid2str(path), i); continue; } @@ -593,7 +605,8 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, 0, IB_NODE_PORT_GUID_F); } - get_remote_node(fabric, node, port, path, i, dist); + get_remote_node(ibmad_port, fabric, node, port, + path, i, dist); } } } -- 1.5.4.5 From chu11 at llnl.gov Mon Jul 27 10:12:59 2009 From: chu11 at llnl.gov (Al Chu) Date: Mon, 27 Jul 2009 10:12:59 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [5/6] fix libibnetdisc API consistency and bugs Message-ID: <1248714779.16723.328.camel@auk31.llnl.gov> Check input parameters to libibnetdisc functions Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- From: Albert Chu Date: Thu, 23 Jul 2009 14:15:25 -0700 Subject: [PATCH] Check input parameters to libibnetdisc functions Signed-off-by: Albert Chu --- infiniband-diags/libibnetdisc/src/chassis.c | 15 ++++++++ infiniband-diags/libibnetdisc/src/ibnetdisc.c | 47 ++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletions(-) diff --git a/infiniband-diags/libibnetdisc/src/chassis.c b/infiniband-diags/libibnetdisc/src/chassis.c index 78aee1f..5df910b 100644 --- a/infiniband-diags/libibnetdisc/src/chassis.c +++ b/infiniband-diags/libibnetdisc/src/chassis.c @@ -54,6 +54,11 @@ static char *ChassisSlotTypeStr[4] = { "", "Line", "Spine", "SRBD" }; char *ibnd_get_chassis_type(ibnd_node_t *node) { + if (!node) { + IBND_DEBUG("node parameter NULL\n"); + return (NULL); + } + /* Currently, only if Voltaire chassis */ if (mad_get_field(node->info, 0, IB_NODE_VENDORID_F) != VTR_VENDOR_ID) return (NULL); @@ -67,6 +72,11 @@ char *ibnd_get_chassis_type(ibnd_node_t *node) char *ibnd_get_chassis_slot_str(ibnd_node_t *node, char *str, size_t size) { + if (!node) { + IBND_DEBUG("node parameter NULL\n"); + return (NULL); + } + /* Currently, only if Voltaire chassis */ if (mad_get_field(node->info, 0, IB_NODE_VENDORID_F) != VTR_VENDOR_ID) return (NULL); @@ -216,6 +226,11 @@ uint64_t ibnd_get_chassis_guid(ibnd_fabric_t *fabric, unsigned char chassisnum) struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric); ibnd_chassis_t *chassis; + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return 0; + } + chassis = find_chassisnum(f, chassisnum); if (chassis) return chassis->chassisguid; diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index 5c5814d..3f9a3dd 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -239,6 +239,11 @@ ibnd_find_node_guid(ibnd_fabric_t *fabric, uint64_t guid) int hash = HASHGUID(guid) % HTSZ; struct ibnd_node *node; + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return (NULL); + } + for (node = f->nodestbl[hash]; node; node = node->htnext) if (node->node.guid == guid) return (ibnd_node_t *)node; @@ -275,6 +280,16 @@ ibnd_update_node(struct ibmad_port *ibmad_port, ibnd_fabric_t *fabric, ibnd_node if (_check_ibmad_port(ibmad_port) < 0) return (NULL); + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return (NULL); + } + + if (!node) { + IBND_DEBUG("node parameter NULL\n"); + return (NULL); + } + if (query_node_info(ibmad_port, f, n, &(n->node.path_portid))) return (NULL); @@ -313,9 +328,16 @@ ibnd_find_node_dr(ibnd_fabric_t *fabric, char *dr_str) { struct ibnd_fabric *f = CONV_FABRIC_INTERNAL(fabric); int i = 0; - ibnd_node_t *rc = f->fabric.from_node; + ibnd_node_t *rc; ib_dr_path_t path; + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return (NULL); + } + + rc = f->fabric.from_node; + if (str2drpath(&path, dr_str, 0, 0) == -1) { return (NULL); } @@ -640,6 +662,9 @@ ibnd_destroy_fabric(ibnd_fabric_t *fabric) struct ibnd_node *next = NULL; ibnd_chassis_t *ch, *ch_next; + if (!fabric) + return; + ch = f->first_chassis; while (ch) { ch_next = ch->next; @@ -684,6 +709,16 @@ ibnd_iter_nodes(ibnd_fabric_t *fabric, { ibnd_node_t *cur = NULL; + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return; + } + + if (!func) { + IBND_DEBUG("func parameter NULL\n"); + return; + } + for (cur = fabric->nodes; cur; cur = cur->next) { func(cur, user_data); } @@ -700,6 +735,16 @@ ibnd_iter_nodes_type(ibnd_fabric_t *fabric, struct ibnd_node *list = NULL; struct ibnd_node *cur = NULL; + if (!fabric) { + IBND_DEBUG("fabric parameter NULL\n"); + return; + } + + if (!func) { + IBND_DEBUG("func parameter NULL\n"); + return; + } + switch (node_type) { case IB_NODE_SWITCH: list = f->switches; -- 1.5.4.5 From sashak at voltaire.com Mon Jul 27 10:14:19 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 27 Jul 2009 20:14:19 +0300 Subject: [ofa-general] Re: [PATCH] opensm: Add ability to configure SMSL In-Reply-To: <20090723130529.GA27568@comcast.net> References: <20090723130529.GA27568@comcast.net> Message-ID: <20090727171419.GJ18150@me> On 09:05 Thu 23 Jul , Hal Rosenstock wrote: > > Override default SM's SL to use in cases where LASH is not used. > > Signed-off-by: Robert Pearson > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From chu11 at llnl.gov Mon Jul 27 10:13:01 2009 From: chu11 at llnl.gov (Al Chu) Date: Mon, 27 Jul 2009 10:13:01 -0700 Subject: [ofa-general] [infiniband-diags] [PATCH] [6/6] fix libibnetdisc API consistency and bugs Message-ID: <1248714781.16723.329.camel@auk31.llnl.gov> Remove timeout_ms parameter to ibnd_discover_fabric, timeout parameter should be specified via the ibmad_port. Remove extraneous use of global timeout_ms in library. Adjust ibnetdiscover, ibqueryerrors, iblinkinfo, and test code appropriately for adjustment. Al -- Albert Chu chu11 at llnl.gov Computer Scientist High Performance Systems Division Lawrence Livermore National Laboratory -------------- next part -------------- From: Albert Chu Date: Thu, 23 Jul 2009 14:16:14 -0700 Subject: [PATCH] Remove timeout_ms parameter to ibnd_discover_fabric, timeout parameter should be specified via the ibmad_port. Remove extraneous use of global timeout_ms in library. Adjust ibnetdiscover, ibqueryerrors, iblinkinfo, and test code appropriately for adjustment. Signed-off-by: Albert Chu --- .../libibnetdisc/include/infiniband/ibnetdisc.h | 4 --- infiniband-diags/libibnetdisc/src/ibnetdisc.c | 21 +++++++++---------- infiniband-diags/libibnetdisc/test/testleaks.c | 6 +++- infiniband-diags/src/iblinkinfo.c | 7 ++++- infiniband-diags/src/ibnetdiscover.c | 8 +++--- infiniband-diags/src/ibqueryerrors.c | 7 ++++- 6 files changed, 28 insertions(+), 25 deletions(-) diff --git a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h index f65690c..8926e58 100644 --- a/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h +++ b/infiniband-diags/libibnetdisc/include/infiniband/ibnetdisc.h @@ -140,13 +140,9 @@ MAD_EXPORT void ibnd_debug(int i); MAD_EXPORT void ibnd_show_progress(int i); MAD_EXPORT ibnd_fabric_t *ibnd_discover_fabric(struct ibmad_port *ibmad_port, - int timeout_ms, ib_portid_t *from, int hops); /** * open: (required) ibmad_port object from libibmad - * timeout_ms: (required) gives the timeout for a _SINGLE_ query on - * the fabric. So if there are multiple nodes not - * responding this may result in a lengthy delay. * from: (optional) specify the node to start scanning from. * If NULL start from the node we are running on. * hops: (optional) Specify how much of the fabric to traverse. diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index 3f9a3dd..60dd128 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -57,7 +57,6 @@ #include "internal.h" #include "chassis.h" -static int timeout_ms = 2000; static int show_progress = 0; int ibdebug; @@ -80,7 +79,7 @@ get_port_info(struct ibmad_port *ibmad_port, struct ibnd_fabric *fabric, iwidth = mad_get_field(port->port.info, 0, IB_PORT_LINK_WIDTH_ACTIVE_F); ispeed = mad_get_field(port->port.info, 0, IB_PORT_LINK_SPEED_ACTIVE_F); - if (!smp_query_via(port->port.info, portid, IB_ATTR_PORT_INFO, portnum, timeout_ms, + if (!smp_query_via(port->port.info, portid, IB_ATTR_PORT_INFO, portnum, 0, ibmad_port)) return -1; @@ -102,7 +101,7 @@ static int query_node_info(struct ibmad_port *ibmad_port, struct ibnd_fabric *fabric, struct ibnd_node *node, ib_portid_t *portid) { - if (!smp_query_via(&(node->node.info), portid, IB_ATTR_NODE_INFO, 0, timeout_ms, + if (!smp_query_via(&(node->node.info), portid, IB_ATTR_NODE_INFO, 0, 0, ibmad_port)) return -1; @@ -132,11 +131,11 @@ query_node(struct ibmad_port *ibmad_port, struct ibnd_fabric *fabric, port->portnum = mad_get_field(node->info, 0, IB_NODE_LOCAL_PORT_F); port->guid = mad_get_field64(node->info, 0, IB_NODE_PORT_GUID_F); - if (!smp_query_via(nd, portid, IB_ATTR_NODE_DESC, 0, timeout_ms, + if (!smp_query_via(nd, portid, IB_ATTR_NODE_DESC, 0, 0, ibmad_port)) return -1; - if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, 0, timeout_ms, + if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, 0, 0, ibmad_port)) return -1; decode_port_info(port); @@ -148,7 +147,7 @@ query_node(struct ibmad_port *ibmad_port, struct ibnd_fabric *fabric, node->smalmc = port->lmc; /* after we have the sma information find out the real PortInfo for this port */ - if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, port->portnum, timeout_ms, + if (!smp_query_via(port->info, portid, IB_ATTR_PORT_INFO, port->portnum, 0, ibmad_port)) return -1; decode_port_info(port); @@ -156,7 +155,7 @@ query_node(struct ibmad_port *ibmad_port, struct ibnd_fabric *fabric, port->base_lid = (uint16_t) node->smalid; /* LID is still defined by port 0 */ port->lmc = (uint8_t) node->smalmc; - if (!smp_query_via(node->switchinfo, portid, IB_ATTR_SWITCH_INFO, 0, timeout_ms, + if (!smp_query_via(node->switchinfo, portid, IB_ATTR_SWITCH_INFO, 0, 0, ibmad_port)) node->smaenhsp0 = 0; /* assume base SP0 */ else @@ -293,7 +292,7 @@ ibnd_update_node(struct ibmad_port *ibmad_port, ibnd_fabric_t *fabric, ibnd_node if (query_node_info(ibmad_port, f, n, &(n->node.path_portid))) return (NULL); - if (!smp_query_via(nd, &(n->node.path_portid), IB_ATTR_NODE_DESC, 0, timeout_ms, + if (!smp_query_via(nd, &(n->node.path_portid), IB_ATTR_NODE_DESC, 0, 0, ibmad_port)) return (NULL); @@ -306,14 +305,14 @@ ibnd_update_node(struct ibmad_port *ibmad_port, ibnd_fabric_t *fabric, ibnd_node if (n->node.type != IB_NODE_SWITCH) goto done; - if (!smp_query_via(portinfo_port0, &(n->node.path_portid), IB_ATTR_PORT_INFO, 0, timeout_ms, + if (!smp_query_via(portinfo_port0, &(n->node.path_portid), IB_ATTR_PORT_INFO, 0, 0, ibmad_port)) return (NULL); n->node.smalid = mad_get_field(portinfo_port0, 0, IB_PORT_LID_F); n->node.smalmc = mad_get_field(portinfo_port0, 0, IB_PORT_LMC_F); - if (!smp_query_via(node->switchinfo, &(n->node.path_portid), IB_ATTR_SWITCH_INFO, 0, timeout_ms, + if (!smp_query_via(node->switchinfo, &(n->node.path_portid), IB_ATTR_SWITCH_INFO, 0, 0, ibmad_port)) node->smaenhsp0 = 0; /* assume base SP0 */ else @@ -536,7 +535,7 @@ get_remote_node(struct ibmad_port *ibmad_port, struct ibnd_fabric *fabric, } ibnd_fabric_t * -ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, +ibnd_discover_fabric(struct ibmad_port *ibmad_port, ib_portid_t *from, int hops) { struct ibnd_fabric *fabric = NULL; diff --git a/infiniband-diags/libibnetdisc/test/testleaks.c b/infiniband-diags/libibnetdisc/test/testleaks.c index 0d009c3..a8f5300 100644 --- a/infiniband-diags/libibnetdisc/test/testleaks.c +++ b/infiniband-diags/libibnetdisc/test/testleaks.c @@ -161,11 +161,13 @@ main(int argc, char **argv) ibmad_port = mad_rpc_open_port(ca, ca_port, mgmt_classes, 2); + mad_rpc_set_timeout(ibmad_port, timeout_ms); + while (iters == -1 || iters-- > 0) { if (from) { /* only scan part of the fabric */ str2drpath(&(port_id.drpath), from, 0, 0); - if ((fabric = ibnd_discover_fabric(ibmad_port, timeout_ms, + if ((fabric = ibnd_discover_fabric(ibmad_port, &port_id, hops)) == NULL) { fprintf(stderr, "discover failed\n"); rc = 1; @@ -173,7 +175,7 @@ main(int argc, char **argv) } guid = 0; } else { - if ((fabric = ibnd_discover_fabric(ibmad_port, timeout_ms, NULL, -1)) == NULL) { + if ((fabric = ibnd_discover_fabric(ibmad_port, NULL, -1)) == NULL) { fprintf(stderr, "discover failed\n"); rc = 1; goto close_port; diff --git a/infiniband-diags/src/iblinkinfo.c b/infiniband-diags/src/iblinkinfo.c index bba5c5c..5dfadee 100644 --- a/infiniband-diags/src/iblinkinfo.c +++ b/infiniband-diags/src/iblinkinfo.c @@ -308,6 +308,9 @@ main(int argc, char **argv) exit(1); } + if (ibd_timeout) + mad_rpc_set_timeout(ibmad_port, ibd_timeout); + node_name_map = open_node_name_map(node_name_map_file); if (dr_path) { @@ -324,12 +327,12 @@ main(int argc, char **argv) } if (resolved >= 0) - if ((fabric = ibnd_discover_fabric(ibmad_port, ibd_timeout, &port_id, + if ((fabric = ibnd_discover_fabric(ibmad_port, &port_id, hops)) == NULL) IBWARN("Single node discover failed; attempting full scan\n"); if (!fabric) - if ((fabric = ibnd_discover_fabric(ibmad_port, ibd_timeout, NULL, -1)) == NULL) { + if ((fabric = ibnd_discover_fabric(ibmad_port, NULL, -1)) == NULL) { fprintf(stderr, "discover failed\n"); rc = 1; goto close_port; diff --git a/infiniband-diags/src/ibnetdiscover.c b/infiniband-diags/src/ibnetdiscover.c index 1339485..37f769c 100644 --- a/infiniband-diags/src/ibnetdiscover.c +++ b/infiniband-diags/src/ibnetdiscover.c @@ -670,9 +670,6 @@ int main(int argc, char **argv) argc -= optind; argv += optind; - if (ibd_timeout) - timeout = ibd_timeout; - if (ibverbose) ibnd_debug(1); @@ -680,12 +677,15 @@ int main(int argc, char **argv) if (!ibmad_port) IBERROR("Failed to open %s port %d", ibd_ca, ibd_ca_port); + if (ibd_timeout) + mad_rpc_set_timeout(ibmad_port, ibd_timeout); + if (argc && !(f = fopen(argv[0], "w"))) IBERROR("can't open file %s for writing", argv[0]); node_name_map = open_node_name_map(node_name_map_file); - if ((fabric = ibnd_discover_fabric(ibmad_port, ibd_timeout, NULL, -1)) == NULL) + if ((fabric = ibnd_discover_fabric(ibmad_port, NULL, -1)) == NULL) IBERROR("discover failed\n"); if (ports_report) diff --git a/infiniband-diags/src/ibqueryerrors.c b/infiniband-diags/src/ibqueryerrors.c index 09f57c5..2c85423 100644 --- a/infiniband-diags/src/ibqueryerrors.c +++ b/infiniband-diags/src/ibqueryerrors.c @@ -422,6 +422,9 @@ main(int argc, char **argv) if (!ibmad_port) IBERROR("Failed to open port; %s:%d\n", ibd_ca, ibd_ca_port); + if (ibd_timeout) + mad_rpc_set_timeout(ibmad_port, ibd_timeout); + node_name_map = open_node_name_map(node_name_map_file); /* limit the scan the fabric around the target */ @@ -437,12 +440,12 @@ main(int argc, char **argv) } if (resolved >= 0) - if ((fabric = ibnd_discover_fabric(ibmad_port, ibd_timeout, &portid, + if ((fabric = ibnd_discover_fabric(ibmad_port, &portid, 0)) == NULL) IBWARN("Single node discover failed; attempting full scan\n"); if (!fabric) /* do a full scan */ - if ((fabric = ibnd_discover_fabric(ibmad_port, ibd_timeout, NULL, -1)) == NULL) { + if ((fabric = ibnd_discover_fabric(ibmad_port, NULL, -1)) == NULL) { fprintf(stderr, "discover failed\n"); rc = 1; goto close_port; -- 1.5.4.5 From hal.rosenstock at gmail.com Mon Jul 27 10:41:00 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Mon, 27 Jul 2009 13:41:00 -0400 Subject: [ofa-general] Re: [PATCH] opensm/osm_mesh: Fix use after free problem in osm_mesh_node_delete In-Reply-To: <20090727170627.GI18150@me> References: <20090724230303.GA28379@comcast.net> <20090727170627.GI18150@me> Message-ID: On Mon, Jul 27, 2009 at 1:06 PM, Sasha Khapyorsky wrote: > On 19:03 Fri 24 Jul , Hal Rosenstock wrote: > > > > When osm_mesh_node_delete is called, osm_switch_delete may already have > > been called so sw->p_sw is no longer valid to be used although it was > > being used to obtain num_ports. > > And this looks like a bug in mesh code. Why does mesh need mesh node > structure after routing was calculated? LASH needs its switch info in > order to provide proper SL value to SA queries, but mesh info is not > needed then. > > I think that proper way to fix this is to free mesh nodes when it is not > needed anymore and not on a next LASH cycle start (like LASH switches). It could be done this way but it's unclear whether that's the best approach going forward. In the future, to minimize the changes on an update due to topology change, we're not sure how much state will need to be kept from one iteration to the next. We suspect that a good solution to the update problem will require preserving mesh state as well as lash state. -- Hal > > Sasha > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > -------------- next part -------------- An HTML attachment was scrubbed... URL: From davem at systemfabricworks.com Mon Jul 27 11:01:49 2009 From: davem at systemfabricworks.com (David McMillen) Date: Mon, 27 Jul 2009 13:01:49 -0500 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <4A6DB346.6000500@voltaire.com> References: <4A69266E.mail4JH11RL5U@systemfabricworks.com> <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> <4A69DCA5.9040705@opengridcomputing.com> <4A69DE72.4020901@opengridcomputing.com> <4A6C0071.3050305@voltaire.com> <5e3be0f30907260332w3c7097f2g5ff61b2b382f368a@mail.gmail.com> <4A6DB346.6000500@voltaire.com> Message-ID: <5e3be0f30907271101h1ad8eeabib5fcec29fe1f19cf@mail.gmail.com> On Mon, Jul 27, 2009 at 9:01 AM, Or Gerlitz wrote: > David McMillen wrote: > >> >> I am pleased to see the discussions this has raised about possible changes >> to the underlying kernel services. However, this user-mode patch does >> address a real problem in the stack as it stands, and I'd like to see >> agreement that it should be applied >> > > Generally speaking, since you only changed a synthetic program, I don't > have any issue with your patch, except for maybe creating possible > confusion among people that can use this code as reference for their apps, > see more below I would be happy to rework this patch again to make it a proper example. Please let me know what to change. I read through the threads referenced below and did not see anything that relates to this patch. In all versions of rdma_cm, including what is in the about to released OFED 1.5, and assuming a properly functioning fabric, both rdma_resolve_addr and rdma_resolve_route can fail due to ETIMEDOUT, and both can be retried with success. Is there an example that I missed somewhere that shows how I should be doing things? Thanks, Dave > > > I don't know enough about the kernel level code to make comments on the >> best way to improve it, but I would like to make an observation from the >> user level. There is no way for the user level to have an informed decision >> about the proper value for timeouts or retry counts ... >> > The scalability issue is in the air for couple of years now, somehow it > came into of many people being sure the problem is SA scalability, where > personally, I am not sure this is the case. Also, in the past have tried to > set up time for low level technical talking on the matter in ofa meeting, > but it was almost always washed a way or given very tiny time slot since > more important issues such as why ofed is the greatest thing on earth and > what was the content of its last version and what will be the content of its > next version, etc, etc. So what happens is that from time to time this or > that related issue comes to the list, we have a thread on that and things > are left in the air. Recently Sean commented that he works on something > http://lists.openfabrics.org/pipermail/ewg/2009-July/013618.html also at > May we had a related thread "How to establish IB communcation more > effectively?" @ > http://lists.openfabrics.org/pipermail/general/2009-May/thread.html#59574etc etc > > Or. > -------------- next part -------------- An HTML attachment was scrubbed... URL: From sashak at voltaire.com Mon Jul 27 12:27:58 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Mon, 27 Jul 2009 22:27:58 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_mesh: Fix use after free problem in osm_mesh_node_delete In-Reply-To: References: <20090724230303.GA28379@comcast.net> <20090727170627.GI18150@me> Message-ID: <20090727192758.GK18150@me> On 13:41 Mon 27 Jul , Hal Rosenstock wrote: > > It could be done this way but it's unclear whether that's the best approach > going forward. In the future, to minimize the changes on an update due to > topology change, we're not sure how much state will need to be kept from > one iteration to the next. When we will know we will do accordingly. But now mesh node structures are useless between sweeps and freed anyway, so let's just do it in a right place. Sasha From jgunthorpe at obsidianresearch.com Mon Jul 27 12:29:38 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Mon, 27 Jul 2009 13:29:38 -0600 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A6DDFCE.9060009@voltaire.com> References: <4A6DDFCE.9060009@voltaire.com> Message-ID: <20090727192938.GD5794@obsidianresearch.com> On Mon, Jul 27, 2009 at 08:11:42PM +0300, Yossi Etigin wrote: > If the LID of an ipoib neighbour changes without a SM event on the local node, > IPoIB will keep caching the invalid path until the device is flushed. The patch > below will remove the path for every incoming ARP packet where the sender hardware > address does not match the cached lid. This assumes a LMC=0 fabric.. Otherwise there is no guarentee that the slid the sender uses is the same one you are using to send. Jason From hnrose at comcast.net Mon Jul 27 13:22:59 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Mon, 27 Jul 2009 16:22:59 -0400 Subject: [ofa-general] [PATCH] opensm/main.c: Display SMSL when specified Message-ID: <20090727202259.GB23402@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c index 78aba68..35ca365 100644 --- a/opensm/opensm/main.c +++ b/opensm/opensm/main.c @@ -980,6 +980,7 @@ int main(int argc, char *argv[]) return (-1); } opt.sm_sl = (uint8_t) temp; + printf(" SMSL = %d\n", opt.sm_sl); break; case 'h': case '?': From hnrose at comcast.net Mon Jul 27 13:22:14 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Mon, 27 Jul 2009 16:22:14 -0400 Subject: [ofa-general] [PATCH] opensm/osm_ucast_lash.c: Handle malloc failures better Message-ID: <20090727202214.GA23402@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 7133e25..a139bdb 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -296,8 +296,8 @@ static void shortest_path(lash_t * p_lash, int ir) cl_list_destroy(&bfsq); } -static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, - reachable_dest_t ** destinations) +static boolean_t generate_routing_func_for_mst(lash_t * p_lash, int sw_id, + reachable_dest_t ** destinations) { int i, next_switch; switch_t *sw = p_lash->switches[sw_id]; @@ -306,7 +306,8 @@ static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, for (i = 0; i < num_channels; i++) { next_switch = sw->dij_channels[i]; - generate_routing_func_for_mst(p_lash, next_switch, &dest); + if (!generate_routing_func_for_mst(p_lash, next_switch, &dest)) + return FALSE; i_dest = dest; prev = i_dest; @@ -327,9 +328,15 @@ static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, } i_dest = (reachable_dest_t *) malloc(sizeof(reachable_dest_t)); - i_dest->switch_id = sw->id; - i_dest->next = concat_dest; + if (i_dest) { + i_dest->switch_id = sw->id; + i_dest->next = concat_dest; + } *destinations = i_dest; + if (i_dest) + return TRUE; + else + return FALSE; } static void generate_cdg_for_sp(lash_t * p_lash, int sw, int dest_switch, @@ -707,6 +714,8 @@ static int init_lash_structures(lash_t * p_lash) /* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */ p_lash->cdg_vertex_matrix = (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****)); + if (p_lash->cdg_vertex_matrix == NULL) + goto Exit_Mem_Error; for (i = 0; i < vl_min; i++) { p_lash->cdg_vertex_matrix[i] = (cdg_vertex_t ***) malloc(num_switches * @@ -800,7 +809,11 @@ static int lash_core(lash_t * p_lash) for (i = 0; i < num_switches; i++) { shortest_path(p_lash, i); - generate_routing_func_for_mst(p_lash, i, &dests); + if (!generate_routing_func_for_mst(p_lash, i, &dests)) { + OSM_LOG(p_log, OSM_LOG_ERROR, + "generate_routing_func_for_mst failed\n"); + goto Exit; + } idest = dests; while (idest != NULL) { From sebastien.dugue at bull.net Tue Jul 28 00:27:36 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Tue, 28 Jul 2009 09:27:36 +0200 Subject: [ofa-general] Re: [PATCH] ibsim: Drop vendor MADs and vendor specific SM MADs In-Reply-To: <20090727152036.GB18150@me> References: <20090727133801.38451235@frecb007965> <20090727152036.GB18150@me> Message-ID: <20090728092736.4170ade3@frecb007965> Hi Sasha, On Mon, 27 Jul 2009 18:20:36 +0300 Sasha Khapyorsky wrote: > On 13:38 Mon 27 Jul , sebastien dugue wrote: > > > > This is useful for testing applications that send those kind of MADS. > > How this is useful in such test? Because I discovered that when using libibumad, it hangs in umad_recv() for those MADs when the client is OpenSM regardless of the timeout I set. > I see that this just drops every sort > of such MADs without letting chance to client to process it. No? Right, not so good, forget it. I guess I'll have to find out why OpenSM silently drops those MADs and why I get stuck in umad_recv(). Thanks, Sebastien. > > Sasha > > > > > > Signed-off-by: Sebastien Dugue > > --- > > ibsim/sim_mad.c | 19 +++++++++++++++++++ > > 1 files changed, 19 insertions(+), 0 deletions(-) > > > > diff --git a/ibsim/sim_mad.c b/ibsim/sim_mad.c > > index 61d4866..b79a5e7 100644 > > --- a/ibsim/sim_mad.c > > +++ b/ibsim/sim_mad.c > > @@ -1170,6 +1170,25 @@ int process_packet(Client * cl, void *p, int size, Client ** dcl) > > if ((response = decode_sim_MAD(cl, r, &rpc, &path, data)) < 0) > > return -1; > > > > + /* Drop vendor MADs */ > > + if (((rpc.mgtclass >= IB_VENDOR_RANGE1_START_CLASS) && > > + (rpc.mgtclass <= IB_VENDOR_RANGE1_END_CLASS)) || > > + ((rpc.mgtclass >= IB_VENDOR_RANGE2_START_CLASS) && > > + (rpc.mgtclass <= IB_VENDOR_RANGE2_END_CLASS))) { > > + IBWARN("ignoring vendor MAD: class 0x%x, attr 0x%x", > > + rpc.mgtclass, rpc.attr.id); > > + goto _dropped; > > + } > > + > > + /* Drop vendor specific SM MADs */ > > + if (((rpc.mgtclass == IB_SMI_CLASS) || > > + (rpc.mgtclass == IB_SMI_DIRECT_CLASS)) && > > + (rpc.attr.id >= 0xff00)) { > > + IBWARN("ignoring vendor specific SM MAD: attr 0x%x", > > + rpc.attr.id); > > + goto _dropped; > > + } > > + > > if (rpc.method == 0x7) { > > IBWARN("lid %u got trap repress - dropping", ntohs(r->dlid)); > > *dcl = 0; > > -- > > 1.6.3.1 > > > From sebastien.dugue at bull.net Tue Jul 28 00:53:30 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Tue, 28 Jul 2009 09:53:30 +0200 Subject: [ofa-general] Re: [PATCH] ibsim: Allow multi-threaded use of ibsim In-Reply-To: <20090727154722.GD18150@me> References: <20090727133835.4883cee4@frecb007965> <20090727154722.GD18150@me> Message-ID: <20090728095330.7beb0fa6@frecb007965> Hi Sasha, On Mon, 27 Jul 2009 18:47:22 +0300 Sasha Khapyorsky wrote: > On 13:38 Mon 27 Jul , sebastien dugue wrote: > > > > Right now, the socket name used between the client and ibsim is appended > > with the client's pid. However this does not work when several threads with > > the same pid want to register their clients. > > And strictly speaking umad2sim by itself is not thread safe. So enabling > a multi-threaded clients you need to keep this in a mind, may be to add > some notes about this to README. Well in fact I poorly chose the subject for this mail. It should have read: 'Allow multiple agents per process' There's nothing that needs to be thread safe in there. The thing is, if you want multiple threads from a single process to register their own agents, then the agents cannot be attached to the umad2sim_dev instance, but rather to a file instance opened on that device. That way each thread that wants to register an agent does it's own umad_open_port(). > > > instead of the pid, use a combination of the thread pid along with a > > connection number incremented for each new client. > > > > Also, attach the agents to the file instance opened on the device rather > > than on the device itself. > > Would be nice to have this as separate patch. No, it's all the same thing. Sorry for the confusion. More below... > > > > > Signed-off-by: Sebastien Dugue > > --- > > umad2sim/sim_client.c | 11 ++- > > umad2sim/umad2sim.c | 171 +++++++++++++++++++++++++++++++++++++----------- > > 2 files changed, 140 insertions(+), 42 deletions(-) > > > > diff --git a/umad2sim/sim_client.c b/umad2sim/sim_client.c > > index eb42a7c..22a50a2 100644 > > --- a/umad2sim/sim_client.c > > +++ b/umad2sim/sim_client.c > > @@ -209,6 +209,8 @@ static int sim_init(struct sim_client *sc, char *nodeid) > > char *connect_port; > > char *connect_host; > > unsigned short port; > > + static int conn_num = 0; > > + int conn_id; > > > > connect_port = getenv("IBSIM_SERVER_PORT"); > > connect_host = getenv("IBSIM_SERVER_NAME"); > > @@ -228,7 +230,10 @@ static int sim_init(struct sim_client *sc, char *nodeid) > > if ((ctlfd = socket(remote_mode ? PF_INET : PF_LOCAL, SOCK_DGRAM, 0)) < 0) > > IBPANIC("can't get socket (ctlfd)"); > > > > - size = make_name(&name, NULL, 0, "%s:ctl%d", socket_basename, pid); > > + conn_id = ((conn_num & 0xff) << 24) | (pid & 0xffffff); > > + conn_num++; > > BTW why to not use pid/tid combination? This would eliminate the needs in > yet another static (and thread unsafe) variable. That was my first thought, but gettid() is pretty much Linux specific so I tried another approach. I agree though that conn_num and it's use should be protected somehow. Will add a spinlock for that. > > > + > > + size = make_name(&name, NULL, 0, "%s:ctl%d", socket_basename, conn_id); > > > > if (bind(ctlfd, (struct sockaddr *)&name, size) < 0) > > IBPANIC("can't bind ctl socket"); > > @@ -243,7 +248,7 @@ static int sim_init(struct sim_client *sc, char *nodeid) > > > > sc->fd_ctl = ctlfd; > > > > - size = make_name(&name, NULL, 0, "%s:in%d", socket_basename, pid); > > + size = make_name(&name, NULL, 0, "%s:in%d", socket_basename, conn_id); > > > > if (bind(fd, (struct sockaddr *)&name, size) < 0) > > IBPANIC("can't bind input socket"); > > @@ -254,7 +259,7 @@ static int sim_init(struct sim_client *sc, char *nodeid) > > IBPANIC("can't read data from bound socket"); > > port = ntohs(name.name_i.sin_port); > > > > - sc->clientid = sim_connect(sc, remote_mode ? port : pid, 0, nodeid); > > + sc->clientid = sim_connect(sc, remote_mode ? port : conn_id, 0, nodeid); > > if (sc->clientid < 0) > > IBPANIC("connect failed"); > > > > diff --git a/umad2sim/umad2sim.c b/umad2sim/umad2sim.c > > index 55440ec..71f7c67 100644 > > --- a/umad2sim/umad2sim.c > > +++ b/umad2sim/umad2sim.c > > @@ -81,12 +81,17 @@ struct umad2sim_dev { > > char name[32]; > > uint8_t port; > > struct sim_client sim_client; > > Why do we need sim_client field here if it is going to umad2sim_devfile > structure? We need it here too in umad2sim_dev_create() and children to create the sysfs tree. > > > - unsigned agent_idx[256]; > > - struct ib_user_mad_reg_req agents[32]; > > char umad_path[256]; > > char issm_path[256]; > > }; > > > > +struct umad2sim_devfile { > > + struct umad2sim_dev *dev; > > + struct sim_client sim_client; > > + struct ib_user_mad_reg_req agents[32]; > > + unsigned agent_idx[256]; > > +}; > > + > > static int (*real_open) (const char *path, int flags, ...); > > static int (*real_close) (int fd); > > static ssize_t(*real_read) (int fd, void *buf, size_t count); > > @@ -113,6 +118,7 @@ static char umad2sim_sysfs_prefix[32]; > > > > static unsigned umad2sim_initialized; > > static struct umad2sim_dev *devices[32]; > > +static struct umad2sim_devfile *devfiles[1024]; > > > > /* > > * sysfs stuff > > @@ -378,9 +384,69 @@ static int dev_sysfs_create(struct umad2sim_dev *dev) > > * umad2sim device > > * > > */ > > +static int umad2sim_open(struct umad2sim_dev *dev) > > +{ > > + int i; > > + int fd; > > + struct umad2sim_devfile *file; > > + > > + /* Find unused fd */ > > + for (fd = 0; fd < 1024; fd++) > > arrsize() is more suitable here (and in another similar places). Right, will change those. > > > + if (devfiles[fd] == NULL) > > + break; > > + > > + if (fd == 1024) { > > + ERROR("umad2sim_open: No more available files\n"); > > + errno = EMFILE; > > + return -1; > > + } > > > > -static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) > > + if ((file = calloc(1, sizeof(struct umad2sim_devfile))) == NULL) { > > + ERROR("umad2sim_open: Out of memory\n"); > > + errno = ENOMEM; > > + return -1; > > + } > > + > > + file->dev = dev; > > + > > + for (i = 0; i < arrsize(file->agents); i++) > > + file->agents[i].id = (uint32_t)(-1); > > + > > + for (i = 0; i < arrsize(file->agent_idx); i++) > > + file->agent_idx[i] = (unsigned)(-1); > > + > > + if (sim_client_init(&file->sim_client) < 0) { > > + free(file); > > + return -1; > > + } > > + > > + devfiles[fd] = file; > > + > > + return 1024 + fd; > > +} > > + > > +static int umad2sim_close(int fd) > > { > > + struct umad2sim_devfile *file; > > + > > + if ((file = devfiles[fd - 1024]) == NULL) { > > + errno = EBADF; > > + ERROR("umad2sim_close: no such file\n"); > > + return -1; > > + } > > + > > + sim_client_exit(&file->sim_client); > > + > > + free(file); > > + > > + devfiles[fd - 1024] = NULL; > > + > > + return 0; > > +} > > + > > +static ssize_t umad2sim_read(int fd, void *buf, size_t count) > > +{ > > + struct umad2sim_devfile *file; > > struct sim_request req; > > ib_user_mad_t *umad = (ib_user_mad_t *) buf; > > unsigned mgmt_class; > > @@ -388,7 +454,13 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) > > > > DEBUG("umad2sim_read: %zu...\n", count); > > > > - cnt = real_read(dev->sim_client.fd_pktin, &req, sizeof(req)); > > + if ((file = devfiles[fd - 1024]) == NULL) { > > + errno = EBADF; > > + ERROR("umad2sim_read: no such file\n"); > > + return -1; > > + } > > + > > + cnt = real_read(file->sim_client.fd_pktin, &req, sizeof(req)); > > DEBUG("umad2sim_read: got %d...\n", cnt); > > if (cnt < sizeof(req)) { > > ERROR("umad2sim_read: partial request - skip.\n"); > > @@ -406,7 +478,7 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) > > mad_get_field(req.mad, 0, IB_MAD_ATTRID_F), > > mad_get_field(req.mad, 0, IB_MAD_ATTRMOD_F)); > > > > - if (mgmt_class >= arrsize(dev->agent_idx)) { > > + if (mgmt_class >= arrsize(file->agent_idx)) { > > ERROR("bad mgmt_class 0x%x\n", mgmt_class); > > mgmt_class = 0; > > } > > @@ -415,7 +487,7 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) > > uint64_t trid = mad_get_field64(req.mad, 0, IB_MAD_TRID_F); > > umad->agent_id = (trid >> 32) & 0xffff; > > } else > > - umad->agent_id = dev->agent_idx[mgmt_class]; > > + umad->agent_id = file->agent_idx[mgmt_class]; > > > > umad->status = ntohl(req.status); > > umad->timeout_ms = 0; > > @@ -437,9 +509,9 @@ static ssize_t umad2sim_read(struct umad2sim_dev *dev, void *buf, size_t count) > > return umad->length; > > } > > > > -static ssize_t umad2sim_write(struct umad2sim_dev *dev, > > - const void *buf, size_t count) > > +static ssize_t umad2sim_write(int fd, const void *buf, size_t count) > > { > > + struct umad2sim_devfile *file; > > struct sim_request req; > > ib_user_mad_t *umad = (ib_user_mad_t *) buf; > > int cnt; > > @@ -457,12 +529,18 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, > > > > DEBUG("umad2sim_write: %zu...\n", count); > > > > + if ((file = devfiles[fd - 1024]) == NULL) { > > + errno = EBADF; > > + ERROR("umad2sim_write: no such file\n"); > > + return -1; > > + } > > + > > DEBUG("umad2sim_write: umad: agent_id=%u, retries=%u, " > > "agent.class=%x, agent.qpn=%u, " > > "addr.qpn=%u, addr.lid=%u\n", > > umad->agent_id, umad->retries, > > - dev->agents[umad->agent_id].mgmt_class, > > - dev->agents[umad->agent_id].qpn, > > + file->agents[umad->agent_id].mgmt_class, > > + file->agents[umad->agent_id].qpn, > > htonl(umad->addr.qpn), htons(umad->addr.lid)); > > DEBUG("umad2sim_write: mad: method=%x, response=%x, mgmtclass=%x, " > > "attrid=%x, attrmod=%x\n", > > @@ -476,7 +554,7 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, > > req.slid = req.dlid == 0xffff ? 0xffff : 0; /* 0 - means auto > > (supported by ibsim) */ ; > > req.dqp = umad->addr.qpn; > > - req.sqp = htonl(dev->agents[umad->agent_id].qpn); > > + req.sqp = htonl(file->agents[umad->agent_id].qpn); > > req.status = 0; > > > > cnt = count - umad_size(); > > @@ -492,7 +570,7 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, > > mad_set_field64(req.mad, 0, IB_MAD_TRID_F, trid); > > } > > > > - cnt = write(dev->sim_client.fd_pktout, (void *)&req, sizeof(req)); > > + cnt = write(file->sim_client.fd_pktout, (void *)&req, sizeof(req)); > > if (cnt < 0) { > > ERROR("umad2sim_write: cannot write\n"); > > return -1; > > @@ -503,19 +581,21 @@ static ssize_t umad2sim_write(struct umad2sim_dev *dev, > > return count; > > } > > > > -static int register_agent(struct umad2sim_dev *dev, > > +static int register_agent(struct umad2sim_devfile *file, > > struct ib_user_mad_reg_req *req) > > { > > - unsigned i; > > + unsigned i; > > + > > DEBUG("register_agent: id = %u, qpn = %u, mgmt_class = %u," > > " mgmt_class_version = %u, rmpp_version = %u\n", > > req->id, req->qpn, req->mgmt_class, req->mgmt_class_version, > > req->rmpp_version); > > - for (i = 0; i < arrsize(dev->agents); i++) > > - if (dev->agents[i].id == (uint32_t)(-1)) { > > + > > + for (i = 0; i < arrsize(file->agents); i++) > > + if (file->agents[i].id == (uint32_t)(-1)) { > > req->id = i; > > - dev->agents[i] = *req; > > - dev->agent_idx[req->mgmt_class] = i; > > + file->agents[i] = *req; > > + file->agent_idx[req->mgmt_class] = i; > > DEBUG("agent registered: %d\n", i); > > return 0; > > } > > @@ -523,28 +603,36 @@ static int register_agent(struct umad2sim_dev *dev, > > return -1; > > } > > > > -static int unregister_agent(struct umad2sim_dev *dev, unsigned id) > > +static int unregister_agent(struct umad2sim_devfile *file, unsigned id) > > { > > unsigned mgmt_class; > > - if (id >= arrsize(dev->agents)) { > > + if (id >= arrsize(file->agents)) { > > errno = EINVAL; > > return -1; > > } > > - mgmt_class = dev->agents[id].mgmt_class; > > - dev->agents[id].id = (uint32_t)(-1); > > - dev->agent_idx[mgmt_class] = -1; > > + mgmt_class = file->agents[id].mgmt_class; > > + file->agents[id].id = (uint32_t)(-1); > > + file->agent_idx[mgmt_class] = -1; > > return 0; > > } > > > > -static int umad2sim_ioctl(struct umad2sim_dev *dev, unsigned long request, > > - void *arg) > > +static int umad2sim_ioctl(int fd, unsigned long request, void *arg) > > { > > + struct umad2sim_devfile *file; > > + > > DEBUG("umad2sim_ioctl: %lu, %p...\n", request, arg); > > + > > + if ((file = devfiles[fd - 1024]) == NULL) { > > + errno = EBADF; > > + ERROR("umad2sim_ioctl: no such file\n"); > > + return -1; > > + } > > + > > switch (request) { > > case IB_USER_MAD_REGISTER_AGENT: > > - return register_agent(dev, arg); > > + return register_agent(file, arg); > > case IB_USER_MAD_UNREGISTER_AGENT: > > - return unregister_agent(dev, *((unsigned *)arg)); > > + return unregister_agent(file, *((unsigned *)arg)); > > case IB_USER_MAD_ENABLE_PKEY: > > return 0; > > default: > > @@ -556,7 +644,6 @@ static int umad2sim_ioctl(struct umad2sim_dev *dev, unsigned long request, > > static struct umad2sim_dev *umad2sim_dev_create(unsigned num, const char *name) > > { > > struct umad2sim_dev *dev; > > - unsigned i; > > > > DEBUG("umad2sim_dev_create: %s...\n", name); > > > > @@ -573,10 +660,6 @@ static struct umad2sim_dev *umad2sim_dev_create(unsigned num, const char *name) > > > > dev->port = mad_get_field(&dev->sim_client.portinfo, 0, > > IB_PORT_LOCAL_PORT_F); > > - for (i = 0; i < arrsize(dev->agents); i++) > > - dev->agents[i].id = (uint32_t)(-1); > > - for (i = 0; i < arrsize(dev->agent_idx); i++) > > - dev->agent_idx[i] = (unsigned)(-1); > > > > dev_sysfs_create(dev); > > > > @@ -637,6 +720,11 @@ static void umad2sim_cleanup(void) > > char path[1024]; > > unsigned i; > > DEBUG("umad2sim_cleanup...\n"); > > + > > + for (i = 0; i < 1024; i++) > > + if (devfiles[i]) > > + free(devfiles[i]); > > And what about sim_client disconnection? umad2sim_dev_delete() did this. This is now handled in umad2sim_close() called from close(). close() did nothing previously. Hope this shed some light in the intents of the patch. Thanks for your comments, Sebastien. > > Sasha > > > + > > for (i = 0; i < arrsize(devices); i++) > > if (devices[i]) { > > umad2sim_dev_delete(devices[i]); > > @@ -756,7 +844,7 @@ int open(const char *path, int flags, ...) > > if (!(dev = devices[i])) > > continue; > > if (!strncmp(path, dev->umad_path, sizeof(dev->umad_path))) { > > - return 1024 + i; > > + return umad2sim_open(dev); > > } > > if (!strncmp(path, dev->issm_path, sizeof(dev->issm_path))) { > > sim_client_set_sm(&dev->sim_client, 1); > > @@ -779,7 +867,7 @@ int close(int fd) > > sim_client_set_sm(&dev->sim_client, 0); > > return 0; > > } else if (fd >= 1024) { > > - return 0; > > + return umad2sim_close(fd); > > } else > > return real_close(fd); > > } > > @@ -791,7 +879,7 @@ ssize_t read(int fd, void *buf, size_t count) > > if (fd >= 2048) > > return -1; > > else if (fd >= 1024) > > - return umad2sim_read(devices[fd - 1024], buf, count); > > + return umad2sim_read(fd, buf, count); > > else > > return real_read(fd, buf, count); > > } > > @@ -803,7 +891,7 @@ ssize_t write(int fd, const void *buf, size_t count) > > if (fd >= 2048) > > return -1; > > else if (fd >= 1024) > > - return umad2sim_write(devices[fd - 1024], buf, count); > > + return umad2sim_write(fd, buf, count); > > else > > return real_write(fd, buf, count); > > } > > @@ -821,7 +909,7 @@ int ioctl(int fd, unsigned long request, ...) > > if (fd >= 2048) > > return -1; > > else if (fd >= 1024) > > - return umad2sim_ioctl(devices[fd - 1024], request, arg); > > + return umad2sim_ioctl(fd, request, arg); > > else > > return real_ioctl(fd, request, arg); > > } > > @@ -836,9 +924,14 @@ int poll(struct pollfd *pfds, nfds_t nfds, int timeout) > > > > for (i = 0; i < nfds; i++) { > > if (pfds[i].fd >= 1024 && pfds[i].fd < 2048) { > > - struct umad2sim_dev *dev = devices[pfds[i].fd - 1024]; > > + struct umad2sim_devfile *file; > > + > > + if ((file = devfiles[pfds[i].fd - 1024]) == NULL) { > > + errno = EBADF; > > + return -1; > > + } > > saved_fds[i] = pfds[i].fd; > > - pfds[i].fd = dev->sim_client.fd_pktin; > > + pfds[i].fd = file->sim_client.fd_pktin; > > } else > > saved_fds[i] = 0; > > } > > -- > > 1.6.3.1 > > > From jackm at dev.mellanox.co.il Tue Jul 28 02:01:44 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Tue, 28 Jul 2009 12:01:44 +0300 Subject: [ofa-general] OFED 1.5 alpha release is available In-Reply-To: <2ED289D4E09FBD4D92D911E869B97FDD29F156@mtlexch01.mtl.com> References: <5D49E7A8952DC44FB38C38FA0D758EAD02C12252@mtlexch01.mtl.com> <5D49E7A8952DC44FB38C38FA0D758EAD02D5F39D@mtlexch01.mtl.com> <2ED289D4E09FBD4D92D911E869B97FDD29F156@mtlexch01.mtl.com> Message-ID: <200907281201.44707.jackm@dev.mellanox.co.il> On Thursday 23 July 2009 21:17, Tziporet Koren wrote: > OFED 1.5-alpha4 is available > > o Linux Operating Systems: ... > - OpenSuSE 10.3: 2.6.22.5-31 * Correction: OpenSuSE 11: 2.6.25.5-1.1-default * (OpenSuSE 10.3 is not supported under OFED 1.5) - Jack From ogerlitz at voltaire.com Tue Jul 28 02:03:28 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Tue, 28 Jul 2009 12:03:28 +0300 Subject: [ofa-general] [PATCH] perftest Add rdma_cm retries In-Reply-To: <5e3be0f30907271101h1ad8eeabib5fcec29fe1f19cf@mail.gmail.com> References: <4A69266E.mail4JH11RL5U@systemfabricworks.com> <1C21905362174A349D0A1567FB31A67A@amr.corp.intel.com> <4A69DCA5.9040705@opengridcomputing.com> <4A69DE72.4020901@opengridcomputing.com> <4A6C0071.3050305@voltaire.com> <5e3be0f30907260332w3c7097f2g5ff61b2b382f368a@mail.gmail.com> <4A6DB346.6000500@voltaire.com> <5e3be0f30907271101h1ad8eeabib5fcec29fe1f19cf@mail.gmail.com> Message-ID: <4A6EBEE0.7000607@voltaire.com> David McMillen wrote: > I would be happy to rework this patch again to make it a proper > example. Please let me know what to change. I read through the > threads referenced below and did not see anything that relates to this > patch. Over these threads (along with others on which I didn't provide you with a pointer), people were attempting to discuss the root cause of the problem and few proposed approaches/solutions. Your patch simply add retries to a synthetic test program and as I wrote you, I don't have any comments on it, expect for that this may not an end-in-mind, that's it. > In all versions of rdma_cm, [...] and assuming a properly functioning > fabric, both rdma_resolve_addr and rdma_resolve_route can fail due to > ETIMEDOUT, and both can be retried with success. Is there an example > that I missed somewhere that shows how I should be doing things? no. In many cases the retry is done in a higher level such as the SCSI stack or the file system core trying to mount, etc. As I wrote, the retry for address and route resolution may be implemented in the rdma-cm and layers below it, but if you feel this is beyond your scope, no problem, at some point the next generations will have to deal with it. Or. From vlad at lists.openfabrics.org Tue Jul 28 02:55:18 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Tue, 28 Jul 2009 02:55:18 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090728-0200 daily build status Message-ID: <20090728095518.878A810201CF@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Passed on i686 with linux-2.6.18 Passed on i686 with linux-2.6.19 Passed on i686 with linux-2.6.21.1 Passed on i686 with linux-2.6.22 Passed on x86_64 with linux-2.6.18 Passed on x86_64 with linux-2.6.21.1 Passed on x86_64 with linux-2.6.20 Passed on x86_64 with linux-2.6.19 Passed on x86_64 with linux-2.6.22 Passed on ia64 with linux-2.6.18 Passed on ia64 with linux-2.6.19 Passed on ia64 with linux-2.6.21.1 Passed on ia64 with linux-2.6.22 Passed on ppc64 with linux-2.6.18 Passed on ppc64 with linux-2.6.19 Failed: Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.23_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.23_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.23_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090728-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From monis at Voltaire.COM Tue Jul 28 03:14:07 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Tue, 28 Jul 2009 13:14:07 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <20090727192938.GD5794@obsidianresearch.com> References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> Message-ID: <4A6ECF6F.4000008@Voltaire.COM> Jason Gunthorpe wrote: > On Mon, Jul 27, 2009 at 08:11:42PM +0300, Yossi Etigin wrote: >> If the LID of an ipoib neighbour changes without a SM event on the local node, >> IPoIB will keep caching the invalid path until the device is flushed. The patch >> below will remove the path for every incoming ARP packet where the sender hardware >> address does not match the cached lid. > > This assumes a LMC=0 fabric.. Otherwise there is no guarentee that the > slid the sender uses is the same one you are using to send. > > Jason > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > This issue can be solved by a little bit different check instead of be16_to_cpu(path->pathrec.dlid) == lid) do is_in_same_lid_group(be16_to_cpu(path->pathrec.dlid), lid, lmc) what do you think? From monis at Voltaire.COM Tue Jul 28 03:39:50 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Tue, 28 Jul 2009 13:39:50 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A6DDFCE.9060009@voltaire.com> References: <4A6DDFCE.9060009@voltaire.com> Message-ID: <4A6ED576.7050807@Voltaire.COM> > + ipoib_dbg(priv, "Path %pI6: LID changed from 0x%04x to 0x%04x\n", > + path->pathrec.dgid.raw, be16_to_cpu(path->pathrec.dlid), lid); > + > + list_del(&path->list); > + rb_erase(&path->rb_node, &priv->path_tree); > + > + spin_unlock_irqrestore(&priv->lock, flags); > + netif_tx_unlock_bh(dev); > + > + path_free(dev, path); > +} > + Instead of deleting the path you could restart the query with if (!path->query) path_rec_start(dev, path); It's should be the same as in unicast_arp_send() when path->valid == 0 From hal.rosenstock at gmail.com Tue Jul 28 04:27:07 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Tue, 28 Jul 2009 07:27:07 -0400 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A6ECF6F.4000008@Voltaire.COM> References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> Message-ID: On Tue, Jul 28, 2009 at 6:14 AM, Moni Shoua wrote: > Jason Gunthorpe wrote: > > On Mon, Jul 27, 2009 at 08:11:42PM +0300, Yossi Etigin wrote: > >> If the LID of an ipoib neighbour changes without a SM event on the > local node, > >> IPoIB will keep caching the invalid path until the device is flushed. > The patch > >> below will remove the path for every incoming ARP packet where the > sender hardware > >> address does not match the cached lid. > > > > This assumes a LMC=0 fabric.. Otherwise there is no guarentee that the > > slid the sender uses is the same one you are using to send. > > > > Jason > > _______________________________________________ > > general mailing list > > general at lists.openfabrics.org > > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > > > This issue can be solved by a little bit different check > instead of > be16_to_cpu(path->pathrec.dlid) == lid) > do > is_in_same_lid_group(be16_to_cpu(path->pathrec.dlid), lid, > lmc) > > what do you think? Is the LMC in the above from the local port ? Unfortunately, LMC is not required to be uniform across the subnet so the remote port's LMC may not be the same as that on the local port. -- Hal > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > -------------- next part -------------- An HTML attachment was scrubbed... URL: From kliteyn at dev.mellanox.co.il Tue Jul 28 07:39:19 2009 From: kliteyn at dev.mellanox.co.il (Yevgeny Kliteynik) Date: Tue, 28 Jul 2009 17:39:19 +0300 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <20090721180312.GA12491@comcast.net> References: <20090721180312.GA12491@comcast.net> Message-ID: <4A6F0D97.5050402@dev.mellanox.co.il> Sasha, Hal Rosenstock wrote: > Currently, MADs are pipelined to a single switch at a time which > effectively serializes these requests due to processing at the SMA. > This patch pipelines (stripes) them across the switches first before > proceeding with successive blocks. As a result of this striping, > multiple switches can process the set and respond concurrently > which results in an improvement to the subnet initialization time. I have some numbers, and they look very good. I have a small cluster of 17 IS4 switches and 11 HCAs. To artificially increase the cluster I used LMC=7, including ExtendedSwitchPort 0 LMC. With the new code, LFT configuration is more than twice as fast as with the old code :) Current ucast manager ran on avarage for ~250msec, with the new code - 110-120msec. Routing calculation phase of the ucast manager took ~1200 usec, the rest was sending the blocks and waiting for no more pending transactions. I also didn't see any noticeble difference between various max_smps_per_node values. Here are some detailed results of different executions (the number on the left is timer value in usec): Current ucast manager (w/o the optimization): 000000 [LFT]: osm_ucast_mgr_process() - START 001131 [LFT]: ucast_mgr_process_tbl() - START 032251 [LFT]: ucast_mgr_process_tbl() - END 032263 [LFT]: osm_ucast_mgr_process() - END 253416 [LFT]: Done wait_for_pending_transactions() New code, max_smps_per_node=0: 001417 [LFT]: osm_ucast_mgr_process() - START (0 max_smps_per_node) 002690 [LFT]: ucast_mgr_process_tbl() - START 032946 [LFT]: ucast_mgr_process_tbl() - END 032948 [LFT]: osm_ucast_pipeline_tbl() - START 033846 [LFT]: osm_ucast_pipeline_tbl() - END 033858 [LFT]: osm_ucast_mgr_process() - END 108203 [LFT]: Done wait_for_pending_transactions() New code, max_smps_per_node=1: 007474 [LFT]: osm_ucast_mgr_process() - START (1 max_smps_per_node) 008735 [LFT]: ucast_mgr_process_tbl() - START 040071 [LFT]: ucast_mgr_process_tbl() - END 040074 [LFT]: osm_ucast_pipeline_tbl() - START 040103 [LFT]: osm_ucast_pipeline_tbl() - END 040114 [LFT]: osm_ucast_mgr_process() - END 120097 [LFT]: Done wait_for_pending_transactions() New code, max_smps_per_node=4: 004137 [LFT]: osm_ucast_mgr_process() - START (4 max_smps_per_node) 005380 [LFT]: ucast_mgr_process_tbl() - START 037436 [LFT]: ucast_mgr_process_tbl() - END 037439 [LFT]: osm_ucast_pipeline_tbl() - START 037495 [LFT]: osm_ucast_pipeline_tbl() - END 037506 [LFT]: osm_ucast_mgr_process() - END 114983 [LFT]: Done wait_for_pending_transactions() Here's the patch that shows where I added the log messages in current code: >From d7962cc06f5526982bc51c17d53981b6d23256a8 Mon Sep 17 00:00:00 2001 From: Yevgeny Kliteynik Date: Mon, 27 Jul 2009 16:12:27 +0300 Subject: [PATCH] [LFT - master] log messages for LFT parallelization Signed-off-by: Yevgeny Kliteynik --- opensm/opensm/osm_state_mgr.c | 9 ++++++++- opensm/opensm/osm_ucast_mgr.c | 4 ++++ 2 files changed, 12 insertions(+), 1 deletions(-) diff --git a/opensm/opensm/osm_state_mgr.c b/opensm/opensm/osm_state_mgr.c index adc39a0..573a2a7 100644 --- a/opensm/opensm/osm_state_mgr.c +++ b/opensm/opensm/osm_state_mgr.c @@ -1271,11 +1271,18 @@ _repeat_discovery: */ if (!sm->ucast_mgr.cache_valid || - osm_ucast_cache_process(&sm->ucast_mgr)) + osm_ucast_cache_process(&sm->ucast_mgr)) { + OSM_LOG(sm->p_log, OSM_LOG_INFO, "[LFT]: " + "osm_ucast_mgr_process() - START\n"); osm_ucast_mgr_process(&sm->ucast_mgr); + OSM_LOG(sm->p_log, OSM_LOG_INFO, "[LFT]: " + "osm_ucast_mgr_process() - END\n"); + } if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats)) return; + OSM_LOG(sm->p_log, OSM_LOG_INFO, "[LFT]: " + "Done wait_for_pending_transactions()\n"); /* cleanup switch lft buffers */ cl_qmap_apply_func(&sm->p_subn->sw_guid_tbl, cleanup_switch, sm->p_log); diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c index 78a7031..83867a0 100644 --- a/opensm/opensm/osm_ucast_mgr.c +++ b/opensm/opensm/osm_ucast_mgr.c @@ -901,8 +901,12 @@ static int ucast_mgr_build_lfts(osm_ucast_mgr_t * p_mgr) cl_qmap_apply_func(&p_mgr->p_subn->port_guid_tbl, add_port_to_order_list, p_mgr); + OSM_LOG(p_mgr->p_log, OSM_LOG_INFO, "[LFT]: " + "ucast_mgr_process_tbl() - START\n"); cl_qmap_apply_func(&p_mgr->p_subn->sw_guid_tbl, ucast_mgr_process_tbl, p_mgr); + OSM_LOG(p_mgr->p_log, OSM_LOG_INFO, "[LFT]: " + "ucast_mgr_process_tbl() - END\n"); cl_qlist_remove_all(&p_mgr->port_order_list); -- 1.5.1.4 And here's the patch that shows where I added the log messages in Hal's code: >From 36476d885ca0f295c1fa46b4e467c9c883a2173b Mon Sep 17 00:00:00 2001 From: Yevgeny Kliteynik Date: Mon, 27 Jul 2009 15:35:16 +0300 Subject: [PATCH] [LFT] printfs for LFT parallelization in Hal's code Signed-off-by: Yevgeny Kliteynik --- opensm/opensm/osm_state_mgr.c | 11 ++++++++++- opensm/opensm/osm_ucast_mgr.c | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletions(-) diff --git a/opensm/opensm/osm_state_mgr.c b/opensm/opensm/osm_state_mgr.c index 847941f..1a6d1ea 100644 --- a/opensm/opensm/osm_state_mgr.c +++ b/opensm/opensm/osm_state_mgr.c @@ -1272,11 +1272,20 @@ _repeat_discovery: */ if (!sm->ucast_mgr.cache_valid || - osm_ucast_cache_process(&sm->ucast_mgr)) + osm_ucast_cache_process(&sm->ucast_mgr)) { + OSM_LOG(sm->p_log, OSM_LOG_INFO, "[LFT]: " + "osm_ucast_mgr_process() - START " + "(%d max_smps_per_node)\n", + sm->ucast_mgr.p_subn->opt.max_smps_per_node); osm_ucast_mgr_process(&sm->ucast_mgr); + OSM_LOG(sm->p_log, OSM_LOG_INFO, "[LFT]: " + "osm_ucast_mgr_process() - END\n"); + } if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats)) return; + OSM_LOG(sm->p_log, OSM_LOG_INFO, "[LFT]: " + "Done wait_for_pending_transactions()\n"); /* cleanup switch lft buffers */ cl_qmap_apply_func(&sm->p_subn->sw_guid_tbl, cleanup_switch, sm->p_log); diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c index 1507bed..d05c05e 100644 --- a/opensm/opensm/osm_ucast_mgr.c +++ b/opensm/opensm/osm_ucast_mgr.c @@ -932,10 +932,18 @@ static int ucast_mgr_build_lfts(osm_ucast_mgr_t * p_mgr) cl_qmap_apply_func(&p_mgr->p_subn->port_guid_tbl, add_port_to_order_list, p_mgr); + OSM_LOG(p_mgr->p_log, OSM_LOG_INFO, "[LFT]: " + "ucast_mgr_process_tbl() - START\n"); cl_qmap_apply_func(&p_mgr->p_subn->sw_guid_tbl, ucast_mgr_process_tbl, p_mgr); + OSM_LOG(p_mgr->p_log, OSM_LOG_INFO, "[LFT]: " + "ucast_mgr_process_tbl() - END\n"); + OSM_LOG(p_mgr->p_log, OSM_LOG_INFO, "[LFT]: " + "osm_ucast_pipeline_tbl() - START\n"); osm_ucast_pipeline_tbl(p_mgr); + OSM_LOG(p_mgr->p_log, OSM_LOG_INFO, "[LFT]: " + "osm_ucast_pipeline_tbl() - END\n"); cl_qlist_remove_all(&p_mgr->port_order_list); -- 1.5.1.4 From weiny2 at llnl.gov Tue Jul 28 18:57:49 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Tue, 28 Jul 2009 18:57:49 -0700 Subject: [ofa-general] [PATCH] opensm: remove extra "0x" from debug message. Message-ID: <20090728185749.1c083e62.weiny2@llnl.gov> From: Ira Weiny Date: Tue, 28 Jul 2009 18:55:10 -0700 Subject: [PATCH] opensm: remove extra "0x" from debug message. "%p" will print a 0x for you. Signed-off-by: Ira Weiny --- opensm/opensm/osm_vl15intf.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/opensm/opensm/osm_vl15intf.c b/opensm/opensm/osm_vl15intf.c index 8f59f42..43095c9 100644 --- a/opensm/opensm/osm_vl15intf.c +++ b/opensm/opensm/osm_vl15intf.c @@ -314,7 +314,7 @@ void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw) CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY); - OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, "Posting p_madw = 0x%p\n", p_madw); + OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, "Posting p_madw = %p\n", p_madw); /* Determine in which fifo to place the pending madw. -- 1.5.4.5 From niftyompi at niftyegg.com Tue Jul 28 21:30:05 2009 From: niftyompi at niftyegg.com (Nifty Tom Mitchell) Date: Tue, 28 Jul 2009 21:30:05 -0700 Subject: Fwd: [ofa-general] Performance evaluation of Opensm In-Reply-To: <309a667c0907070358n53ad17lafcef8c3ef6e6ef2@mail.gmail.com> References: <617255.53094.qm@web94203.mail.in2.yahoo.com> <4A53156D.2040805@dev.mellanox.co.il> <309a667c0907070358n53ad17lafcef8c3ef6e6ef2@mail.gmail.com> Message-ID: <20090729043005.GA2992@compegg> On Tue, Jul 07, 2009 at 04:28:55PM +0530, Devesh Sharma wrote: > Thanks Yevgeny, for your valuable input. This will surly help for my work. > > On Tue, Jul 7, 2009 at 2:59 PM, Yevgeny > Kliteynik wrote: > > Hi Davesh, > > > > It's kind of hard to talk about "performance of OpenSM". > > Subnet Manager has different phases and modes of operation, > > each of them is completely separate issue: > > > > - Fabric discovery > > - Fabric ports/nodes configuration > > - Unicast routing calculation > > - Unicast routing configuration on fabric switches > > - Multicast routing calculation > > - Multicast routing configuration on fabric switches > > - SA queries processing > > - Memory consumption > > - Different routing algorithms consume different time and memory > > - QoS > > - etc, etc, etc > > > > Most of the above can be measured only on real cluster. ... > But how these can be measured is there any compile time flag available > in the Code? You can edit the code and add a log or time stamps -- but I am not sure that you should bother..... i.e. If there was a compile time flag what would you compare it to? *) N.B. once a fabric is setup up you can kill the subnet manager and the fabric will stay as it is and continue to operate. The implication of this is that the subnet manager is not in a critical performance path for normal operation. It is however in a "correctness + reliability" path which clearly sets the agenda for the authors. *) Much of the time you might measure on a cluster depends on the interaction of the subnet manager and other parts on the fabric. i.e. each node and switch in the cluster has a key component in the process (dt(SM)+dt(SMA)=something). This makes it hard to extract only the performance of the subnet manager. *) Sweeping the fabric to discover changes, new or absent devices can often be lazy. There is a configuration flag to tune this. On massive fabrics the time to rescan will grow with the size of the fabric. A lazy scan is normal. Trap notice SMP processing mitigates any lazy tuning. *) Processor, cache type and size, memory performance, I/O path to IB hardware TLB size and management all come to play. Small test fabric results will have some edges, lumps and bumps in any curves that make extrapolating to "interesting" fabric sizes difficult. Some things have been done. http://nowlab.cse.ohio-state.edu/publications/conf-papers/2005/vishnu-fastos05.pdf -- T o m M i t c h e l l Found me a new hat, now what? From dorfman.eli at gmail.com Wed Jul 29 02:24:31 2009 From: dorfman.eli at gmail.com (Eli Dorfman (Voltaire)) Date: Wed, 29 Jul 2009 12:24:31 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> Message-ID: <4A70154F.7080300@gmail.com> Hal Rosenstock wrote: > > > On Tue, Jul 28, 2009 at 6:14 AM, Moni Shoua > wrote: > > Jason Gunthorpe wrote: > > On Mon, Jul 27, 2009 at 08:11:42PM +0300, Yossi Etigin wrote: > >> If the LID of an ipoib neighbour changes without a SM event on > the local node, > >> IPoIB will keep caching the invalid path until the device is > flushed. The patch > >> below will remove the path for every incoming ARP packet where > the sender hardware > >> address does not match the cached lid. > > > > This assumes a LMC=0 fabric.. Otherwise there is no guarentee that the > > slid the sender uses is the same one you are using to send. > > > > Jason > > _______________________________________________ > > general mailing list > > general at lists.openfabrics.org > > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > > > This issue can be solved by a little bit different check > instead of > be16_to_cpu(path->pathrec.dlid) == lid) > do > is_in_same_lid_group(be16_to_cpu(path->pathrec.dlid), > lid, lmc) > > what do you think? > > > Is the LMC in the above from the local port ? Unfortunately, LMC is not > required to be uniform across the subnet so the remote port's LMC may > not be the same as that on the local port. opensm configures the same LMC to all endports (CA) in the fabric so in which case do you suspect that it will be different? Eli > > -- Hal > > > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > > > > ------------------------------------------------------------------------ > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From vlad at lists.openfabrics.org Wed Jul 29 02:54:21 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Wed, 29 Jul 2009 02:54:21 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090729-0200 daily build status Message-ID: <20090729095421.57D081020345@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Passed on i686 with linux-2.6.19 Passed on i686 with linux-2.6.18 Passed on i686 with linux-2.6.21.1 Passed on i686 with linux-2.6.22 Passed on x86_64 with linux-2.6.18 Passed on x86_64 with linux-2.6.21.1 Passed on x86_64 with linux-2.6.20 Passed on x86_64 with linux-2.6.19 Passed on x86_64 with linux-2.6.22 Passed on ia64 with linux-2.6.18 Passed on ia64 with linux-2.6.19 Passed on ia64 with linux-2.6.21.1 Passed on ia64 with linux-2.6.22 Passed on ppc64 with linux-2.6.18 Passed on ppc64 with linux-2.6.19 Failed: Build failed on i686 with linux-2.6.26 Build failed on i686 with linux-2.6.24 Build failed on i686 with linux-2.6.27 Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.27 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.27_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.27_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.27_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.27_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.27' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_x86_64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_x86_64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_x86_64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.23 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.23_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.23_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.23_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.23_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.23_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.23' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.26 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.26_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.26' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.24 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.24_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.24' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on ia64 with linux-2.6.25 Log: In file included from /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_ia64_check/net/rds/af_rds.c:41: /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: error: expected declaration specifiers or '...' before 'rds_stats' /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: data definition has no type or storage class /home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_ia64_check/net/rds/rds.h:641: warning: type defaults to 'int' in declaration of 'DECLARE_PER_CPU_SHARED_ALIGNED' make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_ia64_check/net/rds/af_rds.o] Error 1 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_ia64_check/net/rds] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090729-0200_linux-2.6.25_ia64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/ia64/linux-2.6.25' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From sashak at voltaire.com Wed Jul 29 02:56:23 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Wed, 29 Jul 2009 12:56:23 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A70154F.7080300@gmail.com> References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> Message-ID: <20090729095623.GM18150@me> On 12:24 Wed 29 Jul , Eli Dorfman (Voltaire) wrote: > > opensm configures the same LMC to all endports (CA) in the fabric so in which case do you suspect that > it will be different? OpenSM doesn't configure LMC > 0 for switch base port 0 (it doesn't support LMC > 0). Also there could be other SMs... :) Sasha From ogerlitz at voltaire.com Wed Jul 29 03:11:18 2009 From: ogerlitz at voltaire.com (Or Gerlitz) Date: Wed, 29 Jul 2009 13:11:18 +0300 Subject: [ofa-general] [PATCH 7/8 v3] mlx4: Add support for RDMAoE - address resolution In-Reply-To: <20090713181743.GA904@mtls03> References: <20090713181743.GA904@mtls03> Message-ID: <4A702046.4010801@voltaire.com> Eli Cohen wrote: > The following path handles address vectors creation for RDMAoE ports. mlx4 needs the MAC address of the remote node to include it in the WQE of a UD QP or in the QP context of connected QPs. Address resolution is done atomically in the case of a link local address or using service from core/addr.c to do that. mlx4 transport packets were changed too to accomodate for RDMAoE. Eli, Liran, Sean As I mentioned over some of the previous threads, doing IP-to-MAC address resolution behind the cover of an IB hardware driver doesn't seem to be the correct approach to me. The address resolution module could do both IP-to-GID and IP-to-MAC. Or, you can claim that in a similar manner to being IB's L2 tuple, since the rdma cm route resolution service does L2 resolution is Ethernet's L2 tuple and route resolution under (over) Ethernet would be resolving that. All in all, there should be a way to extend the rdma-cm data-structures and concepts such that IBoE would also be supported along with IB and iWARP. Doing ARP in the IB hardware driver create-address-handle and qp-modify flows, with hiding the MAC inside the driver, simply b/c the rdma-cm wasn't designed to IBoE from day one, and its non trivial to change the cma to support that, isn't the way to go, thoughts? Or. From hal.rosenstock at gmail.com Wed Jul 29 03:30:07 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 29 Jul 2009 06:30:07 -0400 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A70154F.7080300@gmail.com> References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> Message-ID: On Wed, Jul 29, 2009 at 5:24 AM, Eli Dorfman (Voltaire) < dorfman.eli at gmail.com> wrote: > Hal Rosenstock wrote: > > > > > > On Tue, Jul 28, 2009 at 6:14 AM, Moni Shoua > > wrote: > > > > Jason Gunthorpe wrote: > > > On Mon, Jul 27, 2009 at 08:11:42PM +0300, Yossi Etigin wrote: > > >> If the LID of an ipoib neighbour changes without a SM event on > > the local node, > > >> IPoIB will keep caching the invalid path until the device is > > flushed. The patch > > >> below will remove the path for every incoming ARP packet where > > the sender hardware > > >> address does not match the cached lid. > > > > > > This assumes a LMC=0 fabric.. Otherwise there is no guarentee that > the > > > slid the sender uses is the same one you are using to send. > > > > > > Jason > > > _______________________________________________ > > > general mailing list > > > general at lists.openfabrics.org general at lists.openfabrics.org> > > > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > > > > > To unsubscribe, please visit > > http://openib.org/mailman/listinfo/openib-general > > > > > This issue can be solved by a little bit different check > > instead of > > be16_to_cpu(path->pathrec.dlid) == lid) > > do > > is_in_same_lid_group(be16_to_cpu(path->pathrec.dlid), > > lid, lmc) > > > > what do you think? > > > > > > Is the LMC in the above from the local port ? Unfortunately, LMC is not > > required to be uniform across the subnet so the remote port's LMC may > > not be the same as that on the local port. > > opensm configures the same LMC to all endports (CA) in the fabric so in > which case do you suspect that > it will be different? I wasn't talking about the simplification that OpenSM uses but rather what IBA allows. There may be other SMs which do this or OpenSM could be extended (with additional configuration for this). -- Hal > > > Eli > > > > -- Hal > > > > > > > > _______________________________________________ > > general mailing list > > general at lists.openfabrics.org > > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > > > To unsubscribe, please visit > > http://openib.org/mailman/listinfo/openib-general > > > > > > > > ------------------------------------------------------------------------ > > > > _______________________________________________ > > general mailing list > > general at lists.openfabrics.org > > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > > -------------- next part -------------- An HTML attachment was scrubbed... URL: From monis at Voltaire.COM Wed Jul 29 05:16:36 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Wed, 29 Jul 2009 15:16:36 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> Message-ID: <4A703DA4.9080300@Voltaire.COM> > > Is the LMC in the above from the local port ? Unfortunately, LMC > is not > > required to be uniform across the subnet so the remote port's LMC may > > not be the same as that on the local port. > > opensm configures the same LMC to all endports (CA) in the fabric so > in which case do you suspect that > it will be different? > > > I wasn't talking about the simplification that OpenSM uses but rather > what IBA allows. There may be other SMs which do this or OpenSM could be > extended (with additional configuration for this). > > -- Hal So I guess a possible solutions for that is to query each suspected node before making a decision to flush it. When getting the node info response the true LMC can be put in the LID check function. From todd.rimmer at qlogic.com Wed Jul 29 06:15:01 2009 From: todd.rimmer at qlogic.com (Todd Rimmer) Date: Wed, 29 Jul 2009 08:15:01 -0500 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> Message-ID: <5AEC2602AE03EB46BFC16C6B9B200DA81653EF656C@MNEXMB2.qlogic.org> Hal Rosenstock wrote: >> Is the LMC in the above from the local port ? Unfortunately, LMC is not >> required to be uniform across the subnet so the remote port's LMC may >> not be the same as that on the local port. >opensm configures the same LMC to all endports (CA) in the fabric so in which case do you suspect that > it will be different? > >I wasn't talking about the simplification that OpenSM uses but rather what IBA allows. There may be other SMs which do > this or OpenSM could be extended (with additional configuration for this). Hal is correct, IBTA allows LMC to vary per port. There are existing configurations where the QLogic SM can configure a different LMC value for certain types of nodes in the fabric. All of which need to be able to participate in IPoIB. Todd Rimmer Chief Architect QLogic Network Systems Group Voice: 610-233-4852 Fax: 610-233-4777 Todd.Rimmer at QLogic.com www.QLogic.com From: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Hal Rosenstock Sent: Wednesday, July 29, 2009 6:30 AM To: Eli Dorfman (Voltaire) Cc: Moni Shoua; Roland Dreier; Yossi Etigin; general list Subject: Re: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes On Wed, Jul 29, 2009 at 5:24 AM, Eli Dorfman (Voltaire) > wrote: > > > On Tue, Jul 28, 2009 at 6:14 AM, Moni Shoua > >> wrote: > > Jason Gunthorpe wrote: > > On Mon, Jul 27, 2009 at 08:11:42PM +0300, Yossi Etigin wrote: > >> If the LID of an ipoib neighbour changes without a SM event on > the local node, > >> IPoIB will keep caching the invalid path until the device is > flushed. The patch > >> below will remove the path for every incoming ARP packet where > the sender hardware > >> address does not match the cached lid. > > > > This assumes a LMC=0 fabric.. Otherwise there is no guarentee that the > > slid the sender uses is the same one you are using to send. > > > > Jason > > _______________________________________________ > > general mailing list > > general at lists.openfabrics.org > > > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > > > This issue can be solved by a little bit different check > instead of > be16_to_cpu(path->pathrec.dlid) == lid) > do > is_in_same_lid_group(be16_to_cpu(path->pathrec.dlid), > lid, lmc) > > what do you think? > > Eli > > -- Hal > > > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > > > > ------------------------------------------------------------------------ > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general -------------- next part -------------- An HTML attachment was scrubbed... URL: From hal.rosenstock at gmail.com Wed Jul 29 07:07:06 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 29 Jul 2009 10:07:06 -0400 Subject: [ofa-general] IPoIB post_send failed Message-ID: Hi, I'm seeing the following messages from IPoIB: ib0: post_send failed ib0: post_send failed ib0: post_send failed ib0: post_send failed ib0: post_send failed ib0: post_send failed NETDEV WATCHDOG: ib0: transmit timed out ib0: transmit timeout: latency 1374 msecs ib0: queue stopped 1, tx_head 140245691, tx_tail 140245565 What are the possible (and most likely) causes of post_send failures ? I went through the code for all the errors (some at the driver level) but none popped out at me. Once the transmit queue is stopped, does the interface need to be taken down and then back up to restart this ? Thanks. -- Hal -------------- next part -------------- An HTML attachment was scrubbed... URL: From hal.rosenstock at gmail.com Wed Jul 29 07:15:47 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 29 Jul 2009 10:15:47 -0400 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A703DA4.9080300@Voltaire.COM> References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> Message-ID: On Wed, Jul 29, 2009 at 8:16 AM, Moni Shoua wrote: > > > > Is the LMC in the above from the local port ? Unfortunately, LMC > > is not > > > required to be uniform across the subnet so the remote port's LMC > may > > > not be the same as that on the local port. > > > > opensm configures the same LMC to all endports (CA) in the fabric so > > in which case do you suspect that > > it will be different? > > > > > > I wasn't talking about the simplification that OpenSM uses but rather > > what IBA allows. There may be other SMs which do this or OpenSM could be > > extended (with additional configuration for this). > > > > -- Hal > > So I guess a possible solutions for that is to query each suspected node > before making a decision to flush it. > When getting the node info response the true LMC can be put in the LID > check function. > I'm not following how you are saying to determine the true LMC (of the remote port). -- Hal -------------- next part -------------- An HTML attachment was scrubbed... URL: From monis at Voltaire.COM Wed Jul 29 07:22:50 2009 From: monis at Voltaire.COM (Moni Shoua) Date: Wed, 29 Jul 2009 17:22:50 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> Message-ID: <4A705B3A.7060404@Voltaire.COM> Hal Rosenstock wrote: > > > On Wed, Jul 29, 2009 at 8:16 AM, Moni Shoua > wrote: > > > > > Is the LMC in the above from the local port ? Unfortunately, LMC > > is not > > > required to be uniform across the subnet so the remote > port's LMC may > > > not be the same as that on the local port. > > > > opensm configures the same LMC to all endports (CA) in the > fabric so > > in which case do you suspect that > > it will be different? > > > > > > I wasn't talking about the simplification that OpenSM uses but rather > > what IBA allows. There may be other SMs which do this or OpenSM > could be > > extended (with additional configuration for this). > > > > -- Hal > > So I guess a possible solutions for that is to query each suspected > node before making a decision to flush it. > When getting the node info response the true LMC can be put in the > LID check function. > > > I'm not following how you are saying to determine the true LMC (of the > remote port). > Send PortInfo query for that port From michael.heinz at qlogic.com Wed Jul 29 08:56:49 2009 From: michael.heinz at qlogic.com (Mike Heinz) Date: Wed, 29 Jul 2009 10:56:49 -0500 Subject: [ofa-general] Missing patch for SLES11 IPOIB? Message-ID: <4C2744E8AD2982428C5BFE523DF8CDCB45E7E9380C@MNEXMB1.qlogic.org> We're seeing a problem with IPOIB and non-default pkeys - basically it keeps trying to join the wrong multicast group for the pkey. Reviewing the code, I noticed that most kernel backport directories include the patch "ipoib_mcast_set_pkey_to_2_6_24.patch" which fixes this problem - but it is not included in the backports directory for SLES11. Is there are reason for this, or does it need to be moved from the backports tree to the regular patch tree? From yosefe at voltaire.com Wed Jul 29 09:12:38 2009 From: yosefe at voltaire.com (Yossi Etigin) Date: Wed, 29 Jul 2009 19:12:38 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A6ED576.7050807@Voltaire.COM> References: <4A6DDFCE.9060009@voltaire.com> <4A6ED576.7050807@Voltaire.COM> Message-ID: <4A7074F6.9020008@voltaire.com> On 28/07/09 13:39, Moni Shoua wrote: >> + ipoib_dbg(priv, "Path %pI6: LID changed from 0x%04x to 0x%04x\n", >> + path->pathrec.dgid.raw, be16_to_cpu(path->pathrec.dlid), lid); >> + >> + list_del(&path->list); >> + rb_erase(&path->rb_node, &priv->path_tree); >> + >> + spin_unlock_irqrestore(&priv->lock, flags); >> + netif_tx_unlock_bh(dev); >> + >> + path_free(dev, path); >> +} >> + > Instead of deleting the path you could restart the query with > if (!path->query) > path_rec_start(dev, path); > > It's should be the same as in unicast_arp_send() when path->valid == 0 Yes, you are right. From hal.rosenstock at gmail.com Wed Jul 29 09:35:09 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 29 Jul 2009 12:35:09 -0400 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A705B3A.7060404@Voltaire.COM> References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> <4A705B3A.7060404@Voltaire.COM> Message-ID: On Wed, Jul 29, 2009 at 10:22 AM, Moni Shoua wrote: > Hal Rosenstock wrote: > > > > > > On Wed, Jul 29, 2009 at 8:16 AM, Moni Shoua > > wrote: > > > > > > > > Is the LMC in the above from the local port ? Unfortunately, > LMC > > > is not > > > > required to be uniform across the subnet so the remote > > port's LMC may > > > > not be the same as that on the local port. > > > > > > opensm configures the same LMC to all endports (CA) in the > > fabric so > > > in which case do you suspect that > > > it will be different? > > > > > > > > > I wasn't talking about the simplification that OpenSM uses but > rather > > > what IBA allows. There may be other SMs which do this or OpenSM > > could be > > > extended (with additional configuration for this). > > > > > > -- Hal > > > > So I guess a possible solutions for that is to query each suspected > > node before making a decision to flush it. > > When getting the node info response the true LMC can be put in the > > LID check function. > > > > > > I'm not following how you are saying to determine the true LMC (of the > > remote port). > > > Send PortInfo query for that port from the IPoIB client ? I think this ends up needing to contact the SA to get this info rather than the port directly. Isn't that what you were trying to avoid originally ? If that's the case, one way would've been if ARPs carried the LMC as well as the LID. -------------- next part -------------- An HTML attachment was scrubbed... URL: From arlin.r.davis at intel.com Wed Jul 29 10:03:49 2009 From: arlin.r.davis at intel.com (Arlin Davis) Date: Wed, 29 Jul 2009 10:03:49 -0700 Subject: [ofa-general] [PATCH] uDAPL v2: move implementation of direct CQ objects down to provider level Message-ID: <875099D2AC8641EDA45DB177834095C7@amr.corp.intel.com> DAPL introduced the concept of directly waiting on the CQ for events by adding a compile time flag and special handling in the common code. Rather than using the compile time flag and modifying the common code, let the provider implement the best way to wait for CQ events. This simplifies the code and allows the common openib providers to optimize for Linux and Windows platforms independently, rather than assuming a specific implementation for signaling events. Signed-off-by: Sean Hefty --- dapl/common/dapl_adapter_util.h | 19 +-- dapl/common/dapl_evd_dto_callb.c | 8 +- dapl/common/dapl_evd_util.c | 36 +--- dapl/ibal/dapl_ibal_cq.c | 61 +------ dapl/ibal/dapl_ibal_util.c | 76 +------- dapl/ibal/dapl_ibal_util.h | 1 - dapl/include/dapl.h | 10 +- dapl/openib_common/cq.c | 353 +++++++++++++++++++--------------- dapl/openib_common/dapl_ib_common.h | 3 - dapl/openib_common/qp.c | 11 +- dapl/udapl/dapl_evd_set_unwaitable.c | 7 +- dapl/udapl/dapl_evd_wait.c | 19 +-- 12 files changed, 238 insertions(+), 366 deletions(-) diff --git a/dapl/common/dapl_adapter_util.h b/dapl/common/dapl_adapter_util.h index 1a8b7cc..97ab42e 100755 --- a/dapl/common/dapl_adapter_util.h +++ b/dapl/common/dapl_adapter_util.h @@ -249,25 +249,14 @@ dapls_query_provider_specific_attr( IN DAPL_IA *ia_ptr, IN DAT_PROVIDER_ATTR *attr_ptr ); -#ifdef CQ_WAIT_OBJECT DAT_RETURN -dapls_ib_wait_object_create ( - IN DAPL_EVD *evd_ptr, - IN ib_wait_obj_handle_t *p_cq_wait_obj_handle); - -DAT_RETURN -dapls_ib_wait_object_destroy ( - IN ib_wait_obj_handle_t cq_wait_obj_handle); +dapls_evd_dto_wakeup ( + IN DAPL_EVD *evd_ptr); DAT_RETURN -dapls_ib_wait_object_wakeup ( - IN ib_wait_obj_handle_t cq_wait_obj_handle); - -DAT_RETURN -dapls_ib_wait_object_wait ( - IN ib_wait_obj_handle_t cq_wait_obj_handle, +dapls_evd_dto_wait ( + IN DAPL_EVD *evd_ptr, IN uint32_t timeout); -#endif #ifdef DAT_EXTENSIONS void diff --git a/dapl/common/dapl_evd_dto_callb.c b/dapl/common/dapl_evd_dto_callb.c index 2f0d106..2e8d70e 100755 --- a/dapl/common/dapl_evd_dto_callb.c +++ b/dapl/common/dapl_evd_dto_callb.c @@ -118,13 +118,7 @@ dapl_evd_dto_callback(IN ib_hca_handle_t hca_handle, * We don't need to worry about taking the lock for the * wakeup because wakeups are sticky. */ -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) - dapls_ib_wait_object_wakeup(evd_ptr-> - cq_wait_obj_handle); - else -#endif - dapl_os_wait_object_wakeup(&evd_ptr->wait_object); + dapls_evd_dto_wakeup(evd_ptr); } else if (state == DAPL_EVD_STATE_OPEN) { DAPL_CNO *cno = evd_ptr->cno_ptr; diff --git a/dapl/common/dapl_evd_util.c b/dapl/common/dapl_evd_util.c index 2cfd693..88c3f8f 100644 --- a/dapl/common/dapl_evd_util.c +++ b/dapl/common/dapl_evd_util.c @@ -161,7 +161,7 @@ dapls_evd_internal_create(DAPL_IA * ia_ptr, * be conservative and set producer side locking. Otherwise, no. */ evd_ptr->evd_producer_locking_needed = - ((evd_flags & ~(DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) != 0); + !(evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)); /* Before we setup any callbacks, transition state to OPEN. */ evd_ptr->evd_state = DAPL_EVD_STATE_OPEN; @@ -301,20 +301,6 @@ DAPL_EVD *dapls_evd_alloc(IN DAPL_IA * ia_ptr, evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD; /* FIXME: should be DAPL_EVD_STATE_INIT */ dapl_os_wait_object_init(&evd_ptr->wait_object); -#ifdef CQ_WAIT_OBJECT - /* Create CQ wait object; no CNO and data stream type */ - if ((cno_ptr == NULL) && - ((evd_flags & ~(DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) == 0)) { - dapls_ib_wait_object_create(evd_ptr, - &evd_ptr->cq_wait_obj_handle); - if (evd_ptr->cq_wait_obj_handle == NULL) { - dapl_os_free(evd_ptr, sizeof(DAPL_EVD)); - evd_ptr = NULL; - goto bail; - } - } -#endif - evd_ptr->cno_active_count = 0; if (cno_ptr != NULL) { /* Take a reference count on the CNO */ @@ -517,12 +503,6 @@ DAT_RETURN dapls_evd_dealloc(IN DAPL_EVD * evd_ptr) dapl_os_wait_object_destroy(&evd_ptr->wait_object); -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) { - dapls_ib_wait_object_destroy(evd_ptr->cq_wait_obj_handle); - } -#endif - #ifdef DAPL_COUNTERS dapl_os_free(evd_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_EVD_ALL_COUNTERS); @@ -698,14 +678,12 @@ dapli_evd_post_event(IN DAPL_EVD * evd_ptr, IN const DAT_EVENT * event_ptr) >= evd_ptr->threshold)) { dapl_os_unlock(&evd_ptr->header.lock); -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) - dapls_ib_wait_object_wakeup(evd_ptr-> - cq_wait_obj_handle); - else -#endif - dapl_os_wait_object_wakeup(&evd_ptr-> - wait_object); + if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) { + dapls_evd_dto_wakeup(evd_ptr); + } else { + dapl_os_wait_object_wakeup(&evd_ptr->wait_object); + } + } else { dapl_os_unlock(&evd_ptr->header.lock); } diff --git a/dapl/ibal/dapl_ibal_cq.c b/dapl/ibal/dapl_ibal_cq.c index 28de045..5bc51af 100644 --- a/dapl/ibal/dapl_ibal_cq.c +++ b/dapl/ibal/dapl_ibal_cq.c @@ -159,18 +159,8 @@ dapls_ib_cq_late_alloc ( ibal_ca = (dapl_ibal_ca_t*)evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle; -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) - { - cq_create.h_wait_obj = evd_ptr->cq_wait_obj_handle; - cq_create.pfn_comp_cb = NULL; - } - else -#endif - { - cq_create.h_wait_obj = NULL; - cq_create.pfn_comp_cb = dapli_ib_cq_completion_cb; - } + cq_create.h_wait_obj = NULL; + cq_create.pfn_comp_cb = dapli_ib_cq_completion_cb; ib_status = ib_create_cq ( (ib_ca_handle_t)ibal_ca->h_ca, @@ -227,53 +217,6 @@ bail: * */ -#if 0 - -/* windows delays CQ creation as a PD (Protection Domain) is required - * and we do not have one at this juncture. The follow code is for future - * reference only. - */ - -DAT_RETURN -dapls_ib_cq_alloc ( - IN DAPL_IA *ia_ptr, - IN DAPL_EVD *evd_ptr, - IN DAT_COUNT *cqlen ) -{ - dapl_dbg_log ( DAPL_DBG_TYPE_UTIL, - "dapls_ib_cq_alloc: evd %lx cqlen=%d \n", evd_ptr, *cqlen ); - - struct ibv_comp_channel *channel = ia_ptr->hca_ptr->ib_trans.ib_cq; - -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) - channel = evd_ptr->cq_wait_obj_handle; -#endif - - /* Call IB verbs to create CQ */ - evd_ptr->ib_cq_handle = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, - *cqlen, - evd_ptr, - channel, - 0); - - if (evd_ptr->ib_cq_handle == IB_INVALID_HANDLE) - return DAT_INSUFFICIENT_RESOURCES; - - /* arm cq for events */ - dapls_set_cq_notify(ia_ptr, evd_ptr); - - /* update with returned cq entry size */ - *cqlen = evd_ptr->ib_cq_handle->cqe; - - dapl_dbg_log ( DAPL_DBG_TYPE_UTIL, - "dapls_ib_cq_alloc: new_cq %lx cqlen=%d \n", - evd_ptr->ib_cq_handle, *cqlen ); - - return DAT_SUCCESS; -} -#endif - /* * dapl_ib_cq_free diff --git a/dapl/ibal/dapl_ibal_util.c b/dapl/ibal/dapl_ibal_util.c index ad4acf0..513d7c9 100644 --- a/dapl/ibal/dapl_ibal_util.c +++ b/dapl/ibal/dapl_ibal_util.c @@ -1850,83 +1850,21 @@ dapls_ib_completion_notify ( IN ib_hca_handle_t hca_handle, return dapl_ib_status_convert (ib_status); } - - -DAT_RETURN -dapls_ib_wait_object_create ( - IN cl_waitobj_handle_t* p_cq_wait_obj_handle) -{ - cl_status_t cl_status; - - cl_status = cl_waitobj_create (FALSE /* auto_reset */, p_cq_wait_obj_handle); - - if (cl_status == CL_SUCCESS) - return DAT_SUCCESS; - - return DAT_INTERNAL_ERROR; -} - - -DAT_RETURN -dapls_ib_wait_object_destroy ( - IN cl_waitobj_handle_t cq_wait_obj_handle) -{ - cl_status_t cl_status; - - cl_status = cl_waitobj_destroy (cq_wait_obj_handle); - - if (cl_status == CL_SUCCESS) - return DAT_SUCCESS; - - return DAT_INTERNAL_ERROR; -} - - DAT_RETURN -dapls_ib_wait_object_wakeup ( - IN cl_waitobj_handle_t cq_wait_obj_handle) +dapls_evd_dto_wakeup ( + IN DAPL_EVD *evd_ptr) { - cl_status_t cl_status; - - cl_status = cl_waitobj_signal (cq_wait_obj_handle); - - if (cl_status == CL_SUCCESS) - return DAT_SUCCESS; - - return DAT_INTERNAL_ERROR; + return dapl_os_wait_object_wakeup(&evd_ptr->wait_object); } - DAT_RETURN -dapls_ib_wait_object_wait ( - IN cl_waitobj_handle_t cq_wait_obj_handle, - IN uint32_t timeout) +dapls_evd_dto_wait ( + IN DAPL_EVD *evd_ptr, + IN uint32_t timeout) { - cl_status_t cl_status; - - cl_status = cl_waitobj_wait_on (cq_wait_obj_handle, timeout, TRUE ); - - switch (cl_status) - { - case CL_SUCCESS: - return DAT_SUCCESS; - - case CL_TIMEOUT: - dapl_dbg_log (DAPL_DBG_TYPE_ERR, - "--> wait_object_wait: cl_timeout: %d\n", timeout); - return DAT_TIMEOUT_EXPIRED; - - case CL_NOT_DONE: - return DAT_SUCCESS; - - default: - dapl_dbg_log (DAPL_DBG_TYPE_ERR, - "--> wait_object_wait: cl_error: %d\n", cl_status); - return DAT_INTERNAL_ERROR; - } + return dapl_os_wait_object_wait(&evd_ptr->wait_object, timeout); } - /* * dapls_ib_get_async_event * diff --git a/dapl/ibal/dapl_ibal_util.h b/dapl/ibal/dapl_ibal_util.h index 52dd879..a73d6c1 100644 --- a/dapl/ibal/dapl_ibal_util.h +++ b/dapl/ibal/dapl_ibal_util.h @@ -51,7 +51,6 @@ typedef ib_wr_type_t ib_send_op_type_t; typedef ib_wc_t ib_work_completion_t; typedef uint32_t ib_hca_port_t; typedef uint32_t ib_uint32_t; -typedef uint32_t ib_comp_handle_t; typedef ib_local_ds_t ib_data_segment_t; typedef unsigned __int3264 cl_dev_handle_t; diff --git a/dapl/include/dapl.h b/dapl/include/dapl.h index b13c963..a36b110 100755 --- a/dapl/include/dapl.h +++ b/dapl/include/dapl.h @@ -355,9 +355,6 @@ struct dapl_evd /* Every EVD has a CQ unless it is a SOFTWARE_EVENT only EVD */ ib_cq_handle_t ib_cq_handle; - /* Mellanox Specific completion handle for registration/de-registration */ - ib_comp_handle_t ib_comp_handle; - /* An Event Dispatcher cannot be freed while * it is referenced elsewhere. */ @@ -382,12 +379,7 @@ struct dapl_evd DAT_COUNT cno_active_count; DAPL_CNO *cno_ptr; - DAPL_OS_WAIT_OBJECT wait_object; - -#ifdef CQ_WAIT_OBJECT - /* Some providers support a direct CQ wait object */ - ib_wait_obj_handle_t cq_wait_obj_handle; -#endif + DAPL_OS_WAIT_OBJECT wait_object; DAT_COUNT threshold; DAPL_EVD_COMPLETION completion_type; diff --git a/dapl/openib_common/cq.c b/dapl/openib_common/cq.c index 74a5940..096167c 100644 --- a/dapl/openib_common/cq.c +++ b/dapl/openib_common/cq.c @@ -171,22 +171,36 @@ DAT_RETURN dapls_ib_get_async_event(IN ib_error_record_t * err_record, * DAT_INSUFFICIENT_RESOURCES * */ +#if defined(_WIN32) + DAT_RETURN dapls_ib_cq_alloc(IN DAPL_IA * ia_ptr, IN DAPL_EVD * evd_ptr, IN DAT_COUNT * cqlen) { - struct ibv_comp_channel *channel = evd_ptr->cq_wait_obj_handle; + OVERLAPPED *overlap; + DAT_RETURN ret; dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "dapls_ib_cq_alloc: evd %p cqlen=%d \n", evd_ptr, *cqlen); - /* Call IB verbs to create CQ */ evd_ptr->ib_cq_handle = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, - *cqlen, evd_ptr, channel, 0); + *cqlen, evd_ptr, NULL, 0); if (evd_ptr->ib_cq_handle == IB_INVALID_HANDLE) return DAT_INSUFFICIENT_RESOURCES; + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_create: (%p)\n", evd_ptr); + + overlap = &evd_ptr->ib_cq_handle->comp_entry.Overlap; + overlap->hEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + if (!overlap->hEvent) { + ret = DAT_INSUFFICIENT_RESOURCES; + goto err; + } + + overlap->hEvent = (HANDLE) ((ULONG_PTR) overlap->hEvent | 1); + /* arm cq for events */ dapls_set_cq_notify(ia_ptr, evd_ptr); @@ -198,17 +212,20 @@ dapls_ib_cq_alloc(IN DAPL_IA * ia_ptr, evd_ptr->ib_cq_handle, *cqlen); return DAT_SUCCESS; + +err: + ibv_destroy_cq(evd_ptr->ib_cq_handle); + return ret; } /* - * dapl_ib_cq_resize + * dapls_ib_cq_free * - * Alloc a CQ + * destroy a CQ * * Input: * ia_handle IA handle * evd_ptr pointer to EVD struct - * cqlen minimum QLen * * Output: * none @@ -218,50 +235,186 @@ dapls_ib_cq_alloc(IN DAPL_IA * ia_ptr, * DAT_INVALID_PARAMETER * */ +DAT_RETURN dapls_ib_cq_free(IN DAPL_IA * ia_ptr, IN DAPL_EVD * evd_ptr) +{ + DAT_EVENT event; + ib_work_completion_t wc; + HANDLE hevent; + + if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) { + /* pull off CQ and EVD entries and toss */ + while (ibv_poll_cq(evd_ptr->ib_cq_handle, 1, &wc) == 1) ; + while (dapl_evd_dequeue(evd_ptr, &event) == DAT_SUCCESS) ; + + hevent = evd_ptr->ib_cq_handle->comp_entry.Overlap.hEvent; + if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) + return (dapl_convert_errno(errno, "ibv_destroy_cq")); + + CloseHandle(hevent); + evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; + } + return DAT_SUCCESS; +} + DAT_RETURN -dapls_ib_cq_resize(IN DAPL_IA * ia_ptr, - IN DAPL_EVD * evd_ptr, IN DAT_COUNT * cqlen) +dapls_evd_dto_wakeup(IN DAPL_EVD * evd_ptr) { - ib_cq_handle_t new_cq; - struct ibv_comp_channel *channel = evd_ptr->cq_wait_obj_handle; + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wakeup: evd=%p\n", evd_ptr); - /* IB verbs DOES support resize. REDO THIS. - * Try to re-create CQ - * with new size. Can only be done if QP is not attached. - * destroy EBUSY == QP still attached. - */ + if (!SetEvent(evd_ptr->ib_cq_handle->comp_entry.Overlap.hEvent)) + return DAT_INTERNAL_ERROR; - /* Call IB verbs to create CQ */ - new_cq = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, *cqlen, - evd_ptr, channel, 0); + return DAT_SUCCESS; +} + +DAT_RETURN +dapls_evd_dto_wait(IN DAPL_EVD * evd_ptr, IN uint32_t timeout) +{ + int status; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wait: EVD %p time %d\n", + evd_ptr, timeout); + + status = WaitForSingleObject(evd_ptr->ib_cq_handle-> + comp_entry.Overlap.hEvent, + timeout / 1000); + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wait: EVD %p status 0x%x\n", + evd_ptr, status); + if (status) + return DAT_TIMEOUT_EXPIRED; - if (new_cq == IB_INVALID_HANDLE) + InterlockedExchange(&evd_ptr->ib_cq_handle->comp_entry.Busy, 0); + return DAT_SUCCESS; +} + +#else // WIN32 + +DAT_RETURN +dapls_ib_cq_alloc(IN DAPL_IA * ia_ptr, + IN DAPL_EVD * evd_ptr, IN DAT_COUNT * cqlen) +{ + struct ibv_comp_channel *channel; + DAT_RETURN ret; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + "dapls_ib_cq_alloc: evd %p cqlen=%d \n", evd_ptr, *cqlen); + + channel = ibv_create_comp_channel(ia_ptr->hca_ptr->ib_hca_handle); + if (!channel) return DAT_INSUFFICIENT_RESOURCES; - /* destroy the original and replace if successful */ - if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) { - ibv_destroy_cq(new_cq); - return (dapl_convert_errno(errno, "resize_cq")); - } + evd_ptr->ib_cq_handle = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, + *cqlen, evd_ptr, channel, 0); - /* update EVD with new cq handle and size */ - evd_ptr->ib_cq_handle = new_cq; - *cqlen = new_cq->cqe; + if (evd_ptr->ib_cq_handle == IB_INVALID_HANDLE) { + ret = DAT_INSUFFICIENT_RESOURCES; + goto err; + } /* arm cq for events */ dapls_set_cq_notify(ia_ptr, evd_ptr); + /* update with returned cq entry size */ + *cqlen = evd_ptr->ib_cq_handle->cqe; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + "dapls_ib_cq_alloc: new_cq %p cqlen=%d \n", + evd_ptr->ib_cq_handle, *cqlen); + + return DAT_SUCCESS; + +err: + ibv_destroy_comp_channel(channel); + return ret; +} + +DAT_RETURN dapls_ib_cq_free(IN DAPL_IA * ia_ptr, IN DAPL_EVD * evd_ptr) +{ + DAT_EVENT event; + ib_work_completion_t wc; + struct ibv_comp_channel *channel; + + if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) { + /* pull off CQ and EVD entries and toss */ + while (ibv_poll_cq(evd_ptr->ib_cq_handle, 1, &wc) == 1) ; + while (dapl_evd_dequeue(evd_ptr, &event) == DAT_SUCCESS) ; + + channel = evd_ptr->ib_cq_handle->channel; + if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) + return (dapl_convert_errno(errno, "ibv_destroy_cq")); + + ibv_destroy_comp_channel(channel); + evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; + } + return DAT_SUCCESS; +} + +DAT_RETURN +dapls_evd_dto_wakeup(IN DAPL_EVD * evd_ptr) +{ + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wakeup: evd=%p\n", evd_ptr); + + /* no wake up mechanism */ return DAT_SUCCESS; } +DAT_RETURN +dapls_evd_dto_wait(IN DAPL_EVD * evd_ptr, IN uint32_t timeout) +{ + struct ibv_comp_channel *channel = evd_ptr->ib_cq_handle->channel; + struct ibv_cq *ibv_cq = NULL; + void *context; + int status = 0; + int timeout_ms = -1; + struct pollfd cq_fd = { + .fd = channel->fd, + .events = POLLIN, + .revents = 0 + }; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wait: EVD %p time %d\n", + evd_ptr, timeout); + + /* uDAPL timeout values in usecs */ + if (timeout != DAT_TIMEOUT_INFINITE) + timeout_ms = timeout / 1000; + + status = poll(&cq_fd, 1, timeout_ms); + + /* returned event */ + if (status > 0) { + if (!ibv_get_cq_event(channel, &ibv_cq, &context)) { + ibv_ack_cq_events(ibv_cq, 1); + } + status = 0; + + /* timeout */ + } else if (status == 0) + status = ETIMEDOUT; + + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " cq_object_wait: RET evd %p ibv_cq %p %s\n", + evd_ptr, ibv_cq, strerror(errno)); + + return (dapl_convert_errno(status, "cq_wait_object_wait")); + +} +#endif + /* - * dapls_ib_cq_free + * dapl_ib_cq_resize * - * destroy a CQ + * Alloc a CQ * * Input: * ia_handle IA handle * evd_ptr pointer to EVD struct + * cqlen minimum QLen * * Output: * none @@ -271,20 +424,27 @@ dapls_ib_cq_resize(IN DAPL_IA * ia_ptr, * DAT_INVALID_PARAMETER * */ -DAT_RETURN dapls_ib_cq_free(IN DAPL_IA * ia_ptr, IN DAPL_EVD * evd_ptr) +DAT_RETURN +dapls_ib_cq_resize(IN DAPL_IA * ia_ptr, + IN DAPL_EVD * evd_ptr, IN DAT_COUNT * cqlen) { - DAT_EVENT event; - ib_work_completion_t wc; + ib_cq_handle_t old_cq, new_cq; + DAT_RETURN ret; - if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) { - /* pull off CQ and EVD entries and toss */ - while (ibv_poll_cq(evd_ptr->ib_cq_handle, 1, &wc) == 1) ; - while (dapl_evd_dequeue(evd_ptr, &event) == DAT_SUCCESS) ; - if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) - return (dapl_convert_errno(errno, "ibv_destroy_cq")); - evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; - } + old_cq = evd_ptr->ib_cq_handle; + ret = dapls_ib_cq_alloc(ia_ptr, evd_ptr, cqlen); + if (ret) + goto err; + + new_cq = evd_ptr->ib_cq_handle; + evd_ptr->ib_cq_handle = old_cq; + dapls_ib_cq_free(ia_ptr, evd_ptr); + evd_ptr->ib_cq_handle = new_cq; return DAT_SUCCESS; + +err: + evd_ptr->ib_cq_handle = old_cq; + return ret; } /* @@ -369,119 +529,6 @@ DAT_RETURN dapls_ib_completion_poll(IN DAPL_HCA * hca_ptr, return DAT_QUEUE_EMPTY; } -/* NEW common wait objects for providers with direct CQ wait objects */ -DAT_RETURN -dapls_ib_wait_object_create(IN DAPL_EVD * evd_ptr, - IN ib_wait_obj_handle_t * p_cq_wait_obj_handle) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_create: (%p,%p)\n", - evd_ptr, p_cq_wait_obj_handle); - - /* set cq_wait object to evd_ptr */ - *p_cq_wait_obj_handle = - ibv_create_comp_channel(evd_ptr->header.owner_ia->hca_ptr-> - ib_hca_handle); - - return DAT_SUCCESS; -} - -DAT_RETURN -dapls_ib_wait_object_destroy(IN ib_wait_obj_handle_t p_cq_wait_obj_handle) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_destroy: wait_obj=%p\n", p_cq_wait_obj_handle); - - ibv_destroy_comp_channel(p_cq_wait_obj_handle); - - return DAT_SUCCESS; -} - -DAT_RETURN -dapls_ib_wait_object_wakeup(IN ib_wait_obj_handle_t p_cq_wait_obj_handle) -{ - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wakeup: wait_obj=%p\n", p_cq_wait_obj_handle); - - /* no wake up mechanism */ - return DAT_SUCCESS; -} - -#if defined(_WIN32) || defined(_WIN64) -DAT_RETURN -dapls_ib_wait_object_wait(IN ib_wait_obj_handle_t p_cq_wait_obj_handle, - IN uint32_t timeout) -{ - struct dapl_evd *evd_ptr; - struct ibv_cq *ibv_cq = NULL; - int status = 0; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wait: CQ channel %p time %d\n", - p_cq_wait_obj_handle, timeout); - - /* uDAPL timeout values in usecs */ - p_cq_wait_obj_handle->comp_channel.Milliseconds = timeout / 1000; - - /* returned event */ - status = ibv_get_cq_event(p_cq_wait_obj_handle, &ibv_cq, - (void *)&evd_ptr); - if (status == 0) { - ibv_ack_cq_events(ibv_cq, 1); - } - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wait: RET evd %p ibv_cq %p %s\n", - evd_ptr, ibv_cq, strerror(errno)); - - return (dapl_convert_errno(status, "cq_wait_object_wait")); -} -#else //_WIN32 || _WIN64 -DAT_RETURN -dapls_ib_wait_object_wait(IN ib_wait_obj_handle_t p_cq_wait_obj_handle, - IN uint32_t timeout) -{ - struct dapl_evd *evd_ptr; - struct ibv_cq *ibv_cq = NULL; - int status = 0; - int timeout_ms = -1; - struct pollfd cq_fd = { - .fd = p_cq_wait_obj_handle->fd, - .events = POLLIN, - .revents = 0 - }; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wait: CQ channel %p time %d\n", - p_cq_wait_obj_handle, timeout); - - /* uDAPL timeout values in usecs */ - if (timeout != DAT_TIMEOUT_INFINITE) - timeout_ms = timeout / 1000; - - status = poll(&cq_fd, 1, timeout_ms); - - /* returned event */ - if (status > 0) { - if (!ibv_get_cq_event(p_cq_wait_obj_handle, - &ibv_cq, (void *)&evd_ptr)) { - ibv_ack_cq_events(ibv_cq, 1); - } - status = 0; - - /* timeout */ - } else if (status == 0) - status = ETIMEDOUT; - - dapl_dbg_log(DAPL_DBG_TYPE_UTIL, - " cq_object_wait: RET evd %p ibv_cq %p %s\n", - evd_ptr, ibv_cq, strerror(errno)); - - return (dapl_convert_errno(status, "cq_wait_object_wait")); - -} -#endif //_WIN32 || _WIN64 - /* * Local variables: * c-indent-level: 4 diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h index b61e50e..0b417b8 100644 --- a/dapl/openib_common/dapl_ib_common.h +++ b/dapl/openib_common/dapl_ib_common.h @@ -107,9 +107,6 @@ typedef int ib_bool_t; typedef union ibv_gid GID; typedef char *IB_HCA_NAME; typedef uint16_t ib_hca_port_t; -typedef uint32_t ib_comp_handle_t; - -typedef struct ibv_comp_channel *ib_wait_obj_handle_t; /* Definitions */ #define IB_INVALID_HANDLE NULL diff --git a/dapl/openib_common/qp.c b/dapl/openib_common/qp.c index 9fb7c96..9aa0594 100644 --- a/dapl/openib_common/qp.c +++ b/dapl/openib_common/qp.c @@ -77,15 +77,20 @@ dapls_ib_qp_alloc(IN DAPL_IA * ia_ptr, else if (!ia_ptr->hca_ptr->ib_trans.ib_cq_empty) rcv_cq = ia_ptr->hca_ptr->ib_trans.ib_cq_empty; else { - struct ibv_comp_channel *channel = - rcv_evd->cq_wait_obj_handle; + struct ibv_comp_channel *channel; + + channel = ibv_create_comp_channel(ia_ptr->hca_ptr->ib_hca_handle); + if (!channel) + return (dapl_convert_errno(ENOMEM, "create_cq")); /* Call IB verbs to create CQ */ rcv_cq = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, 0, NULL, channel, 0); - if (rcv_cq == IB_INVALID_HANDLE) + if (rcv_cq == IB_INVALID_HANDLE) { + ibv_destroy_comp_channel(channel); return (dapl_convert_errno(ENOMEM, "create_cq")); + } ia_ptr->hca_ptr->ib_trans.ib_cq_empty = rcv_cq; } diff --git a/dapl/udapl/dapl_evd_set_unwaitable.c b/dapl/udapl/dapl_evd_set_unwaitable.c index bf41662..718e433 100644 --- a/dapl/udapl/dapl_evd_set_unwaitable.c +++ b/dapl/udapl/dapl_evd_set_unwaitable.c @@ -80,12 +80,9 @@ DAT_RETURN DAT_API dapl_evd_set_unwaitable(IN DAT_EVD_HANDLE evd_handle) * thing. */ if (evd_ptr->evd_state == DAPL_EVD_STATE_WAITED) { -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) - dapls_ib_wait_object_wakeup(evd_ptr-> - cq_wait_obj_handle); + if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) + dapls_evd_dto_wakeup(evd_ptr); else -#endif dapl_os_wait_object_wakeup(&evd_ptr->wait_object); } bail: diff --git a/dapl/udapl/dapl_evd_wait.c b/dapl/udapl/dapl_evd_wait.c index 8d82d63..2e501ae 100644 --- a/dapl/udapl/dapl_evd_wait.c +++ b/dapl/udapl/dapl_evd_wait.c @@ -182,8 +182,7 @@ DAT_RETURN DAT_API dapl_evd_wait(IN DAT_EVD_HANDLE evd_handle, * a DTO_EVD or RMR_BIND_EVD */ if ((!notify_requested) && - ((evd_ptr->evd_flags & DAT_EVD_DTO_FLAG) || - (evd_ptr->evd_flags & DAT_EVD_RMR_BIND_FLAG))) { + (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG))) { dat_status = dapls_ib_completion_notify(evd_ptr->header. owner_ia->hca_ptr-> @@ -216,17 +215,11 @@ DAT_RETURN DAT_API dapl_evd_wait(IN DAT_EVD_HANDLE evd_handle, DAPL_CNTR(evd_ptr, DCNT_EVD_WAIT_BLOCKED); dapl_os_unlock(&evd_ptr->header.lock); -#ifdef CQ_WAIT_OBJECT - if (evd_ptr->cq_wait_obj_handle) - dat_status = - dapls_ib_wait_object_wait(evd_ptr-> - cq_wait_obj_handle, - time_out); - else -#endif - dat_status = - dapl_os_wait_object_wait(&evd_ptr->wait_object, - time_out); + if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) { + dat_status = dapls_evd_dto_wait(evd_ptr, time_out); + } else { + dat_status = dapl_os_wait_object_wait(&evd_ptr->wait_object, time_out); + } dapl_os_lock(&evd_ptr->header.lock); -- 1.5.2.5 From arlin.r.davis at intel.com Wed Jul 29 10:05:31 2009 From: arlin.r.davis at intel.com (Arlin Davis) Date: Wed, 29 Jul 2009 10:05:31 -0700 Subject: [ofa-general] [PATCH] uDAPL v2: windows platform specific: include w2tcpip.h for sockaddr_in6 definitions. Message-ID: <8928F0AB9E9645AA977358926EBF12E4@amr.corp.intel.com> Signed-off-by: Arlin Davis --- dat/include/dat2/dat_platform_specific.h | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/dat/include/dat2/dat_platform_specific.h b/dat/include/dat2/dat_platform_specific.h index 79e8d3a..addea83 100644 --- a/dat/include/dat2/dat_platform_specific.h +++ b/dat/include/dat2/dat_platform_specific.h @@ -196,6 +196,8 @@ typedef struct sockaddr_in6 DAT_SOCK_ADDR6; /* Socket address header native #elif defined(_MSC_VER) || defined(_WIN32) || defined(_WIN64) /* NT. MSC compiler, Win32/64 platform */ +#include + typedef unsigned __int32 DAT_UINT32; /* Unsigned host order, 32 bits */ typedef unsigned __int64 DAT_UINT64; /* unsigned host order, 64 bits */ typedef unsigned long DAT_UVERYLONG; /* unsigned longest native to compiler */ -- 1.5.2.5 From pradeeps at linux.vnet.ibm.com Wed Jul 29 11:14:40 2009 From: pradeeps at linux.vnet.ibm.com (Pradeep Satyanarayana) Date: Wed, 29 Jul 2009 11:14:40 -0700 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: References: Message-ID: <4A709190.20006@linux.vnet.ibm.com> Hal Rosenstock wrote: > Hi, > > I'm seeing the following messages from IPoIB: > ib0: post_send failed > ib0: post_send failed > ib0: post_send failed > ib0: post_send failed > ib0: post_send failed > ib0: post_send failed > NETDEV WATCHDOG: ib0: transmit timed out > ib0: transmit timeout: latency 1374 msecs > ib0: queue stopped 1, tx_head 140245691, tx_tail 140245565 > > What are the possible (and most likely) causes of post_send failures ? I > went through the code for all the errors (some at the driver level) but > none popped out at me. > Is it possible that the receiver is overwhelmed and hence the tx_ring is full? Is this a UDP application? > Once the transmit queue is stopped, does the interface need to be taken > down and then back up to restart this ? One does not need to take down the interface. It should be able to recover on it's own. There is a timer that kicks in and checks if the tx_ring is still full or not- the transmits should start again. Thanks! Pradeep From hal.rosenstock at gmail.com Wed Jul 29 11:31:41 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 29 Jul 2009 14:31:41 -0400 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: <4A709190.20006@linux.vnet.ibm.com> References: <4A709190.20006@linux.vnet.ibm.com> Message-ID: Hi Pradeep, On Wed, Jul 29, 2009 at 2:14 PM, Pradeep Satyanarayana < pradeeps at linux.vnet.ibm.com> wrote: > Hal Rosenstock wrote: > > Hi, > > > > I'm seeing the following messages from IPoIB: > > ib0: post_send failed > > ib0: post_send failed > > ib0: post_send failed > > ib0: post_send failed > > ib0: post_send failed > > ib0: post_send failed > > NETDEV WATCHDOG: ib0: transmit timed out > > ib0: transmit timeout: latency 1374 msecs > > ib0: queue stopped 1, tx_head 140245691, tx_tail 140245565 > > > > What are the possible (and most likely) causes of post_send failures ? I > > went through the code for all the errors (some at the driver level) but > > none popped out at me. > > > > Is it possible that the receiver is overwhelmed and hence the tx_ring is > full? It's possible but from the message you can't tell whether the tx_ring is full. Does it make sense to increase the transmit ring size via send_queue_size mod param ? > > Is this a UDP application? There is at least some UDP and there are many concurrent clients. > > > > Once the transmit queue is stopped, does the interface need to be taken > > down and then back up to restart this ? > > One does not need to take down the interface. It should be able to recover > on it's > own. There is a timer that kicks in and checks if the tx_ring is still full > or not- > the transmits should start again. Thanks! Thanks for the help! -- Hal > > > Pradeep > > -------------- next part -------------- An HTML attachment was scrubbed... URL: From rdreier at cisco.com Wed Jul 29 11:37:58 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 11:37:58 -0700 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: (Hal Rosenstock's message of "Wed, 29 Jul 2009 10:07:06 -0400") References: Message-ID: > I'm seeing the following messages from IPoIB: > ib0: post_send failed > ib0: post_send failed Most likely the send queue has overflowed, which should "never happen". What version of ipoib are you running, and what low-level HCA driver? What kernel version? - R. From hal.rosenstock at gmail.com Wed Jul 29 11:42:43 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 29 Jul 2009 14:42:43 -0400 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: References: Message-ID: On Wed, Jul 29, 2009 at 2:37 PM, Roland Dreier wrote: > > > I'm seeing the following messages from IPoIB: > > ib0: post_send failed > > ib0: post_send failed > > Most likely the send queue has overflowed, which should "never happen". > What version of ipoib are you running, and what low-level HCA driver? > What kernel version? I know I'm going to hear it but it's not under my control :-) It's whatever is in OFED 1.4.1. kernel is some 2.6.18 variant using mlx4.v1.0 (April 4, 2008) using x86_64 arch. -- Hal - R. -------------- next part -------------- An HTML attachment was scrubbed... URL: From pradeeps at linux.vnet.ibm.com Wed Jul 29 11:48:19 2009 From: pradeeps at linux.vnet.ibm.com (Pradeep Satyanarayana) Date: Wed, 29 Jul 2009 11:48:19 -0700 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: References: <4A709190.20006@linux.vnet.ibm.com> Message-ID: <4A709973.2060803@linux.vnet.ibm.com> Hal Rosenstock wrote: > Hi Pradeep, > > On Wed, Jul 29, 2009 at 2:14 PM, Pradeep Satyanarayana > > wrote: > > Hal Rosenstock wrote: > > Hi, > > > > I'm seeing the following messages from IPoIB: > > ib0: post_send failed > > ib0: post_send failed > > ib0: post_send failed > > ib0: post_send failed > > ib0: post_send failed > > ib0: post_send failed > > NETDEV WATCHDOG: ib0: transmit timed out > > ib0: transmit timeout: latency 1374 msecs > > ib0: queue stopped 1, tx_head 140245691, tx_tail 140245565 > > > > What are the possible (and most likely) causes of post_send > failures ? I > > went through the code for all the errors (some at the driver > level) but > > none popped out at me. > > > > Is it possible that the receiver is overwhelmed and hence the > tx_ring is full? > > > It's possible but from the message you can't tell whether the tx_ring is > full. > > Does it make sense to increase the transmit ring size via > send_queue_size mod param ? Given that there are many concurrent clients and at least some UDP, I have a suspicion that the receiver is indeed overwhelmed. On the contrary, instead of increasing the send_queue_size on the client, which may make the situation worse, please consider reducing the tx_ring size on the clients and increase the rx_ring on the server. This will sort of throttle the flow. Are you concerned about the messages you see, or is that actually impacting the application? You may still see the messages with the above changes (may need some tuning), but hopefully you will see a reduced impact on the applications. I would be interested in learning what you discover. Thanks! Pradeep > > > > Is this a UDP application? > > > There is at least some UDP and there are many concurrent clients. > > > > > > Once the transmit queue is stopped, does the interface need to be > taken > > down and then back up to restart this ? > > One does not need to take down the interface. It should be able to > recover on it's > own. There is a timer that kicks in and checks if the tx_ring is > still full or not- > the transmits should start again. Thanks! > > > Thanks for the help! > > -- Hal > > > > > Pradeep > > From rdreier at cisco.com Wed Jul 29 11:53:28 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 11:53:28 -0700 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: (Hal Rosenstock's message of "Wed, 29 Jul 2009 14:42:43 -0400") References: Message-ID: > I know I'm going to hear it but it's not under my control :-) > It's whatever is in OFED 1.4.1. kernel is some 2.6.18 variant using > mlx4.v1.0 (April 4, 2008) using x86_64 arch. Yeah, unfortunately that's not really a supportable system, sorry.... From rdreier at cisco.com Wed Jul 29 12:01:54 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 12:01:54 -0700 Subject: [ofa-general] Re: [PATCH] V3 libmlx4 - Optimize memory allocation of buffers In-Reply-To: <20090727110233.437ac165@frecb007965> (sebastien dugue's message of "Mon, 27 Jul 2009 11:02:33 +0200") References: <20090727110233.437ac165@frecb007965> Message-ID: thanks, applied both mxl4 and mthca versions with a couple cleanups. let me know if I broke anything in what I ended up committing. From rdreier at cisco.com Wed Jul 29 12:15:49 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 12:15:49 -0700 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: (Hal Rosenstock's message of "Fri, 24 Jul 2009 11:47:39 -0400") References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> Message-ID: > Based on the spec limiting hop pointer to 255 and not 63, I think the > above should just be a check on hop count and not hop pointer: > if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) Yes, it seems that the current code then properly checks hop_ptr against hop_cnt in all cases. Do we all agree that the following patch is right? If so I'll queue it for 2.6.32: drivers/infiniband/core/smi.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c index 8723675..a10152d 100644 --- a/drivers/infiniband/core/smi.c +++ b/drivers/infiniband/core/smi.c @@ -52,6 +52,10 @@ enum smi_action smi_handle_dr_smp_send(struct ib_smp *smp, hop_cnt = smp->hop_cnt; /* See section 14.2.2.2, Vol 1 IB spec */ + /* C14-6 -- valid hop_cnt values are from 0 to 63 */ + if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) + return IB_SMI_DISCARD; + if (!ib_get_smp_direction(smp)) { /* C14-9:1 */ if (hop_cnt && hop_ptr == 0) { From hal.rosenstock at gmail.com Wed Jul 29 12:22:28 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 29 Jul 2009 15:22:28 -0400 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> Message-ID: On Wed, Jul 29, 2009 at 3:15 PM, Roland Dreier wrote: > > > Based on the spec limiting hop pointer to 255 and not 63, I think the > > above should just be a check on hop count and not hop pointer: > > if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) > > Yes, it seems that the current code then properly checks hop_ptr against > hop_cnt in all cases. Do we all agree that the following patch is > right? If so I'll queue it for 2.6.32: > > drivers/infiniband/core/smi.c | 4 ++++ > 1 files changed, 4 insertions(+), 0 deletions(-) > > diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c > index 8723675..a10152d 100644 > --- a/drivers/infiniband/core/smi.c > +++ b/drivers/infiniband/core/smi.c > @@ -52,6 +52,10 @@ enum smi_action smi_handle_dr_smp_send(struct ib_smp > *smp, > hop_cnt = smp->hop_cnt; > > /* See section 14.2.2.2, Vol 1 IB spec */ > + /* C14-6 -- valid hop_cnt values are from 0 to 63 */ > + if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) > + return IB_SMI_DISCARD; > + > if (!ib_get_smp_direction(smp)) { > /* C14-9:1 */ > if (hop_cnt && hop_ptr == 0) { > That looks right to me on the send side. Shouldn't there be the same check on the recv side (smi_handle_dr_smp_recv) which was the intent of Roel's original patch ? -- Hal -------------- next part -------------- An HTML attachment was scrubbed... URL: From roel.kluin at gmail.com Wed Jul 29 12:34:00 2009 From: roel.kluin at gmail.com (Roel Kluin) Date: Wed, 29 Jul 2009 21:34:00 +0200 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> Message-ID: <4A70A428.5010804@gmail.com> Ensure index stays within smp->return_path[] and ->initial_path[]. A hop_cnt or hop_ptr greater or equal to IB_SMP_MAX_PATH_HOPS is invalid. Signed-off-by: Roel Kluin --- Op 29-07-09 21:22, Hal Rosenstock schreef: > > On Wed, Jul 29, 2009 at 3:15 PM, Roland Dreier > wrote: > > > > Based on the spec limiting hop pointer to 255 and not 63, I think the > > above should just be a check on hop count and not hop pointer: > > if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) > > Yes, it seems that the current code then properly checks hop_ptr against > hop_cnt in all cases. Do we all agree that the following patch is > right? If so I'll queue it for 2.6.32: > That looks right to me on the send side. Shouldn't there be the same > check on the recv side (smi_handle_dr_smp_recv) which was the intent of > Roel's original patch ? > > -- Hal > I think we also need the `hop_ptr + 1 >= IB_SMP_MAX_PATH_HOPS' test, right? diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c index 8723675..c623da7 100644 --- a/drivers/infiniband/core/smi.c +++ b/drivers/infiniband/core/smi.c @@ -52,6 +52,10 @@ enum smi_action smi_handle_dr_smp_send(struct ib_smp *smp, hop_cnt = smp->hop_cnt; /* See section 14.2.2.2, Vol 1 IB spec */ + /* C14-6 -- valid hop_cnt values are from 0 to 63 */ + if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) + return IB_SMI_DISCARD; + if (!ib_get_smp_direction(smp)) { /* C14-9:1 */ if (hop_cnt && hop_ptr == 0) { @@ -132,6 +136,9 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, u8 node_type, hop_ptr = smp->hop_ptr; hop_cnt = smp->hop_cnt; + if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) + return IB_SMI_DISCARD; + /* See section 14.2.2.2, Vol 1 IB spec */ if (!ib_get_smp_direction(smp)) { /* C14-9:1 -- sender should have incremented hop_ptr */ @@ -140,7 +147,8 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, u8 node_type, /* C14-9:2 -- intermediate hop */ if (hop_ptr && hop_ptr < hop_cnt) { - if (node_type != RDMA_NODE_IB_SWITCH) + if (node_type != RDMA_NODE_IB_SWITCH || + hop_ptr + 1 >= IB_SMP_MAX_PATH_HOPS) return IB_SMI_DISCARD; smp->return_path[hop_ptr] = port_num; From hal.rosenstock at gmail.com Wed Jul 29 12:31:42 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 29 Jul 2009 15:31:42 -0400 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> Message-ID: On Wed, Jul 29, 2009 at 3:22 PM, Hal Rosenstock wrote: > > > On Wed, Jul 29, 2009 at 3:15 PM, Roland Dreier wrote: > >> >> > Based on the spec limiting hop pointer to 255 and not 63, I think the >> > above should just be a check on hop count and not hop pointer: >> > if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) >> >> Yes, it seems that the current code then properly checks hop_ptr against >> hop_cnt in all cases. Do we all agree that the following patch is >> right? If so I'll queue it for 2.6.32: >> >> drivers/infiniband/core/smi.c | 4 ++++ >> 1 files changed, 4 insertions(+), 0 deletions(-) >> >> diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c >> index 8723675..a10152d 100644 >> --- a/drivers/infiniband/core/smi.c >> +++ b/drivers/infiniband/core/smi.c >> @@ -52,6 +52,10 @@ enum smi_action smi_handle_dr_smp_send(struct ib_smp >> *smp, >> hop_cnt = smp->hop_cnt; >> >> /* See section 14.2.2.2, Vol 1 IB spec */ >> + /* C14-6 -- valid hop_cnt values are from 0 to 63 */ >> + if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) >> + return IB_SMI_DISCARD; >> + >> if (!ib_get_smp_direction(smp)) { >> /* C14-9:1 */ >> if (hop_cnt && hop_ptr == 0) { >> > > That looks right to me on the send side. Shouldn't there be the same check > on the recv side (smi_handle_dr_smp_recv) which was the intent of Roel's > original patch ? > There's also one thing on the send side I'm not sure about. It looks to me like c14-9:3 might break if hop_cnt is max'd out as hop_ptr is incremented but the array is not touched. -- Hal > > -- Hal > > -------------- next part -------------- An HTML attachment was scrubbed... URL: From rdreier at cisco.com Wed Jul 29 12:32:35 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 12:32:35 -0700 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: (Hal Rosenstock's message of "Wed, 29 Jul 2009 15:22:28 -0400") References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> Message-ID: > That looks right to me on the send side. Shouldn't there be the same check > on the recv side (smi_handle_dr_smp_recv) which was the intent of Roel's > original patch ? Duh, yeah... got confused by two sets of similar-looking code. But it seems we want it in both places, right? So something like: drivers/infiniband/core/smi.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c index 8723675..5855e44 100644 --- a/drivers/infiniband/core/smi.c +++ b/drivers/infiniband/core/smi.c @@ -52,6 +52,10 @@ enum smi_action smi_handle_dr_smp_send(struct ib_smp *smp, hop_cnt = smp->hop_cnt; /* See section 14.2.2.2, Vol 1 IB spec */ + /* C14-6 -- valid hop_cnt values are from 0 to 63 */ + if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) + return IB_SMI_DISCARD; + if (!ib_get_smp_direction(smp)) { /* C14-9:1 */ if (hop_cnt && hop_ptr == 0) { @@ -133,6 +137,10 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, u8 node_type, hop_cnt = smp->hop_cnt; /* See section 14.2.2.2, Vol 1 IB spec */ + /* C14-6 -- valid hop_cnt values are from 0 to 63 */ + if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) + return IB_SMI_DISCARD; + if (!ib_get_smp_direction(smp)) { /* C14-9:1 -- sender should have incremented hop_ptr */ if (hop_cnt && hop_ptr == 0) From hal.rosenstock at gmail.com Wed Jul 29 12:33:25 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 29 Jul 2009 15:33:25 -0400 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: <4A70A428.5010804@gmail.com> References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> <4A70A428.5010804@gmail.com> Message-ID: On Wed, Jul 29, 2009 at 3:34 PM, Roel Kluin wrote: > Ensure index stays within smp->return_path[] and ->initial_path[]. > A hop_cnt or hop_ptr greater or equal to IB_SMP_MAX_PATH_HOPS is invalid. > > Signed-off-by: Roel Kluin > --- > Op 29-07-09 21:22, Hal Rosenstock schreef: > > > > On Wed, Jul 29, 2009 at 3:15 PM, Roland Dreier > > wrote: > > > > > > > Based on the spec limiting hop pointer to 255 and not 63, I think > the > > > above should just be a check on hop count and not hop pointer: > > > if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) > > > > Yes, it seems that the current code then properly checks hop_ptr > against > > hop_cnt in all cases. Do we all agree that the following patch is > > right? If so I'll queue it for 2.6.32: > > > That looks right to me on the send side. Shouldn't there be the same > > check on the recv side (smi_handle_dr_smp_recv) which was the intent of > > Roel's original patch ? > > > > -- Hal > > > > I think we also need the `hop_ptr + 1 >= IB_SMP_MAX_PATH_HOPS' test, right? Where are you referring to add this ? Like in your original patch ? -- Hal > > > diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c > index 8723675..c623da7 100644 > --- a/drivers/infiniband/core/smi.c > +++ b/drivers/infiniband/core/smi.c > @@ -52,6 +52,10 @@ enum smi_action smi_handle_dr_smp_send(struct ib_smp > *smp, > hop_cnt = smp->hop_cnt; > > /* See section 14.2.2.2, Vol 1 IB spec */ > + /* C14-6 -- valid hop_cnt values are from 0 to 63 */ > + if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) > + return IB_SMI_DISCARD; > + > if (!ib_get_smp_direction(smp)) { > /* C14-9:1 */ > if (hop_cnt && hop_ptr == 0) { > @@ -132,6 +136,9 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp > *smp, u8 node_type, > hop_ptr = smp->hop_ptr; > hop_cnt = smp->hop_cnt; > > + if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) > + return IB_SMI_DISCARD; > + > /* See section 14.2.2.2, Vol 1 IB spec */ > if (!ib_get_smp_direction(smp)) { > /* C14-9:1 -- sender should have incremented hop_ptr */ > @@ -140,7 +147,8 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp > *smp, u8 node_type, > > /* C14-9:2 -- intermediate hop */ > if (hop_ptr && hop_ptr < hop_cnt) { > - if (node_type != RDMA_NODE_IB_SWITCH) > + if (node_type != RDMA_NODE_IB_SWITCH || > + hop_ptr + 1 >= > IB_SMP_MAX_PATH_HOPS) > return IB_SMI_DISCARD; > > smp->return_path[hop_ptr] = port_num; > -------------- next part -------------- An HTML attachment was scrubbed... URL: From sean.hefty at intel.com Wed Jul 29 12:36:50 2009 From: sean.hefty at intel.com (Sean Hefty) Date: Wed, 29 Jul 2009 12:36:50 -0700 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: <4A70A428.5010804@gmail.com> References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> <4A70A428.5010804@gmail.com> Message-ID: <545128380B754947ABDA75F7E215401B@amr.corp.intel.com> >@@ -132,6 +136,9 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, >u8 node_type, > hop_ptr = smp->hop_ptr; > hop_cnt = smp->hop_cnt; > >+ if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) >+ return IB_SMI_DISCARD; >+ > /* See section 14.2.2.2, Vol 1 IB spec */ > if (!ib_get_smp_direction(smp)) { > /* C14-9:1 -- sender should have incremented hop_ptr */ >@@ -140,7 +147,8 @@ enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, >u8 node_type, > > /* C14-9:2 -- intermediate hop */ > if (hop_ptr && hop_ptr < hop_cnt) { >- if (node_type != RDMA_NODE_IB_SWITCH) >+ if (node_type != RDMA_NODE_IB_SWITCH || >+ hop_ptr + 1 >= IB_SMP_MAX_PATH_HOPS) I believe at this point: hop_ptr < hop_cnt < IB_SMP_MAX_PATH_HOPS so, this test will always fail. - Sean From rdreier at cisco.com Wed Jul 29 12:38:29 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 12:38:29 -0700 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: <4A70A428.5010804@gmail.com> (Roel Kluin's message of "Wed, 29 Jul 2009 21:34:00 +0200") References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> <4A70A428.5010804@gmail.com> Message-ID: > + if (hop_cnt >= IB_SMP_MAX_PATH_HOPS) > + return IB_SMI_DISCARD; > /* C14-9:2 -- intermediate hop */ > if (hop_ptr && hop_ptr < hop_cnt) { > - if (node_type != RDMA_NODE_IB_SWITCH) > + if (node_type != RDMA_NODE_IB_SWITCH || > + hop_ptr + 1 >= IB_SMP_MAX_PATH_HOPS) hmm, is the second test necessary? That's the case where hop_ptr is less than hop_cnt but hop_ptr + 1 is more than IB_SMP_MAX_PATH_HOPS... let's see... the biggest value of hop_ptr that could get into that code is hop_cnt-1 (due to the test just above), and the first test we're adding ensures hop_cnt is at most IB_SMP_MAX_PATH_HOPS-1. So the biggest value of hop_ptr that passes the existing tests plus the first test is IB_SMP_MAX_PATH_HOPS-1-1 ie IB_SMP_MAX_PATH_HOPS-2 which means the second test you're adding is redundant, no? - R. From rdreier at cisco.com Wed Jul 29 12:40:32 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 12:40:32 -0700 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: (Hal Rosenstock's message of "Wed, 29 Jul 2009 15:31:42 -0400") References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> Message-ID: > There's also one thing on the send side I'm not sure about. It looks to me > like c14-9:3 might break if hop_cnt is max'd out as hop_ptr is incremented > but the array is not touched. Isn't that increment at the end of the DR part done to handle the pre-decrement that will be done as part of c14-13? I think it's OK. - R. From hal.rosenstock at gmail.com Wed Jul 29 12:45:11 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Wed, 29 Jul 2009 15:45:11 -0400 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> Message-ID: On Wed, Jul 29, 2009 at 3:40 PM, Roland Dreier wrote: > > > There's also one thing on the send side I'm not sure about. It looks to > me > > like c14-9:3 might break if hop_cnt is max'd out as hop_ptr is > incremented > > but the array is not touched. > > Isn't that increment at the end of the DR part done to handle the > pre-decrement that will be done as part of c14-13? It was the other direction: c14-9 case 3 increments hop ptr and returns so it looks like this could be hop_ptr 65 if it were 64 coming in to this case and I don't see that prevented. Hope that's clearer... -- Hal > I think it's OK. > > - R. > -------------- next part -------------- An HTML attachment was scrubbed... URL: From rdreier at cisco.com Wed Jul 29 12:53:40 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 12:53:40 -0700 Subject: [ofa-general] [PATCH] IB: Possible write outside array bounds In-Reply-To: (Hal Rosenstock's message of "Wed, 29 Jul 2009 15:45:11 -0400") References: <4A699B6D.6060309@gmail.com> <25e057c00907240529wf08691bidd528358e4552171@mail.gmail.com> <4A69D71C.4090400@gmail.com> Message-ID: > > Isn't that increment at the end of the DR part done to handle the > > pre-decrement that will be done as part of c14-13? > It was the other direction: c14-9 case 3 increments hop ptr and returns so > it looks like this could be hop_ptr 65 if it were 64 coming in to this case > and I don't see that prevented. Hope that's clearer... I think I understood. c14-9:3 is in the send path, when the SMI is at the end of the DR part. And it is used when hop_ptr is equal to hop_cnt; since hop_cnt can't be bigger than 63 with the new checks, hop_ptr can't end up bigger than 64. And I think as I said that the returning DR handling will pre-decrement the hop_ptr as described in c14-13, so that it will never be bigger than 63. - R. From or.gerlitz at gmail.com Wed Jul 29 14:18:33 2009 From: or.gerlitz at gmail.com (Or Gerlitz) Date: Thu, 30 Jul 2009 00:18:33 +0300 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: References: Message-ID: <15ddcffd0907291418j5c4420a0odf34af50fdc51e1d@mail.gmail.com> On Wed, Jul 29, 2009 at 9:53 PM, Roland Dreier wrote: > > > I know I'm going to hear it but it's not under my control :-) > > It's whatever is in OFED 1.4.1. kernel is some 2.6.18 variant using > > mlx4.v1.0 (April 4, 2008) using x86_64 arch. > > Yeah, unfortunately that's not really a supportable system, sorry.... > sounds like the fact that ofed isn't supportable from the view point of the Linux kernel IB stack maintainer is quite a big secret and unknown to many of the IB industry players, do people have ideas how to make this information public domain? Or. -------------- next part -------------- An HTML attachment was scrubbed... URL: From akepner at sgi.com Wed Jul 29 14:19:22 2009 From: akepner at sgi.com (Arthur Kepner) Date: Wed, 29 Jul 2009 16:19:22 -0500 Subject: [ofa-general] [PATCH] iblinkinfo: warn only on degragded link speed, width Message-ID: iblinkinfo.pl warns when the link speed (or width) is higher than one of the supported values. For example, we're seeing: 168 11[ ] ==( 4X 10.0 Gbps Active / LinkUp)==> 169 \ 11[ ] "Infiniscale-IV Mellanox Technologies" ( Could be 5.0 Gbps) It's complaining that 10Gbps was negotiated even though 5.0Gbps is available, which I doubt is the intended behavior. The following patch causes iblinkinfo to warn only when the link speed (or width) is lower than the greatest common value supported by both ports. Signed-off-by: Arthur Kepner --- diff -rup a/scripts/iblinkinfo.pl b/scripts/iblinkinfo.pl --- a/scripts/iblinkinfo.pl 2009-07-29 13:16:02.889073624 -0700 +++ b/scripts/iblinkinfo.pl 2009-07-29 13:17:24.902170284 -0700 @@ -168,9 +168,11 @@ sub main } my @lines = split("\n", $data); my $speed = ""; + my $nspeed = -1; # in Gbps my $speed_sup = ""; my $speed_enable = ""; my $width = ""; + my $nwidth = -1; my $width_sup = ""; my $width_enable = ""; my $state = ""; @@ -179,12 +181,18 @@ sub main my $phy_link_state = ""; foreach my $line (@lines) { - if ($line =~ /^LinkSpeedActive:\.+(.*)/) { $speed = $1; } + if ($line =~ /^LinkSpeedActive:\.+(\d+\.\d+)(.*)/) { + $nspeed = $1; + $speed = $1.$2; + } if ($line =~ /^LinkSpeedEnabled:\.+(.*)/) { $speed_enable = $1; } if ($line =~ /^LinkSpeedSupported:\.+(.*)/) { $speed_sup = $1; } - if ($line =~ /^LinkWidthActive:\.+(.*)/) { $width = $1; } + if ($line =~ /^LinkWidthActive:\.+(\d+)(.*)/) { + $nwidth = $1; + $width = $1.$2; + } if ($line =~ /^LinkWidthEnabled:\.+(.*)/) { $width_enable = $1; } @@ -243,19 +251,19 @@ sub main if ($rem_width_enable ne "" && $rem_width_sup ne "") { if ( $width_enable =~ /12X/ && $rem_width_enable =~ /12X/ - && $width !~ /12X/) + && $nwidth < 12) { $width_msg = "Could be 12X"; } else { if ( $width_enable =~ /8X/ && $rem_width_enable =~ /8X/ - && $width !~ /8X/) + && $nwidth < 8) { $width_msg = "Could be 8X"; } else { if ( $width_enable =~ /4X/ && $rem_width_enable =~ /4X/ - && $width !~ /4X/) + && $nwidth < 4) { $width_msg = "Could be 4X"; } @@ -265,13 +273,13 @@ sub main if ($rem_speed_enable ne "" && $rem_speed_sup ne "") { if ( $speed_enable =~ /10\.0/ && $rem_speed_enable =~ /10\.0/ - && $speed !~ /10\.0/) + && $nspeed < 10.0) { $speed_msg = "Could be 10.0 Gbps"; } else { if ( $speed_enable =~ /5\.0/ && $rem_speed_enable =~ /5\.0/ - && $speed !~ /5\.0/) + && $nspeed < 5.0) { $speed_msg = "Could be 5.0 Gbps"; } From rdreier at cisco.com Wed Jul 29 14:29:33 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 14:29:33 -0700 Subject: [ofa-general] [PATCH ibverbs] Do not use enum object types for bitfields In-Reply-To: <20090723160229.GA22400@obsidianresearch.com> (Jason Gunthorpe's message of "Thu, 23 Jul 2009 10:02:29 -0600") References: <20090723160229.GA22400@obsidianresearch.com> Message-ID: By the way, I just did some research and indicates that on current arm ABI, enums are the same size as int. and indicates that sparc32 and sparc64 both use int for enum. and I found a mips cross compiler and tested there, and int works there as well. So I'm pretty comfortable with this patch (once the man pages and other doc get fixed to match the changes). Thanks, Roland From weiny2 at llnl.gov Wed Jul 29 17:07:53 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Wed, 29 Jul 2009 17:07:53 -0700 Subject: [ofa-general] Re: [PATCH] iblinkinfo: warn only on degragded link speed, width In-Reply-To: References: Message-ID: <20090729170753.df99c3e9.weiny2@llnl.gov> Arthur, Thanks for the patch. However, iblinkinfo has been reimplemented in C. Luckily I don't see this problem with that version. # iblinkinfo Switch 0x0002c9020040b1c8 Infiniscale-IV Mellanox Technologies: 2 1[ ] ==( 4X 10.0 Gbps Active/ LinkUp)==> 1 1[ ] "dos0" ( ) 2 2[ ] ==( 4X 10.0 Gbps Active/ LinkUp)==> 5 1[ ] "dos1" ( ) 2 3[ ] ==( 4X 10.0 Gbps Active/ LinkUp)==> 6 1[ ] "dos2" ( ) 2 4[ ] ==( 4X 10.0 Gbps Active/ LinkUp)==> 4 1[ ] "dos3" ( ) 2 5[ ] ==( 4X 10.0 Gbps Active/ LinkUp)==> 3 1[ ] "dos4" ( ) 2 6[ ] ==( 4X 2.5 Gbps Down/ Polling)==> [ ] "" ( ) 2 7[ ] ==( 4X 2.5 Gbps Down/ Polling)==> [ ] "" ( ) ... # iblinkinfo Switch 0x0008f10500200220 Voltaire 4036 - 36 QDR ports switch: ... 1 17[ ] ==( 4X 2.5 Gbps Down/ Polling)==> [ ] "" ( ) 1 18[ ] ==( 4X 2.5 Gbps Down/ Polling)==> [ ] "" ( ) 1 19[ ] ==( 4X 5.0 Gbps Active/ LinkUp)==> 7 5[ ] "Cisco Switch SFS7000D" ( ) 1 20[ ] ==( 4X 2.5 Gbps Down/ Polling)==> [ ] "" ( ) 1 21[ ] ==( 4X 2.5 Gbps Down/ Polling)==> [ ] "" ( ) ... Thanks, Ira On Wed, 29 Jul 2009 16:19:22 -0500 Arthur Kepner wrote: > > iblinkinfo.pl warns when the link speed (or width) is higher than > one of the supported values. For example, we're seeing: > > 168 11[ ] ==( 4X 10.0 Gbps Active / LinkUp)==> 169 \ > 11[ ] "Infiniscale-IV Mellanox Technologies" ( Could be 5.0 Gbps) > > It's complaining that 10Gbps was negotiated even though 5.0Gbps is > available, which I doubt is the intended behavior. > > The following patch causes iblinkinfo to warn only when the link > speed (or width) is lower than the greatest common value supported > by both ports. > > Signed-off-by: Arthur Kepner > --- > > diff -rup a/scripts/iblinkinfo.pl b/scripts/iblinkinfo.pl > --- a/scripts/iblinkinfo.pl 2009-07-29 13:16:02.889073624 -0700 > +++ b/scripts/iblinkinfo.pl 2009-07-29 13:17:24.902170284 -0700 > @@ -168,9 +168,11 @@ sub main > } > my @lines = split("\n", $data); > my $speed = ""; > + my $nspeed = -1; # in Gbps > my $speed_sup = ""; > my $speed_enable = ""; > my $width = ""; > + my $nwidth = -1; > my $width_sup = ""; > my $width_enable = ""; > my $state = ""; > @@ -179,12 +181,18 @@ sub main > my $phy_link_state = ""; > > foreach my $line (@lines) { > - if ($line =~ /^LinkSpeedActive:\.+(.*)/) { $speed = $1; } > + if ($line =~ /^LinkSpeedActive:\.+(\d+\.\d+)(.*)/) { > + $nspeed = $1; > + $speed = $1.$2; > + } > if ($line =~ /^LinkSpeedEnabled:\.+(.*)/) { > $speed_enable = $1; > } > if ($line =~ /^LinkSpeedSupported:\.+(.*)/) { $speed_sup = $1; } > - if ($line =~ /^LinkWidthActive:\.+(.*)/) { $width = $1; } > + if ($line =~ /^LinkWidthActive:\.+(\d+)(.*)/) { > + $nwidth = $1; > + $width = $1.$2; > + } > if ($line =~ /^LinkWidthEnabled:\.+(.*)/) { > $width_enable = $1; > } > @@ -243,19 +251,19 @@ sub main > if ($rem_width_enable ne "" && $rem_width_sup ne "") { > if ( $width_enable =~ /12X/ > && $rem_width_enable =~ /12X/ > - && $width !~ /12X/) > + && $nwidth < 12) > { > $width_msg = "Could be 12X"; > } else { > if ( $width_enable =~ /8X/ > && $rem_width_enable =~ /8X/ > - && $width !~ /8X/) > + && $nwidth < 8) > { > $width_msg = "Could be 8X"; > } else { > if ( $width_enable =~ /4X/ > && $rem_width_enable =~ /4X/ > - && $width !~ /4X/) > + && $nwidth < 4) > { > $width_msg = "Could be 4X"; > } > @@ -265,13 +273,13 @@ sub main > if ($rem_speed_enable ne "" && $rem_speed_sup ne "") { > if ( $speed_enable =~ /10\.0/ > && $rem_speed_enable =~ /10\.0/ > - && $speed !~ /10\.0/) > + && $nspeed < 10.0) > { > $speed_msg = "Could be 10.0 Gbps"; > } else { > if ( $speed_enable =~ /5\.0/ > && $rem_speed_enable =~ /5\.0/ > - && $speed !~ /5\.0/) > + && $nspeed < 5.0) > { > $speed_msg = "Could be 5.0 Gbps"; > } > -- Ira Weiny Math Programmer/Computer Scientist Lawrence Livermore National Lab weiny2 at llnl.gov From David.Brean at Sun.COM Wed Jul 29 18:19:55 2009 From: David.Brean at Sun.COM (David Brean) Date: Wed, 29 Jul 2009 21:19:55 -0400 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: References: Message-ID: <4A70F53B.2050809@Sun.COM> Last week I saw this message appear a few times while a NFS test was executing. It seems to be reproducible. The machine x86 with ConnectX HCA and a 2.6.31 kernel from kernel.org. -David Roland Dreier wrote: > > I know I'm going to hear it but it's not under my control :-) > > > It's whatever is in OFED 1.4.1. kernel is some 2.6.18 variant using > > mlx4.v1.0 (April 4, 2008) using x86_64 arch. > > Yeah, unfortunately that's not really a supportable system, sorry.... > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From rdreier at cisco.com Wed Jul 29 20:14:38 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 20:14:38 -0700 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: <4A70F53B.2050809@Sun.COM> (David Brean's message of "Wed, 29 Jul 2009 21:19:55 -0400") References: <4A70F53B.2050809@Sun.COM> Message-ID: > Last week I saw this message appear a few times while a NFS test was > executing. It seems to be reproducible. The machine x86 with > ConnectX HCA and a 2.6.31 kernel from kernel.org. Were you using datagram or connected mode? From rdreier at cisco.com Wed Jul 29 20:27:17 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 20:27:17 -0700 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: <4A70F53B.2050809@Sun.COM> (David Brean's message of "Wed, 29 Jul 2009 21:19:55 -0400") References: <4A70F53B.2050809@Sun.COM> Message-ID: Also if it's reproducible for you, it would be interesting to apply the patch below (which says why the send is failing) and rerun, and send the kernel log when you get the message about sends failing. Thanks, Roland drivers/infiniband/hw/mlx4/qp.c | 11 +++++++++-- 1 files changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index c4a0264..6b475ae 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -1471,12 +1471,17 @@ static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_send_wr *wr, * This is a temporary limitation and will be removed in * a forthcoming FW release: */ - if (unlikely(halign > 64)) + if (unlikely(halign > 64)) { + printk(KERN_ERR "%s: halign %d (hlen %d)\n", __func__, halign, wr->wr.ud.hlen); return -EINVAL; + } if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) && - wr->num_sge > qp->sq.max_gs - (halign >> 4))) + wr->num_sge > qp->sq.max_gs - (halign >> 4))) { + printk(KERN_ERR "%s: num_sge %d, max_gs %d, halign %d\n", + __func__, wr->num_sge, qp->sq.max_gs, halign); return -EINVAL; + } memcpy(wqe->header, wr->wr.ud.header, wr->wr.ud.hlen); @@ -1528,12 +1533,14 @@ int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, lso_wqe = &dummy; if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { + printk(KERN_ERR "%s: wq overflow\n", __func__); err = -ENOMEM; *bad_wr = wr; goto out; } if (unlikely(wr->num_sge > qp->sq.max_gs)) { + printk(KERN_ERR "%s: num_sge %d, max_gs %d\n", __func__, wr->num_sge, qp->sq.max_gs); err = -EINVAL; *bad_wr = wr; goto out; From David.Brean at Sun.COM Wed Jul 29 21:13:52 2009 From: David.Brean at Sun.COM (David Brean) Date: Thu, 30 Jul 2009 00:13:52 -0400 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: References: <4A70F53B.2050809@Sun.COM> Message-ID: <4A711E00.6050708@Sun.COM> Datagram mode. -David Roland Dreier wrote: > > Last week I saw this message appear a few times while a NFS test was > > executing. It seems to be reproducible. The machine x86 with > > ConnectX HCA and a 2.6.31 kernel from kernel.org. > > Were you using datagram or connected mode? > From bart.vanassche at gmail.com Wed Jul 29 22:54:19 2009 From: bart.vanassche at gmail.com (Bart Van Assche) Date: Thu, 30 Jul 2009 07:54:19 +0200 Subject: [ofa-general] [PATCH ibverbs] Do not use enum object types for bitfields In-Reply-To: References: <20090723160229.GA22400@obsidianresearch.com> Message-ID: On Wed, Jul 29, 2009 at 11:29 PM, Roland Dreier wrote: > By the way, I just did some research and > > indicates that on current arm ABI, enums are the same size as int. > > and > > indicates that sparc32 and sparc64 both use int for enum. > > and I found a mips cross compiler and tested there, and int works there > as well. > > So I'm pretty comfortable with this patch (once the man pages and other > doc get fixed to match the changes). If I'm not mistaken, the ANSI/ISO C standard requires that enums have the same size as an int. Two quotes from JTC1/SC22/WG14 working group, *Draft standard for the ANSI C programming language *, August 3, 1998: *6.2.5 Types [ ... ] 16. An enumeration comprises a set of named integer constant values. Each distinct enumeration constitutes a different enumerated type. * *6.7.2.2 Enumeration specifiers [ ... ] 2 The expression that defines the value of an enumeration constant shall be an integer constant expression that has a value representable as an int.* This does not hold for C++ however. A C++ compiler may use an integral type larger or smaller than an int to represent an enumeration type. A quote from Bjarne Stroustrup, *The C++ Programming Language*, Special Edition, Addison Wesley, 2000: *The sizeof an enumeration is the sizeof some integral type that can hold its range and not larger size of size of than sizeof(int), unless an enumerator cannot be represented as an int or as an unsigned int. * Bart. -------------- next part -------------- An HTML attachment was scrubbed... URL: From rdreier at cisco.com Wed Jul 29 23:12:05 2009 From: rdreier at cisco.com (Roland Dreier) Date: Wed, 29 Jul 2009 23:12:05 -0700 Subject: [ofa-general] [PATCH ibverbs] Do not use enum object types for bitfields In-Reply-To: (Bart Van Assche's message of "Thu, 30 Jul 2009 07:54:19 +0200") References: <20090723160229.GA22400@obsidianresearch.com> Message-ID: > If I'm not mistaken, the ANSI/ISO C standard requires that enums have the > same size as an int. The issue is that, although as you point out, the C standard requires that enum members have type int, the actual enum type can have a different size -- the specific quote is: Each enumerated type shall be compatible with an integer type. The choice of type is implementation-defined, but shall be capable of representing the values of all the members of the enumeration. Do a web search for "fno-short-enums" to see how this actually affects the ARM ABI. Fortunately modern Linux ARM ABIs seem to use -fno-short-enums so we're OK there. - R. From sebastien.dugue at bull.net Thu Jul 30 00:21:18 2009 From: sebastien.dugue at bull.net (sebastien dugue) Date: Thu, 30 Jul 2009 09:21:18 +0200 Subject: [ofa-general] Re: [PATCH] V3 libmlx4 - Optimize memory allocation of buffers In-Reply-To: References: <20090727110233.437ac165@frecb007965> Message-ID: <20090730092118.2cc4945e@frecb007965> Hi Roland, On Wed, 29 Jul 2009 12:01:54 -0700 Roland Dreier wrote: > thanks, applied both mxl4 and mthca versions with a couple cleanups. > let me know if I broke anything in what I ended up committing. > your cleanups are most welcome, thanks. I tested libmthca - OK. However I did not see the commit in the libmlx4 tree and I just realized I'm pulling from: git://git.openfabrics.org/ofed_1_4/libmlx4.git rather than: git://git.kernel.org/pub/scm/libs/infiniband/libmlx4.git? The only difference I can see between the 2 trees are the XRC bits. How often is the openfabrics tree updated from you tree? Thanks, Sebastien. From jackm at dev.mellanox.co.il Thu Jul 30 00:25:09 2009 From: jackm at dev.mellanox.co.il (Jack Morgenstein) Date: Thu, 30 Jul 2009 10:25:09 +0300 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: References: Message-ID: <200907301025.09427.jackm@dev.mellanox.co.il> On Wednesday 29 July 2009 21:42, Hal Rosenstock wrote: > > > I know I'm going to hear it but it's not under my control :-) > > It's whatever is in OFED 1.4.1. kernel is some 2.6.18 variant using > mlx4.v1.0 (April 4, 2008) using x86_64 arch. > Hal, 1. Did you install userspace from OFED 1.4.1, or did you take userspace (libmlx4) from www.kernel.org/git? 2. Which 2.6.18 variant? From kernel.org? -Jack From sashak at voltaire.com Thu Jul 30 02:33:04 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Thu, 30 Jul 2009 12:33:04 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Handle malloc failures better In-Reply-To: <20090727202214.GA23402@comcast.net> References: <20090727202214.GA23402@comcast.net> Message-ID: <20090730093304.GA19871@me> Hi Hal, On 16:22 Mon 27 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock > --- > diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c > index 7133e25..a139bdb 100644 > --- a/opensm/opensm/osm_ucast_lash.c > +++ b/opensm/opensm/osm_ucast_lash.c > @@ -296,8 +296,8 @@ static void shortest_path(lash_t * p_lash, int ir) > cl_list_destroy(&bfsq); > } > > -static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, > - reachable_dest_t ** destinations) > +static boolean_t generate_routing_func_for_mst(lash_t * p_lash, int sw_id, > + reachable_dest_t ** destinations) I think that 'int' is more suitable and simpler for using as return status value. > { > int i, next_switch; > switch_t *sw = p_lash->switches[sw_id]; > @@ -306,7 +306,8 @@ static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, > > for (i = 0; i < num_channels; i++) { > next_switch = sw->dij_channels[i]; > - generate_routing_func_for_mst(p_lash, next_switch, &dest); > + if (!generate_routing_func_for_mst(p_lash, next_switch, &dest)) > + return FALSE; BTW, it looks like a BFS. Could recursion be avoided here? > > i_dest = dest; > prev = i_dest; > @@ -327,9 +328,15 @@ static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, > } > > i_dest = (reachable_dest_t *) malloc(sizeof(reachable_dest_t)); > - i_dest->switch_id = sw->id; > - i_dest->next = concat_dest; > + if (i_dest) { > + i_dest->switch_id = sw->id; > + i_dest->next = concat_dest; > + } > *destinations = i_dest; > + if (i_dest) > + return TRUE; > + else > + return FALSE; > } And then: i_dest = malloc(sizeof(reachable_dest_t)); if (!i_dest) return -1; , that's all. Sasha From sashak at voltaire.com Thu Jul 30 02:33:47 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Thu, 30 Jul 2009 12:33:47 +0300 Subject: [ofa-general] Re: [PATCH] opensm/main.c: Display SMSL when specified In-Reply-To: <20090727202259.GB23402@comcast.net> References: <20090727202259.GB23402@comcast.net> Message-ID: <20090730093347.GB19871@me> On 16:22 Mon 27 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Thu Jul 30 02:34:42 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Thu, 30 Jul 2009 12:34:42 +0300 Subject: [ofa-general] Re: [PATCH] opensm: remove extra "0x" from debug message. In-Reply-To: <20090728185749.1c083e62.weiny2@llnl.gov> References: <20090728185749.1c083e62.weiny2@llnl.gov> Message-ID: <20090730093442.GC19871@me> On 18:57 Tue 28 Jul , Ira Weiny wrote: > > From: Ira Weiny > Date: Tue, 28 Jul 2009 18:55:10 -0700 > Subject: [PATCH] opensm: remove extra "0x" from debug message. > > "%p" will print a 0x for you. > > Signed-off-by: Ira Weiny Applied. Thanks. Sasha From hal.rosenstock at gmail.com Thu Jul 30 02:49:01 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Thu, 30 Jul 2009 05:49:01 -0400 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Handle malloc failures better In-Reply-To: <20090730093304.GA19871@me> References: <20090727202214.GA23402@comcast.net> <20090730093304.GA19871@me> Message-ID: Hi Sasha, On Thu, Jul 30, 2009 at 5:33 AM, Sasha Khapyorsky wrote: > Hi Hal, > > On 16:22 Mon 27 Jul , Hal Rosenstock wrote: > > > > Signed-off-by: Hal Rosenstock > > --- > > diff --git a/opensm/opensm/osm_ucast_lash.c > b/opensm/opensm/osm_ucast_lash.c > > index 7133e25..a139bdb 100644 > > --- a/opensm/opensm/osm_ucast_lash.c > > +++ b/opensm/opensm/osm_ucast_lash.c > > @@ -296,8 +296,8 @@ static void shortest_path(lash_t * p_lash, int ir) > > cl_list_destroy(&bfsq); > > } > > > > -static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, > > - reachable_dest_t ** destinations) > > +static boolean_t generate_routing_func_for_mst(lash_t * p_lash, int > sw_id, > > + reachable_dest_t ** > destinations) > > I think that 'int' is more suitable and simpler for using as return > status value. > > > { > > int i, next_switch; > > switch_t *sw = p_lash->switches[sw_id]; > > @@ -306,7 +306,8 @@ static void generate_routing_func_for_mst(lash_t * > p_lash, int sw_id, > > > > for (i = 0; i < num_channels; i++) { > > next_switch = sw->dij_channels[i]; > > - generate_routing_func_for_mst(p_lash, next_switch, &dest); > > + if (!generate_routing_func_for_mst(p_lash, next_switch, > &dest)) > > + return FALSE; > > BTW, it looks like a BFS. Could recursion be avoided here? Why do you want to eliminate the recursion ? > > > > > > i_dest = dest; > > prev = i_dest; > > @@ -327,9 +328,15 @@ static void generate_routing_func_for_mst(lash_t * > p_lash, int sw_id, > > } > > > > i_dest = (reachable_dest_t *) malloc(sizeof(reachable_dest_t)); > > - i_dest->switch_id = sw->id; > > - i_dest->next = concat_dest; > > + if (i_dest) { > > + i_dest->switch_id = sw->id; > > + i_dest->next = concat_dest; > > + } > > *destinations = i_dest; > > + if (i_dest) > > + return TRUE; > > + else > > + return FALSE; > > } > > And then: > > i_dest = malloc(sizeof(reachable_dest_t)); > if (!i_dest) > return -1; > > , that's all. I'm not following what you mean here. -- Hal > > Sasha > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit > http://openib.org/mailman/listinfo/openib-general > -------------- next part -------------- An HTML attachment was scrubbed... URL: From vlad at lists.openfabrics.org Thu Jul 30 03:02:10 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Thu, 30 Jul 2009 03:02:10 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090730-0200 daily build status Message-ID: <20090730100210.453ADE61C0C@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Passed on i686 with linux-2.6.18 Passed on i686 with linux-2.6.21.1 Passed on i686 with linux-2.6.19 Passed on i686 with linux-2.6.26 Passed on i686 with linux-2.6.24 Passed on i686 with linux-2.6.22 Passed on i686 with linux-2.6.27 Passed on x86_64 with linux-2.6.18 Passed on x86_64 with linux-2.6.19 Passed on x86_64 with linux-2.6.20 Passed on x86_64 with linux-2.6.21.1 Passed on x86_64 with linux-2.6.24 Passed on x86_64 with linux-2.6.22 Passed on x86_64 with linux-2.6.26 Passed on x86_64 with linux-2.6.27 Passed on x86_64 with linux-2.6.25 Passed on ia64 with linux-2.6.18 Passed on ia64 with linux-2.6.19 Passed on ia64 with linux-2.6.23 Passed on ia64 with linux-2.6.21.1 Passed on ia64 with linux-2.6.22 Passed on ia64 with linux-2.6.26 Passed on ia64 with linux-2.6.24 Passed on ia64 with linux-2.6.25 Passed on ppc64 with linux-2.6.18 Passed on ppc64 with linux-2.6.19 Failed: Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-1.2798.fc6 Log: /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.22.5-31-default Log: /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From bart.vanassche at gmail.com Thu Jul 30 04:00:49 2009 From: bart.vanassche at gmail.com (Bart Van Assche) Date: Thu, 30 Jul 2009 13:00:49 +0200 Subject: [ofa-general] Re: 2.6.30.1: possible irq lock inversion dependency detected In-Reply-To: References: Message-ID: On Fri, Jul 10, 2009 at 10:42 PM, Roland Dreier wrote: > >  > Thanks for the patch. With the patch applied the lockdep warning >  > indeed occurs sooner and the output is now indeed shorter. You can >  > find the new lockdep output here: >  > http://bugzilla.kernel.org/attachment.cgi?id=22305. > > Thanks, that actually looks like a completely different issue (that I > can actually understand).  I was able to reproduce that here: the issue > is doing skb_orphan() inside of priv->lock, and the network stack > locking is not irq-safe.  So the following hacky patch fixes that. > > This would be a short-term solution for the immediate issue at least.  A > better solution would be if we didn't need to make priv->lock > hardirq-safe: the only place that requires it is the QP event handler in > ipoib_cm.c, and that might be a little dicy to fix.  Need to think about that. > > However with this patch applied I don't see any further lockdep reports > here.  It would be great if you could retest yet again with this applied > (on top of my earlier patch to make priv->lock hardirq-safe as early as > possible). Hello Roland, Sorry but I'm afraid that the two kernel patches posted in this thread are not sufficient to fix all outstanding locking issues in 2.6.30 IB subsystem. I encountered the following kernel messages today: OpenSM[8074]: SM port is down OpenSM[8074]: SM port is down OpenSM[8074]: SM port is down OpenSM[8074]: Entering MASTER state ib_srpt: ASYNC event= 17 on device= mlx4_0 ib_srpt: ASYNC event= 11 on device= mlx4_0 ib_srpt: ASYNC event= 9 on device= mlx4_0 OpenSM[8074]: SUBNET UP ADDRCONF(NETDEV_CHANGE): ib0: link becomes ready ====================================================== [ INFO: HARDIRQ-safe -> HARDIRQ-unsafe lock order detected ] 2.6.30.3-scst-debug #1 ------------------------------------------------------ firefox/4069 [HC0[0]:SC1[2]:HE0:SE0] is trying to acquire: (&mad_agent_priv->lock){..-...}, at: [] ib_post_send_mad+0xe2/0x7d0 [ib_mad] and this task is already holding: (&priv->lock){-.-...}, at: [] ipoib_path_lookup+0x4d/0x2f0 [ib_ipoib] which would create a new lock dependency: (&priv->lock){-.-...} -> (&mad_agent_priv->lock){..-...} but this new dependency connects a HARDIRQ-irq-safe lock: (&priv->lock){-.-...} ... which became HARDIRQ-irq-safe at: [] 0xffffffffffffffff to a HARDIRQ-irq-unsafe lock: (&(&mad_agent_priv->timed_work)->timer){+.-...} ... which became HARDIRQ-irq-unsafe at: ... [] 0xffffffffffffffff [ ... ] stack backtrace: Pid: 4069, comm: firefox Not tainted 2.6.30.3-scst-debug #1 Call Trace: [] check_usage+0x3ba/0x470 [] check_irq_usage+0x64/0x100 [] __lock_acquire+0xff9/0x1c80 [] lock_acquire+0x108/0x150 [] ? ib_post_send_mad+0xe2/0x7d0 [ib_mad] [] _spin_lock_irqsave+0x41/0x60 [] ? ib_post_send_mad+0xe2/0x7d0 [ib_mad] [] ib_post_send_mad+0xe2/0x7d0 [ib_mad] [] ? idr_get_new_above_int+0x1c/0x90 [] send_mad+0xb4/0x110 [ib_sa] [] ? ib_pack+0x17f/0x210 [ib_core] [] ib_sa_path_rec_get+0x1ed/0x260 [ib_sa] [] path_rec_start+0x89/0xf0 [ib_ipoib] [] ? path_rec_completion+0x0/0x540 [ib_ipoib] [] ipoib_path_lookup+0x2c9/0x2f0 [ib_ipoib] [] ipoib_start_xmit+0x17d/0x440 [ib_ipoib] [] dev_hard_start_xmit+0x2bd/0x340 [] ? dev_hard_start_xmit+0x57/0x340 [] __qdisc_run+0x25e/0x2b0 [] dev_queue_xmit+0x2f0/0x4c0 [] ? dev_queue_xmit+0x52/0x4c0 [] neigh_connected_output+0xa9/0xe0 [] neigh_update+0x265/0x510 [] ? neigh_lookup+0x129/0x160 [] arp_process+0x392/0x8c0 [] ? arp_process+0x0/0x8c0 [] ? trace_hardirqs_on_caller+0x6d/0x1a0 [] arp_rcv+0x119/0x130 [] netif_receive_skb+0x392/0x4e0 [] ? netif_receive_skb+0x110/0x4e0 [] ipoib_ib_handle_rx_wc+0x166/0x2a0 [ib_ipoib] [] ipoib_poll+0x181/0x1e0 [ib_ipoib] [] net_rx_action+0x17a/0x260 [] ? net_rx_action+0xf3/0x260 [] ? __do_softirq+0x59/0x230 [] __do_softirq+0xef/0x230 [] call_softirq+0x1c/0x30 [] do_softirq+0x75/0xb0 [] irq_exit+0x95/0xa0 [] do_IRQ+0x8d/0xf0 [] ret_from_intr+0x0/0xf [] ? _spin_unlock_irq+0x31/0x60 [] ? finish_task_switch+0x89/0x110 [] ? finish_task_switch+0x46/0x110 [] ? ib_mad_completion_handler+0x0/0x800 [ib_mad] [] ? thread_return+0x52/0x85b [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] ? trace_hardirqs_on_caller+0x14d/0x1a0 [] ? trace_hardirqs_on_thunk+0x3a/0x3f [] ? schedule+0x13/0x40 [] ? retint_careful+0x12/0x2e ib0: no IPv6 routers present Bart. From eli at mellanox.co.il Thu Jul 30 06:04:34 2009 From: eli at mellanox.co.il (Eli Cohen) Date: Thu, 30 Jul 2009 16:04:34 +0300 Subject: [ofa-general] [PATCH] mlx4_core: map sufficient ICM memory for EQs Message-ID: <20090730130434.GA21428@mtls03> Current implementation allocates a single host page for EQ context memory. As the number of CPU cores increases, and since the number of required EQs depends on this number, this patch removes the hard coded limit and makes the allocation dependent on EQ entry size and the number of required EQs. Signed-off-by: Eli Cohen --- drivers/net/mlx4/eq.c | 42 ++++++++++++++++++++++++------------------ drivers/net/mlx4/main.c | 1 + drivers/net/mlx4/mlx4.h | 1 + include/linux/mlx4/device.h | 1 + 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c index b9ceddd..1d41b1a 100644 --- a/drivers/net/mlx4/eq.c +++ b/drivers/net/mlx4/eq.c @@ -530,29 +530,34 @@ int mlx4_map_eq_icm(struct mlx4_dev *dev, u64 icm_virt) { struct mlx4_priv *priv = mlx4_priv(dev); int ret; + int host_pages, icm_pages; + int i; - /* - * We assume that mapping one page is enough for the whole EQ - * context table. This is fine with all current HCAs, because - * we only use 32 EQs and each EQ uses 64 bytes of context - * memory, or 1 KB total. - */ + host_pages = ALIGN(min_t(int, dev->caps.num_eqs, num_possible_cpus() + 1) * + dev->caps.eqc_entry_size, PAGE_SIZE) >> PAGE_SHIFT; + priv->eq_table.order = ilog2(roundup_pow_of_two(host_pages)); priv->eq_table.icm_virt = icm_virt; - priv->eq_table.icm_page = alloc_page(GFP_HIGHUSER); + priv->eq_table.icm_page = alloc_pages(GFP_HIGHUSER, priv->eq_table.order); if (!priv->eq_table.icm_page) return -ENOMEM; priv->eq_table.icm_dma = pci_map_page(dev->pdev, priv->eq_table.icm_page, 0, - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + PAGE_SIZE << priv->eq_table.order, + PCI_DMA_BIDIRECTIONAL); if (pci_dma_mapping_error(dev->pdev, priv->eq_table.icm_dma)) { - __free_page(priv->eq_table.icm_page); + __free_pages(priv->eq_table.icm_page, priv->eq_table.order); return -ENOMEM; } - ret = mlx4_MAP_ICM_page(dev, priv->eq_table.icm_dma, icm_virt); - if (ret) { - pci_unmap_page(dev->pdev, priv->eq_table.icm_dma, PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); - __free_page(priv->eq_table.icm_page); + icm_pages = (PAGE_SIZE / MLX4_ICM_PAGE_SIZE) * (1 << priv->eq_table.order); + for (i = 0; i < icm_pages; ++i) { + ret = mlx4_MAP_ICM_page(dev, priv->eq_table.icm_dma, icm_virt + i * MLX4_ICM_PAGE_SIZE); + if (ret) { + mlx4_UNMAP_ICM(dev, priv->eq_table.icm_virt, i); + pci_unmap_page(dev->pdev, priv->eq_table.icm_dma, PAGE_SIZE, + PCI_DMA_BIDIRECTIONAL); + __free_pages(priv->eq_table.icm_page, priv->eq_table.order); + break; + } } return ret; @@ -561,11 +566,12 @@ int mlx4_map_eq_icm(struct mlx4_dev *dev, u64 icm_virt) void mlx4_unmap_eq_icm(struct mlx4_dev *dev) { struct mlx4_priv *priv = mlx4_priv(dev); + int icm_pages = (PAGE_SIZE / MLX4_ICM_PAGE_SIZE) * (1 << priv->eq_table.order); - mlx4_UNMAP_ICM(dev, priv->eq_table.icm_virt, 1); - pci_unmap_page(dev->pdev, priv->eq_table.icm_dma, PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); - __free_page(priv->eq_table.icm_page); + mlx4_UNMAP_ICM(dev, priv->eq_table.icm_virt, icm_pages); + pci_unmap_page(dev->pdev, priv->eq_table.icm_dma, + PAGE_SIZE << priv->eq_table.order, PCI_DMA_BIDIRECTIONAL); + __free_pages(priv->eq_table.icm_page, priv->eq_table.order); } int mlx4_alloc_eq_table(struct mlx4_dev *dev) diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c index dac621b..872becd 100644 --- a/drivers/net/mlx4/main.c +++ b/drivers/net/mlx4/main.c @@ -207,6 +207,7 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) dev->caps.max_cqes = dev_cap->max_cq_sz - 1; dev->caps.reserved_cqs = dev_cap->reserved_cqs; dev->caps.reserved_eqs = dev_cap->reserved_eqs; + dev->caps.eqc_entry_size = dev_cap->eqc_entry_sz; dev->caps.mtts_per_seg = 1 << log_mtts_per_seg; dev->caps.reserved_mtts = DIV_ROUND_UP(dev_cap->reserved_mtts, dev->caps.mtts_per_seg); diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h index 5bd79c2..1a20fa3 100644 --- a/drivers/net/mlx4/mlx4.h +++ b/drivers/net/mlx4/mlx4.h @@ -211,6 +211,7 @@ struct mlx4_eq_table { struct mlx4_icm_table cmpt_table; int have_irq; u8 inta_pin; + int order; }; struct mlx4_srq_table { diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h index ce7cc6c..8923c9b 100644 --- a/include/linux/mlx4/device.h +++ b/include/linux/mlx4/device.h @@ -206,6 +206,7 @@ struct mlx4_caps { int max_cqes; int reserved_cqs; int num_eqs; + int eqc_entry_size; int reserved_eqs; int num_comp_vectors; int num_mpts; -- 1.6.3.3 From dledford at redhat.com Thu Jul 30 08:17:32 2009 From: dledford at redhat.com (Doug Ledford) Date: Thu, 30 Jul 2009 11:17:32 -0400 Subject: [ofa-general] Re: [ewg] [Patch mthca backport] Don't use kmalloc > 128k In-Reply-To: References: <356B6978-3308-4EE9-8C00-00199558BDEA@redhat.com> <200907231121.00140.jackm@dev.mellanox.co.il> Message-ID: <9DEBF68F-72D1-4485-B1C7-A81E9504A8DD@redhat.com> On Jul 27, 2009, at 1:10 PM, Roland Dreier wrote: >>> And I don't think the upstream kernel has that limit on kmalloc size >>> either (at least with SLUB, not sure about SLAB). >> >> This patch was actually written as an emulation of the upstream SLUB >> behavior, which is exactly the same thing: on large allocations >> forward to __g_f_p(). See include/linux/slub_def.h's definition of >> kmalloc_large and kmalloc. > > Right. But does upstream SLAB also pass through to the page allocator > the same as SLUB? No, slab just fails, in which case you have to do your own __g_f_p call. > How about SLQB? No clue. -- Doug Ledford GPG KeyID: CFBFF194 http://people.redhat.com/dledford InfiniBand Specific RPMS http://people.redhat.com/dledford/Infiniband -------------- next part -------------- A non-text attachment was scrubbed... Name: PGP.sig Type: application/pgp-signature Size: 203 bytes Desc: This is a digitally signed message part URL: From jon at opengridcomputing.com Thu Jul 30 08:37:12 2009 From: jon at opengridcomputing.com (Jon Mason) Date: Thu, 30 Jul 2009 10:37:12 -0500 Subject: [ofa-general] ofa_1_5_kernel 20090730-0200 daily build status In-Reply-To: <20090730100210.453ADE61C0C@openfabrics.org> References: <20090730100210.453ADE61C0C@openfabrics.org> Message-ID: <20090730153711.GA5391@opengridcomputing.com> On Thu, Jul 30, 2009 at 03:02:10AM -0700, Vladimir Sokolovsky (Mellanox) wrote: > This email was generated automatically, please do not reply > > > git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git > git_branch: ofed_kernel_1_5 > > Common build parameters: > > Passed: > Passed on i686 with linux-2.6.18 > Passed on i686 with linux-2.6.21.1 > Passed on i686 with linux-2.6.19 > Passed on i686 with linux-2.6.26 > Passed on i686 with linux-2.6.24 > Passed on i686 with linux-2.6.22 > Passed on i686 with linux-2.6.27 > Passed on x86_64 with linux-2.6.18 > Passed on x86_64 with linux-2.6.19 > Passed on x86_64 with linux-2.6.20 > Passed on x86_64 with linux-2.6.21.1 > Passed on x86_64 with linux-2.6.24 > Passed on x86_64 with linux-2.6.22 > Passed on x86_64 with linux-2.6.26 > Passed on x86_64 with linux-2.6.27 > Passed on x86_64 with linux-2.6.25 > Passed on ia64 with linux-2.6.18 > Passed on ia64 with linux-2.6.19 > Passed on ia64 with linux-2.6.23 > Passed on ia64 with linux-2.6.21.1 > Passed on ia64 with linux-2.6.22 > Passed on ia64 with linux-2.6.26 > Passed on ia64 with linux-2.6.24 > Passed on ia64 with linux-2.6.25 > Passed on ppc64 with linux-2.6.18 > Passed on ppc64 with linux-2.6.19 > > Failed: > Build failed on x86_64 with linux-2.6.16.60-0.21-smp > Log: > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function > make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 > make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 > make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 > make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 > make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' > make: *** [kernel] Error 2 > ---------------------------------------------------------------------------------- > Build failed on x86_64 with linux-2.6.18-1.2798.fc6 > Log: > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:376: error: implicit declaration of function 'ipv4_is_loopback' > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' > make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core/addr.o] Error 1 > make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband/core] Error 2 > make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check/drivers/infiniband] Error 2 > make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-1.2798.fc6_x86_64_check] Error 2 > make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-1.2798.fc6' > make: *** [kernel] Error 2 Per Tziporet's e-mail (http://lists.openfabrics.org/pipermail/ewg/2009-June/013277.html), FC6 is not a supported distro. > ---------------------------------------------------------------------------------- > Build failed on x86_64 with linux-2.6.18-128.el5 > Log: > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function > make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 > make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 > make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 > make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 > make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' > make: *** [kernel] Error 2 > ---------------------------------------------------------------------------------- > Build failed on x86_64 with linux-2.6.18-93.el5 > Log: > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function > make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 > make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 > make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 > make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 > make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' > make: *** [kernel] Error 2 > ---------------------------------------------------------------------------------- > Build failed on x86_64 with linux-2.6.22.5-31-default > Log: > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:394: error: 'for_each_netdev' undeclared (first use in this function) > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:395: error: expected ';' before 'if' > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.c:410: error: implicit declaration of function 'ipv6_addr_loopback' > make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core/addr.o] Error 1 > make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband/core] Error 2 > make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check/drivers/infiniband] Error 2 > make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.22.5-31-default_x86_64_check] Error 2 > make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.22.5-31-default' > make: *** [kernel] Error 2 I believe this is the OpenSUSE 10.3 kernel, which is also not supported in OFED 1.5. Can we get this removed also? > ---------------------------------------------------------------------------------- > Build failed on x86_64 with linux-2.6.9-67.ELsmp > Log: > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function > make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 > make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 > make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 > make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 > make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' > make: *** [kernel] Error 2 > ---------------------------------------------------------------------------------- > Build failed on x86_64 with linux-2.6.9-78.ELsmp > Log: > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': > /home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function > make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 > make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 > make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 > make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090730-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 > make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' > make: *** [kernel] Error 2 > ---------------------------------------------------------------------------------- > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general From hal.rosenstock at gmail.com Thu Jul 30 09:11:52 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Thu, 30 Jul 2009 12:11:52 -0400 Subject: [ofa-general] IPoIB post_send failed In-Reply-To: References: Message-ID: On Wed, Jul 29, 2009 at 2:53 PM, Roland Dreier wrote: > > > I know I'm going to hear it but it's not under my control :-) > > > It's whatever is in OFED 1.4.1. kernel is some 2.6.18 variant using > > mlx4.v1.0 (April 4, 2008) using x86_64 arch. > > Yeah, unfortunately that's not really a supportable system, sorry.... > I was mistaken about the kernel being OFED patched. It is a default CentOS 5.3 kernel (pulled directly from the CentOS base SRPMs), and then modified with lustre-server patches. There are no patches that come from lustre that modify IB-specific code. -- Hal -------------- next part -------------- An HTML attachment was scrubbed... URL: From sashak at voltaire.com Thu Jul 30 09:25:22 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Thu, 30 Jul 2009 19:25:22 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_ucast_lash.c: Handle malloc failures better In-Reply-To: References: <20090727202214.GA23402@comcast.net> <20090730093304.GA19871@me> Message-ID: <20090730162522.GB24822@me> On 05:49 Thu 30 Jul , Hal Rosenstock wrote: > > I think that 'int' is more suitable and simpler for using as return > > status value. > > > > > { > > > int i, next_switch; > > > switch_t *sw = p_lash->switches[sw_id]; > > > @@ -306,7 +306,8 @@ static void generate_routing_func_for_mst(lash_t * > > p_lash, int sw_id, > > > > > > for (i = 0; i < num_channels; i++) { > > > next_switch = sw->dij_channels[i]; > > > - generate_routing_func_for_mst(p_lash, next_switch, &dest); > > > + if (!generate_routing_func_for_mst(p_lash, next_switch, > > &dest)) > > > + return FALSE; > > > > BTW, it looks like a BFS. Could recursion be avoided here? > > > Why do you want to eliminate the recursion ? Non-recursive looping doesn't require stack allocations and normally faster (and simpler), it is almost always better to not use recursion when possible. > > > @@ -327,9 +328,15 @@ static void generate_routing_func_for_mst(lash_t * > > p_lash, int sw_id, > > > } > > > > > > i_dest = (reachable_dest_t *) malloc(sizeof(reachable_dest_t)); > > > - i_dest->switch_id = sw->id; > > > - i_dest->next = concat_dest; > > > + if (i_dest) { > > > + i_dest->switch_id = sw->id; > > > + i_dest->next = concat_dest; > > > + } > > > *destinations = i_dest; > > > + if (i_dest) > > > + return TRUE; > > > + else > > > + return FALSE; > > > } > > > > And then: > > > > i_dest = malloc(sizeof(reachable_dest_t)); > > if (!i_dest) > > return -1; > > > > , that's all. > > > I'm not following what you mean here. Those three lines do the same job as this chunk of the patch, but shorter. Sasha From hnrose at comcast.net Thu Jul 30 10:55:51 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Thu, 30 Jul 2009 13:55:51 -0400 Subject: [ofa-general] [PATCH] opensm/osm_lid_mgr.c: Trivial simplification in osm_lid_mgr_process_subnet Message-ID: <20090730175551.GA19618@comcast.net> Signed-off-by: Hal Rosenstock --- diff --git a/opensm/opensm/osm_lid_mgr.c b/opensm/opensm/osm_lid_mgr.c index 34625ba..64157e0 100644 --- a/opensm/opensm/osm_lid_mgr.c +++ b/opensm/opensm/osm_lid_mgr.c @@ -1228,9 +1228,8 @@ int osm_lid_mgr_process_subnet(IN osm_lid_mgr_t * p_mgr) min_lid_ho, max_lid_ho); /* the proc returns the fact it sent a set port info */ - if (lid_mgr_set_physp_pi(p_mgr, p_port, p_port->p_physp, - cl_hton16(min_lid_ho))) - ret = -1; + ret = lid_mgr_set_physp_pi(p_mgr, p_port, p_port->p_physp, + cl_hton16(min_lid_ho)); } /* all ports */ /* store the guid to lid table in persistent db */ From hnrose at comcast.net Thu Jul 30 10:54:57 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Thu, 30 Jul 2009 13:54:57 -0400 Subject: [ofa-general] [PATCHv2] opensm/osm_ucast_lash.c: Handle malloc failures better Message-ID: <20090730175457.GA19613@comcast.net> Signed-off-by: Hal Rosenstock --- Changes since v1: Changed generate_routing_func_for_mst to return int (rather than boolean) Also, simplified malloc failure handling in that routine Both pointers from Sasha diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 7133e25..4bcb68a 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -296,8 +296,8 @@ static void shortest_path(lash_t * p_lash, int ir) cl_list_destroy(&bfsq); } -static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, - reachable_dest_t ** destinations) +static int generate_routing_func_for_mst(lash_t * p_lash, int sw_id, + reachable_dest_t ** destinations) { int i, next_switch; switch_t *sw = p_lash->switches[sw_id]; @@ -306,7 +306,8 @@ static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, for (i = 0; i < num_channels; i++) { next_switch = sw->dij_channels[i]; - generate_routing_func_for_mst(p_lash, next_switch, &dest); + if (generate_routing_func_for_mst(p_lash, next_switch, &dest)) + return -1; i_dest = dest; prev = i_dest; @@ -327,9 +328,12 @@ static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, } i_dest = (reachable_dest_t *) malloc(sizeof(reachable_dest_t)); + if (!i_dest) + return -1; i_dest->switch_id = sw->id; i_dest->next = concat_dest; *destinations = i_dest; + return 0; } static void generate_cdg_for_sp(lash_t * p_lash, int sw, int dest_switch, @@ -707,6 +711,8 @@ static int init_lash_structures(lash_t * p_lash) /* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */ p_lash->cdg_vertex_matrix = (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****)); + if (p_lash->cdg_vertex_matrix == NULL) + goto Exit_Mem_Error; for (i = 0; i < vl_min; i++) { p_lash->cdg_vertex_matrix[i] = (cdg_vertex_t ***) malloc(num_switches * @@ -800,7 +806,11 @@ static int lash_core(lash_t * p_lash) for (i = 0; i < num_switches; i++) { shortest_path(p_lash, i); - generate_routing_func_for_mst(p_lash, i, &dests); + if (generate_routing_func_for_mst(p_lash, i, &dests)) { + OSM_LOG(p_log, OSM_LOG_ERROR, + "generate_routing_func_for_mst failed\n"); + goto Exit; + } idest = dests; while (idest != NULL) { From celaloztu at gmail.com Thu Jul 30 11:27:45 2009 From: celaloztu at gmail.com (Celal Ozturk) Date: Thu, 30 Jul 2009 14:27:45 -0400 Subject: [ofa-general] Question about addr_resolve_remote Message-ID: <5ac865d20907301127u7a2c7c83oa645c23576dc116c@mail.gmail.com> Hi, I'm using OFA 1.4.1 and I have a NetEffects RNIC. IP of my RNIC is 192.168.0.1 and IP of my NIC is 10.127.227.60. When I call rdma_resolve_addr for an IP that doesn't exist, like 4.5.7.8, the result state of neigh_lookup is set to NUD_REACHABLE. This happens because after ip_route_output_key is called, rt->rt_gateway is set to 10.127.227.1, while rt->rt_dst is 4.5.7.8 I don't have much knowledge about linux networking but when I use rt->rt_dst instead of rt->rt_gateway, I get the appropriate response. This NUD_REACHABLE state is not a problem for OFA because it checks the gids before calling cma_attach_to_dev, but still, it looks weird to me to get a NUD_REACHABLE state. Do you think it would be a problem if I changed neigh = neigh_lookup(&arp_tbl, &rt->rt_gateway, rt->idev->dev); to neigh = neigh_lookup(&arp_tbl, &rt->rt_dst, rt->idev->dev); Thanks, Celal -------------- next part -------------- An HTML attachment was scrubbed... URL: From rdreier at cisco.com Thu Jul 30 11:31:14 2009 From: rdreier at cisco.com (Roland Dreier) Date: Thu, 30 Jul 2009 11:31:14 -0700 Subject: [ofa-general] Question about addr_resolve_remote In-Reply-To: <5ac865d20907301127u7a2c7c83oa645c23576dc116c@mail.gmail.com> (Celal Ozturk's message of "Thu, 30 Jul 2009 14:27:45 -0400") References: <5ac865d20907301127u7a2c7c83oa645c23576dc116c@mail.gmail.com> Message-ID: > Do you think it would be a problem if I changed > neigh = neigh_lookup(&arp_tbl, &rt->rt_gateway, rt->idev->dev); > to > neigh = neigh_lookup(&arp_tbl, &rt->rt_dst, rt->idev->dev); Yes I think it would be a problem, because what if you are trying to connect to a destination that is behind a router? (and keep in mind, in your example, it is perfectly correct to try and connect to 4.5.7.8 if it happens to be reachable via a router from your RNIC of 192.168.0.1) - R. From pop.matt at gmail.com Thu Jul 30 12:52:39 2009 From: pop.matt at gmail.com (mattt) Date: Thu, 30 Jul 2009 15:52:39 -0400 Subject: [ofa-general] Anyone have documentation for topspin 90? Message-ID: <72345420907301252k7a722a32gde6d053e691772e@mail.gmail.com> I recently obtained a topspin 90 IB switch, and I need the manual for it, does anyone have one lying around they could email? Was it ever released in PDF? I can't seem to find one. Thanks, matt. From kliteyn at dev.mellanox.co.il Thu Jul 30 14:33:55 2009 From: kliteyn at dev.mellanox.co.il (Yevgeny Kliteynik) Date: Fri, 31 Jul 2009 00:33:55 +0300 Subject: [ofa-general] [PATCH] opensm: Parallelize (Stripe) LFT sets across switches In-Reply-To: <4A6F0D97.5050402@dev.mellanox.co.il> References: <20090721180312.GA12491@comcast.net> <4A6F0D97.5050402@dev.mellanox.co.il> Message-ID: <4A7211C3.9070607@dev.mellanox.co.il> I have more numbers, this time its on Qlogic switches, which do not handle DR packets forwarding in HW. Fabric of ~1100 HCAs, ~280 switches. Current OSM configures LFTs in ~2 seconds. New algorithm does the same job in 1.4-1.6 seconds (30%-20% speed up), depending on the max_smps_per_node value. As in case of IS4 switches, the shortest config time was obtained with max_smps_per_node=0, which is unlimited pipeline. -- Yevgeny Yevgeny Kliteynik wrote: > Sasha, > > Hal Rosenstock wrote: >> Currently, MADs are pipelined to a single switch at a time which >> effectively serializes these requests due to processing at the SMA. >> This patch pipelines (stripes) them across the switches first before >> proceeding with successive blocks. As a result of this striping, >> multiple switches can process the set and respond concurrently >> which results in an improvement to the subnet initialization time. > > I have some numbers, and they look very good. > > I have a small cluster of 17 IS4 switches and 11 HCAs. > To artificially increase the cluster I used LMC=7, including > ExtendedSwitchPort 0 LMC. > > With the new code, LFT configuration is more than twice as > fast as with the old code :) > Current ucast manager ran on avarage for ~250msec, with the > new code - 110-120msec. > > Routing calculation phase of the ucast manager took ~1200 usec, > the rest was sending the blocks and waiting for no more pending > transactions. > > I also didn't see any noticeble difference between various > max_smps_per_node values. > Here are some detailed results of different executions (the > number on the left is timer value in usec): > > Current ucast manager (w/o the optimization): > > 000000 [LFT]: osm_ucast_mgr_process() - START > 001131 [LFT]: ucast_mgr_process_tbl() - START > 032251 [LFT]: ucast_mgr_process_tbl() - END > 032263 [LFT]: osm_ucast_mgr_process() - END > 253416 [LFT]: Done wait_for_pending_transactions() > > New code, max_smps_per_node=0: > > 001417 [LFT]: osm_ucast_mgr_process() - START (0 max_smps_per_node) > 002690 [LFT]: ucast_mgr_process_tbl() - START > 032946 [LFT]: ucast_mgr_process_tbl() - END > 032948 [LFT]: osm_ucast_pipeline_tbl() - START > 033846 [LFT]: osm_ucast_pipeline_tbl() - END > 033858 [LFT]: osm_ucast_mgr_process() - END > 108203 [LFT]: Done wait_for_pending_transactions() > > New code, max_smps_per_node=1: > > 007474 [LFT]: osm_ucast_mgr_process() - START (1 max_smps_per_node) > 008735 [LFT]: ucast_mgr_process_tbl() - START > 040071 [LFT]: ucast_mgr_process_tbl() - END > 040074 [LFT]: osm_ucast_pipeline_tbl() - START > 040103 [LFT]: osm_ucast_pipeline_tbl() - END > 040114 [LFT]: osm_ucast_mgr_process() - END > 120097 [LFT]: Done wait_for_pending_transactions() > > New code, max_smps_per_node=4: > > 004137 [LFT]: osm_ucast_mgr_process() - START (4 max_smps_per_node) > 005380 [LFT]: ucast_mgr_process_tbl() - START > 037436 [LFT]: ucast_mgr_process_tbl() - END > 037439 [LFT]: osm_ucast_pipeline_tbl() - START > 037495 [LFT]: osm_ucast_pipeline_tbl() - END > 037506 [LFT]: osm_ucast_mgr_process() - END > 114983 [LFT]: Done wait_for_pending_transactions() > > Here's the patch that shows where I added the log messages in current code: > >> From d7962cc06f5526982bc51c17d53981b6d23256a8 Mon Sep 17 00:00:00 2001 > From: Yevgeny Kliteynik > Date: Mon, 27 Jul 2009 16:12:27 +0300 > Subject: [PATCH] [LFT - master] log messages for LFT parallelization > > Signed-off-by: Yevgeny Kliteynik > --- > opensm/opensm/osm_state_mgr.c | 9 ++++++++- > opensm/opensm/osm_ucast_mgr.c | 4 ++++ > 2 files changed, 12 insertions(+), 1 deletions(-) > > diff --git a/opensm/opensm/osm_state_mgr.c b/opensm/opensm/osm_state_mgr.c > index adc39a0..573a2a7 100644 > --- a/opensm/opensm/osm_state_mgr.c > +++ b/opensm/opensm/osm_state_mgr.c > @@ -1271,11 +1271,18 @@ _repeat_discovery: > */ > > if (!sm->ucast_mgr.cache_valid || > - osm_ucast_cache_process(&sm->ucast_mgr)) > + osm_ucast_cache_process(&sm->ucast_mgr)) { > + OSM_LOG(sm->p_log, OSM_LOG_INFO, "[LFT]: " > + "osm_ucast_mgr_process() - START\n"); > osm_ucast_mgr_process(&sm->ucast_mgr); > + OSM_LOG(sm->p_log, OSM_LOG_INFO, "[LFT]: " > + "osm_ucast_mgr_process() - END\n"); > + } > > if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats)) > return; > + OSM_LOG(sm->p_log, OSM_LOG_INFO, "[LFT]: " > + "Done wait_for_pending_transactions()\n"); > > /* cleanup switch lft buffers */ > cl_qmap_apply_func(&sm->p_subn->sw_guid_tbl, cleanup_switch, > sm->p_log); > diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c > index 78a7031..83867a0 100644 > --- a/opensm/opensm/osm_ucast_mgr.c > +++ b/opensm/opensm/osm_ucast_mgr.c > @@ -901,8 +901,12 @@ static int ucast_mgr_build_lfts(osm_ucast_mgr_t * > p_mgr) > cl_qmap_apply_func(&p_mgr->p_subn->port_guid_tbl, > add_port_to_order_list, p_mgr); > > + OSM_LOG(p_mgr->p_log, OSM_LOG_INFO, "[LFT]: " > + "ucast_mgr_process_tbl() - START\n"); > cl_qmap_apply_func(&p_mgr->p_subn->sw_guid_tbl, ucast_mgr_process_tbl, > p_mgr); > + OSM_LOG(p_mgr->p_log, OSM_LOG_INFO, "[LFT]: " > + "ucast_mgr_process_tbl() - END\n"); > > cl_qlist_remove_all(&p_mgr->port_order_list); > From jgunthorpe at obsidianresearch.com Thu Jul 30 15:58:16 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Thu, 30 Jul 2009 16:58:16 -0600 Subject: [ofa-general] [PATCH] Do not use enum object types for bitfields Message-ID: <20090730225816.GA32073@obsidianresearch.com> Arithmetic operations on enum members does not result in the enum type. So all flag enums result in compile errors from g++ when they are or'd togeth. Replace all flag enum objects with 'int'. 'int' was selected to preserve the ABI and validated on i386, x86-64, ppc32, ppc64, ia64, and mips arm and sparc also appear compatible with this choice. Signed-off-by: Jason Gunthorpe --- include/infiniband/driver.h | 8 ++++---- include/infiniband/verbs.h | 28 ++++++++++++++-------------- man/ibv_modify_qp.3 | 2 +- man/ibv_modify_srq.3 | 2 +- man/ibv_poll_cq.3 | 2 +- man/ibv_post_send.3 | 2 +- man/ibv_query_qp.3 | 6 ++++-- man/ibv_reg_mr.3 | 3 +-- src/cmd.c | 10 +++++----- src/compat-1_0.c | 18 +++++++++--------- src/verbs.c | 8 ++++---- 11 files changed, 45 insertions(+), 44 deletions(-) diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h index e54e0e3..9a81416 100644 --- a/include/infiniband/driver.h +++ b/include/infiniband/driver.h @@ -77,7 +77,7 @@ int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd, int ibv_cmd_dealloc_pd(struct ibv_pd *pd); #define IBV_CMD_REG_MR_HAS_RESP_PARAMS int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, - uint64_t hca_va, enum ibv_access_flags access, + uint64_t hca_va, int access, struct ibv_mr *mr, struct ibv_reg_mr *cmd, size_t cmd_size, struct ibv_reg_mr_resp *resp, size_t resp_size); @@ -101,7 +101,7 @@ int ibv_cmd_create_srq(struct ibv_pd *pd, struct ibv_create_srq_resp *resp, size_t resp_size); int ibv_cmd_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask, + int srq_attr_mask, struct ibv_modify_srq *cmd, size_t cmd_size); int ibv_cmd_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, @@ -113,11 +113,11 @@ int ibv_cmd_create_qp(struct ibv_pd *pd, struct ibv_create_qp *cmd, size_t cmd_size, struct ibv_create_qp_resp *resp, size_t resp_size); int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *qp_init_attr, struct ibv_query_qp *cmd, size_t cmd_size); int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_modify_qp *cmd, size_t cmd_size); int ibv_cmd_destroy_qp(struct ibv_qp *qp); int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h index 226d85e..0f1cb2e 100644 --- a/include/infiniband/verbs.h +++ b/include/infiniband/verbs.h @@ -269,7 +269,7 @@ struct ibv_wc { uint32_t imm_data; /* in network byte order */ uint32_t qp_num; uint32_t src_qp; - enum ibv_wc_flags wc_flags; + int wc_flags; uint16_t pkey_index; uint16_t slid; uint8_t sl; @@ -508,7 +508,7 @@ struct ibv_send_wr { struct ibv_sge *sg_list; int num_sge; enum ibv_wr_opcode opcode; - enum ibv_send_flags send_flags; + int send_flags; uint32_t imm_data; /* in network byte order */ union { struct { @@ -541,8 +541,8 @@ struct ibv_mw_bind { struct ibv_mr *mr; void *addr; size_t length; - enum ibv_send_flags send_flags; - enum ibv_access_flags mw_access_flags; + int send_flags; + int mw_access_flags; }; struct ibv_srq { @@ -633,12 +633,12 @@ struct ibv_context_ops { struct ibv_pd * (*alloc_pd)(struct ibv_context *context); int (*dealloc_pd)(struct ibv_pd *pd); struct ibv_mr * (*reg_mr)(struct ibv_pd *pd, void *addr, size_t length, - enum ibv_access_flags access); + int access); struct ibv_mr * (*rereg_mr)(struct ibv_mr *mr, - enum ibv_rereg_mr_flags flags, + int flags, struct ibv_pd *pd, void *addr, size_t length, - enum ibv_access_flags access); + int access); int (*dereg_mr)(struct ibv_mr *mr); struct ibv_mw * (*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type); int (*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw, @@ -656,7 +656,7 @@ struct ibv_context_ops { struct ibv_srq_init_attr *srq_init_attr); int (*modify_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask); + int srq_attr_mask); int (*query_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); int (*destroy_srq)(struct ibv_srq *srq); @@ -665,10 +665,10 @@ struct ibv_context_ops { struct ibv_recv_wr **bad_recv_wr); struct ibv_qp * (*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr); int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask); + int attr_mask); int (*destroy_qp)(struct ibv_qp *qp); int (*post_send)(struct ibv_qp *qp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr); @@ -793,7 +793,7 @@ int ibv_dealloc_pd(struct ibv_pd *pd); * ibv_reg_mr - Register a memory region */ struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr, - size_t length, enum ibv_access_flags access); + size_t length, int access); /** * ibv_dereg_mr - Deregister a memory region @@ -926,7 +926,7 @@ struct ibv_srq *ibv_create_srq(struct ibv_pd *pd, */ int ibv_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask); + int srq_attr_mask); /** * ibv_query_srq - Returns the attribute list and current values for the @@ -966,7 +966,7 @@ struct ibv_qp *ibv_create_qp(struct ibv_pd *pd, * ibv_modify_qp - Modify a queue pair. */ int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask); + int attr_mask); /** * ibv_query_qp - Returns the attribute list and current values for the @@ -980,7 +980,7 @@ int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, * selected attributes. */ int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr); /** diff --git a/man/ibv_modify_qp.3 b/man/ibv_modify_qp.3 index a022dac..9eabcdf 100644 --- a/man/ibv_modify_qp.3 +++ b/man/ibv_modify_qp.3 @@ -8,7 +8,7 @@ ibv_modify_qp \- modify the attributes of a queue pair (QP) .B #include .sp .BI "int ibv_modify_qp(struct ibv_qp " "*qp" ", struct ibv_qp_attr " "*attr" , -.BI " enum ibv_qp_attr_mask " "attr_mask" ); +.BI " int " "attr_mask" ); .fi .SH "DESCRIPTION" .B ibv_modify_qp() diff --git a/man/ibv_modify_srq.3 b/man/ibv_modify_srq.3 index fa23c3a..9cc0497 100644 --- a/man/ibv_modify_srq.3 +++ b/man/ibv_modify_srq.3 @@ -9,7 +9,7 @@ ibv_modify_srq \- modify attributes of a shared receive queue (SRQ) .sp .BI "int ibv_modify_srq(struct ibv_srq " "*srq" , .BI " struct ibv_srq_attr " "*srq_attr" , -.BI " enum ibv_srq_attr_mask " "srq_attr_mask" ); +.BI " int " "srq_attr_mask" ); .fi .SH "DESCRIPTION" .B ibv_modify_srq() diff --git a/man/ibv_poll_cq.3 b/man/ibv_poll_cq.3 index cf70efc..57c6daa 100644 --- a/man/ibv_poll_cq.3 +++ b/man/ibv_poll_cq.3 @@ -33,7 +33,7 @@ uint32_t byte_len; /* Number of bytes transferred */ uint32_t imm_data; /* Immediate data (in network byte order) */ uint32_t qp_num; /* Local QP number of completed WR */ uint32_t src_qp; /* Source QP number (remote QP number) of completed WR (valid only for UD QPs) */ -enum ibv_wc_flags wc_flags; /* Flags of the completed WR */ +int wc_flags; /* Flags of the completed WR */ uint16_t pkey_index; /* P_Key index (valid only for GSI QPs) */ uint16_t slid; /* Source LID */ uint8_t sl; /* Service Level */ diff --git a/man/ibv_post_send.3 b/man/ibv_post_send.3 index 51fc1cf..33fbb50 100644 --- a/man/ibv_post_send.3 +++ b/man/ibv_post_send.3 @@ -33,7 +33,7 @@ struct ibv_send_wr *next; /* Pointer to next WR in list, N struct ibv_sge *sg_list; /* Pointer to the s/g array */ int num_sge; /* Size of the s/g array */ enum ibv_wr_opcode opcode; /* Operation type */ -enum ibv_send_flags send_flags; /* Flags of the WR properties */ +int send_flags; /* Flags of the WR properties */ uint32_t imm_data; /* Immediate data (in network byte order) */ union { .in +8 diff --git a/man/ibv_query_qp.3 b/man/ibv_query_qp.3 index 7101dc2..3893ec8 100644 --- a/man/ibv_query_qp.3 +++ b/man/ibv_query_qp.3 @@ -8,7 +8,7 @@ ibv_query_qp \- get the attributes of a queue pair (QP) .B #include .sp .BI "int ibv_query_qp(struct ibv_qp " "*qp" ", struct ibv_qp_attr " "*attr" , -.BI " enum ibv_qp_attr_mask " "attr_mask" , +.BI " int " "attr_mask" , .BI " struct ibv_qp_init_attr " "*init_attr" ); .fi .SH "DESCRIPTION" @@ -69,7 +69,9 @@ The argument .I attr_mask is a hint that specifies the minimum list of attributes to retrieve. Some RDMA devices may return extra attributes not requested, for -example if the value can be returned cheaply. +example if the value can be returned cheaply. This has the same +form as in +.B ibv_modify_qp()\fR. .PP Attribute values are valid if they have been set using .B ibv_modify_qp()\fR. diff --git a/man/ibv_reg_mr.3 b/man/ibv_reg_mr.3 index 7cd3de8..7723771 100644 --- a/man/ibv_reg_mr.3 +++ b/man/ibv_reg_mr.3 @@ -8,8 +8,7 @@ ibv_reg_mr, ibv_dereg_mr \- register or deregister a memory region (MR) .B #include .sp .BI "struct ibv_mr *ibv_reg_mr(struct ibv_pd " "*pd" ", void " "*addr" , -.BI " size_t " "length" , -.BI " enum ibv_access_flags " "access" ); +.BI " size_t " "length" ", int " "access" ); .sp .BI "int ibv_dereg_mr(struct ibv_mr " "*mr" ); .fi diff --git a/src/cmd.c b/src/cmd.c index c2e76cf..cbd5288 100644 --- a/src/cmd.c +++ b/src/cmd.c @@ -231,7 +231,7 @@ int ibv_cmd_dealloc_pd(struct ibv_pd *pd) } int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, - uint64_t hca_va, enum ibv_access_flags access, + uint64_t hca_va, int access, struct ibv_mr *mr, struct ibv_reg_mr *cmd, size_t cmd_size, struct ibv_reg_mr_resp *resp, size_t resp_size) @@ -485,7 +485,7 @@ int ibv_cmd_create_srq(struct ibv_pd *pd, static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask, + int srq_attr_mask, struct ibv_modify_srq *new_cmd, size_t new_cmd_size) { @@ -513,7 +513,7 @@ static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq, int ibv_cmd_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask, + int srq_attr_mask, struct ibv_modify_srq *cmd, size_t cmd_size) { if (abi_ver == 3) @@ -651,7 +651,7 @@ int ibv_cmd_create_qp(struct ibv_pd *pd, } int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr, struct ibv_query_qp *cmd, size_t cmd_size) { @@ -733,7 +733,7 @@ int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, } int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_modify_qp *cmd, size_t cmd_size) { IBV_INIT_CMD(cmd, cmd_size, MODIFY_QP); diff --git a/src/compat-1_0.c b/src/compat-1_0.c index 459ade9..3f5ff35 100644 --- a/src/compat-1_0.c +++ b/src/compat-1_0.c @@ -88,7 +88,7 @@ struct ibv_send_wr_1_0 { struct ibv_sge *sg_list; int num_sge; enum ibv_wr_opcode opcode; - enum ibv_send_flags send_flags; + int send_flags; uint32_t imm_data; /* in network byte order */ union { struct { @@ -172,7 +172,7 @@ struct ibv_context_ops_1_0 { struct ibv_pd * (*alloc_pd)(struct ibv_context *context); int (*dealloc_pd)(struct ibv_pd *pd); struct ibv_mr * (*reg_mr)(struct ibv_pd *pd, void *addr, size_t length, - enum ibv_access_flags access); + int access); int (*dereg_mr)(struct ibv_mr *mr); struct ibv_cq * (*create_cq)(struct ibv_context *context, int cqe, struct ibv_comp_channel *channel, @@ -188,7 +188,7 @@ struct ibv_context_ops_1_0 { struct ibv_srq_init_attr *srq_init_attr); int (*modify_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask); + int srq_attr_mask); int (*query_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); int (*destroy_srq)(struct ibv_srq *srq); @@ -197,10 +197,10 @@ struct ibv_context_ops_1_0 { struct ibv_recv_wr_1_0 **bad_recv_wr); struct ibv_qp * (*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr); int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask); + int attr_mask); int (*destroy_qp)(struct ibv_qp *qp); int (*post_send)(struct ibv_qp_1_0 *qp, struct ibv_send_wr_1_0 *wr, @@ -596,7 +596,7 @@ int __ibv_dealloc_pd_1_0(struct ibv_pd_1_0 *pd) symver(__ibv_dealloc_pd_1_0, ibv_dealloc_pd, IBVERBS_1.0); struct ibv_mr_1_0 *__ibv_reg_mr_1_0(struct ibv_pd_1_0 *pd, void *addr, - size_t length, enum ibv_access_flags access) + size_t length, int access) { struct ibv_mr *real_mr; struct ibv_mr_1_0 *mr; @@ -736,7 +736,7 @@ symver(__ibv_create_srq_1_0, ibv_create_srq, IBVERBS_1.0); int __ibv_modify_srq_1_0(struct ibv_srq_1_0 *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask) + int srq_attr_mask) { return ibv_modify_srq(srq->real_srq, srq_attr, srq_attr_mask); } @@ -806,7 +806,7 @@ struct ibv_qp_1_0 *__ibv_create_qp_1_0(struct ibv_pd_1_0 *pd, symver(__ibv_create_qp_1_0, ibv_create_qp, IBVERBS_1.0); int __ibv_query_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr_1_0 *init_attr) { struct ibv_qp_init_attr real_init_attr; @@ -829,7 +829,7 @@ int __ibv_query_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr, symver(__ibv_query_qp_1_0, ibv_query_qp, IBVERBS_1.0); int __ibv_modify_qp_1_0(struct ibv_qp_1_0 *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask) + int attr_mask) { return ibv_modify_qp(qp->real_qp, attr, attr_mask); } diff --git a/src/verbs.c b/src/verbs.c index 477e412..ba3c0a4 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -155,7 +155,7 @@ int __ibv_dealloc_pd(struct ibv_pd *pd) default_symver(__ibv_dealloc_pd, ibv_dealloc_pd); struct ibv_mr *__ibv_reg_mr(struct ibv_pd *pd, void *addr, - size_t length, enum ibv_access_flags access) + size_t length, int access) { struct ibv_mr *mr; @@ -377,7 +377,7 @@ default_symver(__ibv_create_srq, ibv_create_srq); int __ibv_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, - enum ibv_srq_attr_mask srq_attr_mask) + int srq_attr_mask) { return srq->context->ops.modify_srq(srq, srq_attr, srq_attr_mask); } @@ -419,7 +419,7 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd, default_symver(__ibv_create_qp, ibv_create_qp); int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask, + int attr_mask, struct ibv_qp_init_attr *init_attr) { int ret; @@ -436,7 +436,7 @@ int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, default_symver(__ibv_query_qp, ibv_query_qp); int __ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, - enum ibv_qp_attr_mask attr_mask) + int attr_mask) { int ret; -- 1.6.0.4 From hnrose at comcast.net Thu Jul 30 16:28:48 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Thu, 30 Jul 2009 19:28:48 -0400 Subject: [ofa-general] [PATCHv2] opensm: Parallelize (Stripe) LFT sets across switches Message-ID: <20090730232848.GA22660@comcast.net> Currently, MADs are pipelined to a single switch at a time which effectively serializes these requests due to processing at the SMA. This patch pipelines (stripes) them across the switches first before proceeding with successive blocks. As a result of this striping, multiple switches can process the set and respond concurrently which results in an improvement to the subnet initialization time. This patch also introduces a new config option (max_smps_per_node) which indicates how deep the per node pipeline is (current default is 4). This also has the effect of limiting the number of times that the switch list is traversed. Maybe this embellishment is unnecessary. All unicast routing protocols are updated for this with the exception of file. A similar subsequent change will do this for MFTs. Yevgeny Kliteynik wrote: With a small cluster of 17 IS4 switches and 11 HCAs and to artificially increase the cluster, LMC of 7 was used including EnhancedSwitchPort 0 LMC. With the new code, LFT configuration is more than twice as fast as with the old code :) Current ucast manager ran on avarage for ~250msec, with the new code - 110-120msec. Routing calculation phase of the ucast manager took ~1200 usec, the rest was sending the blocks and waiting for no more pending transactions. No noticeable difference between various max_smps_per_node values was observed. Here are some detailed results of different executions (the number on the left is timer value in usec): Current ucast manager (w/o the optimization): 000000 [LFT]: osm_ucast_mgr_process() - START 001131 [LFT]: ucast_mgr_process_tbl() - START 032251 [LFT]: ucast_mgr_process_tbl() - END 032263 [LFT]: osm_ucast_mgr_process() - END 253416 [LFT]: Done wait_for_pending_transactions() New code, max_smps_per_node=0: 001417 [LFT]: osm_ucast_mgr_process() - START (0 max_smps_per_node) 002690 [LFT]: ucast_mgr_process_tbl() - START 032946 [LFT]: ucast_mgr_process_tbl() - END 032948 [LFT]: osm_ucast_pipeline_tbl() - START 033846 [LFT]: osm_ucast_pipeline_tbl() - END 033858 [LFT]: osm_ucast_mgr_process() - END 108203 [LFT]: Done wait_for_pending_transactions() New code, max_smps_per_node=1: 007474 [LFT]: osm_ucast_mgr_process() - START (1 max_smps_per_node) 008735 [LFT]: ucast_mgr_process_tbl() - START 040071 [LFT]: ucast_mgr_process_tbl() - END 040074 [LFT]: osm_ucast_pipeline_tbl() - START 040103 [LFT]: osm_ucast_pipeline_tbl() - END 040114 [LFT]: osm_ucast_mgr_process() - END 120097 [LFT]: Done wait_for_pending_transactions() New code, max_smps_per_node=4: 004137 [LFT]: osm_ucast_mgr_process() - START (4 max_smps_per_node) 005380 [LFT]: ucast_mgr_process_tbl() - START 037436 [LFT]: ucast_mgr_process_tbl() - END 037439 [LFT]: osm_ucast_pipeline_tbl() - START 037495 [LFT]: osm_ucast_pipeline_tbl() - END 037506 [LFT]: osm_ucast_mgr_process() - END 114983 [LFT]: Done wait_for_pending_transactions() With IS3 based Qlogic switches, which do not handle DR packets forwarding in HW, with a fabric of ~1100 HCAs, ~280 switches: Current OSM configures LFTs in ~2 seconds. New algorithm does the same job in 1.4-1.6 seconds (30%-20% speed up), depending on the max_smps_per_node value. As in case of IS4 switches, the shortest config time was obtained with max_smps_per_node=0, which is unlimited pipeline. Signed-off-by: Hal Rosenstock --- Changes since v1: Added Yevgeny's performance data to patch description above No change to actual patch diff --git a/opensm/include/opensm/osm_base.h b/opensm/include/opensm/osm_base.h index 0537002..617e8a9 100644 --- a/opensm/include/opensm/osm_base.h +++ b/opensm/include/opensm/osm_base.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2006 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * @@ -449,6 +449,18 @@ BEGIN_C_DECLS */ #define OSM_DEFAULT_SMP_MAX_ON_WIRE 4 /***********/ +/****d* OpenSM: Base/OSM_DEFAULT_SMP_MAX_PER_NODE +* NAME +* OSM_DEFAULT_SMP_MAX_PER_NODE +* +* DESCRIPTION +* Specifies the default number of VL15 SMP MADs allowed +* per node for certain attributes. +* +* SYNOPSIS +*/ +#define OSM_DEFAULT_SMP_MAX_PER_NODE 4 +/***********/ /****d* OpenSM: Base/OSM_SM_DEFAULT_QP0_RCV_SIZE * NAME * OSM_SM_DEFAULT_QP0_RCV_SIZE diff --git a/opensm/include/opensm/osm_sm.h b/opensm/include/opensm/osm_sm.h index cc8321d..1776380 100644 --- a/opensm/include/opensm/osm_sm.h +++ b/opensm/include/opensm/osm_sm.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -130,6 +130,7 @@ typedef struct osm_sm { osm_sm_mad_ctrl_t mad_ctrl; osm_lid_mgr_t lid_mgr; osm_ucast_mgr_t ucast_mgr; + boolean_t lfts_updated; cl_disp_reg_handle_t sweep_fail_disp_h; cl_disp_reg_handle_t ni_disp_h; cl_disp_reg_handle_t pi_disp_h; @@ -524,6 +525,45 @@ osm_resp_send(IN osm_sm_t * sm, * *********/ +/****f* OpenSM: SM/osm_sm_set_next_lft_block +* NAME +* osm_sm_set_next_lft_block +* +* DESCRIPTION +* Set the next LFT (LinearForwardingTable) block in the indicated switch. +* +* SYNOPSIS +*/ +void +osm_sm_set_next_lft_block(IN osm_sm_t *p_sm, IN osm_switch_t *p_sw, + IN uint8_t *p_block, IN osm_dr_path_t *p_path, + IN osm_madw_context_t *p_context); +/* +* PARAMETERS +* p_sm +* [in] Pointer to an osm_sm_t object. +* +* p_switch +* [in] Pointer to the switch object. +* +* p_block +* [in] Pointer to the forwarding table block. +* +* p_path +* [in] Pointer to a directed route path object. +* +* p_context +* [in] Mad wrapper context structure to be copied into the wrapper +* context, and thus visible to the recipient of the response. +* +* RETURN VALUES +* None +* +* NOTES +* +* SEE ALSO +*********/ + /****f* OpenSM: SM/osm_sm_mcgrp_join * NAME * osm_sm_mcgrp_join diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 59a32ad..f12afae 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. * @@ -147,6 +147,7 @@ typedef struct osm_subn_opt { uint32_t sweep_interval; uint32_t max_wire_smps; uint32_t transaction_timeout; + uint32_t max_smps_per_node; uint8_t sm_priority; uint8_t lmc; boolean_t lmc_esp0; diff --git a/opensm/include/opensm/osm_switch.h b/opensm/include/opensm/osm_switch.h index 7ce28c5..e12113f 100644 --- a/opensm/include/opensm/osm_switch.h +++ b/opensm/include/opensm/osm_switch.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -102,6 +102,7 @@ typedef struct osm_switch { osm_port_profile_t *p_prof; uint8_t *lft; uint8_t *new_lft; + uint16_t lft_block_id_ho; osm_mcast_tbl_t mcast_tbl; unsigned endport_links; unsigned need_update; diff --git a/opensm/include/opensm/osm_ucast_mgr.h b/opensm/include/opensm/osm_ucast_mgr.h index a040476..fdea49a 100644 --- a/opensm/include/opensm/osm_ucast_mgr.h +++ b/opensm/include/opensm/osm_ucast_mgr.h @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -233,17 +233,42 @@ osm_ucast_mgr_init(IN osm_ucast_mgr_t * const p_mgr, IN struct osm_sm * sm); * osm_ucast_mgr_destroy *********/ -/****f* OpenSM: Unicast Manager/osm_ucast_mgr_set_fwd_table +/****f* OpenSM: Unicast Manager/osm_ucast_pipeline_tbl * NAME -* osm_ucast_mgr_set_fwd_table +* osm_ucast_pipeline_tbl * * DESCRIPTION -* Setup forwarding table for the switch (from prepared new_lft). +* The osm_ucast_pipeline_tbl function pipelines the LFT +* (LinearForwardingTable) sets across the switches +* (from prepared new_lft). * * SYNOPSIS */ -int osm_ucast_mgr_set_fwd_table(IN osm_ucast_mgr_t * const p_mgr, - IN osm_switch_t * const p_sw); +void osm_ucast_pipeline_tbl(IN osm_ucast_mgr_t * p_mgr); +/* +* PARAMETERS +* p_mgr +* [in] Pointer to an osm_ucast_mgr_t object. +* +* RETURN VALUES +* None. +* +* NOTES +* +* SEE ALSO +*********/ + +/****f* OpenSM: Unicast Manager/osm_ucast_mgr_set_fwd_tbl_top +* NAME +* osm_ucast_mgr_set_fwd_tbl_top +* +* DESCRIPTION +* Setup LinearFDBTop for the switch. +* +* SYNOPSIS +*/ +int osm_ucast_mgr_set_fwd_tbl_top(IN osm_ucast_mgr_t * const p_mgr, + IN osm_switch_t * const p_sw); /* * PARAMETERS * p_mgr diff --git a/opensm/opensm/osm_lin_fwd_rcv.c b/opensm/opensm/osm_lin_fwd_rcv.c index 2edb8d3..cb131b4 100644 --- a/opensm/opensm/osm_lin_fwd_rcv.c +++ b/opensm/opensm/osm_lin_fwd_rcv.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -36,7 +36,7 @@ /* * Abstract: * Implementation of osm_lft_rcv_t. - * This object represents the NodeDescription Receiver object. + * This object represents the Linear Forwarding Table Receiver object. * This object is part of the opensm family of objects. */ @@ -55,6 +55,7 @@ void osm_lft_rcv_process(IN void *context, IN void *data) { osm_sm_t *sm = context; osm_madw_t *p_madw = data; + osm_dr_path_t *p_path; ib_smp_t *p_smp; uint32_t block_num; osm_switch_t *p_sw; @@ -62,6 +63,8 @@ void osm_lft_rcv_process(IN void *context, IN void *data) uint8_t *p_block; ib_net64_t node_guid; ib_api_status_t status; + uint8_t block[IB_SMP_DATA_SIZE]; + osm_madw_context_t mad_context; CL_ASSERT(sm); @@ -94,6 +97,16 @@ void osm_lft_rcv_process(IN void *context, IN void *data) "\n\t\t\t\tSwitch 0x%" PRIx64 "\n", ib_get_err_str(status), cl_ntoh64(node_guid)); } + + p_path = osm_physp_get_dr_path_ptr(osm_node_get_physp_ptr(p_sw->p_node, 0)); + + mad_context.lft_context.node_guid = node_guid; + mad_context.lft_context.set_method = TRUE; + + osm_sm_set_next_lft_block(sm, p_sw, &block[0], p_path, + &mad_context); + + p_sw->lft_block_id_ho++; } CL_PLOCK_RELEASE(sm->p_lock); diff --git a/opensm/opensm/osm_sm.c b/opensm/opensm/osm_sm.c index daa60ff..4e0fd2a 100644 --- a/opensm/opensm/osm_sm.c +++ b/opensm/opensm/osm_sm.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. * @@ -441,6 +441,45 @@ Exit: /********************************************************************** **********************************************************************/ +void osm_sm_set_next_lft_block(IN osm_sm_t *p_sm, IN osm_switch_t *p_sw, + IN uint8_t *p_block, IN osm_dr_path_t *p_path, + IN osm_madw_context_t *context) +{ + ib_api_status_t status; + + for (; + osm_switch_get_lft_block(p_sw, p_sw->lft_block_id_ho, p_block); + p_sw->lft_block_id_ho++) { + if (!p_sw->need_update && !p_sm->p_subn->need_update && + !memcmp(p_block, + p_sw->new_lft + p_sw->lft_block_id_ho * IB_SMP_DATA_SIZE, + IB_SMP_DATA_SIZE)) + continue; + + p_sm->lfts_updated = 1; + + OSM_LOG(p_sm->p_log, OSM_LOG_DEBUG, + "Writing FT block %u to switch 0x%" PRIx64 "\n", + p_sw->lft_block_id_ho, + cl_ntoh64(context->lft_context.node_guid)); + + status = osm_req_set(p_sm, p_path, + p_sw->new_lft + + p_sw->lft_block_id_ho * IB_SMP_DATA_SIZE, + IB_SMP_DATA_SIZE, IB_MAD_ATTR_LIN_FWD_TBL, + cl_hton32(p_sw->lft_block_id_ho), + CL_DISP_MSGID_NONE, context); + + if (status != IB_SUCCESS) + OSM_LOG(p_sm->p_log, OSM_LOG_ERROR, "ERR 2E11: " + "Sending linear fwd. tbl. block failed (%s)\n", + ib_get_err_str(status)); + break; + } +} + +/********************************************************************** + **********************************************************************/ static ib_api_status_t sm_mgrp_process(IN osm_sm_t * p_sm, IN osm_mgrp_t * p_mgrp) { diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index ec15f8a..1964b7f 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2008 Xsigo Systems Inc. All rights reserved. * @@ -295,6 +295,7 @@ static const opt_rec_t opt_tbl[] = { { "m_key_lease_period", OPT_OFFSET(m_key_lease_period), opts_parse_net16, NULL, 1 }, { "sweep_interval", OPT_OFFSET(sweep_interval), opts_parse_uint32, NULL, 1 }, { "max_wire_smps", OPT_OFFSET(max_wire_smps), opts_parse_uint32, NULL, 1 }, + { "max_smps_per_node", OPT_OFFSET(max_smps_per_node), opts_parse_uint32, NULL, 1 }, { "console", OPT_OFFSET(console), opts_parse_charp, NULL, 0 }, { "console_port", OPT_OFFSET(console_port), opts_parse_uint16, NULL, 0 }, { "transaction_timeout", OPT_OFFSET(transaction_timeout), opts_parse_uint32, NULL, 1 }, @@ -671,6 +672,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt) p_opt->m_key_lease_period = 0; p_opt->sweep_interval = OSM_DEFAULT_SWEEP_INTERVAL_SECS; p_opt->max_wire_smps = OSM_DEFAULT_SMP_MAX_ON_WIRE; + p_opt->max_smps_per_node = OSM_DEFAULT_SMP_MAX_PER_NODE; p_opt->console = strdup(OSM_DEFAULT_CONSOLE); p_opt->console_port = OSM_DEFAULT_CONSOLE_PORT; p_opt->transaction_timeout = OSM_DEFAULT_TRANS_TIMEOUT_MILLISEC; @@ -1461,6 +1463,10 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) "max_wire_smps %u\n\n" "# The maximum time in [msec] allowed for a transaction to complete\n" "transaction_timeout %u\n\n" + "# Maximum number of SMPs per node sent in parallel\n" + "# (0 means unlimited)\n" + "# Only applies to certain attributes\n" + "max_smps_per_node %u\n\n" "# Maximal time in [msec] a message can stay in the incoming message queue.\n" "# If there is more than one message in the queue and the last message\n" "# stayed in the queue more than this value, any SA request will be\n" @@ -1470,6 +1476,7 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts) "single_thread %s\n\n", p_opts->max_wire_smps, p_opts->transaction_timeout, + p_opts->max_smps_per_node, p_opts->max_msg_fifo_timeout, p_opts->single_thread ? "TRUE" : "FALSE"); diff --git a/opensm/opensm/osm_ucast_cache.c b/opensm/opensm/osm_ucast_cache.c index 216b496..31c930b 100644 --- a/opensm/opensm/osm_ucast_cache.c +++ b/opensm/opensm/osm_ucast_cache.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2008,2009 Mellanox Technologies LTD. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -1085,9 +1085,11 @@ int osm_ucast_cache_process(osm_ucast_mgr_t * p_mgr) memset(p_sw->lft, OSM_NO_PATH, IB_LID_UCAST_END_HO + 1); } - osm_ucast_mgr_set_fwd_table(p_mgr, p_sw); + osm_ucast_mgr_set_fwd_tbl_top(p_mgr, p_sw); } + osm_ucast_pipeline_tbl(p_mgr); + return 0; } diff --git a/opensm/opensm/osm_ucast_file.c b/opensm/opensm/osm_ucast_file.c index 2505c46..099e8ba 100644 --- a/opensm/opensm/osm_ucast_file.c +++ b/opensm/opensm/osm_ucast_file.c @@ -168,8 +168,8 @@ static int do_ucast_file_load(void *context) "routing algorithm\n"); } else if (!strncmp(p, "Unicast lids", 12)) { if (p_sw) - osm_ucast_mgr_set_fwd_table(&p_osm->sm. - ucast_mgr, p_sw); + osm_ucast_mgr_set_fwd_tbl_top(&p_osm->sm. + ucast_mgr, p_sw); q = strstr(p, " guid 0x"); if (!q) { OSM_LOG(&p_osm->log, OSM_LOG_ERROR, @@ -247,7 +247,7 @@ static int do_ucast_file_load(void *context) } if (p_sw) - osm_ucast_mgr_set_fwd_table(&p_osm->sm.ucast_mgr, p_sw); + osm_ucast_mgr_set_fwd_tbl_top(&p_osm->sm.ucast_mgr, p_sw); fclose(file); return 0; diff --git a/opensm/opensm/osm_ucast_ftree.c b/opensm/opensm/osm_ucast_ftree.c index bde6dbd..d65c685 100644 --- a/opensm/opensm/osm_ucast_ftree.c +++ b/opensm/opensm/osm_ucast_ftree.c @@ -2,7 +2,7 @@ * Copyright (c) 2009 Simula Research Laboratory. All rights reserved. * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2007 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -1905,8 +1905,8 @@ static void set_sw_fwd_table(IN cl_map_item_t * const p_map_item, ftree_fabric_t *p_ftree = (ftree_fabric_t *) context; p_sw->p_osm_sw->max_lid_ho = p_ftree->lft_max_lid; - osm_ucast_mgr_set_fwd_table(&p_ftree->p_osm->sm.ucast_mgr, - p_sw->p_osm_sw); + osm_ucast_mgr_set_fwd_tbl_top(&p_ftree->p_osm->sm.ucast_mgr, + p_sw->p_osm_sw); } /*************************************************** @@ -4005,6 +4005,8 @@ static int do_routing(IN void *context) /* for each switch, set its fwd table */ cl_qmap_apply_func(&p_ftree->sw_tbl, set_sw_fwd_table, (void *)p_ftree); + osm_ucast_pipeline_tbl(&p_ftree->p_osm->sm.ucast_mgr); + /* write out hca ordering file */ fabric_dump_hca_ordering(p_ftree); diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 12b5e34..adf5f6c 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2007 Simula Research Laboratory. All rights reserved. * Copyright (c) 2007 Silicon Graphics Inc. All rights reserved. @@ -1045,8 +1045,11 @@ static void populate_fwd_tbls(lash_t * p_lash) physical_egress_port); } } /* for */ - osm_ucast_mgr_set_fwd_table(&p_osm->sm.ucast_mgr, p_sw); + osm_ucast_mgr_set_fwd_tbl_top(&p_osm->sm.ucast_mgr, p_sw); } + + osm_ucast_pipeline_tbl(&p_osm->sm.ucast_mgr); + OSM_LOG_EXIT(p_log); } diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c index 78a7031..86d1c98 100644 --- a/opensm/opensm/osm_ucast_mgr.c +++ b/opensm/opensm/osm_ucast_mgr.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. - * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2002-2009 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two @@ -315,16 +315,14 @@ Exit: /********************************************************************** **********************************************************************/ -int osm_ucast_mgr_set_fwd_table(IN osm_ucast_mgr_t * p_mgr, - IN osm_switch_t * p_sw) +int osm_ucast_mgr_set_fwd_tbl_top(IN osm_ucast_mgr_t * p_mgr, + IN osm_switch_t * p_sw) { osm_node_t *p_node; osm_dr_path_t *p_path; osm_madw_context_t context; ib_api_status_t status; ib_switch_info_t si; - uint16_t block_id_ho = 0; - uint8_t block[IB_SMP_DATA_SIZE]; boolean_t set_swinfo_require = FALSE; uint16_t lin_top; uint8_t life_state; @@ -382,48 +380,8 @@ int osm_ucast_mgr_set_fwd_table(IN osm_ucast_mgr_t * p_mgr, ib_get_err_str(status)); } - /* - Send linear forwarding table blocks to the switch - as long as the switch indicates it has blocks needing - configuration. - */ - - context.lft_context.node_guid = osm_node_get_node_guid(p_node); - context.lft_context.set_method = TRUE; - - if (!p_sw->new_lft) { - /* any routing should provide the new_lft */ - CL_ASSERT(p_mgr->p_subn->opt.use_ucast_cache && - p_mgr->cache_valid && !p_sw->need_update); - goto Exit; - } - - for (block_id_ho = 0; - osm_switch_get_lft_block(p_sw, block_id_ho, block); - block_id_ho++) { - if (!p_sw->need_update && !p_mgr->p_subn->need_update && - !memcmp(block, - p_sw->new_lft + block_id_ho * IB_SMP_DATA_SIZE, - IB_SMP_DATA_SIZE)) - continue; - - OSM_LOG(p_mgr->p_log, OSM_LOG_DEBUG, - "Writing FT block %u\n", block_id_ho); - - status = osm_req_set(p_mgr->sm, p_path, - p_sw->new_lft + - block_id_ho * IB_SMP_DATA_SIZE, - sizeof(block), IB_MAD_ATTR_LIN_FWD_TBL, - cl_hton32(block_id_ho), CL_DISP_MSGID_NONE, - &context); + p_sw->lft_block_id_ho = 0; - if (status != IB_SUCCESS) - OSM_LOG(p_mgr->p_log, OSM_LOG_ERROR, "ERR 3A05: " - "Sending linear fwd. tbl. block failed (%s)\n", - ib_get_err_str(status)); - } - -Exit: OSM_LOG_EXIT(p_mgr->p_log); return 0; } @@ -508,7 +466,7 @@ static void ucast_mgr_process_tbl(IN cl_map_item_t * p_map_item, } } - osm_ucast_mgr_set_fwd_table(p_mgr, p_sw); + osm_ucast_mgr_set_fwd_tbl_top(p_mgr, p_sw); if (p_mgr->p_subn->opt.lmc) free_ports_priv(p_mgr); @@ -516,6 +474,47 @@ static void ucast_mgr_process_tbl(IN cl_map_item_t * p_map_item, OSM_LOG_EXIT(p_mgr->p_log); } +static void ucast_mgr_pipeline_tbl(IN osm_switch_t *p_sw, + IN osm_ucast_mgr_t *p_mgr) +{ + osm_dr_path_t *p_path; + osm_madw_context_t mad_context; + uint8_t block[IB_SMP_DATA_SIZE]; + + OSM_LOG_ENTER(p_mgr->p_log); + + CL_ASSERT(p_sw && p_sw->p_node); + + OSM_LOG(p_mgr->p_log, OSM_LOG_DEBUG, + "Processing switch 0x%" PRIx64 "\n", + cl_ntoh64(osm_node_get_node_guid(p_sw->p_node))); + + /* + Send linear forwarding table blocks to the switch + as long as the switch indicates it has blocks needing + configuration. + */ + if (!p_sw->new_lft) { + /* any routing should provide the new_lft */ + CL_ASSERT(p_mgr->p_subn->opt.use_ucast_cache && + p_mgr->cache_valid && !p_sw->need_update); + goto Exit; + } + + p_path = osm_physp_get_dr_path_ptr(osm_node_get_physp_ptr(p_sw->p_node, 0)); + + mad_context.lft_context.node_guid = osm_node_get_node_guid(p_sw->p_node); + mad_context.lft_context.set_method = TRUE; + + osm_sm_set_next_lft_block(p_mgr->sm, p_sw, &block[0], p_path, + &mad_context); + + p_sw->lft_block_id_ho++; + +Exit: + OSM_LOG_EXIT(p_mgr->p_log); +} + /********************************************************************** **********************************************************************/ static void ucast_mgr_process_neighbors(IN cl_map_item_t * p_map_item, @@ -870,6 +869,28 @@ static void sort_ports_by_switch_load(osm_ucast_mgr_t * m) add_sw_endports_to_order_list(s[i], m); } +void osm_ucast_pipeline_tbl(osm_ucast_mgr_t * p_mgr) +{ + cl_qmap_t *p_sw_tbl; + osm_switch_t *p_sw; + int i; + + for (i = 0; + !p_mgr->p_subn->opt.max_smps_per_node || + i < p_mgr->p_subn->opt.max_smps_per_node; + i++) { + p_mgr->sm->lfts_updated = 0; + p_sw_tbl = &p_mgr->p_subn->sw_guid_tbl; + p_sw = (osm_switch_t *) cl_qmap_head(p_sw_tbl); + while (p_sw != (osm_switch_t *) cl_qmap_end(p_sw_tbl)) { + ucast_mgr_pipeline_tbl(p_sw, p_mgr); + p_sw = (osm_switch_t *) cl_qmap_next(&p_sw->map_item); + } + if (!p_mgr->sm->lfts_updated) + break; + } +} + static int ucast_mgr_build_lfts(osm_ucast_mgr_t * p_mgr) { cl_qlist_init(&p_mgr->port_order_list); @@ -904,6 +925,8 @@ static int ucast_mgr_build_lfts(osm_ucast_mgr_t * p_mgr) cl_qmap_apply_func(&p_mgr->p_subn->sw_guid_tbl, ucast_mgr_process_tbl, p_mgr); + osm_ucast_pipeline_tbl(p_mgr); + cl_qlist_remove_all(&p_mgr->port_order_list); return 0; From weiny2 at llnl.gov Thu Jul 30 18:40:54 2009 From: weiny2 at llnl.gov (Ira Weiny) Date: Thu, 30 Jul 2009 18:40:54 -0700 Subject: [ofa-general] [PATCH] infiniband-diags/libibnetdisc: fix DBGFLAGS variable Message-ID: <20090730184054.1a6f9f25.weiny2@llnl.gov> From: Ira Weiny Date: Thu, 30 Jul 2009 18:40:12 -0700 Subject: [PATCH] infiniband-diags/libibnetdisc: fix DBGFLAGS variable Signed-off-by: Ira Weiny --- infiniband-diags/libibnetdisc/Makefile.am | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/infiniband-diags/libibnetdisc/Makefile.am b/infiniband-diags/libibnetdisc/Makefile.am index ddcf053..c10c972 100644 --- a/infiniband-diags/libibnetdisc/Makefile.am +++ b/infiniband-diags/libibnetdisc/Makefile.am @@ -10,7 +10,11 @@ if ENABLE_TEST_UTILS sbin_PROGRAMS += test/testleaks endif -DBGFLAGS = -g +if DEBUG +DBGFLAGS = -ggdb -D_DEBUG_ +else +DBGFLAGS = +endif if HAVE_LD_VERSION_SCRIPT libibnetdisc_version_script = -Wl,--version-script=$(srcdir)/src/libibnetdisc.map -- 1.5.4.5 From swise at opengridcomputing.com Thu Jul 30 21:38:32 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Thu, 30 Jul 2009 23:38:32 -0500 Subject: [ofa-general] Question about addr_resolve_remote In-Reply-To: References: <5ac865d20907301127u7a2c7c83oa645c23576dc116c@mail.gmail.com> Message-ID: <4A727548.2010201@opengridcomputing.com> Roland Dreier wrote: > > Do you think it would be a problem if I changed > > neigh = neigh_lookup(&arp_tbl, &rt->rt_gateway, rt->idev->dev); > > to > > neigh = neigh_lookup(&arp_tbl, &rt->rt_dst, rt->idev->dev); > > Yes I think it would be a problem, because what if you are trying to > connect to a destination that is behind a router? (and keep in mind, in > your example, it is perfectly correct to try and connect to 4.5.7.8 if > it happens to be reachable via a router from your RNIC of 192.168.0.1) > > - R. > In other words, the neigh entry and its state pertain to the next hop, not the final destination (unless the next hop -is- the final destination). And if you really are trying to get traffic to 4.5.7.8 via your rnic, you need to make sure the linux routing table is setup to provide that route. Based on your example, your local routing table is setup such that 4.5.7.8 is reachable only via your NIC interface (maybe via the default route?). Steve. From vlad at lists.openfabrics.org Fri Jul 31 02:56:59 2009 From: vlad at lists.openfabrics.org (Vladimir Sokolovsky Mellanox) Date: Fri, 31 Jul 2009 02:56:59 -0700 (PDT) Subject: [ofa-general] ofa_1_5_kernel 20090731-0200 daily build status Message-ID: <20090731095700.357A4E61CBE@openfabrics.org> This email was generated automatically, please do not reply git_url: git://git.openfabrics.org/ofed_1_5/linux-2.6.git git_branch: ofed_kernel_1_5 Common build parameters: Passed: Passed on i686 with linux-2.6.18 Passed on i686 with linux-2.6.21.1 Passed on i686 with linux-2.6.19 Passed on i686 with linux-2.6.26 Passed on i686 with linux-2.6.24 Passed on i686 with linux-2.6.22 Passed on i686 with linux-2.6.27 Passed on x86_64 with linux-2.6.18 Passed on x86_64 with linux-2.6.19 Passed on x86_64 with linux-2.6.20 Passed on x86_64 with linux-2.6.21.1 Passed on x86_64 with linux-2.6.22 Passed on x86_64 with linux-2.6.26 Passed on x86_64 with linux-2.6.24 Passed on x86_64 with linux-2.6.25 Passed on x86_64 with linux-2.6.27 Passed on ia64 with linux-2.6.18 Passed on ia64 with linux-2.6.21.1 Passed on ia64 with linux-2.6.19 Passed on ia64 with linux-2.6.22 Passed on ia64 with linux-2.6.23 Passed on ia64 with linux-2.6.24 Passed on ia64 with linux-2.6.25 Passed on ia64 with linux-2.6.26 Passed on ppc64 with linux-2.6.18 Passed on ppc64 with linux-2.6.19 Failed: Build failed on x86_64 with linux-2.6.16.60-0.21-smp Log: /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.16.60-0.21-smp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.16.60-0.21-smp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.16.60-0.21-smp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-128.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-128.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-128.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-128.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.18-93.el5 Log: /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:262: error: 'struct net_device' has no member named 'stats' /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-93.el5_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.18-93.el5_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.18-93.el5' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-78.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-78.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-78.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-78.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- Build failed on x86_64 with linux-2.6.9-67.ELsmp Log: /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:375: warning: pointer targets in assignment differ in signedness /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c: In function 'vnic_get_stats': /home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.c:214: warning: control reaches end of non-void function make[4]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic/vnic_main.o] Error 1 make[3]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband/ulp/qlgc_vnic] Error 2 make[2]: *** [/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-67.ELsmp_x86_64_check/drivers/infiniband] Error 2 make[1]: *** [_module_/home/vlad/tmp/ofa_1_5_kernel-20090731-0200_linux-2.6.9-67.ELsmp_x86_64_check] Error 2 make[1]: Leaving directory `/home/vlad/kernel.org/x86_64/linux-2.6.9-67.ELsmp' make: *** [kernel] Error 2 ---------------------------------------------------------------------------------- From hnrose at comcast.net Fri Jul 31 06:53:16 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Fri, 31 Jul 2009 09:53:16 -0400 Subject: [ofa-general] [PATCH] opensm: Change ib_smp_init_new to return success/failure status Message-ID: <20090731135316.GB10365@comcast.net> based on valid/invalid hop count rather than relying on debug assert Handle invalid status appropriate in callers of ib_smp_init_new Signed-off-by: Hal Rosenstock --- diff --git a/opensm/include/iba/ib_types.h b/opensm/include/iba/ib_types.h index beb7492..6668d96 100644 --- a/opensm/include/iba/ib_types.h +++ b/opensm/include/iba/ib_types.h @@ -4091,11 +4092,11 @@ static inline boolean_t OSM_API ib_smp_is_d(IN const ib_smp_t * const p_smp) * * TODO * This is too big for inlining, but leave it here for now -* since there is not yet another convient spot. +* since there is not yet another convenient spot. * * SYNOPSIS */ -static inline void OSM_API +static inline boolean_t OSM_API ib_smp_init_new(IN ib_smp_t * const p_smp, IN const uint8_t method, IN const ib_net64_t trans_id, @@ -4107,7 +4108,9 @@ ib_smp_init_new(IN ib_smp_t * const p_smp, IN const ib_net16_t dr_slid, IN const ib_net16_t dr_dlid) { CL_ASSERT(p_smp); - CL_ASSERT(hop_count < IB_SUBNET_PATH_HOPS_MAX); + + if (hop_count >= IB_SUBNET_PATH_HOPS_MAX) + return FALSE; p_smp->base_ver = 1; p_smp->mgmt_class = IB_MCLASS_SUBN_DIR; p_smp->class_ver = 1; @@ -4130,6 +4133,7 @@ ib_smp_init_new(IN ib_smp_t * const p_smp, /* copy the path */ memcpy(&p_smp->initial_path, path_out, sizeof(p_smp->initial_path)); + return TRUE; } /* diff --git a/opensm/opensm/osm_req.c b/opensm/opensm/osm_req.c index be9a92b..7934173 100644 --- a/opensm/opensm/osm_req.c +++ b/opensm/opensm/osm_req.c @@ -102,14 +102,21 @@ osm_req_get(IN osm_sm_t * sm, ib_get_sm_attr_str(attr_id), cl_ntoh16(attr_id), cl_ntoh32(attr_mod), cl_ntoh64(tid)); - ib_smp_init_new(osm_madw_get_smp_ptr(p_madw), - IB_MAD_METHOD_GET, - tid, - attr_id, - attr_mod, - p_path->hop_count, - sm->p_subn->opt.m_key, - p_path->path, IB_LID_PERMISSIVE, IB_LID_PERMISSIVE); + if (!ib_smp_init_new(osm_madw_get_smp_ptr(p_madw), + IB_MAD_METHOD_GET, + tid, + attr_id, + attr_mod, + p_path->hop_count, + sm->p_subn->opt.m_key, + p_path->path, + IB_LID_PERMISSIVE, IB_LID_PERMISSIVE)) { + OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 1108: " + "ib_smp_init_new failed: hop count %d\n", + p_path->hop_count); + status = IB_INVALID_PARAMETER; + goto Exit; + } p_madw->mad_addr.dest_lid = IB_LID_PERMISSIVE; p_madw->mad_addr.addr_type.smi.source_lid = IB_LID_PERMISSIVE; @@ -180,14 +187,21 @@ osm_req_set(IN osm_sm_t * sm, ib_get_sm_attr_str(attr_id), cl_ntoh16(attr_id), cl_ntoh32(attr_mod), cl_ntoh64(tid)); - ib_smp_init_new(osm_madw_get_smp_ptr(p_madw), - IB_MAD_METHOD_SET, - tid, - attr_id, - attr_mod, - p_path->hop_count, - sm->p_subn->opt.m_key, - p_path->path, IB_LID_PERMISSIVE, IB_LID_PERMISSIVE); + if (!ib_smp_init_new(osm_madw_get_smp_ptr(p_madw), + IB_MAD_METHOD_SET, + tid, + attr_id, + attr_mod, + p_path->hop_count, + sm->p_subn->opt.m_key, + p_path->path, + IB_LID_PERMISSIVE, IB_LID_PERMISSIVE)) { + OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 1107: " + "ib_smp_init_new failed: hop count %d\n", + p_path->hop_count); + status = IB_INVALID_PARAMETER; + goto Exit; + } p_madw->mad_addr.dest_lid = IB_LID_PERMISSIVE; p_madw->mad_addr.addr_type.smi.source_lid = IB_LID_PERMISSIVE; From hnrose at comcast.net Fri Jul 31 06:51:47 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Fri, 31 Jul 2009 09:51:47 -0400 Subject: [ofa-general] [PATCHv2] opensm/mesh/lash: Fix use after free problem in osm_mesh_node_delete Message-ID: <20090731135147.GA10365@comcast.net> When osm_mesh_node_delete is called, osm_switch_delete may already have been called so sw->p_sw is no longer valid to be used although it was being used to obtain num_ports. Fix this by performing osm_mesh_delete_switches at the end of lash_process. Signed-off-by: Hal Rosenstock --- Changes since v1: Rather than saving num_ports in the mesh node structure on creation and using this on deletion, mesh switches deletion should occur at end of the lash calculation as none of this state is needed after that Approach proposed by Sasha diff --git a/opensm/include/opensm/osm_mesh.h b/opensm/include/opensm/osm_mesh.h index 173fa86..89c07e5 100644 --- a/opensm/include/opensm/osm_mesh.h +++ b/opensm/include/opensm/osm_mesh.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2088 System Fabric Works, Inc. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -70,6 +71,7 @@ typedef struct _mesh_node { } mesh_node_t; void osm_mesh_node_delete(struct _lash *p_lash, struct _switch *sw); +void osm_mesh_delete_switches(struct _lash *p_lash); int osm_mesh_node_create(struct _lash *p_lash, struct _switch *sw); int osm_do_mesh_analysis(struct _lash *p_lash); diff --git a/opensm/opensm/osm_mesh.c b/opensm/opensm/osm_mesh.c index 23fad87..b22fe6e 100644 --- a/opensm/opensm/osm_mesh.c +++ b/opensm/opensm/osm_mesh.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2008,2009 System Fabric Works, Inc. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -1358,6 +1359,20 @@ void osm_mesh_node_delete(lash_t *p_lash, switch_t *sw) } /* + * osm_mesh_delete_switches - cleanup switches resources + */ +void osm_mesh_delete_switches(lash_t *p_lash) +{ + if (p_lash->switches) { + unsigned id; + for (id = 0; ((int)id) < p_lash->num_switches; id++) + if (p_lash->switches[id]) + osm_mesh_node_delete(p_lash, + p_lash->switches[id]); + } +} + +/* * osm_mesh_node_create - allocate per switch resources */ int osm_mesh_node_create(lash_t *p_lash, switch_t *sw) diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 7133e25..2b50fe5 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -5,6 +5,7 @@ * Copyright (c) 2007 Simula Research Laboratory. All rights reserved. * Copyright (c) 2007 Silicon Graphics Inc. All rights reserved. * Copyright (c) 2008,2009 System Fabric Works, Inc. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -1208,7 +1209,7 @@ static int lash_process(void *context) return_status = discover_network_properties(p_lash); if (return_status != IB_SUCCESS) - goto Exit; + goto Exit2; return_status = init_lash_structures(p_lash); if (return_status != IB_SUCCESS) @@ -1223,6 +1224,9 @@ static int lash_process(void *context) populate_fwd_tbls(p_lash); Exit: + osm_mesh_delete_switches(p_lash); + +Exit2: if (p_lash->vl_min) free_lash_structures(p_lash); OSM_LOG_EXIT(p_log); From hnrose at comcast.net Fri Jul 31 07:57:29 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Fri, 31 Jul 2009 10:57:29 -0400 Subject: [ofa-general] [PATCH] libibmad/mad.c: Validate DR hop count for SMPs and handle errors Message-ID: <20090731145729.GA24420@comcast.net> In mad_encode, validate DR hop count for SMPs In mad_build_pkt, handle mad_encode error Signed-off-by: Hal Rosenstock --- diff --git a/libibmad/src/mad.c b/libibmad/src/mad.c index 3f04da0..89ff978 100644 --- a/libibmad/src/mad.c +++ b/libibmad/src/mad.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2004-2007 Voltaire Inc. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -79,6 +80,10 @@ void *mad_encode(void *buf, ib_rpc_t * rpc, ib_dr_path_t * drpath, void *data) IBWARN("encoding dr mad without drpath (null)"); return 0; } + if (drpath->cnt >= IB_SUBNET_PATH_HOPS_MAX) { + IBWARN("dr path with hop count %d", drpath->cnt); + return 0; + } mad_set_field(buf, 0, IB_DRSMP_HOPCNT_F, drpath->cnt); mad_set_field(buf, 0, IB_DRSMP_HOPPTR_F, is_resp ? drpath->cnt + 1 : 0x0); @@ -157,6 +162,8 @@ int mad_build_pkt(void *umad, ib_rpc_t * rpc, ib_portid_t * dport, mad = umad_get_mad(umad); p = mad_encode(mad, rpc, lid_routed ? 0 : &dport->drpath, data); + if (!p) + return -1; if (!is_smi && rmpp) { mad_set_field(mad, 0, IB_SA_RMPP_VERS_F, 1); From yosefe at voltaire.com Fri Jul 31 09:13:12 2009 From: yosefe at voltaire.com (Yossi Etigin) Date: Fri, 31 Jul 2009 19:13:12 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> <4A705B3A.7060404@Voltaire.COM> Message-ID: <4A731818.3060500@voltaire.com> On 29/07/09 19:35, Hal Rosenstock wrote: > > > On Wed, Jul 29, 2009 at 10:22 AM, Moni Shoua > wrote: > > Hal Rosenstock wrote: > > > > > > On Wed, Jul 29, 2009 at 8:16 AM, Moni Shoua > > >> wrote: > > > > > > > > Is the LMC in the above from the local port ? > Unfortunately, LMC > > > is not > > > > required to be uniform across the subnet so the remote > > port's LMC may > > > > not be the same as that on the local port. > > > > > > opensm configures the same LMC to all endports (CA) in the > > fabric so > > > in which case do you suspect that > > > it will be different? > > > > > > > > > I wasn't talking about the simplification that OpenSM uses > but rather > > > what IBA allows. There may be other SMs which do this or OpenSM > > could be > > > extended (with additional configuration for this). > > > > > > -- Hal > > > > So I guess a possible solutions for that is to query each > suspected > > node before making a decision to flush it. > > When getting the node info response the true LMC can be put in the > > LID check function. > > > > > > I'm not following how you are saying to determine the true LMC (of the > > remote port). > > > Send PortInfo query for that port > > > from the IPoIB client ? I think this ends up needing to contact the SA > to get this info rather than the port directly. Isn't that what you were > trying to avoid originally ? If that's the case, one way would've been > if ARPs carried the LMC as well as the LID. > What if we query the remote port LMC once, when the path is resolved, and then use it to mask the LID until the path is refreshed again? From sashak at voltaire.com Fri Jul 31 09:55:19 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Fri, 31 Jul 2009 19:55:19 +0300 Subject: [ofa-general] Re: [PATCH] opensm/osm_lid_mgr.c: Trivial simplification in osm_lid_mgr_process_subnet In-Reply-To: <20090730175551.GA19618@comcast.net> References: <20090730175551.GA19618@comcast.net> Message-ID: <20090731165519.GE24822@me> Hi Hal, On 13:55 Thu 30 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock > --- > diff --git a/opensm/opensm/osm_lid_mgr.c b/opensm/opensm/osm_lid_mgr.c > index 34625ba..64157e0 100644 > --- a/opensm/opensm/osm_lid_mgr.c > +++ b/opensm/opensm/osm_lid_mgr.c > @@ -1228,9 +1228,8 @@ int osm_lid_mgr_process_subnet(IN osm_lid_mgr_t * p_mgr) > min_lid_ho, max_lid_ho); > > /* the proc returns the fact it sent a set port info */ > - if (lid_mgr_set_physp_pi(p_mgr, p_port, p_port->p_physp, > - cl_hton16(min_lid_ho))) > - ret = -1; > + ret = lid_mgr_set_physp_pi(p_mgr, p_port, p_port->p_physp, > + cl_hton16(min_lid_ho)); Actually it is not correct (and I tried to explain this to you some time ago) - subsequent successive PortInfo set run will potentially overwrite a previous failure status. Sasha From todd.rimmer at qlogic.com Fri Jul 31 09:58:13 2009 From: todd.rimmer at qlogic.com (Todd Rimmer) Date: Fri, 31 Jul 2009 11:58:13 -0500 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A731818.3060500@voltaire.com> References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> <4A705B3A.7060404@Voltaire.COM> <4A731818.3060500@voltaire.com> Message-ID: <5AEC2602AE03EB46BFC16C6B9B200DA81653EF68BF@MNEXMB2.qlogic.org> > Yossi Etigin wrote: > What if we query the remote port LMC once, when the path is resolved, > and then > use it to mask the LID until the path is refreshed again? Doing the LMC query when the PathRecord query is made makes, sense. I assume you are proposing a query of the SA for the PortInfoRecord? Note that a direct query of the remote SMA would be a bad idea and has at least the following issues: 1. There are limited VL15 queuing resources and no VL15 flow control, hence if multiple nodes queried the same SMA at the same time, packet loss could be significant which would impact SM operation as well (note how carefully the recent opensm work for parallel queries has had to tune operations to SMAs) 2. If MKey security is enabled, the client will not (and should not) know the MKey of the remote SMA and hence can't make a query. 3. The intent of the IB Architecture is for the SM to be the only entity which interacts with SMAs. A handful of diagnostic tools (like ibdiagnet) may be an acceptable exception, but it would not be recommended to make non-SM queries of an SMA as part of a normal protocol. Todd Rimmer Chief Architect QLogic Network Systems Group Voice: 610-233-4852 Fax: 610-233-4777 Todd.Rimmer at QLogic.com www.QLogic.com From sashak at voltaire.com Fri Jul 31 10:01:12 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Fri, 31 Jul 2009 20:01:12 +0300 Subject: [ofa-general] Re: [PATCHv2] opensm/osm_ucast_lash.c: Handle malloc failures better In-Reply-To: <20090730175457.GA19613@comcast.net> References: <20090730175457.GA19613@comcast.net> Message-ID: <20090731170112.GF24822@me> On 13:54 Thu 30 Jul , Hal Rosenstock wrote: > @@ -800,7 +806,11 @@ static int lash_core(lash_t * p_lash) > for (i = 0; i < num_switches; i++) { > > shortest_path(p_lash, i); > - generate_routing_func_for_mst(p_lash, i, &dests); > + if (generate_routing_func_for_mst(p_lash, i, &dests)) { > + OSM_LOG(p_log, OSM_LOG_ERROR, > + "generate_routing_func_for_mst failed\n"); > + goto Exit; In this case lash_core() will return value of 'status' variable which is zero and indicates a success. No? Sasha > + } > > idest = dests; > while (idest != NULL) { > From hnrose at comcast.net Fri Jul 31 10:36:40 2009 From: hnrose at comcast.net (Hal Rosenstock) Date: Fri, 31 Jul 2009 13:36:40 -0400 Subject: [ofa-general] [PATCHv3] opensm/osm_ucast_lash.c: Handle malloc failures better Message-ID: <20090731173640.GA25877@comcast.net> Signed-off-by: Hal Rosenstock --- Changes since v2: In lash_core, return error status when generate_routing_func_for_mst fails Changes since v1: Changed generate_routing_func_for_mst to return int (rather than boolean) Also, simplified malloc failure handling in that routine Both pointers from Sasha diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c index 7133e25..1c55a90 100644 --- a/opensm/opensm/osm_ucast_lash.c +++ b/opensm/opensm/osm_ucast_lash.c @@ -296,8 +296,8 @@ static void shortest_path(lash_t * p_lash, int ir) cl_list_destroy(&bfsq); } -static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, - reachable_dest_t ** destinations) +static int generate_routing_func_for_mst(lash_t * p_lash, int sw_id, + reachable_dest_t ** destinations) { int i, next_switch; switch_t *sw = p_lash->switches[sw_id]; @@ -306,7 +306,8 @@ static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, for (i = 0; i < num_channels; i++) { next_switch = sw->dij_channels[i]; - generate_routing_func_for_mst(p_lash, next_switch, &dest); + if (generate_routing_func_for_mst(p_lash, next_switch, &dest)) + return -1; i_dest = dest; prev = i_dest; @@ -327,9 +328,12 @@ static void generate_routing_func_for_mst(lash_t * p_lash, int sw_id, } i_dest = (reachable_dest_t *) malloc(sizeof(reachable_dest_t)); + if (!i_dest) + return -1; i_dest->switch_id = sw->id; i_dest->next = concat_dest; *destinations = i_dest; + return 0; } static void generate_cdg_for_sp(lash_t * p_lash, int sw, int dest_switch, @@ -707,6 +711,8 @@ static int init_lash_structures(lash_t * p_lash) /* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */ p_lash->cdg_vertex_matrix = (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****)); + if (p_lash->cdg_vertex_matrix == NULL) + goto Exit_Mem_Error; for (i = 0; i < vl_min; i++) { p_lash->cdg_vertex_matrix[i] = (cdg_vertex_t ***) malloc(num_switches * @@ -800,7 +806,12 @@ static int lash_core(lash_t * p_lash) for (i = 0; i < num_switches; i++) { shortest_path(p_lash, i); - generate_routing_func_for_mst(p_lash, i, &dests); + if (generate_routing_func_for_mst(p_lash, i, &dests)) { + status = -1; + OSM_LOG(p_log, OSM_LOG_ERROR, + "generate_routing_func_for_mst failed\n"); + goto Exit; + } idest = dests; while (idest != NULL) { From hal.rosenstock at gmail.com Fri Jul 31 10:50:51 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Fri, 31 Jul 2009 13:50:51 -0400 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A731818.3060500@voltaire.com> References: <4A6DDFCE.9060009@voltaire.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> <4A705B3A.7060404@Voltaire.COM> <4A731818.3060500@voltaire.com> Message-ID: On Fri, Jul 31, 2009 at 12:13 PM, Yossi Etigin wrote: > On 29/07/09 19:35, Hal Rosenstock wrote: > > > > > > On Wed, Jul 29, 2009 at 10:22 AM, Moni Shoua > > wrote: > > > > Hal Rosenstock wrote: > > > > > > > > > On Wed, Jul 29, 2009 at 8:16 AM, Moni Shoua > > > > >> wrote: > > > > > > > > > > > Is the LMC in the above from the local port ? > > Unfortunately, LMC > > > > is not > > > > > required to be uniform across the subnet so the remote > > > port's LMC may > > > > > not be the same as that on the local port. > > > > > > > > opensm configures the same LMC to all endports (CA) in > the > > > fabric so > > > > in which case do you suspect that > > > > it will be different? > > > > > > > > > > > > I wasn't talking about the simplification that OpenSM uses > > but rather > > > > what IBA allows. There may be other SMs which do this or > OpenSM > > > could be > > > > extended (with additional configuration for this). > > > > > > > > -- Hal > > > > > > So I guess a possible solutions for that is to query each > > suspected > > > node before making a decision to flush it. > > > When getting the node info response the true LMC can be put in > the > > > LID check function. > > > > > > > > > I'm not following how you are saying to determine the true LMC (of > the > > > remote port). > > > > > Send PortInfo query for that port > > > > > > from the IPoIB client ? I think this ends up needing to contact the SA > > to get this info rather than the port directly. Isn't that what you were > > trying to avoid originally ? If that's the case, one way would've been > > if ARPs carried the LMC as well as the LID. > > > > What if we query the remote port LMC once, when the path is resolved, and > then > use it to mask the LID until the path is refreshed again? Just like the LID, remote LMC could change. -------------- next part -------------- An HTML attachment was scrubbed... URL: From yosefe at voltaire.com Fri Jul 31 11:51:16 2009 From: yosefe at voltaire.com (Yossi Etigin) Date: Fri, 31 Jul 2009 21:51:16 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: References: <4A6DDFCE.9060009@voltaire.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> <4A705B3A.7060404@Voltaire.COM> <4A731818.3060500@voltaire.com> Message-ID: <4A733D24.3040201@voltaire.com> On 31/07/09 20:50, Hal Rosenstock wrote: > > > On Fri, Jul 31, 2009 at 12:13 PM, Yossi Etigin > wrote: > > On 29/07/09 19:35, Hal Rosenstock wrote: > > > > > > On Wed, Jul 29, 2009 at 10:22 AM, Moni Shoua > > >> wrote: > > > > Hal Rosenstock wrote: > > > > > > > > > On Wed, Jul 29, 2009 at 8:16 AM, Moni Shoua > > > > > > > > >>> wrote: > > > > > > > > > > > Is the LMC in the above from the local port ? > > Unfortunately, LMC > > > > is not > > > > > required to be uniform across the subnet so the > remote > > > port's LMC may > > > > > not be the same as that on the local port. > > > > > > > > opensm configures the same LMC to all endports > (CA) in the > > > fabric so > > > > in which case do you suspect that > > > > it will be different? > > > > > > > > > > > > I wasn't talking about the simplification that OpenSM uses > > but rather > > > > what IBA allows. There may be other SMs which do this > or OpenSM > > > could be > > > > extended (with additional configuration for this). > > > > > > > > -- Hal > > > > > > So I guess a possible solutions for that is to query each > > suspected > > > node before making a decision to flush it. > > > When getting the node info response the true LMC can be > put in the > > > LID check function. > > > > > > > > > I'm not following how you are saying to determine the true > LMC (of the > > > remote port). > > > > > Send PortInfo query for that port > > > > > > from the IPoIB client ? I think this ends up needing to contact the SA > > to get this info rather than the port directly. Isn't that what > you were > > trying to avoid originally ? If that's the case, one way would've been > > if ARPs carried the LMC as well as the LID. > > > > What if we query the remote port LMC once, when the path is > resolved, and then > use it to mask the LID until the path is refreshed again? > > > Just like the LID, remote LMC could change. > Yes, but AFAIK the only "bad" case is if the LID stays the same but LMC changes to a lower value. In this case the path refresh will not happen when it is supposed to. If LMC changes to a higher value it might trigger an unneeded path refresh, which is not bad because it will refresh the LMC as well. From yosefe at voltaire.com Fri Jul 31 11:56:00 2009 From: yosefe at voltaire.com (Yossi Etigin) Date: Fri, 31 Jul 2009 21:56:00 +0300 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <5AEC2602AE03EB46BFC16C6B9B200DA81653EF68BF@MNEXMB2.qlogic.org> References: <4A6DDFCE.9060009@voltaire.com> <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> <4A705B3A.7060404@Voltaire.COM> <4A731818.3060500@voltaire.com> <5AEC2602AE03EB46BFC16C6B9B200DA81653EF68BF@MNEXMB2.qlogic.org> Message-ID: <4A733E40.6090506@voltaire.com> On 31/07/09 19:58, Todd Rimmer wrote: >> Yossi Etigin wrote: > >> What if we query the remote port LMC once, when the path is resolved, >> and then >> use it to mask the LID until the path is refreshed again? > > Doing the LMC query when the PathRecord query is made makes, sense. > > I assume you are proposing a query of the SA for the PortInfoRecord? > > Note that a direct query of the remote SMA would be a bad idea and has at least the following issues: > 1. There are limited VL15 queuing resources and no VL15 flow control, hence if multiple nodes queried the same SMA at the same time, packet loss could be significant which would impact SM operation as well (note how carefully the recent opensm work for parallel queries has had to tune operations to SMAs) > 2. If MKey security is enabled, the client will not (and should not) know the MKey of the remote SMA and hence can't make a query. > 3. The intent of the IB Architecture is for the SM to be the only entity which interacts with SMAs. A handful of diagnostic tools (like ibdiagnet) may be an acceptable exception, but it would not be recommended to make non-SM queries of an SMA as part of a normal protocol. > Actually I wasn't sure which is better, but given all these issues SM should be the one and not the SA. From hal.rosenstock at gmail.com Fri Jul 31 12:01:58 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Fri, 31 Jul 2009 15:01:58 -0400 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A733E40.6090506@voltaire.com> References: <4A6DDFCE.9060009@voltaire.com> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> <4A705B3A.7060404@Voltaire.COM> <4A731818.3060500@voltaire.com> <5AEC2602AE03EB46BFC16C6B9B200DA81653EF68BF@MNEXMB2.qlogic.org> <4A733E40.6090506@voltaire.com> Message-ID: On Fri, Jul 31, 2009 at 2:56 PM, Yossi Etigin wrote: > On 31/07/09 19:58, Todd Rimmer wrote: > >> Yossi Etigin wrote: > > > >> What if we query the remote port LMC once, when the path is resolved, > >> and then > >> use it to mask the LID until the path is refreshed again? > > > > Doing the LMC query when the PathRecord query is made makes, sense. > > > > I assume you are proposing a query of the SA for the PortInfoRecord? > > > > Note that a direct query of the remote SMA would be a bad idea and has at > least the following issues: > > 1. There are limited VL15 queuing resources and no VL15 flow control, > hence if multiple nodes queried the same SMA at the same time, packet loss > could be significant which would impact SM operation as well (note how > carefully the recent opensm work for parallel queries has had to tune > operations to SMAs) > > 2. If MKey security is enabled, the client will not (and should not) know > the MKey of the remote SMA and hence can't make a query. > > 3. The intent of the IB Architecture is for the SM to be the only entity > which interacts with SMAs. A handful of diagnostic tools (like ibdiagnet) > may be an acceptable exception, but it would not be recommended to make > non-SM queries of an SMA as part of a normal protocol. > > > > Actually I wasn't sure which is better, but given all these issues SM > should be the one and not the SA. > > Do you mean SA rather than SM ? -------------- next part -------------- An HTML attachment was scrubbed... URL: From hal.rosenstock at gmail.com Fri Jul 31 12:05:07 2009 From: hal.rosenstock at gmail.com (Hal Rosenstock) Date: Fri, 31 Jul 2009 15:05:07 -0400 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A733D24.3040201@voltaire.com> References: <4A6DDFCE.9060009@voltaire.com> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> <4A705B3A.7060404@Voltaire.COM> <4A731818.3060500@voltaire.com> <4A733D24.3040201@voltaire.com> Message-ID: On Fri, Jul 31, 2009 at 2:51 PM, Yossi Etigin wrote: > On 31/07/09 20:50, Hal Rosenstock wrote: > > > > > > On Fri, Jul 31, 2009 at 12:13 PM, Yossi Etigin > > wrote: > > > > On 29/07/09 19:35, Hal Rosenstock wrote: > > > > > > > > > On Wed, Jul 29, 2009 at 10:22 AM, Moni Shoua > > > > >> wrote: > > > > > > Hal Rosenstock wrote: > > > > > > > > > > > > On Wed, Jul 29, 2009 at 8:16 AM, Moni Shoua > > > > > > > > > > > > >>> wrote: > > > > > > > > > > > > > > Is the LMC in the above from the local port ? > > > Unfortunately, LMC > > > > > is not > > > > > > required to be uniform across the subnet so the > > remote > > > > port's LMC may > > > > > > not be the same as that on the local port. > > > > > > > > > > opensm configures the same LMC to all endports > > (CA) in the > > > > fabric so > > > > > in which case do you suspect that > > > > > it will be different? > > > > > > > > > > > > > > > I wasn't talking about the simplification that OpenSM > uses > > > but rather > > > > > what IBA allows. There may be other SMs which do this > > or OpenSM > > > > could be > > > > > extended (with additional configuration for this). > > > > > > > > > > -- Hal > > > > > > > > So I guess a possible solutions for that is to query each > > > suspected > > > > node before making a decision to flush it. > > > > When getting the node info response the true LMC can be > > put in the > > > > LID check function. > > > > > > > > > > > > I'm not following how you are saying to determine the true > > LMC (of the > > > > remote port). > > > > > > > Send PortInfo query for that port > > > > > > > > > from the IPoIB client ? I think this ends up needing to contact the > SA > > > to get this info rather than the port directly. Isn't that what > > you were > > > trying to avoid originally ? If that's the case, one way would've > been > > > if ARPs carried the LMC as well as the LID. > > > > > > > What if we query the remote port LMC once, when the path is > > resolved, and then > > use it to mask the LID until the path is refreshed again? > > > > > > Just like the LID, remote LMC could change. > > > > Yes, but AFAIK the only "bad" case is if the LID stays the same but LMC > changes to a lower > value. In this case the path refresh will not happen when it is supposed > to. What's the impact of that ? Also the LID can change at the same time as the LMC. I can't tell if all the possible cases are handled properly. Are they ? > If LMC changes to a higher value it might trigger an unneeded path refresh, > which is not bad because it will refresh the LMC as well. -------------- next part -------------- An HTML attachment was scrubbed... URL: From swise at opengridcomputing.com Fri Jul 31 12:32:25 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Fri, 31 Jul 2009 14:32:25 -0500 Subject: [ofa-general] [PATCH linux-next 1/5] RDMA/cxgb3: unregister leaks memory. Message-ID: <20090731193225.2550.35448.stgit@build.ogc.int> The iwcm struct mem is never freed. Signed-off-by: Steve Wise --- drivers/infiniband/hw/cxgb3/iwch_provider.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index e2a6321..72aa57c 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -1418,6 +1418,7 @@ int iwch_register_device(struct iwch_dev *dev) bail2: ib_unregister_device(&dev->ibdev); bail1: + kfree(dev->ibdev.iwcm); return ret; } @@ -1430,5 +1431,6 @@ void iwch_unregister_device(struct iwch_dev *dev) device_remove_file(&dev->ibdev.dev, iwch_class_attributes[i]); ib_unregister_device(&dev->ibdev); + kfree(dev->ibdev.iwcm); return; } From swise at opengridcomputing.com Fri Jul 31 12:32:30 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Fri, 31 Jul 2009 14:32:30 -0500 Subject: [ofa-general] [PATCH linux-next 2/5] RDMA/cxgb3: Don't free the endpoint early. In-Reply-To: <20090731193225.2550.35448.stgit@build.ogc.int> References: <20090731193225.2550.35448.stgit@build.ogc.int> Message-ID: <20090731193230.2550.42865.stgit@build.ogc.int> - Keep ref on connection request endpoints until either accepted or rejected so it doesn't get freed early. - Endpoint flags now need to be set via atomic bitops because they can be set on both the iw_cxgb3 workqueue thread and user disconnect threads. - Don't move out of CLOSING too early due to multiple calls to iwch_ep_disconnect. Signed-off-by: Steve Wise --- drivers/infiniband/hw/cxgb3/iwch_cm.c | 55 +++++++++++++++++++-------------- drivers/infiniband/hw/cxgb3/iwch_cm.h | 9 +++-- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c index 52d7bb0..da38eff 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_cm.c +++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c @@ -286,7 +286,7 @@ void __free_ep(struct kref *kref) ep = container_of(container_of(kref, struct iwch_ep_common, kref), struct iwch_ep, com); PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]); - if (ep->com.flags & RELEASE_RESOURCES) { + if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) { cxgb3_remove_tid(ep->com.tdev, (void *)ep, ep->hwtid); dst_release(ep->dst); l2t_release(L2DATA(ep->com.tdev), ep->l2t); @@ -297,7 +297,7 @@ void __free_ep(struct kref *kref) static void release_ep_resources(struct iwch_ep *ep) { PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid); - ep->com.flags |= RELEASE_RESOURCES; + set_bit(RELEASE_RESOURCES, &ep->com.flags); put_ep(&ep->com); } @@ -786,10 +786,12 @@ static void connect_request_upcall(struct iwch_ep *ep) event.private_data_len = ep->plen; event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); event.provider_data = ep; - if (state_read(&ep->parent_ep->com) != DEAD) + if (state_read(&ep->parent_ep->com) != DEAD) { + get_ep(&ep->com); ep->parent_ep->com.cm_id->event_handler( ep->parent_ep->com.cm_id, &event); + } put_ep(&ep->parent_ep->com); ep->parent_ep = NULL; } @@ -1156,8 +1158,8 @@ static int abort_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) * We get 2 abort replies from the HW. The first one must * be ignored except for scribbling that we need one more. */ - if (!(ep->com.flags & ABORT_REQ_IN_PROGRESS)) { - ep->com.flags |= ABORT_REQ_IN_PROGRESS; + if (!test_bit(ABORT_REQ_IN_PROGRESS, &ep->com.flags)) { + set_bit(ABORT_REQ_IN_PROGRESS, &ep->com.flags); return CPL_RET_BUF_DONE; } @@ -1480,7 +1482,6 @@ static int peer_close(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) * rejects the CR. */ __state_set(&ep->com, CLOSING); - get_ep(&ep->com); break; case MPA_REP_SENT: __state_set(&ep->com, CLOSING); @@ -1561,8 +1562,8 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) * We get 2 peer aborts from the HW. The first one must * be ignored except for scribbling that we need one more. */ - if (!(ep->com.flags & PEER_ABORT_IN_PROGRESS)) { - ep->com.flags |= PEER_ABORT_IN_PROGRESS; + if (!test_bit(PEER_ABORT_IN_PROGRESS, &ep->com.flags)) { + set_bit(PEER_ABORT_IN_PROGRESS, &ep->com.flags); return CPL_RET_BUF_DONE; } @@ -1591,7 +1592,6 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) * the reference on it until the ULP accepts or * rejects the CR. */ - get_ep(&ep->com); break; case MORIBUND: case CLOSING: @@ -1797,6 +1797,7 @@ int iwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) err = send_mpa_reject(ep, pdata, pdata_len); err = iwch_ep_disconnect(ep, 0, GFP_KERNEL); } + put_ep(&ep->com); return 0; } @@ -1810,8 +1811,10 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) struct iwch_qp *qp = get_qhp(h, conn_param->qpn); PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); - if (state_read(&ep->com) == DEAD) - return -ECONNRESET; + if (state_read(&ep->com) == DEAD) { + err = -ECONNRESET; + goto err; + } BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); BUG_ON(!qp); @@ -1819,7 +1822,8 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) if ((conn_param->ord > qp->rhp->attr.max_rdma_read_qp_depth) || (conn_param->ird > qp->rhp->attr.max_rdma_reads_per_qp)) { abort_connection(ep, NULL, GFP_KERNEL); - return -EINVAL; + err = -EINVAL; + goto err; } cm_id->add_ref(cm_id); @@ -1836,8 +1840,6 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord); - get_ep(&ep->com); - /* bind QP to EP and move to RTS */ attrs.mpa_attr = ep->mpa_attr; attrs.max_ird = ep->ird; @@ -1855,30 +1857,31 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) err = iwch_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); if (err) - goto err; + goto err1; /* if needed, wait for wr_ack */ if (iwch_rqes_posted(qp)) { wait_event(ep->com.waitq, ep->com.rpl_done); err = ep->com.rpl_err; if (err) - goto err; + goto err1; } err = send_mpa_reply(ep, conn_param->private_data, conn_param->private_data_len); if (err) - goto err; + goto err1; state_set(&ep->com, FPDU_MODE); established_upcall(ep); put_ep(&ep->com); return 0; -err: +err1: ep->com.cm_id = NULL; ep->com.qp = NULL; cm_id->rem_ref(cm_id); +err: put_ep(&ep->com); return err; } @@ -2097,14 +2100,18 @@ int iwch_ep_disconnect(struct iwch_ep *ep, int abrupt, gfp_t gfp) ep->com.state = CLOSING; start_ep_timer(ep); } + set_bit(CLOSE_SENT, &ep->com.flags); break; case CLOSING: - close = 1; - if (abrupt) { - stop_ep_timer(ep); - ep->com.state = ABORTING; - } else - ep->com.state = MORIBUND; + if (!test_bit(CLOSE_SENT, &ep->com.flags)) { + close = 1; + if (abrupt) { + stop_ep_timer(ep); + ep->com.state = ABORTING; + } else + ep->com.state = MORIBUND; + set_bit(CLOSE_SENT, &ep->com.flags); + } break; case MORIBUND: case ABORTING: diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.h b/drivers/infiniband/hw/cxgb3/iwch_cm.h index 43c0aea..b9efadf 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_cm.h +++ b/drivers/infiniband/hw/cxgb3/iwch_cm.h @@ -145,9 +145,10 @@ enum iwch_ep_state { }; enum iwch_ep_flags { - PEER_ABORT_IN_PROGRESS = (1 << 0), - ABORT_REQ_IN_PROGRESS = (1 << 1), - RELEASE_RESOURCES = (1 << 2), + PEER_ABORT_IN_PROGRESS = 0, + ABORT_REQ_IN_PROGRESS = 1, + RELEASE_RESOURCES = 2, + CLOSE_SENT = 3, }; struct iwch_ep_common { @@ -162,7 +163,7 @@ struct iwch_ep_common { wait_queue_head_t waitq; int rpl_done; int rpl_err; - u32 flags; + unsigned long flags; }; struct iwch_listen_ep { From swise at opengridcomputing.com Fri Jul 31 12:32:35 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Fri, 31 Jul 2009 14:32:35 -0500 Subject: [ofa-general] [PATCH linux-next 3/5] RDMA/cxgb3: wake up any waiters on peer close/abort. In-Reply-To: <20090731193225.2550.35448.stgit@build.ogc.int> References: <20090731193225.2550.35448.stgit@build.ogc.int> Message-ID: <20090731193235.2550.20835.stgit@build.ogc.int> A close/abort while waiting for a wr_ack during connection migration can cause a hung process in iwch_accept_cr/iwch_reject_cr. The fix is to set rpl_error/rpl_done and wake up the waiters when we get a close/abort while in MPA_REQ_RCVD state. Signed-off-by: Steve Wise --- drivers/infiniband/hw/cxgb3/iwch_cm.c | 16 ++++++++++++---- 1 files changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c index da38eff..d852b3c 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_cm.c +++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c @@ -1479,9 +1479,14 @@ static int peer_close(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) /* * We're gonna mark this puppy DEAD, but keep * the reference on it until the ULP accepts or - * rejects the CR. + * rejects the CR. Also wake up anyone waiting + * in rdma connection migration (see iwch_accept_cr()). */ __state_set(&ep->com, CLOSING); + ep->com.rpl_done = 1; + ep->com.rpl_err = -ECONNRESET; + PDBG("waking up ep %p\n", ep); + wake_up(&ep->com.waitq); break; case MPA_REP_SENT: __state_set(&ep->com, CLOSING); @@ -1590,8 +1595,13 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) /* * We're gonna mark this puppy DEAD, but keep * the reference on it until the ULP accepts or - * rejects the CR. + * rejects the CR. Also wake up anyone waiting + * in rdma connection migration (see iwch_accept_cr()). */ + ep->com.rpl_done = 1; + ep->com.rpl_err = -ECONNRESET; + PDBG("waking up ep %p\n", ep); + wake_up(&ep->com.waitq); break; case MORIBUND: case CLOSING: @@ -1830,8 +1840,6 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) ep->com.cm_id = cm_id; ep->com.qp = qp; - ep->com.rpl_done = 0; - ep->com.rpl_err = 0; ep->ird = conn_param->ird; ep->ord = conn_param->ord; From swise at opengridcomputing.com Fri Jul 31 12:32:41 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Fri, 31 Jul 2009 14:32:41 -0500 Subject: [ofa-general] [PATCH linux-next 4/5] RDMA/cxgb3: Set the appropriate IO channel in rdma_init work requests. In-Reply-To: <20090731193225.2550.35448.stgit@build.ogc.int> References: <20090731193225.2550.35448.stgit@build.ogc.int> Message-ID: <20090731193241.2550.43016.stgit@build.ogc.int> Signed-off-by: Steve Wise --- drivers/infiniband/hw/cxgb3/cxio_hal.c | 4 +++- drivers/infiniband/hw/cxgb3/cxio_wr.h | 6 ++++++ drivers/infiniband/hw/cxgb3/iwch_qp.c | 1 + 3 files changed, 10 insertions(+), 1 deletions(-) diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c index 62f9cf2..4dec515 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.c +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c @@ -852,7 +852,9 @@ int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr) wqe->qpcaps = attr->qpcaps; wqe->ulpdu_size = cpu_to_be16(attr->tcp_emss); wqe->rqe_count = cpu_to_be16(attr->rqe_count); - wqe->flags_rtr_type = cpu_to_be16(attr->flags|V_RTR_TYPE(attr->rtr_type)); + wqe->flags_rtr_type = cpu_to_be16(attr->flags | + V_RTR_TYPE(attr->rtr_type) | + V_CHAN(attr->chan)); wqe->ord = cpu_to_be32(attr->ord); wqe->ird = cpu_to_be32(attr->ird); wqe->qp_dma_addr = cpu_to_be64(attr->qp_dma_addr); diff --git a/drivers/infiniband/hw/cxgb3/cxio_wr.h b/drivers/infiniband/hw/cxgb3/cxio_wr.h index 32e3b14..a197a5b 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_wr.h +++ b/drivers/infiniband/hw/cxgb3/cxio_wr.h @@ -327,6 +327,11 @@ enum rdma_init_rtr_types { #define V_RTR_TYPE(x) ((x) << S_RTR_TYPE) #define G_RTR_TYPE(x) ((((x) >> S_RTR_TYPE)) & M_RTR_TYPE) +#define S_CHAN 4 +#define M_CHAN 0x3 +#define V_CHAN(x) ((x) << S_CHAN) +#define G_CHAN(x) ((((x) >> S_CHAN)) & M_CHAN) + struct t3_rdma_init_attr { u32 tid; u32 qpid; @@ -346,6 +351,7 @@ struct t3_rdma_init_attr { u16 flags; u16 rqe_count; u32 irs; + u32 chan; }; struct t3_rdma_init_wr { diff --git a/drivers/infiniband/hw/cxgb3/iwch_qp.c b/drivers/infiniband/hw/cxgb3/iwch_qp.c index 27bbdc8..6e86534 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_qp.c +++ b/drivers/infiniband/hw/cxgb3/iwch_qp.c @@ -889,6 +889,7 @@ static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp, init_attr.qp_dma_size = (1UL << qhp->wq.size_log2); init_attr.rqe_count = iwch_rqes_posted(qhp); init_attr.flags = qhp->attr.mpa_attr.initiator ? MPA_INITIATOR : 0; + init_attr.chan = qhp->ep->l2t->smt_idx; if (peer2peer) { init_attr.rtr_type = RTR_READ; if (init_attr.ord == 0 && qhp->attr.mpa_attr.initiator) From swise at opengridcomputing.com Fri Jul 31 12:32:46 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Fri, 31 Jul 2009 14:32:46 -0500 Subject: [ofa-general] [PATCH linux-next 5/5] RDMA/cxgb3: Handle port events properly. In-Reply-To: <20090731193225.2550.35448.stgit@build.ogc.int> References: <20090731193225.2550.35448.stgit@build.ogc.int> Message-ID: <20090731193246.2550.26405.stgit@build.ogc.int> Massage the err_handler upcall into an even handler upcall. Pass netdev port events to the cxgb3 ULPs. Generate RDMA port events based on LLD port events. Signed-off-by: Steve Wise --- drivers/infiniband/hw/cxgb3/iwch.c | 28 ++++++++++++++++++++-------- drivers/net/cxgb3/cxgb3_main.c | 6 ++++-- drivers/net/cxgb3/cxgb3_offload.c | 6 +++--- drivers/net/cxgb3/cxgb3_offload.h | 8 +++++--- drivers/scsi/cxgb3i/cxgb3i_init.c | 12 ++++++------ 5 files changed, 38 insertions(+), 22 deletions(-) diff --git a/drivers/infiniband/hw/cxgb3/iwch.c b/drivers/infiniband/hw/cxgb3/iwch.c index 26fc0a4..5796170 100644 --- a/drivers/infiniband/hw/cxgb3/iwch.c +++ b/drivers/infiniband/hw/cxgb3/iwch.c @@ -51,7 +51,7 @@ cxgb3_cpl_handler_func t3c_handlers[NUM_CPL_CMDS]; static void open_rnic_dev(struct t3cdev *); static void close_rnic_dev(struct t3cdev *); -static void iwch_err_handler(struct t3cdev *, u32, u32); +static void iwch_event_handler(struct t3cdev *, u32, u32); struct cxgb3_client t3c_client = { .name = "iw_cxgb3", @@ -59,7 +59,7 @@ struct cxgb3_client t3c_client = { .remove = close_rnic_dev, .handlers = t3c_handlers, .redirect = iwch_ep_redirect, - .err_handler = iwch_err_handler + .event_handler = iwch_event_handler }; static LIST_HEAD(dev_list); @@ -162,21 +162,33 @@ static void close_rnic_dev(struct t3cdev *tdev) mutex_unlock(&dev_mutex); } -static void iwch_err_handler(struct t3cdev *tdev, u32 status, u32 error) +static void iwch_event_handler(struct t3cdev *tdev, u32 evt, u32 port_id) { struct cxio_rdev *rdev = tdev->ulp; struct iwch_dev *rnicp = rdev_to_iwch_dev(rdev); struct ib_event event; + u32 portnum = port_id + 1; - if (status == OFFLOAD_STATUS_DOWN) { + switch (evt) { + case OFFLOAD_STATUS_DOWN: { rdev->flags = CXIO_ERROR_FATAL; - - event.device = &rnicp->ibdev; event.event = IB_EVENT_DEVICE_FATAL; - event.element.port_num = 0; - ib_dispatch_event(&event); + break; + } + case OFFLOAD_PORT_DOWN: { + event.event = IB_EVENT_PORT_ERR; + break; + } + case OFFLOAD_PORT_UP: { + event.event = IB_EVENT_PORT_ACTIVE; + break; + } } + event.device = &rnicp->ibdev; + event.element.port_num = portnum; + ib_dispatch_event(&event); + return; } diff --git a/drivers/net/cxgb3/cxgb3_main.c b/drivers/net/cxgb3/cxgb3_main.c index fb5df5c..c97ab82 100644 --- a/drivers/net/cxgb3/cxgb3_main.c +++ b/drivers/net/cxgb3/cxgb3_main.c @@ -1286,6 +1286,7 @@ static int cxgb_open(struct net_device *dev) if (!other_ports) schedule_chk_task(adapter); + cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id); return 0; } @@ -1318,6 +1319,7 @@ static int cxgb_close(struct net_device *dev) if (!adapter->open_device_map) cxgb_down(adapter); + cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id); return 0; } @@ -2717,7 +2719,7 @@ static int t3_adapter_error(struct adapter *adapter, int reset) if (is_offload(adapter) && test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) { - cxgb3_err_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0); + cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0); offload_close(&adapter->tdev); } @@ -2782,7 +2784,7 @@ static void t3_resume_ports(struct adapter *adapter) } if (is_offload(adapter) && !ofld_disable) - cxgb3_err_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0); + cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0); } /* diff --git a/drivers/net/cxgb3/cxgb3_offload.c b/drivers/net/cxgb3/cxgb3_offload.c index f9f54b5..75064ee 100644 --- a/drivers/net/cxgb3/cxgb3_offload.c +++ b/drivers/net/cxgb3/cxgb3_offload.c @@ -153,14 +153,14 @@ void cxgb3_remove_clients(struct t3cdev *tdev) mutex_unlock(&cxgb3_db_lock); } -void cxgb3_err_notify(struct t3cdev *tdev, u32 status, u32 error) +void cxgb3_event_notify(struct t3cdev *tdev, u32 event, u32 port) { struct cxgb3_client *client; mutex_lock(&cxgb3_db_lock); list_for_each_entry(client, &client_list, client_list) { - if (client->err_handler) - client->err_handler(tdev, status, error); + if (client->event_handler) + client->event_handler(tdev, event, port); } mutex_unlock(&cxgb3_db_lock); } diff --git a/drivers/net/cxgb3/cxgb3_offload.h b/drivers/net/cxgb3/cxgb3_offload.h index 55945f4..670aa62 100644 --- a/drivers/net/cxgb3/cxgb3_offload.h +++ b/drivers/net/cxgb3/cxgb3_offload.h @@ -64,14 +64,16 @@ void cxgb3_register_client(struct cxgb3_client *client); void cxgb3_unregister_client(struct cxgb3_client *client); void cxgb3_add_clients(struct t3cdev *tdev); void cxgb3_remove_clients(struct t3cdev *tdev); -void cxgb3_err_notify(struct t3cdev *tdev, u32 status, u32 error); +void cxgb3_event_notify(struct t3cdev *tdev, u32 event, u32 port); typedef int (*cxgb3_cpl_handler_func)(struct t3cdev *dev, struct sk_buff *skb, void *ctx); enum { OFFLOAD_STATUS_UP, - OFFLOAD_STATUS_DOWN + OFFLOAD_STATUS_DOWN, + OFFLOAD_PORT_DOWN, + OFFLOAD_PORT_UP }; struct cxgb3_client { @@ -82,7 +84,7 @@ struct cxgb3_client { int (*redirect)(void *ctx, struct dst_entry *old, struct dst_entry *new, struct l2t_entry *l2t); struct list_head client_list; - void (*err_handler)(struct t3cdev *tdev, u32 status, u32 error); + void (*event_handler)(struct t3cdev *tdev, u32 event, u32 port); }; /* diff --git a/drivers/scsi/cxgb3i/cxgb3i_init.c b/drivers/scsi/cxgb3i/cxgb3i_init.c index 042d9bc..d0ab23a 100644 --- a/drivers/scsi/cxgb3i/cxgb3i_init.c +++ b/drivers/scsi/cxgb3i/cxgb3i_init.c @@ -26,7 +26,7 @@ MODULE_VERSION(DRV_MODULE_VERSION); static void open_s3_dev(struct t3cdev *); static void close_s3_dev(struct t3cdev *); -static void s3_err_handler(struct t3cdev *tdev, u32 status, u32 error); +static void s3_event_handler(struct t3cdev *tdev, u32 event, u32 port); static cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS]; static struct cxgb3_client t3c_client = { @@ -34,7 +34,7 @@ static struct cxgb3_client t3c_client = { .handlers = cxgb3i_cpl_handlers, .add = open_s3_dev, .remove = close_s3_dev, - .err_handler = s3_err_handler, + .event_handler = s3_event_handler, }; /** @@ -66,16 +66,16 @@ static void close_s3_dev(struct t3cdev *t3dev) cxgb3i_ddp_cleanup(t3dev); } -static void s3_err_handler(struct t3cdev *tdev, u32 status, u32 error) +static void s3_event_handler(struct t3cdev *tdev, u32 event, u32 port) { struct cxgb3i_adapter *snic = cxgb3i_adapter_find_by_tdev(tdev); - cxgb3i_log_info("snic 0x%p, tdev 0x%p, status 0x%x, err 0x%x.\n", - snic, tdev, status, error); + cxgb3i_log_info("snic 0x%p, tdev 0x%p, event 0x%x, port 0x%x.\n", + snic, tdev, event, port); if (!snic) return; - switch (status) { + switch (event) { case OFFLOAD_STATUS_DOWN: snic->flags |= CXGB3I_ADAPTER_FLAG_RESET; break; From swise at opengridcomputing.com Fri Jul 31 12:37:48 2009 From: swise at opengridcomputing.com (Steve Wise) Date: Fri, 31 Jul 2009 14:37:48 -0500 Subject: [ofa-general] [PATCH linux-next 5/5] RDMA/cxgb3: Handle port events properly. In-Reply-To: <20090731193246.2550.26405.stgit@build.ogc.int> References: <20090731193225.2550.35448.stgit@build.ogc.int> <20090731193246.2550.26405.stgit@build.ogc.int> Message-ID: <4A73480C.7000900@opengridcomputing.com> Note this patch hits the low level driver and the iscsi driver in addition to iw_cxgb3... Steve Wise wrote: > Massage the err_handler upcall into an even handler upcall. > > Pass netdev port events to the cxgb3 ULPs. > > Generate RDMA port events based on LLD port events. > > Signed-off-by: Steve Wise > --- > > drivers/infiniband/hw/cxgb3/iwch.c | 28 ++++++++++++++++++++-------- > drivers/net/cxgb3/cxgb3_main.c | 6 ++++-- > drivers/net/cxgb3/cxgb3_offload.c | 6 +++--- > drivers/net/cxgb3/cxgb3_offload.h | 8 +++++--- > drivers/scsi/cxgb3i/cxgb3i_init.c | 12 ++++++------ > 5 files changed, 38 insertions(+), 22 deletions(-) > > diff --git a/drivers/infiniband/hw/cxgb3/iwch.c b/drivers/infiniband/hw/cxgb3/iwch.c > index 26fc0a4..5796170 100644 > --- a/drivers/infiniband/hw/cxgb3/iwch.c > +++ b/drivers/infiniband/hw/cxgb3/iwch.c > @@ -51,7 +51,7 @@ cxgb3_cpl_handler_func t3c_handlers[NUM_CPL_CMDS]; > > static void open_rnic_dev(struct t3cdev *); > static void close_rnic_dev(struct t3cdev *); > -static void iwch_err_handler(struct t3cdev *, u32, u32); > +static void iwch_event_handler(struct t3cdev *, u32, u32); > > struct cxgb3_client t3c_client = { > .name = "iw_cxgb3", > @@ -59,7 +59,7 @@ struct cxgb3_client t3c_client = { > .remove = close_rnic_dev, > .handlers = t3c_handlers, > .redirect = iwch_ep_redirect, > - .err_handler = iwch_err_handler > + .event_handler = iwch_event_handler > }; > > static LIST_HEAD(dev_list); > @@ -162,21 +162,33 @@ static void close_rnic_dev(struct t3cdev *tdev) > mutex_unlock(&dev_mutex); > } > > -static void iwch_err_handler(struct t3cdev *tdev, u32 status, u32 error) > +static void iwch_event_handler(struct t3cdev *tdev, u32 evt, u32 port_id) > { > struct cxio_rdev *rdev = tdev->ulp; > struct iwch_dev *rnicp = rdev_to_iwch_dev(rdev); > struct ib_event event; > + u32 portnum = port_id + 1; > > - if (status == OFFLOAD_STATUS_DOWN) { > + switch (evt) { > + case OFFLOAD_STATUS_DOWN: { > rdev->flags = CXIO_ERROR_FATAL; > - > - event.device = &rnicp->ibdev; > event.event = IB_EVENT_DEVICE_FATAL; > - event.element.port_num = 0; > - ib_dispatch_event(&event); > + break; > + } > + case OFFLOAD_PORT_DOWN: { > + event.event = IB_EVENT_PORT_ERR; > + break; > + } > + case OFFLOAD_PORT_UP: { > + event.event = IB_EVENT_PORT_ACTIVE; > + break; > + } > } > > + event.device = &rnicp->ibdev; > + event.element.port_num = portnum; > + ib_dispatch_event(&event); > + > return; > } > > diff --git a/drivers/net/cxgb3/cxgb3_main.c b/drivers/net/cxgb3/cxgb3_main.c > index fb5df5c..c97ab82 100644 > --- a/drivers/net/cxgb3/cxgb3_main.c > +++ b/drivers/net/cxgb3/cxgb3_main.c > @@ -1286,6 +1286,7 @@ static int cxgb_open(struct net_device *dev) > if (!other_ports) > schedule_chk_task(adapter); > > + cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id); > return 0; > } > > @@ -1318,6 +1319,7 @@ static int cxgb_close(struct net_device *dev) > if (!adapter->open_device_map) > cxgb_down(adapter); > > + cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id); > return 0; > } > > @@ -2717,7 +2719,7 @@ static int t3_adapter_error(struct adapter *adapter, int reset) > > if (is_offload(adapter) && > test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) { > - cxgb3_err_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0); > + cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0); > offload_close(&adapter->tdev); > } > > @@ -2782,7 +2784,7 @@ static void t3_resume_ports(struct adapter *adapter) > } > > if (is_offload(adapter) && !ofld_disable) > - cxgb3_err_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0); > + cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0); > } > > /* > diff --git a/drivers/net/cxgb3/cxgb3_offload.c b/drivers/net/cxgb3/cxgb3_offload.c > index f9f54b5..75064ee 100644 > --- a/drivers/net/cxgb3/cxgb3_offload.c > +++ b/drivers/net/cxgb3/cxgb3_offload.c > @@ -153,14 +153,14 @@ void cxgb3_remove_clients(struct t3cdev *tdev) > mutex_unlock(&cxgb3_db_lock); > } > > -void cxgb3_err_notify(struct t3cdev *tdev, u32 status, u32 error) > +void cxgb3_event_notify(struct t3cdev *tdev, u32 event, u32 port) > { > struct cxgb3_client *client; > > mutex_lock(&cxgb3_db_lock); > list_for_each_entry(client, &client_list, client_list) { > - if (client->err_handler) > - client->err_handler(tdev, status, error); > + if (client->event_handler) > + client->event_handler(tdev, event, port); > } > mutex_unlock(&cxgb3_db_lock); > } > diff --git a/drivers/net/cxgb3/cxgb3_offload.h b/drivers/net/cxgb3/cxgb3_offload.h > index 55945f4..670aa62 100644 > --- a/drivers/net/cxgb3/cxgb3_offload.h > +++ b/drivers/net/cxgb3/cxgb3_offload.h > @@ -64,14 +64,16 @@ void cxgb3_register_client(struct cxgb3_client *client); > void cxgb3_unregister_client(struct cxgb3_client *client); > void cxgb3_add_clients(struct t3cdev *tdev); > void cxgb3_remove_clients(struct t3cdev *tdev); > -void cxgb3_err_notify(struct t3cdev *tdev, u32 status, u32 error); > +void cxgb3_event_notify(struct t3cdev *tdev, u32 event, u32 port); > > typedef int (*cxgb3_cpl_handler_func)(struct t3cdev *dev, > struct sk_buff *skb, void *ctx); > > enum { > OFFLOAD_STATUS_UP, > - OFFLOAD_STATUS_DOWN > + OFFLOAD_STATUS_DOWN, > + OFFLOAD_PORT_DOWN, > + OFFLOAD_PORT_UP > }; > > struct cxgb3_client { > @@ -82,7 +84,7 @@ struct cxgb3_client { > int (*redirect)(void *ctx, struct dst_entry *old, > struct dst_entry *new, struct l2t_entry *l2t); > struct list_head client_list; > - void (*err_handler)(struct t3cdev *tdev, u32 status, u32 error); > + void (*event_handler)(struct t3cdev *tdev, u32 event, u32 port); > }; > > /* > diff --git a/drivers/scsi/cxgb3i/cxgb3i_init.c b/drivers/scsi/cxgb3i/cxgb3i_init.c > index 042d9bc..d0ab23a 100644 > --- a/drivers/scsi/cxgb3i/cxgb3i_init.c > +++ b/drivers/scsi/cxgb3i/cxgb3i_init.c > @@ -26,7 +26,7 @@ MODULE_VERSION(DRV_MODULE_VERSION); > > static void open_s3_dev(struct t3cdev *); > static void close_s3_dev(struct t3cdev *); > -static void s3_err_handler(struct t3cdev *tdev, u32 status, u32 error); > +static void s3_event_handler(struct t3cdev *tdev, u32 event, u32 port); > > static cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS]; > static struct cxgb3_client t3c_client = { > @@ -34,7 +34,7 @@ static struct cxgb3_client t3c_client = { > .handlers = cxgb3i_cpl_handlers, > .add = open_s3_dev, > .remove = close_s3_dev, > - .err_handler = s3_err_handler, > + .event_handler = s3_event_handler, > }; > > /** > @@ -66,16 +66,16 @@ static void close_s3_dev(struct t3cdev *t3dev) > cxgb3i_ddp_cleanup(t3dev); > } > > -static void s3_err_handler(struct t3cdev *tdev, u32 status, u32 error) > +static void s3_event_handler(struct t3cdev *tdev, u32 event, u32 port) > { > struct cxgb3i_adapter *snic = cxgb3i_adapter_find_by_tdev(tdev); > > - cxgb3i_log_info("snic 0x%p, tdev 0x%p, status 0x%x, err 0x%x.\n", > - snic, tdev, status, error); > + cxgb3i_log_info("snic 0x%p, tdev 0x%p, event 0x%x, port 0x%x.\n", > + snic, tdev, event, port); > if (!snic) > return; > > - switch (status) { > + switch (event) { > case OFFLOAD_STATUS_DOWN: > snic->flags |= CXGB3I_ADAPTER_FLAG_RESET; > break; > > _______________________________________________ > general mailing list > general at lists.openfabrics.org > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general > > To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general > From jgunthorpe at obsidianresearch.com Fri Jul 31 12:40:03 2009 From: jgunthorpe at obsidianresearch.com (Jason Gunthorpe) Date: Fri, 31 Jul 2009 13:40:03 -0600 Subject: [ofa-general] [PATCH] ipoib: refresh path when remote lid changes In-Reply-To: <4A731818.3060500@voltaire.com> References: <20090727192938.GD5794@obsidianresearch.com> <4A6ECF6F.4000008@Voltaire.COM> <4A70154F.7080300@gmail.com> <4A703DA4.9080300@Voltaire.COM> <4A705B3A.7060404@Voltaire.COM> <4A731818.3060500@voltaire.com> Message-ID: <20090731194003.GV30626@obsidianresearch.com> On Fri, Jul 31, 2009 at 07:13:12PM +0300, Yossi Etigin wrote: > What if we query the remote port LMC once, when the path is > resolved, and then use it to mask the LID until the path is > refreshed again? What are you trying to fix here? Most SMs have a persistent LID stability feature, so why would the LID change very often anyhow? Jason From sashak at voltaire.com Fri Jul 31 13:22:19 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Fri, 31 Jul 2009 23:22:19 +0300 Subject: [ofa-general] Re: [PATCH] infiniband-diags/libibnetdisc: fix DBGFLAGS variable In-Reply-To: <20090730184054.1a6f9f25.weiny2@llnl.gov> References: <20090730184054.1a6f9f25.weiny2@llnl.gov> Message-ID: <20090731202219.GA16708@me> On 18:40 Thu 30 Jul , Ira Weiny wrote: > > From: Ira Weiny > Date: Thu, 30 Jul 2009 18:40:12 -0700 > Subject: [PATCH] infiniband-diags/libibnetdisc: fix DBGFLAGS variable > > > Signed-off-by: Ira Weiny Applied. Thanks. Sasha From sashak at voltaire.com Fri Jul 31 13:24:24 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Fri, 31 Jul 2009 23:24:24 +0300 Subject: [ofa-general] Re: [PATCH] libibmad/mad.c: Validate DR hop count for SMPs and handle errors In-Reply-To: <20090731145729.GA24420@comcast.net> References: <20090731145729.GA24420@comcast.net> Message-ID: <20090731202424.GB16708@me> On 10:57 Fri 31 Jul , Hal Rosenstock wrote: > > In mad_encode, validate DR hop count for SMPs > In mad_build_pkt, handle mad_encode error > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Fri Jul 31 13:25:28 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Fri, 31 Jul 2009 23:25:28 +0300 Subject: [ofa-general] Re: [PATCHv3] opensm/osm_ucast_lash.c: Handle malloc failures better In-Reply-To: <20090731173640.GA25877@comcast.net> References: <20090731173640.GA25877@comcast.net> Message-ID: <20090731202528.GC16708@me> On 13:36 Fri 31 Jul , Hal Rosenstock wrote: > > Signed-off-by: Hal Rosenstock Applied. Thanks. Sasha From sashak at voltaire.com Fri Jul 31 16:46:13 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sat, 1 Aug 2009 02:46:13 +0300 Subject: [ofa-general] [PATCH] libibmad: use NULL as null pointer Message-ID: <20090731234613.GD16708@me> Use 'NULL' (instead of '0') as null pointer when functions return bad status. It is in order to prevent cut-and-paste errors when a tested statements are used in functions which return integers. Signed-off-by: Sasha Khapyorsky --- libibmad/src/bm.c | 2 +- libibmad/src/fields.c | 6 +++--- libibmad/src/gs.c | 4 ++-- libibmad/src/mad.c | 4 ++-- libibmad/src/rpc.c | 14 +++++++------- libibmad/src/sa.c | 2 +- libibmad/src/vendor.c | 4 ++-- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/libibmad/src/bm.c b/libibmad/src/bm.c index e335d92..2983855 100644 --- a/libibmad/src/bm.c +++ b/libibmad/src/bm.c @@ -58,7 +58,7 @@ uint8_t *bm_call_via(void *data, ib_portid_t * portid, ib_bm_call_t * call, DEBUG("route %s data %p", portid2str(portid), data); if (portid->lid <= 0) { IBWARN("only lid routes are supported"); - return 0; + return NULL; } resp_expected = response_expected(call->method); diff --git a/libibmad/src/fields.c b/libibmad/src/fields.c index 129f7e5..c8e4e79 100644 --- a/libibmad/src/fields.c +++ b/libibmad/src/fields.c @@ -633,7 +633,7 @@ static char *_mad_dump_field(const ib_field_t * f, const char *name, char *buf, int l, n; if (bufsz <= 32) - return 0; /* buf too small */ + return NULL; /* buf too small */ if (!name) name = f->name; @@ -680,14 +680,14 @@ int mad_print_field(enum MAD_FIELDS field, const char *name, void *val) char *mad_dump_field(enum MAD_FIELDS field, char *buf, int bufsz, void *val) { if (field <= IB_NO_FIELD || field >= IB_FIELD_LAST_) - return 0; + return NULL; return _mad_dump_field(ib_mad_f + field, 0, buf, bufsz, val); } char *mad_dump_val(enum MAD_FIELDS field, char *buf, int bufsz, void *val) { if (field <= IB_NO_FIELD || field >= IB_FIELD_LAST_) - return 0; + return NULL; return _mad_dump_val(ib_mad_f + field, buf, bufsz, val); } diff --git a/libibmad/src/gs.c b/libibmad/src/gs.c index c7e4ff6..4701866 100644 --- a/libibmad/src/gs.c +++ b/libibmad/src/gs.c @@ -56,7 +56,7 @@ uint8_t *pma_query_via(void *rcvbuf, ib_portid_t * dest, int port, if (lid == -1) { IBWARN("only lid routed is supported"); - return 0; + return NULL; } rpc.mgtclass = IB_PERFORMANCE_CLASS; @@ -89,7 +89,7 @@ uint8_t *performance_reset_via(void *rcvbuf, ib_portid_t * dest, if (lid == -1) { IBWARN("only lid routed is supported"); - return 0; + return NULL; } if (!mask) diff --git a/libibmad/src/mad.c b/libibmad/src/mad.c index 89ff978..8defabd 100644 --- a/libibmad/src/mad.c +++ b/libibmad/src/mad.c @@ -78,11 +78,11 @@ void *mad_encode(void *buf, ib_rpc_t * rpc, ib_dr_path_t * drpath, void *data) if (rpc->mgtclass == IB_SMI_DIRECT_CLASS) { if (!drpath) { IBWARN("encoding dr mad without drpath (null)"); - return 0; + return NULL; } if (drpath->cnt >= IB_SUBNET_PATH_HOPS_MAX) { IBWARN("dr path with hop count %d", drpath->cnt); - return 0; + return NULL; } mad_set_field(buf, 0, IB_DRSMP_HOPCNT_F, drpath->cnt); mad_set_field(buf, 0, IB_DRSMP_HOPPTR_F, diff --git a/libibmad/src/rpc.c b/libibmad/src/rpc.c index 0b989da..c6fd392 100644 --- a/libibmad/src/rpc.c +++ b/libibmad/src/rpc.c @@ -218,7 +218,7 @@ void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc, memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) - return 0; + return NULL; timeout = rpc->timeout ? rpc->timeout : port->timeout ? port->timeout : madrpc_timeout; @@ -228,7 +228,7 @@ void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc, port->class_agents[rpc->mgtclass], len, timeout, retries)) < 0) { IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); - return 0; + return NULL; } mad = umad_get_mad(rcvbuf); @@ -248,7 +248,7 @@ void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc, if (status != 0) { ERRS("MAD completed with error status 0x%x; dport (%s)", status, portid2str(dport)); - return 0; + return NULL; } if (ibdebug) { @@ -274,7 +274,7 @@ void *mad_rpc_rmpp(const struct ibmad_port *port, ib_rpc_t * rpc, DEBUG("rmpp %p data %p", rmpp, data); if ((len = mad_build_pkt(sndbuf, rpc, dport, rmpp, data)) < 0) - return 0; + return NULL; timeout = rpc->timeout ? rpc->timeout : port->timeout ? port->timeout : madrpc_timeout; @@ -284,7 +284,7 @@ void *mad_rpc_rmpp(const struct ibmad_port *port, ib_rpc_t * rpc, port->class_agents[rpc->mgtclass], len, timeout, retries)) < 0) { IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); - return 0; + return NULL; } mad = umad_get_mad(rcvbuf); @@ -292,7 +292,7 @@ void *mad_rpc_rmpp(const struct ibmad_port *port, ib_rpc_t * rpc, if ((status = mad_get_field(mad, 0, IB_MAD_STATUS_F)) != 0) { ERRS("MAD completed with error status 0x%x; dport (%s)", status, portid2str(dport)); - return 0; + return NULL; } if (ibdebug) { @@ -306,7 +306,7 @@ void *mad_rpc_rmpp(const struct ibmad_port *port, ib_rpc_t * rpc, if ((rmpp->flags & 0x3) && mad_get_field(mad, 0, IB_SA_RMPP_VERS_F) != 1) { IBWARN("bad rmpp version"); - return 0; + return NULL; } rmpp->type = mad_get_field(mad, 0, IB_SA_RMPP_TYPE_F); rmpp->status = mad_get_field(mad, 0, IB_SA_RMPP_STATUS_F); diff --git a/libibmad/src/sa.c b/libibmad/src/sa.c index 27e6ee9..5f34039 100644 --- a/libibmad/src/sa.c +++ b/libibmad/src/sa.c @@ -56,7 +56,7 @@ uint8_t *sa_rpc_call(const struct ibmad_port *ibmad_port, void *rcvbuf, if (portid->lid <= 0) { IBWARN("only lid routes are supported"); - return 0; + return NULL; } rpc.mgtclass = IB_SA_CLASS; diff --git a/libibmad/src/vendor.c b/libibmad/src/vendor.c index 4bf9561..6736f32 100644 --- a/libibmad/src/vendor.c +++ b/libibmad/src/vendor.c @@ -66,11 +66,11 @@ uint8_t *ib_vendor_call_via(void *data, ib_portid_t * portid, DEBUG("route %s data %p", portid2str(portid), data); if (portid->lid <= 0) - return 0; /* no direct SMI */ + return NULL; /* no direct SMI */ if (!(range1 = mad_is_vendor_range1(call->mgmt_class)) && !(mad_is_vendor_range2(call->mgmt_class))) - return 0; + return NULL; resp_expected = response_expected(call->method); -- 1.6.4 From sashak at voltaire.com Fri Jul 31 16:52:21 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sat, 1 Aug 2009 02:52:21 +0300 Subject: [ofa-general] Re: [infiniband-diags] [PATCH] [1/6] fix libibnetdisc API consistency and bugs In-Reply-To: <1248714752.16723.324.camel@auk31.llnl.gov> References: <1248714752.16723.324.camel@auk31.llnl.gov> Message-ID: <20090731235221.GE16708@me> On 10:12 Mon 27 Jul , Al Chu wrote: > In libibnetdisc, do not automatically output messages to stderr/stdout > for warnings or non-fatal errors. Use IBND_DEBUG instead of IBPANIC or > IBWARN to allow users of the lib to decide output conditions. > > Al > > -- > Albert Chu > chu11 at llnl.gov > Computer Scientist > High Performance Systems Division > Lawrence Livermore National Laboratory > From: Albert Chu > Date: Thu, 23 Jul 2009 11:56:53 -0700 > Subject: [PATCH] In libibnetdisc, do not automatically output messages to stderr/stdout for warnings or non-fatal errors. Use IBND_DEBUG instead of IBPANIC or IBWARN to allow users of the lib to decide output conditions. > > > Signed-off-by: Albert Chu Applied. Thanks. Sasha From sashak at voltaire.com Fri Jul 31 16:52:40 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sat, 1 Aug 2009 02:52:40 +0300 Subject: [ofa-general] Re: [infiniband-diags] [PATCH] [2/6] fix libibnetdisc API consistency and bugs In-Reply-To: <1248714756.16723.325.camel@auk31.llnl.gov> References: <1248714756.16723.325.camel@auk31.llnl.gov> Message-ID: <20090731235240.GF16708@me> On 10:12 Mon 27 Jul , Al Chu wrote: > Use IBPANIC consistently in libibnetdisc, in particular, since IBPANIC > calls exit, there's no use in returning a value after an error. > > Al > > -- > Albert Chu > chu11 at llnl.gov > Computer Scientist > High Performance Systems Division > Lawrence Livermore National Laboratory > From: Albert Chu > Date: Thu, 23 Jul 2009 11:57:35 -0700 > Subject: [PATCH] Use IBPANIC consistently in libibnetdisc, in particular, since IBPANIC calls exit, there's no use in returning a value after an error. > > > Signed-off-by: Albert Chu Applied. Thanks. Sasha From sashak at voltaire.com Fri Jul 31 16:55:16 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sat, 1 Aug 2009 02:55:16 +0300 Subject: [ofa-general] Re: [infiniband-diags] [PATCH] [3/6] fix libibnetdisc API consistency and bugs In-Reply-To: <1248714761.16723.326.camel@auk31.llnl.gov> References: <1248714761.16723.326.camel@auk31.llnl.gov> Message-ID: <20090731235516.GG16708@me> On 10:12 Mon 27 Jul , Al Chu wrote: > Fix potential memleak in ibnd_discover_fabric error path. > > Al > > -- > Albert Chu > chu11 at llnl.gov > Computer Scientist > High Performance Systems Division > Lawrence Livermore National Laboratory > From: Albert Chu > Date: Thu, 23 Jul 2009 11:58:45 -0700 > Subject: [PATCH] Fix potential memleak in ibnd_discover_fabric error path. > > > Signed-off-by: Albert Chu Applied. Thanks. Sasha From sashak at voltaire.com Fri Jul 31 17:04:17 2009 From: sashak at voltaire.com (Sasha Khapyorsky) Date: Sat, 1 Aug 2009 03:04:17 +0300 Subject: [ofa-general] [PATCH] bibnetdisc/ibnetdisc.c: fix parameter incompatibility warning In-Reply-To: <1248714761.16723.326.camel@auk31.llnl.gov> References: <1248714761.16723.326.camel@auk31.llnl.gov> Message-ID: <20090801000417.GH16708@me> Fix function parameter pointer type incompatibility compilation warning. Signed-off-by: Sasha Khapyorsky --- On 10:12 Mon 27 Jul , Al Chu wrote: > --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c > +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c > @@ -602,7 +602,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, > > return ((ibnd_fabric_t *)fabric); > error: > - free(fabric); > + ibnd_destroy_fabric(fabric); > return (NULL); > } infiniband-diags/libibnetdisc/src/ibnetdisc.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/infiniband-diags/libibnetdisc/src/ibnetdisc.c b/infiniband-diags/libibnetdisc/src/ibnetdisc.c index 230b3ea..7ce07e3 100644 --- a/infiniband-diags/libibnetdisc/src/ibnetdisc.c +++ b/infiniband-diags/libibnetdisc/src/ibnetdisc.c @@ -615,7 +615,7 @@ ibnd_discover_fabric(struct ibmad_port *ibmad_port, int timeout_ms, return ((ibnd_fabric_t *)fabric); error: - ibnd_destroy_fabric(fabric); + ibnd_destroy_fabric((ibnd_fabric_t *)fabric); return (NULL); } -- 1.6.4