[openib-general] [PATCH 4/4] osm: building LID matrices in fat-tree routing

Yevgeny Kliteynik kliteyn at dev.mellanox.co.il
Mon Jan 1 05:31:35 PST 2007


Building LID matrices in fat-tree routing

Lid matrices are now built while setting routing
w/o any additional complexity, and removed the
temporary ftree dump function (that differed from
the ucast mgr dump function only by not dumping
anything that is related to lid matrices).

Signed-off-by: Yevgeny Kliteynik <kliteyn at dev.mellanox.co.il>
---
 osm/opensm/osm_ucast_ftree.c |  173 ++++++++++++++++++++++++++++--------------
 1 files changed, 115 insertions(+), 58 deletions(-)

diff --git a/osm/opensm/osm_ucast_ftree.c b/osm/opensm/osm_ucast_ftree.c
index 9a1ac69..6f72791 100644
--- a/osm/opensm/osm_ucast_ftree.c
+++ b/osm/opensm/osm_ucast_ftree.c
@@ -770,6 +770,26 @@ __osm_ftree_sw_get_fwd_table_block(
    return p_sw->lft_buf[lid_ho];
 }
 
+/***************************************************/
+
+static inline cl_status_t
+__osm_ftree_sw_set_hops(
+   IN  ftree_sw_t     * p_sw,
+   IN  uint16_t         max_lid_ho,
+   IN  uint16_t         lid_ho,
+   IN  uint8_t          port_num,
+   IN  uint8_t          hops)
+{
+   /* make sure the lid matrix has enough room */
+   osm_switch_set_min_lid_size(p_sw->p_osm_sw, max_lid_ho);
+
+   /* set local min hop table(LID) */
+   return osm_switch_set_hops(p_sw->p_osm_sw,
+                              lid_ho,
+                              port_num,
+                              hops);
+}
+
 /***************************************************
  **
  ** ftree_hca_t functions
@@ -1747,8 +1767,10 @@ __osm_ftree_fabric_route_upgoing_by_goin
    IN  ftree_sw_t     * p_sw,
    IN  ftree_sw_t     * p_prev_sw,
    IN  ib_net16_t       target_lid,
+   IN  uint8_t          target_rank,
    IN  boolean_t        is_real_lid,
-   IN  boolean_t        is_main_path)
+   IN  boolean_t        is_main_path,
+   IN  uint8_t          highest_rank_in_route)
 {
    ftree_sw_t          * p_remote_sw;
    uint16_t              ports_num;
@@ -1813,6 +1835,7 @@ __osm_ftree_fabric_route_upgoing_by_goin
        *      - going DOWN(TRUE,TRUE) through ALL the groups
        *         + promoting port counter
        *         + setting path in remote switch fwd tbl
+       *         + setting hops in remote switch on all the ports of each group
        *      
        *  2. is_real_lid == TRUE && is_main_path == FALSE: 
        *      - going DOWN(TRUE,FALSE) through ALL the groups but only if
@@ -1820,11 +1843,14 @@ __osm_ftree_fabric_route_upgoing_by_goin
        *        for this target LID
        *         + NOT promoting port counter
        *         + setting path in remote switch fwd tbl if it hasn't been set yet
+       *         + setting hops in remote switch on all the ports of each group
+       *           if it hasn't been set yet
        *
        *  3. is_real_lid == FALSE && is_main_path == TRUE: 
        *      - going DOWN(FALSE,TRUE) through ALL the groups
        *         + promoting port counter
        *         + NOT setting path in remote switch fwd tbl
+       *         + NOT setting hops in remote switch
        *
        *  4. is_real_lid == FALSE && is_main_path == FALSE: 
        *      - illegal state - we shouldn't get here
@@ -1849,6 +1875,23 @@ __osm_ftree_fabric_route_upgoing_by_goin
                  __osm_ftree_tuple_to_str(p_remote_sw->tuple),
                  cl_ntoh16(target_lid),
                  p_min_port->remote_port_num);
+
+         /* On the remote switch that is pointed by the p_group,
+            set hops for ALL the ports in the remote group. */
+
+         for (j = 0; j < ports_num; j++)
+         {
+            cl_ptr_vector_at(&p_group->ports, j, (void **)&p_port);
+
+            __osm_ftree_sw_set_hops(p_remote_sw,
+                                    p_ftree->lft_max_lid_ho,
+                                    cl_ntoh16(target_lid),
+                                    p_port->remote_port_num,
+                                    ( (target_rank - highest_rank_in_route) +
+                                      (p_remote_sw->rank - highest_rank_in_route) ));
+         }
+
+
       }
    
       /* The number of upgoing routes is tracked in the 
@@ -1869,8 +1912,10 @@ __osm_ftree_fabric_route_upgoing_by_goin
                p_remote_sw,   /* remote switch - used as a route-upgoing alg. start point */
                NULL,          /* prev. position - NULL to mark that we went down and not up */
                target_lid,    /* LID that we're routing to */
+               target_rank,   /* rank of the LID that we're routing to */
                is_real_lid,   /* whether the target LID is real or dummy */
-               is_main_path); /* whether this is path to HCA that should by tracked by counters */
+               is_main_path,  /* whether this is path to HCA that should by tracked by counters */
+               highest_rank_in_route); /* highest visited point in the tree before going down */
       }
    }
    /* done scanning all the down-going port groups */
@@ -1896,6 +1941,7 @@ __osm_ftree_fabric_route_downgoing_by_go
    IN  ftree_sw_t     * p_sw,
    IN  ftree_sw_t     * p_prev_sw,
    IN  ib_net16_t       target_lid,
+   IN  uint8_t          target_rank,
    IN  boolean_t        is_real_lid,
    IN  boolean_t        is_main_path)
 {
@@ -1920,8 +1966,10 @@ __osm_ftree_fabric_route_downgoing_by_go
          p_sw,          /* local switch - used as a route-upgoing alg. start point */
          p_prev_sw,     /* switch that we went up from (NULL means that we went down) */
          target_lid,    /* LID that we're routing to */
+         target_rank,   /* rank of the LID that we're routing to */
          is_real_lid,   /* whether this target LID is real or dummy */
-         is_main_path); /* whether this path to HCA should by tracked by counters */
+         is_main_path,  /* whether this path to HCA should by tracked by counters */
+         p_sw->rank);   /* the highest visited point in the tree before going down */
    }
 
    /* recursion stop condition - if it's a root switch, */
@@ -1971,9 +2019,12 @@ __osm_ftree_fabric_route_downgoing_by_go
     *      - going UP(TRUE,TRUE) on selected min_group and min_port
     *         + promoting port counter
     *         + setting path in remote switch fwd tbl
+    *         + setting hops in remote switch on all the ports of selected group
     *      - going UP(TRUE,FALSE) on rest of the groups, each time on port 0
     *         + NOT promoting port counter
     *         + setting path in remote switch fwd tbl if it hasn't been set yet
+    *         + setting hops in remote switch on all the ports of each group
+    *           if it hasn't been set yet
     *      
     *  2. is_real_lid == TRUE && is_main_path == FALSE: 
     *      - going UP(TRUE,FALSE) on ALL the groups, each time on port 0,
@@ -1981,11 +2032,14 @@ __osm_ftree_fabric_route_downgoing_by_go
     *        configured for this target LID
     *         + NOT promoting port counter
     *         + setting path in remote switch fwd tbl if it hasn't been set yet
+    *         + setting hops in remote switch on all the ports of each group
+    *           if it hasn't been set yet
     *
     *  3. is_real_lid == FALSE && is_main_path == TRUE: 
     *      - going UP(FALSE,TRUE) ONLY on selected min_group and min_port
     *         + promoting port counter
     *         + NOT setting path in remote switch fwd tbl
+    *         + NOT setting hops in remote switch
     *
     *  4. is_real_lid == FALSE && is_main_path == FALSE: 
     *      - illegal state - we shouldn't get here
@@ -2013,12 +2067,25 @@ __osm_ftree_fabric_route_downgoing_by_go
          __osm_ftree_sw_set_fwd_table_block(p_remote_sw,
                                             cl_ntoh16(target_lid),
                                             p_min_port->remote_port_num);
-         p_remote_sw->lft_buf[cl_ntoh16(target_lid)] = p_min_port->remote_port_num;
          osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
                  "__osm_ftree_fabric_route_downgoing_by_going_up: "
                  "Switch %s: set path to HCA LID 0x%x through port %u\n",
                  __osm_ftree_tuple_to_str(p_remote_sw->tuple),
                  cl_ntoh16(target_lid),p_min_port->remote_port_num);
+
+         /* On the remote switch that is pointed by the min_group,
+            set hops for ALL the ports in the remote group. */
+
+         ports_num = (uint16_t)cl_ptr_vector_get_size(&p_min_group->ports);
+         for (j = 0; j < ports_num; j++)
+         {
+            cl_ptr_vector_at(&p_min_group->ports, j, (void **)&p_port);
+            __osm_ftree_sw_set_hops(p_remote_sw,
+                                    p_ftree->lft_max_lid_ho,
+                                    cl_ntoh16(target_lid),
+                                    p_port->remote_port_num,
+                                    target_rank - p_remote_sw->rank);
+         }
       }
 
       /* Recursion step: 
@@ -2028,6 +2095,7 @@ __osm_ftree_fabric_route_downgoing_by_go
             p_remote_sw,    /* remote switch - used as a route-downgoing alg. next step point */
             p_sw,           /* this switch - prev. position switch for the function */
             target_lid,     /* LID that we're routing to */
+            target_rank,    /* rank of the LID that we're routing to */
             is_real_lid,    /* whether this target LID is real or dummy */
             is_main_path);  /* whether this is path to HCA that should by tracked by counters */
    }
@@ -2044,6 +2112,8 @@ __osm_ftree_fabric_route_downgoing_by_go
     *        configured for this target LID
     *         + NOT promoting port counter
     *         + setting path in remote switch fwd tbl if it hasn't been set yet
+    *         + setting hops in remote switch on all the ports of each group
+    *           if it hasn't been set yet
     *      
     *  2. is_real_lid == TRUE && is_main_path == FALSE: 
     *      - going UP(TRUE,FALSE) on ALL the groups, each time on port 0,
@@ -2051,6 +2121,8 @@ __osm_ftree_fabric_route_downgoing_by_go
     *        configured for this target LID
     *         + NOT promoting port counter
     *         + setting path in remote switch fwd tbl if it hasn't been set yet
+    *         + setting hops in remote switch on all the ports of each group
+    *           if it hasn't been set yet
     *
     *  These two rules can be rephrased this way:
     *   - foreach UP port group
@@ -2060,6 +2132,7 @@ __osm_ftree_fabric_route_downgoing_by_go
     *         - select port 0
     *         - do NOT promote port counter
     *         - set path in remote switch fwd tbl
+    *         - set hops in remote switch on all the ports of this group
     *         - go UP(TRUE,FALSE) to the remote switch
     */
 
@@ -2087,6 +2160,22 @@ __osm_ftree_fabric_route_downgoing_by_go
       __osm_ftree_sw_set_fwd_table_block(p_remote_sw,
                                          cl_ntoh16(target_lid),
                                          p_port->remote_port_num);
+
+      /* On the remote switch that is pointed by the p_group,
+         set hops for ALL the ports in the remote group. */
+
+      ports_num = (uint16_t)cl_ptr_vector_get_size(&p_group->ports);
+      for (j = 0; j < ports_num; j++)
+      {
+         cl_ptr_vector_at(&p_group->ports, j, (void **)&p_port);
+
+         __osm_ftree_sw_set_hops(p_remote_sw,
+                                 p_ftree->lft_max_lid_ho,
+                                 cl_ntoh16(target_lid),
+                                 p_port->remote_port_num,
+                                 target_rank - p_remote_sw->rank);
+      }
+
       /* Recursion step: 
          Assign downgoing ports by stepping up, starting on REMOTE switch. */
       __osm_ftree_fabric_route_downgoing_by_going_up(
@@ -2094,6 +2183,7 @@ __osm_ftree_fabric_route_downgoing_by_go
             p_remote_sw, /* remote switch - used as a route-downgoing alg. next step point */
             p_sw,        /* this switch - prev. position switch for the function */
             target_lid,  /* LID that we're routing to */
+            target_rank, /* rank of the LID that we're routing to */
             TRUE,        /* whether the target LID is real or dummy */
             FALSE);      /* whether this is path to HCA that should by tracked by counters */
    }
@@ -2150,17 +2240,26 @@ __osm_ftree_fabric_route_to_hcas(
                  cl_ntoh16(remote_lid),
                  p_port->port_num);
 
+         /* set local min hop table(LID) to route to the CA */
+         __osm_ftree_sw_set_hops(p_sw,
+                                 p_ftree->lft_max_lid_ho,
+                                 cl_ntoh16(remote_lid),
+                                 p_port->port_num,
+                                 1);
+
          /* assign downgoing ports by stepping up */
          __osm_ftree_fabric_route_downgoing_by_going_up(
                p_ftree,
                p_sw,       /* local switch - used as a route-downgoing alg. start point */
                NULL,       /* prev. position switch */
                remote_lid, /* LID that we're routing to */
+               __osm_ftree_fabric_get_rank(p_ftree), /* rank of the LID that we're routing to */
                TRUE,       /* whether this HCA LID is real or dummy */
                TRUE);      /* whether this path to HCA should by tracked by counters */
       }
 
-      /* We're done with the real HCAs. Now route the dummy HCAs that are missing.*/
+      /* We're done with the real HCAs. Now route the dummy HCAs that are missing.
+         When routing to dummy HCAs we don't fill lid matrices. */
 
       if (p_ftree->max_hcas_per_leaf > p_sw->down_port_groups_num)
       {
@@ -2177,6 +2276,7 @@ __osm_ftree_fabric_route_to_hcas(
                   p_sw,    /* local switch - used as a route-downgoing alg. start point */
                   NULL,    /* prev. position switch */
                   0,       /* LID that we're routing to - ignored for dummy HCA */
+                  0,       /* rank of the LID that we're routing to - ignored for dummy HCA */
                   FALSE,   /* whether this HCA LID is real or dummy */
                   TRUE);   /* whether this path to HCA should by tracked by counters */
          }
@@ -2226,11 +2326,19 @@ __osm_ftree_fabric_route_to_switches(
               __osm_ftree_tuple_to_str(p_sw->tuple),
               cl_ntoh16(p_sw->base_lid));
 
+      /* set min hop table of the switch to itself */
+      __osm_ftree_sw_set_hops(p_sw,
+                              p_ftree->lft_max_lid_ho,
+                              cl_ntoh16(p_sw->base_lid),
+                              0, /* port_num */
+                              0);/* hops     */
+
       __osm_ftree_fabric_route_downgoing_by_going_up(
             p_ftree,
             p_sw,           /* local switch - used as a route-downgoing alg. start point */
             NULL,           /* prev. position switch */
             p_sw->base_lid, /* LID that we're routing to */
+            p_sw->rank,     /* rank of the LID that we're routing to */
             TRUE,           /* whether the target LID is a real or dummy */
             FALSE);         /* whether this path should by tracked by counters */
    }
@@ -2971,48 +3079,6 @@ __osm_ftree_do_routing(
 /***************************************************
  ***************************************************/
 
-static int 
-__osm_ftree_routing(
-   IN  void * context)
-{
-   int status = __osm_ftree_construct_fabric(context);
-   if (status != 0)
-      return status;
-
-   __osm_ftree_do_routing(context);
-   return 0;
-}
-
-/***************************************************
- ***************************************************/
-
-void
-ucast_mgr_dump_to_file(
-   IN  osm_ucast_mgr_t *p_mgr,
-   IN  const char *file_name,
-   IN  void (*func)(cl_map_item_t *, void *));
-
-void
-ucast_mgr_dump_lfts(
-   IN  cl_map_item_t *p_map_item,
-   void *cxt);
-
-static void 
-__osm_ftree_dump_tables(
-   IN  void * context)
-{
-   ftree_fabric_t * p_ftree = context;
-   if (!p_ftree)
-      return;
-
-   ucast_mgr_dump_to_file(&p_ftree->p_osm->sm.ucast_mgr,
-                          "opensm-lfts.dump",
-                          ucast_mgr_dump_lfts);
-}
-
-/***************************************************
- ***************************************************/
-
 static void 
 __osm_ftree_delete(
    IN  void * context)
@@ -3034,17 +3100,8 @@ int osm_ucast_ftree_setup(osm_opensm_t *
    p_ftree->p_osm = p_osm;
 
    p_osm->routing_engine.context = (void *)p_ftree;
-   p_osm->routing_engine.ucast_build_fwd_tables = __osm_ftree_routing;
-   /* ToDo: Resolve multicast routing. 
-    *       Until then lid matrices are built, despite the
-    *       fact that FatTree routing doesn't need them.
-    *       When the multicast routing will be resolved,
-    *       __osm_ftree_routing() function should be removed,
-    *       and here's how the FatTree routing will be set:
-    *  p_osm->routing_engine.build_lid_matrices = __osm_ftree_construct_fabric;
-    *  p_osm->routing_engine.ucast_build_fwd_tables = __osm_ftree_do_routing;
-    */
-   p_osm->routing_engine.ucast_dump_tables = __osm_ftree_dump_tables;
+   p_osm->routing_engine.build_lid_matrices = __osm_ftree_construct_fabric;
+   p_osm->routing_engine.ucast_build_fwd_tables = __osm_ftree_do_routing;
    p_osm->routing_engine.delete = __osm_ftree_delete;
    return 0;
 }
-- 
1.4.4.1.GIT





More information about the general mailing list