[ofa-general] [PATCH] IB/CM: add support for routed paths
Hal Rosenstock
hrosenstock at xsigo.com
Tue Dec 11 04:59:45 PST 2007
Sean,
On Mon, 2007-12-10 at 15:53 -0800, Sean Hefty wrote:
> Paths with hop_limit > 1 indicate that the connection will be routed
> between IB subnets. Update the subnet local field in the CM REQ
> based on the hop_limit value. In addition, if the path is routed, then
> set the LIDs in the REQ to the permissive LIDs. This is used to indicate
> to the passive side that it should use the LIDs in the received
> local route header (LRH) associated with the REQ when programming the
> QP.
Just wondering:
Could the subnet local component(s) in the REQ be used on the passive
side instead of permissive LIDs ? That might be more "standard" than
using permissive LIDs.
-- Hal
> This is a temporary work-around to the IB CM to support IB router
> development until the IB router specification is completed. It is not
> anticipated that this work-around will cause any interoperability
> issues with existing stacks or future stacks that will properly support
> IB routers when defined.
>
> Signed-off-by: Sean Hefty <sean.hefty at intel.com>
> ---
> This is a related patch to add support for IB routers in the IB CM. This
> patch differs from previous patches in that the changes are limited to
> the ib_cm module only, and the ib_cm interface remains unchanged. I believe
> that changes to the interface and ULPs will eventually be needed, but
> it would be better to defer those changes once the router spec is more
> defined.
>
> This patch and those posted by Rolf are also available at:
>
> git://git.openfabrics.org/~shefty/rdma-dev.git ib_router
>
>
> drivers/infiniband/core/cm.c | 91 +++++++++++++++++++++++++++++++-----------
> 1 files changed, 67 insertions(+), 24 deletions(-)
>
> diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
> index 2e39236..c3212b9 100644
> --- a/drivers/infiniband/core/cm.c
> +++ b/drivers/infiniband/core/cm.c
> @@ -1,5 +1,5 @@
> /*
> - * Copyright (c) 2004-2006 Intel Corporation. All rights reserved.
> + * Copyright (c) 2004-2007 Intel Corporation. All rights reserved.
> * Copyright (c) 2004 Topspin Corporation. All rights reserved.
> * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved.
> * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
> @@ -895,6 +895,9 @@ static void cm_format_req(struct cm_req_msg *req_msg,
> struct cm_id_private *cm_id_priv,
> struct ib_cm_req_param *param)
> {
> + struct ib_sa_path_rec *pri_path = param->primary_path;
> + struct ib_sa_path_rec *alt_path = param->alternate_path;
> +
> cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
> cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_REQ));
>
> @@ -918,35 +921,46 @@ static void cm_format_req(struct cm_req_msg *req_msg,
> cm_req_set_max_cm_retries(req_msg, param->max_cm_retries);
> cm_req_set_srq(req_msg, param->srq);
>
> - req_msg->primary_local_lid = param->primary_path->slid;
> - req_msg->primary_remote_lid = param->primary_path->dlid;
> - req_msg->primary_local_gid = param->primary_path->sgid;
> - req_msg->primary_remote_gid = param->primary_path->dgid;
> - cm_req_set_primary_flow_label(req_msg, param->primary_path->flow_label);
> - cm_req_set_primary_packet_rate(req_msg, param->primary_path->rate);
> - req_msg->primary_traffic_class = param->primary_path->traffic_class;
> - req_msg->primary_hop_limit = param->primary_path->hop_limit;
> - cm_req_set_primary_sl(req_msg, param->primary_path->sl);
> - cm_req_set_primary_subnet_local(req_msg, 1); /* local only... */
> + if (pri_path->hop_limit <= 1) {
> + req_msg->primary_local_lid = pri_path->slid;
> + req_msg->primary_remote_lid = pri_path->dlid;
> + } else {
> + /* Work-around until there's a way to obtain remote LID info */
> + req_msg->primary_local_lid = IB_LID_PERMISSIVE;
> + req_msg->primary_remote_lid = IB_LID_PERMISSIVE;
> + }
> + req_msg->primary_local_gid = pri_path->sgid;
> + req_msg->primary_remote_gid = pri_path->dgid;
> + cm_req_set_primary_flow_label(req_msg, pri_path->flow_label);
> + cm_req_set_primary_packet_rate(req_msg, pri_path->rate);
> + req_msg->primary_traffic_class = pri_path->traffic_class;
> + req_msg->primary_hop_limit = pri_path->hop_limit;
> + cm_req_set_primary_sl(req_msg, pri_path->sl);
> + cm_req_set_primary_subnet_local(req_msg, (pri_path->hop_limit <= 1));
> cm_req_set_primary_local_ack_timeout(req_msg,
> cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
> - param->primary_path->packet_life_time));
> + pri_path->packet_life_time));
>
> - if (param->alternate_path) {
> - req_msg->alt_local_lid = param->alternate_path->slid;
> - req_msg->alt_remote_lid = param->alternate_path->dlid;
> - req_msg->alt_local_gid = param->alternate_path->sgid;
> - req_msg->alt_remote_gid = param->alternate_path->dgid;
> + if (alt_path) {
> + if (alt_path->hop_limit <= 1) {
> + req_msg->alt_local_lid = alt_path->slid;
> + req_msg->alt_remote_lid = alt_path->dlid;
> + } else {
> + req_msg->alt_local_lid = IB_LID_PERMISSIVE;
> + req_msg->alt_remote_lid = IB_LID_PERMISSIVE;
> + }
> + req_msg->alt_local_gid = alt_path->sgid;
> + req_msg->alt_remote_gid = alt_path->dgid;
> cm_req_set_alt_flow_label(req_msg,
> - param->alternate_path->flow_label);
> - cm_req_set_alt_packet_rate(req_msg, param->alternate_path->rate);
> - req_msg->alt_traffic_class = param->alternate_path->traffic_class;
> - req_msg->alt_hop_limit = param->alternate_path->hop_limit;
> - cm_req_set_alt_sl(req_msg, param->alternate_path->sl);
> - cm_req_set_alt_subnet_local(req_msg, 1); /* local only... */
> + alt_path->flow_label);
> + cm_req_set_alt_packet_rate(req_msg, alt_path->rate);
> + req_msg->alt_traffic_class = alt_path->traffic_class;
> + req_msg->alt_hop_limit = alt_path->hop_limit;
> + cm_req_set_alt_sl(req_msg, alt_path->sl);
> + cm_req_set_alt_subnet_local(req_msg, (alt_path->hop_limit <= 1));
> cm_req_set_alt_local_ack_timeout(req_msg,
> cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
> - param->alternate_path->packet_life_time));
> + alt_path->packet_life_time));
> }
>
> if (param->private_data && param->private_data_len)
> @@ -1359,6 +1373,34 @@ out:
> return listen_cm_id_priv;
> }
>
> +/*
> + * Work-around for inter-subnet connections. If the LIDs are permissive,
> + * we need to override the LID/SL data in the REQ with the LID information
> + * in the work completion.
> + */
> +static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
> +{
> + if (!cm_req_get_primary_subnet_local(req_msg)) {
> + if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) {
> + req_msg->primary_local_lid = cpu_to_be16(wc->slid);
> + cm_req_set_primary_sl(req_msg, wc->sl);
> + }
> +
> + if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE)
> + req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits);
> + }
> +
> + if (!cm_req_get_alt_subnet_local(req_msg)) {
> + if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) {
> + req_msg->alt_local_lid = cpu_to_be16(wc->slid);
> + cm_req_set_alt_sl(req_msg, wc->sl);
> + }
> +
> + if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE)
> + req_msg->alt_remote_lid = cpu_to_be16(wc->dlid_path_bits);
> + }
> +}
> +
> static int cm_req_handler(struct cm_work *work)
> {
> struct ib_cm_id *cm_id;
> @@ -1399,6 +1441,7 @@ static int cm_req_handler(struct cm_work *work)
> cm_id_priv->id.service_id = req_msg->service_id;
> cm_id_priv->id.service_mask = __constant_cpu_to_be64(~0ULL);
>
> + cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
> cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]);
> ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av);
> if (ret) {
>
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
>
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
More information about the general
mailing list