[ofa-general] [PATCH] IB/core: Enhance SMI for switch support
Hal Rosenstock
halr at voltaire.com
Mon Mar 26 15:53:54 PDT 2007
IB/core: Enhance SMI for switch support
SMI is being extended for switch (intermediate hop) support. Care has
been taken to ensure the CA (and router) code paths are as identical as
possible to how they were prior to adding this support.
This has been lightly tested on Tavor HCAs. Is it possible to get
regession testing of this on a wider set of Mellanox HCAs ? Also, can
someone test this on iPath ?
Signed-off-by: Suresh Shelvapille <suri at baymicrosystems.com>
Signed-off-by: Hal Rosenstock <halr at voltaire.com>
diff --git a/drivers/infiniband/core/agent.c b/drivers/infiniband/core/agent.c
index ecd1a30..7583941 100644
--- a/drivers/infiniband/core/agent.c
+++ b/drivers/infiniband/core/agent.c
@@ -3,7 +3,7 @@
* Copyright (c) 2004, 2005 Infinicon Corporation. All rights reserved.
* Copyright (c) 2004, 2005 Intel Corporation. All rights reserved.
* Copyright (c) 2004, 2005 Topspin Corporation. All rights reserved.
- * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved.
+ * Copyright (c) 2004-2007 Voltaire Corporation. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -34,7 +34,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
- * $Id: agent.c 1389 2004-12-27 22:56:47Z roland $
*/
#include <linux/slab.h>
@@ -42,6 +41,7 @@
#include "agent.h"
#include "smi.h"
+#include "mad_priv.h"
#define SPFX "ib_agent: "
@@ -87,8 +87,13 @@ int agent_send_response(struct ib_mad *m
struct ib_mad_send_buf *send_buf;
struct ib_ah *ah;
int ret;
+ struct ib_mad_send_wr_private *mad_send_wr;
+
+ if (device->node_type == RDMA_NODE_IB_SWITCH)
+ port_priv = ib_get_agent_port(device, 0);
+ else
+ port_priv = ib_get_agent_port(device, port_num);
- port_priv = ib_get_agent_port(device, port_num);
if (!port_priv) {
printk(KERN_ERR SPFX "Unable to find port agent\n");
return -ENODEV;
@@ -113,6 +118,14 @@ int agent_send_response(struct ib_mad *m
memcpy(send_buf->mad, mad, sizeof *mad);
send_buf->ah = ah;
+
+ if (device->node_type == RDMA_NODE_IB_SWITCH){
+ mad_send_wr = container_of(send_buf,
+ struct ib_mad_send_wr_private,
+ send_buf);
+ mad_send_wr->send_wr.wr.ud.port_num = port_num;
+ }
+
if ((ret = ib_post_send_mad(send_buf, NULL))) {
printk(KERN_ERR SPFX "ib_post_send_mad error:%d\n", ret);
goto err2;
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index 13efd41..7856e1a 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
* Copyright (c) 2005 Intel Corporation. All rights reserved.
* Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
*
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
- * $Id: mad.c 5596 2006-03-03 01:00:07Z sean.hefty $
*/
#include <linux/dma-mapping.h>
#include <rdma/ib_cache.h>
@@ -676,10 +675,16 @@ static int handle_outgoing_dr_smp(struct
struct ib_mad_port_private *port_priv;
struct ib_mad_agent_private *recv_mad_agent = NULL;
struct ib_device *device = mad_agent_priv->agent.device;
- u8 port_num = mad_agent_priv->agent.port_num;
+ u8 port_num;
struct ib_wc mad_wc;
struct ib_send_wr *send_wr = &mad_send_wr->send_wr;
+ if (device->node_type == RDMA_NODE_IB_SWITCH &&
+ smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
+ port_num = send_wr->wr.ud.port_num;
+ else
+ port_num = mad_agent_priv->agent.port_num;
+
/*
* Directed route handling starts if the initial LID routed part of
* a request or the ending LID routed part of a response is empty.
@@ -693,6 +698,7 @@ static int handle_outgoing_dr_smp(struct
printk(KERN_ERR PFX "Invalid directed route\n");
goto out;
}
+
/* Check to post send on QP or process locally */
ret = smi_check_local_smp(smp, device);
if (!ret)
@@ -1839,6 +1845,7 @@ static void ib_mad_recv_done_handler(str
struct ib_mad_private *recv, *response;
struct ib_mad_list_head *mad_list;
struct ib_mad_agent_private *mad_agent;
+ int port_num;
response = kmem_cache_alloc(ib_mad_cache, GFP_KERNEL);
if (!response)
@@ -1872,21 +1879,49 @@ static void ib_mad_recv_done_handler(str
if (!validate_mad(&recv->mad.mad, qp_info->qp->qp_num))
goto out;
+ if (port_priv->device->node_type == RDMA_NODE_IB_SWITCH)
+ port_num = wc->port_num;
+ else
+ port_num = port_priv->port_num;
+
if (recv->mad.mad.mad_hdr.mgmt_class ==
IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
+ int retsmi;
+
if (!smi_handle_dr_smp_recv(&recv->mad.smp,
port_priv->device->node_type,
- port_priv->port_num,
+ port_num,
port_priv->device->phys_port_cnt))
goto out;
- if (!smi_check_forward_dr_smp(&recv->mad.smp))
+
+ retsmi = smi_check_forward_dr_smp(&recv->mad.smp);
+ if (!retsmi)
goto local;
- if (!smi_handle_dr_smp_send(&recv->mad.smp,
- port_priv->device->node_type,
- port_priv->port_num))
- goto out;
- if (!smi_check_local_smp(&recv->mad.smp, port_priv->device))
+
+ if (retsmi == 1) { /* don't forward */
+ if (!smi_handle_dr_smp_send(&recv->mad.smp,
+ port_priv->device->node_type,
+ port_num))
+ goto out;
+
+ if (!smi_check_local_smp(&recv->mad.smp, port_priv->device))
+ goto out;
+ } else if (port_priv->device->node_type == RDMA_NODE_IB_SWITCH) {
+ /* forward case for switches */
+ memcpy(response, recv, sizeof(*response));
+ response->header.recv_wc.wc = &response->header.wc;
+ response->header.recv_wc.recv_buf.mad = &response->mad.mad;
+ response->header.recv_wc.recv_buf.grh = &response->grh;
+
+ if (!agent_send_response(&response->mad.mad,
+ &response->grh, wc,
+ port_priv->device,
+ smi_get_fwd_port(&recv->mad.smp),
+ qp_info->qp->qp_num))
+ response = NULL;
+
goto out;
+ }
}
local:
@@ -1915,7 +1950,7 @@ local:
agent_send_response(&response->mad.mad,
&recv->grh, wc,
port_priv->device,
- port_priv->port_num,
+ port_num,
qp_info->qp->qp_num);
goto out;
}
diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c
index 54b81e1..654deae 100644
--- a/drivers/infiniband/core/smi.c
+++ b/drivers/infiniband/core/smi.c
@@ -3,7 +3,7 @@
* Copyright (c) 2004, 2005 Infinicon Corporation. All rights reserved.
* Copyright (c) 2004, 2005 Intel Corporation. All rights reserved.
* Copyright (c) 2004, 2005 Topspin Corporation. All rights reserved.
- * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved.
+ * Copyright (c) 2004-2007 Voltaire Corporation. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -34,7 +34,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
- * $Id: smi.c 1389 2004-12-27 22:56:47Z roland $
*/
#include <rdma/ib_smi.h>
@@ -202,6 +201,7 @@ int smi_handle_dr_smp_recv(struct ib_smp
/*
* Return 1 if the received DR SMP should be forwarded to the send queue
* Return 0 if the SMP should be completed up the stack
+ * Return 2 if the SMP should be forwarded (for switches only)
*/
int smi_check_forward_dr_smp(struct ib_smp *smp)
{
@@ -213,7 +213,7 @@ int smi_check_forward_dr_smp(struct ib_s
if (!ib_get_smp_direction(smp)) {
/* C14-9:2 -- intermediate hop */
if (hop_ptr && hop_ptr < hop_cnt)
- return 1;
+ return 2;
/* C14-9:3 -- at the end of the DR segment of path */
if (hop_ptr == hop_cnt)
@@ -223,9 +223,9 @@ int smi_check_forward_dr_smp(struct ib_s
if (hop_ptr == hop_cnt + 1)
return 1;
} else {
- /* C14-13:2 */
+ /* C14-13:2 -- intermediate hop */
if (2 <= hop_ptr && hop_ptr <= hop_cnt)
- return 1;
+ return 2;
/* C14-13:3 -- at the end of the DR segment of path */
if (hop_ptr == 1)
diff --git a/drivers/infiniband/core/smi.h b/drivers/infiniband/core/smi.h
index 3011bfd..0cf0f19 100644
--- a/drivers/infiniband/core/smi.h
+++ b/drivers/infiniband/core/smi.h
@@ -3,7 +3,7 @@
* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
* Copyright (c) 2004 Intel Corporation. All rights reserved.
* Copyright (c) 2004 Topspin Corporation. All rights reserved.
- * Copyright (c) 2004 Voltaire Corporation. All rights reserved.
+ * Copyright (c) 2004-2007 Voltaire Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -33,7 +33,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
- * $Id: smi.h 1389 2004-12-27 22:56:47Z roland $
*/
#ifndef __SMI_H_
@@ -63,4 +62,14 @@ static inline int smi_check_local_smp(st
(smp->hop_ptr == smp->hop_cnt + 1)));
}
+/*
+ * Return the forwarding port number from initial_path for outgoing SMP and
+ * from return_path for returning SMP
+ */
+static inline int smi_get_fwd_port(struct ib_smp *smp)
+{
+ return (!ib_get_smp_direction(smp) ? smp->initial_path[smp->hop_ptr+1] :
+ smp->return_path[smp->hop_ptr-1]);
+}
+
#endif /* __SMI_H_ */
More information about the general
mailing list