[openib-general] [PATCH] mad: use GID/LID on requester side when matching responses to requests
Jack Morgenstein
jackm at mellanox.co.il
Thu Apr 6 04:51:18 PDT 2006
Check GID/LID for requester side when searching for request which matches
received response. This, in order to guarantee uniqueness if use same TID
when requesting via multiple source LIDs (when LMC is not zero). To perform
check, need to add LMC to cache.
(previous patch returned OK unconditionally for LID check, when the check is
performed at the requesting node).
Further, do not perform LID check for direct-routed packets, since permissive
LID makes a proper check impossible.
Signed-off-by: Jack Morgenstein <jackm at mellanox.co.il>
Index: src/drivers/infiniband/include/rdma/ib_verbs.h
===================================================================
--- src/drivers/infiniband/include/rdma/ib_verbs.h (revision 6066)
+++ src/drivers/infiniband/include/rdma/ib_verbs.h (working copy)
@@ -822,6 +822,7 @@ struct ib_cache {
struct ib_event_handler event_handler;
struct ib_pkey_cache **pkey_cache;
struct ib_gid_cache **gid_cache;
+ struct ib_lmc_cache *lmc_cache;
};
struct ib_device {
Index: src/drivers/infiniband/include/rdma/ib_cache.h
===================================================================
--- src/drivers/infiniband/include/rdma/ib_cache.h (revision 6066)
+++ src/drivers/infiniband/include/rdma/ib_cache.h (working copy)
@@ -102,4 +102,17 @@ int ib_find_cached_pkey(struct ib_device
u16 pkey,
u16 *index);
+/**
+ * ib_get_cached_lmc - Returns a cached lmc table entry
+ * @device: The device to query.
+ * @port_num: The port number of the device to query.
+ * @lmc: The lmc value for the specified port for that device.
+ *
+ * ib_get_cached_lmc() fetches the specified lmc table entry stored in
+ * the local software cache.
+ */
+int ib_get_cached_lmc(struct ib_device *device,
+ u8 port_num,
+ u8 *lmc);
+
#endif /* _IB_CACHE_H */
Index: src/drivers/infiniband/core/cache.c
===================================================================
--- src/drivers/infiniband/core/cache.c (revision 6066)
+++ src/drivers/infiniband/core/cache.c (working copy)
@@ -59,6 +59,10 @@ struct ib_update_work {
u8 port_num;
};
+struct ib_lmc_cache {
+ u8 lmc[0];
+};
+
static inline int start_port(struct ib_device *device)
{
return (device->node_type == RDMA_NODE_IB_SWITCH) ? 0 : 1;
@@ -191,6 +195,24 @@ int ib_find_cached_pkey(struct ib_device
}
EXPORT_SYMBOL(ib_find_cached_pkey);
+int ib_get_cached_lmc(struct ib_device *device,
+ u8 port_num,
+ u8 *lmc)
+{
+ unsigned long flags;
+ int ret = 0;
+
+ if (port_num < start_port(device) || port_num > end_port(device))
+ return -EINVAL;
+
+ read_lock_irqsave(&device->cache.lock, flags);
+ *lmc = device->cache.lmc_cache->lmc[port_num - start_port(device)];
+ read_unlock_irqrestore(&device->cache.lock, flags);
+
+ return ret;
+}
+EXPORT_SYMBOL(ib_get_cached_lmc);
+
static void ib_cache_update(struct ib_device *device,
u8 port)
{
@@ -251,6 +273,8 @@ static void ib_cache_update(struct ib_de
device->cache.pkey_cache[port - start_port(device)] = pkey_cache;
device->cache.gid_cache [port - start_port(device)] = gid_cache;
+ device->cache.lmc_cache->lmc[port - start_port(device)] = tprops->lmc;
+
write_unlock_irq(&device->cache.lock);
kfree(old_pkey_cache);
@@ -305,7 +329,13 @@ static void ib_cache_setup_one(struct ib
kmalloc(sizeof *device->cache.pkey_cache *
(end_port(device) - start_port(device) + 1), GFP_KERNEL);
- if (!device->cache.pkey_cache || !device->cache.gid_cache) {
+ device->cache.lmc_cache = kmalloc(sizeof *device->cache.lmc_cache *
+ (end_port(device) -
+ start_port(device) + 1),
+ GFP_KERNEL);
+
+ if (!device->cache.pkey_cache || !device->cache.gid_cache ||
+ !device->cache.lmc_cache) {
printk(KERN_WARNING "Couldn't allocate cache "
"for %s\n", device->name);
goto err;
@@ -333,6 +363,7 @@ err_cache:
err:
kfree(device->cache.pkey_cache);
kfree(device->cache.gid_cache);
+ kfree(device->cache.lmc_cache);
}
static void ib_cache_cleanup_one(struct ib_device *device)
@@ -349,6 +380,7 @@ static void ib_cache_cleanup_one(struct
kfree(device->cache.pkey_cache);
kfree(device->cache.gid_cache);
+ kfree(device->cache.lmc_cache);
}
static struct ib_client cache_client = {
Index: src/drivers/infiniband/core/mad.c
===================================================================
--- src/drivers/infiniband/core/mad.c (revision 6066)
+++ src/drivers/infiniband/core/mad.c (working copy)
@@ -34,6 +34,7 @@
* $Id$
*/
#include <linux/dma-mapping.h>
+#include <rdma/ib_cache.h>
#include "mad_priv.h"
#include "mad_rmpp.h"
@@ -1669,20 +1670,21 @@ static inline int rcv_has_same_class(str
rwc->recv_buf.mad->mad_hdr.mgmt_class;
}
-static inline int rcv_has_same_gid(struct ib_mad_send_wr_private *wr,
+static inline int rcv_has_same_gid(struct ib_mad_agent_private
*mad_agent_priv,
+ struct ib_mad_send_wr_private *wr,
struct ib_mad_recv_wc *rwc )
{
struct ib_ah_attr attr;
u8 send_resp, rcv_resp;
+ union ib_gid sgid;
+ struct ib_device *device = mad_agent_priv->agent.device;
+ u8 port_num = mad_agent_priv->agent.port_num;
+ u8 lmc;
send_resp = ((struct ib_mad *)(wr->send_buf.mad))->
mad_hdr.method & IB_MGMT_METHOD_RESP;
rcv_resp = rwc->recv_buf.mad->mad_hdr.method & IB_MGMT_METHOD_RESP;
- if (!send_resp && rcv_resp)
- /* is request/response. GID/LIDs are both local (same). */
- return 1;
-
if (send_resp == rcv_resp)
/* both requests, or both responses. GIDs different */
return 0;
@@ -1691,48 +1693,78 @@ static inline int rcv_has_same_gid(struc
/* Assume not equal, to avoid false positives. */
return 0;
- if (!(attr.ah_flags & IB_AH_GRH) && !(rwc->wc->wc_flags & IB_WC_GRH))
- return attr.dlid == rwc->wc->slid;
- else if ((attr.ah_flags & IB_AH_GRH) &&
- (rwc->wc->wc_flags & IB_WC_GRH))
- return memcmp(attr.grh.dgid.raw,
- rwc->recv_buf.grh->sgid.raw, 16) == 0;
- else
+ if (!!(attr.ah_flags & IB_AH_GRH) !=
+ !!(rwc->wc->wc_flags & IB_WC_GRH))
/* one has GID, other does not. Assume different */
return 0;
+
+ if (!send_resp && rcv_resp) {
+ /* is request/response. */
+ if (!(attr.ah_flags & IB_AH_GRH)) {
+ if (ib_get_cached_lmc(device, port_num, &lmc))
+ return 0;
+ return (!lmc || !((attr.src_path_bits ^
+ rwc->wc->dlid_path_bits) &
+ ((1 << lmc) - 1)));
+ } else {
+ if (ib_get_cached_gid(device, port_num,
+ attr.grh.sgid_index, &sgid))
+ return 0;
+ return !memcmp(sgid.raw, rwc->recv_buf.grh->dgid.raw,
+ 16);
+ }
+ }
+
+ if (!(attr.ah_flags & IB_AH_GRH))
+ return attr.dlid == rwc->wc->slid;
+ else
+ return !memcmp(attr.grh.dgid.raw, rwc->recv_buf.grh->sgid.raw,
+ 16);
}
+
+static inline int is_direct(u8 class)
+{
+ return (class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE);
+}
+
struct ib_mad_send_wr_private*
ib_find_send_mad(struct ib_mad_agent_private *mad_agent_priv,
- struct ib_mad_recv_wc *mad_recv_wc)
+ struct ib_mad_recv_wc *wc)
{
- struct ib_mad_send_wr_private *mad_send_wr;
+ struct ib_mad_send_wr_private *wr;
struct ib_mad *mad;
- mad = (struct ib_mad *)mad_recv_wc->recv_buf.mad;
+ mad = (struct ib_mad *)wc->recv_buf.mad;
- list_for_each_entry(mad_send_wr, &mad_agent_priv->wait_list,
- agent_list) {
- if ((mad_send_wr->tid == mad->mad_hdr.tid) &&
- rcv_has_same_class(mad_send_wr, mad_recv_wc) &&
- rcv_has_same_gid(mad_send_wr, mad_recv_wc))
- return mad_send_wr;
+ list_for_each_entry(wr, &mad_agent_priv->wait_list, agent_list) {
+ if ((wr->tid == mad->mad_hdr.tid) &&
+ rcv_has_same_class(wr, wc) &&
+ /*
+ * Don't check GID for direct routed MADs.
+ * These might have permissive LIDs.
+ */
+ (is_direct(wc->recv_buf.mad->mad_hdr.mgmt_class) ||
+ rcv_has_same_gid(mad_agent_priv, wr, wc)))
+ return wr;
}
/*
* It's possible to receive the response before we've
* been notified that the send has completed
*/
- list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list,
- agent_list) {
- if (is_data_mad(mad_agent_priv, mad_send_wr->send_buf.mad) &&
- mad_send_wr->tid == mad->mad_hdr.tid &&
- mad_send_wr->timeout &&
- rcv_has_same_class(mad_send_wr, mad_recv_wc) &&
- rcv_has_same_gid(mad_send_wr, mad_recv_wc)) {
+ list_for_each_entry(wr, &mad_agent_priv->send_list, agent_list) {
+ if (is_data_mad(mad_agent_priv, wr->send_buf.mad) &&
+ wr->tid == mad->mad_hdr.tid &&
+ wr->timeout &&
+ rcv_has_same_class(wr, wc) &&
+ /*
+ * Don't check GID for direct routed MADs.
+ * These might have permissive LIDs.
+ */
+ (is_direct(wc->recv_buf.mad->mad_hdr.mgmt_class) ||
+ rcv_has_same_gid(mad_agent_priv, wr, wc)))
/* Verify request has not been canceled */
- return (mad_send_wr->status == IB_WC_SUCCESS) ?
- mad_send_wr : NULL;
- }
+ return (wr->status == IB_WC_SUCCESS) ? wr : NULL;
}
return NULL;
}
More information about the general
mailing list