[ofa-general] [PATCH 2/3 v2] remove ib pkey gid and lmc cache
Yosef Etigin
yosefe at voltaire.com
Thu May 3 02:40:11 PDT 2007
don't use ib cache in core and ulp's
v1:
* Add ib_find_gid and ib_find_pkey, over uncached device queries
* Modify users of the cache in core and ulp's to use the unchached methods
changes from v2:
* don't remove the cache compelely, and still use it within mthca driver.
the mthca changes and complete removal of the cache - in the next patch
Signed-off-by: Yosef Etigin <yosefe at voltaire.com>
---
drivers/infiniband/core/cm.c | 8 -
drivers/infiniband/core/cma.c | 9 --
drivers/infiniband/core/device.c | 137 ++++++++++++++++++++++++++++++--
drivers/infiniband/core/mad.c | 5 -
drivers/infiniband/core/multicast.c | 3
drivers/infiniband/core/sa_query.c | 3
drivers/infiniband/core/verbs.c | 3
drivers/infiniband/ulp/ipoib/ipoib_cm.c | 3
drivers/infiniband/ulp/ipoib/ipoib_ib.c | 2
drivers/infiniband/ulp/srp/ib_srp.c | 6 -
include/rdma/ib_verbs.h | 37 ++++++++
11 files changed, 184 insertions(+), 32 deletions(-)
Index: b/drivers/infiniband/core/device.c
===================================================================
--- a/drivers/infiniband/core/device.c 2007-05-03 11:25:58.878535870 +0300
+++ b/drivers/infiniband/core/device.c 2007-05-03 12:34:12.639368278 +0300
@@ -149,6 +149,20 @@ static int alloc_name(char *name)
return 0;
}
+
+static inline int start_port(struct ib_device *device)
+{
+ return (device->node_type == RDMA_NODE_IB_SWITCH) ? 0 : 1;
+}
+
+
+static inline int end_port(struct ib_device *device)
+{
+ return (device->node_type == RDMA_NODE_IB_SWITCH) ?
+ 0 : device->phys_port_cnt;
+}
+
+
/**
* ib_alloc_device - allocate an IB device struct
* @size:size of structure to allocate
@@ -592,6 +606,122 @@ int ib_modify_port(struct ib_device *dev
}
EXPORT_SYMBOL(ib_modify_port);
+/**
+ * ib_find_gid - Returns the port number and index of a GID
+ * @device: Device to query.
+ * @gid: GID to look for
+ * @port_num: Returned port number
+ * @index: Returned index
+ *
+ * ib_find_gid() returns the index of @pkey in the pkey table
+ * on port @port_num
+ */
+int ib_find_gid(struct ib_device *device, union ib_gid *gid,
+ u8 *port_num, u16 *index)
+{
+ struct ib_port_attr *tprops = NULL;
+ union ib_gid tmp_gid;
+ int ret;
+ int port;
+ int i;
+
+ tprops = kmalloc(sizeof *tprops, GFP_ATOMIC);
+
+ for (port = start_port(device); port <= end_port(device); ++port) {
+ ret = ib_query_port(device, port, tprops);
+ if (ret)
+ continue;
+
+ for (i = 0; i < tprops->gid_tbl_len; ++i) {
+ ret = ib_query_gid(device, port, i, &tmp_gid);
+ if (ret)
+ goto out;
+ if (!memcmp(&tmp_gid, gid, sizeof *gid)) {
+ *port_num = port;
+ *index = i;
+ ret = 0;
+ goto out;
+ }
+ } /* for i */
+ }
+ ret = -ENOENT;
+out:
+ kfree(tprops);
+ return ret;
+}
+EXPORT_SYMBOL(ib_find_gid);
+
+/**
+ * ib_find_pkey - Returns the index of a PKey on a port
+ * @device: Device to query.
+ * @port_num: Port to query on
+ * @pkey: PKey to look for
+ * @index: Returned index
+ *
+ * ib_find_pkey() returns the index of @pkey in the pkey table
+ * on port @port_num
+ */
+int ib_find_pkey(struct ib_device *device,
+ u8 port_num, u16 pkey, u16 *index)
+{
+ struct ib_port_attr *tprops = NULL;
+ int ret;
+ int i = -1;
+ u16 tmp_pkey;
+
+ tprops = kmalloc(sizeof *tprops, GFP_ATOMIC);
+
+ ret = ib_query_port(device, port_num, tprops);
+ if (ret) {
+ printk(KERN_WARNING "ib_query_port failed , ret = %d\n", ret);
+ goto out;
+ }
+
+ for (i = 0; i < tprops->pkey_tbl_len; ++i) {
+ ret = ib_query_pkey(device, port_num, i, &tmp_pkey);
+ if (ret)
+ goto out;
+
+ if (pkey == tmp_pkey) {
+ *index = i;
+ ret = 0;
+ goto out;
+ }
+ }
+ ret = -ENOENT;
+
+out:
+ kfree(tprops);
+ return ret;
+}
+EXPORT_SYMBOL(ib_find_pkey);
+
+/**
+ * ib_query_lmc - Returns the LMC of a port
+ * @device: Device to query.
+ * @port_num: Port to query on
+ * @lmc: Returned LMC
+ *
+ * ib_query_lmc() returns the LID mask control associated
+ * with port @port_num
+ */
+int ib_query_lmc(struct ib_device *device, u8 port_num, u8 *lmc)
+{
+ struct ib_port_attr *tprops = NULL;
+ int ret;
+
+ tprops = kmalloc(sizeof *tprops, GFP_ATOMIC);
+ ret = ib_query_port(device, port_num, tprops);
+ if (ret) goto err;
+
+ *lmc = tprops->lmc;
+err:
+ kfree(tprops);
+ return ret;
+
+}
+EXPORT_SYMBOL(ib_query_lmc);
+
static int __init ib_core_init(void)
{
int ret;
Index: b/include/rdma/ib_verbs.h
===================================================================
--- a/include/rdma/ib_verbs.h 2007-05-03 11:25:59.090497856 +0300
+++ b/include/rdma/ib_verbs.h 2007-05-03 12:34:01.210405046 +0300
@@ -1134,6 +1134,43 @@ int ib_modify_port(struct ib_device *dev
struct ib_port_modify *port_modify);
/**
+ * ib_find_gid - Returns the port number and index of a GID
+ * @device: Device to query.
+ * @gid: GID to look for
+ * @port_num: Returned port number
+ * @index: Returned index
+ *
+ * ib_find_gid() returns the index of @pkey in the pkey table
+ * on port @port_num
+ */
+int ib_find_gid(struct ib_device *device, union ib_gid *gid,
+ u8 *port_num, u16 *index)
+
+/**
+ * ib_find_pkey - Returns the index of a PKey on a port
+ * @device: Device to query.
+ * @port_num: Port to query on
+ * @pkey: PKey to look for
+ * @index: Returned index
+ *
+ * ib_find_pkey() returns the index of @pkey in the pkey table
+ * on port @port_num
+ */
+int ib_find_pkey(struct ib_device *device,
+ u8 port_num, u16 pkey, u16 *index)
+
+/**
+ * ib_query_lmc - Returns the LMC of a port
+ * @device: Device to query.
+ * @port_num: Port to query on
+ * @lmc: Returned LMC
+ *
+ * ib_query_lmc() returns the LID mask control associated
+ * with port @port_num
+ */
+int ib_query_lmc(struct ib_device *device, u8 port_num, u8 *lmc)
+
+/**
* ib_alloc_pd - Allocates an unused protection domain.
* @device: The device on which to allocate the protection domain.
*
Index: b/drivers/infiniband/core/cm.c
===================================================================
--- a/drivers/infiniband/core/cm.c 2007-05-03 11:25:58.900531925 +0300
+++ b/drivers/infiniband/core/cm.c 2007-05-03 11:32:21.376036145 +0300
@@ -46,8 +46,8 @@
#include <linux/spinlock.h>
#include <linux/workqueue.h>
-#include <rdma/ib_cache.h>
#include <rdma/ib_cm.h>
+#include <rdma/ib_verbs.h>
#include "cm_msgs.h"
MODULE_AUTHOR("Sean Hefty");
@@ -275,7 +275,7 @@ static int cm_init_av_by_path(struct ib_
read_lock_irqsave(&cm.device_lock, flags);
list_for_each_entry(cm_dev, &cm.device_list, list) {
- if (!ib_find_cached_gid(cm_dev->device, &path->sgid,
+ if (!ib_find_gid(cm_dev->device, &path->sgid,
&p, NULL)) {
port = &cm_dev->port[p-1];
break;
@@ -286,7 +286,7 @@ static int cm_init_av_by_path(struct ib_
if (!port)
return -EINVAL;
- ret = ib_find_cached_pkey(cm_dev->device, port->port_num,
+ ret = ib_find_pkey(cm_dev->device, port->port_num,
be16_to_cpu(path->pkey), &av->pkey_index);
if (ret)
return ret;
@@ -1382,7 +1382,7 @@ static int cm_req_handler(struct cm_work
cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]);
ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av);
if (ret) {
- ib_get_cached_gid(work->port->cm_dev->device,
+ ib_query_gid(work->port->cm_dev->device,
work->port->port_num, 0, &work->path[0].sgid);
ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
&work->path[0].sgid, sizeof work->path[0].sgid,
Index: b/drivers/infiniband/core/cma.c
===================================================================
--- a/drivers/infiniband/core/cma.c 2007-05-03 11:25:58.916529056 +0300
+++ b/drivers/infiniband/core/cma.c 2007-05-03 11:32:21.401031673 +0300
@@ -41,7 +41,6 @@
#include <rdma/rdma_cm.h>
#include <rdma/rdma_cm_ib.h>
-#include <rdma/ib_cache.h>
#include <rdma/ib_cm.h>
#include <rdma/ib_sa.h>
#include <rdma/iw_cm.h>
@@ -325,7 +324,7 @@ static int cma_acquire_dev(struct rdma_i
}
list_for_each_entry(cma_dev, &dev_list, list) {
- ret = ib_find_cached_gid(cma_dev->device, &gid,
+ ret = ib_find_gid(cma_dev->device, &gid,
&id_priv->id.port_num, NULL);
if (!ret) {
ret = cma_set_qkey(cma_dev->device,
@@ -514,7 +513,7 @@ static int cma_ib_init_qp_attr(struct rd
struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
int ret;
- ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
+ ret = ib_find_pkey(id_priv->id.device, id_priv->id.port_num,
ib_addr_get_pkey(dev_addr),
&qp_attr->pkey_index);
if (ret)
@@ -1658,11 +1657,11 @@ static int cma_bind_loopback(struct rdma
cma_dev = list_entry(dev_list.next, struct cma_device, list);
port_found:
- ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
+ ret = ib_query_gid(cma_dev->device, p, 0, &gid);
if (ret)
goto out;
- ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
+ ret = ib_query_pkey(cma_dev->device, p, 0, &pkey);
if (ret)
goto out;
Index: b/drivers/infiniband/core/mad.c
===================================================================
--- a/drivers/infiniband/core/mad.c 2007-05-03 11:25:58.930526546 +0300
+++ b/drivers/infiniband/core/mad.c 2007-05-03 11:32:21.435025591 +0300
@@ -34,7 +34,6 @@
* $Id: mad.c 5596 2006-03-03 01:00:07Z sean.hefty $
*/
#include <linux/dma-mapping.h>
-#include <rdma/ib_cache.h>
#include "mad_priv.h"
#include "mad_rmpp.h"
@@ -1707,13 +1706,13 @@ static inline int rcv_has_same_gid(struc
if (!send_resp && rcv_resp) {
/* is request/response. */
if (!(attr.ah_flags & IB_AH_GRH)) {
- if (ib_get_cached_lmc(device, port_num, &lmc))
+ if (ib_query_lmc(device, port_num, &lmc))
return 0;
return (!lmc || !((attr.src_path_bits ^
rwc->wc->dlid_path_bits) &
((1 << lmc) - 1)));
} else {
- if (ib_get_cached_gid(device, port_num,
+ if (ib_query_gid(device, port_num,
attr.grh.sgid_index, &sgid))
return 0;
return !memcmp(sgid.raw, rwc->recv_buf.grh->dgid.raw,
Index: b/drivers/infiniband/core/multicast.c
===================================================================
--- a/drivers/infiniband/core/multicast.c 2007-05-03 11:25:58.947523497 +0300
+++ b/drivers/infiniband/core/multicast.c 2007-05-03 11:32:21.454022192 +0300
@@ -38,7 +38,6 @@
#include <linux/bitops.h>
#include <linux/random.h>
-#include <rdma/ib_cache.h>
#include "sa.h"
static void mcast_add_one(struct ib_device *device);
@@ -686,7 +685,7 @@ int ib_init_ah_from_mcmember(struct ib_d
u16 gid_index;
u8 p;
- ret = ib_find_cached_gid(device, &rec->port_gid, &p, &gid_index);
+ ret = ib_find_gid(device, &rec->port_gid, &p, &gid_index);
if (ret)
return ret;
Index: b/drivers/infiniband/core/sa_query.c
===================================================================
--- a/drivers/infiniband/core/sa_query.c 2007-05-03 11:25:58.964520449 +0300
+++ b/drivers/infiniband/core/sa_query.c 2007-05-03 11:32:21.472018972 +0300
@@ -47,7 +47,6 @@
#include <linux/workqueue.h>
#include <rdma/ib_pack.h>
-#include <rdma/ib_cache.h>
#include "sa.h"
MODULE_AUTHOR("Roland Dreier");
@@ -477,7 +476,7 @@ int ib_init_ah_from_path(struct ib_devic
ah_attr->ah_flags = IB_AH_GRH;
ah_attr->grh.dgid = rec->dgid;
- ret = ib_find_cached_gid(device, &rec->sgid, &port_num,
+ ret = ib_find_gid(device, &rec->sgid, &port_num,
&gid_index);
if (ret)
return ret;
Index: b/drivers/infiniband/core/verbs.c
===================================================================
--- a/drivers/infiniband/core/verbs.c 2007-05-03 11:25:58.984516863 +0300
+++ b/drivers/infiniband/core/verbs.c 2007-05-03 11:32:21.499014142 +0300
@@ -43,7 +43,6 @@
#include <linux/string.h>
#include <rdma/ib_verbs.h>
-#include <rdma/ib_cache.h>
int ib_rate_to_mult(enum ib_rate rate)
{
@@ -159,7 +158,7 @@ int ib_init_ah_from_wc(struct ib_device
ah_attr->ah_flags = IB_AH_GRH;
ah_attr->grh.dgid = grh->sgid;
- ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
+ ret = ib_find_gid(device, &grh->dgid, &port_num,
&gid_index);
if (ret)
return ret;
Index: b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
===================================================================
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c 2007-05-03 11:25:59.020510408 +0300
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c 2007-05-03 11:32:21.536007523 +0300
@@ -33,7 +33,6 @@
*/
#include <rdma/ib_cm.h>
-#include <rdma/ib_cache.h>
#include <net/dst.h>
#include <net/icmp.h>
#include <linux/icmpv6.h>
@@ -759,7 +758,7 @@ static int ipoib_cm_modify_tx_init(struc
struct ipoib_dev_priv *priv = netdev_priv(dev);
struct ib_qp_attr qp_attr;
int qp_attr_mask, ret;
- ret = ib_find_cached_pkey(priv->ca, priv->port, priv->pkey, &qp_attr.pkey_index);
+ ret = ib_find_pkey(priv->ca, priv->port, priv->pkey, &qp_attr.pkey_index);
if (ret) {
ipoib_warn(priv, "pkey 0x%x not in cache: %d\n", priv->pkey, ret);
return ret;
Index: b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
===================================================================
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2007-05-03 11:27:12.676301751 +0300
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2007-05-03 11:32:21.563002693 +0300
@@ -38,7 +38,7 @@
#include <linux/delay.h>
#include <linux/dma-mapping.h>
-#include <rdma/ib_cache.h>
+#include <rdma/ib_verbs.h>
#include "ipoib.h"
Index: b/drivers/infiniband/ulp/srp/ib_srp.c
===================================================================
--- a/drivers/infiniband/ulp/srp/ib_srp.c 2007-05-03 11:25:59.064502518 +0300
+++ b/drivers/infiniband/ulp/srp/ib_srp.c 2007-05-03 11:32:21.592997326 +0300
@@ -48,8 +48,6 @@
#include <scsi/scsi_dbg.h>
#include <scsi/srp.h>
-#include <rdma/ib_cache.h>
-
#include "ib_srp.h"
#define DRV_NAME "ib_srp"
@@ -164,7 +162,7 @@ static int srp_init_qp(struct srp_target
if (!attr)
return -ENOMEM;
- ret = ib_find_cached_pkey(target->srp_host->dev->dev,
+ ret = ib_find_pkey(target->srp_host->dev->dev,
target->srp_host->port,
be16_to_cpu(target->path.pkey),
&attr->pkey_index);
@@ -1780,7 +1778,7 @@ static ssize_t srp_create_target(struct
if (ret)
goto err;
- ib_get_cached_gid(host->dev->dev, host->port, 0, &target->path.sgid);
+ ib_query_gid(host->dev->dev, host->port, 0, &target->path.sgid);
printk(KERN_DEBUG PFX "new target: id_ext %016llx ioc_guid %016llx pkey %04x "
"service_id %016llx dgid %04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
More information about the general
mailing list