[openib-general] [PATCH] expand local SA cache to include all path records from source
Sean Hefty
sean.hefty at intel.com
Thu Apr 20 15:28:23 PDT 2006
Modify the local SA cache to store all path records from the local
SGID to all destinations. The current implementation retrieved only
a single path to each destination.
Add API calls that can be used to walk the list of known paths to a
given DGID.
This allows a user to select an alternate path for a connection, and
provide their own filtering on available paths.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
Index: include/rdma/ib_local_sa.h
===================================================================
--- include/rdma/ib_local_sa.h (revision 6418)
+++ include/rdma/ib_local_sa.h (working copy)
@@ -52,4 +52,32 @@
int ib_get_path_rec(struct ib_device *device, u8 port_num, union ib_gid *sgid,
union ib_gid *dgid, u16 pkey, struct ib_sa_path_rec *rec);
+/**
+ * ib_create_path_cursor - Create a cursor that may be used to walk through
+ * a list of path records.
+ * @device: The local device to retrieve path records for.
+ * @port_num: The port of the local device.
+ * @dgid: The destination GID of the path record.
+ *
+ * This call allocates a cursor that is used to walk through a list of locally
+ * cached path records. All path records accessed by the cursor will have the
+ * specified DGID. User should not hold the cursor for an extended period of
+ * time, and must free it by calling ib_free_sa_cursor.
+ */
+struct ib_sa_cursor *ib_create_path_cursor(struct ib_device *device,
+ u8 port_num, union ib_gid *dgid);
+
+/**
+ * ib_free_sa_cursor - Release a cursor.
+ * @cursor: The cursor to free.
+ */
+void ib_free_sa_cursor(struct ib_sa_cursor *cursor);
+
+/**
+ * ib_get_next_sa_attr - Retrieve the next SA attribute referenced by a cursor.
+ * @cursor: A reference to a cursor that points to the next attribute to
+ * retrieve.
+ */
+void *ib_get_next_sa_attr(struct ib_sa_cursor **cursor);
+
#endif /* IB_LOCAL_SA_H */
Index: core/local_sa.c
===================================================================
--- core/local_sa.c (revision 6542)
+++ core/local_sa.c (working copy)
@@ -84,10 +84,10 @@ struct sa_db_port {
struct ib_mad_agent *agent;
struct index_root index;
unsigned long update_time;
+ int update;
struct work_struct work;
union ib_gid gid;
int port_num;
- u16 pkey;
};
struct sa_db_device {
@@ -107,6 +107,25 @@ struct ib_path_rec {
u8 reserved2[20];
};
+struct ib_sa_cursor {
+ struct ib_sa_cursor *next;
+};
+
+struct ib_sa_attr_list {
+ struct ib_sa_cursor cursor;
+ struct ib_sa_cursor *tail;
+ int update;
+};
+
+struct ib_path_rec_info {
+ struct ib_sa_cursor cursor;
+ struct ib_sa_path_rec rec;
+};
+
+enum {
+ IB_MAX_PATHS_PER_QUERY = 0x7F
+};
+
static void send_handler(struct ib_mad_agent *agent,
struct ib_mad_send_wc *mad_send_wc)
{
@@ -114,6 +133,53 @@ static void send_handler(struct ib_mad_a
ib_free_send_mad(mad_send_wc->send_buf);
}
+static void free_attr_list(struct ib_sa_attr_list *attr_list)
+{
+ struct ib_sa_cursor *cur;
+
+ for (cur = attr_list->cursor.next; cur; cur = attr_list->cursor.next) {
+ attr_list->cursor.next = cur->next;
+ kfree(cur);
+ }
+ attr_list->tail = &attr_list->cursor;
+}
+
+static int insert_attr(struct index_root *index, int update, void *key,
+ struct ib_sa_cursor *cursor)
+{
+ struct ib_sa_attr_list *attr_list;
+ void *err;
+
+ attr_list = index_find(index, key);
+ if (!attr_list) {
+ attr_list = kmalloc(sizeof *attr_list, GFP_KERNEL);
+ if (!attr_list)
+ return -ENOMEM;
+
+ attr_list->cursor.next = NULL;
+ attr_list->tail = &attr_list->cursor;
+ attr_list->update = update;
+
+ err = index_insert(index, attr_list, key);
+ if (err) {
+ kfree(attr_list);
+ return PTR_ERR(err);
+ }
+ } else if (attr_list->update != update) {
+ free_attr_list(attr_list);
+ attr_list->update = update;
+ }
+
+ /*
+ * Assume that the SA returned the best attribute first, and insert
+ * attributes on the tail.
+ */
+ attr_list->tail->next = cursor;
+ cursor->next = NULL;
+ attr_list->tail = cursor;
+ return 0;
+}
+
/*
* Copy a path record from a received MAD and insert it into our index.
* The path record in the MAD is in network order, so must be swapped. It
@@ -124,7 +190,7 @@ static void update_path_rec(struct sa_db
{
struct ib_mad_recv_buf *recv_buf;
struct ib_sa_mad *mad = (void *) mad_recv_wc->recv_buf.mad;
- struct ib_sa_path_rec *sa_path, *old_path;
+ struct ib_path_rec_info *path_info;
struct ib_path_rec ib_path, *path = NULL;
int i, attr_size, left, offset = 0;
@@ -132,6 +198,8 @@ static void update_path_rec(struct sa_db
if (attr_size < sizeof ib_path)
return;
+ down_write(&lock);
+ port->update++;
list_for_each_entry(recv_buf, &mad_recv_wc->rmpp_list, list) {
for (i = 0; i < IB_MGMT_SA_DATA;) {
mad = (struct ib_sa_mad *) recv_buf->mad;
@@ -155,28 +223,25 @@ static void update_path_rec(struct sa_db
}
if (!path->slid)
- return;
+ goto unlock;
- sa_path = kmalloc(sizeof *sa_path, GFP_KERNEL);
- if (!sa_path)
- return;
-
- ib_sa_unpack_attr(sa_path, path, IB_SA_ATTR_PATH_REC);
-
- down_write(&lock);
- old_path = index_find_replace(&port->index, sa_path,
- sa_path->dgid.raw);
- if (old_path)
- kfree(old_path);
- else if (index_insert(&port->index, sa_path,
- sa_path->dgid.raw)) {
- up_write(&lock);
- kfree(sa_path);
- return;
+ path_info = kmalloc(sizeof *path_info, GFP_KERNEL);
+ if (!path_info)
+ goto unlock;
+
+ ib_sa_unpack_attr(&path_info->rec, path,
+ IB_SA_ATTR_PATH_REC);
+
+ if (insert_attr(&port->index, port->update,
+ path_info->rec.dgid.raw,
+ &path_info->cursor)) {
+ kfree(path_info);
+ goto unlock;
}
- up_write(&lock);
}
}
+unlock:
+ up_write(&lock);
}
static void recv_handler(struct ib_mad_agent *mad_agent,
@@ -251,12 +316,10 @@ static void format_path_req(struct sa_db
mad->mad_hdr.attr_id = cpu_to_be16(IB_SA_ATTR_PATH_REC);
mad->mad_hdr.tid = form_tid(msg->mad_agent->hi_tid);
- mad->sa_hdr.comp_mask = IB_SA_PATH_REC_SGID | IB_SA_PATH_REC_PKEY |
- IB_SA_PATH_REC_NUMB_PATH;
+ mad->sa_hdr.comp_mask = IB_SA_PATH_REC_SGID | IB_SA_PATH_REC_NUMB_PATH;
path_rec.sgid = port->gid;
- path_rec.pkey = port->pkey;
- path_rec.numb_path = 1;
+ path_rec.numb_path = IB_MAX_PATHS_PER_QUERY;
ib_sa_pack_attr(mad->data, &path_rec, IB_SA_ATTR_PATH_REC);
}
@@ -320,36 +383,73 @@ static void handle_event(struct ib_event
int ib_get_path_rec(struct ib_device *device, u8 port_num, union ib_gid *sgid,
union ib_gid *dgid, u16 pkey, struct ib_sa_path_rec *rec)
{
+ struct ib_sa_cursor *cursor;
+ struct ib_sa_path_rec *path;
+ int ret = -ENODATA;
+
+ cursor = ib_create_path_cursor(device, port_num, dgid);
+ if (IS_ERR(cursor))
+ return PTR_ERR(cursor);
+
+ for (path = ib_get_next_sa_attr(&cursor); path;
+ path = ib_get_next_sa_attr(&cursor)) {
+ if (pkey == path->pkey &&
+ !memcmp(sgid, path->sgid.raw, sizeof *sgid)) {
+ memcpy(rec, path, sizeof *rec);
+ ret = 0;
+ break;
+ }
+ }
+
+ ib_free_sa_cursor(cursor);
+ return ret;
+}
+EXPORT_SYMBOL(ib_get_path_rec);
+
+struct ib_sa_cursor *ib_create_path_cursor(struct ib_device *device,
+ u8 port_num, union ib_gid *dgid)
+{
struct sa_db_device *dev;
struct sa_db_port *port;
- struct ib_sa_path_rec *path_rec;
- int ret = 0;
+ struct ib_sa_attr_list *list;
+ int ret;
down_read(&lock);
dev = ib_get_client_data(device, &sa_db_client);
if (!dev) {
ret = -ENODEV;
- goto unlock;
+ goto err;
}
port = &dev->port[port_num - 1];
- if (memcmp(&port->gid, sgid, sizeof *sgid) || port->pkey != pkey) {
+ list = index_find(&port->index, dgid->raw);
+ if (!list) {
ret = -ENODATA;
- goto unlock;
+ goto err;
}
- path_rec = index_find(&port->index, dgid->raw);
- if (!path_rec) {
- ret = -ENODATA;
- goto unlock;
- }
+ return &list->cursor;
+err:
+ up_read(&lock);
+ return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(ib_create_path_cursor);
- memcpy(rec, path_rec, sizeof *path_rec);
-unlock:
+void ib_free_sa_cursor(struct ib_sa_cursor *cursor)
+{
up_read(&lock);
- return ret;
}
-EXPORT_SYMBOL(ib_get_path_rec);
+EXPORT_SYMBOL(ib_free_sa_cursor);
+
+void *ib_get_next_sa_attr(struct ib_sa_cursor **cursor)
+{
+ *cursor = (*cursor)->next;
+ if (*cursor)
+ return ((void *)(*cursor)) + sizeof(**cursor);
+ else
+ return NULL;
+}
+EXPORT_SYMBOL(ib_get_next_sa_attr);
static void sa_db_free_data(void *context, void *data)
{
@@ -375,11 +475,11 @@ static void sa_db_add_one(struct ib_devi
port->dev = dev;
port->port_num = i;
port->update_time = jiffies - hold_time;
+ port->update = 0;
INIT_WORK(&port->work, update_cache, port);
index_init(&port->index, sizeof (union ib_gid), GFP_KERNEL);
- if (ib_get_cached_gid(device, i, 0, &port->gid) ||
- ib_get_cached_pkey(device, i, 0, &port->pkey))
+ if (ib_get_cached_gid(device, i, 0, &port->gid))
goto err;
port->agent = ib_register_mad_agent(device, i, IB_QPT_GSI,
More information about the general
mailing list