[ewg] [eli at mellanox.co.il: [PATCH 4/9 v2] ib_core: Add RDMAoE SA support]
Eli Cohen
eli at dev.mellanox.co.il
Thu Jun 25 08:44:32 PDT 2009
----- Forwarded message from Eli Cohen <eli at mellanox.co.il> -----
Date: Thu, 25 Jun 2009 17:59:00 +0300
From: Eli Cohen <eli at mellanox.co.il>
User-Agent: Mutt/1.5.18 (2008-05-17)
To: Sean Hefty <sean.hefty at intel.com>, Roland Dreier <rdreier at cisco.com>
Subject: [PATCH 4/9 v2] ib_core: Add RDMAoE SA support
Add support for resolving paths and joining multicast group for RDMAoE ports.
The Ethernet specific code will complete immediately but will call the callback
from a workqueue context to avoid deadloks. Ethernet code is implemented in a
new file, rdmaoe_sa.c, and a header file was also added to expose the new API
it supports. Consumers who want to use this API need to be changed.
Signed-off-by: Eli Cohen <eli at mellanox.co.il>
---
I changed this patch such that RDMAoE functionality is put in it own
file. Some struct definitions were moved to sa.h and to a new
multicast.h file. This way, there are less changes to core SA files.
Let me know what you think of this approach.
drivers/infiniband/core/Makefile | 2 +-
drivers/infiniband/core/multicast.c | 43 +--------
drivers/infiniband/core/multicast.h | 79 ++++++++++++++
drivers/infiniband/core/rdmaoe_sa.c | 194 +++++++++++++++++++++++++++++++++++
drivers/infiniband/core/sa.h | 24 +++++
drivers/infiniband/core/sa_query.c | 26 +----
include/rdma/rdmaoe_sa.h | 61 +++++++++++
7 files changed, 364 insertions(+), 65 deletions(-)
create mode 100644 drivers/infiniband/core/multicast.h
create mode 100644 drivers/infiniband/core/rdmaoe_sa.c
create mode 100644 include/rdma/rdmaoe_sa.h
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index cb1ab3e..88d1c23 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -13,7 +13,7 @@ ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o
ib_mad-y := mad.o smi.o agent.o mad_rmpp.o
-ib_sa-y := sa_query.o multicast.o
+ib_sa-y := sa_query.o multicast.o rdmaoe_sa.o
ib_cm-y := cm.o
diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c
index 107f170..727a55a 100644
--- a/drivers/infiniband/core/multicast.c
+++ b/drivers/infiniband/core/multicast.c
@@ -39,6 +39,7 @@
#include <rdma/ib_cache.h>
#include "sa.h"
+#include "multicast.h"
static void mcast_add_one(struct ib_device *device);
static void mcast_remove_one(struct ib_device *device);
@@ -72,52 +73,10 @@ struct mcast_device {
struct mcast_port port[0];
};
-enum mcast_state {
- MCAST_JOINING,
- MCAST_MEMBER,
- MCAST_ERROR,
-};
-
-enum mcast_group_state {
- MCAST_IDLE,
- MCAST_BUSY,
- MCAST_GROUP_ERROR,
- MCAST_PKEY_EVENT
-};
-
enum {
MCAST_INVALID_PKEY_INDEX = 0xFFFF
};
-struct mcast_member;
-
-struct mcast_group {
- struct ib_sa_mcmember_rec rec;
- struct rb_node node;
- struct mcast_port *port;
- spinlock_t lock;
- struct work_struct work;
- struct list_head pending_list;
- struct list_head active_list;
- struct mcast_member *last_join;
- int members[3];
- atomic_t refcount;
- enum mcast_group_state state;
- struct ib_sa_query *query;
- int query_id;
- u16 pkey_index;
-};
-
-struct mcast_member {
- struct ib_sa_multicast multicast;
- struct ib_sa_client *client;
- struct mcast_group *group;
- struct list_head list;
- enum mcast_state state;
- atomic_t refcount;
- struct completion comp;
-};
-
static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
void *context);
static void leave_handler(int status, struct ib_sa_mcmember_rec *rec,
diff --git a/drivers/infiniband/core/multicast.h b/drivers/infiniband/core/multicast.h
new file mode 100644
index 0000000..17eb9fe
--- /dev/null
+++ b/drivers/infiniband/core/multicast.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2009 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef MULTICAST_H
+#define MULTICAST_H
+
+enum mcast_state {
+ MCAST_JOINING,
+ MCAST_MEMBER,
+ MCAST_ERROR,
+};
+
+enum mcast_group_state {
+ MCAST_IDLE,
+ MCAST_BUSY,
+ MCAST_GROUP_ERROR,
+ MCAST_PKEY_EVENT
+};
+
+struct mcast_member;
+
+struct mcast_group {
+ struct ib_sa_mcmember_rec rec;
+ struct rb_node node;
+ struct mcast_port *port;
+ spinlock_t lock;
+ struct work_struct work;
+ struct list_head pending_list;
+ struct list_head active_list;
+ struct mcast_member *last_join;
+ int members[3];
+ atomic_t refcount;
+ enum mcast_group_state state;
+ struct ib_sa_query *query;
+ int query_id;
+ u16 pkey_index;
+};
+
+struct mcast_member {
+ struct ib_sa_multicast multicast;
+ struct ib_sa_client *client;
+ struct mcast_group *group;
+ struct list_head list;
+ enum mcast_state state;
+ atomic_t refcount;
+ struct completion comp;
+};
+
+#endif /* MULTICAST_H */
+
diff --git a/drivers/infiniband/core/rdmaoe_sa.c b/drivers/infiniband/core/rdmaoe_sa.c
new file mode 100644
index 0000000..f751f81
--- /dev/null
+++ b/drivers/infiniband/core/rdmaoe_sa.c
@@ -0,0 +1,194 @@
+/*
+ * Copyright (c) 2009 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/workqueue.h>
+#include <rdma/ib_sa.h>
+#include <rdma/rdmaoe_sa.h>
+#include "sa.h"
+#include "multicast.h"
+
+struct eth_work {
+ struct work_struct work;
+ struct mcast_member *member;
+ struct ib_device *device;
+ u8 port_num;
+};
+
+static void deref_member(struct mcast_member *member)
+{
+ if (atomic_dec_and_test(&member->refcount))
+ complete(&member->comp);
+}
+
+static void eth_mcast_work_handler(struct work_struct *work)
+{
+ struct eth_work *w = container_of(work, struct eth_work, work);
+ int err;
+ struct ib_port_attr port_attr;
+ int status = 0;
+
+ err = ib_query_port(w->device, w->port_num, &port_attr);
+ if (err)
+ status = err;
+ else if (port_attr.state != IB_PORT_ACTIVE)
+ status = -EAGAIN;
+
+ w->member->multicast.rec.qkey = cpu_to_be32(0xc2c);
+ err = w->member->multicast.callback(status, &w->member->multicast);
+ deref_member(w->member);
+ if (err)
+ ib_sa_free_multicast(&w->member->multicast);
+
+ kfree(w);
+}
+
+struct ib_sa_multicast *
+rdmaoe_sa_join_multicast(struct ib_sa_client *client,
+ struct ib_device *device, u8 port_num,
+ struct ib_sa_mcmember_rec *rec,
+ ib_sa_comp_mask comp_mask, gfp_t gfp_mask,
+ int (*callback)(int status,
+ struct ib_sa_multicast *multicast),
+ void *context)
+{
+ struct eth_work *w;
+ struct mcast_member *member;
+ int err;
+
+ member = kzalloc(sizeof *member, gfp_mask);
+ if (!member)
+ return ERR_PTR(-ENOMEM);
+
+ w = kzalloc(sizeof *w, gfp_mask);
+ if (!w) {
+ err = -ENOMEM;
+ goto out1;
+ }
+ w->member = member;
+ w->device = device;
+ w->port_num = port_num;
+
+ member->multicast.context = context;
+ member->multicast.callback = callback;
+ member->client = client;
+ member->multicast.rec = *rec;
+ init_completion(&member->comp);
+ atomic_set(&member->refcount, 1);
+
+ atomic_inc(&member->refcount);
+ ib_sa_client_get(client);
+ INIT_WORK(&w->work, eth_mcast_work_handler);
+ schedule_work(&w->work);
+
+ return &member->multicast;
+
+out1:
+ kfree(member);
+ return ERR_PTR(err);
+}
+EXPORT_SYMBOL(rdmaoe_sa_join_multicast);
+
+void rdmaoe_sa_free_multicast(struct ib_sa_multicast *multicast)
+{
+ struct mcast_member *member;
+
+ member = container_of(multicast, struct mcast_member, multicast);
+
+ deref_member(member);
+ wait_for_completion(&member->comp);
+ ib_sa_client_put(member->client);
+ kfree(member);
+}
+EXPORT_SYMBOL(rdmaoe_sa_free_multicast);
+
+struct eth_work_container {
+ struct work_struct work;
+ struct ib_sa_path_query *query;
+};
+
+static void resolve_callback(struct work_struct *work)
+{
+ struct eth_work_container *eth =
+ container_of(work, struct eth_work_container, work);
+ struct ib_sa_path_query *query = eth->query;
+ struct ib_sa_path_rec res = {};
+
+ res.dgid = query->dgid;
+ res.sgid = query->sgid;
+ res.hop_limit = 2; /* TBD fix this */
+ res.mtu = IB_MTU_1024; /* TBD fix me */
+ query->callback(0, &res, query->context);
+
+ ib_sa_client_put(query->sa_query.client);
+}
+
+int rdmaoe_sa_path_rec_get(struct ib_sa_client *client,
+ struct ib_device *device, u8 port_num,
+ struct ib_sa_path_rec *rec,
+ ib_sa_comp_mask comp_mask,
+ int timeout_ms, gfp_t gfp_mask,
+ void (*callback)(int status,
+ struct ib_sa_path_rec *resp,
+ void *context),
+ void *context,
+ struct ib_sa_query **sa_query)
+{
+ struct ib_sa_path_query *query;
+ struct eth_work_container *eth;
+
+ query = kzalloc(sizeof *query, gfp_mask);
+ if (!query)
+ return -ENOMEM;
+
+ eth = kzalloc(sizeof *eth, gfp_mask);
+ if (!eth) {
+ kfree(query);
+ return -ENOMEM;
+ }
+
+ ib_sa_client_get(client);
+ query->sa_query.client = client;
+ query->callback = callback;
+ query->context = context;
+ query->dgid = rec->dgid;
+ query->sgid = rec->sgid;
+
+ *sa_query = &query->sa_query;
+
+ eth->query = query;
+ INIT_WORK(ð->work, resolve_callback);
+ schedule_work(ð->work);
+
+ return 0;
+}
+EXPORT_SYMBOL(rdmaoe_sa_path_rec_get);
+
diff --git a/drivers/infiniband/core/sa.h b/drivers/infiniband/core/sa.h
index b1d4bbf..da780cf 100644
--- a/drivers/infiniband/core/sa.h
+++ b/drivers/infiniband/core/sa.h
@@ -37,6 +37,30 @@
#include <rdma/ib_sa.h>
+struct ib_sa_query {
+ void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *);
+ void (*release)(struct ib_sa_query *);
+ struct ib_sa_client *client;
+ struct ib_sa_port *port;
+ struct ib_mad_send_buf *mad_buf;
+ struct ib_sa_sm_ah *sm_ah;
+ int id;
+};
+
+struct ib_sa_service_query {
+ void (*callback)(int, struct ib_sa_service_rec *, void *);
+ void *context;
+ struct ib_sa_query sa_query;
+};
+
+struct ib_sa_path_query {
+ void (*callback)(int, struct ib_sa_path_rec *, void *);
+ void *context;
+ struct ib_sa_query sa_query;
+ union ib_gid dgid;
+ union ib_gid sgid;
+};
+
static inline void ib_sa_client_get(struct ib_sa_client *client)
{
atomic_inc(&client->users);
diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
index 1865049..0625e10 100644
--- a/drivers/infiniband/core/sa_query.c
+++ b/drivers/infiniband/core/sa_query.c
@@ -46,6 +46,7 @@
#include <rdma/ib_pack.h>
#include <rdma/ib_cache.h>
#include "sa.h"
+#include "multicast.h"
MODULE_AUTHOR("Roland Dreier");
MODULE_DESCRIPTION("InfiniBand subnet administration query support");
@@ -72,28 +73,6 @@ struct ib_sa_device {
struct ib_sa_port port[0];
};
-struct ib_sa_query {
- void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *);
- void (*release)(struct ib_sa_query *);
- struct ib_sa_client *client;
- struct ib_sa_port *port;
- struct ib_mad_send_buf *mad_buf;
- struct ib_sa_sm_ah *sm_ah;
- int id;
-};
-
-struct ib_sa_service_query {
- void (*callback)(int, struct ib_sa_service_rec *, void *);
- void *context;
- struct ib_sa_query sa_query;
-};
-
-struct ib_sa_path_query {
- void (*callback)(int, struct ib_sa_path_rec *, void *);
- void *context;
- struct ib_sa_query sa_query;
-};
-
struct ib_sa_mcmember_query {
void (*callback)(int, struct ib_sa_mcmember_rec *, void *);
void *context;
@@ -363,6 +342,9 @@ static void update_sm_ah(struct work_struct *work)
struct ib_port_attr port_attr;
struct ib_ah_attr ah_attr;
+ if (ib_get_port_link_type(port->agent->device, port->port_num) != PORT_LINK_IB)
+ return;
+
if (ib_query_port(port->agent->device, port->port_num, &port_attr)) {
printk(KERN_WARNING "Couldn't query port\n");
return;
diff --git a/include/rdma/rdmaoe_sa.h b/include/rdma/rdmaoe_sa.h
new file mode 100644
index 0000000..a672cf7
--- /dev/null
+++ b/include/rdma/rdmaoe_sa.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2009 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RDMAOE_SA_H
+#define RDMAOE_SA_H
+
+#include <rdma/ib_sa.h>
+
+struct ib_sa_multicast *
+rdmaoe_sa_join_multicast(struct ib_sa_client *client,
+ struct ib_device *device, u8 port_num,
+ struct ib_sa_mcmember_rec *rec,
+ ib_sa_comp_mask comp_mask, gfp_t gfp_mask,
+ int (*callback)(int status,
+ struct ib_sa_multicast *multicast),
+ void *context);
+
+void rdmaoe_sa_free_multicast(struct ib_sa_multicast *multicast);
+
+int rdmaoe_sa_path_rec_get(struct ib_sa_client *client,
+ struct ib_device *device, u8 port_num,
+ struct ib_sa_path_rec *rec,
+ ib_sa_comp_mask comp_mask,
+ int timeout_ms, gfp_t gfp_mask,
+ void (*callback)(int status,
+ struct ib_sa_path_rec *resp,
+ void *context),
+ void *context,
+ struct ib_sa_query **sa_query);
+
+#endif /* RDMAOE_SA_H */
+
--
1.6.3.3
----- End forwarded message -----
More information about the ewg
mailing list