[ewg] [PATCH 5/9] ib_core: CMA device binding

Eli Cohen eli at mellanox.co.il
Mon Jun 15 00:34:59 PDT 2009


Add support for RDMAoE device binding and IP --> GID resolution.

Signed-off-by: Eli Cohen <eli at mellanox.co.il>
---
 drivers/infiniband/core/addr.c |   20 ++++++++-------
 drivers/infiniband/core/cma.c  |   39 ++++++++++++++++++++++++++----
 include/rdma/ib_addr.h         |   51 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+), 14 deletions(-)

diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index ce511d8..440e613 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -64,7 +64,7 @@ struct addr_req {
 
 static void process_req(struct work_struct *work);
 
-static DEFINE_MUTEX(lock);
+static DEFINE_SPINLOCK(lock);
 static LIST_HEAD(req_list);
 static DECLARE_DELAYED_WORK(work, process_req);
 static struct workqueue_struct *addr_wq;
@@ -163,7 +163,7 @@ static void queue_req(struct addr_req *req)
 {
 	struct addr_req *temp_req;
 
-	mutex_lock(&lock);
+	spin_lock(&lock);
 	list_for_each_entry_reverse(temp_req, &req_list, list) {
 		if (time_after_eq(req->timeout, temp_req->timeout))
 			break;
@@ -173,7 +173,7 @@ static void queue_req(struct addr_req *req)
 
 	if (req_list.next == &req->list)
 		set_timeout(req->timeout);
-	mutex_unlock(&lock);
+	spin_unlock(&lock);
 }
 
 static void addr_send_arp(struct sockaddr *dst_in)
@@ -207,7 +207,9 @@ static void addr_send_arp(struct sockaddr *dst_in)
 		if (!dst)
 			return;
 
-		neigh_event_send(dst->neighbour, NULL);
+		if (dst->neighbour)
+			neigh_event_send(dst->neighbour, NULL);
+
 		dst_release(dst);
 		break;
 	}
@@ -322,7 +324,7 @@ static void process_req(struct work_struct *work)
 
 	INIT_LIST_HEAD(&done_list);
 
-	mutex_lock(&lock);
+	spin_lock(&lock);
 	list_for_each_entry_safe(req, temp_req, &req_list, list) {
 		if (req->status == -ENODATA) {
 			src_in = (struct sockaddr *) &req->src_addr;
@@ -341,7 +343,7 @@ static void process_req(struct work_struct *work)
 		req = list_entry(req_list.next, struct addr_req, list);
 		set_timeout(req->timeout);
 	}
-	mutex_unlock(&lock);
+	spin_unlock(&lock);
 
 	list_for_each_entry_safe(req, temp_req, &done_list, list) {
 		list_del(&req->list);
@@ -439,7 +441,7 @@ int rdma_resolve_ip(struct rdma_addr_client *client,
 	struct addr_req *req;
 	int ret = 0;
 
-	req = kzalloc(sizeof *req, GFP_KERNEL);
+	req = kzalloc(sizeof *req, GFP_ATOMIC);
 	if (!req)
 		return -ENOMEM;
 
@@ -483,7 +485,7 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr)
 {
 	struct addr_req *req, *temp_req;
 
-	mutex_lock(&lock);
+	spin_lock(&lock);
 	list_for_each_entry_safe(req, temp_req, &req_list, list) {
 		if (req->addr == addr) {
 			req->status = -ECANCELED;
@@ -493,7 +495,7 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr)
 			break;
 		}
 	}
-	mutex_unlock(&lock);
+	spin_unlock(&lock);
 }
 EXPORT_SYMBOL(rdma_addr_cancel);
 
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 851de83..0f15d9d 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -327,22 +327,30 @@ static int cma_acquire_dev(struct rdma_id_private *id_priv)
 {
 	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 	struct cma_device *cma_dev;
-	union ib_gid gid;
+	union ib_gid gid, llgid, *tmp;
 	int ret = -ENODEV;
 
 	switch (rdma_node_get_transport(dev_addr->dev_type)) {
 	case RDMA_TRANSPORT_IB:
 		ib_addr_get_sgid(dev_addr, &gid);
+		tmp = &gid;
 		break;
 	case RDMA_TRANSPORT_IWARP:
 		iw_addr_get_sgid(dev_addr, &gid);
+		rdma_mac_to_ll_addr(dev_addr->src_dev_addr, &llgid);
 		break;
 	default:
 		return -ENODEV;
 	}
 
 	list_for_each_entry(cma_dev, &dev_list, list) {
-		ret = ib_find_cached_gid(cma_dev->device, &gid,
+		if (ib_get_port_link_type(cma_dev->device, id_priv->id.port_num)
+		    == PORT_LINK_ETH &&
+		    rdma_node_get_transport(dev_addr->dev_type) ==
+		    RDMA_TRANSPORT_IWARP)
+			tmp = &llgid;
+
+		ret = ib_find_cached_gid(cma_dev->device, tmp,
 					 &id_priv->id.port_num, NULL);
 		if (!ret) {
 			cma_attach_to_dev(id_priv, cma_dev);
@@ -1032,7 +1040,19 @@ static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
 	if (rt->num_paths == 2)
 		rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
 
-	ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
+
+	switch (ib_get_port_link_type(listen_id->device,
+				     ib_event->param.req_rcvd.port)) {
+	case PORT_LINK_IB:
+		ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
+		break;
+	case PORT_LINK_ETH:
+		iw_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
+		break;
+	default:
+		printk(KERN_ERR "RDMA CMA: unknown rdma port link type\n");
+		goto destroy_id;
+	}
 	ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
 				&id->route.addr.dev_addr);
 	if (ret)
@@ -1563,10 +1583,19 @@ static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
 	struct ib_sa_path_rec path_rec;
 	ib_sa_comp_mask comp_mask;
 	struct sockaddr_in6 *sin6;
+	enum ib_port_link_type lt;
 
+	lt = ib_get_port_link_type(id_priv->id.device, id_priv->id.port_num);
 	memset(&path_rec, 0, sizeof path_rec);
-	ib_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
-	ib_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
+	if (lt == PORT_LINK_IB ||
+	    (ib_addr_hw_addr_is_gid(addr->dev_addr.src_dev_addr) &&
+	     ib_addr_hw_addr_is_gid(addr->dev_addr.dst_dev_addr))) {
+		ib_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
+		ib_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
+	} else {
+		rdma_mac_to_ll_addr(addr->dev_addr.src_dev_addr, &path_rec.sgid);
+		rdma_mac_to_ll_addr(addr->dev_addr.dst_dev_addr, &path_rec.dgid);
+	}
 	path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
 	path_rec.numb_path = 1;
 	path_rec.reversible = 1;
diff --git a/include/rdma/ib_addr.h b/include/rdma/ib_addr.h
index 483057b..9636a97 100644
--- a/include/rdma/ib_addr.h
+++ b/include/rdma/ib_addr.h
@@ -151,10 +151,61 @@ static inline void iw_addr_get_sgid(struct rdma_dev_addr *dev_addr,
 	memcpy(gid, dev_addr->src_dev_addr, sizeof *gid);
 }
 
+static inline void iw_addr_set_dgid(struct rdma_dev_addr *dev_addr,
+				    union ib_gid *gid)
+{
+	memcpy(dev_addr->dst_dev_addr, gid, sizeof *gid);
+}
+
 static inline void iw_addr_get_dgid(struct rdma_dev_addr *dev_addr,
 				    union ib_gid *gid)
 {
 	memcpy(gid, dev_addr->dst_dev_addr, sizeof *gid);
 }
 
+static inline int rdma_link_local_addr(struct in6_addr *addr)
+{
+	if (addr->s6_addr32[0] == cpu_to_be32(0xfe800000) &&
+	    addr->s6_addr32[1] == 0)
+		return 1;
+	else
+		return 0;
+}
+
+static inline void rdma_mac_to_ll_addr(u8 *mac, union ib_gid *gid)
+{
+	memset(gid->raw, 0, 16);
+	*((u32 *)gid->raw) = cpu_to_be32(0xfe800000);
+	gid->raw[12] = 0xfe;
+	gid->raw[11] = 0xff;
+	memcpy(gid->raw + 13, mac + 3, 3);
+	memcpy(gid->raw + 8, mac, 3);
+	gid->raw[8] ^= 2;
+}
+
+static inline int rdma_is_multicast_addr(struct in6_addr *addr)
+{
+	return addr->s6_addr[0] = 0xff ? 1 : 0;
+}
+
+static inline void rdma_get_ll_mac(struct in6_addr *addr, u8 *mac)
+{
+	memcpy(mac, &addr->s6_addr[8], 3);
+	memcpy(mac + 3, &addr->s6_addr[13], 3);
+	mac[0] ^= 2;
+}
+
+static inline void rdma_get_mcast_mac(struct in6_addr *addr, u8 *mac)
+{
+	int i;
+
+	for (i = 0; i < 6; ++i)
+		mac[i] = 0xff;
+}
+
+ static inline int ib_addr_hw_addr_is_gid(u8 *addr)
+ {
+	 return be16_to_cpu(*(u16 *)(addr + 4)) == 0xfe80;
+ }
+
 #endif /* IB_ADDR_H */
-- 
1.6.3.1




More information about the ewg mailing list