[openib-general] [RFC/PATCH v3] rdma/cma: add RDMA_PS_IPOIB port space
Or Gerlitz
ogerlitz at voltaire.com
Tue Jan 23 07:38:07 PST 2007
Add to the RDMA CM an IPoIB port space (RDMA_PS_IPOIB) whose semantics are similar
to those of RDMA_PS_UDP where RDMA_PS_IPOIB IDs allow for inter operability with
IPoIB on some traffic patterns.
For RDMA_PS_UDP and RDMA_PS_IPOIB IDs, the qkey is stored in struct rdma_id_private
and delivered also in ADDR_RESOLVED and CONNECT_REQUEST events. The user space library
learns the qkey from these events and use them when it is called to create UD QP.
The IB UD qkey used by RDMA_PS_IPOIB IDs is that of the related ipoib broadcast group
where the qkey used by RDMA_PS_UDP IDs is hard defined "rdma cm qkey".
Creation of RDMA_PS_IPOIB IDs by proceeses is controlled by the linux kernel
capabilities subsystem.
Signed-off-by: Or Gerlitz <ogerlitz at voltaire.com>
Index: rdma-dev/drivers/infiniband/core/cma.c
===================================================================
--- rdma-dev.orig/drivers/infiniband/core/cma.c 2007-01-21 12:11:16.000000000 +0200
+++ rdma-dev/drivers/infiniband/core/cma.c 2007-01-23 15:45:52.000000000 +0200
@@ -71,6 +71,7 @@ static struct workqueue_struct *cma_wq;
static DEFINE_IDR(sdp_ps);
static DEFINE_IDR(tcp_ps);
static DEFINE_IDR(udp_ps);
+static DEFINE_IDR(ipoib_ps);
struct cma_device {
struct list_head list;
@@ -136,6 +137,7 @@ struct rdma_id_private {
u32 seq_num;
u32 qp_num;
u8 srq;
+ u32 qkey;
};
struct cma_multicast {
@@ -323,6 +325,10 @@ struct rdma_cm_id *rdma_create_id(rdma_c
{
struct rdma_id_private *id_priv;
+ /* XXX - work around this till capabilities work fine for non root users */
+ if (ps == RDMA_PS_IPOIB && !capable(CAP_NET_BROADCAST))
+ return ERR_PTR(-EACCES);
+
id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
if (!id_priv)
return ERR_PTR(-ENOMEM);
@@ -884,6 +890,31 @@ out:
return ret;
}
+static int cma_set_qkey(struct rdma_id_private *id_priv, struct rdma_cm_event *event)
+{
+ struct ib_sa_mcmember_rec rec;
+ struct rdma_dev_addr *dev_addr;
+ int ret;
+
+ if (id_priv->id.ps == RDMA_PS_IPOIB) {
+ dev_addr = &id_priv->id.route.addr.dev_addr;
+ ib_addr_get_mgid(dev_addr, &rec.mgid);
+ ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
+ &rec.mgid, &rec);
+ if (ret)
+ return -EINVAL;
+ id_priv->qkey = rec.qkey;
+ event->param.ud.qkey = rec.qkey;
+ }
+
+ if (id_priv->id.ps == RDMA_PS_UDP) {
+ id_priv->qkey = RDMA_UD_QKEY;
+ event->param.ud.qkey = RDMA_UD_QKEY;
+ }
+
+ return 0;
+}
+
static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
struct ib_cm_event *ib_event)
{
@@ -999,7 +1030,7 @@ static int cma_req_handler(struct ib_cm_
memset(&event, 0, sizeof event);
offset = cma_user_data_offset(listen_id->id.ps);
event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
- if (listen_id->id.ps == RDMA_PS_UDP) {
+ if (listen_id->id.ps == RDMA_PS_UDP || listen_id->id.ps == RDMA_PS_IPOIB) {
conn_id = cma_new_udp_id(&listen_id->id, ib_event);
event.param.ud.private_data = ib_event->private_data + offset;
event.param.ud.private_data_len =
@@ -1020,7 +1051,11 @@ static int cma_req_handler(struct ib_cm_
mutex_unlock(&lock);
if (ret)
goto release_conn_id;
-
+
+ ret = cma_set_qkey(conn_id, &event);
+ if (ret)
+ goto release_conn_id;
+
conn_id->cm_id.ib = cm_id;
cm_id->context = conn_id;
cm_id->cm_handler = cma_ib_handler;
@@ -1600,6 +1635,7 @@ static void addr_handler(int status, str
{
struct rdma_id_private *id_priv = context;
struct rdma_cm_event event;
+ int ret;
memset(&event, 0, sizeof event);
atomic_inc(&id_priv->dev_remove);
@@ -1627,6 +1663,11 @@ static void addr_handler(int status, str
memcpy(&id_priv->id.route.addr.src_addr, src_addr,
ip_addr_size(src_addr));
event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
+ ret = cma_set_qkey(id_priv, &event);
+ if (ret) {
+ event.event = RDMA_CM_EVENT_ADDR_ERROR;
+ event.status = ret;
+ }
}
if (id_priv->id.event_handler(&id_priv->id, &event)) {
@@ -1822,6 +1863,9 @@ static int cma_get_port(struct rdma_id_p
case RDMA_PS_UDP:
ps = &udp_ps;
break;
+ case RDMA_PS_IPOIB:
+ ps = &ipoib_ps;
+ break;
default:
return -EPROTONOSUPPORT;
}
@@ -1936,7 +1980,9 @@ static int cma_sidr_rep_handler(struct i
event.status = ib_event->param.sidr_rep_rcvd.status;
break;
}
- if (rep->qkey != RDMA_UD_QKEY) {
+ if (rep->qkey != id_priv->qkey) {
+ printk(KERN_WARNING "qkey mismatch %.8x client qkey %.8x\n",
+ rep->qkey, id_priv->qkey);
event.event = RDMA_CM_EVENT_UNREACHABLE;
event.status = -EINVAL;
break;
@@ -2135,7 +2181,7 @@ int rdma_connect(struct rdma_cm_id *id,
switch (rdma_node_get_transport(id->device->node_type)) {
case RDMA_TRANSPORT_IB:
- if (id->ps == RDMA_PS_UDP)
+ if (id->ps == RDMA_PS_UDP || id->ps == RDMA_PS_IPOIB)
ret = cma_resolve_ib_udp(id_priv, conn_param);
else
ret = cma_connect_ib(id_priv, conn_param);
@@ -2231,7 +2277,7 @@ static int cma_send_sidr_rep(struct rdma
rep.status = status;
if (status == IB_SIDR_SUCCESS) {
rep.qp_num = id_priv->qp_num;
- rep.qkey = RDMA_UD_QKEY;
+ rep.qkey = id_priv->qkey;
}
rep.private_data = private_data;
rep.private_data_len = private_data_len;
@@ -2255,7 +2301,7 @@ int rdma_accept(struct rdma_cm_id *id, s
switch (rdma_node_get_transport(id->device->node_type)) {
case RDMA_TRANSPORT_IB:
- if (id->ps == RDMA_PS_UDP)
+ if (id->ps == RDMA_PS_UDP || id->ps == RDMA_PS_IPOIB)
ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
conn_param->private_data,
conn_param->private_data_len);
@@ -2316,7 +2362,7 @@ int rdma_reject(struct rdma_cm_id *id, c
switch (rdma_node_get_transport(id->device->node_type)) {
case RDMA_TRANSPORT_IB:
- if (id->ps == RDMA_PS_UDP)
+ if (id->ps == RDMA_PS_UDP || id->ps == RDMA_PS_IPOIB)
ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
private_data, private_data_len);
else
Index: rdma-dev/include/rdma/rdma_cm.h
===================================================================
--- rdma-dev.orig/include/rdma/rdma_cm.h 2007-01-18 13:46:06.000000000 +0200
+++ rdma-dev/include/rdma/rdma_cm.h 2007-01-23 12:35:35.000000000 +0200
@@ -58,6 +58,7 @@ enum rdma_cm_event_type {
enum rdma_port_space {
RDMA_PS_SDP = 0x0001,
+ RDMA_PS_IPOIB = 0x0002,
RDMA_PS_TCP = 0x0106,
RDMA_PS_UDP = 0x0111,
RDMA_PS_SCTP = 0x0183
Index: rdma-dev/drivers/infiniband/core/ucma.c
===================================================================
--- rdma-dev.orig/drivers/infiniband/core/ucma.c 2007-01-18 13:46:06.000000000 +0200
+++ rdma-dev/drivers/infiniband/core/ucma.c 2007-01-23 15:45:05.000000000 +0200
@@ -257,7 +257,7 @@ static int ucma_event_handler(struct rdm
ucma_set_event_context(ctx, event, uevent);
uevent->resp.event = event->event;
uevent->resp.status = event->status;
- if (cm_id->ps == RDMA_PS_UDP)
+ if (cm_id->ps == RDMA_PS_UDP || cm_id->ps == RDMA_PS_IPOIB)
ucma_copy_ud_event(&uevent->resp.param.ud, &event->param.ud);
else
ucma_copy_conn_event(&uevent->resp.param.conn,
More information about the general
mailing list