[openib-general] [PATCH] [CM] Replace QP pointer with necessary values only
Sean Hefty
mshefty at ichips.intel.com
Tue Mar 8 15:59:25 PST 2005
This patch modifies the CM API to take the necessary QP values, rather
than the QP pointer itself. This should simplify the implementation
of the usermode CM.
I did _not_ change the device caching information for this until I
can convince myself that clients should be able to gain access to the
device structure in this fashion.
As a side effect of this change, SIDR now works in theory.
The CM continues to try to send MADs using the same path as that
provided in the CM REQ message.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
Index: infiniband/core/cm.c
===================================================================
--- infiniband/core/cm.c (revision 1964)
+++ infiniband/core/cm.c (working copy)
@@ -61,6 +61,8 @@ static struct ib_client cm_client = {
static struct ib_cm {
spinlock_t lock;
+ struct list_head device_list;
+ rwlock_t device_lock;
struct rb_root listen_service_table;
/* struct rb_root peer_service_table; todo: fix peer to peer */
struct rb_root remote_qp_table;
@@ -71,13 +73,19 @@ static struct ib_cm {
} cm;
struct cm_port {
+ struct cm_device *cm_dev;
struct ib_mad_agent *mad_agent;
- u64 ca_guid;
- spinlock_t lock;
struct ib_mr *mr;
u8 port_num;
};
+struct cm_device {
+ struct list_head list;
+ struct ib_device *device;
+ u64 ca_guid;
+ struct cm_port port[0];
+};
+
struct cm_msg {
struct cm_id_private *cm_id_priv;
struct ib_send_wr send_wr;
@@ -214,48 +222,6 @@ static void cm_free_msg(struct cm_msg *m
kfree(msg);
}
-static struct cm_port * cm_find_port(struct ib_device *device,
- union ib_gid *gid)
-{
- struct cm_port *port;
- int ret;
- u8 p;
-
- port = (struct cm_port *)ib_get_client_data(device, &cm_client);
- if (!port)
- return NULL;
-
- ret = ib_find_cached_gid(device, gid, &p, NULL);
- if (ret)
- port = NULL;
- else
- port = &port[p-1];
-
- return port;
-}
-
-static int cm_find_device(union ib_gid *gid, struct ib_device **device,
- struct cm_port **port)
-{
- int ret;
- u8 p;
-
- /* todo: (high priority if SIDR is needed, low otherwise)
- write me - need call in ib_cache_* stuff? */
- /* see static device_list in device.c */
- /* ret = ib_find_cached_device_gid(gid, device, &p, NULL); */
- ret = -EINVAL;
- if (ret)
- return ret;
-
- *port = (struct cm_port *)ib_get_client_data(*device, &cm_client);
- if (!*port)
- return -EINVAL;
-
- *port = &(*port[p-1]);
- return 0;
-}
-
static void cm_set_ah_attr(struct ib_ah_attr *ah_attr, u8 port_num,
u16 dlid, u8 sl, u16 src_path_bits)
{
@@ -266,20 +232,33 @@ static void cm_set_ah_attr(struct ib_ah_
ah_attr->port_num = port_num;
}
-static int cm_init_av(struct ib_device *device, struct ib_sa_path_rec *path,
- struct cm_av *av)
+static int cm_init_av(struct ib_sa_path_rec *path, struct cm_av *av)
{
+ struct cm_device *cm_dev;
+ struct cm_port *port = NULL;
+ unsigned long flags;
int ret;
+ u8 p;
+
+ read_lock_irqsave(&cm.device_lock, flags);
+ list_for_each_entry(cm_dev, &cm.device_list, list) {
+ if (!ib_find_cached_gid(cm_dev->device, &path->sgid,
+ &p, NULL)) {
+ port = &cm_dev->port[p-1];
+ break;
+ }
+ }
+ read_unlock_irqrestore(&cm.device_lock, flags);
- av->port = cm_find_port(device, &path->sgid);
- if (!av->port)
+ if (!port)
return -EINVAL;
- ret = ib_find_cached_pkey(device, av->port->port_num, path->pkey,
- &av->pkey_index);
+ ret = ib_find_cached_pkey(cm_dev->device, port->port_num,
+ be16_to_cpu(path->pkey), &av->pkey_index);
if (ret)
return ret;
+ av->port = port;
av->dgid = path->dgid;
cm_set_ah_attr(&av->ah_attr, av->port->port_num, path->dlid,
path->sl, path->slid & 0x7F);
@@ -660,8 +639,9 @@ retest:
case IB_CM_MRA_REP_RCVD:
spin_unlock_irqrestore(&cm_id_priv->lock, flags);
ib_send_cm_rej(cm_id, IB_CM_REJ_TIMEOUT,
- &cm_id_priv->av.port->ca_guid,
- sizeof &cm_id_priv->av.port->ca_guid, NULL, 0);
+ &cm_id_priv->av.port->cm_dev->ca_guid,
+ sizeof &cm_id_priv->av.port->cm_dev->ca_guid,
+ NULL, 0);
break;
case IB_CM_ESTABLISHED:
spin_unlock_irqrestore(&cm_id_priv->lock, flags);
@@ -744,13 +724,13 @@ static void cm_format_req(struct cm_req_
req_msg->local_comm_id = cm_id_priv->id.local_id;
req_msg->service_id = param->service_id;
- req_msg->local_ca_guid = cm_id_priv->av.port->ca_guid;
- cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp->qp_num));
+ req_msg->local_ca_guid = cm_id_priv->av.port->cm_dev->ca_guid;
+ cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num));
cm_req_set_resp_res(req_msg, param->responder_resources);
cm_req_set_init_depth(req_msg, param->initiator_depth);
cm_req_set_remote_resp_timeout(req_msg,
param->remote_cm_response_timeout);
- cm_req_set_qp_type(req_msg, param->qp->qp_type);
+ cm_req_set_qp_type(req_msg, param->qp_type);
cm_req_set_flow_ctrl(req_msg, param->flow_control);
cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn));
cm_req_set_local_resp_timeout(req_msg,
@@ -760,7 +740,7 @@ static void cm_format_req(struct cm_req_
cm_req_set_path_mtu(req_msg, param->primary_path->mtu);
cm_req_set_rnr_retry_count(req_msg, param->rnr_retry_count);
cm_req_set_max_cm_retries(req_msg, param->max_cm_retries);
- cm_req_set_srq(req_msg, (param->qp->srq != NULL));
+ cm_req_set_srq(req_msg, param->srq);
req_msg->primary_local_lid = param->primary_path->slid;
req_msg->primary_remote_lid = param->primary_path->dlid;
@@ -798,10 +778,10 @@ static void cm_format_req(struct cm_req_
static inline int cm_validate_req_param(struct ib_cm_req_param *param)
{
- if (!param->qp || !param->primary_path)
+ if (!param->primary_path)
return -EINVAL;
- if (param->qp->qp_type != IB_QPT_RC && param->qp->qp_type != IB_QPT_UC)
+ if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC)
return -EINVAL;
if (param->private_data &&
@@ -839,13 +819,11 @@ int ib_send_cm_req(struct ib_cm_id *cm_i
}
spin_unlock_irqrestore(&cm_id_priv->lock, flags);
- ret = cm_init_av(param->qp->device, param->primary_path,
- &cm_id_priv->av);
+ ret = cm_init_av(param->primary_path, &cm_id_priv->av);
if (ret)
goto out;
if (param->alternate_path) {
- ret = cm_init_av(param->qp->device, param->alternate_path,
- &cm_id_priv->alt_av);
+ ret = cm_init_av(param->alternate_path, &cm_id_priv->alt_av);
if (ret)
goto out;
}
@@ -1078,13 +1056,11 @@ static int cm_req_handler(struct cm_work
cm_id_priv->id.service_mask = ~0ULL;
cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]);
- ret = cm_init_av(work->port->mad_agent->device, &work->path[0],
- &cm_id_priv->av);
+ ret = cm_init_av(&work->path[0], &cm_id_priv->av);
if (ret)
goto error3;
if (req_msg->alt_local_lid) {
- ret = cm_init_av(work->port->mad_agent->device, &work->path[1],
- &cm_id_priv->alt_av);
+ ret = cm_init_av(&work->path[1], &cm_id_priv->alt_av);
if (ret)
goto error3;
}
@@ -1124,7 +1100,7 @@ static void cm_format_rep(struct cm_rep_
rep_msg->local_comm_id = cm_id_priv->id.local_id;
rep_msg->remote_comm_id = cm_id_priv->id.remote_id;
- cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp->qp_num));
+ cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num));
cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn));
rep_msg->resp_resources = param->responder_resources;
rep_msg->initiator_depth = param->initiator_depth;
@@ -1132,29 +1108,14 @@ static void cm_format_rep(struct cm_rep_
cm_rep_set_failover(rep_msg, param->failover_accepted);
cm_rep_set_flow_ctrl(rep_msg, param->flow_control);
cm_rep_set_rnr_retry_count(rep_msg, param->rnr_retry_count);
- cm_rep_set_srq(rep_msg, (param->qp->srq != NULL));
- rep_msg->local_ca_guid = cm_id_priv->av.port->ca_guid;
+ cm_rep_set_srq(rep_msg, param->srq);
+ rep_msg->local_ca_guid = cm_id_priv->av.port->cm_dev->ca_guid;
if (param->private_data && param->private_data_len)
memcpy(rep_msg->private_data, param->private_data,
param->private_data_len);
}
-static inline int cm_validate_rep_param(struct ib_cm_rep_param *param)
-{
- if (!param->qp)
- return -EINVAL;
-
- if (param->qp->qp_type != IB_QPT_RC && param->qp->qp_type != IB_QPT_UC)
- return -EINVAL;
-
- if (param->private_data &&
- param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
- return -EINVAL;
-
- return 0;
-}
-
int ib_send_cm_rep(struct ib_cm_id *cm_id,
struct ib_cm_rep_param *param)
{
@@ -1165,9 +1126,11 @@ int ib_send_cm_rep(struct ib_cm_id *cm_i
unsigned long flags;
int ret;
- ret = cm_validate_rep_param(param);
- if (ret)
+ if (param->private_data &&
+ param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE) {
+ ret = -EINVAL;
goto out;
+ }
cm_id_priv = container_of(cm_id, struct cm_id_private, id);
ret = cm_alloc_msg(cm_id_priv, &msg);
@@ -2313,7 +2276,6 @@ static void cm_format_sidr_req(struct cm
int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
struct ib_cm_sidr_req_param *param)
{
- struct ib_device *device;
struct cm_id_private *cm_id_priv;
struct cm_msg *msg;
struct ib_send_wr *bad_send_wr;
@@ -2325,11 +2287,7 @@ int ib_send_cm_sidr_req(struct ib_cm_id
return -EINVAL;
cm_id_priv = container_of(cm_id, struct cm_id_private, id);
- ret = cm_find_device(¶m->path->sgid, &device, &cm_id_priv->av.port);
- if (ret)
- goto out;
-
- ret = cm_init_av(device, param->path, &cm_id_priv->av);
+ ret = cm_init_av(param->path, &cm_id_priv->av);
if (ret)
goto out;
@@ -2966,7 +2924,8 @@ static u64 cm_get_ca_guid(struct ib_devi
static void cm_add_one(struct ib_device *device)
{
- struct cm_port *port_array, *port;
+ struct cm_device *cm_dev;
+ struct cm_port *port;
struct ib_mad_reg_req reg_req = {
.mgmt_class = IB_MGMT_CLASS_CM,
.mgmt_class_version = IB_CM_CLASS_VERSION
@@ -2974,24 +2933,25 @@ static void cm_add_one(struct ib_device
struct ib_port_modify port_modify = {
.set_port_cap_mask = IB_PORT_CM_SUP
};
- u64 ca_guid;
- u8 i;
+ unsigned long flags;
int ret;
+ u8 i;
- ca_guid = cm_get_ca_guid(device);
- if (!ca_guid)
+ cm_dev = kmalloc(sizeof(*cm_dev) + sizeof(*port) *
+ device->phys_port_cnt, GFP_KERNEL);
+ if (!cm_dev)
return;
- port_array = kmalloc(sizeof *port * device->phys_port_cnt, GFP_KERNEL);
- if (!port_array)
- return;
+ cm_dev->device = device;
+ cm_dev->ca_guid = cm_get_ca_guid(device);
+ if (!cm_dev->ca_guid)
+ goto error1;
set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
- for (i = 1, port = port_array; i <= device->phys_port_cnt; i++, port++){
- spin_lock_init(&port->lock);
- port->ca_guid = ca_guid;
+ for (i = 1; i <= device->phys_port_cnt; i++) {
+ port = &cm_dev->port[i-1];
+ port->cm_dev = cm_dev;
port->port_num = i;
-
port->mad_agent = ib_register_mad_agent(device, i,
IB_QPT_GSI,
®_req,
@@ -3000,54 +2960,64 @@ static void cm_add_one(struct ib_device
cm_recv_handler,
port);
if (IS_ERR(port->mad_agent))
- goto error1;
+ goto error2;
port->mr = ib_get_dma_mr(port->mad_agent->qp->pd,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(port->mr))
- goto error2;
+ goto error3;
ret = ib_modify_port(device, i, 0, &port_modify);
if (ret)
- goto error3;
+ goto error4;
}
- ib_set_client_data(device, &cm_client, port_array);
+ ib_set_client_data(device, &cm_client, cm_dev);
+
+ write_lock_irqsave(&cm.device_lock, flags);
+ list_add_tail(&cm_dev->list, &cm.device_list);
+ write_unlock_irqrestore(&cm.device_lock, flags);
return;
-error3:
+error4:
ib_dereg_mr(port->mr);
-error2:
+error3:
ib_unregister_mad_agent(port->mad_agent);
-error1:
+error2:
port_modify.set_port_cap_mask = 0;
port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
- while (port != port_array) {
- --port;
+ while (--i) {
+ port = &cm_dev->port[i];
ib_modify_port(device, port->port_num, 0, &port_modify);
- ib_dereg_mr(port->mr);
ib_unregister_mad_agent(port->mad_agent);
}
- kfree(port_array);
+error1:
+ kfree(cm_dev);
}
static void cm_remove_one(struct ib_device *device)
{
- struct cm_port *port_array, *port;
+ struct cm_device *cm_dev;
+ struct cm_port *port;
struct ib_port_modify port_modify = {
.clr_port_cap_mask = IB_PORT_CM_SUP
};
+ unsigned long flags;
int i;
- port_array = (struct cm_port *)ib_get_client_data(device, &cm_client);
- if (!port_array)
+ cm_dev = ib_get_client_data(device, &cm_client);
+ if (!cm_dev)
return;
- for (i = 1, port = port_array; i <= device->phys_port_cnt; i++, port++){
+ write_lock_irqsave(&cm.device_lock, flags);
+ list_del(&cm_dev->list);
+ write_unlock_irqrestore(&cm.device_lock, flags);
+
+ for (i = 1; i <= device->phys_port_cnt; i++) {
+ port = &cm_dev->port[i];
ib_modify_port(device, port->port_num, 0, &port_modify);
- ib_dereg_mr(port->mr);
ib_unregister_mad_agent(port->mad_agent);
}
- kfree(port_array);
+ kfree(cm_dev);
}
static int __init ib_cm_init(void)
@@ -3055,6 +3025,8 @@ static int __init ib_cm_init(void)
int ret;
memset(&cm, 0, sizeof cm);
+ INIT_LIST_HEAD(&cm.device_list);
+ rwlock_init(&cm.device_lock);
spin_lock_init(&cm.lock);
cm.listen_service_table = RB_ROOT;
cm.remote_id_table = RB_ROOT;
More information about the general
mailing list