[openib-general] [PATCH 1/2] RDMA CM: allow user to set IB CM timeout and retries
Sean Hefty
sean.hefty at intel.com
Mon Jun 5 17:05:25 PDT 2006
Allow users to override the default number of retries and timeout used
by the RDMA CM when connecting over Infiniband. Some applications, like
MPI, are unable to connect within the default timeout value when scaling
up.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
Index: include/rdma/rdma_user_cm.h
===================================================================
--- include/rdma/rdma_user_cm.h (revision 7619)
+++ include/rdma/rdma_user_cm.h (working copy)
@@ -203,6 +203,7 @@ enum {
/* IB specific option names for get/set. */
enum {
IB_PATH_OPTIONS = 1,
+ IB_CM_REQ_OPTIONS = 2,
};
struct rdma_ucm_get_option_resp {
Index: include/rdma/rdma_cm_ib.h
===================================================================
--- include/rdma/rdma_cm_ib.h (revision 7619)
+++ include/rdma/rdma_cm_ib.h (working copy)
@@ -44,4 +44,26 @@
int rdma_set_ib_paths(struct rdma_cm_id *id,
struct ib_sa_path_rec *path_rec, int num_paths);
+struct ib_cm_req_opt {
+ u8 remote_cm_response_timeout;
+ u8 local_cm_response_timeout;
+ u8 max_cm_retries;
+};
+
+/**
+ * rdma_get_ib_req_info - Retrieves the current IB CM REQ / SIDR REQ values
+ * that will be used when connection, or performing service ID resolution.
+ * @id: Connection identifier associated with the request.
+ * @info: Current values for CM REQ messages.
+ */
+int rdma_get_ib_req_info(struct rdma_cm_id *id, struct ib_cm_req_opt *info);
+
+/**
+ * rdma_set_ib_req_info - Sets the current IB CM REQ / SIDR REQ values
+ * that will be used when connection, or performing service ID resolution.
+ * @id: Connection identifier associated with the request.
+ * @info: New values for CM REQ messages.
+ */
+int rdma_set_ib_req_info(struct rdma_cm_id *id, struct ib_cm_req_opt *info);
+
#endif /* RDMA_CM_IB_H */
Index: core/ucma_ib.c
===================================================================
--- core/ucma_ib.c (revision 7619)
+++ core/ucma_ib.c (working copy)
@@ -81,12 +81,37 @@ static int ucma_get_paths(struct rdma_cm
return ret;
}
+static int ucma_get_req_opt(struct rdma_cm_id *id, void __user *opt,
+ int *optlen)
+{
+ struct ib_cm_req_opt req_opt;
+ int ret = 0;
+
+ if (!opt)
+ goto out;
+
+ if (*optlen < sizeof req_opt) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ ret = rdma_get_ib_req_info(id, &req_opt);
+ if (!ret)
+ if (copy_to_user(opt, &req_opt, sizeof req_opt))
+ ret = -EFAULT;
+out:
+ *optlen = sizeof req_opt;
+ return ret;
+}
+
int ucma_get_ib_option(struct rdma_cm_id *id, int optname,
void *optval, int *optlen)
{
switch (optname) {
case IB_PATH_OPTIONS:
return ucma_get_paths(id, optval, optlen);
+ case IB_CM_REQ_OPTIONS:
+ return ucma_get_req_opt(id, optval, optlen);
default:
return -EINVAL;
}
@@ -132,12 +157,27 @@ out:
return ret;
}
+static int ucma_set_req_opt(struct rdma_cm_id *id, void __user *opt, int optlen)
+{
+ struct ib_cm_req_opt req_opt;
+
+ if (optlen != sizeof req_opt)
+ return -EINVAL;
+
+ if (copy_from_user(&req_opt, opt, sizeof req_opt))
+ return -EFAULT;
+
+ return rdma_set_ib_req_info(id, &req_opt);
+}
+
int ucma_set_ib_option(struct rdma_cm_id *id, int optname,
void *optval, int optlen)
{
switch (optname) {
case IB_PATH_OPTIONS:
return ucma_set_paths(id, optval, optlen);
+ case IB_CM_REQ_OPTIONS:
+ return ucma_set_req_opt(id, optval, optlen);
default:
return -EINVAL;
}
Index: core/cma.c
===================================================================
--- core/cma.c (revision 7619)
+++ core/cma.c (working copy)
@@ -126,6 +126,10 @@ struct rdma_id_private {
struct ib_cm_id *ib;
} cm_id;
+ union {
+ struct ib_cm_req_opt *req;
+ } options;
+
u32 seq_num;
u32 qp_num;
enum ib_qp_type qp_type;
@@ -710,6 +714,7 @@ void rdma_destroy_id(struct rdma_cm_id *
wait_for_completion(&id_priv->comp);
kfree(id_priv->id.route.path_rec);
+ kfree(id_priv->options.req);
kfree(id_priv);
}
EXPORT_SYMBOL(rdma_destroy_id);
@@ -1240,6 +1245,65 @@ err:
}
EXPORT_SYMBOL(rdma_set_ib_paths);
+static inline u8 cma_get_ib_remote_timeout(struct rdma_id_private *id_priv)
+{
+ return id_priv->options.req ?
+ id_priv->options.req->remote_cm_response_timeout :
+ CMA_CM_RESPONSE_TIMEOUT;
+}
+
+static inline u8 cma_get_ib_local_timeout(struct rdma_id_private *id_priv)
+{
+ return id_priv->options.req ?
+ id_priv->options.req->local_cm_response_timeout :
+ CMA_CM_RESPONSE_TIMEOUT;
+}
+
+static inline u8 cma_get_ib_cm_retries(struct rdma_id_private *id_priv)
+{
+ return id_priv->options.req ?
+ id_priv->options.req->max_cm_retries : CMA_MAX_CM_RETRIES;
+}
+
+int rdma_get_ib_req_info(struct rdma_cm_id *id, struct ib_cm_req_opt *info)
+{
+ struct rdma_id_private *id_priv;
+
+ id_priv = container_of(id, struct rdma_id_private, id);
+ if (!cma_comp(id_priv, CMA_ROUTE_RESOLVED))
+ return -EINVAL;
+
+ info->remote_cm_response_timeout = cma_get_ib_remote_timeout(id_priv);
+ info->local_cm_response_timeout = cma_get_ib_local_timeout(id_priv);
+ info->max_cm_retries = cma_get_ib_cm_retries(id_priv);
+ return 0;
+}
+EXPORT_SYMBOL(rdma_get_ib_req_info);
+
+int rdma_set_ib_req_info(struct rdma_cm_id *id, struct ib_cm_req_opt *info)
+{
+ struct rdma_id_private *id_priv;
+
+ if (info->remote_cm_response_timeout > 0x1F ||
+ info->local_cm_response_timeout > 0x1F ||
+ info->max_cm_retries > 0xF)
+ return -EINVAL;
+
+ id_priv = container_of(id, struct rdma_id_private, id);
+ if (!cma_comp(id_priv, CMA_ROUTE_RESOLVED))
+ return -EINVAL;
+
+ if (!id_priv->options.req) {
+ id_priv->options.req = kmalloc(sizeof *info, GFP_KERNEL);
+ if (!id_priv->options.req)
+ return -ENOMEM;
+ }
+
+ *id_priv->options.req = *info;
+ return 0;
+}
+EXPORT_SYMBOL(rdma_set_ib_req_info);
+
int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
{
struct rdma_id_private *id_priv;
@@ -1646,9 +1710,9 @@ static int cma_connect_ib(struct rdma_id
req.flow_control = conn_param->flow_control;
req.retry_count = conn_param->retry_count;
req.rnr_retry_count = conn_param->rnr_retry_count;
- req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
- req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
- req.max_cm_retries = CMA_MAX_CM_RETRIES;
+ req.remote_cm_response_timeout = cma_get_ib_remote_timeout(id_priv);
+ req.local_cm_response_timeout = cma_get_ib_local_timeout(id_priv);
+ req.max_cm_retries = cma_get_ib_cm_retries(id_priv);
req.srq = id_priv->srq ? 1 : 0;
ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
@@ -1707,7 +1771,7 @@ static int cma_accept_ib(struct rdma_id_
rep.private_data_len = conn_param->private_data_len;
rep.responder_resources = conn_param->responder_resources;
rep.initiator_depth = conn_param->initiator_depth;
- rep.target_ack_delay = CMA_CM_RESPONSE_TIMEOUT;
+ rep.target_ack_delay = cma_get_ib_local_timeout(id_priv);
rep.failover_accepted = 0;
rep.flow_control = conn_param->flow_control;
rep.rnr_retry_count = conn_param->rnr_retry_count;
More information about the general
mailing list