[openib-general] [PATCH] uDAPL openib-cma provider - add support for IB_CM_REQ_OPTIONS
Arlin Davis
arlin.r.davis at intel.com
Mon Jun 5 17:16:31 PDT 2006
James,
Here is a patch to the openib-cma provider that uses the new set_option feature of the uCMA to
adjust connect request timeout and retry values. The defaults are a little quick for some consumers.
They are now bumped up from 3 retries to 15 and are tunable with uDAPL environment variables. Also,
included a fix to disallow any event after a disconnect event.
You need to sync up the commit with Sean's patch for the uCMA get/set IB_CM_REQ_OPTIONS.
I would like to get this in OFED RC6 if possible.
Thanks,
-arlin
Signed-off by: Arlin Davis ardavis at ichips.intel.com
Index: dapl/openib_cma/dapl_ib_util.c
===================================================================
--- dapl/openib_cma/dapl_ib_util.c (revision 7694)
+++ dapl/openib_cma/dapl_ib_util.c (working copy)
@@ -264,7 +264,15 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_N
/* set inline max with env or default, get local lid and gid 0 */
hca_ptr->ib_trans.max_inline_send =
dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_DEFAULT);
-
+
+ /* set CM timer defaults */
+ hca_ptr->ib_trans.max_cm_timeout =
+ dapl_os_get_env_val("DAPL_MAX_CM_RESPONSE_TIME",
+ IB_CM_RESPONSE_TIMEOUT);
+ hca_ptr->ib_trans.max_cm_retries =
+ dapl_os_get_env_val("DAPL_MAX_CM_RETRIES",
+ IB_CM_RETRIES);
+
/* EVD events without direct CQ channels, non-blocking */
hca_ptr->ib_trans.ib_cq =
ibv_create_comp_channel(hca_ptr->ib_hca_handle);
Index: dapl/openib_cma/dapl_ib_cm.c
===================================================================
--- dapl/openib_cma/dapl_ib_cm.c (revision 7694)
+++ dapl/openib_cma/dapl_ib_cm.c (working copy)
@@ -58,6 +58,7 @@
#include "dapl_ib_util.h"
#include <sys/poll.h>
#include <signal.h>
+#include <rdma/rdma_cma_ib.h>
extern struct rdma_event_channel *g_cm_events;
@@ -85,7 +86,6 @@ static inline uint64_t cpu_to_be64(uint6
(unsigned short)((SID % IB_PORT_MOD) + IB_PORT_BASE) :\
(unsigned short)SID)
-
static void dapli_addr_resolve(struct dapl_cm_id *conn)
{
int ret;
@@ -114,6 +114,8 @@ static void dapli_addr_resolve(struct da
static void dapli_route_resolve(struct dapl_cm_id *conn)
{
int ret;
+ size_t optlen = sizeof(struct ib_cm_req_opt);
+ struct ib_cm_req_opt req_opt;
#ifdef DAPL_DBG
struct rdma_addr *ipaddr = &conn->cm_id->route.addr;
struct ib_addr *ibaddr = &conn->cm_id->route.addr.addr.ibaddr;
@@ -143,13 +145,43 @@ static void dapli_route_resolve(struct d
cpu_to_be64(ibaddr->dgid.global.interface_id));
dapl_dbg_log(DAPL_DBG_TYPE_CM,
- " rdma_connect: cm_id %p pdata %p plen %d rr %d ind %d\n",
+ " route_resolve: cm_id %p pdata %p plen %d rr %d ind %d\n",
conn->cm_id,
conn->params.private_data,
conn->params.private_data_len,
conn->params.responder_resources,
conn->params.initiator_depth );
+ /* Get default connect request timeout values, and adjust */
+ ret = rdma_get_option(conn->cm_id, RDMA_PROTO_IB, IB_CM_REQ_OPTIONS,
+ (void*)&req_opt, &optlen);
+ if (ret) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " rdma_get_option failed: %s\n",
+ strerror(errno));
+ goto bail;
+ }
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM, " route_resolve: "
+ "Set CR times - response %d to %d, retry %d to %d\n",
+ req_opt.remote_cm_response_timeout,
+ conn->hca->ib_trans.max_cm_timeout,
+ req_opt.max_cm_retries,
+ conn->hca->ib_trans.max_cm_retries);
+
+ /* Use hca response time setting for connect requests */
+ req_opt.max_cm_retries = conn->hca->ib_trans.max_cm_retries;
+ req_opt.remote_cm_response_timeout =
+ conn->hca->ib_trans.max_cm_timeout;
+ req_opt.local_cm_response_timeout =
+ req_opt.remote_cm_response_timeout;
+ ret = rdma_set_option(conn->cm_id, RDMA_PROTO_IB, IB_CM_REQ_OPTIONS,
+ (void*)&req_opt, optlen);
+ if (ret) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " rdma_set_option failed: %s\n",
+ strerror(errno));
+ goto bail;
+ }
+
ret = rdma_connect(conn->cm_id, &conn->params);
if (ret) {
dapl_dbg_log(DAPL_DBG_TYPE_ERR, " rdma_connect failed: %s\n",
@@ -273,14 +305,37 @@ static void dapli_cm_active_cb(struct da
}
dapl_os_unlock(&conn->lock);
+ /* There is a chance that we can get events after
+ * the consumer calls disconnect in a pending state
+ * since the IB CM and uDAPL states are not shared.
+ * In some cases, IB CM could generate either a DCONN
+ * or CONN_ERR after the consumer returned from
+ * dapl_ep_disconnect with a DISCONNECTED event
+ * already queued. Check state here and bail to
+ * avoid any events after a disconnect.
+ */
+ if (DAPL_BAD_HANDLE(conn->ep, DAPL_MAGIC_EP))
+ return;
+
+ dapl_os_lock(&conn->ep->header.lock);
+ if (conn->ep->param.ep_state == DAT_EP_STATE_DISCONNECTED) {
+ dapl_os_unlock(&conn->ep->header.lock);
+ return;
+ }
+ if (event->event == RDMA_CM_EVENT_DISCONNECTED)
+ conn->ep->param.ep_state = DAT_EP_STATE_DISCONNECTED;
+
+ dapl_os_unlock(&conn->ep->header.lock);
+
switch (event->event) {
case RDMA_CM_EVENT_UNREACHABLE:
case RDMA_CM_EVENT_CONNECT_ERROR:
- dapl_dbg_log(
- DAPL_DBG_TYPE_WARN,
- " dapli_cm_active_handler: CONN_ERR "
- " event=0x%x status=%d\n",
- event->event, event->status);
+ dapl_dbg_log(
+ DAPL_DBG_TYPE_WARN,
+ " dapli_cm_active_handler: CONN_ERR "
+ " event=0x%x status=%d %s\n",
+ event->event, event->status,
+ (event->status == -110)?"TIMEOUT":"" );
dapl_evd_connection_callback(conn,
IB_CME_DESTINATION_UNREACHABLE,
@@ -368,25 +423,23 @@ static void dapli_cm_passive_cb(struct d
event->private_data, new_conn->sp);
break;
case RDMA_CM_EVENT_UNREACHABLE:
- dapls_cr_callback(conn, IB_CME_DESTINATION_UNREACHABLE,
- NULL, conn->sp);
-
case RDMA_CM_EVENT_CONNECT_ERROR:
dapl_dbg_log(
- DAPL_DBG_TYPE_WARN,
- " dapli_cm_passive: CONN_ERR "
- " event=0x%x status=%d",
- " on SRC 0x%x,0x%x DST 0x%x,0x%x\n",
- event->event, event->status,
- ntohl(((struct sockaddr_in *)
- &ipaddr->src_addr)->sin_addr.s_addr),
- ntohs(((struct sockaddr_in *)
- &ipaddr->src_addr)->sin_port),
- ntohl(((struct sockaddr_in *)
- &ipaddr->dst_addr)->sin_addr.s_addr),
- ntohs(((struct sockaddr_in *)
- &ipaddr->dst_addr)->sin_port));
+ DAPL_DBG_TYPE_WARN,
+ " dapli_cm_passive: CONN_ERR "
+ " event=0x%x status=%d %s"
+ " on SRC 0x%x,0x%x DST 0x%x,0x%x\n",
+ event->event, event->status,
+ (event->status == -110)?"TIMEOUT":"",
+ ntohl(((struct sockaddr_in *)
+ &ipaddr->src_addr)->sin_addr.s_addr),
+ ntohs(((struct sockaddr_in *)
+ &ipaddr->src_addr)->sin_port),
+ ntohl(((struct sockaddr_in *)
+ &ipaddr->dst_addr)->sin_addr.s_addr),
+ ntohs(((struct sockaddr_in *)
+ &ipaddr->dst_addr)->sin_port));
dapls_cr_callback(conn, IB_CME_DESTINATION_UNREACHABLE,
NULL, conn->sp);
Index: dapl/openib_cma/dapl_ib_util.h
===================================================================
--- dapl/openib_cma/dapl_ib_util.h (revision 7694)
+++ dapl/openib_cma/dapl_ib_util.h (working copy)
@@ -67,8 +67,8 @@ typedef ib_hca_handle_t dapl_ibal_ca_t;
#define IB_RC_RETRY_COUNT 7
#define IB_RNR_RETRY_COUNT 7
-#define IB_CM_RESPONSE_TIMEOUT 18 /* 1 sec */
-#define IB_MAX_CM_RETRIES 7
+#define IB_CM_RESPONSE_TIMEOUT 20 /* 4 sec */
+#define IB_CM_RETRIES 15
#define IB_REQ_MRA_TIMEOUT 27 /* a little over 9 minutes */
#define IB_MAX_AT_RETRY 3
#define IB_TARGET_MAX 4 /* max_qp_ous_rd_atom */
@@ -252,6 +252,8 @@ typedef struct _ib_hca_transport
ib_async_cq_handler_t async_cq_error;
ib_async_dto_handler_t async_cq;
ib_async_qp_handler_t async_qp_error;
+ uint8_t max_cm_timeout;
+ uint8_t max_cm_retries;
} ib_hca_transport_t;
More information about the general
mailing list