[openib-general] [PATCH] uDAPL - rdma_ucm branch: add changes to support rr/init exchange

[James Lentini]

Looks good Arlin.

On Fri, 19 Jan 2007, Arlin Davis wrote:

> Some uDAPL changes to support exchanging and validation of the device responder_resources and the
> initiator_depth during connection establishment.
> 
> Signed-off by: Arlin Davis ardavis at ichips.intel.com
> 
> 
> diff --git a/dapl/openib_cma/dapl_ib_cm.c b/dapl/openib_cma/dapl_ib_cm.c
> old mode 100644
> new mode 100755
> index 0f24244..8bdd0eb
> --- a/dapl/openib_cma/dapl_ib_cm.c
> +++ b/dapl/openib_cma/dapl_ib_cm.c
> @@ -259,6 +259,18 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
>  		new_conn->sp = conn->sp;
>  		new_conn->hca = conn->hca;
>  		
> +		/* Get requesters connect data, setup for accept */
> +		new_conn->params.responder_resources = 
> +			DAPL_MIN(event->param.conn.initiator_depth,
> +				 conn->hca->ib_trans.max_rdma_rd_in);
> +		new_conn->params.initiator_depth = 
> +			DAPL_MIN(event->param.conn.responder_resources,
> +				 conn->hca->ib_trans.max_rdma_rd_out);
> +
> +		new_conn->params.flow_control = event->param.conn.flow_control;
> +		new_conn->params.rnr_retry_count = event->param.conn.rnr_retry_count;
> +		new_conn->params.retry_count = event->param.conn.retry_count;
> +
>  		/* save private data */
>  		if (event->param.conn.private_data_len) {
>  			dapl_os_memcpy(new_conn->p_data, 
> @@ -279,7 +291,8 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
>  			     event->param.conn.private_data, event->param.conn.private_data_len);
>  		
>  		dapl_dbg_log(DAPL_DBG_TYPE_CM, " passive_cb: "
> -			     "REQ: IP SRC %x PORT %d DST %x PORT %d\n", 
> +			     "REQ: IP SRC %x PORT %d DST %x PORT %d "
> +			     "rr %d init %d\n", 
>  			     ntohl(((struct sockaddr_in *)
>  				&ipaddr->src_addr)->sin_addr.s_addr),
>  			     ntohs(((struct sockaddr_in *)
> @@ -287,7 +300,9 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
>  			     ntohl(((struct sockaddr_in *)
>  				&ipaddr->dst_addr)->sin_addr.s_addr),
>  			     ntohs(((struct sockaddr_in *)
> -				&ipaddr->dst_addr)->sin_port));
> +				&ipaddr->dst_addr)->sin_port),
> +			     new_conn->params.responder_resources,
> +			     new_conn->params.initiator_depth);
>  	}
>  	return new_conn;
>  }
> @@ -556,8 +571,8 @@ DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle,
>  
>  	/* Setup QP/CM parameters and private data in cm_id */
>  	(void)dapl_os_memzero(&conn->params, sizeof(conn->params));
> -	conn->params.responder_resources = IB_TARGET_MAX;
> -	conn->params.initiator_depth = IB_INITIATOR_DEPTH;
> +	conn->params.responder_resources = conn->hca->ib_trans.max_rdma_rd_in;
> +	conn->params.initiator_depth = conn->hca->ib_trans.max_rdma_rd_out;
>  	conn->params.flow_control = 1;
>  	conn->params.rnr_retry_count = IB_RNR_RETRY_COUNT;
>  	conn->params.retry_count = IB_RC_RETRY_COUNT;
> @@ -814,7 +829,6 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
>  	struct dapl_cm_id *cr_conn = cr_ptr->ib_cm_handle;
>  	int ret;
>  	DAT_RETURN dat_status;
> -	struct rdma_conn_param conn_params;
>  	
>  	dapl_dbg_log(DAPL_DBG_TYPE_CM,
>  		     " accept(cr %p conn %p, id %p, p_data %p, p_sz=%d)\n",
> @@ -867,16 +881,10 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
>  	ep_ptr->qp_handle = cr_conn;
>  	ep_ptr->cm_handle = cr_conn;
>  	cr_conn->ep = ep_ptr;
> +	cr_conn->params.private_data = p_data;
> +	cr_conn->params.private_data_len = p_size;
>  
> -	memset(&conn_params, 0, sizeof(conn_params));
> -	conn_params.private_data = p_data;
> -	conn_params.private_data_len = p_size;
> -	conn_params.responder_resources = IB_TARGET_MAX;
> -	conn_params.initiator_depth = IB_INITIATOR_DEPTH;
> -	conn_params.flow_control = 1;
> -	conn_params.rnr_retry_count = IB_RNR_RETRY_COUNT;
> -
> -	ret = rdma_accept(cr_conn->cm_id, &conn_params);
> +	ret = rdma_accept(cr_conn->cm_id, &cr_conn->params);
>  	if (ret) {
>  		dapl_dbg_log(DAPL_DBG_TYPE_ERR," accept: ERROR %d\n", ret);
>  		dat_status = dapl_convert_errno(ret, "accept");
> diff --git a/dapl/openib_cma/dapl_ib_util.c b/dapl/openib_cma/dapl_ib_util.c
> old mode 100644
> new mode 100755
> index 6bb35f6..0606312
> --- a/dapl/openib_cma/dapl_ib_util.c
> +++ b/dapl/openib_cma/dapl_ib_util.c
> @@ -469,6 +469,9 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA *hca_ptr,
>  		ia_attr->num_vendor_attr          = 0;
>  		ia_attr->vendor_attr              = NULL;
>  		ia_attr->max_iov_segments_per_rdma_read = dev_attr.max_sge;
> +		/* save rd_atom for peer validation during connect requests */
> +		hca_ptr->ib_trans.max_rdma_rd_in  = dev_attr.max_qp_rd_atom;
> +		hca_ptr->ib_trans.max_rdma_rd_out = dev_attr.max_qp_rd_atom;
>  
>  		dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 
>  			" query_hca: (ver=%x) ep %d ep_q %d evd %d evd_q %d\n", 
> diff --git a/dapl/openib_cma/dapl_ib_util.h b/dapl/openib_cma/dapl_ib_util.h
> old mode 100644
> new mode 100755
> index 1fadb08..457d26b
> --- a/dapl/openib_cma/dapl_ib_util.h
> +++ b/dapl/openib_cma/dapl_ib_util.h
> @@ -73,10 +73,7 @@ typedef ib_hca_handle_t		dapl_ibal_ca_t;
>  #define IB_ARP_RETRY_COUNT	15	/* 60 sec total */
>  #define IB_ROUTE_TIMEOUT	4000	/* 4 sec */
>  #define IB_ROUTE_RETRY_COUNT	15	/* 60 sec total */
> -#define IB_REQ_MRA_TIMEOUT	27	/* a little over 9 minutes */
>  #define IB_MAX_AT_RETRY		3
> -#define IB_TARGET_MAX		4	/* max_qp_ous_rd_atom */
> -#define IB_INITIATOR_DEPTH	4	/* max_qp_init_rd_atom */
>  
>  typedef enum {
>  	IB_CME_CONNECTED,
> @@ -272,6 +269,9 @@ typedef struct _ib_hca_transport
>  	ib_async_qp_handler_t	async_qp_error;
>  	uint8_t			max_cm_timeout;
>  	uint8_t			max_cm_retries;
> +	/* device attributes */
> +	int			max_rdma_rd_in;
> +	int			max_rdma_rd_out;
>  
>  } ib_hca_transport_t;
>  
>