[ofa-general] [PATCH] uDAPL v2: openib_cma: cleanup destroy/accept, add SID to PORT macros, fix init/resp settings in accept

[Arlin Davis]

openib_cma: fix cleanup issues in destroy_conn and accept
            macros to convert SID and PORT, network order
            fix init/responder settings in accept
 
Signed-off by: Arlin Davis <ardavis at ichips.intel.com>

diff --git a/dapl/openib_cma/dapl_ib_cm.c b/dapl/openib_cma/dapl_ib_cm.c
index e8c33f2..f7d83e0 100755
--- a/dapl/openib_cma/dapl_ib_cm.c
+++ b/dapl/openib_cma/dapl_ib_cm.c
@@ -69,13 +69,15 @@ static inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); }
 static inline uint64_t cpu_to_be64(uint64_t x) { return x; }
 #endif
 
-/* cma requires 16 bit SID */
+/* cma requires 16 bit SID, in network order */
 #define IB_PORT_MOD 32001
 #define IB_PORT_BASE (65535 - IB_PORT_MOD)
-#define MAKE_PORT(SID) \
+#define SID_TO_PORT(SID) \
     (SID > 0xffff ? \
-    (unsigned short)((SID % IB_PORT_MOD) + IB_PORT_BASE) :\
-    (unsigned short)SID)
+    htons((unsigned short)((SID % IB_PORT_MOD) + IB_PORT_BASE)) :\
+    htons((unsigned short)SID))
+
+#define PORT_TO_SID(p) ntohs(p)
 
 static void dapli_addr_resolve(struct dapl_cm_id *conn)
 {
@@ -173,8 +175,10 @@ void dapli_destroy_conn(struct dapl_cm_id *conn)
 	dapl_os_lock(&conn->lock);
 	conn->destroy = 1;
 	
-	if (conn->ep)
+	if (conn->ep) {
 		conn->ep->cm_handle = IB_INVALID_HANDLE;
+		conn->ep->qp_handle = IB_INVALID_HANDLE;
+	}
 
 	cm_id = conn->cm_id;
 	conn->cm_id = NULL;
@@ -220,10 +224,10 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
 		
 		/* Get requesters connect data, setup for accept */
 		new_conn->params.responder_resources = 
-			DAPL_MIN(event->param.conn.initiator_depth,
+			DAPL_MIN(event->param.conn.responder_resources,
 				 conn->hca->ib_trans.max_rdma_rd_in);
 		new_conn->params.initiator_depth = 
-			DAPL_MIN(event->param.conn.responder_resources,
+			DAPL_MIN(event->param.conn.initiator_depth,
 				 conn->hca->ib_trans.max_rdma_rd_out);
 
 		new_conn->params.flow_control = event->param.conn.flow_control;
@@ -348,8 +352,8 @@ static void dapli_cm_active_cb(struct dapl_cm_id *conn,
 			&conn->cm_id->route.addr.dst_addr)->sin_addr.s_addr));
 
 		/* setup local and remote ports for ep query */
-		conn->ep->param.remote_port_qual = rdma_get_dst_port(conn->cm_id);
-		conn->ep->param.local_port_qual = rdma_get_src_port(conn->cm_id);
+		conn->ep->param.remote_port_qual = PORT_TO_SID(rdma_get_dst_port(conn->cm_id));
+		conn->ep->param.local_port_qual = PORT_TO_SID(rdma_get_src_port(conn->cm_id));
 
 		dapl_evd_connection_callback(conn, IB_CME_CONNECTED,
 					     event->param.conn.private_data, conn->ep);
@@ -526,8 +530,9 @@ DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle,
 	if (NULL == ep_ptr) 
 		return DAT_SUCCESS;
 
-	dapl_dbg_log(DAPL_DBG_TYPE_CM, " connect: rSID %d, pdata %p, ln %d\n", 
-		     r_qual,p_data,p_size);
+	dapl_dbg_log(DAPL_DBG_TYPE_CM, 
+		     " connect: rSID 0x%llx rPort %d, pdata %p, ln %d\n", 
+		     r_qual,ntohs(SID_TO_PORT(r_qual)),p_data,p_size);
 			
 	/* rdma conn and cm_id pre-bound; reference via qp_handle */
 	conn = ep_ptr->cm_handle = ep_ptr->qp_handle;
@@ -549,7 +554,7 @@ DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle,
 	dapl_os_memcpy(&conn->r_addr, r_addr, sizeof(*r_addr));
 
 	/* Resolve remote address, src already bound during QP create */
-	((struct sockaddr_in*)&conn->r_addr)->sin_port = htons(MAKE_PORT(r_qual));
+	((struct sockaddr_in*)&conn->r_addr)->sin_port = SID_TO_PORT(r_qual);
 	((struct sockaddr_in*)&conn->r_addr)->sin_family = AF_INET;
 
 	if (rdma_resolve_addr(conn->cm_id, NULL, 
@@ -593,7 +598,7 @@ dapls_ib_disconnect(IN DAPL_EP *ep_ptr,
 		     " disconnect(ep %p, conn %p, id %d flags %x)\n",
 		     ep_ptr,conn, (conn?conn->cm_id:0),close_flags);
 
-	if (conn == IB_INVALID_HANDLE)
+        if ((conn == IB_INVALID_HANDLE) || (conn->cm_id == NULL))
 		return DAT_SUCCESS;
 
 	/* no graceful half-pipe disconnect option */
@@ -682,8 +687,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA *ia_ptr,
 	/* open identifies the local device; per DAT specification */
 	/* Get family and address then set port to consumer's ServiceID */
 	dapl_os_memcpy(&addr, &ia_ptr->hca_ptr->hca_address, sizeof(addr));
-	((struct sockaddr_in *)&addr)->sin_port = htons(MAKE_PORT(ServiceID));
-
+	((struct sockaddr_in *)&addr)->sin_port = SID_TO_PORT(ServiceID);
 
 	if (rdma_bind_addr(conn->cm_id,(struct sockaddr *)&addr)) {
 		if (errno == EBUSY)
@@ -695,8 +699,8 @@ dapls_ib_setup_conn_listener(IN DAPL_IA *ia_ptr,
 	}
 
 	dapl_dbg_log(DAPL_DBG_TYPE_CM,
-		" listen(ia_ptr %p SID %d sp %p conn %p id %d)\n",
-		ia_ptr, MAKE_PORT(ServiceID), 
+		" listen(ia_ptr %p SID 0x%llx Port %d sp %p conn %p id %d)\n",
+		ia_ptr, ServiceID, ntohs(SID_TO_PORT(ServiceID)), 
 		sp_ptr, conn, conn->cm_id);
 
 	sp_ptr->cm_srvc_handle = conn;
@@ -841,9 +845,6 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
 	}
 
    	cr_ptr->param.local_ep_handle = ep_handle;
-	ep_ptr->qp_handle = cr_conn;
-	ep_ptr->cm_handle = cr_conn;
-	cr_conn->ep = ep_ptr;
 	cr_conn->params.private_data = p_data;
 	cr_conn->params.private_data_len = p_size;
 
@@ -854,9 +855,15 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
 		goto bail;
 	}
 
+	/* save accepted conn and EP reference */
+	ep_ptr->qp_handle = cr_conn;
+	ep_ptr->cm_handle = cr_conn;
+	cr_conn->ep = ep_ptr;
+
 	/* setup local and remote ports for ep query */
-	ep_ptr->param.remote_port_qual = rdma_get_dst_port(cr_conn->cm_id);
-	ep_ptr->param.local_port_qual = rdma_get_src_port(cr_conn->cm_id);
+	/* Note: port qual in network order */
+	ep_ptr->param.remote_port_qual = PORT_TO_SID(rdma_get_dst_port(cr_conn->cm_id));
+	ep_ptr->param.local_port_qual = PORT_TO_SID(rdma_get_src_port(cr_conn->cm_id));
 
 	return DAT_SUCCESS;
 bail: