[ofa-general] [PATCH 4/5] [DAPL] dapl/ibal: update IBAL provider with latest WinOF codebase

Sean Hefty sean.hefty at intel.com
Fri Jan 30 10:58:06 PST 2009


From: Stan Smith <stan.smith at intel.com>

Merge SVN IBAL provider code back into the main git tree.

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
Actual codng changes were made by Stan.  I'm just submitting the patch
to update the DAPL git repository.

 dapl/dirs                          |    2 
 dapl/ibal/dapl_ibal_cm.c           |  274 ++++++++++++++++++++++--------------
 dapl/ibal/dapl_ibal_cq.c           |    4 -
 dapl/ibal/dapl_ibal_dto.h          |   10 +
 dapl/ibal/dapl_ibal_extensions.c   |   17 --
 dapl/ibal/dapl_ibal_name_service.h |    2 
 dapl/ibal/dapl_ibal_qp.c           |   12 +-
 dapl/ibal/dapl_ibal_util.c         |   63 ++++++++
 dapl/ibal/dapl_ibal_util.h         |   85 ++++++++++-
 9 files changed, 322 insertions(+), 147 deletions(-)

diff --git a/dapl/dirs b/dapl/dirs
index a091968..6fb48fa 100644
--- a/dapl/dirs
+++ b/dapl/dirs
@@ -1 +1 @@
-DIRS=udapl udapl_scm
+DIRS = ibal ibal-scm
diff --git a/dapl/ibal/dapl_ibal_cm.c b/dapl/ibal/dapl_ibal_cm.c
index a986430..fe5501a 100644
--- a/dapl/ibal/dapl_ibal_cm.c
+++ b/dapl/ibal/dapl_ibal_cm.c
@@ -30,6 +30,7 @@
 #include "dapl_ibal_util.h"
 #include "dapl_name_service.h"
 #include "dapl_ibal_name_service.h"
+#include "dapl_cookie.h"
 
 #define IB_INFINITE_SERVICE_LEASE   0xFFFFFFFF
 #define  DAPL_ATS_SERVICE_ID        ATS_SERVICE_ID //0x10000CE100415453
@@ -78,15 +79,35 @@ dapli_ib_cm_event_str(ib_cm_events_t e)
 }
 
 
-static void 
-dapli_ibal_listen_err_cb (
-        IN    ib_listen_err_rec_t          *p_listen_err_rec )
+#if defined(DAPL_DBG)
+
+void dapli_print_private_data( char *prefix, const uint8_t *pd, int len )
 {
-    UNUSED_PARAM( p_listen_err_rec );
+    int i;
+            
+    if ( !pd || len <= 0 )
+	return;
+
+    dapl_log ( DAPL_DBG_TYPE_CM, "--> %s: private_data:\n    ",prefix);
 
-    dapl_dbg_log (DAPL_DBG_TYPE_CM, "--> %s: CM callback listen error\n",
-                  "DiLEcb");
+    if (len > IB_MAX_REP_PDATA_SIZE)
+    {
+    	dapl_log ( DAPL_DBG_TYPE_ERR,
+		"    Private data size(%d) > Max(%d), ignored.\n    ",
+					len,DAPL_MAX_PRIVATE_DATA_SIZE);
+	len = IB_MAX_REP_PDATA_SIZE;
+    }
+
+    for ( i = 0 ; i < len; i++ )
+    {
+	dapl_log ( DAPL_DBG_TYPE_CM, "%2x ", pd[i]);
+	if ( ((i+1) % 20) == 0 ) 
+	    dapl_log ( DAPL_DBG_TYPE_CM, "\n    ");
+    }
+   dapl_log ( DAPL_DBG_TYPE_CM, "\n");
 }
+#endif
+
 
 static void 
 dapli_ib_cm_apr_cb (
@@ -110,6 +131,7 @@ dapli_ib_cm_lap_cb (
 
 /*
  * Connection Disconnect Request callback
+ * We received a DREQ, return a DREP (disconnect reply).
  */
 
 static void 
@@ -124,23 +146,34 @@ dapli_ib_cm_dreq_cb (
 
     ep_ptr  = (DAPL_EP * __ptr64) p_cm_dreq_rec->qp_context;
 
-    if ( ep_ptr == NULL ||
-         ep_ptr->header.magic == DAPL_MAGIC_INVALID )
+    if ( DAPL_BAD_PTR(ep_ptr) )
     {
         dapl_dbg_log (DAPL_DBG_TYPE_ERR, 
-                      "--> DiCDcb: EP %lx invalid or FREED\n", ep_ptr);
+                      "--> %s: BAD_PTR EP %lx\n", __FUNCTION__, ep_ptr);
+        return;
+    }
+    if ( ep_ptr->header.magic != DAPL_MAGIC_EP  )
+    {
+        if ( ep_ptr->header.magic == DAPL_MAGIC_INVALID )
+            return;
+
+        dapl_dbg_log (DAPL_DBG_TYPE_ERR,
+                      "--> %s: EP %p BAD_EP_MAGIC %x != wanted %x\n",
+		       __FUNCTION__, ep_ptr, ep_ptr->header.magic,
+		       DAPL_MAGIC_EP );
         return;
     }
+
     dapl_dbg_log (DAPL_DBG_TYPE_CM, 
-                  "--> %s() QP %lx EP %lx state %s sent_discreq %d\n",
-                  __FUNCTION__, ep_ptr->qp_handle, ep_ptr,
+                  "--> %s() EP %p, %s sent_discreq %s\n",
+                  __FUNCTION__,ep_ptr,
                   dapl_get_ep_state_str(ep_ptr->param.ep_state),
-                  ep_ptr->sent_discreq );
+                  (ep_ptr->sent_discreq == DAT_TRUE ? "TRUE":"FALSE"));
 
     dapl_os_lock (&ep_ptr->header.lock);
     if ( ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED
          /*|| ( ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECT_PENDING
-           && ep_ptr->sent_discreq == DAT_FALSE)*/ )
+           && ep_ptr->sent_discreq == DAT_TRUE)*/ )
     {
         dapl_os_unlock (&ep_ptr->header.lock);
         dapl_dbg_log (DAPL_DBG_TYPE_CM, 
@@ -150,19 +183,18 @@ dapli_ib_cm_dreq_cb (
     }
 
     ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECT_PENDING;
-
     ep_ptr->recv_discreq = DAT_TRUE;
     dapl_os_unlock (&ep_ptr->header.lock);
 
-    dapl_os_memzero (&cm_drep, sizeof ( ib_cm_drep_t));
+    dapl_os_memzero (&cm_drep, sizeof(ib_cm_drep_t));
 
     /* Could fail if we received reply from other side, no need to retry */
-    /* Wait for any transaction in process holding reference */
-    while ( dapl_os_atomic_read(&ep_ptr->req_count) && bail-- > 0 )
+    /* Wait for any send ops in process holding reference */
+    while (dapls_cb_pending(&ep_ptr->req_buffer) && bail-- > 0 )
     {
     	dapl_dbg_log (DAPL_DBG_TYPE_CM, 
                   "--> DiCDcb: WAIT for EP=%lx req_count(%d) != 0\n", 
-                  ep_ptr, dapl_os_atomic_read(&ep_ptr->req_count));
+                  ep_ptr, dapls_cb_pending(&ep_ptr->req_buffer));
     	dapl_os_sleep_usec (100);
     }
 
@@ -195,6 +227,7 @@ dapli_ib_cm_dreq_cb (
 
 /*
  * Connection Disconnect Reply callback
+ * We sent a DREQ and received a DREP.
  */
 
 static void 
@@ -207,16 +240,22 @@ dapli_ib_cm_drep_cb (
 
     ep_ptr  = (DAPL_EP * __ptr64) p_cm_drep_rec->qp_context;
 
-    if ( !ep_ptr || DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP) )
+    if (p_cm_drep_rec->cm_status)
     {
          dapl_dbg_log (DAPL_DBG_TYPE_CM,
+                  "--> %s: DREP cm_status(%s) EP=%p\n", __FUNCTION__,
+                  ib_get_err_str(p_cm_drep_rec->cm_status), ep_ptr); 
+    }
+
+    if ( DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP) )
+    {
+         dapl_dbg_log (DAPL_DBG_TYPE_ERR,
                   "--> %s: BAD EP Handle EP=%lx\n", __FUNCTION__,ep_ptr); 
         return;
     }
     
     dapl_dbg_log (DAPL_DBG_TYPE_CM, 
-                  "--> DiCDpcb: QP %lx EP %lx state %s cm_hdl %lx\n",
-                  ep_ptr->qp_handle, ep_ptr,
+	"--> DiCDpcb: EP %p state %s cm_hdl %p\n",ep_ptr,
                   dapl_get_ep_state_str(ep_ptr->param.ep_state),
                   ep_ptr->cm_handle);
 
@@ -254,6 +293,9 @@ dapli_ib_cm_drep_cb (
     }
 }
 
+/*
+ * CM reply callback
+ */
 
 static void 
 dapli_ib_cm_rep_cb (
@@ -268,12 +310,14 @@ dapli_ib_cm_rep_cb (
         
     dapl_os_assert (p_cm_rep_rec != NULL);
 
-    dapl_os_memzero (&cm_rtu, sizeof ( ib_cm_rtu_t ));
-
-    dapl_os_assert ( ((DAPL_HEADER * __ptr64)
-                     p_cm_rep_rec->qp_context)->magic == DAPL_MAGIC_EP );
-                         
     ep_ptr  = (DAPL_EP * __ptr64) p_cm_rep_rec->qp_context;
+
+    if ( DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP) )
+    {
+        dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: EP %lx invalid or FREED\n",
+                      __FUNCTION__, ep_ptr);
+        return;
+    }
     dapl_dbg_log (DAPL_DBG_TYPE_CM, 
                   "--> DiCRpcb: EP = %lx local_max_rdma_read_in %d\n", 
                   ep_ptr, p_cm_rep_rec->resp_res);
@@ -281,6 +325,7 @@ dapli_ib_cm_rep_cb (
     p_ca   = (dapl_ibal_ca_t *) 
              ep_ptr->header.owner_ia->hca_ptr->ib_hca_handle;
 
+    dapl_os_memzero (&cm_rtu, sizeof ( ib_cm_rtu_t ));
     cm_rtu.pfn_cm_apr_cb  = dapli_ib_cm_apr_cb;
     cm_rtu.pfn_cm_dreq_cb = dapli_ib_cm_dreq_cb;
     cm_rtu.p_rtu_pdata    = NULL;
@@ -302,7 +347,7 @@ dapli_ib_cm_rep_cb (
         cm_cb_op = IB_CME_CONNECTED;
         dapl_dbg_log (DAPL_DBG_TYPE_CM, 
                   "--> DiCRpcb: EP %lx Connected req_count %d\n", 
-                  ep_ptr, dapl_os_atomic_read(&ep_ptr->req_count));
+                  ep_ptr, dapls_cb_pending(&ep_ptr->req_buffer));
     }
     else
     {
@@ -311,23 +356,10 @@ dapli_ib_cm_rep_cb (
 
     prd_ptr = (DAPL_PRIVATE * __ptr64) p_cm_rep_rec->p_rep_pdata;
 
-#ifdef DAPL_DBG
-#if 0
-    {
-        int i;
-            
-        dapl_dbg_log ( DAPL_DBG_TYPE_EP, "--> DiCRpcb: private_data: ");
-
-        for ( i = 0 ; i < IB_MAX_REP_PDATA_SIZE ; i++ )
-        {
-            dapl_dbg_log ( DAPL_DBG_TYPE_EP, 
-                                "0x%x ", prd_ptr->private_data[i]);
-                                 
-        }
-        dapl_dbg_log ( DAPL_DBG_TYPE_EP, "\n");
-                            
-    }
-#endif
+#if defined(DAPL_DBG) && 0
+    dapli_print_private_data( "DiCRpcb",
+			      prd_ptr->private_data,
+			      IB_MAX_REP_PDATA_SIZE);
 #endif
 
     dapl_evd_connection_callback ( 
@@ -349,6 +381,13 @@ dapli_ib_cm_rej_cb (
 
     ep_ptr = (DAPL_EP * __ptr64) p_cm_rej_rec->qp_context;
 
+    if ( DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP) )
+    {
+        dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: EP %lx invalid or FREED\n",
+                      __FUNCTION__, ep_ptr);
+        return;
+    }
+
     dapl_dbg_log (DAPL_DBG_TYPE_CM, 
                   "--> DiCRjcb: EP = %lx QP = %lx rej reason = 0x%x\n", 
                   ep_ptr,ep_ptr->qp_handle,CL_NTOH16(p_cm_rej_rec->rej_status));
@@ -589,6 +628,13 @@ dapli_ib_cm_rtu_cb (
    
     ep_ptr = (DAPL_EP * __ptr64) p_cm_rtu_rec->qp_context;
 
+    if ( DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP) )
+    {
+        dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> %s: EP %lx invalid or FREED\n",
+                      __FUNCTION__, ep_ptr);
+        return;
+    }
+
     dapl_dbg_log (DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK, 
                   "--> DiCRucb: EP %lx QP %lx\n", ep_ptr, ep_ptr->qp_handle); 
 
@@ -708,8 +754,8 @@ dapls_ib_connect (
         IN        DAT_EP_HANDLE                ep_handle,
         IN        DAT_IA_ADDRESS_PTR           remote_ia_address,
         IN        DAT_CONN_QUAL                remote_conn_qual,
-        IN        DAT_COUNT                    prd_size,
-        IN        DAPL_PRIVATE                 *prd_ptr )
+        IN        DAT_COUNT                    private_data_size,
+        IN        DAT_PVOID                    private_data )
 {
     DAPL_EP                      *ep_ptr;
     DAPL_IA                      *ia_ptr;
@@ -874,8 +920,9 @@ dapls_ib_connect (
     cm_req.p_alt_path       = NULL;
     cm_req.h_qp             = ep_ptr->qp_handle;
     cm_req.qp_type          = IB_QPT_RELIABLE_CONN;
-    cm_req.p_req_pdata      = (uint8_t *) prd_ptr;
-    cm_req.req_length       = (uint8_t)prd_size;
+    cm_req.p_req_pdata      = (uint8_t *) private_data;
+    cm_req.req_length       = (uint8_t)
+				min(private_data_size,IB_MAX_REQ_PDATA_SIZE);
     /* cm retry to send this request messages, IB max of 4 bits */
     cm_req.max_cm_retries   = 15; /* timer outside of call, s/be infinite */
     /* qp retry to send any wr */
@@ -886,13 +933,13 @@ dapls_ib_connect (
     cm_req.init_depth       = (uint8_t)ep_ptr->param.ep_attr.max_rdma_read_out;
 
     /* time wait before retrying a pkt after receiving a RNR NAK */
-    cm_req.rnr_nak_timeout  = 12; /* 163.84ms */
+    cm_req.rnr_nak_timeout  = IB_RNR_NAK_TIMEOUT;
     
 	/* 
      * number of time local QP should retry after receiving RNR NACK before
      * reporting an error
      */
-    cm_req.rnr_retry_cnt       = 6;	/* 7 is infinite */
+    cm_req.rnr_retry_cnt       = IB_RNR_RETRY_CNT;
 
     cm_req.remote_resp_timeout = 16;	/* 250ms */
     cm_req.local_resp_timeout  = 16;	/* 250ms */
@@ -962,13 +1009,11 @@ DAT_RETURN
 dapls_ib_disconnect ( IN   DAPL_EP           *ep_ptr,
                       IN   DAT_CLOSE_FLAGS   disconnect_flags )
 {
-    DAPL_IA                        *ia_ptr;
-    ib_api_status_t                ib_status;
+    ib_api_status_t	ib_status = IB_SUCCESS;
     ib_cm_dreq_t                   cm_dreq;
 
-    //UNUSED_PARAM( disconnect_flags );
-
     dapl_os_assert(ep_ptr);
+
     if ( DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP) )
     {
          dapl_dbg_log (DAPL_DBG_TYPE_CM,
@@ -982,27 +1027,25 @@ dapls_ib_disconnect ( IN   DAPL_EP           *ep_ptr,
         return DAT_SUCCESS;
     }
 
-    dapl_dbg_log (DAPL_DBG_TYPE_CM, "--> %s() DsD: EP %lx QP %lx ep_state %s "
-                  "rx_drq %d tx_drq %d Close %s\n", __FUNCTION__,
-                  ep_ptr, ep_ptr->qp_handle, 
-                  dapl_get_ep_state_str (ep_ptr->param.ep_state),
+    dapl_dbg_log (DAPL_DBG_TYPE_CM,
+	"--> %s() EP %p %s rx_drq %d tx_drq %d Close %s\n", __FUNCTION__,
+	ep_ptr, dapl_get_ep_state_str(ep_ptr->param.ep_state),
                   ep_ptr->recv_discreq, ep_ptr->sent_discreq,
-		  (disconnect_flags == DAT_CLOSE_ABRUPT_FLAG
-                     ? "Abrupt":"Graceful"));
+	(disconnect_flags == DAT_CLOSE_ABRUPT_FLAG ? "Abrupt":"Graceful"));
 
-    if ( disconnect_flags == DAT_CLOSE_ABRUPT_FLAG ) {
-        dapl_ep_legacy_post_disconnect(ep_ptr, disconnect_flags);
+    if ( disconnect_flags == DAT_CLOSE_ABRUPT_FLAG )
+    {
+	if ( ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED )
         return DAT_SUCCESS;
-    }
 
-    if ( ep_ptr->param.ep_state != DAT_EP_STATE_CONNECTED )
+	if ( ep_ptr->param.ep_state != DAT_EP_STATE_DISCONNECT_PENDING )
     {
-        dapl_dbg_log (DAPL_DBG_TYPE_CM,
-                      "--> DsD: EP %lx NOT connected state %s\n",
-                      ep_ptr, dapl_get_ep_state_str (ep_ptr->param.ep_state));
+	    dapl_dbg_log(DAPL_DBG_TYPE_CM,
+			"%s() calling legacy_post_disconnect()\n",__FUNCTION__);
+            dapl_ep_legacy_post_disconnect(ep_ptr, disconnect_flags);
+            return DAT_SUCCESS;
+	}
     }
-    ia_ptr    = ep_ptr->header.owner_ia;
-    ib_status = IB_SUCCESS;
 
     dapl_os_memzero(&cm_dreq, sizeof(ib_cm_dreq_t));
 
@@ -1017,19 +1060,32 @@ dapls_ib_disconnect ( IN   DAPL_EP           *ep_ptr,
     cm_dreq.p_dreq_pdata   = NULL;
     cm_dreq.flags          = IB_FLAGS_SYNC;
 
+    /*
+     * still need to send DREQ (disconnect request)?
+     */
     if ( (ep_ptr->recv_discreq == DAT_FALSE)
-               && (ep_ptr->sent_discreq == DAT_FALSE) )
+          && (ep_ptr->sent_discreq == DAT_FALSE)
+	  && (ep_ptr->qp_state != IB_QPS_RESET) )
     {
         ep_ptr->sent_discreq = DAT_TRUE;
-
         ib_status = ib_cm_dreq ( &cm_dreq );
+	/* tolerate INVALID_STATE error as the other side can race ahead and
+	 * generate a DREQ before we do.
+	 */
+	if ( ib_status == IB_INVALID_STATE )
+	    ib_status = IB_SUCCESS;
 
+	if (ib_status)
+	{
+	    dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+			"%s() EP %p ib_cm_dreq() status %s\n",
+			__FUNCTION__,ep_ptr,ib_get_err_str(ib_status));
+	}
+	if ( ib_status == IB_SUCCESS )
         dapl_dbg_log (DAPL_DBG_TYPE_CM,
-                      "--> DsD: EP %lx  QP %lx  DREQ SENT status %s\n", 
-                      ep_ptr, ep_ptr->qp_handle,ib_get_err_str(ib_status));
+				"--> DsD: EP %p  DREQ SENT\n", ep_ptr);
     }
-
-    return dapl_ib_status_convert (ib_status);
+    return ib_status;
 }
 
 
@@ -1127,7 +1183,6 @@ dapls_ib_setup_conn_listener (
 
     ib_status = ib_cm_listen ( dapl_ibal_root.h_al,
                                &cm_listen,
-                               dapli_ibal_listen_err_cb,
                                (void *) sp_ptr,
                                &sp_ptr->cm_srvc_handle );
 
@@ -1256,8 +1311,22 @@ dapls_ib_reject_connection ( IN  dp_ib_cm_handle_t   ib_cm_handle,
     cm_rej.rej_status   = IB_REJ_USER_DEFINED;
     cm_rej.p_ari        = (ib_ari_t *)&rej_table[reject_reason]; 
     cm_rej.ari_length   = (uint8_t)strlen (rej_table[reject_reason]);
-    cm_rej.p_rej_pdata  = NULL;
-    cm_rej.rej_length   = 0;
+
+    if (private_data_size > 
+	dapls_ib_private_data_size(NULL,DAPL_PDATA_CONN_REJ,NULL))
+    {
+        dapl_dbg_log ( DAPL_DBG_TYPE_ERR,
+                       "--> DsRjC: private_data size(%d) > Max(%d)\n", 
+                       private_data_size, IB_MAX_REJ_PDATA_SIZE );
+	return DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG3);
+    }
+
+    cm_rej.p_rej_pdata  = private_data;
+    cm_rej.rej_length   = private_data_size;
+
+#if defined(DAPL_DBG) && 0
+    dapli_print_private_data("DsRjC",private_data,private_data_size);
+#endif
 
     ib_status = ib_cm_rej ( *ib_cm_handle, &cm_rej);
 
@@ -1269,7 +1338,6 @@ dapls_ib_reject_connection ( IN  dp_ib_cm_handle_t   ib_cm_handle,
     }
 
     return ( dapl_ib_status_convert ( ib_status ) );
-
 }
 
 
@@ -1286,7 +1354,6 @@ dapli_query_qp( ib_qp_handle_t qp_handle, ib_qp_attr_t  *qpa )
         dapl_dbg_log ( DAPL_DBG_TYPE_ERR,"ib_query_qp(%lx) '%s'\n",
                 qp_handle, ib_get_err_str(ib_status) );
     }
-#if 1
     else
     {
         dapl_dbg_log ( DAPL_DBG_TYPE_CM, "--> QP(%lx) state %s "
@@ -1297,7 +1364,6 @@ dapli_query_qp( ib_qp_handle_t qp_handle, ib_qp_attr_t  *qpa )
                        qpa->init_depth,
                        qpa->access_ctrl );
     }
-#endif
 }
 #endif
 
@@ -1310,8 +1376,8 @@ dapli_query_qp( ib_qp_handle_t qp_handle, ib_qp_attr_t  *qpa )
  * Input:
  *        cr_handle
  *        ep_handle
- *        private_data_size - ignored as DAT layer sets 0
- *        private_data      - ignored as DAT layer sets NULL
+ *        private_data_size
+ *        private_data
  *
  * Output:
  *         none
@@ -1326,8 +1392,8 @@ DAT_RETURN
 dapls_ib_accept_connection (
         IN        DAT_CR_HANDLE            cr_handle,
         IN        DAT_EP_HANDLE            ep_handle,
-        IN        DAT_COUNT                p_size,
-        IN        DAPL_PRIVATE             *prd_ptr )
+        IN        DAT_COUNT                private_data_size,
+        IN const  DAT_PVOID                private_data )
 {
     DAPL_CR                *cr_ptr;
     DAPL_EP                *ep_ptr;
@@ -1414,24 +1480,21 @@ dapls_ib_accept_connection (
 
     cm_rep.h_qp           = ep_ptr->qp_handle;
     cm_rep.qp_type        = IB_QPT_RELIABLE_CONN;
-    cm_rep.p_rep_pdata    = (uint8_t *) cr_ptr->private_data;
-    cm_rep.rep_length     = IB_REQ_PDATA_SIZE;
-
-#if defined(DAPL_DBG) && 0
-    {
-        int i;
-            
-        dapl_dbg_log ( DAPL_DBG_TYPE_EP, "--> DsAC: private_data: ");
 
-        for ( i = 0 ; i < IB_MAX_REP_PDATA_SIZE ; i++ )
-        {
-            dapl_dbg_log ( DAPL_DBG_TYPE_EP, 
-                                "0x%x ", prd_ptr->private_data[i]);
+    if (private_data_size > IB_MAX_REP_PDATA_SIZE) {
+    	dapl_dbg_log (DAPL_DBG_TYPE_ERR,
+			"--> DsIBAC: private_data_size(%d) > Max(%d)\n",
+			private_data_size, IB_MAX_REP_PDATA_SIZE);
+	return DAT_ERROR(DAT_LENGTH_ERROR, DAT_NO_SUBTYPE);
                                  
         }
-        dapl_dbg_log ( DAPL_DBG_TYPE_EP, "\n");
+    cm_rep.p_rep_pdata    = (const uint8_t *)private_data;
+    cm_rep.rep_length     = private_data_size;
                             
-    }
+#if defined(DAPL_DBG) && 0
+    dapli_print_private_data( "DsIBAC",
+			      (const uint8_t*)private_data,
+ 			      private_data_size );
 #endif
 
     cm_rep.pfn_cm_rej_cb = dapli_ib_cm_rej_cb;
@@ -1467,15 +1530,15 @@ dapls_ib_accept_connection (
     cm_rep.flags             = 0;
     cm_rep.failover_accepted = IB_FAILOVER_ACCEPT_UNSUPPORTED;
     cm_rep.target_ack_delay  = 14;
-    cm_rep.rnr_nak_timeout   = 12;
-    cm_rep.rnr_retry_cnt     = 6;
+    cm_rep.rnr_nak_timeout   = IB_RNR_NAK_TIMEOUT;
+    cm_rep.rnr_retry_cnt     = IB_RNR_RETRY_CNT;
     cm_rep.pp_recv_failure   = NULL;
     cm_rep.p_recv_wr         = NULL;
      
     dapl_dbg_log (DAPL_DBG_TYPE_CM,
                  "--> DsIBAC: cm_rep: acc %x init %d qp_type %x req_count %d\n",
 	         cm_rep.access_ctrl, cm_rep.init_depth,cm_rep.qp_type,
-                 dapl_os_atomic_read(&ep_ptr->req_count));
+                 dapls_cb_pending(&ep_ptr->req_buffer));
 
     ib_status = ib_cm_rep ( *ep_ptr->cm_handle, &cm_rep );
 
@@ -1681,9 +1744,9 @@ dapls_ib_cr_handoff (
  * Return the size of private data given a connection op type
  *
  * Input:
- *	hca_ptr		hca pointer, needed for transport type
  *	prd_ptr		private data pointer
  *	conn_op		connection operation type
+ *	hca_ptr		hca pointer, needed for transport type
  *
  * If prd_ptr is NULL, this is a query for the max size supported by
  * the provider, otherwise it is the actual size of the private data
@@ -1700,13 +1763,14 @@ dapls_ib_cr_handoff (
  */
 int
 dapls_ib_private_data_size (
-	IN	DAPL_HCA		*hca_ptr,
 	IN      DAPL_PRIVATE		*prd_ptr,
-	IN	DAPL_PDATA_OP		conn_op)
+	IN	DAPL_PDATA_OP		conn_op,
+	IN	DAPL_HCA		*hca_ptr)
 {
     int  size;
 
     UNUSED_PARAM( prd_ptr );
+    UNUSED_PARAM( hca_ptr );
 
     switch (conn_op)
     {
diff --git a/dapl/ibal/dapl_ibal_cq.c b/dapl/ibal/dapl_ibal_cq.c
index 2ac5814..28de045 100644
--- a/dapl/ibal/dapl_ibal_cq.c
+++ b/dapl/ibal/dapl_ibal_cq.c
@@ -82,7 +82,9 @@ dapli_ibal_cq_async_error_callback ( IN  ib_async_event_rec_t  *p_err_rec )
 
     /* maps to dapl_evd_cq_async_error_callback(), context is EVD */
     evd_cb->pfn_async_cq_err_cb( (ib_hca_handle_t)p_ca, 
-				(ib_error_record_t*)&p_err_rec->code, evd_ptr);
+				 evd_ptr->ib_cq_handle,
+				 (ib_error_record_t*)&p_err_rec->code,
+				 evd_ptr );
 
 }
 
diff --git a/dapl/ibal/dapl_ibal_dto.h b/dapl/ibal/dapl_ibal_dto.h
index 283fd91..4694072 100644
--- a/dapl/ibal/dapl_ibal_dto.h
+++ b/dapl/ibal/dapl_ibal_dto.h
@@ -146,19 +146,16 @@ dapls_ib_post_recv (
     {
 	return DAT_SUCCESS;
     }
-    else
-    {
+
         dapl_dbg_log (DAPL_DBG_TYPE_EP, "--> DsPR: post_recv status %s\n", 
                       ib_get_err_str(ib_status));
-
         /*
          * Moving QP to error state; 
          */
-        ib_status = dapls_modify_qp_state_to_error ( ep_ptr->qp_handle);
+    (void) dapls_modify_qp_state_to_error ( ep_ptr->qp_handle);
         ep_ptr->qp_state = IB_QPS_ERROR;
 
 	return (dapl_ib_status_convert (ib_status));
-    }
 }
 
 
@@ -185,6 +182,7 @@ dapls_ib_post_send (
     if (ep_ptr->param.ep_state != DAT_EP_STATE_CONNECTED)
     {
     	ib_qp_attr_t             qp_attr;
+
     	ib_query_qp ( ep_ptr->qp_handle, &qp_attr );
 
     	dapl_dbg_log (DAPL_DBG_TYPE_ERR, "--> DsPS: !CONN EP (%p) ep_state=%d "
@@ -274,7 +272,7 @@ dapls_ib_post_send (
     /*
      * Moving QP to error state; 
      */
-    ib_status = dapls_modify_qp_state_to_error ( ep_ptr->qp_handle);
+    (void) dapls_modify_qp_state_to_error ( ep_ptr->qp_handle);
     ep_ptr->qp_state = IB_QPS_ERROR;
 
     return (dapl_ib_status_convert (ib_status));
diff --git a/dapl/ibal/dapl_ibal_extensions.c b/dapl/ibal/dapl_ibal_extensions.c
index 48c0dfe..b05c0bb 100644
--- a/dapl/ibal/dapl_ibal_extensions.c
+++ b/dapl/ibal/dapl_ibal_extensions.c
@@ -194,10 +194,7 @@ dapli_post_ext( IN       DAT_EP_HANDLE		ep_handle,
 	/*
 	 * Synchronization ok since this buffer is only used for send
 	 * requests, which aren't allowed to race with each other.
-	 * only if completion is expected
 	 */
-	if (!(DAT_COMPLETION_SUPPRESS_FLAG & flags)) {
-
 		dat_status = dapls_dto_cookie_alloc(
 						&ep_ptr->req_buffer,
 						DAPL_DTO_TYPE_EXTENSION,
@@ -207,21 +204,13 @@ dapli_post_ext( IN       DAT_EP_HANDLE		ep_handle,
 		if ( dat_status != DAT_SUCCESS )
 		{
 #ifdef DAPL_DBG
-			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
-					"%s() cookie alloc faulure %x\n",
+		dapl_dbg_log(DAPL_DBG_TYPE_ERR,"%s() cookie alloc faulure %x\n",
 					__FUNCTION__,dat_status);
 #endif
 			goto bail;
 		}
 		
 		/*
-		 * Take reference before posting to avoid race conditions with
-		 * completions
-		 */
-		dapl_os_atomic_inc(&ep_ptr->req_count);
-	}
-
-	/*
 	 * Invoke provider specific routine to post DTO
 	 */
 	dat_status = dapls_ib_post_ext_send(ep_ptr,	
@@ -237,9 +226,6 @@ dapli_post_ext( IN       DAT_EP_HANDLE		ep_handle,
 
 	if (dat_status != DAT_SUCCESS)
 	{
-		if ( cookie != NULL )
-		{
-			dapl_os_atomic_dec(&ep_ptr->req_count);
 			dapls_cookie_dealloc(&ep_ptr->req_buffer, cookie);
 #ifdef DAPL_DBG
 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
@@ -247,7 +233,6 @@ dapli_post_ext( IN       DAT_EP_HANDLE		ep_handle,
 					__FUNCTION__,dat_status,__LINE__);
 #endif
 		}
-	}
 
 bail:
 	return dat_status;
diff --git a/dapl/ibal/dapl_ibal_name_service.h b/dapl/ibal/dapl_ibal_name_service.h
index e55eead..d322d71 100644
--- a/dapl/ibal/dapl_ibal_name_service.h
+++ b/dapl/ibal/dapl_ibal_name_service.h
@@ -58,8 +58,6 @@ dapli_ib_sa_query_cb (
         IN    ib_query_rec_t       *p_query_rec );
 
 
-//DAT_RETURN dapls_ns_init (void);
-
 #ifdef NO_NAME_SERVICE
 
 DAT_RETURN dapls_ns_lookup_address (
diff --git a/dapl/ibal/dapl_ibal_qp.c b/dapl/ibal/dapl_ibal_qp.c
index 9f26c82..cc8c394 100644
--- a/dapl/ibal/dapl_ibal_qp.c
+++ b/dapl/ibal/dapl_ibal_qp.c
@@ -81,7 +81,7 @@ dapli_ib_qp_async_error_cb( IN  ib_async_event_rec_t* p_err_rec )
 	if ((evd_cb == NULL) || (evd_cb->pfn_async_qp_err_cb == NULL))
 	{
 		dapl_dbg_log (DAPL_DBG_TYPE_ERR,
-			"--> DiQpAEC: no ERROR cb on %p found \n", p_ca);
+			"--> DiQpAEC: no ERROR cb on p_ca %p found\n", p_ca);
 		return;
 	}
 
@@ -94,8 +94,9 @@ dapli_ib_qp_async_error_cb( IN  ib_async_event_rec_t* p_err_rec )
 	
 	/* maps to dapl_evd_qp_async_error_callback(), context is EP */
 	evd_cb->pfn_async_qp_err_cb(	(ib_hca_handle_t)p_ca, 
+				     ep_ptr->qp_handle,
 					(ib_error_record_t*)&p_err_rec->code,
-					ep_ptr);
+				     ep_ptr );
 }
 
 /*
@@ -556,9 +557,9 @@ dapls_modify_qp_state_to_rtr (
     qp_mod.state.rtr.primary_av.conn.path_mtu = p_port->p_attr->mtu;
     qp_mod.state.rtr.primary_av.conn.local_ack_timeout = 7;
     qp_mod.state.rtr.primary_av.conn.seq_err_retry_cnt = 7;
-    qp_mod.state.rtr.primary_av.conn.rnr_retry_cnt     = 7;
+    qp_mod.state.rtr.primary_av.conn.rnr_retry_cnt = IB_RNR_RETRY_CNT;
     qp_mod.state.rtr.resp_res               = 4; // in-flight RDMAs
-    qp_mod.state.rtr.rnr_nak_timeout        = 7;
+    qp_mod.state.rtr.rnr_nak_timeout        = IB_RNR_NAK_TIMEOUT;
  
     ib_status = ib_modify_qp (qp_handle, &qp_mod);
     
@@ -579,7 +580,8 @@ dapls_modify_qp_state_to_rts ( ib_qp_handle_t  qp_handle )
     qp_mod.req_state                   = IB_QPS_RTS;
     qp_mod.state.rts.sq_psn            = DAPL_IBAL_START_PSN;
     qp_mod.state.rts.retry_cnt         = 7;
-    qp_mod.state.rts.rnr_retry_cnt     = 6;
+    qp_mod.state.rts.rnr_retry_cnt     = IB_RNR_RETRY_CNT;
+    qp_mod.state.rtr.rnr_nak_timeout   = IB_RNR_NAK_TIMEOUT;
     qp_mod.state.rts.local_ack_timeout = 7;
     qp_mod.state.rts.init_depth        = 4; 
 
diff --git a/dapl/ibal/dapl_ibal_util.c b/dapl/ibal/dapl_ibal_util.c
index fe9e3b8..ad4acf0 100644
--- a/dapl/ibal/dapl_ibal_util.c
+++ b/dapl/ibal/dapl_ibal_util.c
@@ -29,7 +29,7 @@
 #include "dapl_ring_buffer_util.h"
 
 #ifdef DAT_EXTENSIONS
-#include <dat\dat_ib_extensions.h>
+#include <dat2\dat_ib_extensions.h>
 #endif
 
 #ifndef NO_NAME_SERVICE
@@ -1542,6 +1542,63 @@ dapls_ib_setup_async_callback (
 
 
 /*
+ * dapls_ib_query_gid
+ *
+ * Query the hca for the gid of the 1st active port.
+ *
+ * Input:
+ *	hca_handl		hca handle	
+ *	ep_attr			attribute of the ep
+ *
+ * Output:
+ * 	none
+ *
+ * Returns:
+ * 	DAT_SUCCESS
+ *	DAT_INVALID_HANDLE
+ *	DAT_INVALID_PARAMETER
+ */
+
+DAT_RETURN
+dapls_ib_query_gid( IN  DAPL_HCA	*hca_ptr,
+		    IN  GID		*gid )
+{
+    dapl_ibal_ca_t    *p_ca;
+    ib_ca_attr_t      *p_hca_attr;
+    ib_api_status_t   ib_status;
+    ib_hca_port_t     port_num;
+
+    p_ca = (dapl_ibal_ca_t *) hca_ptr->ib_hca_handle;
+
+    if (p_ca == NULL)
+    {
+        dapl_dbg_log ( DAPL_DBG_TYPE_ERR,
+			"%s() invalid hca_ptr %p", __FUNCTION__, hca_ptr);
+        return DAT_INVALID_HANDLE;
+    }
+
+    ib_status = ib_query_ca (
+                          p_ca->h_ca,
+                          p_ca->p_ca_attr,
+                          &p_ca->ca_attr_size);
+    if (ib_status != IB_SUCCESS)
+    {
+        dapl_dbg_log ( DAPL_DBG_TYPE_ERR,
+                       "%s() ib_query_ca returned failed status = %s\n", 
+                       ib_get_err_str(ib_status));
+        return dapl_ib_status_convert (ib_status);
+    }
+
+    p_hca_attr = p_ca->p_ca_attr;
+    port_num = hca_ptr->port_num - 1;
+
+    gid->gid_prefix = p_hca_attr->p_port_attr[port_num].p_gid_table->unicast.prefix;
+    gid->guid = p_hca_attr->p_port_attr[port_num].p_gid_table->unicast.interface_id;
+    return DAT_SUCCESS;
+}
+
+
+/*
  * dapls_ib_query_hca
  *
  * Query the hca attribute
@@ -2201,7 +2258,9 @@ static DAT_NAMED_ATTR	*ib_attrs = NULL;
 #define SPEC_ATTR_SIZE( x )	0
 #endif
 
-void dapls_query_provider_specific_attr( IN  DAT_PROVIDER_ATTR   *attr_ptr )
+void dapls_query_provider_specific_attr(
+	IN	DAPL_IA				*ia_ptr,
+	IN	DAT_PROVIDER_ATTR	*attr_ptr )
 {
     attr_ptr->num_provider_specific_attr = SPEC_ATTR_SIZE(ib_attrs);
     attr_ptr->provider_specific_attr     = ib_attrs;
diff --git a/dapl/ibal/dapl_ibal_util.h b/dapl/ibal/dapl_ibal_util.h
index 033bc8d..52dd879 100644
--- a/dapl/ibal/dapl_ibal_util.h
+++ b/dapl/ibal/dapl_ibal_util.h
@@ -29,7 +29,7 @@
 #include <complib/cl_atomic.h>
 
 #ifdef DAT_EXTENSIONS
-#include <dat\dat_ib_extensions.h>
+#include <dat2\dat_ib_extensions.h>
 #endif
 
 /*
@@ -62,6 +62,18 @@ typedef void (*ib_async_handler_t)(
     IN    ib_error_record_t  *err_code,
     IN    void               *context);
 
+typedef void (*ib_async_qp_handler_t)(
+    IN    ib_hca_handle_t    ib_hca_handle,
+    IN    ib_qp_handle_t     ib_qp_handle,
+    IN    ib_error_record_t  *err_code,
+    IN    void               *context);
+
+typedef void (*ib_async_cq_handler_t)(
+    IN    ib_hca_handle_t    ib_hca_handle,
+    IN    ib_cq_handle_t     ib_cq_handle,
+    IN    ib_error_record_t  *err_code,
+    IN    void               *context);
+
 typedef ib_net64_t   ib_guid_t;
 typedef ib_net16_t   ib_lid_t;
 typedef boolean_t    ib_bool_t;
@@ -112,6 +124,34 @@ typedef struct _ib_hca_name
 #define          IB_MAX_DREQ_PDATA_SIZE     220
 #define          IB_MAX_DREP_PDATA_SIZE     224
 
+
+/* Resource Not Ready
+	1-6 is an actual retry count which is decremented to zero before
+        an error condition is set.
+    7 is 'magic' in that it implies Infinite retry, just keeps trying.
+*/
+#define		IB_RNR_RETRY_CNT   7
+
+/*
+IB 1.2 spec, page 331, table 45, RNR NAK timeout encoding (5-bits)
+ 
+00000=655.36ms(milliseconds)
+00001=0.01ms
+00010=0.02ms
+00011=0.03ms
+00100=0.04ms
+00101=0.06ms
+00110=0.08ms
+00111=0.12ms
+
+11100=163.84ms 28d
+11101=245.76ms 29d
+11110=327.68ms 30d
+11111=491.52ms 31d
+*/
+#define		IB_RNR_NAK_TIMEOUT   0
+
+
 typedef void
 (*dapl_ibal_pfn_destructor_t)(
     IN    void*    context );
@@ -164,8 +204,8 @@ typedef struct _dapl_ibal_evd_cb
 {
     cl_list_item_t     next;        // peer CA list
     ib_async_handler_t pfn_async_err_cb;
-    ib_async_handler_t pfn_async_qp_err_cb;
-    ib_async_handler_t pfn_async_cq_err_cb;
+    ib_async_qp_handler_t  pfn_async_qp_err_cb;
+    ib_async_cq_handler_t  pfn_async_cq_err_cb;
     void               *context;
 } dapl_ibal_evd_cb_t;
 
@@ -370,6 +410,7 @@ struct ib_llist_entry
 };
 
 #ifdef SOCK_CM
+
 typedef enum
 {
 	IB_THREAD_INIT,
@@ -378,7 +419,22 @@ typedef enum
 	IB_THREAD_EXIT
 
 } ib_thread_state_t;
-#endif
+
+typedef enum scm_state 
+{
+	SCM_INIT,
+	SCM_LISTEN,
+	SCM_CONN_PENDING,
+	SCM_ACCEPTING,
+	SCM_ACCEPTED,
+	SCM_REJECTED,
+	SCM_CONNECTED,
+	SCM_DISCONNECTED,
+	SCM_DESTROY
+
+} SCM_STATE;
+
+#endif	/* SOCK_CM */
 
 typedef struct _ib_hca_transport
 { 
@@ -409,29 +465,40 @@ typedef uint32_t ib_shm_transport_t;
 
 /* CM mappings use SOCKETS */
 
-/* destination info to exchange until real IB CM shows up */
+/* destination info exchanged between dapl, define wire protocol version */
+#define DSCM_VER 2
+
 typedef struct _ib_qp_cm
 { 
-	ib_net32_t		qpn;
+	ib_net16_t	ver;
+	ib_net16_t	rej;
 	ib_net16_t		lid;
 	ib_net16_t		port;
+	ib_net32_t	qpn;
 	ib_net32_t		p_size;
 	DAT_SOCK_ADDR6		ia_address;
+	GID		gid;
 
 } ib_qp_cm_t;
 
 struct ib_cm_handle
 { 
 	struct ib_llist_entry	entry;
+	DAPL_OS_LOCK		lock;
+	SCM_STATE		state;
 	int			socket;
 	int			l_socket; 
-	struct dapl_hca		*hca_ptr;
-	DAT_HANDLE		cr;
+	struct dapl_hca		*hca;
 	DAT_HANDLE		sp;	
+	DAT_HANDLE		cr;
+	struct dapl_ep		*ep;
 	ib_qp_cm_t		dst;
 	unsigned char		p_data[256];
 };
-#endif
+
+DAT_RETURN dapli_init_sock_cm ( IN DAPL_HCA  *hca_ptr );
+
+#endif	/* SOCK_CM */
 
 /*
  * Prototype






More information about the general mailing list