[ofa-general] [PATCH 1/7][v2.0] dapl: endpoint pending request count is wrong

Davis, Arlin R arlin.r.davis at intel.com
Fri Jun 20 11:54:41 PDT 2008


The code assumes every cookie allocated during posting of
requests gets completed. This incorrect assumption results in
wrong pending count. Remove request_pending field and replace
with direct call, dapl_cb_pending, to provide accurate
data to consumer.

Add debug print if consumer overruns request queue.

Signed-off by: Arlin Davis ardavis at ichips.intel.com
---
 dapl/common/dapl_cookie.c        |   28 ++++++++++++++++++++++++++++
 dapl/common/dapl_cookie.h        |    4 ++++
 dapl/common/dapl_ep_get_status.c |    4 ++--
 dapl/common/dapl_ep_post_recv.c  |    7 -------
 dapl/common/dapl_ep_util.c       |   22 ++++++++++------------
 dapl/common/dapl_evd_util.c      |   16 ++++------------
 dapl/common/dapl_rmr_bind.c      |   14 --------------
 7 files changed, 48 insertions(+), 47 deletions(-)

diff --git a/dapl/common/dapl_cookie.c b/dapl/common/dapl_cookie.c
index 7998d7f..1462fe9 100644
--- a/dapl/common/dapl_cookie.c
+++ b/dapl/common/dapl_cookie.c
@@ -260,6 +260,34 @@ dapls_cb_put (
     return DAT_SUCCESS;
 }
 
+/*
+ * dapls_cb_pending
+ *
+ * snapshot of active entries on cookie ring buffer 
+ *
+ * Input:
+ *	buffer		pointer to DAPL_COOKIE_BUFFER
+ *
+ * Returns:
+ *	DAT_COUNT	number of active/pending cookies
+ *
+ */
+extern DAT_COUNT 
+dapls_cb_pending (
+    DAPL_COOKIE_BUFFER	*buffer )
+{
+	DAT_COUNT head, tail;
+
+	head = dapl_os_atomic_read(&buffer->head);
+	tail = dapl_os_atomic_read(&buffer->tail);
+
+	if (head == tail)
+		return 0;
+	else if (head > tail)
+		return (head - tail);
+	else 
+		return ((buffer->pool_size - tail) + head);
+}
 
 /*
  * dapls_rmr_cookie_alloc
diff --git a/dapl/common/dapl_cookie.h b/dapl/common/dapl_cookie.h
index 20fbec6..f953b28 100644
--- a/dapl/common/dapl_cookie.h
+++ b/dapl/common/dapl_cookie.h
@@ -50,6 +50,10 @@ extern void
 dapls_cb_free (
     DAPL_COOKIE_BUFFER		*buffer );
 
+extern DAT_COUNT 
+dapls_cb_pending (
+    DAPL_COOKIE_BUFFER		*buffer );
+
 extern DAT_RETURN
 dapls_rmr_cookie_alloc (
     IN  DAPL_COOKIE_BUFFER	*buffer,
diff --git a/dapl/common/dapl_ep_get_status.c
b/dapl/common/dapl_ep_get_status.c
index d55b512..853afff 100644
--- a/dapl/common/dapl_ep_get_status.c
+++ b/dapl/common/dapl_ep_get_status.c
@@ -98,12 +98,12 @@ dapl_ep_get_status (
 
     if ( in_dto_idle != NULL )
     {
-	*in_dto_idle = (dapl_os_atomic_read (&ep_ptr->recv_count)) ?
DAT_FALSE : DAT_TRUE;
+	*in_dto_idle = (dapls_cb_pending(&ep_ptr->recv_buffer)) ?
DAT_FALSE : DAT_TRUE;
     }
 
     if ( out_dto_idle != NULL )
     {
-	*out_dto_idle = (dapl_os_atomic_read (&ep_ptr->req_count)) ?
DAT_FALSE : DAT_TRUE;
+	*out_dto_idle = (dapls_cb_pending(&ep_ptr->req_buffer)) ?
DAT_FALSE : DAT_TRUE;
     }
 
  bail:
diff --git a/dapl/common/dapl_ep_post_recv.c
b/dapl/common/dapl_ep_post_recv.c
index 2e52411..12724cd 100644
--- a/dapl/common/dapl_ep_post_recv.c
+++ b/dapl/common/dapl_ep_post_recv.c
@@ -109,19 +109,12 @@ dapl_ep_post_recv (
     }
 
     /*
-     * Take reference before posting to avoid race conditions with
-     * completions
-     */
-    dapl_os_atomic_inc (&ep_ptr->recv_count);
-
-    /*
      * Invoke provider specific routine to post DTO
      */
     dat_status = dapls_ib_post_recv (ep_ptr, cookie, num_segments,
local_iov);
 
     if ( dat_status != DAT_SUCCESS )
     {
-	dapl_os_atomic_dec (&ep_ptr->recv_count);
 	dapls_cookie_dealloc (&ep_ptr->recv_buffer, cookie);
     }
 
diff --git a/dapl/common/dapl_ep_util.c b/dapl/common/dapl_ep_util.c
index e29e287..f3f88bc 100644
--- a/dapl/common/dapl_ep_util.c
+++ b/dapl/common/dapl_ep_util.c
@@ -161,9 +161,6 @@ dapl_ep_alloc (
     ep_ptr->qp_state  = DAPL_QP_STATE_UNATTACHED;
     ep_ptr->cm_handle = IB_INVALID_HANDLE;
 
-    dapl_os_atomic_set (&ep_ptr->req_count, 0);
-    dapl_os_atomic_set (&ep_ptr->recv_count, 0);
-
     if ( DAT_SUCCESS != dapls_cb_create (
 	     &ep_ptr->req_buffer,
 	     ep_ptr,
@@ -431,18 +428,20 @@ dapl_ep_post_send_req (
 		 dto_type,
 		 user_cookie,
 		 &cookie );
-    if ( dat_status != DAT_SUCCESS )
-    {
+    if (dat_status != DAT_SUCCESS) {
+	dapl_log(DAPL_DBG_TYPE_ERR,
+		 " dapl_post_req resource ERR:"
+		 " dtos pending = %d, max_dtos %d, max_cb %d hd %d tl
%d\n",
+		 dapls_cb_pending(&ep_ptr->req_buffer),
+		 ep_ptr->param.ep_attr.max_request_dtos,
+		 ep_ptr->req_buffer.pool_size,
+		 ep_ptr->req_buffer.head,
+		 ep_ptr->req_buffer.tail);
+
 	goto bail;
     }
 
     /*
-     * Take reference before posting to avoid race conditions with
-     * completions
-     */
-    dapl_os_atomic_inc (&ep_ptr->req_count);
-
-    /*
      * Invoke provider specific routine to post DTO
      */
     dat_status = dapls_ib_post_send ( ep_ptr,
@@ -455,7 +454,6 @@ dapl_ep_post_send_req (
 
     if ( dat_status != DAT_SUCCESS )
     {
-	dapl_os_atomic_dec (&ep_ptr->req_count);
 	dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie);
     }
 
diff --git a/dapl/common/dapl_evd_util.c b/dapl/common/dapl_evd_util.c
index 293759f..87a0194 100755
--- a/dapl/common/dapl_evd_util.c
+++ b/dapl/common/dapl_evd_util.c
@@ -1099,17 +1099,11 @@ dapli_evd_cqe_to_event (
 	    }
 #endif
 
-	    if ( DAPL_DTO_TYPE_RECV == cookie->val.dto.type )
-	    {
-		dapl_os_atomic_dec (&ep_ptr->recv_count);
-		buffer = &ep_ptr->recv_buffer;
-	    }
+	    if (DAPL_DTO_TYPE_RECV == cookie->val.dto.type)
+	    	buffer = &ep_ptr->recv_buffer;
 	    else
-	    {
-		dapl_os_atomic_dec (&ep_ptr->req_count);
-		buffer = &ep_ptr->req_buffer;
-	    }
-
+	    	buffer = &ep_ptr->req_buffer;
+	    
 	    event_ptr->event_number = DAT_DTO_COMPLETION_EVENT;
 	    event_ptr->event_data.dto_completion_event_data.ep_handle =
 		cookie->ep;
@@ -1136,8 +1130,6 @@ dapli_evd_cqe_to_event (
 
 	case DAPL_COOKIE_TYPE_RMR:
 	{
-	    dapl_os_atomic_dec (&ep_ptr->req_count);
-
 	    event_ptr->event_number = DAT_RMR_BIND_COMPLETION_EVENT;
 
 	    event_ptr->event_data.rmr_completion_event_data.rmr_handle =
diff --git a/dapl/common/dapl_rmr_bind.c b/dapl/common/dapl_rmr_bind.c
index 01d06c3..12a98f6 100755
--- a/dapl/common/dapl_rmr_bind.c
+++ b/dapl/common/dapl_rmr_bind.c
@@ -151,12 +151,6 @@ dapli_rmr_bind_fuse (
 
     is_signaled = (completion_flags & DAT_COMPLETION_SUPPRESS_FLAG) ?
DAT_FALSE : DAT_TRUE;
 
-    /*
-     * Take reference before posting to avoid race conditions with
-     * completions
-     */
-    dapl_os_atomic_inc (&ep_ptr->req_count);
-
     dat_status = dapls_ib_mw_bind (rmr,
 				   lmr,
 				   ep_ptr,
@@ -167,7 +161,6 @@ dapli_rmr_bind_fuse (
 				   is_signaled);
     if ( DAT_SUCCESS != dat_status )
     {
-	dapl_os_atomic_dec (&ep_ptr->req_count);
 	dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie);
 	goto bail;
     }
@@ -255,19 +248,12 @@ dapli_rmr_bind_unfuse (
 
     is_signaled = (completion_flags & DAT_COMPLETION_UNSIGNALLED_FLAG)
? DAT_FALSE : DAT_TRUE;
 
-    /*
-     * Take reference before posting to avoid race conditions with 
-     * completions 
-     */
-    dapl_os_atomic_inc (&ep_ptr->req_count);
-
     dat_status = dapls_ib_mw_unbind (rmr,
 				     ep_ptr,
 				     cookie,
 				     is_signaled);
     if ( DAT_SUCCESS != dat_status )
     {
-	dapl_os_atomic_dec (&ep_ptr->req_count);
 	dapls_cookie_dealloc (&ep_ptr->req_buffer, cookie);
 	goto bail1;
     }
-- 
1.5.2.5




More information about the general mailing list