[ofa-general] [PATCH] uDAPL v2 - Add 64 bit counters for IA, EP, and EVD's

Arlin Davis arlin.r.davis at intel.com
Tue Mar 3 12:50:48 PST 2009


 Add full set of 64-bit counters to uDAPL common code, cma, and scm 
 providers. Replaced the current global set of counters. 
  
 Details:

 Use -DDAPL_COUNTERS to build-in counters for cma and scm providers.
 New extension calls in dat_ib_extensions.h for counters
	dat_print_counters, dat_query_counters
 Added counters for operations, async errors, and data
 Update dtestx (-p) with print and query counter examples

Signed-off-by: Arlin Davis <ardavis at ichips.intel.com>
---
 dapl/common/dapl_debug.c                    |  202 +++++++++++++++++++++-----
 dapl/common/dapl_ep_connect.c               |    2 +-
 dapl/common/dapl_ep_create.c                |    2 +-
 dapl/common/dapl_ep_disconnect.c            |    2 +-
 dapl/common/dapl_ep_free.c                  |    2 +-
 dapl/common/dapl_ep_get_status.c            |    4 +-
 dapl/common/dapl_ep_post_rdma_read.c        |    1 -
 dapl/common/dapl_ep_post_rdma_write.c       |    1 -
 dapl/common/dapl_ep_post_recv.c             |    1 -
 dapl/common/dapl_ep_post_send.c             |    1 -
 dapl/common/dapl_ep_util.c                  |   15 ++
 dapl/common/dapl_evd_connection_callb.c     |    3 +-
 dapl/common/dapl_evd_cq_async_error_callb.c |    1 +
 dapl/common/dapl_evd_dequeue.c              |    9 +-
 dapl/common/dapl_evd_dto_callb.c            |    2 +-
 dapl/common/dapl_evd_free.c                 |    3 +-
 dapl/common/dapl_evd_qp_async_error_callb.c |    1 +
 dapl/common/dapl_evd_un_async_error_callb.c |    1 +
 dapl/common/dapl_evd_util.c                 |   15 ++
 dapl/common/dapl_ia_util.c                  |   14 ++
 dapl/common/dapl_lmr_free.c                 |   11 +-
 dapl/common/dapl_psp_create.c               |    3 +-
 dapl/common/dapl_psp_create_any.c           |    2 +
 dapl/common/dapl_psp_free.c                 |    2 +
 dapl/common/dapl_pz_create.c                |    1 +
 dapl/common/dapl_pz_free.c                  |    2 +
 dapl/common/dapl_rmr_create.c               |    2 +-
 dapl/common/dapl_rmr_free.c                 |    3 +
 dapl/common/dapl_rsp_create.c               |    2 +
 dapl/common/dapl_rsp_free.c                 |    2 +
 dapl/common/dapl_srq_create.c               |    2 +
 dapl/common/dapl_srq_free.c                 |    2 +
 dapl/common/dapl_timer_util.c               |    2 -
 dapl/ibal/dapl_ibal_extensions.c            |    2 -
 dapl/include/dapl.h                         |   10 ++
 dapl/include/dapl_debug.h                   |   55 ++------
 dapl/openib_cma/dapl_ib_dto.h               |   62 ++++++++-
 dapl/openib_cma/dapl_ib_extensions.c        |   35 +++++-
 dapl/openib_cma/dapl_ib_util.c              |    5 +
 dapl/openib_scm/dapl_ib_dto.h               |   74 +++++++++-
 dapl/openib_scm/dapl_ib_extensions.c        |   35 +++++-
 dapl/openib_scm/dapl_ib_util.c              |    5 +
 dapl/udapl/dapl_evd_create.c                |    3 +-
 dapl/udapl/dapl_evd_wait.c                  |    6 +-
 dapl/udapl/dapl_init.c                      |    2 -
 dapl/udapl/dapl_lmr_create.c                |    2 +
 dapl/udapl/linux/dapl_osd.h                 |    9 +-
 dat/include/dat2/dat_ib_extensions.h        |  133 +++++++++++++++++-
 test/dtest/dtestx.c                         |   88 +++++++++++-
 49 files changed, 706 insertions(+), 138 deletions(-)

diff --git a/dapl/common/dapl_debug.c b/dapl/common/dapl_debug.c
index e717591..7601fc8 100644
--- a/dapl/common/dapl_debug.c
+++ b/dapl/common/dapl_debug.c
@@ -25,7 +25,6 @@
  * and/or other materials provided with the distribution.
  */
 
-#include "dapl_debug.h"
 #include "dapl.h"
 #if !defined(__KDAPL__)
 #include <stdarg.h>
@@ -67,52 +66,177 @@ void dapl_internal_dbg_log ( DAPL_DBG_TYPE type, const char *fmt, ...)
     }
 }
 
-#if defined(DAPL_COUNTERS)
-int dapl_dbg_counters[DCNT_NUM_COUNTERS] = { 0 };
+#ifdef DAPL_COUNTERS
 
 /*
- * The order of this list must match exactly with the #defines
- * in dapl_debug.h
+ * The order of this list must match the DAT counter definitions 
  */
-char  *dapl_dbg_counter_names[] = {
-	"dapl_ep_create",
-	"dapl_ep_free",
-	"dapl_ep_connect",
-	"dapl_ep_disconnect",
-	"dapl_ep_post_send",
-	"dapl_ep_post_recv",
-	"dapl_ep_post_rdma_write",
-	"dapl_ep_post_rdma_read",
-	"dapl_evd_create",
-	"dapl_evd_free",
-	"dapl_evd_wait",
-	"dapl_evd_blocked",
-	"dapl_evd_completion_notify",
-	"dapl_evd_dto_callback",
-	"dapl_evd_connection_callback",
-	"dapl_evd_dequeue",
-	"dapl_evd_poll",
-	"dapl_evd_found",
-	"dapl_evd_not_found",
-	"dapls_timer_set",
-	"dapls_timer_cancel",
-	"dapl_extensions"
+static char *ia_cntr_names[] = {
+	"DCNT_IA_PZ_CREATE",
+	"DCNT_IA_PZ_FREE",
+	"DCNT_IA_LMR_CREATE",
+	"DCNT_IA_LMR_FREE",
+	"DCNT_IA_RMR_CREATE",
+	"DCNT_IA_RMR_FREE",
+	"DCNT_IA_PSP_CREATE",
+	"DCNT_IA_PSP_CREATE_ANY",
+	"DCNT_IA_PSP_FREE",
+	"DCNT_IA_RSP_CREATE",
+	"DCNT_IA_RSP_FREE",
+	"DCNT_IA_EVD_CREATE",
+	"DCNT_IA_EVD_FREE",
+	"DCNT_IA_EP_CREATE",
+	"DCNT_IA_EP_FREE",
+	"DCNT_IA_SRQ_CREATE",
+	"DCNT_IA_SRQ_FREE",
+	"DCNT_IA_SP_CR",
+	"DCNT_IA_SP_CR_ACCEPTED",
+	"DCNT_IA_SP_CR_REJECTED",
+	"DCNT_IA_MEM_ALLOC",
+	"DCNT_IA_MEM_ALLOC_DATA",
+	"DCNT_IA_MEM_FREE",
+	"DCNT_IA_ASYNC_ERROR",
+	"DCNT_IA_ASYNC_QP_ERROR",
+	"DCNT_IA_ASYNC_CQ_ERROR"
 };
 
-void dapl_dump_cntr( int cntr )
+static char *ep_cntr_names[] = {
+	"DCNT_EP_CONNECT",
+	"DCNT_EP_DISCONNECT",
+	"DCNT_EP_POST_SEND",
+	"DCNT_EP_POST_SEND_DATA",
+	"DCNT_EP_POST_SEND_UD",
+	"DCNT_EP_POST_SEND_UD_DATA",
+	"DCNT_EP_POST_RECV",
+	"DCNT_EP_POST_RECV_DATA",
+	"DCNT_EP_POST_WRITE",
+	"DCNT_EP_POST_WRITE_DATA",
+	"DCNT_EP_POST_WRITE_IMM",
+	"DCNT_EP_POST_WRITE_IMM_DATA",
+	"DCNT_EP_POST_READ",
+	"DCNT_EP_POST_READ_DATA",
+	"DCNT_EP_POST_CMP_SWAP",
+	"DCNT_EP_POST_FETCH_ADD",
+	"DCNT_EP_RECV",
+	"DCNT_EP_RECV_DATA",
+	"DCNT_EP_RECV_UD",
+	"DCNT_EP_RECV_UD_DATA",
+	"DCNT_EP_RECV_IMM",
+	"DCNT_EP_RECV_IMM_DATA",
+	"DCNT_EP_RECV_RDMA_IMM",
+	"DCNT_EP_RECV_RDMA_IMM_DATA",
+};
+
+static char *evd_cntr_names[] = {
+	"DCNT_EVD_WAIT",
+	"DCNT_EVD_WAIT_BLOCKED",
+	"DCNT_EVD_WAIT_NOTIFY",
+	"DCNT_EVD_DEQUEUE",
+	"DCNT_EVD_DEQUEUE_FOUND",
+	"DCNT_EVD_DEQUEUE_NOT_FOUND",
+	"DCNT_EVD_DEQUEUE_POLL",
+	"DCNT_EVD_DEQUEUE_POLL_FOUND",
+	"DCNT_EVD_CONN_CALLBACK",
+	"DCNT_EVD_DTO_CALLBACK",
+};
+
+DAT_RETURN dapl_query_counter(DAT_HANDLE dh, 
+			      int counter, 
+			      void *p_cntrs_out,
+			      int reset) 
 {
-    int i;
+	int i, max;
+	DAT_UINT64 *p_cntrs;
+	DAT_HANDLE_TYPE type = 0;
+		
+	dat_get_handle_type(dh, &type);
 
-    for ( i = 0; i < DCNT_NUM_COUNTERS; i++ )
-    {
-        if (( cntr == i ) || ( cntr == DCNT_ALL_COUNTERS ))
-        {
-            dapl_dbg_log (  DAPL_DBG_TYPE_CNTR,
-                            "DAPL Counter: %s = %lu \n",
-                            dapl_dbg_counter_names[i],
-                            dapl_dbg_counters[i] );
+	switch(type) {
+		case DAT_HANDLE_TYPE_IA:
+			max = DCNT_IA_ALL_COUNTERS;
+			p_cntrs = ((DAPL_IA*)dh)->cntrs;
+			break;
+		case DAT_HANDLE_TYPE_EP:
+			max = DCNT_EP_ALL_COUNTERS;
+			p_cntrs = ((DAPL_EP*)dh)->cntrs;
+			break;
+		case DAT_HANDLE_TYPE_EVD:
+			max = DCNT_EVD_ALL_COUNTERS;
+			p_cntrs = ((DAPL_EVD*)dh)->cntrs;
+			break;
+		default:
+			return DAT_INVALID_HANDLE;
+	}
+                              
+	for (i=0; i < max; i++)	{
+		if ((counter == i) || (counter == max)) {
+			((DAT_UINT64*)p_cntrs_out)[i] = p_cntrs[i];
+			if (reset) p_cntrs[i]=0;
+		}
+	}
+	return DAT_SUCCESS;
+}
+
+char *dapl_query_counter_name(DAT_HANDLE dh, int counter) 
+{
+	DAT_HANDLE_TYPE type = 0;
+		
+	dat_get_handle_type(dh, &type);
+
+	switch(type) {
+		case DAT_HANDLE_TYPE_IA:
+			if (counter < DCNT_IA_ALL_COUNTERS)
+				return ia_cntr_names[counter];
+			break;
+		case DAT_HANDLE_TYPE_EP:
+			if (counter < DCNT_EP_ALL_COUNTERS)
+				return ep_cntr_names[counter];
+			break;
+		case DAT_HANDLE_TYPE_EVD:
+			if (counter < DCNT_EVD_ALL_COUNTERS)
+				return evd_cntr_names[counter];
+			break;
+		default:
+			return NULL;
+	}
+	return NULL;
+}
+
+#include <inttypes.h>
+void dapl_print_counter(DAT_HANDLE dh, int counter, int reset)
+{
+	int i, max;
+	DAT_UINT64 *p_cntrs;
+	DAT_HANDLE_TYPE type = 0;
+		
+	dat_get_handle_type(dh, &type);
+
+	switch(type) {
+		case DAT_HANDLE_TYPE_IA:
+			max = DCNT_IA_ALL_COUNTERS;
+			p_cntrs = ((DAPL_IA*)dh)->cntrs;
+			break;
+		case DAT_HANDLE_TYPE_EP:
+			max = DCNT_EP_ALL_COUNTERS;
+			p_cntrs = ((DAPL_EP*)dh)->cntrs;
+			break;
+		case DAT_HANDLE_TYPE_EVD:
+			max = DCNT_EVD_ALL_COUNTERS;
+			p_cntrs = ((DAPL_EVD*)dh)->cntrs;
+			break;
+		default:
+			return;
+	}
+                              
+	for (i=0; i < max; i++)	{
+		if ((counter == i) || (counter == max)) {
+			printf( " %s = " F64u " \n",
+				dapl_query_counter_name(dh, i),
+				p_cntrs[i] );
+			if (reset) p_cntrs[i]=0;
+		}
         }
-    }
+	return;
 }
 
 #endif /* DAPL_COUNTERS */
diff --git a/dapl/common/dapl_ep_connect.c b/dapl/common/dapl_ep_connect.c
index 0c3f10a..9a72531 100755
--- a/dapl/common/dapl_ep_connect.c
+++ b/dapl/common/dapl_ep_connect.c
@@ -97,7 +97,6 @@ dapl_ep_connect (
 		private_data,
 		qos,
 		connect_flags);
-    DAPL_CNTR (DCNT_EP_CONNECT);
 
     dat_status = DAT_SUCCESS;
     ep_ptr = (DAPL_EP *) ep_handle;
@@ -124,6 +123,7 @@ dapl_ep_connect (
 	dat_status = DAT_ERROR (DAT_INVALID_PARAMETER, DAT_INVALID_ARG4);
 	goto bail;
     }
+    DAPL_CNTR (ep_ptr, DCNT_EP_CONNECT);
 
     /*
      * If the endpoint needs a QP, associated the QP with it.
diff --git a/dapl/common/dapl_ep_create.c b/dapl/common/dapl_ep_create.c
index ff233b0..8982876 100644
--- a/dapl/common/dapl_ep_create.c
+++ b/dapl/common/dapl_ep_create.c
@@ -89,7 +89,6 @@ dapl_ep_create (
 		  connect_evd_handle, 
 		  ep_attr, 
 		  ep_handle);
-    DAPL_CNTR(DCNT_EP_CREATE);
 
     ia_ptr = (DAPL_IA *)ia_handle;
     dat_status = DAT_SUCCESS;
@@ -102,6 +101,7 @@ dapl_ep_create (
 	dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA);
 	goto bail;
     }
+    DAPL_CNTR(ia_ptr, DCNT_IA_EP_CREATE);
 
     /*
      * Verify non-required parameters.
diff --git a/dapl/common/dapl_ep_disconnect.c b/dapl/common/dapl_ep_disconnect.c
index fabce92..631d809 100644
--- a/dapl/common/dapl_ep_disconnect.c
+++ b/dapl/common/dapl_ep_disconnect.c
@@ -75,7 +75,6 @@ dapl_ep_disconnect (
 		 "dapl_ep_disconnect (%p, %x)\n",
 		  ep_handle,
 		  disconnect_flags);
-    DAPL_CNTR(DCNT_EP_DISCONNECT);
 
     ep_ptr = (DAPL_EP *) ep_handle;
 
@@ -87,6 +86,7 @@ dapl_ep_disconnect (
 	dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP);
 	goto bail;
     }
+    DAPL_CNTR(ep_ptr, DCNT_EP_DISCONNECT);
 
     /*
      * Do the verification of parameters and the state change
diff --git a/dapl/common/dapl_ep_free.c b/dapl/common/dapl_ep_free.c
index 62d9644..76b34f0 100644
--- a/dapl/common/dapl_ep_free.c
+++ b/dapl/common/dapl_ep_free.c
@@ -73,7 +73,6 @@ dapl_ep_free (
 
     dapl_dbg_log (DAPL_DBG_TYPE_API | DAPL_DBG_TYPE_CM,
                   "dapl_ep_free (%p)\n", ep_handle);
-    DAPL_CNTR(DCNT_EP_FREE);
 
     ep_ptr = (DAPL_EP *) ep_handle;
     param = &ep_ptr->param;
@@ -86,6 +85,7 @@ dapl_ep_free (
 	dat_status = DAT_ERROR (DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP);
 	goto bail;
     }
+    DAPL_CNTR(ep_ptr->header.owner_ia, DCNT_IA_EP_FREE);
 
     if ( ep_ptr->param.ep_state == DAT_EP_STATE_RESERVED ||
 	 ep_ptr->param.ep_state == DAT_EP_STATE_PASSIVE_CONNECTION_PENDING ||
diff --git a/dapl/common/dapl_ep_get_status.c b/dapl/common/dapl_ep_get_status.c
index 3af7f9a..3f856d3 100644
--- a/dapl/common/dapl_ep_get_status.c
+++ b/dapl/common/dapl_ep_get_status.c
@@ -99,12 +99,12 @@ dapl_ep_get_status (
 
     if ( in_dto_idle != NULL )
     {
-	*in_dto_idle = (dapls_cb_pending(&ep_ptr->recv_buffer)) ? DAT_FALSE : DAT_TRUE;
+	*in_dto_idle = dapls_cb_pending(&ep_ptr->recv_buffer);
     }
 
     if ( out_dto_idle != NULL )
     {
-	*out_dto_idle = (dapls_cb_pending(&ep_ptr->req_buffer)) ? DAT_FALSE : DAT_TRUE;
+	*out_dto_idle = dapls_cb_pending(&ep_ptr->req_buffer);
     }
 
  bail:
diff --git a/dapl/common/dapl_ep_post_rdma_read.c b/dapl/common/dapl_ep_post_rdma_read.c
index 95a7e05..dc6c39c 100644
--- a/dapl/common/dapl_ep_post_rdma_read.c
+++ b/dapl/common/dapl_ep_post_rdma_read.c
@@ -86,7 +86,6 @@ dapl_ep_post_rdma_read (
 		  user_cookie.as_64,
 		  remote_iov, 
 		  completion_flags);
-    DAPL_CNTR(DCNT_POST_RDMA_READ);
 
     dat_status = dapl_ep_post_send_req(ep_handle, 
                                  num_segments, 
diff --git a/dapl/common/dapl_ep_post_rdma_write.c b/dapl/common/dapl_ep_post_rdma_write.c
index 6b9ae94..ef79d0c 100644
--- a/dapl/common/dapl_ep_post_rdma_write.c
+++ b/dapl/common/dapl_ep_post_rdma_write.c
@@ -85,7 +85,6 @@ dapl_ep_post_rdma_write (
 		  user_cookie.as_64,
 		  remote_iov, 
 		  completion_flags);
-    DAPL_CNTR(DCNT_POST_RDMA_WRITE);
 
     dat_status = dapl_ep_post_send_req(ep_handle, 
                                  num_segments, 
diff --git a/dapl/common/dapl_ep_post_recv.c b/dapl/common/dapl_ep_post_recv.c
index 12724cd..b11eb96 100644
--- a/dapl/common/dapl_ep_post_recv.c
+++ b/dapl/common/dapl_ep_post_recv.c
@@ -85,7 +85,6 @@ dapl_ep_post_recv (
 		  local_iov,
 		  user_cookie.as_64,
 		  completion_flags);
-    DAPL_CNTR (DCNT_POST_RECV);
 
     if ( DAPL_BAD_HANDLE (ep_handle, DAPL_MAGIC_EP) )
     {
diff --git a/dapl/common/dapl_ep_post_send.c b/dapl/common/dapl_ep_post_send.c
index c13a095..0229132 100644
--- a/dapl/common/dapl_ep_post_send.c
+++ b/dapl/common/dapl_ep_post_send.c
@@ -84,7 +84,6 @@ dapl_ep_post_send (
 		  local_iov, 
 		  user_cookie.as_64,
 		  completion_flags);
-    DAPL_CNTR(DCNT_POST_SEND);
 
     dat_status = dapl_ep_post_send_req(ep_handle,
                                  num_segments,
diff --git a/dapl/common/dapl_ep_util.c b/dapl/common/dapl_ep_util.c
index 782e09a..f75c059 100644
--- a/dapl/common/dapl_ep_util.c
+++ b/dapl/common/dapl_ep_util.c
@@ -125,6 +125,16 @@ dapl_ep_alloc (
     /* zero the structure */
     dapl_os_memzero (ep_ptr, sizeof (DAPL_EP) + sizeof (DAT_SOCK_ADDR));
 
+#ifdef DAPL_COUNTERS
+    /* Allocate counters */
+    ep_ptr->cntrs = dapl_os_alloc(sizeof(DAT_UINT64) * DCNT_EP_ALL_COUNTERS);
+    if (ep_ptr->cntrs == NULL) {
+	dapl_os_free(ep_ptr, sizeof (DAPL_EP) + sizeof (DAT_SOCK_ADDR));
+	return (NULL);
+    }
+    dapl_os_memzero (ep_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_EP_ALL_COUNTERS);
+#endif /* DAPL_COUNTERS */
+
     /*
      * initialize the header
      */
@@ -227,6 +237,11 @@ dapl_ep_dealloc (
         ep_ptr->ibal_cm_handle = NULL;
     }
 #endif
+
+#ifdef DAPL_COUNTERS
+    dapl_os_free(ep_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_EP_ALL_COUNTERS);
+#endif /* DAPL_COUNTERS */
+
     dapl_os_free (ep_ptr, sizeof (DAPL_EP) + sizeof (DAT_SOCK_ADDR) );
 }
 
diff --git a/dapl/common/dapl_evd_connection_callb.c b/dapl/common/dapl_evd_connection_callb.c
index 7f994b0..d5a9750 100644
--- a/dapl/common/dapl_evd_connection_callb.c
+++ b/dapl/common/dapl_evd_connection_callb.c
@@ -84,8 +84,6 @@ dapl_evd_connection_callback (
 	ib_cm_event,
 	(void *) ib_cm_handle );
 
-    DAPL_CNTR(DCNT_EVD_CONN_CALLBACK);
-
     /*
      * Determine the type of handle passed back to us in the context
      * and sort out key parameters.
@@ -101,6 +99,7 @@ dapl_evd_connection_callback (
      */
     ep_ptr  = (DAPL_EP *) context;
     evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle;
+    DAPL_CNTR(evd_ptr, DCNT_EVD_CONN_CALLBACK);
 
     prd_ptr = (DAPL_PRIVATE *)private_data_ptr;
     private_data_size = 0;
diff --git a/dapl/common/dapl_evd_cq_async_error_callb.c b/dapl/common/dapl_evd_cq_async_error_callb.c
index 2b0251a..ca4bc8d 100644
--- a/dapl/common/dapl_evd_cq_async_error_callb.c
+++ b/dapl/common/dapl_evd_cq_async_error_callb.c
@@ -79,6 +79,7 @@ dapl_evd_cq_async_error_callback (
 
     evd = (DAPL_EVD *) context;
     async_evd = evd->header.owner_ia->async_error_evd;
+    DAPL_CNTR(evd->header.owner_ia, DCNT_IA_ASYNC_CQ_ERROR);
 
     dat_status = dapls_evd_post_async_error_event(
 	async_evd,
diff --git a/dapl/common/dapl_evd_dequeue.c b/dapl/common/dapl_evd_dequeue.c
index f6faeeb..5a409f3 100644
--- a/dapl/common/dapl_evd_dequeue.c
+++ b/dapl/common/dapl_evd_dequeue.c
@@ -75,7 +75,6 @@ DAT_RETURN DAT_API dapl_evd_dequeue (
 		  "dapl_evd_dequeue (%p, %p)\n",
 		  evd_handle, 
 		  event);
-    DAPL_CNTR(DCNT_EVD_DEQUEUE);
 
     evd_ptr = (DAPL_EVD *)evd_handle;
     dat_status = DAT_SUCCESS;
@@ -91,6 +90,7 @@ DAT_RETURN DAT_API dapl_evd_dequeue (
 	dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2);
 	goto bail;
     }
+    DAPL_CNTR(evd_ptr, DCNT_EVD_DEQUEUE);
 
     /*
      * We need to dequeue under lock, as the IB OS Access API
@@ -122,17 +122,18 @@ DAT_RETURN DAT_API dapl_evd_dequeue (
 	*event = *local_event;
 	dat_status = dapls_rbuf_add (&evd_ptr->free_event_queue, 
 				     local_event);
-	DAPL_CNTR(DCNT_EVD_DEQUEUE_FOUND);
+	DAPL_CNTR(evd_ptr, DCNT_EVD_DEQUEUE_FOUND);
+
     }
     else if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE)
     {
 	dat_status = dapls_evd_cq_poll_to_event(evd_ptr, event);
-	DAPL_CNTR(DCNT_EVD_DEQUEUE_POLL);
+	DAPL_CNTR(evd_ptr, DCNT_EVD_DEQUEUE_POLL);
     }
     else
     {
 	dat_status = DAT_ERROR (DAT_QUEUE_EMPTY,0);
-	DAPL_CNTR(DCNT_EVD_DEQUEUE_NOT_FOUND);
+	DAPL_CNTR(evd_ptr, DCNT_EVD_DEQUEUE_NOT_FOUND);
     }
 
     dapl_os_unlock ( &evd_ptr->header.lock );
diff --git a/dapl/common/dapl_evd_dto_callb.c b/dapl/common/dapl_evd_dto_callb.c
index bb3e1f8..347c2e9 100755
--- a/dapl/common/dapl_evd_dto_callb.c
+++ b/dapl/common/dapl_evd_dto_callb.c
@@ -84,9 +84,9 @@ dapl_evd_dto_callback (
 		  hca_handle, 
 		  cq_handle, 
 		  user_context);
-    DAPL_CNTR(DCNT_EVD_DTO_CALLBACK);
 
     evd_ptr = (DAPL_EVD *) user_context;
+    DAPL_CNTR(evd_ptr, DCNT_EVD_DTO_CALLBACK);
 
     dapl_os_assert (hca_handle == evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle);
     dapl_os_assert (evd_ptr->ib_cq_handle == cq_handle);
diff --git a/dapl/common/dapl_evd_free.c b/dapl/common/dapl_evd_free.c
index 4d126b2..8e4d082 100755
--- a/dapl/common/dapl_evd_free.c
+++ b/dapl/common/dapl_evd_free.c
@@ -68,7 +68,6 @@ DAT_RETURN DAT_API dapl_evd_free (
     DAT_RETURN  dat_status;
 
     dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_evd_free (%p)\n", evd_handle);
-    DAPL_CNTR (DCNT_EVD_FREE);
 
     dat_status = DAT_SUCCESS;
     evd_ptr = (DAPL_EVD *)evd_handle;
@@ -79,6 +78,8 @@ DAT_RETURN DAT_API dapl_evd_free (
 	goto bail;
     }
 
+    DAPL_CNTR(evd_ptr->header.owner_ia, DCNT_IA_EVD_FREE);
+
     if (dapl_os_atomic_read (&evd_ptr->evd_ref_count) != 0)
     {
 	dat_status = DAT_ERROR (DAT_INVALID_STATE, DAT_INVALID_STATE_EVD_IN_USE);
diff --git a/dapl/common/dapl_evd_qp_async_error_callb.c b/dapl/common/dapl_evd_qp_async_error_callb.c
index b14db3c..27b6ddc 100644
--- a/dapl/common/dapl_evd_qp_async_error_callb.c
+++ b/dapl/common/dapl_evd_qp_async_error_callb.c
@@ -114,6 +114,7 @@ dapl_evd_qp_async_error_callback (
 
     ia_ptr    = ep_ptr->header.owner_ia;
     async_evd = (DAPL_EVD *) ia_ptr->async_error_evd;
+    DAPL_CNTR(ia_ptr, DCNT_IA_ASYNC_QP_ERROR);
 
     dapl_dbg_log (
 	DAPL_DBG_TYPE_CALLBACK | DAPL_DBG_TYPE_EXCEPTION,
diff --git a/dapl/common/dapl_evd_un_async_error_callb.c b/dapl/common/dapl_evd_un_async_error_callb.c
index 5ba0f3e..987aee2 100644
--- a/dapl/common/dapl_evd_un_async_error_callb.c
+++ b/dapl/common/dapl_evd_un_async_error_callb.c
@@ -78,6 +78,7 @@ dapl_evd_un_async_error_callback (
     }
 
     async_evd = (DAPL_EVD *) context;
+    DAPL_CNTR(async_evd->header.owner_ia, DCNT_IA_ASYNC_ERROR);
 
     dat_status = dapls_ib_get_async_event(cause_ptr, &async_event);
 
diff --git a/dapl/common/dapl_evd_util.c b/dapl/common/dapl_evd_util.c
index c6c7463..ad3d13d 100644
--- a/dapl/common/dapl_evd_util.c
+++ b/dapl/common/dapl_evd_util.c
@@ -273,6 +273,17 @@ dapls_evd_alloc (
     /* zero the structure */
     dapl_os_memzero (evd_ptr, sizeof (DAPL_EVD));
 
+#ifdef DAPL_COUNTERS
+    /* Allocate counters */
+    evd_ptr->cntrs = dapl_os_alloc(sizeof(DAT_UINT64) * DCNT_EVD_ALL_COUNTERS);
+    if (evd_ptr->cntrs == NULL) {
+	dapl_os_free(evd_ptr, sizeof(DAPL_EVD));
+	return (NULL);
+    }
+    dapl_os_memzero(evd_ptr->cntrs, 
+		    sizeof(DAT_UINT64) * DCNT_EVD_ALL_COUNTERS);
+#endif /* DAPL_COUNTERS */
+
     /*
      * initialize the header
      */
@@ -542,6 +553,10 @@ dapls_evd_dealloc (
     }
 #endif
 
+#ifdef DAPL_COUNTERS
+    dapl_os_free(evd_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_EVD_ALL_COUNTERS);
+#endif /* DAPL_COUNTERS */
+
     dapl_os_free (evd_ptr, sizeof (DAPL_EVD));
 
 bail:
diff --git a/dapl/common/dapl_ia_util.c b/dapl/common/dapl_ia_util.c
index 7b03afc..a42063b 100755
--- a/dapl/common/dapl_ia_util.c
+++ b/dapl/common/dapl_ia_util.c
@@ -78,6 +78,16 @@ dapl_ia_alloc ( DAT_PROVIDER * provider, DAPL_HCA * hca_ptr )
     /* zero the structure */
     dapl_os_memzero (ia_ptr, sizeof (DAPL_IA));
 
+#ifdef DAPL_COUNTERS
+    /* Allocate counters */
+    ia_ptr->cntrs = dapl_os_alloc(sizeof(DAT_UINT64) * DCNT_IA_ALL_COUNTERS);
+    if (ia_ptr->cntrs == NULL) {
+	dapl_os_free (ia_ptr, sizeof(DAPL_IA));
+	return (NULL);
+    }
+    dapl_os_memzero(ia_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_IA_ALL_COUNTERS);
+#endif /* DAPL_COUNTERS */
+
     /*
      * initialize the header
      */
@@ -596,6 +606,10 @@ dapls_ia_free ( DAPL_IA *ia_ptr )
     ia_ptr->header.magic = DAPL_MAGIC_INVALID; /* reset magic to prevent reuse */
     dapl_os_lock_destroy (&ia_ptr->header.lock);
 
+#ifdef DAPL_COUNTERS
+    dapl_os_free(ia_ptr->cntrs, sizeof(DAT_UINT64) * DCNT_IA_ALL_COUNTERS);
+#endif /* DAPL_COUNTERS */
+
     dapl_os_free (ia_ptr, sizeof (DAPL_IA));
 }
 
diff --git a/dapl/common/dapl_lmr_free.c b/dapl/common/dapl_lmr_free.c
index f4106b4..3fa67b2 100644
--- a/dapl/common/dapl_lmr_free.c
+++ b/dapl/common/dapl_lmr_free.c
@@ -62,6 +62,7 @@ dapl_lmr_free (
     IN	DAT_LMR_HANDLE  lmr_handle )
 {
     DAPL_LMR 		*lmr;
+    DAPL_PZ		*pz;
     DAT_RETURN		dat_status;
 
     dapl_dbg_log (DAPL_DBG_TYPE_API, "dapl_lmr_free (%p)\n", lmr_handle);
@@ -72,7 +73,10 @@ dapl_lmr_free (
 	goto bail;
     }
 
-    lmr = (DAPL_LMR *) lmr_handle;
+    lmr = (DAPL_LMR *)lmr_handle;
+    pz = (DAPL_PZ *)lmr->param.pz_handle;
+    
+    DAPL_CNTR(pz->header.owner_ia, DCNT_IA_LMR_FREE);
 
     switch (lmr->param.mem_type)
     {
@@ -85,8 +89,6 @@ dapl_lmr_free (
 	case DAT_MEM_TYPE_VIRTUAL:
 	case DAT_MEM_TYPE_LMR:
 	{
-	    DAPL_PZ 	*pz;
-
 	    if ( 0 != dapl_os_atomic_read (&lmr->lmr_ref_count) )
 	    {
 		return DAT_INVALID_STATE;
@@ -103,10 +105,7 @@ dapl_lmr_free (
 
 	    if (dat_status == DAT_SUCCESS)
 	    {
-
-		pz = (DAPL_PZ *) lmr->param.pz_handle;
 		dapl_os_atomic_dec (&pz->pz_ref_count);
-
 		dapl_lmr_dealloc (lmr);
 	    }
 	    else
diff --git a/dapl/common/dapl_psp_create.c b/dapl/common/dapl_psp_create.c
index 9d2e945..a6eb31c 100644
--- a/dapl/common/dapl_psp_create.c
+++ b/dapl/common/dapl_psp_create.c
@@ -114,7 +114,8 @@ dapl_psp_create (
 	dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG4);
 	goto bail;
     }
-	
+
+    DAPL_CNTR(ia_ptr, DCNT_IA_PSP_CREATE);	
 
     /*
      * See if we have a quiescent listener to use for this PSP, else
diff --git a/dapl/common/dapl_psp_create_any.c b/dapl/common/dapl_psp_create_any.c
index e2faa4a..9b9b575 100644
--- a/dapl/common/dapl_psp_create_any.c
+++ b/dapl/common/dapl_psp_create_any.c
@@ -133,6 +133,8 @@ dapl_psp_create_any (
 	goto bail;
     }
 
+    DAPL_CNTR(ia_ptr, DCNT_IA_PSP_CREATE_ANY);	
+
     /*
      * Fill out the args for a PSP
      */
diff --git a/dapl/common/dapl_psp_free.c b/dapl/common/dapl_psp_free.c
index 28913b4..0117e19 100644
--- a/dapl/common/dapl_psp_free.c
+++ b/dapl/common/dapl_psp_free.c
@@ -82,6 +82,8 @@ dapl_psp_free (
 
     ia_ptr = sp_ptr->header.owner_ia;
 
+    DAPL_CNTR(ia_ptr->header.owner_ia, DCNT_IA_PSP_FREE);
+
     /* 
      * Remove the connection listener if it has been established
      * and there are no current connections in progress.
diff --git a/dapl/common/dapl_pz_create.c b/dapl/common/dapl_pz_create.c
index a7105aa..efca311 100644
--- a/dapl/common/dapl_pz_create.c
+++ b/dapl/common/dapl_pz_create.c
@@ -90,6 +90,7 @@ dapl_pz_create (
 	dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY);
 	goto bail;
     }
+    DAPL_CNTR(ia, DCNT_IA_PZ_CREATE);
 
     dat_status = dapls_ib_pd_alloc (ia, pz);
     if ( DAT_SUCCESS != dat_status )
diff --git a/dapl/common/dapl_pz_free.c b/dapl/common/dapl_pz_free.c
index b2510f7..270dd38 100644
--- a/dapl/common/dapl_pz_free.c
+++ b/dapl/common/dapl_pz_free.c
@@ -72,6 +72,8 @@ dapl_pz_free (
 
     pz = (DAPL_PZ *) pz_handle;
 
+    DAPL_CNTR(pz->header.owner_ia, DCNT_IA_PZ_FREE);
+
     if ( 0 != dapl_os_atomic_read (&pz->pz_ref_count) )
     {
 	dat_status = DAT_ERROR (DAT_INVALID_STATE,DAT_INVALID_STATE_PZ_IN_USE);
diff --git a/dapl/common/dapl_rmr_create.c b/dapl/common/dapl_rmr_create.c
index 9a7dbd9..d3e1661 100755
--- a/dapl/common/dapl_rmr_create.c
+++ b/dapl/common/dapl_rmr_create.c
@@ -71,7 +71,6 @@ dapl_rmr_create (
     }
 
     pz = (DAPL_PZ *) pz_handle;
-
     rmr = dapl_rmr_alloc (pz);
 
     if ( rmr == NULL )
@@ -79,6 +78,7 @@ dapl_rmr_create (
 	dat_status = DAT_ERROR (DAT_INSUFFICIENT_RESOURCES,DAT_RESOURCE_MEMORY);
 	goto bail;
     }
+    DAPL_CNTR(pz->header.owner_ia, DCNT_IA_RMR_CREATE);
 
     dat_status = dapls_ib_mw_alloc (rmr);
 
diff --git a/dapl/common/dapl_rmr_free.c b/dapl/common/dapl_rmr_free.c
index a4edb13..f75aaed 100644
--- a/dapl/common/dapl_rmr_free.c
+++ b/dapl/common/dapl_rmr_free.c
@@ -58,6 +58,7 @@ dapl_rmr_free (
     IN	DAT_RMR_HANDLE  rmr_handle )
 {
     DAPL_RMR	 	*rmr;
+    DAPL_PZ		*pz;
     DAT_RETURN		dat_status;
 
     dat_status = DAT_SUCCESS;
@@ -69,6 +70,8 @@ dapl_rmr_free (
     }
 
     rmr = (DAPL_RMR *) rmr_handle;
+    pz = (DAPL_PZ *)rmr->param.pz_handle;
+    DAPL_CNTR(pz->header.owner_ia, DCNT_IA_RMR_FREE);
 
     /*
      * If the user did not perform an unbind op, release
diff --git a/dapl/common/dapl_rsp_create.c b/dapl/common/dapl_rsp_create.c
index a9c9b5f..cf41dd9 100644
--- a/dapl/common/dapl_rsp_create.c
+++ b/dapl/common/dapl_rsp_create.c
@@ -126,6 +126,8 @@ dapl_rsp_create (
 	goto bail;
     }
 
+    DAPL_CNTR(ia_ptr, DCNT_IA_RSP_CREATE);	
+
     sp_ptr = dapls_ia_sp_search (ia_ptr, conn_qual, DAT_FALSE);
     sp_found = DAT_TRUE;
     if (sp_ptr == NULL)
diff --git a/dapl/common/dapl_rsp_free.c b/dapl/common/dapl_rsp_free.c
index b7bc40c..e016f0d 100644
--- a/dapl/common/dapl_rsp_free.c
+++ b/dapl/common/dapl_rsp_free.c
@@ -85,6 +85,8 @@ dapl_rsp_free (
     /* ia_ptr = (DAPL_IA *)sp_ptr->header.owner_ia; */
     ia_ptr = sp_ptr->header.owner_ia;
 
+    DAPL_CNTR(ia_ptr, DCNT_IA_RSP_FREE);	
+
     /*
      * Remove the connection listener if there are no connections.  If
      * we defer removing the sp it becomes something of a zombie
diff --git a/dapl/common/dapl_srq_create.c b/dapl/common/dapl_srq_create.c
index b03bbd6..ef45826 100644
--- a/dapl/common/dapl_srq_create.c
+++ b/dapl/common/dapl_srq_create.c
@@ -97,6 +97,8 @@ dapl_srq_create (
 	goto bail;
     }
 
+    DAPL_CNTR(ia_ptr, DCNT_IA_SRQ_CREATE);
+
     /*
      * Verify non-required parameters.
      * N.B. Assumption: any parameter that can be
diff --git a/dapl/common/dapl_srq_free.c b/dapl/common/dapl_srq_free.c
index 55d74fc..9decf54 100644
--- a/dapl/common/dapl_srq_free.c
+++ b/dapl/common/dapl_srq_free.c
@@ -96,6 +96,8 @@ dapl_srq_free (
 
     ia_ptr = srq_ptr->header.owner_ia;
 
+    DAPL_CNTR(ia_ptr, DCNT_IA_SRQ_FREE);
+
     /*
      * Do verification of parameters and the state change atomically.
      */
diff --git a/dapl/common/dapl_timer_util.c b/dapl/common/dapl_timer_util.c
index c7407c1..12c41bc 100644
--- a/dapl/common/dapl_timer_util.c
+++ b/dapl/common/dapl_timer_util.c
@@ -107,7 +107,6 @@ dapls_timer_set (
     DAPL_OS_TIMEVAL		cur_time;
     DAT_BOOLEAN			wakeup_tmo_thread;
 
-    DAPL_CNTR(DCNT_TIMER_SET);
     /*
      * Start the timer thread the first time we need a timer
      */
@@ -216,7 +215,6 @@ void
 dapls_timer_cancel (
 	IN  DAPL_OS_TIMER		*timer)
 {
-    DAPL_CNTR(DCNT_TIMER_CANCEL);
     dapl_os_lock ( &g_daplTimerHead.lock );
     /*
      * make sure the entry has not been removed by another thread
diff --git a/dapl/ibal/dapl_ibal_extensions.c b/dapl/ibal/dapl_ibal_extensions.c
index b05c0bb..643125d 100644
--- a/dapl/ibal/dapl_ibal_extensions.c
+++ b/dapl/ibal/dapl_ibal_extensions.c
@@ -93,8 +93,6 @@ dapl_extensions(IN DAT_HANDLE		dat_handle,
 		     "dapl_extensions(hdl %p operation %d, ...)\n",
 		     dat_handle, ext_op);
 
-	DAPL_CNTR(DCNT_EXTENSION);
-
 	switch ((int)ext_op)
 	{
 	
diff --git a/dapl/include/dapl.h b/dapl/include/dapl.h
index e2025ce..b13c963 100755
--- a/dapl/include/dapl.h
+++ b/dapl/include/dapl.h
@@ -313,6 +313,9 @@ struct dapl_ia
     DAPL_LLIST_HEAD	psp_list_head;		/* PSP queue */
     DAPL_LLIST_HEAD	rsp_list_head;		/* RSP queue */
     DAPL_LLIST_HEAD	srq_list_head;		/* SRQ queue */
+#ifdef DAPL_COUNTERS
+    void		*cntrs;
+#endif
 };
 
 /* DAPL_CNO maps to DAT_CNO_HANDLE */
@@ -388,6 +391,10 @@ struct dapl_evd
 
     DAT_COUNT		threshold;
     DAPL_EVD_COMPLETION	completion_type;
+
+#ifdef DAPL_COUNTERS
+    void		*cntrs;
+#endif
 };
 
 /* DAPL_PRIVATE used to pass private data in a connection */
@@ -472,6 +479,9 @@ struct dapl_ep
     DAT_BOOLEAN         sent_discreq;
     dp_ib_cm_handle_t   ibal_cm_handle;
 #endif
+#ifdef DAPL_COUNTERS
+    void		*cntrs;
+#endif
 };
 
 /* DAPL_SRQ maps to DAT_SRQ_HANDLE */
diff --git a/dapl/include/dapl_debug.h b/dapl/include/dapl_debug.h
index f0de7c8..d034227 100644
--- a/dapl/include/dapl_debug.h
+++ b/dapl/include/dapl_debug.h
@@ -99,50 +99,25 @@ static __inline void dapl_dbg_log ( DAPL_DBG_TYPE type,  const char *fmt,  ...)
 
 #endif /* !DAPL_DBG */
 
-/*
- * Counters
- */
-#define DCNT_EP_CREATE			0
-#define DCNT_EP_FREE			1
-#define DCNT_EP_CONNECT			2
-#define DCNT_EP_DISCONNECT		3
-#define DCNT_POST_SEND			4
-#define DCNT_POST_RECV			5
-#define DCNT_POST_RDMA_WRITE		6
-#define DCNT_POST_RDMA_READ		7
-#define DCNT_EVD_CREATE			8
-#define DCNT_EVD_FREE			9
-#define DCNT_EVD_WAIT			10
-#define DCNT_EVD_WAIT_BLOCKED		11
-#define DCNT_EVD_WAIT_CMP_NTFY		12
-#define DCNT_EVD_DTO_CALLBACK		13
-#define DCNT_EVD_CONN_CALLBACK		14
-#define DCNT_EVD_DEQUEUE		15
-#define DCNT_EVD_DEQUEUE_POLL		16
-#define DCNT_EVD_DEQUEUE_FOUND		17
-#define DCNT_EVD_DEQUEUE_NOT_FOUND	18
-#define DCNT_TIMER_SET			19
-#define DCNT_TIMER_CANCEL		20
-#define DCNT_EXTENSION			21
-#define DCNT_NUM_COUNTERS		22
-#define DCNT_ALL_COUNTERS               DCNT_NUM_COUNTERS
-
-#if defined(DAPL_COUNTERS)
-
-extern void dapl_dump_cntr( int cntr );
-extern int dapl_dbg_counters[];
-
-#define DAPL_CNTR(cntr)         dapl_os_atomic_inc (&dapl_dbg_counters[cntr]);
-#define DAPL_DUMP_CNTR(cntr)    dapl_dump_cntr( cntr );
-#define DAPL_COUNTERS_INIT()
+#include <dat2/dat_ib_extensions.h>
+
+#ifdef DAPL_COUNTERS
+
+#define DAPL_CNTR(h_ptr, cntr) ((DAT_UINT64*)h_ptr->cntrs)[cntr]++
+#define DAPL_CNTR_DATA(h_ptr, cntr, data) ((DAT_UINT64*)h_ptr->cntrs)[cntr]+= data
+
+DAT_RETURN dapl_query_counter(DAT_HANDLE dh, 
+			      int counter, 
+			      void *p_cntrs_out,
+			      int reset);
+char *dapl_query_counter_name(DAT_HANDLE dh, int counter);
+void dapl_print_counter(DAT_HANDLE dh, int counter, int reset);
 
 #else
 
-#define DAPL_CNTR(cntr)
-#define DAPL_DUMP_CNTR(cntr) 
-#define DAPL_COUNTERS_INIT()
+#define DAPL_CNTR(handle, cntr)
+#define DAPL_CNTR_DATA(handle, cntr, data)
 
 #endif /* DAPL_COUNTERS */
 
-
 #endif /* _DAPL_DEBUG_H_ */
diff --git a/dapl/openib_cma/dapl_ib_dto.h b/dapl/openib_cma/dapl_ib_dto.h
index 2b01963..eba19b7 100644
--- a/dapl/openib_cma/dapl_ib_dto.h
+++ b/dapl/openib_cma/dapl_ib_dto.h
@@ -89,7 +89,7 @@ dapls_ib_post_recv (
 
 	if (NULL == ds_array_p)
 		return (DAT_INSUFFICIENT_RESOURCES);
-	
+
 	/* setup work request */
 	total_len = 0;
 	wr.next = 0;
@@ -126,6 +126,9 @@ dapls_ib_post_recv (
 	if (ret)
 		return( dapl_convert_errno(errno,"ibv_recv") );
 
+	DAPL_CNTR(ep_ptr, DCNT_EP_POST_RECV);
+	DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_RECV_DATA, total_len);
+
 	return DAT_SUCCESS;
 }
 
@@ -237,6 +240,25 @@ dapls_ib_post_send (
 	if (ret)
 		return( dapl_convert_errno(errno,"ibv_send") );
 
+#ifdef DAPL_COUNTERS
+	switch (op_type) {
+	case OP_SEND:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND);
+		DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_DATA,total_len);
+		break;
+	case OP_RDMA_WRITE:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE);
+		DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_WRITE_DATA,total_len);
+		break;	
+	case OP_RDMA_READ:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_READ);
+		DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_READ_DATA,total_len);
+		break;
+	default:
+		break;
+	}
+#endif /* DAPL_COUNTERS */
+
 	dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n");
 	return DAT_SUCCESS;
 }
@@ -416,7 +438,25 @@ dapls_ib_post_ext_send (
 
 	if (ret)
 		return( dapl_convert_errno(errno,"ibv_send") );
-	
+
+#ifdef DAPL_COUNTERS
+	switch (op_type) {
+	case OP_RDMA_WRITE_IMM:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE_IMM);
+		DAPL_CNTR_DATA(ep_ptr, 
+			       DCNT_EP_POST_WRITE_IMM_DATA, total_len);
+		break;
+	case OP_COMP_AND_SWAP:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_CMP_SWAP);
+		break;	
+	case OP_FETCH_AND_ADD:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_FETCH_ADD);
+		break;
+	default:
+		break;
+	}
+#endif /* DAPL_COUNTERS */
+
 	dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n");
 	return DAT_SUCCESS;
 }
@@ -434,6 +474,10 @@ dapls_ib_optional_prv_dat(
 /* map Work Completions to DAPL WR operations */
 STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p)
 {
+#ifdef DAPL_COUNTERS
+	DAPL_COOKIE *cookie = (DAPL_COOKIE *)(uintptr_t)cqe_p->wr_id;
+#endif /* DAPL_COUNTERS */
+
 	switch (cqe_p->opcode) {
 	case IBV_WC_SEND:
 		return (OP_SEND);
@@ -451,11 +495,21 @@ STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p)
 	case IBV_WC_BIND_MW:
 		return (OP_BIND_MW);
 	case IBV_WC_RECV:
-		if (cqe_p->wc_flags & IBV_WC_WITH_IMM)
+		if (cqe_p->wc_flags & IBV_WC_WITH_IMM) {
+			DAPL_CNTR(cookie->ep, DCNT_EP_RECV_IMM);
+			DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_IMM_DATA, 
+				       cqe_p->byte_len);
 			return (OP_RECEIVE_IMM);
-		else
+		} else {
+			DAPL_CNTR(cookie->ep, DCNT_EP_RECV);
+			DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_DATA, 
+				       cqe_p->byte_len);
 			return (OP_RECEIVE);
+		}
 	case IBV_WC_RECV_RDMA_WITH_IMM:
+		DAPL_CNTR(cookie->ep, DCNT_EP_RECV_RDMA_IMM);
+		DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_RDMA_IMM_DATA, 
+			       cqe_p->byte_len);
 		return (OP_RECEIVE_IMM);
 	default:
 		return (OP_INVALID);
diff --git a/dapl/openib_cma/dapl_ib_extensions.c b/dapl/openib_cma/dapl_ib_extensions.c
index 1402057..af34128 100755
--- a/dapl/openib_cma/dapl_ib_extensions.c
+++ b/dapl/openib_cma/dapl_ib_extensions.c
@@ -94,8 +94,6 @@ dapl_extensions(IN DAT_HANDLE		dat_handle,
 		     "dapl_extensions(hdl %p operation %d, ...)\n",
 		     dat_handle, ext_op);
 
-	DAPL_CNTR(DCNT_EXTENSION);
-
 	switch ((int)ext_op)
 	{
 	
@@ -150,6 +148,39 @@ dapl_extensions(IN DAT_HANDLE		dat_handle,
 
     		break;
 
+#ifdef DAPL_COUNTERS
+	case DAT_QUERY_COUNTERS_OP:
+	{
+		int cntr, reset;
+		DAT_UINT64 * p_cntr_out;
+
+    		dapl_dbg_log(DAPL_DBG_TYPE_RTN, 
+			     " Query counter extension call\n");
+		
+		cntr = va_arg(args, int);
+		p_cntr_out = va_arg(args, DAT_UINT64*);
+		reset  = va_arg(args, int);
+				
+		status = dapl_query_counter(dat_handle, cntr, 
+					    p_cntr_out, reset);
+    		break;
+	}
+	case DAT_PRINT_COUNTERS_OP:
+	{
+		int cntr, reset;
+		
+    		dapl_dbg_log(DAPL_DBG_TYPE_RTN, 
+			     " Print counter extension call\n");
+		
+		cntr = va_arg(args, int);
+		reset  = va_arg(args, int);
+
+		dapl_print_counter(dat_handle, cntr, reset);
+		status = DAT_SUCCESS;
+		break;
+	}
+#endif /* DAPL_COUNTERS */
+
 	default:
     		dapl_dbg_log(DAPL_DBG_TYPE_ERR, 
 			     "unsupported extension(%d)\n", (int)ext_op);
diff --git a/dapl/openib_cma/dapl_ib_util.c b/dapl/openib_cma/dapl_ib_util.c
index 36b534e..c5641f0 100755
--- a/dapl/openib_cma/dapl_ib_util.c
+++ b/dapl/openib_cma/dapl_ib_util.c
@@ -970,6 +970,11 @@ DAT_NAMED_ATTR	ib_attrs[] = {
     {
     	DAT_IB_ATTR_IMMED_DATA, "TRUE"
     },
+#ifdef DAPL_COUNTERS
+    {
+        DAT_ATTR_COUNTERS, "TRUE"
+    },
+#endif /* DAPL_COUNTERS */
 #endif
 };
 
diff --git a/dapl/openib_scm/dapl_ib_dto.h b/dapl/openib_scm/dapl_ib_dto.h
index fa19d01..ff338fc 100644
--- a/dapl/openib_scm/dapl_ib_dto.h
+++ b/dapl/openib_scm/dapl_ib_dto.h
@@ -92,7 +92,7 @@ dapls_ib_post_recv (
 
 	if (NULL == ds_array_p)
 		return (DAT_INSUFFICIENT_RESOURCES);
-	
+
 	/* setup work request */
 	total_len = 0;
 	wr.next = 0;
@@ -129,6 +129,9 @@ dapls_ib_post_recv (
 	if (ret)
 		return( dapl_convert_errno(errno,"ibv_recv") );
 
+	DAPL_CNTR(ep_ptr, DCNT_EP_POST_RECV);
+	DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_RECV_DATA, total_len);
+
 	return DAT_SUCCESS;
 }
 
@@ -244,6 +247,25 @@ dapls_ib_post_send (
 	if (ret)
 		return( dapl_convert_errno(errno,"ibv_send") );
 
+#ifdef DAPL_COUNTERS
+	switch (op_type) {
+	case OP_SEND:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND);
+		DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_DATA,total_len);
+		break;
+	case OP_RDMA_WRITE:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE);
+		DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_WRITE_DATA,total_len);
+		break;	
+	case OP_RDMA_READ:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_READ);
+		DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_READ_DATA,total_len);
+		break;
+	default:
+		break;
+	}
+#endif /* DAPL_COUNTERS */
+
 	dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n");
 	return DAT_SUCCESS;
 }
@@ -452,8 +474,30 @@ dapls_ib_post_ext_send (
 	if (ret)
 		return( dapl_convert_errno(errno,"ibv_send") );
 	
+#ifdef DAPL_COUNTERS
+	switch (op_type) {
+	case OP_RDMA_WRITE_IMM:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE_IMM);
+		DAPL_CNTR_DATA(ep_ptr, 
+			       DCNT_EP_POST_WRITE_IMM_DATA, total_len);
+		break;
+	case OP_COMP_AND_SWAP:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_CMP_SWAP);
+		break;	
+	case OP_FETCH_AND_ADD:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_FETCH_ADD);
+		break;
+	case OP_SEND_UD:
+		DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND_UD);
+		DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_UD_DATA, total_len);
+		break;
+	default:
+		break;
+	}
+#endif /* DAPL_COUNTERS */
+
 	dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n");
-	return DAT_SUCCESS;
+        return DAT_SUCCESS;
 }
 #endif
 
@@ -470,6 +514,10 @@ dapls_ib_optional_prv_dat(
 /* map Work Completions to DAPL WR operations */
 STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p)
 {
+#ifdef DAPL_COUNTERS
+	DAPL_COOKIE *cookie = (DAPL_COOKIE *)(uintptr_t)cqe_p->wr_id;
+#endif /* DAPL_COUNTERS */
+
 	switch (cqe_p->opcode) {
 	case IBV_WC_SEND:
 		if (CQE_WR_TYPE_UD(cqe_p->wr_id))
@@ -490,13 +538,27 @@ STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p)
 	case IBV_WC_BIND_MW:
 		return (OP_BIND_MW);
 	case IBV_WC_RECV:
-		if (CQE_WR_TYPE_UD(cqe_p->wr_id)) 
+		if (CQE_WR_TYPE_UD(cqe_p->wr_id)) {
+			DAPL_CNTR(cookie->ep, DCNT_EP_RECV_UD);
+			DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_UD_DATA, 
+				       cqe_p->byte_len);
 			return (OP_RECV_UD);
-		else if (cqe_p->wc_flags & IBV_WC_WITH_IMM)
-			return (OP_RECEIVE_MSG_IMM);
-		else
+		}
+		else if (cqe_p->wc_flags & IBV_WC_WITH_IMM) {
+			DAPL_CNTR(cookie->ep, DCNT_EP_RECV_IMM);
+			DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_IMM_DATA, 
+				       cqe_p->byte_len);
+			return (OP_RECEIVE_IMM);
+		} else {
+			DAPL_CNTR(cookie->ep, DCNT_EP_RECV);
+			DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_DATA, 
+				       cqe_p->byte_len);
 			return (OP_RECEIVE);
+		}
 	case IBV_WC_RECV_RDMA_WITH_IMM:
+		DAPL_CNTR(cookie->ep, DCNT_EP_RECV_RDMA_IMM);
+		DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_RDMA_IMM_DATA, 
+			       cqe_p->byte_len);
 		return (OP_RECEIVE_IMM);
 	default:
 		return (OP_INVALID);
diff --git a/dapl/openib_scm/dapl_ib_extensions.c b/dapl/openib_scm/dapl_ib_extensions.c
index 692c8a8..1d23ca3 100755
--- a/dapl/openib_scm/dapl_ib_extensions.c
+++ b/dapl/openib_scm/dapl_ib_extensions.c
@@ -96,8 +96,6 @@ dapl_extensions(IN DAT_HANDLE		dat_handle,
 		     "dapl_extensions(hdl %p operation %d, ...)\n",
 		     dat_handle, ext_op);
 
-	DAPL_CNTR(DCNT_EXTENSION);
-
 	switch ((int)ext_op)
 	{
 	
@@ -167,6 +165,39 @@ dapl_extensions(IN DAT_HANDLE		dat_handle,
 					OP_SEND_UD, comp_flags, ah);
     		break;
 
+#ifdef DAPL_COUNTERS
+	case DAT_QUERY_COUNTERS_OP:
+	{
+		int cntr, reset;
+		DAT_UINT64 * p_cntr_out;
+
+    		dapl_dbg_log(DAPL_DBG_TYPE_RTN, 
+			     " Query counter extension call\n");
+		
+		cntr = va_arg(args, int);
+		p_cntr_out = va_arg(args, DAT_UINT64*);
+		reset  = va_arg(args, int);
+				
+		status = dapl_query_counter(dat_handle, cntr, 
+					    p_cntr_out, reset);
+    		break;
+	}
+	case DAT_PRINT_COUNTERS_OP:
+	{
+		int cntr, reset;
+		
+    		dapl_dbg_log(DAPL_DBG_TYPE_RTN, 
+			     " Print counter extension call\n");
+		
+		cntr = va_arg(args, int);
+		reset  = va_arg(args, int);
+				
+		dapl_print_counter(dat_handle, cntr, reset);
+    		status = DAT_SUCCESS;
+		break;
+	}
+#endif /* DAPL_COUNTERS */
+
 	default:
     		dapl_dbg_log(DAPL_DBG_TYPE_ERR, 
 			     "unsupported extension(%d)\n", (int)ext_op);
diff --git a/dapl/openib_scm/dapl_ib_util.c b/dapl/openib_scm/dapl_ib_util.c
index d82d3f5..7623529 100644
--- a/dapl/openib_scm/dapl_ib_util.c
+++ b/dapl/openib_scm/dapl_ib_util.c
@@ -618,6 +618,11 @@ DAT_NAMED_ATTR  ib_attrs[] = {
     {
         DAT_IB_ATTR_UD, "TRUE"
     },
+#ifdef DAPL_COUNTERS
+    {
+        DAT_ATTR_COUNTERS, "TRUE"
+    },
+#endif /* DAPL_COUNTERS */
 #endif
 };
 
diff --git a/dapl/udapl/dapl_evd_create.c b/dapl/udapl/dapl_evd_create.c
index 7687a81..5ca5c22 100644
--- a/dapl/udapl/dapl_evd_create.c
+++ b/dapl/udapl/dapl_evd_create.c
@@ -95,7 +95,6 @@ DAT_RETURN DAT_API dapl_evd_create (
 		  cno_handle, 
 		  evd_flags, 
 		  evd_handle);
-    DAPL_CNTR(DCNT_EVD_CREATE);
 
     ia_ptr      = (DAPL_IA *)ia_handle;
     cno_ptr     = (DAPL_CNO *)cno_handle;
@@ -109,6 +108,8 @@ DAT_RETURN DAT_API dapl_evd_create (
 	goto bail;
     }
 
+    DAPL_CNTR(ia_ptr, DCNT_IA_EVD_CREATE);
+
     if ( evd_min_qlen <= 0 )
     {
 	dat_status = DAT_ERROR (DAT_INVALID_PARAMETER,DAT_INVALID_ARG2);
diff --git a/dapl/udapl/dapl_evd_wait.c b/dapl/udapl/dapl_evd_wait.c
index 578041a..9fc0ba2 100644
--- a/dapl/udapl/dapl_evd_wait.c
+++ b/dapl/udapl/dapl_evd_wait.c
@@ -85,7 +85,6 @@ DAT_RETURN DAT_API dapl_evd_wait (
 		  threshold, 
 		  event, 
 		  nmore);
-    DAPL_CNTR(DCNT_EVD_WAIT);
 
     evd_ptr = (DAPL_EVD *)evd_handle;
     dat_status = DAT_SUCCESS;
@@ -122,6 +121,7 @@ DAT_RETURN DAT_API dapl_evd_wait (
 	dat_status = DAT_ERROR (DAT_INVALID_STATE,0);
 	goto bail;
     }
+    DAPL_CNTR(evd_ptr, DCNT_EVD_WAIT);
 
     dapl_dbg_log (DAPL_DBG_TYPE_EVD, 
 	          "dapl_evd_wait: EVD %p, CQ %p\n", 
@@ -206,7 +206,7 @@ DAT_RETURN DAT_API dapl_evd_wait (
 		(evd_ptr->completion_type == DAPL_EVD_STATE_SOLICITED_WAIT) ?
 		     IB_NOTIFY_ON_SOLIC_COMP : IB_NOTIFY_ON_NEXT_COMP );  
 
-	    DAPL_CNTR(DCNT_EVD_WAIT_CMP_NTFY);
+	    DAPL_CNTR(evd_ptr, DCNT_EVD_WAIT_NOTIFY);
 	    /* FIXME report error */
 	    dapl_os_assert(dat_status == DAT_SUCCESS);
 
@@ -225,7 +225,7 @@ DAT_RETURN DAT_API dapl_evd_wait (
 	 */
 	evd_ptr->threshold = threshold;	
 
-	DAPL_CNTR(DCNT_EVD_WAIT_BLOCKED);
+	DAPL_CNTR(evd_ptr, DCNT_EVD_WAIT_BLOCKED);
 
 #ifdef CQ_WAIT_OBJECT
 	if (evd_ptr->cq_wait_obj_handle)
diff --git a/dapl/udapl/dapl_init.c b/dapl/udapl/dapl_init.c
index a4afba5..f19468b 100644
--- a/dapl/udapl/dapl_init.c
+++ b/dapl/udapl/dapl_init.c
@@ -115,8 +115,6 @@ void dapl_init ( void )
 	goto bail;
     }
 
-    DAPL_COUNTERS_INIT();
-
     return;
 
 bail:
diff --git a/dapl/udapl/dapl_lmr_create.c b/dapl/udapl/dapl_lmr_create.c
index 99b184a..dd93445 100644
--- a/dapl/udapl/dapl_lmr_create.c
+++ b/dapl/udapl/dapl_lmr_create.c
@@ -489,6 +489,8 @@ dapl_lmr_create (
     ia = (DAPL_IA *) ia_handle;
     pz = (DAPL_PZ *) pz_handle;
 
+    DAPL_CNTR(ia, DCNT_IA_LMR_CREATE);	
+
     switch (mem_type)
     {
 	case DAT_MEM_TYPE_VIRTUAL:
diff --git a/dapl/udapl/linux/dapl_osd.h b/dapl/udapl/linux/dapl_osd.h
index ae02944..974a0f0 100644
--- a/dapl/udapl/linux/dapl_osd.h
+++ b/dapl/udapl/linux/dapl_osd.h
@@ -539,10 +539,11 @@ DAT_RETURN dapl_os_get_time (DAPL_OS_TIMEVAL *);
  * ability to define 64 bit formats, which unfortunatly are non standard
  * in the C compiler world. E.g. %llx for gcc, %I64x for Windows
  */
-#define F64d   "%lld"
-#define F64u   "%llu"
-#define F64x   "%llx"
-#define F64X   "%llX"
+#include <inttypes.h>
+#define F64d   "%"PRId64
+#define F64u   "%"PRIu64
+#define F64x   "%"PRIx64
+#define F64X   "%"PRIX64
 
 
 /*
diff --git a/dat/include/dat2/dat_ib_extensions.h b/dat/include/dat2/dat_ib_extensions.h
index eb10714..2c082b1 100755
--- a/dat/include/dat2/dat_ib_extensions.h
+++ b/dat/include/dat2/dat_ib_extensions.h
@@ -58,7 +58,21 @@
  *	DAT_NAMED_ATTR	name == extended operations and version, 
  *			version_value = version number of extension API
  */
-#define DAT_IB_EXTENSION_VERSION	202	/* 2.0.2 */
+
+/* 2.0.1 - Initial IB extension support, atomic and immed data
+ *         dat_ib_post_fetch_and_add()
+ *         dat_ib_post_cmp_and_swap()
+ *         dat_ib_post_rdma_write_immed()
+ *		
+ * 2.0.2 - Add UD support, post send and remote_ah via connect events 
+ *         dat_ib_post_send_ud()
+ *
+ * 2.0.3 - Add query/print counter support for IA, EP, and EVD's 
+ *         dat_query_counters(), dat_print_counters()
+ *
+ */
+#define DAT_IB_EXTENSION_VERSION	203	/* 2.0.3 */
+#define DAT_ATTR_COUNTERS		"DAT_COUNTERS"
 #define DAT_IB_ATTR_FETCH_AND_ADD	"DAT_IB_FETCH_AND_ADD"
 #define DAT_IB_ATTR_CMP_AND_SWAP	"DAT_IB_CMP_AND_SWAP"
 #define DAT_IB_ATTR_IMMED_DATA		"DAT_IB_IMMED_DATA"
@@ -87,7 +101,9 @@ typedef enum dat_ib_op
 	DAT_IB_FETCH_AND_ADD_OP,
 	DAT_IB_CMP_AND_SWAP_OP,
 	DAT_IB_RDMA_WRITE_IMMED_OP,
-	DAT_IB_UD_SEND_OP
+	DAT_IB_UD_SEND_OP,
+	DAT_QUERY_COUNTERS_OP,
+	DAT_PRINT_COUNTERS_OP
 	
 } DAT_IB_OP;
 
@@ -247,6 +263,92 @@ typedef enum dat_ib_service_type
 
 } DAT_IB_SERVICE_TYPE;
 
+/*
+ * Definitions for 64-bit IA Counters
+ */
+typedef enum dat_ia_counters
+{
+	DCNT_IA_PZ_CREATE,
+	DCNT_IA_PZ_FREE,
+	DCNT_IA_LMR_CREATE,
+	DCNT_IA_LMR_FREE,
+	DCNT_IA_RMR_CREATE,
+	DCNT_IA_RMR_FREE,
+	DCNT_IA_PSP_CREATE,
+	DCNT_IA_PSP_CREATE_ANY,
+	DCNT_IA_PSP_FREE,
+	DCNT_IA_RSP_CREATE,
+	DCNT_IA_RSP_FREE,
+	DCNT_IA_EVD_CREATE,
+	DCNT_IA_EVD_FREE,
+	DCNT_IA_EP_CREATE,
+	DCNT_IA_EP_FREE,
+	DCNT_IA_SRQ_CREATE,
+	DCNT_IA_SRQ_FREE,
+	DCNT_IA_SP_CR,
+	DCNT_IA_SP_CR_ACCEPTED,
+	DCNT_IA_SP_CR_REJECTED,
+	DCNT_IA_MEM_ALLOC,
+	DCNT_IA_MEM_ALLOC_DATA,
+	DCNT_IA_MEM_FREE,
+	DCNT_IA_ASYNC_ERROR,
+	DCNT_IA_ASYNC_QP_ERROR,
+	DCNT_IA_ASYNC_CQ_ERROR,
+	DCNT_IA_ALL_COUNTERS,  /* MUST be last */
+
+} DAT_IA_COUNTERS;
+
+/*
+ * Definitions for 64-bit EP Counters
+ */
+typedef enum dat_ep_counters
+{
+	DCNT_EP_CONNECT,
+	DCNT_EP_DISCONNECT,
+	DCNT_EP_POST_SEND,
+	DCNT_EP_POST_SEND_DATA,
+	DCNT_EP_POST_SEND_UD,
+	DCNT_EP_POST_SEND_UD_DATA,
+	DCNT_EP_POST_RECV,
+	DCNT_EP_POST_RECV_DATA,
+	DCNT_EP_POST_WRITE,
+	DCNT_EP_POST_WRITE_DATA,
+	DCNT_EP_POST_WRITE_IMM,
+	DCNT_EP_POST_WRITE_IMM_DATA,
+	DCNT_EP_POST_READ,
+	DCNT_EP_POST_READ_DATA,
+	DCNT_EP_POST_CMP_SWAP,
+	DCNT_EP_POST_FETCH_ADD,
+	DCNT_EP_RECV,
+	DCNT_EP_RECV_DATA,
+	DCNT_EP_RECV_UD,
+	DCNT_EP_RECV_UD_DATA,
+	DCNT_EP_RECV_IMM,
+	DCNT_EP_RECV_IMM_DATA,
+	DCNT_EP_RECV_RDMA_IMM,
+	DCNT_EP_RECV_RDMA_IMM_DATA,
+	DCNT_EP_ALL_COUNTERS,  /* MUST be last */
+
+} DAT_EP_COUNTERS;
+
+/*
+ * Definitions for 64-bit EVD Counters
+ */
+typedef enum dat_evd_counters
+{
+	DCNT_EVD_WAIT,
+	DCNT_EVD_WAIT_BLOCKED,
+	DCNT_EVD_WAIT_NOTIFY,
+	DCNT_EVD_DEQUEUE,
+	DCNT_EVD_DEQUEUE_FOUND,
+	DCNT_EVD_DEQUEUE_NOT_FOUND,
+	DCNT_EVD_DEQUEUE_POLL,
+	DCNT_EVD_DEQUEUE_POLL_FOUND,
+	DCNT_EVD_CONN_CALLBACK,
+	DCNT_EVD_DTO_CALLBACK,
+	DCNT_EVD_ALL_COUNTERS,  /* MUST be last */
+
+} DAT_EVD_COUNTERS;
 
 /* Extended RETURN and EVENT STATUS string helper functions */
 
@@ -370,5 +472,32 @@ dat_strerror_ext_status (
 				(cookie), \
 				(flgs))
 
+
+/* 
+ * Query counter(s):  
+ * Provide IA, EP, or EVD and call will return appropriate counters
+ * 	DAT_HANDLE dat_handle, enum cntr, *DAT_UINT64 p_cntrs_out, int reset
+ *
+ * use _ALL_COUNTERS to query all
+ */
+#define dat_query_counters(dat_handle, cntr, p_cntrs_out, reset) \
+	     dat_extension_op(  dat_handle, \
+				DAT_QUERY_COUNTERS_OP, \
+				(cntr), \
+				(p_cntrs_out), \
+				(reset))
+/* 
+ * Print counter(s):  
+ * Provide IA, EP, or EVD and call will print appropriate counters
+ * 	DAT_HANDLE dat_handle, int cntr, int reset
+ * 
+ * use _ALL_COUNTERS to print all
+ */
+#define dat_print_counters(dat_handle, cntr, reset) \
+	     dat_extension_op(  dat_handle, \
+				DAT_PRINT_COUNTERS_OP, \
+				(cntr), \
+				(reset))
+
 #endif /* _DAT_IB_EXTENSIONS_H_ */
 
diff --git a/test/dtest/dtestx.c b/test/dtest/dtestx.c
index 032cadd..eafc1c9 100755
--- a/test/dtest/dtestx.c
+++ b/test/dtest/dtestx.c
@@ -60,6 +60,7 @@
 
 #define DAPL_PROVIDER "ofa-v2-ib0"
 #define F64x "%"PRIx64""
+#define F64u "%"PRIu64""
 
 #endif
 
@@ -74,6 +75,7 @@ int disconnect_ep(void);
 	if (status != DAT_SUCCESS) {\
 		dat_strerror(status, &maj_msg, &min_msg);\
 		fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg);\
+		dat_ia_close(ia, DAT_CLOSE_DEFAULT);\
 		exit(1);\
 	} else if (verbose) {\
 		printf("dtestx: %s success\n",str);\
@@ -86,6 +88,7 @@ int disconnect_ep(void);
 	if (status != DAT_SUCCESS) {\
 		dat_strerror(status, &maj_msg, &min_msg);\
 		fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg);\
+		dat_ia_close(ia, DAT_CLOSE_DEFAULT);\
 		exit(1);\
 	} else if (verbose) {\
 		printf("dtestx: %s\n",str);\
@@ -162,6 +165,7 @@ DAT_EVD_HANDLE		con_evd = DAT_HANDLE_NULL;
 DAT_EVD_HANDLE		dto_evd = DAT_HANDLE_NULL;
 DAT_PSP_HANDLE		psp = DAT_HANDLE_NULL;
 int			server = 1;
+int			remote_host = 0;
 int			ud_test = 0;
 int			multi_eps = 0;
 int			buf_size = BUF_SIZE;
@@ -171,6 +175,8 @@ char			hostname[256] = { 0 };
 DAT_IB_ADDR_HANDLE	remote_ah[MAX_EP_COUNT];
 int			eps = 1;
 int			verbose = 0;
+int			counters = 0;
+int			counters_ok = 0;
 
 #define LOGPRINTF if (verbose) printf
 
@@ -178,11 +184,12 @@ void print_usage(void)
 {
     printf("\n dtestx usage \n\n");
     printf("v: verbose\n");
+    printf("p: print counters\n");
     printf("u  unreliable datagram test\n");
     printf("U: unreliable datagram test, UD endpoint count\n");
     printf("m  unreliable datagram test, multiple Server endpoints\n");
     printf("b: buf length to allocate\n");
-    printf("h: hostname/address of Server, specified on Client\n");
+    printf("h: hostname/address of Server, client and UDP server\n");
     printf("c: Client\n");
     printf("s: Server, default\n");
     printf("P: provider name (default = ofa-v2-ib0)\n");
@@ -400,6 +407,7 @@ connect_ep(char *hostname)
 	DAT_RMR_TRIPLET		*r_iov;
 	DAT_DTO_COOKIE		cookie;
 	DAT_CONN_QUAL		conn_qual;
+	DAT_BOOLEAN		in,out;
 	int			i,ii,pdata,ctx;
 	DAT_PROVIDER_ATTR       prov_attrs;
         DAT_DTO_COMPLETION_EVENT_DATA *dto_event = 
@@ -418,6 +426,19 @@ connect_ep(char *hostname)
 		LOGPRINTF(" Provider Specific Attribute[%d] %s=%s\n",
 			  i, prov_attrs.provider_specific_attr[i].name,
 			  prov_attrs.provider_specific_attr[i].value);
+		
+		/* check for counter support */
+		status = strcmp(prov_attrs.provider_specific_attr[i].name,
+				"DAT_COUNTERS");
+		if (!status) 
+			counters_ok = 1;
+	}
+
+	/* make sure provider supports counters */
+	if ((counters) && (!counters_ok)) { 
+		printf("Disable dat_query_counters:"
+		       " Provider not built with counters\n");
+		counters = 0;
 	}
 	
 	status = dat_pz_create(ia, &pz);
@@ -632,6 +653,9 @@ connect_ep(char *hostname)
 		 cookie,
 		 DAT_COMPLETION_SUPPRESS_FLAG);
 
+	dat_ep_get_status(ep[0],NULL,&in,&out);
+	printf("EP[0] status: posted buffers: Req=%d, Rcv=%d\n",in,out);
+
 	/*
 	 *  Wait for their RMR
 	 */
@@ -730,9 +754,49 @@ disconnect_ep(void)
 		_OK2(status, "dat_lmr_free_atomic");
 	}
 	for (i=0;i<eps;i++) {
+		if (counters) { /* examples of query and print */
+			int ii;
+			DAT_UINT64 ep_cntrs[DCNT_EP_ALL_COUNTERS];
+
+			dat_query_counters(ep[i], DCNT_EP_ALL_COUNTERS, 
+					   ep_cntrs, 0);
+			printf(" EP[%d] Cntrs:",i);
+			for (ii=0;ii<DCNT_EP_ALL_COUNTERS;ii++)
+				printf(" "F64u"", ep_cntrs[ii]);
+			printf("\n");
+			dat_print_counters(ep[i],DCNT_EP_ALL_COUNTERS,0);
+		}
 		status = dat_ep_free(ep[i]);
 		_OK2(status, "dat_ep_free");
 	}
+	if (counters) { /* examples of query and print */
+		int ii;
+		DAT_UINT64 evd_cntrs[DCNT_EVD_ALL_COUNTERS];
+
+		dat_query_counters(dto_evd, DCNT_EVD_ALL_COUNTERS, 
+				   evd_cntrs, 0);
+		printf(" DTO_EVD Cntrs:");
+			for (ii=0;ii<DCNT_EVD_ALL_COUNTERS;ii++)
+				printf(" "F64u"", evd_cntrs[ii]);
+		printf("\n");
+		dat_print_counters(dto_evd,DCNT_EVD_ALL_COUNTERS,0);
+		
+		dat_query_counters(con_evd, DCNT_EVD_ALL_COUNTERS, 
+				   evd_cntrs, 0);
+		printf(" CONN_EVD Cntrs:");
+			for (ii=0;ii<DCNT_EVD_ALL_COUNTERS;ii++)
+				printf(" "F64u"", evd_cntrs[ii]);
+		printf("\n");
+		dat_print_counters(con_evd,DCNT_EVD_ALL_COUNTERS,0);
+		
+		dat_query_counters(cr_evd, DCNT_EVD_ALL_COUNTERS, 
+				   evd_cntrs, 0);
+		printf(" CR_EVD Cntrs:");
+			for (ii=0;ii<DCNT_EVD_ALL_COUNTERS;ii++)
+				printf(" "F64u"", evd_cntrs[ii]);
+		printf("\n");
+		dat_print_counters(cr_evd,DCNT_EVD_ALL_COUNTERS,0);
+	}
 	status = dat_evd_free(dto_evd);
 	_OK2(status, "dat_evd_free DTO");
 	
@@ -745,6 +809,17 @@ disconnect_ep(void)
 	status = dat_pz_free(pz);
 	_OK2(status, "dat_pz_free");
 
+	if (counters) { /* examples of query and print */
+		int ii;
+		DAT_UINT64 ia_cntrs[DCNT_IA_ALL_COUNTERS];
+
+		dat_query_counters(ia, DCNT_IA_ALL_COUNTERS, ia_cntrs, 0);
+		printf(" IA Cntrs:");
+			for (ii=0;ii<DCNT_IA_ALL_COUNTERS;ii++)
+				printf(" "F64u"", ia_cntrs[ii]);
+		printf("\n");
+		dat_print_counters(ia, DCNT_IA_ALL_COUNTERS, 0);
+}
 	status = dat_ia_close(ia, DAT_CLOSE_DEFAULT);
 	_OK2(status, "dat_ia_close");
 
@@ -1168,7 +1243,7 @@ main(int argc, char **argv)
 	int rc;
 	
 	/* parse arguments */
-	while ((rc = getopt(argc, argv, "csvumU:h:b:P:")) != -1)
+	while ((rc = getopt(argc, argv, "csvumpU:h:b:P:")) != -1)
 	{
 		switch(rc)
 		{
@@ -1185,8 +1260,11 @@ main(int argc, char **argv)
 			case 's':
 				server = 1;
 				break;
+			case 'p':
+				counters = 1;
+				break;
 			case 'h':
-				server = 0;
+				remote_host = 1;
 				strcpy (hostname, optarg);
 				break;
 			case 'b':
@@ -1222,6 +1300,9 @@ main(int argc, char **argv)
 		}
 	}
 #endif
+	/* for non UD tests, -h is always client */
+	if (remote_host && !ud_test)
+		server = 0;
 
 	if (!server) {
 		printf("\nRunning as Client - %s %s %d endpoint(s)\n",
@@ -1230,6 +1311,7 @@ main(int argc, char **argv)
 		printf("\nRunning as Server - %s %s %d endpoint(s)\n",
 			provider, ud_test?"UD test":"", eps);
 	}
+
 	/*
 	 * connect
 	 */
-- 
1.5.2.5





More information about the general mailing list