[ewg] [PATCH 2/5] openib: add new provider specific attributes

Davis, Arlin R arlin.r.davis at intel.com
Thu Mar 13 10:04:57 PDT 2014


DAT_IB_PROVIDER_NAME = UCM/CMA/SCM
DAT_IB_DEVICE_NAME = ibv_get_device_name
DAT_IB_CONNECTIVITY_MODE = DIRECT/PROXY
DAT_IB_RDMA_READ = TRUE/FALSE
DAT_IB_NODE_GUID = xxxx:xxxx:xxxx:xxxx
DAT_IB_PORT_STATE = ibv_port_state_str

Signed-off-by: Arlin Davis <arlin.r.davis at intel.com>
---
 dapl/openib_cma/dapl_ib_util.h      |    4 ++-
 dapl/openib_common/dapl_ib_common.h |   13 ++++++++-
 dapl/openib_common/dapl_ib_dto.h    |   10 +++++++
 dapl/openib_common/util.c           |   52 ++++++++++++++++++++++++++++-------
 dapl/openib_scm/dapl_ib_util.h      |    4 ++-
 dapl/openib_ucm/dapl_ib_util.h      |    4 ++-
 test/dtest/dtest.c                  |   14 +++++++++-
 7 files changed, 86 insertions(+), 15 deletions(-)

diff --git a/dapl/openib_cma/dapl_ib_util.h b/dapl/openib_cma/dapl_ib_util.h
index 454f7e1..de95485 100755
--- a/dapl/openib_cma/dapl_ib_util.h
+++ b/dapl/openib_cma/dapl_ib_util.h
@@ -120,10 +120,12 @@ typedef struct _ib_hca_transport
 	uint8_t			hop_limit;
 	uint8_t			tclass;
 	uint8_t			mtu;
-	DAT_NAMED_ATTR		named_attr;
 	uint8_t			sl;
 	uint16_t		pkey;
 	int			pkey_idx;
+	uint64_t		guid;
+	char			guid_str[32];
+	ib_named_attr_t		na;
 #ifdef DAT_IB_COLLECTIVES
 	/* Collective member device and address information */
 	ib_thread_state_t 	coll_thread_state;
diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h
index ba805d0..dfc80a9 100644
--- a/dapl/openib_common/dapl_ib_common.h
+++ b/dapl/openib_common/dapl_ib_common.h
@@ -109,6 +109,17 @@ typedef struct _ib_cm_msg
 
 } ib_cm_msg_t;
 
+typedef struct _ib_named_attr
+{
+	 const char *dev;
+	 const char *mode;
+	 const char *read;
+	 const char *guid;
+	 const char *mtu;
+	 const char *port;
+
+} ib_named_attr_t;
+
 /* CM events */
 typedef enum {
 	IB_CME_CONNECTED,
@@ -304,7 +315,7 @@ int32_t	dapls_ib_release(void);
 
 /* util.c */
 enum ibv_mtu dapl_ib_mtu(int mtu);
-char *dapl_ib_mtu_str(enum ibv_mtu mtu);
+const char *dapl_ib_mtu_str(enum ibv_mtu mtu);
 int getipaddr_netdev(char *name, char *addr, int addr_len);
 DAT_RETURN getlocalipaddr(char *addr, int addr_len);
 
diff --git a/dapl/openib_common/dapl_ib_dto.h b/dapl/openib_common/dapl_ib_dto.h
index b93565c..2bd6e7e 100644
--- a/dapl/openib_common/dapl_ib_dto.h
+++ b/dapl/openib_common/dapl_ib_dto.h
@@ -35,6 +35,16 @@
 
 STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p);
 
+#if defined(_OPENIB_CMA_)
+#define PROVIDER_NAME "CMA"
+#elif defined(_OPENIB_UCM_)
+#define PROVIDER_NAME "UCM"
+#elif defined(_OPENIB_SCM_)
+#define PROVIDER_NAME "SCM"
+#else
+#define PROVIDER_NAME ""
+#endif
+
 #define CQE_WR_TYPE_UD(id) \
 	(((DAPL_COOKIE *)(uintptr_t)id)->ep->qp_handle->qp_type == IBV_QPT_UD)
 
diff --git a/dapl/openib_common/util.c b/dapl/openib_common/util.c
index 20fb8b2..258d172 100644
--- a/dapl/openib_common/util.c
+++ b/dapl/openib_common/util.c
@@ -246,7 +246,7 @@ enum ibv_mtu dapl_ib_mtu(int mtu)
 	}
 }
 
-char *dapl_ib_mtu_str(enum ibv_mtu mtu)
+const char *dapl_ib_mtu_str(enum ibv_mtu mtu)
 {
 	switch (mtu) {
 	case IBV_MTU_256:
@@ -264,8 +264,6 @@ char *dapl_ib_mtu_str(enum ibv_mtu mtu)
 	}
 }
 
-
-
 /*
  * dapls_ib_query_hca
  *
@@ -377,10 +375,19 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr,
 		    DAPL_MAX(dev_attr.local_ca_ack_delay,
 			     hca_ptr->ib_trans.ack_timer);
 
-		/* set MTU in transport specific named attribute */
-		hca_ptr->ib_trans.named_attr.name = "DAT_IB_TRANSPORT_MTU";
-		hca_ptr->ib_trans.named_attr.value =
-		    dapl_ib_mtu_str(hca_ptr->ib_trans.mtu);
+		/* set provider/transport specific named attributes */
+		hca_ptr->ib_trans.na.dev = ia_attr->adapter_name;
+		hca_ptr->ib_trans.na.mtu = dapl_ib_mtu_str(hca_ptr->ib_trans.mtu);
+		hca_ptr->ib_trans.na.port = ibv_port_state_str(port_attr.state);
+		hca_ptr->ib_trans.guid = ntohll(ibv_get_device_guid(hca_ptr->ib_trans.ib_dev));
+		sprintf(hca_ptr->ib_trans.guid_str, "%04x:%04x:%04x:%04x",
+			(unsigned) (hca_ptr->ib_trans.guid >> 48) & 0xffff,
+			(unsigned) (hca_ptr->ib_trans.guid >> 32) & 0xffff,
+			(unsigned) (hca_ptr->ib_trans.guid >> 16) & 0xffff,
+			(unsigned) (hca_ptr->ib_trans.guid >>  0) & 0xffff);
+		hca_ptr->ib_trans.na.guid = hca_ptr->ib_trans.guid_str;
+		hca_ptr->ib_trans.na.mode = "DIRECT";
+		hca_ptr->ib_trans.na.read = "TRUE";
 
 		if (hca_ptr->ib_hca_handle->device->transport_type != IBV_TRANSPORT_IB)
 			goto skip_ib;
@@ -635,7 +642,9 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca)
 }
 
 /*
- * dapls_set_provider_specific_attr
+ * dapls_query_provider_specific_attrs
+ *
+ * Common for openib providers: cma, ucm, scm, mcm
  *
  * Input:
  *      attr_ptr        Pointer provider specific attributes
@@ -648,8 +657,26 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca)
  */
 DAT_NAMED_ATTR ib_attrs[] = {
 	{
+	 "DAT_IB_PROVIDER_NAME", PROVIDER_NAME}
+	,
+	{
+	 "DAT_IB_DEVICE_NAME", "OFA_HCA_0000"}
+	,
+	{
+	 "DAT_IB_CONNECTIVITY_MODE", "DIRECT"}
+	,
+	{
+	 "DAT_IB_RDMA_READ", "TRUE"}
+	,
+	{
+	 "DAT_IB_NODE_GUID", "xxxx:xxxx:xxxx:xxxx"}
+	,
+	{
 	 "DAT_IB_TRANSPORT_MTU", "2048"}
 	,
+	{
+	 "DAT_IB_PORT_STATUS", "UNKNOWN"}
+	,
 #ifdef DAT_EXTENSIONS
 	{
 	 "DAT_EXTENSION_INTERFACE", "TRUE"}
@@ -704,8 +731,13 @@ void dapls_query_provider_specific_attr(IN DAPL_IA * ia_ptr,
 	attr_ptr->num_provider_specific_attr = SPEC_ATTR_SIZE(ib_attrs);
 	attr_ptr->provider_specific_attr = ib_attrs;
 
-	/* set MTU to actual settings */
-	ib_attrs[0].value = ia_ptr->hca_ptr->ib_trans.named_attr.value;
+	/* update common attributes from providers */
+	ib_attrs[1].value = ia_ptr->hca_ptr->ib_trans.na.dev;
+	ib_attrs[2].value = ia_ptr->hca_ptr->ib_trans.na.mode;
+	ib_attrs[3].value = ia_ptr->hca_ptr->ib_trans.na.read;
+	ib_attrs[4].value = ia_ptr->hca_ptr->ib_trans.na.guid;
+	ib_attrs[5].value = ia_ptr->hca_ptr->ib_trans.na.mtu;
+	ib_attrs[6].value = ia_ptr->hca_ptr->ib_trans.na.port;
 }
 
 /*
diff --git a/dapl/openib_scm/dapl_ib_util.h b/dapl/openib_scm/dapl_ib_util.h
index 2050c2c..d382226 100644
--- a/dapl/openib_scm/dapl_ib_util.h
+++ b/dapl/openib_scm/dapl_ib_util.h
@@ -106,11 +106,13 @@ typedef struct _ib_hca_transport
 	uint8_t			hop_limit;
 	uint8_t			tclass;
 	uint8_t			mtu;
-	DAT_NAMED_ATTR		named_attr;
 	DAPL_SOCKET		scm[2];
 	uint8_t			sl;
 	uint16_t		pkey;
 	int			pkey_idx;
+	uint64_t		guid;
+	char 			guid_str[32];
+	ib_named_attr_t		na;
 #ifdef DAT_IB_COLLECTIVES
 	/* Collective member device and address information */
 	ib_thread_state_t 	coll_thread_state;
diff --git a/dapl/openib_ucm/dapl_ib_util.h b/dapl/openib_ucm/dapl_ib_util.h
index 469560e..91c8466 100644
--- a/dapl/openib_ucm/dapl_ib_util.h
+++ b/dapl/openib_ucm/dapl_ib_util.h
@@ -93,7 +93,6 @@ typedef struct _ib_hca_transport
 	uint8_t			hop_limit;
 	uint8_t			tclass;
 	uint8_t			mtu;
-	DAT_NAMED_ATTR		named_attr;
 	struct dapl_thread_signal signal;
 	int			cqe;
 	int			qpe;
@@ -121,6 +120,9 @@ typedef struct _ib_hca_transport
 	uint8_t			sl;
 	uint16_t		pkey;
 	int			pkey_idx;
+	uint64_t		guid;
+	char 			guid_str[32];
+	ib_named_attr_t		na;
 #ifdef DAT_IB_COLLECTIVES
 	/* Collective member device and address information */
 	ib_thread_state_t 	coll_thread_state;
diff --git a/test/dtest/dtest.c b/test/dtest/dtest.c
index d43a09b..8f4c80f 100755
--- a/test/dtest/dtest.c
+++ b/test/dtest/dtest.c
@@ -338,6 +338,7 @@ int main(int argc, char **argv)
 	DAT_RETURN ret;
 	DAT_EP_PARAM ep_param;
 	DAT_IA_ATTR ia_attr;
+	DAT_PROVIDER_ATTR pr_attr;
 
 	/* parse arguments */
 	while ((c = getopt(argc, argv, "tscvpq:l:b:d:B:h:P:")) != -1) {
@@ -445,7 +446,10 @@ int main(int argc, char **argv)
 	} else
 		LOGPRINTF("%d Opened Interface Adaptor\n", getpid());
 
-	ret = dat_ia_query(h_ia, 0, DAT_IA_FIELD_ALL, &ia_attr, 0, 0);
+	ret = dat_ia_query(h_ia, 0,
+			   DAT_IA_FIELD_ALL, &ia_attr,
+			   DAT_PROVIDER_FIELD_PROVIDER_SPECIFIC_ATTR,
+			   &pr_attr);
 	if (ret != DAT_SUCCESS) {
 		fprintf(stderr, "%d: Error Adaptor query: %s\n",
 			getpid(), DT_RetToStr(ret));
@@ -453,6 +457,14 @@ int main(int argc, char **argv)
 	}
 	print_ia_address(ia_attr.ia_address_ptr);
 
+	/* Provider specific attributes */
+	for (i=0; i<pr_attr.num_provider_specific_attr; i++) {
+		LOGPRINTF("%d Provider_attr[%d] %s = %s \n",
+			  getpid(), i,
+			  pr_attr.provider_specific_attr[i].name,
+			  pr_attr.provider_specific_attr[i].value);
+	}
+
 	/* Create Protection Zone */
 	start = get_time();
 	LOGPRINTF("%d Create Protection Zone\n", getpid());
-- 
1.7.3





More information about the ewg mailing list