[ewg] [PATCH 2/5] openib: add new provider specific attributes
Davis, Arlin R
arlin.r.davis at intel.com
Thu Mar 13 10:04:57 PDT 2014
DAT_IB_PROVIDER_NAME = UCM/CMA/SCM
DAT_IB_DEVICE_NAME = ibv_get_device_name
DAT_IB_CONNECTIVITY_MODE = DIRECT/PROXY
DAT_IB_RDMA_READ = TRUE/FALSE
DAT_IB_NODE_GUID = xxxx:xxxx:xxxx:xxxx
DAT_IB_PORT_STATE = ibv_port_state_str
Signed-off-by: Arlin Davis <arlin.r.davis at intel.com>
---
dapl/openib_cma/dapl_ib_util.h | 4 ++-
dapl/openib_common/dapl_ib_common.h | 13 ++++++++-
dapl/openib_common/dapl_ib_dto.h | 10 +++++++
dapl/openib_common/util.c | 52 ++++++++++++++++++++++++++++-------
dapl/openib_scm/dapl_ib_util.h | 4 ++-
dapl/openib_ucm/dapl_ib_util.h | 4 ++-
test/dtest/dtest.c | 14 +++++++++-
7 files changed, 86 insertions(+), 15 deletions(-)
diff --git a/dapl/openib_cma/dapl_ib_util.h b/dapl/openib_cma/dapl_ib_util.h
index 454f7e1..de95485 100755
--- a/dapl/openib_cma/dapl_ib_util.h
+++ b/dapl/openib_cma/dapl_ib_util.h
@@ -120,10 +120,12 @@ typedef struct _ib_hca_transport
uint8_t hop_limit;
uint8_t tclass;
uint8_t mtu;
- DAT_NAMED_ATTR named_attr;
uint8_t sl;
uint16_t pkey;
int pkey_idx;
+ uint64_t guid;
+ char guid_str[32];
+ ib_named_attr_t na;
#ifdef DAT_IB_COLLECTIVES
/* Collective member device and address information */
ib_thread_state_t coll_thread_state;
diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h
index ba805d0..dfc80a9 100644
--- a/dapl/openib_common/dapl_ib_common.h
+++ b/dapl/openib_common/dapl_ib_common.h
@@ -109,6 +109,17 @@ typedef struct _ib_cm_msg
} ib_cm_msg_t;
+typedef struct _ib_named_attr
+{
+ const char *dev;
+ const char *mode;
+ const char *read;
+ const char *guid;
+ const char *mtu;
+ const char *port;
+
+} ib_named_attr_t;
+
/* CM events */
typedef enum {
IB_CME_CONNECTED,
@@ -304,7 +315,7 @@ int32_t dapls_ib_release(void);
/* util.c */
enum ibv_mtu dapl_ib_mtu(int mtu);
-char *dapl_ib_mtu_str(enum ibv_mtu mtu);
+const char *dapl_ib_mtu_str(enum ibv_mtu mtu);
int getipaddr_netdev(char *name, char *addr, int addr_len);
DAT_RETURN getlocalipaddr(char *addr, int addr_len);
diff --git a/dapl/openib_common/dapl_ib_dto.h b/dapl/openib_common/dapl_ib_dto.h
index b93565c..2bd6e7e 100644
--- a/dapl/openib_common/dapl_ib_dto.h
+++ b/dapl/openib_common/dapl_ib_dto.h
@@ -35,6 +35,16 @@
STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p);
+#if defined(_OPENIB_CMA_)
+#define PROVIDER_NAME "CMA"
+#elif defined(_OPENIB_UCM_)
+#define PROVIDER_NAME "UCM"
+#elif defined(_OPENIB_SCM_)
+#define PROVIDER_NAME "SCM"
+#else
+#define PROVIDER_NAME ""
+#endif
+
#define CQE_WR_TYPE_UD(id) \
(((DAPL_COOKIE *)(uintptr_t)id)->ep->qp_handle->qp_type == IBV_QPT_UD)
diff --git a/dapl/openib_common/util.c b/dapl/openib_common/util.c
index 20fb8b2..258d172 100644
--- a/dapl/openib_common/util.c
+++ b/dapl/openib_common/util.c
@@ -246,7 +246,7 @@ enum ibv_mtu dapl_ib_mtu(int mtu)
}
}
-char *dapl_ib_mtu_str(enum ibv_mtu mtu)
+const char *dapl_ib_mtu_str(enum ibv_mtu mtu)
{
switch (mtu) {
case IBV_MTU_256:
@@ -264,8 +264,6 @@ char *dapl_ib_mtu_str(enum ibv_mtu mtu)
}
}
-
-
/*
* dapls_ib_query_hca
*
@@ -377,10 +375,19 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr,
DAPL_MAX(dev_attr.local_ca_ack_delay,
hca_ptr->ib_trans.ack_timer);
- /* set MTU in transport specific named attribute */
- hca_ptr->ib_trans.named_attr.name = "DAT_IB_TRANSPORT_MTU";
- hca_ptr->ib_trans.named_attr.value =
- dapl_ib_mtu_str(hca_ptr->ib_trans.mtu);
+ /* set provider/transport specific named attributes */
+ hca_ptr->ib_trans.na.dev = ia_attr->adapter_name;
+ hca_ptr->ib_trans.na.mtu = dapl_ib_mtu_str(hca_ptr->ib_trans.mtu);
+ hca_ptr->ib_trans.na.port = ibv_port_state_str(port_attr.state);
+ hca_ptr->ib_trans.guid = ntohll(ibv_get_device_guid(hca_ptr->ib_trans.ib_dev));
+ sprintf(hca_ptr->ib_trans.guid_str, "%04x:%04x:%04x:%04x",
+ (unsigned) (hca_ptr->ib_trans.guid >> 48) & 0xffff,
+ (unsigned) (hca_ptr->ib_trans.guid >> 32) & 0xffff,
+ (unsigned) (hca_ptr->ib_trans.guid >> 16) & 0xffff,
+ (unsigned) (hca_ptr->ib_trans.guid >> 0) & 0xffff);
+ hca_ptr->ib_trans.na.guid = hca_ptr->ib_trans.guid_str;
+ hca_ptr->ib_trans.na.mode = "DIRECT";
+ hca_ptr->ib_trans.na.read = "TRUE";
if (hca_ptr->ib_hca_handle->device->transport_type != IBV_TRANSPORT_IB)
goto skip_ib;
@@ -635,7 +642,9 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca)
}
/*
- * dapls_set_provider_specific_attr
+ * dapls_query_provider_specific_attrs
+ *
+ * Common for openib providers: cma, ucm, scm, mcm
*
* Input:
* attr_ptr Pointer provider specific attributes
@@ -648,8 +657,26 @@ void dapli_async_event_cb(struct _ib_hca_transport *hca)
*/
DAT_NAMED_ATTR ib_attrs[] = {
{
+ "DAT_IB_PROVIDER_NAME", PROVIDER_NAME}
+ ,
+ {
+ "DAT_IB_DEVICE_NAME", "OFA_HCA_0000"}
+ ,
+ {
+ "DAT_IB_CONNECTIVITY_MODE", "DIRECT"}
+ ,
+ {
+ "DAT_IB_RDMA_READ", "TRUE"}
+ ,
+ {
+ "DAT_IB_NODE_GUID", "xxxx:xxxx:xxxx:xxxx"}
+ ,
+ {
"DAT_IB_TRANSPORT_MTU", "2048"}
,
+ {
+ "DAT_IB_PORT_STATUS", "UNKNOWN"}
+ ,
#ifdef DAT_EXTENSIONS
{
"DAT_EXTENSION_INTERFACE", "TRUE"}
@@ -704,8 +731,13 @@ void dapls_query_provider_specific_attr(IN DAPL_IA * ia_ptr,
attr_ptr->num_provider_specific_attr = SPEC_ATTR_SIZE(ib_attrs);
attr_ptr->provider_specific_attr = ib_attrs;
- /* set MTU to actual settings */
- ib_attrs[0].value = ia_ptr->hca_ptr->ib_trans.named_attr.value;
+ /* update common attributes from providers */
+ ib_attrs[1].value = ia_ptr->hca_ptr->ib_trans.na.dev;
+ ib_attrs[2].value = ia_ptr->hca_ptr->ib_trans.na.mode;
+ ib_attrs[3].value = ia_ptr->hca_ptr->ib_trans.na.read;
+ ib_attrs[4].value = ia_ptr->hca_ptr->ib_trans.na.guid;
+ ib_attrs[5].value = ia_ptr->hca_ptr->ib_trans.na.mtu;
+ ib_attrs[6].value = ia_ptr->hca_ptr->ib_trans.na.port;
}
/*
diff --git a/dapl/openib_scm/dapl_ib_util.h b/dapl/openib_scm/dapl_ib_util.h
index 2050c2c..d382226 100644
--- a/dapl/openib_scm/dapl_ib_util.h
+++ b/dapl/openib_scm/dapl_ib_util.h
@@ -106,11 +106,13 @@ typedef struct _ib_hca_transport
uint8_t hop_limit;
uint8_t tclass;
uint8_t mtu;
- DAT_NAMED_ATTR named_attr;
DAPL_SOCKET scm[2];
uint8_t sl;
uint16_t pkey;
int pkey_idx;
+ uint64_t guid;
+ char guid_str[32];
+ ib_named_attr_t na;
#ifdef DAT_IB_COLLECTIVES
/* Collective member device and address information */
ib_thread_state_t coll_thread_state;
diff --git a/dapl/openib_ucm/dapl_ib_util.h b/dapl/openib_ucm/dapl_ib_util.h
index 469560e..91c8466 100644
--- a/dapl/openib_ucm/dapl_ib_util.h
+++ b/dapl/openib_ucm/dapl_ib_util.h
@@ -93,7 +93,6 @@ typedef struct _ib_hca_transport
uint8_t hop_limit;
uint8_t tclass;
uint8_t mtu;
- DAT_NAMED_ATTR named_attr;
struct dapl_thread_signal signal;
int cqe;
int qpe;
@@ -121,6 +120,9 @@ typedef struct _ib_hca_transport
uint8_t sl;
uint16_t pkey;
int pkey_idx;
+ uint64_t guid;
+ char guid_str[32];
+ ib_named_attr_t na;
#ifdef DAT_IB_COLLECTIVES
/* Collective member device and address information */
ib_thread_state_t coll_thread_state;
diff --git a/test/dtest/dtest.c b/test/dtest/dtest.c
index d43a09b..8f4c80f 100755
--- a/test/dtest/dtest.c
+++ b/test/dtest/dtest.c
@@ -338,6 +338,7 @@ int main(int argc, char **argv)
DAT_RETURN ret;
DAT_EP_PARAM ep_param;
DAT_IA_ATTR ia_attr;
+ DAT_PROVIDER_ATTR pr_attr;
/* parse arguments */
while ((c = getopt(argc, argv, "tscvpq:l:b:d:B:h:P:")) != -1) {
@@ -445,7 +446,10 @@ int main(int argc, char **argv)
} else
LOGPRINTF("%d Opened Interface Adaptor\n", getpid());
- ret = dat_ia_query(h_ia, 0, DAT_IA_FIELD_ALL, &ia_attr, 0, 0);
+ ret = dat_ia_query(h_ia, 0,
+ DAT_IA_FIELD_ALL, &ia_attr,
+ DAT_PROVIDER_FIELD_PROVIDER_SPECIFIC_ATTR,
+ &pr_attr);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d: Error Adaptor query: %s\n",
getpid(), DT_RetToStr(ret));
@@ -453,6 +457,14 @@ int main(int argc, char **argv)
}
print_ia_address(ia_attr.ia_address_ptr);
+ /* Provider specific attributes */
+ for (i=0; i<pr_attr.num_provider_specific_attr; i++) {
+ LOGPRINTF("%d Provider_attr[%d] %s = %s \n",
+ getpid(), i,
+ pr_attr.provider_specific_attr[i].name,
+ pr_attr.provider_specific_attr[i].value);
+ }
+
/* Create Protection Zone */
start = get_time();
LOGPRINTF("%d Create Protection Zone\n", getpid());
--
1.7.3
More information about the ewg
mailing list