[openib-general] [PATCH/RFC] change ibv_get_devices() to ibv_get_device_list()
Roland Dreier
rdreier at cisco.com
Fri Dec 9 17:22:32 PST 2005
This patch converts the ibv_get_devices() API to a better
ibv_get_device_list(). The old API was bad because it exposed the
dlist data structure exposed by libsysfs, which was not thread-safe
and was just plain overly complex for what it was used for.
In addition, I've converted over all the in-tree users of
ibv_get_devices() that I could find -- DAPL, libehca, libibcm,
librdmacm and mvapich.
I'm planning to commit this early next week; any objections, comments,
or suggestions before I do so?
Thanks,
Roland
dapl/dapl/openib/dapl_ib_util.c | 36 ++++++-----
dapl/dapl/openib_scm/dapl_ib_util.c | 35 +++++++----
libehca/configure.in | 2
libibcm/configure.in | 4 -
libibcm/examples/cmpost.c | 7 --
libibverbs/ChangeLog | 13 +++-
libibverbs/examples/asyncwatch.c | 16 +----
libibverbs/examples/device_list.c | 13 ++--
libibverbs/examples/devinfo.c | 101 +++++++++++++++-----------------
libibverbs/examples/rc_pingpong.c | 9 +-
libibverbs/examples/srq_pingpong.c | 9 +-
libibverbs/examples/uc_pingpong.c | 9 +-
libibverbs/examples/ud_pingpong.c | 9 +-
libibverbs/include/infiniband/verbs.h | 16 ++++-
libibverbs/src/device.c | 27 ++++++--
libibverbs/src/ibverbs.h | 11 +--
libibverbs/src/init.c | 72 +++++++++++++---------
libibverbs/src/libibverbs.map | 3
librdmacm/configure.in | 4 -
librdmacm/src/cma.c | 14 ++--
mpi/mvapich-gen2/mpid/ch_gen2/viainit.c | 11 +++
perftest/rdma_bw.c | 9 +-
perftest/rdma_lat.c | 9 +-
23 files changed, 255 insertions(+), 184 deletions(-)
--- userspace/libibverbs/include/infiniband/verbs.h (revision 4360)
+++ userspace/libibverbs/include/infiniband/verbs.h (working copy)
@@ -585,9 +585,21 @@ struct ibv_context {
};
/**
- * ibv_get_devices - Return list of IB devices
+ * ibv_get_device_list - Get list of IB devices currently available
+ * @num_devices: optional. if non-NULL, set to the number of devices
+ * returned in the array.
+ *
+ * Return a NULL-terminated array of IB devices. The array can be
+ * released with ibv_free_device_list().
+ */
+extern struct ibv_device **ibv_get_device_list(int *num_devices);
+
+/**
+ * ibv_free_device_list - Free list from ibv_get_device_list()
+ *
+ * Free an array of devices returned from ibv_get_device_list()
*/
-extern struct dlist *ibv_get_devices(void);
+extern void ibv_free_device_list(struct ibv_device **list);
/**
* ibv_get_device_name - Return kernel device name
--- userspace/libibverbs/ChangeLog (revision 4360)
+++ userspace/libibverbs/ChangeLog (working copy)
@@ -1,4 +1,15 @@
-2005-11-10 Sean Hefty <sean.hefty at intel.com>
+2005-11-11 Roland Dreier <roland at cisco.com>
+
+ * examples/asyncwatch.c, examples/rc_pingpong.c,
+ examples/srq_pingpong.c, examples/uc_pingpong.c,
+ examples/ud_pingpong.c, examples/device_list.c,
+ examples/devinfo.c: Update examples to match new API.
+
+ * include/infiniband/verbs.h, src/device.c, src/init.c,
+ src/ibverbs.h: Change from dlist-based ibv_get_devices() API to
+ simpler ibv_get_device_list() and ibv_free_device_list() API.
+
+2005-11-10 Sean Hefty <sean.hefty at intel.com>
* include/infiniband/sa-kern-abi.h: New include file to contain
definitions of SA structures passed between userspace and kernel.
--- userspace/libibverbs/src/libibverbs.map (revision 4360)
+++ userspace/libibverbs/src/libibverbs.map (working copy)
@@ -1,6 +1,7 @@
IBVERBS_1.0 {
global:
- ibv_get_devices;
+ ibv_get_device_list;
+ ibv_free_device_list;
ibv_get_device_name;
ibv_get_device_guid;
ibv_open_device;
--- userspace/libibverbs/src/device.c (revision 4360)
+++ userspace/libibverbs/src/device.c (working copy)
@@ -49,21 +49,36 @@
#include "ibverbs.h"
static pthread_mutex_t device_list_lock = PTHREAD_MUTEX_INITIALIZER;
-static struct dlist *device_list;
+static int num_devices;
+static struct ibv_device **device_list;
-struct dlist *ibv_get_devices(void)
+struct ibv_device **ibv_get_device_list(int *num)
{
- struct dlist *l;
+ struct ibv_device **l;
+ int i;
pthread_mutex_lock(&device_list_lock);
- if (!device_list)
- device_list = ibverbs_init();
- l = device_list;
+
+ if (!num_devices)
+ num_devices = ibverbs_init(&device_list);
+
+ l = calloc(num_devices, sizeof (struct ibv_device *));
+ for (i = 0; i < num_devices; ++i)
+ l[i] = device_list[i];
+
pthread_mutex_unlock(&device_list_lock);
+ if (num)
+ *num = l ? num_devices : 0;
+
return l;
}
+void ibv_free_device_list(struct ibv_device **list)
+{
+ free(list);
+}
+
const char *ibv_get_device_name(struct ibv_device *device)
{
return device->ibdev->name;
--- userspace/libibverbs/src/ibverbs.h (revision 4360)
+++ userspace/libibverbs/src/ibverbs.h (working copy)
@@ -47,7 +47,8 @@
#define PFX "libibverbs: "
struct ibv_driver {
- ibv_driver_init_func init_func;
+ ibv_driver_init_func init_func;
+ struct ibv_driver *next;
};
struct ibv_abi_compat_v2 {
@@ -57,11 +58,11 @@ struct ibv_abi_compat_v2 {
extern HIDDEN int abi_ver;
-extern struct dlist *ibverbs_init(void);
+extern HIDDEN int ibverbs_init(struct ibv_device ***list);
-extern int ibv_init_mem_map(void);
-extern int ibv_lock_range(void *base, size_t size);
-extern int ibv_unlock_range(void *base, size_t size);
+extern HIDDEN int ibv_init_mem_map(void);
+extern HIDDEN int ibv_lock_range(void *base, size_t size);
+extern HIDDEN int ibv_unlock_range(void *base, size_t size);
#define IBV_INIT_CMD(cmd, size, opcode) \
do { \
--- userspace/libibverbs/src/init.c (revision 4360)
+++ userspace/libibverbs/src/init.c (working copy)
@@ -55,7 +55,7 @@ HIDDEN int abi_ver;
static char default_path[] = DRIVER_PATH;
static const char *user_path;
-static struct dlist *driver_list;
+static struct ibv_driver *driver_list;
static void load_driver(char *so_path)
{
@@ -82,7 +82,8 @@ static void load_driver(char *so_path)
}
driver->init_func = init_func;
- dlist_push(driver_list, driver);
+ driver->next = driver_list;
+ driver_list = driver;
}
static void find_drivers(char *dir)
@@ -112,8 +113,7 @@ static void find_drivers(char *dir)
load_driver(so_glob.gl_pathv[i]);
}
-static void init_drivers(struct sysfs_class_device *verbs_dev,
- struct dlist *device_list)
+static struct ibv_device *init_drivers(struct sysfs_class_device *verbs_dev)
{
struct sysfs_class_device *ib_dev;
struct sysfs_attribute *attr;
@@ -125,7 +125,7 @@ static void init_drivers(struct sysfs_cl
if (!attr) {
fprintf(stderr, PFX "Warning: no ibdev class attr for %s\n",
verbs_dev->name);
- return;
+ return NULL;
}
sscanf(attr->value, "%63s", ibdev_name);
@@ -134,19 +134,17 @@ static void init_drivers(struct sysfs_cl
if (!ib_dev) {
fprintf(stderr, PFX "Warning: no infiniband class device %s for %s\n",
attr->value, verbs_dev->name);
- return;
+ return NULL;
}
- dlist_for_each_data(driver_list, driver, struct ibv_driver) {
+ for (driver = driver_list; driver; driver = driver->next) {
dev = driver->init_func(verbs_dev);
if (dev) {
dev->dev = verbs_dev;
dev->ibdev = ib_dev;
dev->driver = driver;
- dlist_push(device_list, dev);
-
- return;
+ return dev;
}
}
@@ -155,6 +153,8 @@ static void init_drivers(struct sysfs_cl
if (user_path)
fprintf(stderr, "%s:", user_path);
fprintf(stderr, "%s\n", default_path);
+
+ return NULL;
}
static int check_abi_version(void)
@@ -188,28 +188,23 @@ static int check_abi_version(void)
}
-struct dlist *ibverbs_init(void)
+HIDDEN int ibverbs_init(struct ibv_device ***list)
{
char *wr_path, *dir;
struct sysfs_class *cls;
struct dlist *verbs_dev_list;
- struct dlist *device_list;
struct sysfs_class_device *verbs_dev;
+ struct ibv_device *device;
+ struct ibv_device **new_list;
+ int num_devices = 0;
+ int list_size = 0;
- driver_list = dlist_new(sizeof (struct ibv_driver));
- device_list = dlist_new(sizeof (struct ibv_device));
- if (!driver_list || !device_list) {
- fprintf(stderr, PFX "Fatal: couldn't allocate device/driver list.\n");
- abort();
- }
+ *list = NULL;
if (ibv_init_mem_map())
- return NULL;
+ return 0;
- /*
- * Check if a driver is statically linked, and if so load it first.
- */
- load_driver(NULL);
+ find_drivers(default_path);
/*
* Only follow the path passed in through the calling user's
@@ -224,25 +219,42 @@ struct dlist *ibverbs_init(void)
}
}
- find_drivers(default_path);
+ /*
+ * Now check if a driver is statically linked. Since we push
+ * drivers onto our driver list, the last driver we find will
+ * be the first one we try.
+ */
+ load_driver(NULL);
cls = sysfs_open_class("infiniband_verbs");
if (!cls) {
fprintf(stderr, PFX "Fatal: couldn't open sysfs class 'infiniband_verbs'.\n");
- return NULL;
+ return 0;
}
if (check_abi_version())
- return NULL;
+ return 0;
verbs_dev_list = sysfs_get_class_devices(cls);
if (!verbs_dev_list) {
fprintf(stderr, PFX "Fatal: no infiniband class devices found.\n");
- return NULL;
+ return 0;
}
- dlist_for_each_data(verbs_dev_list, verbs_dev, struct sysfs_class_device)
- init_drivers(verbs_dev, device_list);
+ dlist_for_each_data(verbs_dev_list, verbs_dev, struct sysfs_class_device) {
+ device = init_drivers(verbs_dev);
+ if (device) {
+ if (list_size <= num_devices) {
+ list_size = list_size ? list_size * 2 : 1;
+ new_list = realloc(*list, list_size * sizeof (struct ibv_device *));
+ if (!new_list)
+ goto out;
+ *list = new_list;
+ }
+ *list[num_devices++] = device;
+ }
+ }
- return device_list;
+out:
+ return num_devices;
}
--- userspace/libibverbs/examples/asyncwatch.c (revision 4360)
+++ userspace/libibverbs/examples/asyncwatch.c (working copy)
@@ -50,34 +50,30 @@ static inline uint64_t be64_to_cpu(uint6
int main(int argc, char *argv[])
{
- struct dlist *dev_list;
- struct ibv_device *ib_dev;
+ struct ibv_device **dev_list;
struct ibv_context *context;
struct ibv_async_event event;
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
- dlist_start(dev_list);
- ib_dev = dlist_next(dev_list);
-
- if (!ib_dev) {
+ if (!*dev_list) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
- context = ibv_open_device(ib_dev);
+ context = ibv_open_device(*dev_list);
if (!context) {
fprintf(stderr, "Couldn't get context for %s\n",
- ibv_get_device_name(ib_dev));
+ ibv_get_device_name(*dev_list));
return 1;
}
printf("%s: async event FD %d\n",
- ibv_get_device_name(ib_dev), context->async_fd);
+ ibv_get_device_name(*dev_list), context->async_fd);
while (1) {
if (ibv_get_async_event(context, &event))
--- userspace/libibverbs/examples/rc_pingpong.c (revision 4360)
+++ userspace/libibverbs/examples/rc_pingpong.c (working copy)
@@ -447,7 +447,7 @@ static void usage(const char *argv0)
int main(int argc, char *argv[])
{
- struct dlist *dev_list;
+ struct ibv_device **dev_list;
struct ibv_device *ib_dev;
struct pingpong_context *ctx;
struct pingpong_dest my_dest;
@@ -536,21 +536,20 @@ int main(int argc, char *argv[])
page_size = sysconf(_SC_PAGESIZE);
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
- dlist_start(dev_list);
if (!ib_devname) {
- ib_dev = dlist_next(dev_list);
+ ib_dev = *dev_list;
if (!ib_dev) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
} else {
- dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
+ for (ib_dev = *dev_list; ib_dev; ++dev_list)
if (!strcmp(ibv_get_device_name(ib_dev), ib_devname))
break;
if (!ib_dev) {
--- userspace/libibverbs/examples/srq_pingpong.c (revision 4360)
+++ userspace/libibverbs/examples/srq_pingpong.c (working copy)
@@ -509,7 +509,7 @@ static void usage(const char *argv0)
int main(int argc, char *argv[])
{
- struct dlist *dev_list;
+ struct ibv_device **dev_list;
struct ibv_device *ib_dev;
struct pingpong_context *ctx;
struct pingpong_dest my_dest[MAX_QP];
@@ -605,21 +605,20 @@ int main(int argc, char *argv[])
page_size = sysconf(_SC_PAGESIZE);
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
- dlist_start(dev_list);
if (!ib_devname) {
- ib_dev = dlist_next(dev_list);
+ ib_dev = *dev_list;
if (!ib_dev) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
} else {
- dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
+ for (ib_dev = *dev_list; ib_dev; ++dev_list)
if (!strcmp(ibv_get_device_name(ib_dev), ib_devname))
break;
if (!ib_dev) {
--- userspace/libibverbs/examples/uc_pingpong.c (revision 4360)
+++ userspace/libibverbs/examples/uc_pingpong.c (working copy)
@@ -435,7 +435,7 @@ static void usage(const char *argv0)
int main(int argc, char *argv[])
{
- struct dlist *dev_list;
+ struct ibv_device **dev_list;
struct ibv_device *ib_dev;
struct pingpong_context *ctx;
struct pingpong_dest my_dest;
@@ -524,21 +524,20 @@ int main(int argc, char *argv[])
page_size = sysconf(_SC_PAGESIZE);
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
- dlist_start(dev_list);
if (!ib_devname) {
- ib_dev = dlist_next(dev_list);
+ ib_dev = *dev_list;
if (!ib_dev) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
} else {
- dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
+ for (ib_dev = *dev_list; ib_dev; ++dev_list)
if (!strcmp(ibv_get_device_name(ib_dev), ib_devname))
break;
if (!ib_dev) {
--- userspace/libibverbs/examples/ud_pingpong.c (revision 4360)
+++ userspace/libibverbs/examples/ud_pingpong.c (working copy)
@@ -443,7 +443,7 @@ static void usage(const char *argv0)
int main(int argc, char *argv[])
{
- struct dlist *dev_list;
+ struct ibv_device **dev_list;
struct ibv_device *ib_dev;
struct pingpong_context *ctx;
struct pingpong_dest my_dest;
@@ -532,21 +532,20 @@ int main(int argc, char *argv[])
page_size = sysconf(_SC_PAGESIZE);
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
- dlist_start(dev_list);
if (!ib_devname) {
- ib_dev = dlist_next(dev_list);
+ ib_dev = *dev_list;
if (!ib_dev) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
} else {
- dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
+ for (ib_dev = *dev_list; ib_dev; ++dev_list)
if (!strcmp(ibv_get_device_name(ib_dev), ib_devname))
break;
if (!ib_dev) {
--- userspace/libibverbs/examples/device_list.c (revision 4360)
+++ userspace/libibverbs/examples/device_list.c (working copy)
@@ -51,10 +51,9 @@ static inline uint64_t be64_to_cpu(uint6
int main(int argc, char *argv[])
{
- struct dlist *dev_list;
- struct ibv_device *ib_dev;
+ struct ibv_device **dev_list;
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
fprintf(stderr, "No IB devices found\n");
return 1;
@@ -63,10 +62,12 @@ int main(int argc, char *argv[])
printf(" %-16s\t node GUID\n", "device");
printf(" %-16s\t----------------\n", "------");
- dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
+ while (*dev_list) {
printf(" %-16s\t%016llx\n",
- ibv_get_device_name(ib_dev),
- (unsigned long long) be64_to_cpu(ibv_get_device_guid(ib_dev)));
+ ibv_get_device_name(*dev_list),
+ (unsigned long long) be64_to_cpu(ibv_get_device_guid(*dev_list)));
+ ++dev_list;
+ }
return 0;
}
--- userspace/libibverbs/examples/devinfo.c (revision 4360)
+++ userspace/libibverbs/examples/devinfo.c (working copy)
@@ -299,11 +299,11 @@ cleanup:
static void usage(const char *argv0)
{
- printf("Usage: %s print the ca attributes\n", argv0);
- printf("\n");
- printf("Options:\n");
- printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
- printf(" -i, --ib-port=<port> use port <port> of IB device (default all ports)\n");
+ printf("Usage: %s print the ca attributes\n", argv0);
+ printf("\n");
+ printf("Options:\n");
+ printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
+ printf(" -i, --ib-port=<port> use port <port> of IB device (default all ports)\n");
printf(" -l, --list print only the IB devices names\n");
printf(" -v, --verbose print all the attributes of the IB device(s)\n");
}
@@ -312,60 +312,56 @@ int main(int argc, char *argv[])
{
char *ib_devname = NULL;
int ret = 0;
- struct dlist *dev_list;
- struct ibv_device *ib_dev;
+ struct ibv_device **dev_list;
int num_of_hcas;
int ib_port = 0;
/* parse command line options */
while (1) {
int c;
- static struct option long_options[] = {
- { .name = "ib-dev", .has_arg = 1, .val = 'd' },
- { .name = "ib-port", .has_arg = 1, .val = 'i' },
+ static struct option long_options[] = {
+ { .name = "ib-dev", .has_arg = 1, .val = 'd' },
+ { .name = "ib-port", .has_arg = 1, .val = 'i' },
{ .name = "list", .has_arg = 0, .val = 'l' },
- { .name = "verbose", .has_arg = 0, .val = 'v' },
- { 0, 0, 0, 0}
- };
+ { .name = "verbose", .has_arg = 0, .val = 'v' },
+ { 0, 0, 0, 0}
+ };
- c = getopt_long(argc, argv, "d:i:lv", long_options, NULL);
- if (c == -1)
- break;
-
- switch (c) {
- case 'd':
- ib_devname = strdup(optarg);
- break;
-
- case 'i':
- ib_port = strtol(optarg, NULL, 0);
- if (ib_port < 0) {
- usage(argv[0]);
- return 1;
- }
- break;
+ c = getopt_long(argc, argv, "d:i:lv", long_options, NULL);
+ if (c == -1)
+ break;
+
+ switch (c) {
+ case 'd':
+ ib_devname = strdup(optarg);
+ break;
+
+ case 'i':
+ ib_port = strtol(optarg, NULL, 0);
+ if (ib_port < 0) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
case 'v':
- verbose = 1;
- break;
+ verbose = 1;
+ break;
case 'l':
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(&num_of_hcas);
if (!dev_list) {
fprintf(stderr, "Failed to get IB devices list");
return -1;
}
- num_of_hcas = 0;
- dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
- num_of_hcas ++;
-
printf("%d HCA%s found:\n", num_of_hcas,
num_of_hcas != 1 ? "s" : "");
- dlist_start(dev_list);
- dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
- printf("\t%s\n", ibv_get_device_name(ib_dev));
+ while (*dev_list) {
+ printf("\t%s\n", ibv_get_device_name(*dev_list));
+ ++dev_list;
+ }
printf("\n");
return 0;
@@ -376,28 +372,31 @@ int main(int argc, char *argv[])
}
}
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
fprintf(stderr, "Failed to get IB device list\n");
return -1;
}
- dlist_start(dev_list);
+
if (ib_devname) {
- dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
- if (!strcmp(ibv_get_device_name(ib_dev), ib_devname))
+ while (*dev_list) {
+ if (!strcmp(ibv_get_device_name(*dev_list), ib_devname))
break;
- if (!ib_dev) {
+ ++dev_list;
+ }
+
+ if (!*dev_list) {
fprintf(stderr, "IB device '%s' wasn't found\n", ib_devname);
return -1;
}
- ret |= print_hca_cap(ib_dev, ib_port);
+
+ ret |= print_hca_cap(*dev_list, ib_port);
} else {
- ib_dev = dlist_next(dev_list);
- if (!ib_dev) {
- fprintf(stderr, "No IB devices found\n");
- return -1;
- }
- ret |= print_hca_cap(ib_dev, ib_port);
+ if (!*dev_list) {
+ fprintf(stderr, "No IB devices found\n");
+ return -1;
+ }
+ ret |= print_hca_cap(*dev_list, ib_port);
}
if (ib_devname)
--- userspace/dapl/dapl/openib/dapl_ib_util.c (revision 4360)
+++ userspace/dapl/dapl/openib/dapl_ib_util.c (working copy)
@@ -206,29 +206,34 @@ DAT_RETURN dapls_ib_open_hca (
IN IB_HCA_NAME hca_name,
IN DAPL_HCA *hca_ptr)
{
- struct dlist *dev_list;
+ struct ibv_device **dev_list;
long opts;
+ int i;
dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
" open_hca: %s - %p\n", hca_name, hca_ptr );
/* Get list of all IB devices, find match, open */
- dev_list = ibv_get_devices();
- dlist_start(dev_list);
- dlist_for_each_data(dev_list,
- hca_ptr->ib_trans.ib_dev,
- struct ibv_device) {
- if (!strcmp(ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
- hca_name))
- break;
- }
-
- if (!hca_ptr->ib_trans.ib_dev) {
+ dev_list = ibv_get_device_list(NULL);
+ if (!dev_list) {
dapl_dbg_log (DAPL_DBG_TYPE_ERR,
- " open_hca: IB device %s not found\n",
+ " open_hca: ibv_get_device_list() failed\n",
hca_name);
return DAT_INTERNAL_ERROR;
}
+
+ for (i = 0; dev_list[i]; ++i) {
+ hca_ptr->ib_trans.ib_dev = dev_list[i];
+ if (!strcmp(ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
+ hca_name))
+ goto found;
+ }
+
+ dapl_dbg_log (DAPL_DBG_TYPE_ERR,
+ " open_hca: IB device %s not found\n",
+ hca_name);
+ goto err;
+found:
dapl_dbg_log (
DAPL_DBG_TYPE_UTIL," open_hca: Found dev %s %016llx\n",
ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
@@ -240,7 +245,7 @@ DAT_RETURN dapls_ib_open_hca (
dapl_dbg_log (DAPL_DBG_TYPE_ERR,
" open_hca: IB dev open failed for %s\n",
ibv_get_device_name(hca_ptr->ib_trans.ib_dev));
- return DAT_INTERNAL_ERROR;
+ goto err;
}
hca_ptr->ib_trans.ib_ctx = hca_ptr->ib_hca_handle;
@@ -336,11 +341,14 @@ DAT_RETURN dapls_ib_open_hca (
hca_ptr->ib_trans.max_inline_send );
hca_ptr->ib_trans.d_hca = hca_ptr;
+ ibv_free_device_list(dev_list);
return DAT_SUCCESS;
bail:
ibv_close_device(hca_ptr->ib_hca_handle);
hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
+err:
+ ibv_free_device_list(dev_list);
return DAT_INTERNAL_ERROR;
}
--- userspace/dapl/dapl/openib_scm/dapl_ib_util.c (revision 4360)
+++ userspace/dapl/dapl/openib_scm/dapl_ib_util.c (working copy)
@@ -131,28 +131,35 @@ DAT_RETURN dapls_ib_open_hca (
IN IB_HCA_NAME hca_name,
IN DAPL_HCA *hca_ptr)
{
- struct dlist *dev_list;
+ struct ibv_device **dev_list;
int opts;
+ int i;
DAT_RETURN dat_status = DAT_SUCCESS;
dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
" open_hca: %s - %p\n", hca_name, hca_ptr );
/* Get list of all IB devices, find match, open */
- dev_list = ibv_get_devices();
- dlist_start(dev_list);
- dlist_for_each_data(dev_list,hca_ptr->ib_trans.ib_dev,struct ibv_device) {
- if (!strcmp(ibv_get_device_name(hca_ptr->ib_trans.ib_dev),hca_name))
- break;
- }
-
- if (!hca_ptr->ib_trans.ib_dev) {
+ dev_list = ibv_get_device_list(NULL);
+ if (!dev_list) {
dapl_dbg_log (DAPL_DBG_TYPE_ERR,
- " open_hca: IB device %s not found\n",
+ " open_hca: ibv_get_device_list() failed\n",
hca_name);
return DAT_INTERNAL_ERROR;
}
-
+
+ for (i = 0; dev_list[i]; ++i) {
+ hca_ptr->ib_trans.ib_dev = dev_list[i];
+ if (!strcmp(ibv_get_device_name(hca_ptr->ib_trans.ib_dev),hca_name))
+ goto found;
+ }
+
+ dapl_dbg_log (DAPL_DBG_TYPE_ERR,
+ " open_hca: IB device %s not found\n",
+ hca_name);
+ goto err;
+
+found:
dapl_dbg_log (DAPL_DBG_TYPE_UTIL," open_hca: Found dev %s %016llx\n",
ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
(unsigned long long)bswap_64(ibv_get_device_guid(hca_ptr->ib_trans.ib_dev)));
@@ -162,7 +169,7 @@ DAT_RETURN dapls_ib_open_hca (
dapl_dbg_log (DAPL_DBG_TYPE_ERR,
" open_hca: IB dev open failed for %s\n",
ibv_get_device_name(hca_ptr->ib_trans.ib_dev) );
- return DAT_INTERNAL_ERROR;
+ goto err;
}
/* set inline max with enviroment or default */
@@ -242,10 +249,14 @@ DAT_RETURN dapls_ib_open_hca (
((struct sockaddr_in *)&hca_ptr->hca_address)->sin_addr.s_addr >> 16 & 0xff,
((struct sockaddr_in *)&hca_ptr->hca_address)->sin_addr.s_addr >> 24 & 0xff );
+ ibv_free_device_list(dev_list);
return dat_status;
+
bail:
ibv_close_device(hca_ptr->ib_hca_handle);
hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
+err:
+ ibv_free_device_list(dev_list);
return DAT_INTERNAL_ERROR;
}
--- userspace/mpi/mvapich-gen2/mpid/ch_gen2/viainit.c (revision 4360)
+++ userspace/mpi/mvapich-gen2/mpid/ch_gen2/viainit.c (working copy)
@@ -74,13 +74,22 @@ static void set_malloc_options(void)
static void open_hca(void)
{
- struct dlist *dev_list;
struct ibv_device *ib_dev = NULL;
+#ifdef GEN2_OLD_DEVICE_LIST_VERB
+ struct dlist *dev_list;
+
dev_list = ibv_get_devices();
dlist_start(dev_list);
ib_dev = dlist_next(dev_list);
+#else
+ struct ibv_device **dev_list;
+
+ dev_list = ibv_get_device_list(NULL);
+ ib_dev = dev_list[0];
+ ibv_free_device_list(dev_list);
+#endif
if (!ib_dev) {
fprintf(stderr, "No IB devices found\n");
--- userspace/libehca/configure.in (revision 4360)
+++ userspace/libehca/configure.in (working copy)
@@ -12,7 +12,7 @@ AC_HEADER_STDC
dnl Checks for libraries.
AC_CHECK_LIB(ibverbs,
- ibv_get_devices,
+ ibv_get_device_list,
[],
AC_MSG_ERROR([libibverbs not installed]))
--- userspace/librdmacm/configure.in (revision 4360)
+++ userspace/librdmacm/configure.in (working copy)
@@ -25,8 +25,8 @@ AC_CHECK_SIZEOF(long)
dnl Checks for libraries
if test "$disable_libcheck" != "yes"
then
-AC_CHECK_LIB(ibverbs, ibv_get_devices, [],
- AC_MSG_ERROR([ibv_get_devices() not found. librdmacm requires libibverbs.]))
+AC_CHECK_LIB(ibverbs, ibv_get_device_list, [],
+ AC_MSG_ERROR([ibv_get_device_list() not found. librdmacm requires libibverbs.]))
fi
dnl Checks for header files.
--- userspace/librdmacm/src/cma.c (revision 4360)
+++ userspace/librdmacm/src/cma.c (working copy)
@@ -114,7 +114,7 @@ struct cma_id_private {
uint32_t handle;
};
-static struct dlist *dev_list;
+static struct ibv_device **dev_list;
static struct dlist *cma_dev_list;
static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
static int ucma_initialized;
@@ -141,7 +141,7 @@ static void ucma_cleanup(void)
static int ucma_init(void)
{
- struct ibv_device *dev;
+ int i;
struct cma_device *cma_dev;
struct ibv_device_attr attr;
int ret;
@@ -163,22 +163,22 @@ static int ucma_init(void)
goto err;
}
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
printf("CMA: unable to get RDMA device liste\n");
ret = -ENODEV;
goto err;
}
- dlist_for_each_data(dev_list, dev, struct ibv_device) {
+ for (i = 0; dev_list[i]; ++i) {
cma_dev = malloc(sizeof *cma_dev);
if (!cma_dev) {
ret = -ENOMEM;
goto err;
}
- cma_dev->guid = ibv_get_device_guid(dev);
- cma_dev->verbs = ibv_open_device(dev);
+ cma_dev->guid = ibv_get_device_guid(dev_list[i]);
+ cma_dev->verbs = ibv_open_device(dev_list[i]);
if (!cma_dev->verbs) {
printf("CMA: unable to open RDMA device\n");
ret = -ENODEV;
@@ -201,6 +201,8 @@ out:
err:
ucma_cleanup();
pthread_mutex_unlock(&mut);
+ if (dev_list)
+ ibv_free_device_list(dev_list);
return ret;
}
--- userspace/perftest/rdma_lat.c (revision 4360)
+++ userspace/perftest/rdma_lat.c (working copy)
@@ -105,18 +105,17 @@ static uint16_t pp_get_local_lid(struct
static struct ibv_device *pp_find_dev(const char *ib_devname)
{
- struct dlist *dev_list;
+ struct ibv_device **dev_list;
struct ibv_device *ib_dev = NULL;
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(NULL);
- dlist_start(dev_list);
if (!ib_devname) {
- ib_dev = dlist_next(dev_list);
+ ib_dev = dev_list[0];
if (!ib_dev)
fprintf(stderr, "No IB devices found\n");
} else {
- dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
+ for (ib_dev = *dev_list; ib_dev; ++dev_list)
if (!strcmp(ibv_get_device_name(ib_dev), ib_devname))
break;
if (!ib_dev)
--- userspace/perftest/rdma_bw.c (revision 4360)
+++ userspace/perftest/rdma_bw.c (working copy)
@@ -472,7 +472,7 @@ static void print_report(unsigned int it
int main(int argc, char *argv[])
{
- struct dlist *dev_list;
+ struct ibv_device **dev_list;
struct ibv_device *ib_dev;
struct pingpong_context *ctx;
struct pingpong_dest my_dest;
@@ -587,17 +587,16 @@ int main(int argc, char *argv[])
page_size = sysconf(_SC_PAGESIZE);
- dev_list = ibv_get_devices();
+ dev_list = ibv_get_device_list(NULL);
- dlist_start(dev_list);
if (!ib_devname) {
- ib_dev = dlist_next(dev_list);
+ ib_dev = dev_list[0];
if (!ib_dev) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
} else {
- dlist_for_each_data(dev_list, ib_dev, struct ibv_device)
+ for (ib_dev = *dev_list; ib_dev; ++dev_list)
if (!strcmp(ibv_get_device_name(ib_dev), ib_devname))
break;
if (!ib_dev) {
--- userspace/libibcm/configure.in (revision 4360)
+++ userspace/libibcm/configure.in (working copy)
@@ -25,8 +25,8 @@ AC_CHECK_SIZEOF(long)
dnl Checks for libraries
if test "$disable_libcheck" != "yes"
then
-AC_CHECK_LIB(ibverbs, ibv_get_devices, [],
- AC_MSG_ERROR([ibv_get_devices() not found. libibcm requires libibverbs.]))
+AC_CHECK_LIB(ibverbs, ibv_get_device_list, [],
+ AC_MSG_ERROR([ibv_get_device_list() not found. libibcm requires libibverbs.]))
#AC_CHECK_LIB(rdmacm, rdma_create_id, [],
# AC_MSG_ERROR([rdma_create_id() not found. ucmpost requires librdmacm.]))
fi
--- userspace/libibcm/examples/cmpost.c (revision 4360)
+++ userspace/libibcm/examples/cmpost.c (working copy)
@@ -423,15 +423,14 @@ static void destroy_messages(void)
static int init(void)
{
- struct dlist *dev_list;
+ struct ibv_device **dev_list;
int ret;
test.connects_left = connections;
test.disconnects_left = connections;
- dev_list = ibv_get_devices();
- dlist_start(dev_list);
- test.device = dlist_next(dev_list);
+ dev_list = ibv_get_device_list(NULL);
+ test.device = dev_list[0];
if (!test.device)
return -1;
More information about the general
mailing list