[openib-general] [PATCH] Fix capability mask enums in ib_verbs.h
Roland Dreier
roland at topspin.com
Fri Jan 14 14:28:50 PST 2005
This patch implements setting the IsSM capability bit from userspace
by creating special files /dev/infiniband/issmN that set the bit when
opened and clear it when closed. Additional opens of the same file
will block or return EAGAIN depending on whether the O_NONBLOCK flag
is passed to open(). The precise API is also described in the patch
to the doc file.
Please test this out and let me know if it does what is required.
Also if there are objections to the interface let me know.
(For this patch to set the correct capability bit, the previous patch
to ib_verbs.h is required)
- Roland
Index: infiniband/core/user_mad.c
===================================================================
--- infiniband/core/user_mad.c (revision 1524)
+++ infiniband/core/user_mad.c (working copy)
@@ -46,6 +46,7 @@
#include <linux/ioctl32.h>
#include <asm/uaccess.h>
+#include <asm/semaphore.h>
#include <ib_mad.h>
#include <ib_user_mad.h>
@@ -55,7 +56,7 @@
MODULE_LICENSE("Dual BSD/GPL");
enum {
- IB_UMAD_MAX_PORTS = 256,
+ IB_UMAD_MAX_PORTS = 64,
IB_UMAD_MAX_AGENTS = 32
};
@@ -63,6 +64,12 @@
int devnum;
struct cdev dev;
struct class_device class_dev;
+
+ int sm_devnum;
+ struct cdev sm_dev;
+ struct class_device sm_class_dev;
+ struct semaphore sm_sem;
+
struct ib_device *ib_dev;
struct ib_umad_device *umad_dev;
u8 port_num;
@@ -93,7 +100,7 @@
static dev_t base_dev;
static spinlock_t map_lock;
-static DECLARE_BITMAP(dev_map, IB_UMAD_MAX_PORTS);
+static DECLARE_BITMAP(dev_map, IB_UMAD_MAX_PORTS * 2);
static void ib_umad_add_one(struct ib_device *device);
static void ib_umad_remove_one(struct ib_device *device);
@@ -511,6 +518,54 @@
.release = ib_umad_close
};
+static int ib_umad_sm_open(struct inode *inode, struct file *filp)
+{
+ struct ib_umad_port *port =
+ container_of(inode->i_cdev, struct ib_umad_port, sm_dev);
+ struct ib_port_modify props = {
+ .set_port_cap_mask = IB_PORT_SM
+ };
+ int ret;
+
+ if (filp->f_flags & O_NONBLOCK) {
+ if (down_trylock(&port->sm_sem))
+ return -EAGAIN;
+ } else {
+ if (down_interruptible(&port->sm_sem))
+ return -ERESTARTSYS;
+ }
+
+ ret = ib_modify_port(port->ib_dev, port->port_num, 0, &props);
+ if (ret) {
+ up(&port->sm_sem);
+ return ret;
+ }
+
+ filp->private_data = port;
+
+ return 0;
+}
+
+static int ib_umad_sm_close(struct inode *inode, struct file *filp)
+{
+ struct ib_umad_port *port = filp->private_data;
+ struct ib_port_modify props = {
+ .clr_port_cap_mask = IB_PORT_SM
+ };
+ int ret;
+
+ ret = ib_modify_port(port->ib_dev, port->port_num, 0, &props);
+ up(&port->sm_sem);
+
+ return ret;
+}
+
+static struct file_operations umad_sm_fops = {
+ .owner = THIS_MODULE,
+ .open = ib_umad_sm_open,
+ .release = ib_umad_sm_close
+};
+
static struct ib_client umad_client = {
.name = "umad",
.add = ib_umad_add_one,
@@ -519,10 +574,12 @@
static ssize_t show_dev(struct class_device *class_dev, char *buf)
{
- struct ib_umad_port *port =
- container_of(class_dev, struct ib_umad_port, class_dev);
+ struct ib_umad_port *port = class_get_devdata(class_dev);
- return print_dev_t(buf, port->dev.dev);
+ if (class_dev == &port->class_dev)
+ return print_dev_t(buf, port->dev.dev);
+ else
+ return print_dev_t(buf, port->sm_dev.dev);
}
static CLASS_DEVICE_ATTR(dev, S_IRUGO, show_dev, NULL);
@@ -554,11 +611,16 @@
static void ib_umad_release_port(struct class_device *class_dev)
{
- struct ib_umad_port *port =
- container_of(class_dev, struct ib_umad_port, class_dev);
+ struct ib_umad_port *port = class_get_devdata(class_dev);
- cdev_del(&port->dev);
- clear_bit(port->devnum, dev_map);
+ if (class_dev == &port->class_dev) {
+ cdev_del(&port->dev);
+ clear_bit(port->devnum, dev_map);
+ } else {
+ cdev_del(&port->sm_dev);
+ clear_bit(port->sm_devnum, dev_map);
+ }
+
kref_put(&port->umad_dev->ref, ib_umad_release_dev);
}
@@ -573,6 +635,94 @@
}
static CLASS_ATTR(abi_version, S_IRUGO, show_abi_version, NULL);
+static int ib_umad_init_port(struct ib_device *device, int port_num,
+ struct ib_umad_port *port)
+{
+ spin_lock(&map_lock);
+ port->devnum = find_first_zero_bit(dev_map, IB_UMAD_MAX_PORTS);
+ if (port->devnum >= IB_UMAD_MAX_PORTS) {
+ spin_unlock(&map_lock);
+ return -1;
+ }
+ port->sm_devnum = find_next_zero_bit(dev_map, IB_UMAD_MAX_PORTS * 2, IB_UMAD_MAX_PORTS);
+ if (port->sm_devnum >= IB_UMAD_MAX_PORTS * 2) {
+ spin_unlock(&map_lock);
+ return -1;
+ }
+ set_bit(port->devnum, dev_map);
+ set_bit(port->sm_devnum, dev_map);
+ spin_unlock(&map_lock);
+
+ port->ib_dev = device;
+ port->port_num = port_num;
+ init_MUTEX(&port->sm_sem);
+
+ cdev_init(&port->dev, &umad_fops);
+ port->dev.owner = THIS_MODULE;
+ kobject_set_name(&port->dev.kobj, "umad%d", port->devnum);
+ if (cdev_add(&port->dev, base_dev + port->devnum, 1))
+ return -1;
+
+ port->class_dev.class = &umad_class;
+ port->class_dev.dev = device->dma_device;
+
+ snprintf(port->class_dev.class_id, BUS_ID_SIZE, "umad%d", port->devnum);
+
+ if (class_device_register(&port->class_dev))
+ goto err_cdev;
+
+ class_set_devdata(&port->class_dev, port);
+ kref_get(&port->umad_dev->ref);
+
+ if (class_device_create_file(&port->class_dev, &class_device_attr_dev))
+ goto err_class;
+ if (class_device_create_file(&port->class_dev, &class_device_attr_ibdev))
+ goto err_class;
+ if (class_device_create_file(&port->class_dev, &class_device_attr_port))
+ goto err_class;
+
+ cdev_init(&port->sm_dev, &umad_sm_fops);
+ port->sm_dev.owner = THIS_MODULE;
+ kobject_set_name(&port->dev.kobj, "issm%d", port->sm_devnum - IB_UMAD_MAX_PORTS);
+ if (cdev_add(&port->sm_dev, base_dev + port->sm_devnum, 1))
+ return -1;
+
+ port->sm_class_dev.class = &umad_class;
+ port->sm_class_dev.dev = device->dma_device;
+
+ snprintf(port->sm_class_dev.class_id, BUS_ID_SIZE, "issm%d", port->sm_devnum - IB_UMAD_MAX_PORTS);
+
+ if (class_device_register(&port->sm_class_dev))
+ goto err_sm_cdev;
+
+ class_set_devdata(&port->sm_class_dev, port);
+ kref_get(&port->umad_dev->ref);
+
+ if (class_device_create_file(&port->sm_class_dev, &class_device_attr_dev))
+ goto err_sm_class;
+ if (class_device_create_file(&port->sm_class_dev, &class_device_attr_ibdev))
+ goto err_sm_class;
+ if (class_device_create_file(&port->sm_class_dev, &class_device_attr_port))
+ goto err_sm_class;
+
+ return 0;
+
+err_sm_class:
+ class_device_unregister(&port->sm_class_dev);
+
+err_sm_cdev:
+ cdev_del(&port->sm_dev);
+
+err_class:
+ class_device_unregister(&port->class_dev);
+
+err_cdev:
+ cdev_del(&port->dev);
+ clear_bit(port->devnum, dev_map);
+
+ return -1;
+}
+
static void ib_umad_add_one(struct ib_device *device)
{
struct ib_umad_device *umad_dev;
@@ -601,58 +751,20 @@
for (i = s; i <= e; ++i) {
umad_dev->port[i - s].umad_dev = umad_dev;
- kref_get(&umad_dev->ref);
- spin_lock(&map_lock);
- umad_dev->port[i - s].devnum =
- find_first_zero_bit(dev_map, IB_UMAD_MAX_PORTS);
- if (umad_dev->port[i - s].devnum >= IB_UMAD_MAX_PORTS) {
- spin_unlock(&map_lock);
+ if (ib_umad_init_port(device, i, &umad_dev->port[i - s]))
goto err;
- }
- set_bit(umad_dev->port[i - s].devnum, dev_map);
- spin_unlock(&map_lock);
-
- umad_dev->port[i - s].ib_dev = device;
- umad_dev->port[i - s].port_num = i;
-
- cdev_init(&umad_dev->port[i - s].dev, &umad_fops);
- umad_dev->port[i - s].dev.owner = THIS_MODULE;
- kobject_set_name(&umad_dev->port[i - s].dev.kobj,
- "umad%d", umad_dev->port[i - s].devnum);
- if (cdev_add(&umad_dev->port[i - s].dev, base_dev +
- umad_dev->port[i - s].devnum, 1))
- goto err;
-
- umad_dev->port[i - s].class_dev.class = &umad_class;
- umad_dev->port[i - s].class_dev.dev = device->dma_device;
- snprintf(umad_dev->port[i - s].class_dev.class_id,
- BUS_ID_SIZE, "umad%d", umad_dev->port[i - s].devnum);
- if (class_device_register(&umad_dev->port[i - s].class_dev))
- goto err_class;
-
- if (class_device_create_file(&umad_dev->port[i - s].class_dev,
- &class_device_attr_dev))
- goto err_class;
- if (class_device_create_file(&umad_dev->port[i - s].class_dev,
- &class_device_attr_ibdev))
- goto err_class;
- if (class_device_create_file(&umad_dev->port[i - s].class_dev,
- &class_device_attr_port))
- goto err_class;
}
ib_set_client_data(device, &umad_client, umad_dev);
return;
-err_class:
- cdev_del(&umad_dev->port[i - s].dev);
- clear_bit(umad_dev->port[i - s].devnum, dev_map);
-
err:
- while (--i >= s)
+ while (--i >= s) {
class_device_unregister(&umad_dev->port[i - s].class_dev);
+ class_device_unregister(&umad_dev->port[i - s].sm_class_dev);
+ }
kref_put(&umad_dev->ref, ib_umad_release_dev);
}
@@ -665,8 +777,10 @@
if (!umad_dev)
return;
- for (i = 0; i <= umad_dev->end_port - umad_dev->start_port; ++i)
+ for (i = 0; i <= umad_dev->end_port - umad_dev->start_port; ++i) {
class_device_unregister(&umad_dev->port[i].class_dev);
+ class_device_unregister(&umad_dev->port[i].sm_class_dev);
+ }
kref_put(&umad_dev->ref, ib_umad_release_dev);
}
@@ -677,7 +791,7 @@
spin_lock_init(&map_lock);
- ret = alloc_chrdev_region(&base_dev, 0, IB_UMAD_MAX_PORTS,
+ ret = alloc_chrdev_region(&base_dev, 0, IB_UMAD_MAX_PORTS * 2,
"infiniband_mad");
if (ret) {
printk(KERN_ERR "user_mad: couldn't get device number\n");
@@ -719,7 +833,7 @@
class_unregister(&umad_class);
out_chrdev:
- unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS);
+ unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2);
out:
return ret;
@@ -731,7 +845,7 @@
unregister_ioctl32_conversion(IB_USER_MAD_UNREGISTER_AGENT);
ib_unregister_client(&umad_client);
class_unregister(&umad_class);
- unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS);
+ unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2);
}
module_init(ib_umad_init);
Index: docs/user_mad.txt
===================================================================
--- docs/user_mad.txt (revision 1524)
+++ docs/user_mad.txt (working copy)
@@ -2,9 +2,10 @@
Device files
- Each port of each InfiniBand device has a "umad" device attached.
- For example, a two-port HCA will have two devices, while a switch
- will have one device (for switch port 0).
+ Each port of each InfiniBand device has a "umad" device and an
+ "issm" device attached. For example, a two-port HCA will have two
+ umad devices and two issm devices, while a switch will have one
+ device of each type (for switch port 0).
Creating MAD agents
@@ -63,19 +64,36 @@
if (ret != sizeof mad)
perror("write");
+Setting IsSM Capability Bit
+
+ To set the IsSM capability bit for a port, simply open the
+ corresponding issm device file. If the IsSM bit is already set,
+ then the open call will block until the bit is cleared (or return
+ immediately with errno set to EAGAIN if the O_NONBLOCK flag is
+ passed to open()). The IsSM bit will be cleared when the issm file
+ is closed. No read, write or other operations can be performed on
+ the issm file.
+
/dev files
To create the appropriate character device files automatically with
udev, a rule like
KERNEL="umad*", NAME="infiniband/%k"
+ KERNEL="issm*", NAME="infiniband/%k"
- can be used. This will create a device node named
+ can be used. This will create device nodes named
/dev/infiniband/umad0
+ /dev/infiniband/issm0
for the first port, and so on. The InfiniBand device and port
- associated with this device can be determined from the files
+ associated with these devices can be determined from the files
/sys/class/infiniband_mad/umad0/ibdev
/sys/class/infiniband_mad/umad0/port
+
+ and
+
+ /sys/class/infiniband_mad/issm0/ibdev
+ /sys/class/infiniband_mad/issm0/port
More information about the general
mailing list