[ofa-general] [PATCH] rdma_cm: Add debugfs entries to monitor rdma_cm connections

Moni Shoua monis at Voltaire.COM
Thu Apr 16 05:09:05 PDT 2009


Create a virtual file under debugfs for each cma device and use it to print
information about each rdma_id that is attached to this device.

Here is an example of 'cat /sys/kernel/debug/rdma_cm/mthca0_rdma_id'. This
example is for a host that runs a rping server (when a remote client is
connected to it)  and a rping client to a remote server.

TP DEV     PO NDEV SRC                                                 DST                                                 PS   ST QPN
0  mthca0  0       0.0.0.0:7174                                                                                            262  8  0
1  mthca0  1  ib0  192.30.3.249:50113                                  192.30.3.248:7174                                   262  5  66566
1  mthca0  1  ib0  192.30.3.249:7174                                   192.30.3.248:42560                                  262  5  66567

Signed-off-by: Moni Shoua <monis at voltaire.com>
-- 

 drivers/infiniband/core/cma.c |  161 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 161 insertions(+)

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 2a2e508..ea16d44 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -51,6 +51,9 @@
 #include <rdma/ib_sa.h>
 #include <rdma/iw_cm.h>
 
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+
 MODULE_AUTHOR("Sean Hefty");
 MODULE_DESCRIPTION("Generic RDMA CM Agent");
 MODULE_LICENSE("Dual BSD/GPL");
@@ -59,6 +62,8 @@ MODULE_LICENSE("Dual BSD/GPL");
 #define CMA_MAX_CM_RETRIES 15
 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
 
+static struct dentry *cma_root_dentry;
+
 static void cma_add_one(struct ib_device *device);
 static void cma_remove_one(struct ib_device *device);
 
@@ -86,6 +91,7 @@ struct cma_device {
 	struct completion	comp;
 	atomic_t		refcount;
 	struct list_head	id_list;
+	struct dentry 		*rdma_id_dentry;
 };
 
 enum cma_state {
@@ -2850,6 +2856,150 @@ static struct notifier_block cma_nb = {
 	.notifier_call = cma_netdev_callback
 };
 
+static void *cma_rdma_id_seq_start(struct seq_file *file, loff_t *pos)
+{
+	struct cma_device *cma_dev = file->private;
+	void *ret;
+
+	mutex_lock(&lock);
+	if (*pos == 0)
+		return SEQ_START_TOKEN;
+	ret = seq_list_start_head(&cma_dev->id_list, *pos);
+	return ret;
+}
+
+static void *cma_rdma_id_seq_next(struct seq_file *file, void *v, loff_t *pos)
+{
+	void *ret;
+	struct cma_device *cma_dev = file->private;
+	if (v == SEQ_START_TOKEN) {
+		++*pos;
+		if (!list_empty(&cma_dev->id_list))
+			ret = cma_dev->id_list.next;
+		else
+			ret = NULL;
+	} else {
+		ret = seq_list_next(v, &cma_dev->id_list, pos);
+	}
+	return ret;
+}
+
+static void cma_rdma_id_seq_stop(struct seq_file *file, void *iter_ptr)
+{
+	mutex_unlock(&lock);
+}
+
+static void format_addr(struct sockaddr *sa, char* buf)
+{
+	switch (sa->sa_family) {
+	case AF_INET: {
+		struct sockaddr_in *sin = (struct sockaddr_in *)sa;
+		sprintf(buf, "%pI4:%u", &sin->sin_addr.s_addr,
+				be16_to_cpu(cma_port(sa)));
+		break;
+	}
+	case AF_INET6: {
+		struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sa;
+		sprintf(buf, "%pI6:%u", &sin6->sin6_addr,
+				be16_to_cpu(cma_port(sa)));
+		break;
+	}
+	default:
+		buf[0] = 0;
+	}
+}
+
+static int cma_rdma_id_seq_show(struct seq_file *file, void *v)
+{
+	struct rdma_id_private *id_priv;
+	char local_addr[64], remote_addr[64];
+
+	if (!v)
+		return 0;
+	if (v == SEQ_START_TOKEN) {
+		seq_printf(file,
+		       "%-3s"
+		       "%-8s"
+		       "%-3s"
+		       "%-5s"
+		       "%-52s"
+		       "%-52s"
+		       "%-5s"
+		       "%-3s"
+		       "%-8s"
+		       "\n",
+		       "TP", "DEV", "PO", "NDEV", "SRC", "DST", "PS", "ST", "QPN");
+	} else {
+		id_priv = list_entry(v, struct rdma_id_private, list);
+	       format_addr((struct sockaddr *)&id_priv->id.route.addr.src_addr,
+				       local_addr);
+	       format_addr((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
+				       remote_addr);
+
+		       seq_printf(file,
+			       "%-3d"
+			       "%-8s"
+			       "%-3d"
+			       "%-5s"
+			       "%-52s"
+			       "%-52s"
+			       "%-5d"
+			       "%-3d"
+			       "%-8d"
+			       "\n",
+			       id_priv->id.route.addr.dev_addr.dev_type,
+			       (id_priv->id.device) ? id_priv->id.device->name : "",
+			       id_priv->id.port_num,
+			       (id_priv->id.route.addr.dev_addr.src_dev) ? id_priv->id.route.addr.dev_addr.src_dev->name : "",
+			       local_addr,
+			       remote_addr,
+			       id_priv->id.ps,
+			       id_priv->state,
+			       id_priv->qp_num);
+	}
+	return 0;
+}
+
+static const struct seq_operations cma_rdma_id_seq_ops = {
+	.start = cma_rdma_id_seq_start,
+	.next  = cma_rdma_id_seq_next,
+	.stop  = cma_rdma_id_seq_stop,
+	.show  = cma_rdma_id_seq_show,
+};
+
+static int cma_rdma_id_open(struct inode *inode, struct file *file)
+{
+	struct seq_file *seq;
+	int ret;
+
+	ret = seq_open(file, &cma_rdma_id_seq_ops);
+	if (ret)
+		return ret;
+
+	seq = file->private_data;
+	seq->private = inode->i_private;
+
+	return 0;
+}
+
+static const struct file_operations cma_rdma_id_fops = {
+	.owner   = THIS_MODULE,
+	.open    = cma_rdma_id_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = seq_release
+};
+
+void cma_create_debug_files(struct cma_device *cma_dev)
+{
+	char name[IB_DEVICE_NAME_MAX + sizeof "_rdma_id"];
+	snprintf(name, sizeof name, "%s_rdma_id", cma_dev->device->name);
+	cma_dev->rdma_id_dentry = debugfs_create_file(name, S_IFREG | S_IRUGO,
+					cma_root_dentry, cma_dev, &cma_rdma_id_fops);
+	if (!cma_dev->rdma_id_dentry)
+		printk(KERN_WARNING "RDMA CMA: failed to create debugfs file %s\n",  name);
+}
+
 static void cma_add_one(struct ib_device *device)
 {
 	struct cma_device *cma_dev;
@@ -2871,6 +3021,7 @@ static void cma_add_one(struct ib_device *device)
 	list_for_each_entry(id_priv, &listen_any_list, list)
 		cma_listen_on_dev(id_priv, cma_dev);
 	mutex_unlock(&lock);
+	cma_create_debug_files(cma_dev);
 }
 
 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
@@ -2905,6 +3056,8 @@ static void cma_process_remove(struct cma_device *cma_dev)
 	int ret;
 
 	mutex_lock(&lock);
+	if (cma_dev->rdma_id_dentry)
+		debugfs_remove(cma_dev->rdma_id_dentry);
 	while (!list_empty(&cma_dev->id_list)) {
 		id_priv = list_entry(cma_dev->id_list.next,
 				     struct rdma_id_private, list);
@@ -2940,6 +3093,7 @@ static void cma_remove_one(struct ib_device *device)
 	mutex_unlock(&lock);
 
 	cma_process_remove(cma_dev);
+
 	kfree(cma_dev);
 }
 
@@ -2947,6 +3101,12 @@ static int cma_init(void)
 {
 	int ret, low, high, remaining;
 
+	cma_root_dentry = debugfs_create_dir("rdma_cm", NULL);
+	if (!cma_root_dentry) {
+		printk(KERN_ERR "RDMA CMA: failed to create debugfs dir\n");
+		return  -ENOMEM;
+	}
+
 	get_random_bytes(&next_port, sizeof next_port);
 	inet_get_local_port_range(&low, &high);
 	remaining = (high - low) + 1;
@@ -2984,6 +3144,7 @@ static void cma_cleanup(void)
 	idr_destroy(&tcp_ps);
 	idr_destroy(&udp_ps);
 	idr_destroy(&ipoib_ps);
+	debugfs_remove(cma_root_dentry);
 }
 
 module_init(cma_init);



More information about the general mailing list