[ofa-general] [PATCH v2] rdma_cm: Add debugfs entries to monitor rdma_cm connections

Moni Shoua monis at Voltaire.COM
Thu Apr 23 05:10:22 PDT 2009


Create a virtual file under debugfs for each cma device and use it to print
information about each rdma_id that is attached to this device.

Here is an example of 'cat /sys/kernel/debug/rdma_cm/mthca0_rdma_id'. This
example is for a host that runs a rping server (when a remote client is
connected to it)  and a rping client to a remote server.

TYPE DEVICE  PORT NET_DEV SRC_ADDR                                            DST_ADDR                                            SPACE STATE          QP_NUM  
     mthca0  0            0.0.0.0:7174                                                                                            TCP   LISTEN         0       
IB   mthca0  1    ib0     192.30.3.249:46079                                  192.30.3.248:7174                                   TCP   CONNECT        132102  
IB   mthca0  1    ib0     192.30.3.249:7174                                   192.30.3.248:42561                                  TCP   CONNECT        132103  

Signed-off-by: Moni Shoua <monis at voltaire.com>
--

 drivers/infiniband/core/cma.c |  206 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 206 insertions(+)

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 2a2e508..0288cad 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -51,6 +51,9 @@
 #include <rdma/ib_sa.h>
 #include <rdma/iw_cm.h>
 
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+
 MODULE_AUTHOR("Sean Hefty");
 MODULE_DESCRIPTION("Generic RDMA CM Agent");
 MODULE_LICENSE("Dual BSD/GPL");
@@ -59,6 +62,10 @@ MODULE_LICENSE("Dual BSD/GPL");
 #define CMA_MAX_CM_RETRIES 15
 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
 
+#define CASE_RET(val, ret) case val: return #ret;
+
+static struct dentry *cma_root_dentry;
+
 static void cma_add_one(struct ib_device *device);
 static void cma_remove_one(struct ib_device *device);
 
@@ -86,6 +93,7 @@ struct cma_device {
 	struct completion	comp;
 	atomic_t		refcount;
 	struct list_head	id_list;
+	struct dentry 		*rdma_id_dentry;
 };
 
 enum cma_state {
@@ -102,6 +110,49 @@ enum cma_state {
 	CMA_DESTROYING
 };
 
+static const char *format_cma_state(enum cma_state s)
+{
+	switch (s) {
+		CASE_RET(CMA_IDLE,           IDLE);
+		CASE_RET(CMA_ADDR_QUERY,     ADDR_QUERY);
+		CASE_RET(CMA_ADDR_RESOLVED,  ADDR_RESOLVED);
+		CASE_RET(CMA_ROUTE_QUERY,    ROUTE_QUERY);
+		CASE_RET(CMA_ROUTE_RESOLVED, ROUTE_RESOLVED);
+		CASE_RET(CMA_CONNECT,        CONNECT);
+		CASE_RET(CMA_DISCONNECT,     DISCONNECT);
+		CASE_RET(CMA_ADDR_BOUND,     ADDR_BOUND);
+		CASE_RET(CMA_LISTEN,         LISTEN);
+		CASE_RET(CMA_DEVICE_REMOVAL, DEVICE_REMOVAL);
+		CASE_RET(CMA_DESTROYING,     DESTROYING);
+	}
+	return "";
+}
+
+static const char *format_port_space(enum rdma_port_space ps)
+{
+	switch (ps) {
+		CASE_RET(RDMA_PS_SDP,   SDP);
+		CASE_RET(RDMA_PS_IPOIB, IPOIB);
+		CASE_RET(RDMA_PS_TCP,   TCP);
+		CASE_RET(RDMA_PS_UDP,   UDP);
+		CASE_RET(RDMA_PS_SCTP,  SCTP);
+	}
+	return "";
+}
+
+static const char *format_node_type(enum rdma_node_type nt)
+{
+	enum rdma_transport_type tt;
+	if (nt) {
+		tt = rdma_node_get_transport(nt);
+		switch (tt) {
+			CASE_RET(RDMA_TRANSPORT_IB,    IB);
+			CASE_RET(RDMA_TRANSPORT_IWARP, IW);
+		}
+	}
+	return "";
+}
+
 struct rdma_bind_list {
 	struct idr		*ps;
 	struct hlist_head	owners;
@@ -2850,6 +2901,150 @@ static struct notifier_block cma_nb = {
 	.notifier_call = cma_netdev_callback
 };
 
+static void *cma_rdma_id_seq_start(struct seq_file *file, loff_t *pos)
+{
+	struct cma_device *cma_dev = file->private;
+	void *ret;
+
+	mutex_lock(&lock);
+	if (*pos == 0)
+		return SEQ_START_TOKEN;
+	ret = seq_list_start_head(&cma_dev->id_list, *pos);
+	return ret;
+}
+
+static void *cma_rdma_id_seq_next(struct seq_file *file, void *v, loff_t *pos)
+{
+	void *ret;
+	struct cma_device *cma_dev = file->private;
+	if (v == SEQ_START_TOKEN) {
+		++*pos;
+		if (!list_empty(&cma_dev->id_list))
+			ret = cma_dev->id_list.next;
+		else
+			ret = NULL;
+	} else {
+		ret = seq_list_next(v, &cma_dev->id_list, pos);
+	}
+	return ret;
+}
+
+static void cma_rdma_id_seq_stop(struct seq_file *file, void *iter_ptr)
+{
+	mutex_unlock(&lock);
+}
+
+static void format_addr(struct sockaddr *sa, char* buf)
+{
+	switch (sa->sa_family) {
+	case AF_INET: {
+		struct sockaddr_in *sin = (struct sockaddr_in *)sa;
+		sprintf(buf, "%pI4:%u", &sin->sin_addr.s_addr,
+				be16_to_cpu(cma_port(sa)));
+		break;
+	}
+	case AF_INET6: {
+		struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sa;
+		sprintf(buf, "%pI6:%u", &sin6->sin6_addr,
+				be16_to_cpu(cma_port(sa)));
+		break;
+	}
+	default:
+		buf[0] = 0;
+	}
+}
+
+static int cma_rdma_id_seq_show(struct seq_file *file, void *v)
+{
+	struct rdma_id_private *id_priv;
+	char local_addr[64], remote_addr[64];
+
+	if (!v)
+		return 0;
+	if (v == SEQ_START_TOKEN) {
+		seq_printf(file,
+		       "%-5s"
+		       "%-8s"
+		       "%-5s"
+		       "%-8s"
+		       "%-52s"
+		       "%-52s"
+		       "%-6s"
+		       "%-15s"
+		       "%-8s"
+		       "\n",
+		       "TYPE", "DEVICE", "PORT", "NET_DEV", "SRC_ADDR", "DST_ADDR", "SPACE", "STATE", "QP_NUM");
+	} else {
+		id_priv = list_entry(v, struct rdma_id_private, list);
+	       format_addr((struct sockaddr *)&id_priv->id.route.addr.src_addr,
+				       local_addr);
+	       format_addr((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
+				       remote_addr);
+
+		       seq_printf(file,
+			       "%-5s"
+			       "%-8s"
+			       "%-5d"
+			       "%-8s"
+			       "%-52s"
+			       "%-52s"
+			       "%-6s"
+			       "%-15s"
+			       "%-8d"
+			       "\n",
+			       format_node_type(id_priv->id.route.addr.dev_addr.dev_type),
+			       (id_priv->id.device) ? id_priv->id.device->name : "",
+			       id_priv->id.port_num,
+			       (id_priv->id.route.addr.dev_addr.src_dev) ? id_priv->id.route.addr.dev_addr.src_dev->name : "",
+			       local_addr,
+			       remote_addr,
+			       format_port_space(id_priv->id.ps),
+			       format_cma_state(id_priv->state),
+			       id_priv->qp_num);
+	}
+	return 0;
+}
+
+static const struct seq_operations cma_rdma_id_seq_ops = {
+	.start = cma_rdma_id_seq_start,
+	.next  = cma_rdma_id_seq_next,
+	.stop  = cma_rdma_id_seq_stop,
+	.show  = cma_rdma_id_seq_show,
+};
+
+static int cma_rdma_id_open(struct inode *inode, struct file *file)
+{
+	struct seq_file *seq;
+	int ret;
+
+	ret = seq_open(file, &cma_rdma_id_seq_ops);
+	if (ret)
+		return ret;
+
+	seq = file->private_data;
+	seq->private = inode->i_private;
+
+	return 0;
+}
+
+static const struct file_operations cma_rdma_id_fops = {
+	.owner   = THIS_MODULE,
+	.open    = cma_rdma_id_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = seq_release
+};
+
+void cma_create_debug_files(struct cma_device *cma_dev)
+{
+	char name[IB_DEVICE_NAME_MAX + sizeof "_rdma_id"];
+	snprintf(name, sizeof name, "%s_rdma_id", cma_dev->device->name);
+	cma_dev->rdma_id_dentry = debugfs_create_file(name, S_IFREG | S_IRUGO,
+					cma_root_dentry, cma_dev, &cma_rdma_id_fops);
+	if (!cma_dev->rdma_id_dentry)
+		printk(KERN_WARNING "RDMA CMA: failed to create debugfs file %s\n",  name);
+}
+
 static void cma_add_one(struct ib_device *device)
 {
 	struct cma_device *cma_dev;
@@ -2871,6 +3066,7 @@ static void cma_add_one(struct ib_device *device)
 	list_for_each_entry(id_priv, &listen_any_list, list)
 		cma_listen_on_dev(id_priv, cma_dev);
 	mutex_unlock(&lock);
+	cma_create_debug_files(cma_dev);
 }
 
 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
@@ -2905,6 +3101,8 @@ static void cma_process_remove(struct cma_device *cma_dev)
 	int ret;
 
 	mutex_lock(&lock);
+	if (cma_dev->rdma_id_dentry)
+		debugfs_remove(cma_dev->rdma_id_dentry);
 	while (!list_empty(&cma_dev->id_list)) {
 		id_priv = list_entry(cma_dev->id_list.next,
 				     struct rdma_id_private, list);
@@ -2940,6 +3138,7 @@ static void cma_remove_one(struct ib_device *device)
 	mutex_unlock(&lock);
 
 	cma_process_remove(cma_dev);
+
 	kfree(cma_dev);
 }
 
@@ -2947,6 +3146,12 @@ static int cma_init(void)
 {
 	int ret, low, high, remaining;
 
+	cma_root_dentry = debugfs_create_dir("rdma_cm", NULL);
+	if (!cma_root_dentry) {
+		printk(KERN_ERR "RDMA CMA: failed to create debugfs dir\n");
+		return  -ENOMEM;
+	}
+
 	get_random_bytes(&next_port, sizeof next_port);
 	inet_get_local_port_range(&low, &high);
 	remaining = (high - low) + 1;
@@ -2984,6 +3189,7 @@ static void cma_cleanup(void)
 	idr_destroy(&tcp_ps);
 	idr_destroy(&udp_ps);
 	idr_destroy(&ipoib_ps);
+	debugfs_remove(cma_root_dentry);
 }
 
 module_init(cma_init);



More information about the general mailing list