[ofa-general] [PATCH v2 RESEND] rdma_cm: Add debugfs entries to monitor rdma_cm connections

Moni Shoua monis at Voltaire.COM
Sun Jun 21 07:38:11 PDT 2009


This patch was sent on Apr, 23 2009. Nothing was changed except for fixing offsets and verifying it under 2.6.30

--------------------

Create a virtual file under debugfs for each cma device and use it to print
information about each rdma_id that is attached to this device.

Here is an example of 'cat /sys/kernel/debug/rdma_cm/mthca0_rdma_id'. This
example is for a host that runs a rping server (when a remote client is
connected to it)  and a rping client to a remote server.

TYPE DEVICE  PORT NET_DEV SRC_ADDR                                            DST_ADDR                                            SPACE STATE          QP_NUM  
     mthca0  0            0.0.0.0:7174                                                                                            TCP   LISTEN         0       
IB   mthca0  1    ib0     192.30.3.249:46079                                  192.30.3.248:7174                                   TCP   CONNECT        132102  
IB   mthca0  1    ib0     192.30.3.249:7174                                   192.30.3.248:42561                                  TCP   CONNECT        132103  

Signed-off-by: Moni Shoua <monis at voltaire.com>
--

 drivers/infiniband/core/cma.c |  183 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 183 insertions(+)

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 851de83..746ab14 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -51,6 +51,9 @@
 #include <rdma/ib_sa.h>
 #include <rdma/iw_cm.h>
 
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+
 MODULE_AUTHOR("Sean Hefty");
 MODULE_DESCRIPTION("Generic RDMA CM Agent");
 MODULE_LICENSE("Dual BSD/GPL");
@@ -59,6 +62,8 @@ MODULE_LICENSE("Dual BSD/GPL");
 #define CMA_MAX_CM_RETRIES 15
 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
 
+static struct dentry *cma_root_dentry;
+
 static void cma_add_one(struct ib_device *device);
 static void cma_remove_one(struct ib_device *device);
 
@@ -86,6 +91,7 @@ struct cma_device {
 	struct completion	comp;
 	atomic_t		refcount;
 	struct list_head	id_list;
+	struct dentry 		*rdma_id_dentry;
 };
 
 enum cma_state {
@@ -102,6 +108,47 @@ enum cma_state {
 	CMA_DESTROYING
 };
 
+static const char *format_cma_state(enum cma_state s)
+{
+	switch (s) {
+	case CMA_IDLE:           return "IDLE";
+	case CMA_ADDR_QUERY:     return "ADDR_QUERY";
+	case CMA_ADDR_RESOLVED:  return "ADDR_RESOLVED";
+	case CMA_ROUTE_QUERY:    return "ROUTE_QUERY";
+	case CMA_ROUTE_RESOLVED: return "ROUTE_RESOLVED";
+	case CMA_CONNECT:        return "CONNECT";
+	case CMA_DISCONNECT:     return "DISCONNECT";
+	case CMA_ADDR_BOUND:     return "ADDR_BOUND";
+	case CMA_LISTEN:         return "LISTEN";
+	case CMA_DEVICE_REMOVAL: return "DEVICE_REMOVAL";
+	case CMA_DESTROYING:     return "DESTROYING";
+	}
+	return "";
+}
+
+static const char *format_port_space(enum rdma_port_space ps)
+{
+	switch (ps) {
+	case RDMA_PS_SDP:       return "SDP";
+	case RDMA_PS_IPOIB:     return "IPOIB";
+	case RDMA_PS_TCP:       return "TCP";
+	case RDMA_PS_UDP:       return "UDP";
+	case RDMA_PS_SCTP:      return "SCTP";
+	}
+	return "";
+}
+
+static const char *format_node_type(enum rdma_node_type nt)
+{
+	if (nt) {
+		switch (rdma_node_get_transport(nt)) {
+		case RDMA_TRANSPORT_IB:		return "IB";
+		case RDMA_TRANSPORT_IWARP:	return "IW";
+		}
+	}
+	return "";
+}
+
 struct rdma_bind_list {
 	struct idr		*ps;
 	struct hlist_head	owners;
@@ -2867,6 +2914,131 @@ static struct notifier_block cma_nb = {
 	.notifier_call = cma_netdev_callback
 };
 
+static void *cma_rdma_id_seq_start(struct seq_file *file, loff_t *pos)
+{
+	struct cma_device *cma_dev = file->private;
+	void *ret;
+
+	mutex_lock(&lock);
+	if (*pos == 0)
+		return SEQ_START_TOKEN;
+	ret = seq_list_start_head(&cma_dev->id_list, *pos);
+	return ret;
+}
+
+static void *cma_rdma_id_seq_next(struct seq_file *file, void *v, loff_t *pos)
+{
+	void *ret;
+	struct cma_device *cma_dev = file->private;
+	if (v == SEQ_START_TOKEN) {
+		++*pos;
+		if (!list_empty(&cma_dev->id_list))
+			ret = cma_dev->id_list.next;
+		else
+			ret = NULL;
+	} else {
+		ret = seq_list_next(v, &cma_dev->id_list, pos);
+	}
+	return ret;
+}
+
+static void cma_rdma_id_seq_stop(struct seq_file *file, void *iter_ptr)
+{
+	mutex_unlock(&lock);
+}
+
+static void format_addr(struct sockaddr *sa, char* buf)
+{
+	switch (sa->sa_family) {
+	case AF_INET: {
+		struct sockaddr_in *sin = (struct sockaddr_in *)sa;
+		sprintf(buf, "%pI4:%u", &sin->sin_addr.s_addr,
+				be16_to_cpu(cma_port(sa)));
+		break;
+	}
+	case AF_INET6: {
+		struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sa;
+		sprintf(buf, "%pI6:%u", &sin6->sin6_addr,
+				be16_to_cpu(cma_port(sa)));
+		break;
+	}
+	default:
+		buf[0] = 0;
+	}
+}
+
+static int cma_rdma_id_seq_show(struct seq_file *file, void *v)
+{
+	struct rdma_id_private *id_priv;
+	char local_addr[64], remote_addr[64];
+
+	if (!v)
+		return 0;
+	if (v == SEQ_START_TOKEN) {
+		seq_printf(file,
+			"%-5s %-8s %-5s %-8s %-52s %-52s %-6s %-15s %-8s \n",
+		       "TYPE", "DEVICE", "PORT", "NET_DEV", "SRC_ADDR", "DST_ADDR", "SPACE", "STATE", "QP_NUM");
+	} else {
+		id_priv = list_entry(v, struct rdma_id_private, list);
+		format_addr((struct sockaddr *)&id_priv->id.route.addr.src_addr,
+				local_addr);
+		format_addr((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
+				remote_addr);
+
+		       seq_printf(file,
+				"%-5s %-8s %-5d %-8s %-52s %-52s %-6s %-15s %-8d \n",
+			       format_node_type(id_priv->id.route.addr.dev_addr.dev_type),
+			       (id_priv->id.device) ? id_priv->id.device->name : "",
+			       id_priv->id.port_num,
+			       (id_priv->id.route.addr.dev_addr.src_dev) ? id_priv->id.route.addr.dev_addr.src_dev->name : "",
+			       local_addr, remote_addr,
+			       format_port_space(id_priv->id.ps),
+			       format_cma_state(id_priv->state),
+			       id_priv->qp_num);
+	}
+	return 0;
+}
+
+static const struct seq_operations cma_rdma_id_seq_ops = {
+	.start = cma_rdma_id_seq_start,
+	.next  = cma_rdma_id_seq_next,
+	.stop  = cma_rdma_id_seq_stop,
+	.show  = cma_rdma_id_seq_show,
+};
+
+static int cma_rdma_id_open(struct inode *inode, struct file *file)
+{
+	struct seq_file *seq;
+	int ret;
+
+	ret = seq_open(file, &cma_rdma_id_seq_ops);
+	if (ret)
+		return ret;
+
+	seq = file->private_data;
+	seq->private = inode->i_private;
+
+	return 0;
+}
+
+static const struct file_operations cma_rdma_id_fops = {
+	.owner   = THIS_MODULE,
+	.open    = cma_rdma_id_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = seq_release
+};
+
+void cma_create_debug_files(struct cma_device *cma_dev)
+{
+	char name[IB_DEVICE_NAME_MAX + sizeof "_rdma_id"];
+	snprintf(name, sizeof name, "%s_rdma_id", cma_dev->device->name);
+	cma_dev->rdma_id_dentry = debugfs_create_file(name, S_IFREG | S_IRUGO,
+					cma_root_dentry, cma_dev, &cma_rdma_id_fops);
+	if (!cma_dev->rdma_id_dentry)
+		printk(KERN_WARNING "RDMA CMA: failed to create debugfs file %s\n",  name);
+}
+
 static void cma_add_one(struct ib_device *device)
 {
 	struct cma_device *cma_dev;
@@ -2888,6 +3060,7 @@ static void cma_add_one(struct ib_device *device)
 	list_for_each_entry(id_priv, &listen_any_list, list)
 		cma_listen_on_dev(id_priv, cma_dev);
 	mutex_unlock(&lock);
+	cma_create_debug_files(cma_dev);
 }
 
 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
@@ -2922,6 +3095,8 @@ static void cma_process_remove(struct cma_device *cma_dev)
 	int ret;
 
 	mutex_lock(&lock);
+	if (cma_dev->rdma_id_dentry)
+		debugfs_remove(cma_dev->rdma_id_dentry);
 	while (!list_empty(&cma_dev->id_list)) {
 		id_priv = list_entry(cma_dev->id_list.next,
 				     struct rdma_id_private, list);
@@ -2957,6 +3132,7 @@ static void cma_remove_one(struct ib_device *device)
 	mutex_unlock(&lock);
 
 	cma_process_remove(cma_dev);
+
 	kfree(cma_dev);
 }
 
@@ -2964,6 +3140,12 @@ static int cma_init(void)
 {
 	int ret, low, high, remaining;
 
+	cma_root_dentry = debugfs_create_dir("rdma_cm", NULL);
+	if (!cma_root_dentry) {
+		printk(KERN_ERR "RDMA CMA: failed to create debugfs dir\n");
+		return  -ENOMEM;
+	}
+
 	get_random_bytes(&next_port, sizeof next_port);
 	inet_get_local_port_range(&low, &high);
 	remaining = (high - low) + 1;
@@ -3001,6 +3183,7 @@ static void cma_cleanup(void)
 	idr_destroy(&tcp_ps);
 	idr_destroy(&udp_ps);
 	idr_destroy(&ipoib_ps);
+	debugfs_remove(cma_root_dentry);
 }
 
 module_init(cma_init);



More information about the general mailing list