[ofa-general] RE: [PATCH v2] rdma_cm: Add debugfs entries to monitor rdma_cm connections
Moni Shoua
monis at Voltaire.COM
Sun Apr 26 02:45:36 PDT 2009
Thanks Sean. I think this takes care of the issues you brought up.
------------------------------------
Create a virtual file under debugfs for each cma device and use it to print
information about each rdma_id that is attached to this device.
Here is an example of 'cat /sys/kernel/debug/rdma_cm/mthca0_rdma_id'. This
example is for a host that runs a rping server (when a remote client is
connected to it) and a rping client to a remote server.
TYPE DEVICE PORT NET_DEV SRC_ADDR DST_ADDR SPACE STATE QP_NUM
mthca0 0 0.0.0.0:7174 TCP LISTEN 0
IB mthca0 1 ib0 192.30.3.249:46079 192.30.3.248:7174 TCP CONNECT 132102
IB mthca0 1 ib0 192.30.3.249:7174 192.30.3.248:42561 TCP CONNECT 132103
Signed-off-by: Moni Shoua <monis at voltaire.com>
--
drivers/infiniband/core/cma.c | 183 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 183 insertions(+)
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 2a2e508..ce393e7 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -51,6 +51,9 @@
#include <rdma/ib_sa.h>
#include <rdma/iw_cm.h>
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+
MODULE_AUTHOR("Sean Hefty");
MODULE_DESCRIPTION("Generic RDMA CM Agent");
MODULE_LICENSE("Dual BSD/GPL");
@@ -59,6 +62,8 @@ MODULE_LICENSE("Dual BSD/GPL");
#define CMA_MAX_CM_RETRIES 15
#define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
+static struct dentry *cma_root_dentry;
+
static void cma_add_one(struct ib_device *device);
static void cma_remove_one(struct ib_device *device);
@@ -86,6 +91,7 @@ struct cma_device {
struct completion comp;
atomic_t refcount;
struct list_head id_list;
+ struct dentry *rdma_id_dentry;
};
enum cma_state {
@@ -102,6 +108,47 @@ enum cma_state {
CMA_DESTROYING
};
+static const char *format_cma_state(enum cma_state s)
+{
+ switch (s) {
+ case CMA_IDLE: return "IDLE";
+ case CMA_ADDR_QUERY: return "ADDR_QUERY";
+ case CMA_ADDR_RESOLVED: return "ADDR_RESOLVED";
+ case CMA_ROUTE_QUERY: return "ROUTE_QUERY";
+ case CMA_ROUTE_RESOLVED: return "ROUTE_RESOLVED";
+ case CMA_CONNECT: return "CONNECT";
+ case CMA_DISCONNECT: return "DISCONNECT";
+ case CMA_ADDR_BOUND: return "ADDR_BOUND";
+ case CMA_LISTEN: return "LISTEN";
+ case CMA_DEVICE_REMOVAL: return "DEVICE_REMOVAL";
+ case CMA_DESTROYING: return "DESTROYING";
+ }
+ return "";
+}
+
+static const char *format_port_space(enum rdma_port_space ps)
+{
+ switch (ps) {
+ case RDMA_PS_SDP: return "SDP";
+ case RDMA_PS_IPOIB: return "IPOIB";
+ case RDMA_PS_TCP: return "TCP";
+ case RDMA_PS_UDP: return "UDP";
+ case RDMA_PS_SCTP: return "SCTP";
+ }
+ return "";
+}
+
+static const char *format_node_type(enum rdma_node_type nt)
+{
+ if (nt) {
+ switch (rdma_node_get_transport(nt)) {
+ case RDMA_TRANSPORT_IB: return "IB";
+ case RDMA_TRANSPORT_IWARP: return "IW";
+ }
+ }
+ return "";
+}
+
struct rdma_bind_list {
struct idr *ps;
struct hlist_head owners;
@@ -2850,6 +2897,131 @@ static struct notifier_block cma_nb = {
.notifier_call = cma_netdev_callback
};
+static void *cma_rdma_id_seq_start(struct seq_file *file, loff_t *pos)
+{
+ struct cma_device *cma_dev = file->private;
+ void *ret;
+
+ mutex_lock(&lock);
+ if (*pos == 0)
+ return SEQ_START_TOKEN;
+ ret = seq_list_start_head(&cma_dev->id_list, *pos);
+ return ret;
+}
+
+static void *cma_rdma_id_seq_next(struct seq_file *file, void *v, loff_t *pos)
+{
+ void *ret;
+ struct cma_device *cma_dev = file->private;
+ if (v == SEQ_START_TOKEN) {
+ ++*pos;
+ if (!list_empty(&cma_dev->id_list))
+ ret = cma_dev->id_list.next;
+ else
+ ret = NULL;
+ } else {
+ ret = seq_list_next(v, &cma_dev->id_list, pos);
+ }
+ return ret;
+}
+
+static void cma_rdma_id_seq_stop(struct seq_file *file, void *iter_ptr)
+{
+ mutex_unlock(&lock);
+}
+
+static void format_addr(struct sockaddr *sa, char* buf)
+{
+ switch (sa->sa_family) {
+ case AF_INET: {
+ struct sockaddr_in *sin = (struct sockaddr_in *)sa;
+ sprintf(buf, "%pI4:%u", &sin->sin_addr.s_addr,
+ be16_to_cpu(cma_port(sa)));
+ break;
+ }
+ case AF_INET6: {
+ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sa;
+ sprintf(buf, "%pI6:%u", &sin6->sin6_addr,
+ be16_to_cpu(cma_port(sa)));
+ break;
+ }
+ default:
+ buf[0] = 0;
+ }
+}
+
+static int cma_rdma_id_seq_show(struct seq_file *file, void *v)
+{
+ struct rdma_id_private *id_priv;
+ char local_addr[64], remote_addr[64];
+
+ if (!v)
+ return 0;
+ if (v == SEQ_START_TOKEN) {
+ seq_printf(file,
+ "%-5s %-8s %-5s %-8s %-52s %-52s %-6s %-15s %-8s \n",
+ "TYPE", "DEVICE", "PORT", "NET_DEV", "SRC_ADDR", "DST_ADDR", "SPACE", "STATE", "QP_NUM");
+ } else {
+ id_priv = list_entry(v, struct rdma_id_private, list);
+ format_addr((struct sockaddr *)&id_priv->id.route.addr.src_addr,
+ local_addr);
+ format_addr((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
+ remote_addr);
+
+ seq_printf(file,
+ "%-5s %-8s %-5d %-8s %-52s %-52s %-6s %-15s %-8d \n",
+ format_node_type(id_priv->id.route.addr.dev_addr.dev_type),
+ (id_priv->id.device) ? id_priv->id.device->name : "",
+ id_priv->id.port_num,
+ (id_priv->id.route.addr.dev_addr.src_dev) ? id_priv->id.route.addr.dev_addr.src_dev->name : "",
+ local_addr, remote_addr,
+ format_port_space(id_priv->id.ps),
+ format_cma_state(id_priv->state),
+ id_priv->qp_num);
+ }
+ return 0;
+}
+
+static const struct seq_operations cma_rdma_id_seq_ops = {
+ .start = cma_rdma_id_seq_start,
+ .next = cma_rdma_id_seq_next,
+ .stop = cma_rdma_id_seq_stop,
+ .show = cma_rdma_id_seq_show,
+};
+
+static int cma_rdma_id_open(struct inode *inode, struct file *file)
+{
+ struct seq_file *seq;
+ int ret;
+
+ ret = seq_open(file, &cma_rdma_id_seq_ops);
+ if (ret)
+ return ret;
+
+ seq = file->private_data;
+ seq->private = inode->i_private;
+
+ return 0;
+}
+
+static const struct file_operations cma_rdma_id_fops = {
+ .owner = THIS_MODULE,
+ .open = cma_rdma_id_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release
+};
+
+void cma_create_debug_files(struct cma_device *cma_dev)
+{
+ char name[IB_DEVICE_NAME_MAX + sizeof "_rdma_id"];
+ snprintf(name, sizeof name, "%s_rdma_id", cma_dev->device->name);
+ cma_dev->rdma_id_dentry = debugfs_create_file(name, S_IFREG | S_IRUGO,
+ cma_root_dentry, cma_dev, &cma_rdma_id_fops);
+ if (!cma_dev->rdma_id_dentry)
+ printk(KERN_WARNING "RDMA CMA: failed to create debugfs file %s\n", name);
+}
+
static void cma_add_one(struct ib_device *device)
{
struct cma_device *cma_dev;
@@ -2871,6 +3043,7 @@ static void cma_add_one(struct ib_device *device)
list_for_each_entry(id_priv, &listen_any_list, list)
cma_listen_on_dev(id_priv, cma_dev);
mutex_unlock(&lock);
+ cma_create_debug_files(cma_dev);
}
static int cma_remove_id_dev(struct rdma_id_private *id_priv)
@@ -2905,6 +3078,8 @@ static void cma_process_remove(struct cma_device *cma_dev)
int ret;
mutex_lock(&lock);
+ if (cma_dev->rdma_id_dentry)
+ debugfs_remove(cma_dev->rdma_id_dentry);
while (!list_empty(&cma_dev->id_list)) {
id_priv = list_entry(cma_dev->id_list.next,
struct rdma_id_private, list);
@@ -2940,6 +3115,7 @@ static void cma_remove_one(struct ib_device *device)
mutex_unlock(&lock);
cma_process_remove(cma_dev);
+
kfree(cma_dev);
}
@@ -2947,6 +3123,12 @@ static int cma_init(void)
{
int ret, low, high, remaining;
+ cma_root_dentry = debugfs_create_dir("rdma_cm", NULL);
+ if (!cma_root_dentry) {
+ printk(KERN_ERR "RDMA CMA: failed to create debugfs dir\n");
+ return -ENOMEM;
+ }
+
get_random_bytes(&next_port, sizeof next_port);
inet_get_local_port_range(&low, &high);
remaining = (high - low) + 1;
@@ -2984,6 +3166,7 @@ static void cma_cleanup(void)
idr_destroy(&tcp_ps);
idr_destroy(&udp_ps);
idr_destroy(&ipoib_ps);
+ debugfs_remove(cma_root_dentry);
}
module_init(cma_init);
More information about the general
mailing list