[openib-general] [RFC] [PATCH] ping: Add IB ping server agent
Hal Rosenstock
halr at voltaire.com
Thu Mar 17 12:39:40 PST 2005
ping: Add IB ping server agent (used with ibping diagnostic tool)
Signed-off-by: Shahar Frank <shaharf at voltaire.com>
Signed-off-by: Hal Rosenstock <halr at voltaire.com>
Index: ping_priv.h
===================================================================
--- ping_priv.h (revision 0)
+++ ping_priv.h (revision 0)
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2004 Infinicon Corporation. All rights reserved.
+ * Copyright (c) 2004 Intel Corporation. All rights reserved.
+ * Copyright (c) 2004 Topspin Corporation. All rights reserved.
+ * Copyright (c) 2004 Voltaire Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#ifndef __IB_PING_PRIV_H__
+#define __IB_PING_PRIV_H__
+
+#include <linux/pci.h>
+
+#define SPFX "ib_ping: "
+
+struct ib_ping_send_wr {
+ struct list_head send_list;
+ struct ib_ah *ah;
+ struct ib_mad_private *mad;
+ DECLARE_PCI_UNMAP_ADDR(mapping)
+};
+
+struct ib_ping_port_private {
+ struct list_head port_list;
+ struct list_head send_posted_list;
+ spinlock_t send_list_lock;
+ int port_num;
+ struct ib_mad_agent *pingd_agent; /* OpenIB Ping class */
+};
+
+#endif /* __IB_PING_PRIV_H__ */
Index: ping.h
===================================================================
--- ping.h (revision 0)
+++ ping.h (revision 0)
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2004 Infinicon Corporation. All rights reserved.
+ * Copyright (c) 2004 Intel Corporation. All rights reserved.
+ * Copyright (c) 2004 Topspin Corporation. All rights reserved.
+ * Copyright (c) 2004 Voltaire Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#ifndef __PING_H_
+#define __PING_H_
+
+extern spinlock_t ib_ping_port_list_lock;
+
+extern int ib_ping_port_open(struct ib_device *device,
+ int port_num);
+
+extern int ib_ping_port_close(struct ib_device *device, int port_num);
+
+#endif /* __PING_H_ */
Index: ping.c
===================================================================
--- ping.c (revision 0)
+++ ping.c (revision 0)
@@ -0,0 +1,336 @@
+/*
+ * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2004 Infinicon Corporation. All rights reserved.
+ * Copyright (c) 2004 Intel Corporation. All rights reserved.
+ * Copyright (c) 2004 Topspin Corporation. All rights reserved.
+ * Copyright (c) 2004 Voltaire Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/utsname.h>
+#include <asm/bug.h>
+
+#include "ping_priv.h"
+#include "mad_priv.h"
+#include "ping.h"
+
+spinlock_t ib_ping_port_list_lock;
+static LIST_HEAD(ib_ping_port_list);
+
+/*
+ * Caller must hold ib_ping_port_list_lock
+ */
+static inline struct ib_ping_port_private *
+__ib_get_ping_port(struct ib_device *device, int port_num,
+ struct ib_mad_agent *mad_agent)
+{
+ struct ib_ping_port_private *entry;
+
+ BUG_ON(!(!!device ^ !!mad_agent)); /* Exactly one MUST be (!NULL) */
+
+ if (device) {
+ list_for_each_entry(entry, &ib_ping_port_list, port_list) {
+ if (entry->pingd_agent->device == device &&
+ entry->port_num == port_num)
+ return entry;
+ }
+ } else {
+ list_for_each_entry(entry, &ib_ping_port_list, port_list) {
+ if (entry->pingd_agent == mad_agent)
+ return entry;
+ }
+ }
+ return NULL;
+}
+
+static inline struct ib_ping_port_private *
+ib_get_ping_port(struct ib_device *device, int port_num,
+ struct ib_mad_agent *mad_agent)
+{
+ struct ib_ping_port_private *entry;
+ unsigned long flags;
+
+ spin_lock_irqsave(&ib_ping_port_list_lock, flags);
+ entry = __ib_get_ping_port(device, port_num, mad_agent);
+ spin_unlock_irqrestore(&ib_ping_port_list_lock, flags);
+
+ return entry;
+}
+
+static int ping_mad_send(struct ib_mad_agent *mad_agent,
+ struct ib_ping_port_private *port_priv,
+ struct ib_mad_private *mad_priv,
+ struct ib_grh *grh,
+ struct ib_wc *wc)
+{
+ struct ib_ping_send_wr *ping_send_wr;
+ struct ib_sge gather_list;
+ struct ib_send_wr send_wr;
+ struct ib_send_wr *bad_send_wr;
+ struct ib_ah_attr ah_attr;
+ unsigned long flags;
+ int ret = 1;
+
+ ping_send_wr = kmalloc(sizeof(*ping_send_wr), GFP_KERNEL);
+ if (!ping_send_wr)
+ goto out;
+ ping_send_wr->mad = mad_priv;
+
+ /* PCI mapping */
+ gather_list.addr = dma_map_single(mad_agent->device->dma_device,
+ &mad_priv->mad,
+ sizeof(mad_priv->mad),
+ DMA_TO_DEVICE);
+ gather_list.length = sizeof(mad_priv->mad);
+ gather_list.lkey = mad_agent->mr->lkey;
+
+ send_wr.next = NULL;
+ send_wr.opcode = IB_WR_SEND;
+ send_wr.sg_list = &gather_list;
+ send_wr.num_sge = 1;
+ send_wr.wr.ud.remote_qpn = wc->src_qp; /* DQPN */
+ send_wr.wr.ud.timeout_ms = 0;
+ send_wr.send_flags = IB_SEND_SIGNALED | IB_SEND_SOLICITED;
+
+ ah_attr.dlid = wc->slid;
+ ah_attr.port_num = mad_agent->port_num;
+ ah_attr.src_path_bits = wc->dlid_path_bits;
+ ah_attr.sl = wc->sl;
+ ah_attr.static_rate = 0;
+ ah_attr.ah_flags = 0; /* No GRH */
+ if (mad_priv->mad.mad.mad_hdr.mgmt_class == IB_MGMT_CLASS_OPENIB_PING) {
+ if (wc->wc_flags & IB_WC_GRH) {
+ ah_attr.ah_flags = IB_AH_GRH;
+ /* Should sgid be looked up ? */
+ ah_attr.grh.sgid_index = 0;
+ ah_attr.grh.hop_limit = grh->hop_limit;
+ ah_attr.grh.flow_label = be32_to_cpup(
+ &grh->version_tclass_flow) & 0xfffff;
+ ah_attr.grh.traffic_class = (be32_to_cpup(
+ &grh->version_tclass_flow) >> 20) & 0xff;
+ memcpy(ah_attr.grh.dgid.raw,
+ grh->sgid.raw,
+ sizeof(ah_attr.grh.dgid));
+ }
+ } else {
+ printk(KERN_ERR SPFX "Not OpenIB ping class 0x%x\n",
+ mad_priv->mad.mad.mad_hdr.mgmt_class);
+ kfree(ping_send_wr);
+ goto out;
+ }
+
+ ping_send_wr->ah = ib_create_ah(mad_agent->qp->pd, &ah_attr);
+ if (IS_ERR(ping_send_wr->ah)) {
+ printk(KERN_ERR SPFX "No memory for address handle\n");
+ kfree(ping_send_wr);
+ goto out;
+ }
+
+ send_wr.wr.ud.ah = ping_send_wr->ah;
+ send_wr.wr.ud.pkey_index = wc->pkey_index;
+ send_wr.wr.ud.remote_qkey = IB_QP1_QKEY;
+ send_wr.wr.ud.mad_hdr = &mad_priv->mad.mad.mad_hdr;
+ send_wr.wr_id = (unsigned long)ping_send_wr;
+
+ pci_unmap_addr_set(ping_send_wr, mapping, gather_list.addr);
+
+ /* Send */
+ spin_lock_irqsave(&port_priv->send_list_lock, flags);
+ if (ib_post_send_mad(mad_agent, &send_wr, &bad_send_wr)) {
+ spin_unlock_irqrestore(&port_priv->send_list_lock, flags);
+ dma_unmap_single(mad_agent->device->dma_device,
+ pci_unmap_addr(ping_send_wr, mapping),
+ sizeof(mad_priv->mad),
+ DMA_TO_DEVICE);
+ ib_destroy_ah(ping_send_wr->ah);
+ kfree(ping_send_wr);
+ } else {
+ list_add_tail(&ping_send_wr->send_list,
+ &port_priv->send_posted_list);
+ spin_unlock_irqrestore(&port_priv->send_list_lock, flags);
+ ret = 0;
+ }
+
+out:
+ return ret;
+}
+
+static void pingd_recv_handler(struct ib_mad_agent *mad_agent,
+ struct ib_mad_recv_wc *mad_recv_wc)
+{
+ struct ib_ping_port_private *port_priv;
+ struct ib_vendor_mad *vend;
+ struct ib_mad_private *recv = container_of(mad_recv_wc,
+ struct ib_mad_private,
+ header.recv_wc);
+
+ /* Find matching MAD agent */
+ port_priv = ib_get_ping_port(NULL, 0, mad_agent);
+ if (!port_priv) {
+ kmem_cache_free(ib_mad_cache, recv);
+ printk(KERN_ERR SPFX "pingd_recv_handler: no matching MAD "
+ "agent %p\n", mad_agent);
+ return;
+ }
+
+ vend = (struct ib_vendor_mad *)mad_recv_wc->recv_buf.mad;
+
+ vend->mad_hdr.method |= IB_MGMT_METHOD_RESP;
+ vend->mad_hdr.status = 0;
+ if (!system_utsname.domainname[0])
+ strncpy(vend->data, system_utsname.nodename, sizeof vend->data);
+ else
+ snprintf(vend->data, sizeof vend->data, "%s.%s",
+ system_utsname.nodename, system_utsname.domainname);
+
+ /* Send response */
+ if (ping_mad_send(mad_agent, port_priv, recv,
+ mad_recv_wc->recv_buf.grh, mad_recv_wc->wc)) {
+ kmem_cache_free(ib_mad_cache, recv);
+ printk(KERN_ERR SPFX "pingd_recv_handler: reply failed\n");
+ }
+}
+
+static void pingd_send_handler(struct ib_mad_agent *mad_agent,
+ struct ib_mad_send_wc *mad_send_wc)
+{
+ struct ib_ping_port_private *port_priv;
+ struct ib_ping_send_wr *ping_send_wr;
+ unsigned long flags;
+
+ /* Find matching MAD agent */
+ port_priv = ib_get_ping_port(NULL, 0, mad_agent);
+ if (!port_priv) {
+ printk(KERN_ERR SPFX "pingd_send_handler: no matching MAD "
+ "agent %p\n", mad_agent);
+ return;
+ }
+
+ ping_send_wr = (struct ib_ping_send_wr *)(unsigned long)mad_send_wc->wr_id;
+ spin_lock_irqsave(&port_priv->send_list_lock, flags);
+ /* Remove completed send from posted send MAD list */
+ list_del(&ping_send_wr->send_list);
+ spin_unlock_irqrestore(&port_priv->send_list_lock, flags);
+
+ /* Unmap PCI */
+ dma_unmap_single(mad_agent->device->dma_device,
+ pci_unmap_addr(ping_send_wr, mapping),
+ sizeof(ping_send_wr->mad->mad),
+ DMA_TO_DEVICE);
+
+ ib_destroy_ah(ping_send_wr->ah);
+
+ /* Release allocated memory */
+ kmem_cache_free(ib_mad_cache, ping_send_wr->mad);
+ kfree(ping_send_wr);
+}
+
+int ib_ping_port_open(struct ib_device *device, int port_num)
+{
+ int ret;
+ struct ib_ping_port_private *port_priv;
+ struct ib_mad_reg_req pingd_reg_req;
+ unsigned long flags;
+
+ /* First, check if port already open */
+ port_priv = ib_get_ping_port(device, port_num, NULL);
+ if (port_priv) {
+ printk(KERN_DEBUG SPFX "%s port %d already open\n",
+ device->name, port_num);
+ return 0;
+ }
+
+ /* Create new device info */
+ port_priv = kmalloc(sizeof *port_priv, GFP_KERNEL);
+ if (!port_priv) {
+ printk(KERN_ERR SPFX "No memory for ib_ping_port_private\n");
+ ret = -ENOMEM;
+ goto error1;
+ }
+
+ memset(port_priv, 0, sizeof *port_priv);
+ port_priv->port_num = port_num;
+ spin_lock_init(&port_priv->send_list_lock);
+ INIT_LIST_HEAD(&port_priv->send_posted_list);
+
+ pingd_reg_req.mgmt_class = IB_MGMT_CLASS_OPENIB_PING;
+ pingd_reg_req.mgmt_class_version = 1;
+ pingd_reg_req.oui[0] = (IB_OPENIB_OUI >> 16) & 0xff;
+ pingd_reg_req.oui[1] = (IB_OPENIB_OUI >> 8) & 0xff;
+ pingd_reg_req.oui[2] = IB_OPENIB_OUI & 0xff;
+ set_bit(IB_MGMT_METHOD_GET, pingd_reg_req.method_mask);
+
+ /* Obtain server MAD agent for OpenIB Ping class (GSI QP) */
+ port_priv->pingd_agent = ib_register_mad_agent(device, port_num,
+ IB_QPT_GSI,
+ &pingd_reg_req, 0,
+ &pingd_send_handler,
+ &pingd_recv_handler,
+ NULL);
+ if (IS_ERR(port_priv->pingd_agent)) {
+ ret = PTR_ERR(port_priv->pingd_agent);
+ goto error2;
+ }
+
+ spin_lock_irqsave(&ib_ping_port_list_lock, flags);
+ list_add_tail(&port_priv->port_list, &ib_ping_port_list);
+ spin_unlock_irqrestore(&ib_ping_port_list_lock, flags);
+
+ return 0;
+
+error2:
+ kfree(port_priv);
+error1:
+ return ret;
+}
+
+int ib_ping_port_close(struct ib_device *device, int port_num)
+{
+ struct ib_ping_port_private *port_priv;
+ unsigned long flags;
+
+ spin_lock_irqsave(&ib_ping_port_list_lock, flags);
+ port_priv = __ib_get_ping_port(device, port_num, NULL);
+ if (port_priv == NULL) {
+ spin_unlock_irqrestore(&ib_ping_port_list_lock, flags);
+ printk(KERN_ERR SPFX "Port %d not found\n", port_num);
+ return -ENODEV;
+ }
+ list_del(&port_priv->port_list);
+ spin_unlock_irqrestore(&ib_ping_port_list_lock, flags);
+
+ ib_unregister_mad_agent(port_priv->pingd_agent);
+ kfree(port_priv);
+
+ return 0;
+}
Index: mad.c
===================================================================
--- mad.c (revision 2019)
+++ mad.c (working copy)
@@ -37,7 +37,9 @@
#include "mad_priv.h"
#include "smi.h"
#include "agent.h"
+#include "ping.h"
+
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("kernel IB MAD API");
MODULE_AUTHOR("Hal Rosenstock");
@@ -2624,6 +2626,12 @@
device->name, cur_port);
goto error_device_open;
}
+ ret = ib_ping_port_open(device, cur_port);
+ if (ret) {
+ printk(KERN_ERR PFX "Couldn't open %s port %d "
+ "for ping agent\n",
+ device->name, cur_port);
+ }
}
goto error_device_query;
@@ -2631,6 +2639,12 @@
error_device_open:
while (i > 0) {
cur_port--;
+ ret2 = ib_ping_port_close(device, cur_port);
+ if (ret2) {
+ printk(KERN_ERR PFX "Couldn't close %s port %d "
+ "for ping agent\n",
+ device->name, cur_port);
+ }
ret2 = ib_agent_port_close(device, cur_port);
if (ret2) {
printk(KERN_ERR PFX "Couldn't close %s port %d "
@@ -2661,6 +2675,12 @@
cur_port = 1;
}
for (i = 0; i < num_ports; i++, cur_port++) {
+ ret2 = ib_ping_port_close(device, cur_port);
+ if (ret2) {
+ printk(KERN_ERR PFX "Couldn't close %s port %d "
+ "for ping agent\n",
+ device->name, cur_port);
+ }
ret2 = ib_agent_port_close(device, cur_port);
if (ret2) {
printk(KERN_ERR PFX "Couldn't close %s port %d "
@@ -2691,6 +2711,7 @@
spin_lock_init(&ib_mad_port_list_lock);
spin_lock_init(&ib_agent_port_list_lock);
+ spin_lock_init(&ib_ping_port_list_lock);
ib_mad_cache = kmem_cache_create("ib_mad",
sizeof(struct ib_mad_private),
Index: Makefile
===================================================================
--- Makefile (revision 2017)
+++ Makefile (working copy)
@@ -5,7 +5,7 @@
ib_core-y := packer.o ud_header.o verbs.o sysfs.o \
device.o fmr_pool.o cache.o
-ib_mad-y := mad.o smi.o agent.o
+ib_mad-y := mad.o smi.o agent.o ping.o
ib_cm-y := cm.o
More information about the general
mailing list