[openib-general] [PATCH 6/10] Driver IB files - IB core stack interaction

Ramachandra K rkuchimanchi at silverstorm.com
Mon Oct 2 13:08:02 PDT 2006


Adds the files that implement interaction with the core IB stack.


Signed-off-by: Ramachandra K <rkuchimanchi at silverstorm.com>
---

 drivers/infiniband/ulp/vnic/vnic_ib.c |  709 +++++++++++++++++++++++++++++++++
 drivers/infiniband/ulp/vnic/vnic_ib.h |  167 ++++++++
 2 files changed, 876 insertions(+), 0 deletions(-)

diff --git a/drivers/infiniband/ulp/vnic/vnic_ib.c b/drivers/infiniband/ulp/vnic/vnic_ib.c
new file mode 100644
index 0000000..0c50b83
--- /dev/null
+++ b/drivers/infiniband/ulp/vnic/vnic_ib.c
@@ -0,0 +1,709 @@
+/*
+ * Copyright (c) 2006 SilverStorm Technologies Inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/string.h>
+#include <linux/random.h>
+#include <linux/netdevice.h>
+#include <linux/list.h>
+#include <rdma/ib_cache.h>
+
+#include "vnic_util.h"
+#include "vnic_config.h"
+#include "vnic_ib.h"
+#include "vnic_viport.h"
+#include "vnic_sys.h"
+#include "vnic_main.h"
+
+static int vnic_ib_inited = 0;
+
+static void vnic_add_one(struct ib_device *device);
+static void vnic_remove_one(struct ib_device *device);
+
+static struct ib_client vnic_client = {
+	.name = "vnic",
+	.add = vnic_add_one,
+	.remove = vnic_remove_one
+};
+
+static struct ib_sa_client vnic_sa_client;
+
+static CLASS_DEVICE_ATTR(create_primary, S_IWUSR, NULL, vnic_create_primary);
+static CLASS_DEVICE_ATTR(create_secondary, S_IWUSR, NULL,
+			 vnic_create_secondary);
+
+static CLASS_DEVICE_ATTR(delete_vnic, S_IWUSR, NULL, vnic_delete);
+
+static struct vnic_ib_port *vnic_add_port(struct vnic_ib_device *device, u8 port_num)
+{
+	struct vnic_ib_port *port;
+
+	port = kzalloc(sizeof *port, GFP_KERNEL);
+	if (!port)
+		return NULL;
+
+	init_completion(&port->cdev_info.released);
+	port->dev = device;
+	port->port_num = port_num;
+
+	port->cdev_info.class_dev.class = &vnic_class;
+	port->cdev_info.class_dev.dev = device->dev->dma_device;
+	snprintf(port->cdev_info.class_dev.class_id, BUS_ID_SIZE, "vnic-%s-%d",
+		 device->dev->name, port_num);
+
+	if (class_device_register(&port->cdev_info.class_dev))
+		goto free_port;
+
+	if (class_device_create_file(&port->cdev_info.class_dev,
+				     &class_device_attr_create_primary))
+		goto err_class;
+	if (class_device_create_file(&port->cdev_info.class_dev,
+				     &class_device_attr_create_secondary))
+		goto err_class;
+
+	return port;
+err_class:
+	class_device_unregister(&port->cdev_info.class_dev);
+
+free_port:
+	kfree(port);
+
+	return NULL;
+}
+
+static void vnic_add_one(struct ib_device *device)
+{
+	struct vnic_ib_device *vnic_dev;
+	struct vnic_ib_port *port;
+	int s, e, p;
+
+	vnic_dev = kmalloc(sizeof *vnic_dev, GFP_KERNEL);
+	vnic_dev->dev = device;
+	if (!vnic_dev)
+		return;
+
+	INIT_LIST_HEAD(&vnic_dev->dev_list);
+	if (device->node_type == RDMA_NODE_IB_SWITCH) {
+		s = 0;
+		e = 0;
+
+	} else {
+		s = 1;
+		e = device->phys_port_cnt;
+
+	}
+
+	for (p = s; p <= e; p++) {
+		port = vnic_add_port(vnic_dev, p);
+		if (port)
+			list_add_tail(&port->list, &vnic_dev->dev_list);
+	}
+
+	ib_set_client_data(device, &vnic_client, vnic_dev);
+
+}
+
+static void vnic_remove_one(struct ib_device *device)
+{
+	struct vnic_ib_device *vnic_dev;
+	struct vnic_ib_port *port, *tmp_port;
+
+	vnic_dev = ib_get_client_data(device, &vnic_client);
+	list_for_each_entry_safe(port, tmp_port, &vnic_dev->dev_list, list) {
+		class_device_unregister(&port->cdev_info.class_dev);
+		/*
+		 * wait for sysfs entries to go away, so that no new vnics
+		 * are created
+		 */
+		wait_for_completion(&port->cdev_info.released);
+		kfree(port);
+
+	}
+	kfree(vnic_dev);
+}
+
+BOOLEAN vnic_ib_init()
+{
+	int ret;
+
+	IB_FUNCTION("vnic_ib_init()\n");
+
+	/* class has to be registered before
+	 * calling ib_register_client() because, that call
+	 * will trigger vnic_add_port() which will register
+	 * class_device of vnic_class for the port
+	 */
+	ret = class_register(&vnic_class);
+	if (ret) {
+		printk(KERN_ERR "couldn't register class infiniband_vnic");
+		goto out;
+	}
+
+	ib_sa_register_client(&vnic_sa_client);
+
+	ret = ib_register_client(&vnic_client);
+	if (ret) {
+		printk(KERN_ERR "couldn't register IB client");
+		goto err_ib_reg;
+	}
+
+	interface_cdev.class_dev.class = &vnic_class;
+	snprintf(interface_cdev.class_dev.class_id, BUS_ID_SIZE, "interfaces");
+
+	init_completion(&interface_cdev.released);
+	if (class_device_register(&interface_cdev.class_dev))
+		goto err_class_dev;
+
+	if (class_device_create_file(&interface_cdev.class_dev,
+				     &class_device_attr_delete_vnic))
+		goto err_class_file;
+
+	vnic_ib_inited = 1;
+
+	return TRUE;
+
+err_class_file:
+	class_device_unregister(&interface_cdev.class_dev);
+err_class_dev:
+	ib_unregister_client(&vnic_client);
+err_ib_reg:
+	ib_sa_unregister_client(&vnic_sa_client);
+	class_unregister(&vnic_class);
+out:
+	return FALSE;
+}
+
+void vnic_ib_cleanup()
+{
+	IB_FUNCTION("vnic_ib_cleanup()\n");
+
+	if (!vnic_ib_inited)
+		return;
+
+	class_device_unregister(&interface_cdev.class_dev);
+	wait_for_completion(&interface_cdev.released);
+
+	ib_unregister_client(&vnic_client);
+	ib_sa_unregister_client(&vnic_sa_client);
+	class_unregister(&vnic_class);
+
+	return;
+}
+
+static void vnic_path_rec_completion(int status,
+				     struct ib_sa_path_rec *pathrec,
+				     void *context)
+{
+	struct ib_path_info *p = context;
+	p->status = status;
+	if (!status)
+		p->path = *pathrec;
+
+	complete(&p->done);
+}
+
+BOOLEAN vnic_ib_get_path(struct netpath *netpath, struct vnic * vnic)
+{
+	struct viport_config *config = netpath->viport->config;
+
+	init_completion(&config->path_info.done);
+	IB_INFO("Using SA path rec get time out value of %d\n",
+	       config->sa_path_rec_get_timeout);
+	config->path_info.path_query_id =
+			 ib_sa_path_rec_get(&vnic_sa_client,
+					    config->ibdev,
+					    config->port,
+					    &config->path_info.path,
+					    IB_SA_PATH_REC_DGID      |
+					    IB_SA_PATH_REC_SGID      |
+					    IB_SA_PATH_REC_NUMB_PATH |
+					    IB_SA_PATH_REC_PKEY,
+					    config->sa_path_rec_get_timeout,
+					    GFP_KERNEL,
+					    vnic_path_rec_completion,
+					    &config->path_info,
+					    &config->path_info.path_query);
+
+	if (config->path_info.path_query_id < 0) {
+		IB_ERROR("SA path record query failed\n");
+		return FALSE;
+	}
+
+	wait_for_completion(&config->path_info.done);
+
+	if (config->path_info.status < 0) {
+		printk(KERN_WARNING PFX "path record query failed for dgid "
+		       "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
+		       (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+					dgid.raw[0]),
+		       (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+					dgid.raw[2]),
+		       (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+					dgid.raw[4]),
+		       (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+					dgid.raw[6]),
+		       (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+					dgid.raw[8]),
+		       (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+					dgid.raw[10]),
+		       (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+					dgid.raw[12]),
+		       (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+					dgid.raw[14]));
+
+		if (config->path_info.status == -ETIMEDOUT)
+			printk(KERN_WARNING PFX
+			       "reason: path record query timed out\n");
+		else if (config->path_info.status == -EIO)
+			printk(KERN_WARNING PFX
+			       "reason: error in sending path record query\n");
+
+		netpath_timer(netpath, vnic->config->no_path_timeout);
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static void ib_qp_event(struct ib_event *event, void *context)
+{
+	IB_ERROR("QP event %d\n", event->event);
+}
+
+static void vnic_ib_completion(struct ib_cq *cq, void *ptr)
+{
+	struct ib_wc wc;
+	struct io *io;
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+	struct vnic_ib_conn *ib_conn = ptr;
+	cycles_t           comp_time;
+	u32              comp_num = 0;
+	comp_time = get_cycles();
+	ib_conn->statistics.num_callbacks++;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+	ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
+	while (ib_poll_cq(cq, 1, &wc) > 0) {
+		io = (struct io *)(wc.wr_id);
+
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+		ib_conn->statistics.num_ios++;
+		comp_num++;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+		if (wc.status) {
+#if 0
+			IB_ERROR("completion error "
+				 "wc.status %d wc.opcode %d vendor err 0x%x\n",
+				 wc.status, wc.opcode, wc.vendor_err);
+#endif
+		} else if (io) {
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+			if (io->type == RECV) {
+				io->time = comp_time;
+			} else if (io->type == RDMA) {
+				ib_conn->statistics.rdma_comp_time +=
+							 comp_time - io->time;
+				ib_conn->statistics.rdma_comp_ios++;
+			} else if (io->type == SEND) {
+				ib_conn->statistics.send_comp_time +=
+							 comp_time - io->time;
+				ib_conn->statistics.send_comp_ios++;
+			}
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+			if (io->routine)
+				(*io->routine) (io);
+		}
+	}
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+	if (comp_num > ib_conn->statistics.max_ios)
+		ib_conn->statistics.max_ios = comp_num;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+}
+
+int vnic_ib_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
+{
+	struct vnic_ib_conn *ib_conn = cm_id->context;
+	struct viport *viport = ib_conn->viport;
+	struct ib_qp_attr *qp_attr = NULL;
+	int err = 0;
+	int disconn = 0;
+	int attr_mask = 0;
+
+	switch (event->event) {
+	case IB_CM_REQ_ERROR:
+		IB_ERROR("sending CM REQ failed\n");
+		disconn = 1;
+		break;
+	case IB_CM_REP_RECEIVED:
+		IB_INFO("CM REP recvd\n");
+		qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
+		if (!qp_attr) {
+			err = 1;
+			break;
+		}
+
+		qp_attr->qp_state = IB_QPS_RTR;
+		err = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
+		if (err)
+			break;
+
+		err = ib_modify_qp(ib_conn->qp, qp_attr, attr_mask);
+		if (err)
+			break;
+
+		IB_INFO("QP RTR\n");
+
+		qp_attr->qp_state = IB_QPS_RTS;
+		err = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
+		if (err)
+			break;
+
+		err = ib_modify_qp(ib_conn->qp, qp_attr, attr_mask);
+		if (err)
+			break;
+
+		IB_INFO("QP RTS\n");
+
+		err = ib_send_cm_rtu(cm_id, NULL, 0);
+		if (err)
+			break;
+		ib_conn->state = IB_CONN_CONNECTED;
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+		ib_conn->statistics.connection_time =
+			get_cycles() - ib_conn->statistics.connection_time;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+		IB_INFO("RTU SENT\n");
+		break;
+	case IB_CM_REJ_RECEIVED:
+		printk(KERN_ERR PFX "CM rejected control connection \n");
+		if (event->param.rej_rcvd.reason ==
+		    IB_CM_REJ_INVALID_SERVICE_ID)
+			printk(KERN_ERR "reason: invalid service ID. "
+			       "IOCGUID value specified may be incorrect\n");
+		else
+			printk(KERN_ERR "reason code : 0x%x\n",
+			       event->param.rej_rcvd.reason);
+
+		disconn = 1;
+		break;
+	case IB_CM_MRA_RECEIVED:
+		IB_INFO("CM MRA received\n");
+		break;
+
+	case IB_CM_DREP_RECEIVED:
+		IB_INFO("CM DREP recvd\n");
+		ib_conn->state = IB_CONN_DISCONNECTED;
+		break;
+
+	case IB_CM_TIMEWAIT_EXIT:
+		IB_ERROR("CM timewait exit\n");
+		err = 1;
+		break;
+
+	default:
+		IB_INFO("unhandled CM event %d\n", event->event);
+		break;
+
+	}
+
+	if (err) {
+		ib_conn->state = IB_CONN_DISCONNECTED;
+		viport_failure(viport);
+	}
+
+	if (disconn) {
+		ib_conn->state = IB_CONN_DISCONNECTED;
+		viport_disconnect(viport);
+
+	}
+	complete(&ib_conn->done);
+	return 0;
+}
+
+
+BOOLEAN vnic_ib_cm_connect(struct vnic_ib_conn *ib_conn)
+{
+	struct ib_cm_req_param	*req = NULL;
+	struct viport		*viport;
+	int 			ret;
+
+	if (!ib_conn_initted(ib_conn)) {
+		IB_ERROR("IB Connection out of state for CM connect (%d)\n",
+			 ib_conn->state);
+		return FALSE;
+	}
+
+#ifdef INIC_STATISTICS
+	ib_conn->statistics.connection_time = get_cycles();
+#endif
+
+	req = kzalloc(sizeof *req, GFP_KERNEL);
+
+	if (!req)
+		return -ENOMEM;
+
+	viport	= ib_conn->viport;
+
+	req->primary_path	= &viport->config->path_info.path;
+	req->alternate_path	= NULL;
+	req->qp_num		= ib_conn->qp->qp_num;
+	req->qp_type		= ib_conn->qp->qp_type;
+	req->service_id 	= ib_conn->ib_config->service_id;
+	req->private_data	= &ib_conn->ib_config->conn_data;
+	req->private_data_len	= sizeof(struct vnic_connection_data);
+	req->flow_control	= 1;
+
+	get_random_bytes(&req->starting_psn, 4);
+	req->starting_psn &= 0xffffff;
+
+	/*
+	 * Both responder_resources and initiator_depth are set to zero
+	 * as we do not need RDMA read.
+	 *
+	 * They also must be set to zero, otherwise data connections
+	 * are  rejected by VEx.
+	 */
+	req->responder_resources 	= 0;
+	req->initiator_depth		= 0;
+	req->remote_cm_response_timeout = 20;
+	req->local_cm_response_timeout  = 20;
+	req->retry_count		= ib_conn->ib_config->retry_count;
+	req->rnr_retry_count		= ib_conn->ib_config->rnr_retry_count;
+	req->max_cm_retries		= 15;
+
+	ib_conn->state = IB_CONN_CONNECTING;
+
+	ret = ib_send_cm_req(ib_conn->cm_id, req);
+
+	kfree(req);
+
+	if (ret) {
+		IB_ERROR("CM REQ sending failed %d \n", ret);
+		ib_conn->state = IB_CONN_DISCONNECTED;
+		return FALSE;
+	}
+	return TRUE;
+
+}
+
+BOOLEAN vnic_ib_conn_init(struct vnic_ib_conn *ib_conn, struct viport *viport,
+		  struct ib_pd *pd, uint64_t guid,
+		  struct ib_config *config)
+{
+	struct ib_qp_init_attr *init_attr;
+	struct ib_qp_attr *attr;
+	BOOLEAN ret;
+	int retval;
+	struct viport_config *viport_config = viport->config;
+	unsigned int cq_size = config->num_sends + config->num_recvs;
+
+
+	if (!ib_conn_uninitted(ib_conn)) {
+		IB_ERROR("IB Connection out of state for init (%d)\n",
+			 ib_conn->state);
+		return FALSE;
+	}
+
+	init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
+	if (!init_attr)
+		return FALSE;
+
+	ib_conn->cq = ib_create_cq(viport_config->ibdev, vnic_ib_completion,
+			      NULL, ib_conn, cq_size);
+	if (IS_ERR(ib_conn->cq)) {
+		ret = FALSE;
+		IB_ERROR("could not create CQ\n");
+		goto out;
+	}
+
+	ib_req_notify_cq(ib_conn->cq, IB_CQ_NEXT_COMP);
+
+	init_attr->event_handler	= ib_qp_event;
+	init_attr->cap.max_send_wr	= config->num_sends;
+	init_attr->cap.max_recv_wr	= config->num_recvs;
+	init_attr->cap.max_recv_sge	= config->recv_scatter;
+	init_attr->cap.max_send_sge	= config->send_gather;
+	init_attr->sq_sig_type		= IB_SIGNAL_ALL_WR;
+	init_attr->qp_type		= IB_QPT_RC;
+	init_attr->send_cq		= ib_conn->cq;
+	init_attr->recv_cq		= ib_conn->cq;
+
+	ib_conn->qp = ib_create_qp(pd, init_attr);
+
+	if (IS_ERR(ib_conn->qp)) {
+		ret = FALSE;
+		IB_ERROR("could not create QP\n");
+		ib_destroy_cq(ib_conn->cq);
+		goto out;
+	}
+
+	attr = kmalloc(sizeof *attr, GFP_KERNEL);
+	if (!attr) {
+		ret = FALSE;
+		goto out;
+	}
+
+	retval = ib_find_cached_pkey(viport_config->ibdev,
+				     viport_config->port,
+				     be16_to_cpu(viport_config->path_info.path.
+						 pkey),
+				     &attr->pkey_index);
+	if (retval) {
+		ret = FALSE;
+		IB_ERROR("ib_find_cached_pkey() failed\n");
+		goto freeattr;
+	}
+
+	attr->qp_state		= IB_QPS_INIT;
+	attr->qp_access_flags	= IB_ACCESS_REMOTE_WRITE;
+	attr->port_num		= viport_config->port;
+
+	retval = ib_modify_qp(ib_conn->qp, attr,
+			      IB_QP_STATE |
+			      IB_QP_PKEY_INDEX |
+			      IB_QP_ACCESS_FLAGS | IB_QP_PORT);
+	if (retval) {
+		ret = FALSE;
+		IB_ERROR("could not modify QP\n");
+		ib_destroy_qp(ib_conn->qp);
+		ib_destroy_cq(ib_conn->cq);
+		goto freeattr;
+	}
+
+	ib_conn->conn_lock  = SPIN_LOCK_UNLOCKED;
+	ib_conn->state = IB_CONN_INITTED;
+
+	ret = TRUE;
+freeattr:
+	kfree(attr);
+out:
+	kfree(init_attr);
+	return ret;
+}
+
+BOOLEAN vnic_ib_post_recv(struct vnic_ib_conn * ib_conn, struct io * io)
+{
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+	cycles_t post_time;
+#endif	/* CONFIG_INFINIBAND_VNIC_STATS */
+	struct ib_recv_wr *bad_wr;
+	int ret;
+	unsigned long flags;
+
+	IB_FUNCTION("vnic_ib_post_recv()\n");
+
+	spin_lock_irqsave(&ib_conn->conn_lock, flags);
+
+	if (!ib_conn_initted(ib_conn) && !ib_conn_connected(ib_conn))
+		goto post_fail;
+
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+	io->type = RECV;
+	post_time = get_cycles();
+	if (io->time != 0) {
+		ib_conn->statistics.recv_comp_time += post_time - io->time;
+		ib_conn->statistics.recv_comp_ios++;
+	}
+#endif	/* CONFIG_INFINIBAND_VNIC_STATS */
+	ret = ib_post_recv(ib_conn->qp, &io->rwr, &bad_wr);
+
+	if (ret) {
+		IB_ERROR("error in posting rcv wr\n");
+		goto post_fail;
+	}
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+	post_time = get_cycles() - post_time;
+	ib_conn->statistics.recv_post_time += post_time;
+	ib_conn->statistics.recv_post_ios++;
+#endif	/* CONFIG_INFINIBAND_VNIC_STATS */
+	spin_unlock_irqrestore(&ib_conn->conn_lock, flags);
+	return TRUE;
+post_fail:
+	spin_unlock_irqrestore(&ib_conn->conn_lock, flags);
+	return FALSE;
+
+}
+
+BOOLEAN vnic_ib_post_send(struct vnic_ib_conn * ib_conn, struct io * io)
+{
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+	cycles_t      post_time;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+	unsigned long flags;
+	struct ib_send_wr *bad_wr;
+	int ret;
+
+	IB_FUNCTION("vnic_ib_post_send()\n");
+
+	spin_lock_irqsave(&ib_conn->conn_lock, flags);
+
+	if (!ib_conn_connected(ib_conn)) {
+		IB_ERROR("IB Connection out of state for posting sends (%d)\n",
+			 ib_conn->state);
+		return FALSE;
+	}
+
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+	io->time = post_time = get_cycles();
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+	if (io->swr.opcode == IB_WR_RDMA_WRITE)
+		io->type = RDMA;
+	else
+		io->type = SEND;
+
+	ret = ib_post_send(ib_conn->qp, &io->swr, &bad_wr);
+
+	if (ret)
+		goto send_post_fail;
+
+
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+	post_time = get_cycles() - post_time;
+
+	if (io->swr.opcode == IB_WR_RDMA_WRITE) {
+		ib_conn->statistics.rdma_post_time += post_time;
+		ib_conn->statistics.rdma_post_ios++;
+	} else {
+		ib_conn->statistics.send_post_time += post_time;
+		ib_conn->statistics.send_post_ios++;
+	}
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+	spin_unlock_irqrestore(&ib_conn->conn_lock, flags);
+	return TRUE;
+send_post_fail:
+	spin_unlock_irqrestore(&ib_conn->conn_lock, flags);
+	return FALSE;
+}
diff --git a/drivers/infiniband/ulp/vnic/vnic_ib.h b/drivers/infiniband/ulp/vnic/vnic_ib.h
new file mode 100644
index 0000000..002396f
--- /dev/null
+++ b/drivers/infiniband/ulp/vnic/vnic_ib.h
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2006 SilverStorm Technologies Inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef VNIC_IB_H_INCLUDED
+#define VNIC_IB_H_INCLUDED
+
+#include <linux/timex.h>
+#include <linux/completion.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_pack.h>
+#include <rdma/ib_sa.h>
+#include <rdma/ib_cm.h>
+
+#include "vnic_sys.h"
+#include "vnic_netpath.h"
+#define PFX	"ib_vnic: "
+
+struct io;
+typedef void (comp_routine_t) (struct io * io);
+
+enum ib_conn_state {
+	IB_CONN_UNINITTED = 0,
+	IB_CONN_INITTED,
+	IB_CONN_CONNECTING,
+	IB_CONN_CONNECTED,
+	IB_CONN_DISCONNECTED
+};
+
+struct vnic_ib_conn {
+	struct viport		*viport;
+	struct ib_config	*ib_config;
+	spinlock_t		conn_lock;
+	enum ib_conn_state	state;
+	struct ib_qp		*qp;
+	struct ib_cq		*cq;
+	struct ib_cm_id		*cm_id;
+	struct completion	done;
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+	struct {
+		cycles_t	connection_time;
+		cycles_t	rdma_post_time;
+		u32		rdma_post_ios;
+		cycles_t	rdma_comp_time;
+		u32		rdma_comp_ios;
+		cycles_t	send_post_time;
+		u32		send_post_ios;
+		cycles_t	send_comp_time;
+		u32		send_comp_ios;
+		cycles_t	recv_post_time;
+		u32		recv_post_ios;
+		cycles_t	recv_comp_time;
+		u32		recv_comp_ios;
+		u32		num_ios;
+		u32		num_callbacks;
+		u32		max_ios;
+	} statistics;
+#endif	/* CONFIG_INFINIBAND_VNIC_STATS */
+};
+
+struct ib_path_info {
+	struct ib_sa_path_rec	path;
+	struct ib_sa_query	*path_query;
+	int			path_query_id;
+	int			status;
+	struct			completion done;
+};
+
+struct vnic_ib_device {
+	struct list_head	dev_list;
+	struct ib_device	*dev;
+
+};
+
+struct vnic;
+
+struct vnic_ib_port {
+	struct vnic_ib_device	*dev;
+	u8			port_num;
+	struct class_dev_info	cdev_info;
+	struct list_head	list;
+};
+
+struct io {
+	struct list_head	list_ptrs;
+	struct viport		*viport;
+	comp_routine_t		*routine;
+	struct ib_recv_wr	rwr;
+	struct ib_send_wr	swr;
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+	cycles_t		time;
+#endif	/* CONFIG_INFINIBAND_VNIC_STATS */
+	enum { RECV, RDMA, SEND } type;
+};
+
+struct rdma_io {
+	struct io		io;
+	struct ib_sge		list[2];
+	u16			index;
+	u16			len;
+	u8			*data;
+	dma_addr_t		data_dma;
+	struct sk_buff		*skb;
+	dma_addr_t		skb_data_dma;
+	struct viport_trailer 	*trailer;
+	dma_addr_t 		trailer_dma;
+};
+
+struct send_io {
+	struct io	io;
+	struct ib_sge	list;
+	u8		*virtual_addr;
+};
+
+struct recv_io {
+	struct io	io;
+	struct ib_sge	list;
+	u8		*virtual_addr;
+};
+
+BOOLEAN	vnic_ib_init(void);
+void	vnic_ib_cleanup(void);
+
+BOOLEAN vnic_ib_get_path(struct netpath *netpath, struct vnic * vnic);
+BOOLEAN	vnic_ib_conn_init(struct vnic_ib_conn *ib_conn, struct viport *viport,
+		  	  struct ib_pd *pd, u64 guid, struct ib_config *config);
+
+BOOLEAN	vnic_ib_post_recv(struct vnic_ib_conn *ib_conn, struct io *io);
+BOOLEAN	vnic_ib_post_send(struct vnic_ib_conn *ib_conn, struct io *io);
+BOOLEAN	vnic_ib_cm_connect(struct vnic_ib_conn *ib_conn);
+int	vnic_ib_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
+
+#define	ib_conn_uninitted(ib_conn)	((ib_conn)->state == IB_CONN_UNINITTED)
+#define	ib_conn_initted(ib_conn)	((ib_conn)->state == IB_CONN_INITTED)
+#define	ib_conn_connecting(ib_conn)	((ib_conn)->state == IB_CONN_CONNECTING)
+#define	ib_conn_connected(ib_conn)	((ib_conn)->state == IB_CONN_CONNECTED)
+#define	ib_conn_disconnected(ib_conn)	((ib_conn)->state == IB_CONN_DISCONNECTED)
+
+#endif	/* VNIC_IB_H_INCLUDED */





More information about the general mailing list