[openib-general] [PATCH 6/10] Driver IB files - IB core stack interaction
Ramachandra K
rkuchimanchi at silverstorm.com
Mon Oct 2 13:08:02 PDT 2006
Adds the files that implement interaction with the core IB stack.
Signed-off-by: Ramachandra K <rkuchimanchi at silverstorm.com>
---
drivers/infiniband/ulp/vnic/vnic_ib.c | 709 +++++++++++++++++++++++++++++++++
drivers/infiniband/ulp/vnic/vnic_ib.h | 167 ++++++++
2 files changed, 876 insertions(+), 0 deletions(-)
diff --git a/drivers/infiniband/ulp/vnic/vnic_ib.c b/drivers/infiniband/ulp/vnic/vnic_ib.c
new file mode 100644
index 0000000..0c50b83
--- /dev/null
+++ b/drivers/infiniband/ulp/vnic/vnic_ib.c
@@ -0,0 +1,709 @@
+/*
+ * Copyright (c) 2006 SilverStorm Technologies Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/string.h>
+#include <linux/random.h>
+#include <linux/netdevice.h>
+#include <linux/list.h>
+#include <rdma/ib_cache.h>
+
+#include "vnic_util.h"
+#include "vnic_config.h"
+#include "vnic_ib.h"
+#include "vnic_viport.h"
+#include "vnic_sys.h"
+#include "vnic_main.h"
+
+static int vnic_ib_inited = 0;
+
+static void vnic_add_one(struct ib_device *device);
+static void vnic_remove_one(struct ib_device *device);
+
+static struct ib_client vnic_client = {
+ .name = "vnic",
+ .add = vnic_add_one,
+ .remove = vnic_remove_one
+};
+
+static struct ib_sa_client vnic_sa_client;
+
+static CLASS_DEVICE_ATTR(create_primary, S_IWUSR, NULL, vnic_create_primary);
+static CLASS_DEVICE_ATTR(create_secondary, S_IWUSR, NULL,
+ vnic_create_secondary);
+
+static CLASS_DEVICE_ATTR(delete_vnic, S_IWUSR, NULL, vnic_delete);
+
+static struct vnic_ib_port *vnic_add_port(struct vnic_ib_device *device, u8 port_num)
+{
+ struct vnic_ib_port *port;
+
+ port = kzalloc(sizeof *port, GFP_KERNEL);
+ if (!port)
+ return NULL;
+
+ init_completion(&port->cdev_info.released);
+ port->dev = device;
+ port->port_num = port_num;
+
+ port->cdev_info.class_dev.class = &vnic_class;
+ port->cdev_info.class_dev.dev = device->dev->dma_device;
+ snprintf(port->cdev_info.class_dev.class_id, BUS_ID_SIZE, "vnic-%s-%d",
+ device->dev->name, port_num);
+
+ if (class_device_register(&port->cdev_info.class_dev))
+ goto free_port;
+
+ if (class_device_create_file(&port->cdev_info.class_dev,
+ &class_device_attr_create_primary))
+ goto err_class;
+ if (class_device_create_file(&port->cdev_info.class_dev,
+ &class_device_attr_create_secondary))
+ goto err_class;
+
+ return port;
+err_class:
+ class_device_unregister(&port->cdev_info.class_dev);
+
+free_port:
+ kfree(port);
+
+ return NULL;
+}
+
+static void vnic_add_one(struct ib_device *device)
+{
+ struct vnic_ib_device *vnic_dev;
+ struct vnic_ib_port *port;
+ int s, e, p;
+
+ vnic_dev = kmalloc(sizeof *vnic_dev, GFP_KERNEL);
+ vnic_dev->dev = device;
+ if (!vnic_dev)
+ return;
+
+ INIT_LIST_HEAD(&vnic_dev->dev_list);
+ if (device->node_type == RDMA_NODE_IB_SWITCH) {
+ s = 0;
+ e = 0;
+
+ } else {
+ s = 1;
+ e = device->phys_port_cnt;
+
+ }
+
+ for (p = s; p <= e; p++) {
+ port = vnic_add_port(vnic_dev, p);
+ if (port)
+ list_add_tail(&port->list, &vnic_dev->dev_list);
+ }
+
+ ib_set_client_data(device, &vnic_client, vnic_dev);
+
+}
+
+static void vnic_remove_one(struct ib_device *device)
+{
+ struct vnic_ib_device *vnic_dev;
+ struct vnic_ib_port *port, *tmp_port;
+
+ vnic_dev = ib_get_client_data(device, &vnic_client);
+ list_for_each_entry_safe(port, tmp_port, &vnic_dev->dev_list, list) {
+ class_device_unregister(&port->cdev_info.class_dev);
+ /*
+ * wait for sysfs entries to go away, so that no new vnics
+ * are created
+ */
+ wait_for_completion(&port->cdev_info.released);
+ kfree(port);
+
+ }
+ kfree(vnic_dev);
+}
+
+BOOLEAN vnic_ib_init()
+{
+ int ret;
+
+ IB_FUNCTION("vnic_ib_init()\n");
+
+ /* class has to be registered before
+ * calling ib_register_client() because, that call
+ * will trigger vnic_add_port() which will register
+ * class_device of vnic_class for the port
+ */
+ ret = class_register(&vnic_class);
+ if (ret) {
+ printk(KERN_ERR "couldn't register class infiniband_vnic");
+ goto out;
+ }
+
+ ib_sa_register_client(&vnic_sa_client);
+
+ ret = ib_register_client(&vnic_client);
+ if (ret) {
+ printk(KERN_ERR "couldn't register IB client");
+ goto err_ib_reg;
+ }
+
+ interface_cdev.class_dev.class = &vnic_class;
+ snprintf(interface_cdev.class_dev.class_id, BUS_ID_SIZE, "interfaces");
+
+ init_completion(&interface_cdev.released);
+ if (class_device_register(&interface_cdev.class_dev))
+ goto err_class_dev;
+
+ if (class_device_create_file(&interface_cdev.class_dev,
+ &class_device_attr_delete_vnic))
+ goto err_class_file;
+
+ vnic_ib_inited = 1;
+
+ return TRUE;
+
+err_class_file:
+ class_device_unregister(&interface_cdev.class_dev);
+err_class_dev:
+ ib_unregister_client(&vnic_client);
+err_ib_reg:
+ ib_sa_unregister_client(&vnic_sa_client);
+ class_unregister(&vnic_class);
+out:
+ return FALSE;
+}
+
+void vnic_ib_cleanup()
+{
+ IB_FUNCTION("vnic_ib_cleanup()\n");
+
+ if (!vnic_ib_inited)
+ return;
+
+ class_device_unregister(&interface_cdev.class_dev);
+ wait_for_completion(&interface_cdev.released);
+
+ ib_unregister_client(&vnic_client);
+ ib_sa_unregister_client(&vnic_sa_client);
+ class_unregister(&vnic_class);
+
+ return;
+}
+
+static void vnic_path_rec_completion(int status,
+ struct ib_sa_path_rec *pathrec,
+ void *context)
+{
+ struct ib_path_info *p = context;
+ p->status = status;
+ if (!status)
+ p->path = *pathrec;
+
+ complete(&p->done);
+}
+
+BOOLEAN vnic_ib_get_path(struct netpath *netpath, struct vnic * vnic)
+{
+ struct viport_config *config = netpath->viport->config;
+
+ init_completion(&config->path_info.done);
+ IB_INFO("Using SA path rec get time out value of %d\n",
+ config->sa_path_rec_get_timeout);
+ config->path_info.path_query_id =
+ ib_sa_path_rec_get(&vnic_sa_client,
+ config->ibdev,
+ config->port,
+ &config->path_info.path,
+ IB_SA_PATH_REC_DGID |
+ IB_SA_PATH_REC_SGID |
+ IB_SA_PATH_REC_NUMB_PATH |
+ IB_SA_PATH_REC_PKEY,
+ config->sa_path_rec_get_timeout,
+ GFP_KERNEL,
+ vnic_path_rec_completion,
+ &config->path_info,
+ &config->path_info.path_query);
+
+ if (config->path_info.path_query_id < 0) {
+ IB_ERROR("SA path record query failed\n");
+ return FALSE;
+ }
+
+ wait_for_completion(&config->path_info.done);
+
+ if (config->path_info.status < 0) {
+ printk(KERN_WARNING PFX "path record query failed for dgid "
+ "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
+ (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+ dgid.raw[0]),
+ (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+ dgid.raw[2]),
+ (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+ dgid.raw[4]),
+ (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+ dgid.raw[6]),
+ (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+ dgid.raw[8]),
+ (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+ dgid.raw[10]),
+ (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+ dgid.raw[12]),
+ (int)be16_to_cpu(*(__be16 *) &config->path_info.path.
+ dgid.raw[14]));
+
+ if (config->path_info.status == -ETIMEDOUT)
+ printk(KERN_WARNING PFX
+ "reason: path record query timed out\n");
+ else if (config->path_info.status == -EIO)
+ printk(KERN_WARNING PFX
+ "reason: error in sending path record query\n");
+
+ netpath_timer(netpath, vnic->config->no_path_timeout);
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static void ib_qp_event(struct ib_event *event, void *context)
+{
+ IB_ERROR("QP event %d\n", event->event);
+}
+
+static void vnic_ib_completion(struct ib_cq *cq, void *ptr)
+{
+ struct ib_wc wc;
+ struct io *io;
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ struct vnic_ib_conn *ib_conn = ptr;
+ cycles_t comp_time;
+ u32 comp_num = 0;
+ comp_time = get_cycles();
+ ib_conn->statistics.num_callbacks++;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+ ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
+ while (ib_poll_cq(cq, 1, &wc) > 0) {
+ io = (struct io *)(wc.wr_id);
+
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ ib_conn->statistics.num_ios++;
+ comp_num++;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+ if (wc.status) {
+#if 0
+ IB_ERROR("completion error "
+ "wc.status %d wc.opcode %d vendor err 0x%x\n",
+ wc.status, wc.opcode, wc.vendor_err);
+#endif
+ } else if (io) {
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ if (io->type == RECV) {
+ io->time = comp_time;
+ } else if (io->type == RDMA) {
+ ib_conn->statistics.rdma_comp_time +=
+ comp_time - io->time;
+ ib_conn->statistics.rdma_comp_ios++;
+ } else if (io->type == SEND) {
+ ib_conn->statistics.send_comp_time +=
+ comp_time - io->time;
+ ib_conn->statistics.send_comp_ios++;
+ }
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+ if (io->routine)
+ (*io->routine) (io);
+ }
+ }
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ if (comp_num > ib_conn->statistics.max_ios)
+ ib_conn->statistics.max_ios = comp_num;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+}
+
+int vnic_ib_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
+{
+ struct vnic_ib_conn *ib_conn = cm_id->context;
+ struct viport *viport = ib_conn->viport;
+ struct ib_qp_attr *qp_attr = NULL;
+ int err = 0;
+ int disconn = 0;
+ int attr_mask = 0;
+
+ switch (event->event) {
+ case IB_CM_REQ_ERROR:
+ IB_ERROR("sending CM REQ failed\n");
+ disconn = 1;
+ break;
+ case IB_CM_REP_RECEIVED:
+ IB_INFO("CM REP recvd\n");
+ qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
+ if (!qp_attr) {
+ err = 1;
+ break;
+ }
+
+ qp_attr->qp_state = IB_QPS_RTR;
+ err = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
+ if (err)
+ break;
+
+ err = ib_modify_qp(ib_conn->qp, qp_attr, attr_mask);
+ if (err)
+ break;
+
+ IB_INFO("QP RTR\n");
+
+ qp_attr->qp_state = IB_QPS_RTS;
+ err = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
+ if (err)
+ break;
+
+ err = ib_modify_qp(ib_conn->qp, qp_attr, attr_mask);
+ if (err)
+ break;
+
+ IB_INFO("QP RTS\n");
+
+ err = ib_send_cm_rtu(cm_id, NULL, 0);
+ if (err)
+ break;
+ ib_conn->state = IB_CONN_CONNECTED;
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ ib_conn->statistics.connection_time =
+ get_cycles() - ib_conn->statistics.connection_time;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+ IB_INFO("RTU SENT\n");
+ break;
+ case IB_CM_REJ_RECEIVED:
+ printk(KERN_ERR PFX "CM rejected control connection \n");
+ if (event->param.rej_rcvd.reason ==
+ IB_CM_REJ_INVALID_SERVICE_ID)
+ printk(KERN_ERR "reason: invalid service ID. "
+ "IOCGUID value specified may be incorrect\n");
+ else
+ printk(KERN_ERR "reason code : 0x%x\n",
+ event->param.rej_rcvd.reason);
+
+ disconn = 1;
+ break;
+ case IB_CM_MRA_RECEIVED:
+ IB_INFO("CM MRA received\n");
+ break;
+
+ case IB_CM_DREP_RECEIVED:
+ IB_INFO("CM DREP recvd\n");
+ ib_conn->state = IB_CONN_DISCONNECTED;
+ break;
+
+ case IB_CM_TIMEWAIT_EXIT:
+ IB_ERROR("CM timewait exit\n");
+ err = 1;
+ break;
+
+ default:
+ IB_INFO("unhandled CM event %d\n", event->event);
+ break;
+
+ }
+
+ if (err) {
+ ib_conn->state = IB_CONN_DISCONNECTED;
+ viport_failure(viport);
+ }
+
+ if (disconn) {
+ ib_conn->state = IB_CONN_DISCONNECTED;
+ viport_disconnect(viport);
+
+ }
+ complete(&ib_conn->done);
+ return 0;
+}
+
+
+BOOLEAN vnic_ib_cm_connect(struct vnic_ib_conn *ib_conn)
+{
+ struct ib_cm_req_param *req = NULL;
+ struct viport *viport;
+ int ret;
+
+ if (!ib_conn_initted(ib_conn)) {
+ IB_ERROR("IB Connection out of state for CM connect (%d)\n",
+ ib_conn->state);
+ return FALSE;
+ }
+
+#ifdef INIC_STATISTICS
+ ib_conn->statistics.connection_time = get_cycles();
+#endif
+
+ req = kzalloc(sizeof *req, GFP_KERNEL);
+
+ if (!req)
+ return -ENOMEM;
+
+ viport = ib_conn->viport;
+
+ req->primary_path = &viport->config->path_info.path;
+ req->alternate_path = NULL;
+ req->qp_num = ib_conn->qp->qp_num;
+ req->qp_type = ib_conn->qp->qp_type;
+ req->service_id = ib_conn->ib_config->service_id;
+ req->private_data = &ib_conn->ib_config->conn_data;
+ req->private_data_len = sizeof(struct vnic_connection_data);
+ req->flow_control = 1;
+
+ get_random_bytes(&req->starting_psn, 4);
+ req->starting_psn &= 0xffffff;
+
+ /*
+ * Both responder_resources and initiator_depth are set to zero
+ * as we do not need RDMA read.
+ *
+ * They also must be set to zero, otherwise data connections
+ * are rejected by VEx.
+ */
+ req->responder_resources = 0;
+ req->initiator_depth = 0;
+ req->remote_cm_response_timeout = 20;
+ req->local_cm_response_timeout = 20;
+ req->retry_count = ib_conn->ib_config->retry_count;
+ req->rnr_retry_count = ib_conn->ib_config->rnr_retry_count;
+ req->max_cm_retries = 15;
+
+ ib_conn->state = IB_CONN_CONNECTING;
+
+ ret = ib_send_cm_req(ib_conn->cm_id, req);
+
+ kfree(req);
+
+ if (ret) {
+ IB_ERROR("CM REQ sending failed %d \n", ret);
+ ib_conn->state = IB_CONN_DISCONNECTED;
+ return FALSE;
+ }
+ return TRUE;
+
+}
+
+BOOLEAN vnic_ib_conn_init(struct vnic_ib_conn *ib_conn, struct viport *viport,
+ struct ib_pd *pd, uint64_t guid,
+ struct ib_config *config)
+{
+ struct ib_qp_init_attr *init_attr;
+ struct ib_qp_attr *attr;
+ BOOLEAN ret;
+ int retval;
+ struct viport_config *viport_config = viport->config;
+ unsigned int cq_size = config->num_sends + config->num_recvs;
+
+
+ if (!ib_conn_uninitted(ib_conn)) {
+ IB_ERROR("IB Connection out of state for init (%d)\n",
+ ib_conn->state);
+ return FALSE;
+ }
+
+ init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
+ if (!init_attr)
+ return FALSE;
+
+ ib_conn->cq = ib_create_cq(viport_config->ibdev, vnic_ib_completion,
+ NULL, ib_conn, cq_size);
+ if (IS_ERR(ib_conn->cq)) {
+ ret = FALSE;
+ IB_ERROR("could not create CQ\n");
+ goto out;
+ }
+
+ ib_req_notify_cq(ib_conn->cq, IB_CQ_NEXT_COMP);
+
+ init_attr->event_handler = ib_qp_event;
+ init_attr->cap.max_send_wr = config->num_sends;
+ init_attr->cap.max_recv_wr = config->num_recvs;
+ init_attr->cap.max_recv_sge = config->recv_scatter;
+ init_attr->cap.max_send_sge = config->send_gather;
+ init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
+ init_attr->qp_type = IB_QPT_RC;
+ init_attr->send_cq = ib_conn->cq;
+ init_attr->recv_cq = ib_conn->cq;
+
+ ib_conn->qp = ib_create_qp(pd, init_attr);
+
+ if (IS_ERR(ib_conn->qp)) {
+ ret = FALSE;
+ IB_ERROR("could not create QP\n");
+ ib_destroy_cq(ib_conn->cq);
+ goto out;
+ }
+
+ attr = kmalloc(sizeof *attr, GFP_KERNEL);
+ if (!attr) {
+ ret = FALSE;
+ goto out;
+ }
+
+ retval = ib_find_cached_pkey(viport_config->ibdev,
+ viport_config->port,
+ be16_to_cpu(viport_config->path_info.path.
+ pkey),
+ &attr->pkey_index);
+ if (retval) {
+ ret = FALSE;
+ IB_ERROR("ib_find_cached_pkey() failed\n");
+ goto freeattr;
+ }
+
+ attr->qp_state = IB_QPS_INIT;
+ attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
+ attr->port_num = viport_config->port;
+
+ retval = ib_modify_qp(ib_conn->qp, attr,
+ IB_QP_STATE |
+ IB_QP_PKEY_INDEX |
+ IB_QP_ACCESS_FLAGS | IB_QP_PORT);
+ if (retval) {
+ ret = FALSE;
+ IB_ERROR("could not modify QP\n");
+ ib_destroy_qp(ib_conn->qp);
+ ib_destroy_cq(ib_conn->cq);
+ goto freeattr;
+ }
+
+ ib_conn->conn_lock = SPIN_LOCK_UNLOCKED;
+ ib_conn->state = IB_CONN_INITTED;
+
+ ret = TRUE;
+freeattr:
+ kfree(attr);
+out:
+ kfree(init_attr);
+ return ret;
+}
+
+BOOLEAN vnic_ib_post_recv(struct vnic_ib_conn * ib_conn, struct io * io)
+{
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ cycles_t post_time;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+ struct ib_recv_wr *bad_wr;
+ int ret;
+ unsigned long flags;
+
+ IB_FUNCTION("vnic_ib_post_recv()\n");
+
+ spin_lock_irqsave(&ib_conn->conn_lock, flags);
+
+ if (!ib_conn_initted(ib_conn) && !ib_conn_connected(ib_conn))
+ goto post_fail;
+
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ io->type = RECV;
+ post_time = get_cycles();
+ if (io->time != 0) {
+ ib_conn->statistics.recv_comp_time += post_time - io->time;
+ ib_conn->statistics.recv_comp_ios++;
+ }
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+ ret = ib_post_recv(ib_conn->qp, &io->rwr, &bad_wr);
+
+ if (ret) {
+ IB_ERROR("error in posting rcv wr\n");
+ goto post_fail;
+ }
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ post_time = get_cycles() - post_time;
+ ib_conn->statistics.recv_post_time += post_time;
+ ib_conn->statistics.recv_post_ios++;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+ spin_unlock_irqrestore(&ib_conn->conn_lock, flags);
+ return TRUE;
+post_fail:
+ spin_unlock_irqrestore(&ib_conn->conn_lock, flags);
+ return FALSE;
+
+}
+
+BOOLEAN vnic_ib_post_send(struct vnic_ib_conn * ib_conn, struct io * io)
+{
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ cycles_t post_time;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+ unsigned long flags;
+ struct ib_send_wr *bad_wr;
+ int ret;
+
+ IB_FUNCTION("vnic_ib_post_send()\n");
+
+ spin_lock_irqsave(&ib_conn->conn_lock, flags);
+
+ if (!ib_conn_connected(ib_conn)) {
+ IB_ERROR("IB Connection out of state for posting sends (%d)\n",
+ ib_conn->state);
+ return FALSE;
+ }
+
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ io->time = post_time = get_cycles();
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+
+ if (io->swr.opcode == IB_WR_RDMA_WRITE)
+ io->type = RDMA;
+ else
+ io->type = SEND;
+
+ ret = ib_post_send(ib_conn->qp, &io->swr, &bad_wr);
+
+ if (ret)
+ goto send_post_fail;
+
+
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ post_time = get_cycles() - post_time;
+
+ if (io->swr.opcode == IB_WR_RDMA_WRITE) {
+ ib_conn->statistics.rdma_post_time += post_time;
+ ib_conn->statistics.rdma_post_ios++;
+ } else {
+ ib_conn->statistics.send_post_time += post_time;
+ ib_conn->statistics.send_post_ios++;
+ }
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+ spin_unlock_irqrestore(&ib_conn->conn_lock, flags);
+ return TRUE;
+send_post_fail:
+ spin_unlock_irqrestore(&ib_conn->conn_lock, flags);
+ return FALSE;
+}
diff --git a/drivers/infiniband/ulp/vnic/vnic_ib.h b/drivers/infiniband/ulp/vnic/vnic_ib.h
new file mode 100644
index 0000000..002396f
--- /dev/null
+++ b/drivers/infiniband/ulp/vnic/vnic_ib.h
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2006 SilverStorm Technologies Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef VNIC_IB_H_INCLUDED
+#define VNIC_IB_H_INCLUDED
+
+#include <linux/timex.h>
+#include <linux/completion.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_pack.h>
+#include <rdma/ib_sa.h>
+#include <rdma/ib_cm.h>
+
+#include "vnic_sys.h"
+#include "vnic_netpath.h"
+#define PFX "ib_vnic: "
+
+struct io;
+typedef void (comp_routine_t) (struct io * io);
+
+enum ib_conn_state {
+ IB_CONN_UNINITTED = 0,
+ IB_CONN_INITTED,
+ IB_CONN_CONNECTING,
+ IB_CONN_CONNECTED,
+ IB_CONN_DISCONNECTED
+};
+
+struct vnic_ib_conn {
+ struct viport *viport;
+ struct ib_config *ib_config;
+ spinlock_t conn_lock;
+ enum ib_conn_state state;
+ struct ib_qp *qp;
+ struct ib_cq *cq;
+ struct ib_cm_id *cm_id;
+ struct completion done;
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ struct {
+ cycles_t connection_time;
+ cycles_t rdma_post_time;
+ u32 rdma_post_ios;
+ cycles_t rdma_comp_time;
+ u32 rdma_comp_ios;
+ cycles_t send_post_time;
+ u32 send_post_ios;
+ cycles_t send_comp_time;
+ u32 send_comp_ios;
+ cycles_t recv_post_time;
+ u32 recv_post_ios;
+ cycles_t recv_comp_time;
+ u32 recv_comp_ios;
+ u32 num_ios;
+ u32 num_callbacks;
+ u32 max_ios;
+ } statistics;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+};
+
+struct ib_path_info {
+ struct ib_sa_path_rec path;
+ struct ib_sa_query *path_query;
+ int path_query_id;
+ int status;
+ struct completion done;
+};
+
+struct vnic_ib_device {
+ struct list_head dev_list;
+ struct ib_device *dev;
+
+};
+
+struct vnic;
+
+struct vnic_ib_port {
+ struct vnic_ib_device *dev;
+ u8 port_num;
+ struct class_dev_info cdev_info;
+ struct list_head list;
+};
+
+struct io {
+ struct list_head list_ptrs;
+ struct viport *viport;
+ comp_routine_t *routine;
+ struct ib_recv_wr rwr;
+ struct ib_send_wr swr;
+#ifdef CONFIG_INFINIBAND_VNIC_STATS
+ cycles_t time;
+#endif /* CONFIG_INFINIBAND_VNIC_STATS */
+ enum { RECV, RDMA, SEND } type;
+};
+
+struct rdma_io {
+ struct io io;
+ struct ib_sge list[2];
+ u16 index;
+ u16 len;
+ u8 *data;
+ dma_addr_t data_dma;
+ struct sk_buff *skb;
+ dma_addr_t skb_data_dma;
+ struct viport_trailer *trailer;
+ dma_addr_t trailer_dma;
+};
+
+struct send_io {
+ struct io io;
+ struct ib_sge list;
+ u8 *virtual_addr;
+};
+
+struct recv_io {
+ struct io io;
+ struct ib_sge list;
+ u8 *virtual_addr;
+};
+
+BOOLEAN vnic_ib_init(void);
+void vnic_ib_cleanup(void);
+
+BOOLEAN vnic_ib_get_path(struct netpath *netpath, struct vnic * vnic);
+BOOLEAN vnic_ib_conn_init(struct vnic_ib_conn *ib_conn, struct viport *viport,
+ struct ib_pd *pd, u64 guid, struct ib_config *config);
+
+BOOLEAN vnic_ib_post_recv(struct vnic_ib_conn *ib_conn, struct io *io);
+BOOLEAN vnic_ib_post_send(struct vnic_ib_conn *ib_conn, struct io *io);
+BOOLEAN vnic_ib_cm_connect(struct vnic_ib_conn *ib_conn);
+int vnic_ib_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
+
+#define ib_conn_uninitted(ib_conn) ((ib_conn)->state == IB_CONN_UNINITTED)
+#define ib_conn_initted(ib_conn) ((ib_conn)->state == IB_CONN_INITTED)
+#define ib_conn_connecting(ib_conn) ((ib_conn)->state == IB_CONN_CONNECTING)
+#define ib_conn_connected(ib_conn) ((ib_conn)->state == IB_CONN_CONNECTED)
+#define ib_conn_disconnected(ib_conn) ((ib_conn)->state == IB_CONN_DISCONNECTED)
+
+#endif /* VNIC_IB_H_INCLUDED */
More information about the general
mailing list