[openib-general] [PATCH 18/18] [RFC] cxbg3 User Mode Library
Steve Wise
swise at opengridcomputing.com
Mon Mar 6 10:12:16 PST 2006
--- old/src/userspace/libcxgb3/src/iwch.c 1969-12-31 18:00:00.000000000 -0600
+++ new/src/userspace/libcxgb3/src/iwch.c 2006-03-06 09:26:20.000000000 -0600
@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <pthread.h>
+
+#include "iwch.h"
+#include "iwch-abi.h"
+
+#ifndef PCI_VENDOR_ID_CHELSIO
+#define PCI_VENDOR_ID_CHELSIO 0x1425
+#endif
+
+#ifndef PCI_DEVICE_ID_CHELSIO_IWCH
+#define PCI_DEVICE_ID_CHELSIO_IWCH 0x0020
+#endif
+
+
+#define HCA(v, d, t) \
+ { .vendor = PCI_VENDOR_ID_##v, \
+ .device = PCI_DEVICE_ID_CHELSIO_##d, \
+ .type = CHELSIO_##t }
+
+struct {
+ unsigned vendor;
+ unsigned device;
+ enum iwch_hca_type type;
+} hca_table[] = {
+HCA(CHELSIO, IWCH, IWCH),};
+
+static struct ibv_context_ops iwch_ctx_ops = {
+ .query_device = iwch_query_device,
+ .query_port = iwch_query_port,
+ .alloc_pd = iwch_alloc_pd,
+ .dealloc_pd = iwch_free_pd,
+ .reg_mr = iwch_reg_mr,
+ .dereg_mr = iwch_dereg_mr,
+ .create_cq = iwch_create_cq,
+ .resize_cq = iwch_resize_cq,
+ .poll_cq = iwch_poll_cq,
+ .destroy_cq = iwch_destroy_cq,
+ .create_srq = iwch_create_srq,
+ .modify_srq = iwch_modify_srq,
+ .destroy_srq = iwch_destroy_srq,
+ .create_qp = iwch_create_qp,
+ .modify_qp = iwch_modify_qp,
+ .destroy_qp = iwch_destroy_qp,
+ .create_ah = iwch_create_ah,
+ .destroy_ah = iwch_destroy_ah,
+ .attach_mcast = iwch_attach_mcast,
+ .detach_mcast = iwch_detach_mcast
+};
+
+static struct ibv_context *iwch_alloc_context(struct ibv_device *ibdev,
+ int cmd_fd)
+{
+ struct iwch_context *context;
+ struct ibv_get_context cmd;
+ struct iwch_alloc_ucontext_resp resp;
+
+ context = malloc(sizeof *context);
+ if (!context)
+ return NULL;
+
+ context->ibv_ctx.cmd_fd = cmd_fd;
+
+ if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
+ &resp.ibv_resp, sizeof resp))
+ goto err_free;
+
+ context->ibv_ctx.device = ibdev;
+ context->ibv_ctx.ops = iwch_ctx_ops;
+ context->ibv_ctx.ops.req_notify_cq = iwch_arm_cq;
+ context->ibv_ctx.ops.cq_event = NULL;
+ context->ibv_ctx.ops.post_send = iwch_post_send;
+ context->ibv_ctx.ops.post_recv = iwch_post_recv;
+ context->ibv_ctx.ops.post_srq_recv = iwch_post_srq_recv;
+
+ return &context->ibv_ctx;
+err_free:
+ free(context);
+ return NULL;
+}
+
+static void iwch_free_context(struct ibv_context *ibctx)
+{
+ struct iwch_context *context = to_iwch_ctx(ibctx);
+
+ free(context);
+}
+
+static struct ibv_device_ops iwch_dev_ops = {
+ .alloc_context = iwch_alloc_context,
+ .free_context = iwch_free_context
+};
+
+struct ibv_device *openib_driver_init(struct sysfs_class_device *sysdev)
+{
+ struct sysfs_device *pcidev;
+ struct sysfs_attribute *attr;
+ struct iwch_device *dev;
+ unsigned vendor, device;
+ int i;
+
+ pcidev = sysfs_get_classdev_device(sysdev);
+ if (!pcidev)
+ return NULL;
+ fprintf(stderr, PFX "pcidev 0x%p\n", pcidev);
+
+ attr = sysfs_get_device_attr(pcidev, "vendor");
+ if (!attr)
+ return NULL;
+ sscanf(attr->value, "%i", &vendor);
+ fprintf(stderr, PFX "vendor 0x%p\n", attr->value);
+ sysfs_close_attribute(attr);
+
+ attr = sysfs_get_device_attr(pcidev, "device");
+ fprintf(stderr, PFX "device 0x%p\n", attr->value);
+ if (!attr)
+ return NULL;
+ sscanf(attr->value, "%i", &device);
+ sysfs_close_attribute(attr);
+
+ for (i = 0; i < sizeof hca_table / sizeof hca_table[0]; ++i)
+ if (vendor == hca_table[i].vendor &&
+ device == hca_table[i].device)
+ goto found;
+
+ return NULL;
+
+found:
+ dev = malloc(sizeof *dev);
+ if (!dev) {
+ fprintf(stderr, PFX "Fatal: couldn't allocate device for %s\n",
+ sysdev->name);
+ abort();
+ }
+
+ dev->ibv_dev.ops = iwch_dev_ops;
+ dev->hca_type = hca_table[i].type;
+ dev->page_size = sysconf(_SC_PAGESIZE);
+
+ return &dev->ibv_dev;
+}
--- old/src/userspace/libcxgb3/src/iwch.h 1969-12-31 18:00:00.000000000 -0600
+++ new/src/userspace/libcxgb3/src/iwch.h 2006-03-06 09:26:20.000000000 -0600
@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef IWCH_H
+#define IWCH_H
+
+#include <infiniband/driver.h>
+#include <infiniband/arch.h>
+
+#define HIDDEN __attribute__((visibility ("hidden")))
+
+#define PFX "cxgb3: "
+
+enum iwch_hca_type {
+ CHELSIO_IWCH
+};
+
+struct iwch_device {
+ struct ibv_device ibv_dev;
+ enum iwch_hca_type hca_type;
+ int page_size;
+};
+
+struct iwch_context {
+ struct ibv_context ibv_ctx;
+};
+
+struct iwch_pd {
+ struct ibv_pd ibv_pd;
+};
+
+struct iwch_cq {
+ struct ibv_cq ibv_cq;
+ __u32 cqid;
+ __u32 entries;
+ __u64 physaddr;
+ __u64 queue;
+};
+
+struct iwch_qp {
+ struct ibv_qp ibv_qp;
+ __u32 qpid;
+ __u32 entries;
+ __u64 physaddr;
+ __u64 physsize;
+ __u64 queue;
+};
+
+#define to_iwch_xxx(xxx, type) \
+ ((struct iwch_##type *) \
+ ((void *) ib##xxx - offsetof(struct iwch_##type, ibv_##xxx)))
+
+static inline struct iwch_device *to_iwch_dev(struct ibv_device *ibdev)
+{
+ return to_iwch_xxx(dev, device);
+}
+
+static inline struct iwch_context *to_iwch_ctx(struct ibv_context *ibctx)
+{
+ return to_iwch_xxx(ctx, context);
+}
+
+static inline struct iwch_pd *to_iwch_pd(struct ibv_pd *ibpd)
+{
+ return to_iwch_xxx(pd, pd);
+}
+
+static inline struct iwch_cq *to_iwch_cq(struct ibv_cq *ibcq)
+{
+ return to_iwch_xxx(cq, cq);
+}
+
+static inline struct iwch_qp *to_iwch_qp(struct ibv_qp *ibqp)
+{
+ return to_iwch_xxx(qp, qp);
+}
+
+
+extern int iwch_query_device(struct ibv_context *context,
+ struct ibv_device_attr *attr);
+extern int iwch_query_port(struct ibv_context *context, uint8_t port,
+ struct ibv_port_attr *attr);
+
+extern struct ibv_pd *iwch_alloc_pd(struct ibv_context *context);
+extern int iwch_free_pd(struct ibv_pd *pd);
+
+extern struct ibv_mr *iwch_reg_mr(struct ibv_pd *pd, void *addr,
+ size_t length, enum ibv_access_flags access);
+extern int iwch_dereg_mr(struct ibv_mr *mr);
+
+struct ibv_cq *iwch_create_cq(struct ibv_context *context, int cqe,
+ struct ibv_comp_channel *channel,
+ int comp_vector);
+extern int iwch_resize_cq(struct ibv_cq *cq, int cqe);
+extern int iwch_destroy_cq(struct ibv_cq *cq);
+extern int iwch_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
+extern int iwch_arm_cq(struct ibv_cq *cq, int solicited);
+extern void iwch_cq_event(struct ibv_cq *cq);
+extern void iwch_init_cq_buf(struct iwch_cq *cq, int nent);
+
+extern struct ibv_srq *iwch_create_srq(struct ibv_pd *pd,
+ struct ibv_srq_init_attr *attr);
+extern int iwch_modify_srq(struct ibv_srq *srq,
+ struct ibv_srq_attr *attr,
+ enum ibv_srq_attr_mask mask);
+extern int iwch_destroy_srq(struct ibv_srq *srq);
+extern int iwch_post_srq_recv(struct ibv_srq *ibsrq,
+ struct ibv_recv_wr *wr,
+ struct ibv_recv_wr **bad_wr);
+
+extern struct ibv_qp *iwch_create_qp(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *attr);
+extern int iwch_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+ enum ibv_qp_attr_mask attr_mask);
+extern int iwch_destroy_qp(struct ibv_qp *qp);
+extern int iwch_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
+ struct ibv_send_wr **bad_wr);
+extern int iwch_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
+ struct ibv_recv_wr **bad_wr);
+extern struct ibv_ah *iwch_create_ah(struct ibv_pd *pd,
+ struct ibv_ah_attr *ah_attr);
+extern int iwch_destroy_ah(struct ibv_ah *ah);
+extern int iwch_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid,
+ uint16_t lid);
+extern int iwch_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid,
+ uint16_t lid);
+
+#endif /* IWCH_H */
--- old/src/userspace/libcxgb3/src/cq.c 1969-12-31 18:00:00.000000000 -0600
+++ new/src/userspace/libcxgb3/src/cq.c 2006-03-06 09:26:20.000000000 -0600
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdio.h>
+#include <netinet/in.h>
+#include <pthread.h>
+
+#include <infiniband/opcode.h>
+
+#include "iwch.h"
+#include "iwch-abi.h"
+
+
+int iwch_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc)
+{
+ return ibv_cmd_poll_cq(ibcq, ne, wc);
+}
+
+
+int iwch_arm_cq(struct ibv_cq *cq, int solicited)
+{
+ return ibv_cmd_req_notify_cq(cq, solicited);
+}
+
+
--- old/src/userspace/libcxgb3/src/iwch-abi.h 1969-12-31 18:00:00.000000000 -0600
+++ new/src/userspace/libcxgb3/src/iwch-abi.h 2006-03-06 09:26:20.000000000 -0600
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef IWCH_ABI_H
+#define IWCH_ABI_H
+
+#include <infiniband/kern-abi.h>
+
+struct iwch_alloc_ucontext_resp {
+ struct ibv_get_context_resp ibv_resp;
+};
+
+struct iwch_alloc_pd_resp {
+ struct ibv_alloc_pd_resp ibv_resp;
+};
+
+struct iwch_create_cq {
+ struct ibv_create_cq ibv_cmd;
+};
+
+
+struct iwch_create_cq_resp {
+ struct ibv_create_cq_resp ibv_resp;
+ __u32 cqid;
+ __u32 entries;
+ __u64 physaddr; /* library mmaps this to get addressability */
+ __u64 queue;
+};
+
+struct iwch_create_qp {
+ struct ibv_create_qp ibv_cmd;
+};
+
+struct iwch_create_qp_resp {
+ struct ibv_create_qp_resp ibv_resp;
+ __u32 qpid;
+ __u32 entries; /* actual number of entries after creation */
+ __u64 physaddr; /* library mmaps this to get addressability */
+ __u64 physsize; /* library mmaps this to get addressability */
+ __u64 queue;
+};
+
+
+struct t3_cqe {
+ __u32 header:32;
+ __u32 len:32;
+ __u32 wrid_hi_stag:32;
+ __u32 wrid_low_msn:32;
+};
+
+#endif /* IWCH_ABI_H */
--- old/src/userspace/libcxgb3/src/qp.c 1969-12-31 18:00:00.000000000 -0600
+++ new/src/userspace/libcxgb3/src/qp.c 2006-03-06 09:26:20.000000000 -0600
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <netinet/in.h>
+#include <pthread.h>
+
+#include "iwch.h"
+#include <stdio.h>
+
+int iwch_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
+ struct ibv_send_wr **bad_wr)
+{
+ return ibv_cmd_post_send(ibqp, wr, bad_wr);
+}
+
+int iwch_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
+ struct ibv_recv_wr **bad_wr)
+{
+ return ibv_cmd_post_recv(ibqp, wr, bad_wr);
+}
+
--- old/src/userspace/libcxgb3/src/verbs.c 1969-12-31 18:00:00.000000000 -0600
+++ new/src/userspace/libcxgb3/src/verbs.c 2006-03-06 09:26:20.000000000 -0600
@@ -0,0 +1,308 @@
+/*
+ * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <strings.h>
+#include <errno.h>
+#include <pthread.h>
+#include <sys/mman.h>
+#include <netinet/in.h>
+
+#include "iwch.h"
+#include "iwch-abi.h"
+
+
+int iwch_query_device(struct ibv_context *context, struct ibv_device_attr *attr)
+{
+ struct ibv_query_device cmd;
+ uint64_t raw_fw_ver;
+ unsigned major, minor, sub_minor;
+ int ret;
+
+ ret =
+ ibv_cmd_query_device(context, attr, &raw_fw_ver, &cmd, sizeof cmd);
+ fprintf(stderr, "ibv_cmd_query_device ret = 0x%x\n", ret);
+ if (ret)
+ return ret;
+
+ major = (raw_fw_ver >> 32) & 0xffff;
+ minor = (raw_fw_ver >> 16) & 0xffff;
+ sub_minor = raw_fw_ver & 0xffff;
+
+ snprintf(attr->fw_ver, sizeof attr->fw_ver,
+ "%d.%d.%d", major, minor, sub_minor);
+
+ return 0;
+}
+
+int iwch_query_port(struct ibv_context *context, uint8_t port,
+ struct ibv_port_attr *attr)
+{
+ struct ibv_query_port cmd;
+
+ return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
+}
+
+struct ibv_pd *iwch_alloc_pd(struct ibv_context *context)
+{
+ struct ibv_alloc_pd cmd;
+ struct iwch_alloc_pd_resp resp;
+ struct iwch_pd *pd;
+
+ pd = malloc(sizeof *pd);
+ if (!pd)
+ return NULL;
+
+ if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof cmd,
+ &resp.ibv_resp, sizeof resp)) {
+ free(pd);
+ return NULL;
+ }
+
+ return &pd->ibv_pd;
+}
+
+int iwch_free_pd(struct ibv_pd *pd)
+{
+ int ret;
+
+ ret = ibv_cmd_dealloc_pd(pd);
+ if (ret)
+ return ret;
+
+ free(pd);
+ return 0;
+}
+
+static struct ibv_mr *__iwch_reg_mr(struct ibv_pd *pd, void *addr,
+ size_t length, uint64_t hca_va,
+ enum ibv_access_flags access)
+{
+ struct ibv_mr *mr;
+ struct ibv_reg_mr cmd;
+
+ mr = malloc(sizeof *mr);
+ if (!mr)
+ return NULL;
+
+ if (ibv_cmd_reg_mr(pd, addr, length, hca_va,
+ access, mr, &cmd, sizeof cmd)) {
+ fprintf(stderr, "ibv_cmd_reg_mr failed\n");
+ free(mr);
+ return NULL;
+ }
+
+ return mr;
+}
+
+struct ibv_mr *iwch_reg_mr(struct ibv_pd *pd, void *addr,
+ size_t length, enum ibv_access_flags access)
+{
+ return __iwch_reg_mr(pd, addr, length, (uintptr_t) addr, access);
+}
+
+int iwch_dereg_mr(struct ibv_mr *mr)
+{
+ int ret;
+
+ ret = ibv_cmd_dereg_mr(mr);
+ if (ret)
+ return ret;
+
+ free(mr);
+ return 0;
+}
+
+struct ibv_cq *iwch_create_cq(struct ibv_context *context, int cqe,
+ struct ibv_comp_channel *channel, int comp_vector)
+{
+ struct iwch_create_cq cmd;
+ struct iwch_create_cq_resp resp;
+ struct iwch_cq *cq;
+ int ret;
+
+ fprintf(stderr, "iwch_create_cq called\n");
+
+ cq = malloc(sizeof *cq);
+ if (!cq) {
+ goto err;
+ }
+
+ fprintf(stderr, "Calling ibv_cmd_create_cq\n");
+ ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
+ &cq->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
+ &resp.ibv_resp, sizeof resp);
+ if (ret)
+ goto err;
+
+#if 0 /* A reminder for bypass functionality */
+ cq->physaddr = resp.physaddr;
+ cq->queue =
+ (unsigned long) mmap(NULL, cqe * sizeof(struct t3_cqe), PROT_WRITE,
+ MAP_SHARED, context->cmd_fd, cq->physaddr);
+#endif
+
+ return &cq->ibv_cq;
+
+
+err:
+ free(cq);
+
+ return NULL;
+}
+
+int iwch_resize_cq(struct ibv_cq *cq, int cqe)
+{
+ int ret;
+ struct ibv_resize_cq cmd;
+
+ ret = ibv_cmd_resize_cq(cq, cqe, &cmd, sizeof cmd);
+ if (ret)
+ return ret;
+ /* We will need to unmap and remap when we implement user mode */
+
+ return 0;
+}
+
+int iwch_destroy_cq(struct ibv_cq *cq)
+{
+ int ret;
+
+ ret = ibv_cmd_destroy_cq(cq);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+struct ibv_srq *iwch_create_srq(struct ibv_pd *pd,
+ struct ibv_srq_init_attr *attr)
+{
+ return (void *) -ENOSYS;
+}
+
+int iwch_modify_srq(struct ibv_srq *srq,
+ struct ibv_srq_attr *attr, enum ibv_srq_attr_mask attr_mask)
+{
+ return -ENOSYS;
+}
+
+int iwch_destroy_srq(struct ibv_srq *srq)
+{
+ return -ENOSYS;
+}
+
+int iwch_post_srq_recv(struct ibv_srq *ibsrq,
+ struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr)
+{
+ return -ENOSYS;
+}
+
+struct ibv_qp *iwch_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
+{
+ struct iwch_create_qp cmd;
+ struct iwch_create_qp_resp resp;
+ struct iwch_qp *qp;
+ int ret;
+
+ /* Sanity check QP size before proceeding */
+ if (attr->cap.max_send_wr > 65536 ||
+ attr->cap.max_recv_wr > 65536 ||
+ attr->cap.max_send_sge > 4 ||
+ attr->cap.max_recv_sge > 4 || attr->cap.max_inline_data > 1024)
+ return NULL;
+
+ qp = malloc(sizeof *qp);
+ if (!qp)
+ return NULL;
+
+ ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd,
+ &resp.ibv_resp, sizeof resp);
+ if (ret)
+ return NULL;
+
+#if 0 /* A reminder for bypass functionality */
+ qp->physaddr = resp.physaddr;
+#endif
+
+ return &qp->ibv_qp;
+
+
+ return NULL;
+}
+
+int iwch_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+ enum ibv_qp_attr_mask attr_mask)
+{
+ struct ibv_modify_qp cmd;
+
+ return ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof cmd);
+}
+
+int iwch_destroy_qp(struct ibv_qp *qp)
+{
+ int ret;
+
+ ret = ibv_cmd_destroy_qp(qp);
+ if (ret)
+ return ret;
+
+ free(qp);
+
+ return 0;
+}
+
+struct ibv_ah *iwch_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
+{
+ return (void *) -ENOSYS;
+}
+
+int iwch_destroy_ah(struct ibv_ah *ah)
+{
+ return -ENOSYS;
+}
+
+int iwch_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
+{
+ return -ENOSYS;
+}
+
+int iwch_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
+{
+ return -ENOSYS;
+}
+
More information about the general
mailing list