[openib-general] [PATCH v1 1/2] AMSO1100 Verbs Library.

Steve Wise swise at opengridcomputing.com
Tue Jun 20 13:04:34 PDT 2006


This code implements user verbs for the Ammasso 1100 device.  This library
doesn't do kernel bypass (but it could someday).
---

 libamso/src/amso-abi.h |   79 +++++++++++++
 libamso/src/amso.c     |  180 +++++++++++++++++++++++++++++
 libamso/src/amso.h     |  156 +++++++++++++++++++++++++
 libamso/src/amso.map   |    6 +
 libamso/src/cq.c       |   57 +++++++++
 libamso/src/qp.c       |   55 +++++++++
 libamso/src/verbs.c    |  303 ++++++++++++++++++++++++++++++++++++++++++++++++
 7 files changed, 836 insertions(+), 0 deletions(-)

diff --git a/libamso/src/amso-abi.h b/libamso/src/amso-abi.h
new file mode 100644
index 0000000..a3df617
--- /dev/null
+++ b/libamso/src/amso-abi.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef AMSO_ABI_H
+#define AMSO_ABI_H
+
+#include <infiniband/kern-abi.h>
+
+struct amso_alloc_ucontext_resp {
+	struct ibv_get_context_resp ibv_resp;
+};
+
+struct amso_alloc_pd_resp {
+	struct ibv_alloc_pd_resp ibv_resp;
+};
+
+struct amso_create_cq {
+	struct ibv_create_cq ibv_cmd;
+};
+
+
+struct amso_create_cq_resp {
+	struct ibv_create_cq_resp ibv_resp;
+	__u32 cqid;
+	__u32 entries;
+	__u64 physaddr;		/* library mmaps this to get addressability */
+	__u64 queue;
+};
+
+struct amso_create_qp {
+	struct ibv_create_qp ibv_cmd;
+};
+
+struct amso_create_qp_resp {
+	struct ibv_create_qp_resp ibv_resp;
+	__u32 qpid;
+	__u32 entries;		/* actual number of entries after creation */
+	__u64 physaddr;		/* library mmaps this to get addressability */
+	__u64 physsize;		/* library mmaps this to get addressability */
+	__u64 queue;
+};
+
+
+struct t3_cqe {
+	__u32 header:32;
+	__u32 len:32;
+	__u32 wrid_hi_stag:32;
+	__u32 wrid_low_msn:32;
+};
+
+#endif				/* AMSO_ABI_H */
diff --git a/libamso/src/amso.c b/libamso/src/amso.c
new file mode 100644
index 0000000..c017281
--- /dev/null
+++ b/libamso/src/amso.c
@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif				/* HAVE_CONFIG_H */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <pthread.h>
+#include <sysfs/libsysfs.h>
+
+#include "amso.h"
+#include "amso-abi.h"
+
+#define PCI_VENDOR_ID_AMSO		0x18b8
+#define PCI_DEVICE_ID_AMSO_1100		0xb001
+
+#define HCA(v, d, t) \
+	{ .vendor = PCI_VENDOR_ID_##v,			\
+	  .device = PCI_DEVICE_ID_AMSO_##d,		\
+	  .type = AMSO_##t }
+
+struct {
+	unsigned vendor;
+	unsigned device;
+	enum amso_hca_type type;
+} hca_table[] = {
+	HCA(AMSO, 1100, 1100),
+};
+
+static struct ibv_context_ops amso_ctx_ops = {
+	.query_device = amso_query_device,
+	.query_port = amso_query_port,
+	.alloc_pd = amso_alloc_pd,
+	.dealloc_pd = amso_free_pd,
+	.reg_mr = amso_reg_mr,
+	.dereg_mr = amso_dereg_mr,
+	.create_cq = amso_create_cq,
+	.resize_cq = amso_resize_cq,
+	.poll_cq = amso_poll_cq,
+	.destroy_cq = amso_destroy_cq,
+	.create_srq = amso_create_srq,
+	.modify_srq = amso_modify_srq,
+	.destroy_srq = amso_destroy_srq,
+	.create_qp = amso_create_qp,
+	.modify_qp = amso_modify_qp,
+	.destroy_qp = amso_destroy_qp,
+	.create_ah = amso_create_ah,
+	.destroy_ah = amso_destroy_ah,
+	.attach_mcast = amso_attach_mcast,
+	.detach_mcast = amso_detach_mcast
+};
+
+static struct ibv_context *amso_alloc_context(struct ibv_device *ibdev,
+					      int cmd_fd)
+{
+	struct amso_context *context;
+	struct ibv_get_context cmd;
+	struct amso_alloc_ucontext_resp resp;
+
+	context = malloc(sizeof *context);
+	if (!context)
+		return NULL;
+
+	context->ibv_ctx.cmd_fd = cmd_fd;
+
+	if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
+				&resp.ibv_resp, sizeof resp))
+		goto err_free;
+
+	context->ibv_ctx.device = ibdev;
+	context->ibv_ctx.ops = amso_ctx_ops;
+	context->ibv_ctx.ops.req_notify_cq = amso_arm_cq;
+	context->ibv_ctx.ops.cq_event = NULL;
+	context->ibv_ctx.ops.post_send = amso_post_send;
+	context->ibv_ctx.ops.post_recv = amso_post_recv;
+	context->ibv_ctx.ops.post_srq_recv = amso_post_srq_recv;
+
+	return &context->ibv_ctx;
+err_free:
+	free(context);
+	return NULL;
+}
+
+static void amso_free_context(struct ibv_context *ibctx)
+{
+	struct amso_context *context = to_amso_ctx(ibctx);
+
+	free(context);
+}
+
+static struct ibv_device_ops amso_dev_ops = {
+	.alloc_context = amso_alloc_context,
+	.free_context = amso_free_context
+};
+
+struct ibv_device *ibv_driver_init(const char *uverbs_sys_path,
+				   int abi_version)
+{
+	char value[8];
+	struct amso_device *dev;
+	unsigned vendor, device;
+	int i;
+
+	if (ibv_read_sysfs_file(uverbs_sys_path, "device/vendor",
+				value, sizeof value) < 0)
+		return NULL;
+	sscanf(value, "%i", &vendor);
+
+	if (ibv_read_sysfs_file(uverbs_sys_path, "device/device",
+				value, sizeof value) < 0)
+		return NULL;
+	sscanf(value, "%i", &device);
+
+
+	for (i = 0; i < sizeof hca_table / sizeof hca_table[0]; ++i)
+		if (vendor == hca_table[i].vendor &&
+		    device == hca_table[i].device)
+			goto found;
+
+	return NULL;
+
+found:
+	dev = malloc(sizeof *dev);
+	if (!dev) {
+		return NULL;
+	}
+
+	dev->ibv_dev.ops = amso_dev_ops;
+	dev->hca_type = hca_table[i].type;
+	dev->page_size = sysconf(_SC_PAGESIZE);
+
+	return &dev->ibv_dev;
+}
+
+#ifdef HAVE_SYSFS_LIBSYSFS_H
+struct ibv_device *openib_driver_init(struct sysfs_class_device *sysdev)
+{
+	int abi_ver = 0;
+	char value[8];
+
+	if (ibv_read_sysfs_file(sysdev->path, "abi_version",
+				value, sizeof value) > 0)
+		abi_ver = strtol(value, NULL, 10);
+
+	return ibv_driver_init(sysdev->path, abi_ver);
+}
+#endif /* HAVE_SYSFS_LIBSYSFS_H */
diff --git a/libamso/src/amso.h b/libamso/src/amso.h
new file mode 100644
index 0000000..eea4319
--- /dev/null
+++ b/libamso/src/amso.h
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef AMSO_H
+#define AMSO_H
+
+#include <infiniband/driver.h>
+#include <infiniband/arch.h>
+
+#define HIDDEN		__attribute__((visibility ("hidden")))
+
+#define PFX		"amso: "
+
+enum amso_hca_type {
+	AMSO_1100
+};
+
+struct amso_device {
+	struct ibv_device ibv_dev;
+	enum amso_hca_type hca_type;
+	int page_size;
+};
+
+struct amso_context {
+	struct ibv_context ibv_ctx;
+};
+
+struct amso_pd {
+	struct ibv_pd ibv_pd;
+};
+
+struct amso_cq {
+	struct ibv_cq ibv_cq;
+	__u32 cqid;
+	__u32 entries;
+	__u64 physaddr;
+	__u64 queue;
+};
+
+struct amso_qp {
+	struct ibv_qp ibv_qp;
+	__u32 qpid;
+	__u32 entries;
+	__u64 physaddr;
+	__u64 physsize;
+	__u64 queue;
+};
+
+#define to_amso_xxx(xxx, type)						\
+	((struct amso_##type *)					\
+	 ((void *) ib##xxx - offsetof(struct amso_##type, ibv_##xxx)))
+
+static inline struct amso_device *to_amso_dev(struct ibv_device *ibdev)
+{
+	return to_amso_xxx(dev, device);
+}
+
+static inline struct amso_context *to_amso_ctx(struct ibv_context *ibctx)
+{
+	return to_amso_xxx(ctx, context);
+}
+
+static inline struct amso_pd *to_amso_pd(struct ibv_pd *ibpd)
+{
+	return to_amso_xxx(pd, pd);
+}
+
+static inline struct amso_cq *to_amso_cq(struct ibv_cq *ibcq)
+{
+	return to_amso_xxx(cq, cq);
+}
+
+static inline struct amso_qp *to_amso_qp(struct ibv_qp *ibqp)
+{
+	return to_amso_xxx(qp, qp);
+}
+
+
+extern int amso_query_device(struct ibv_context *context,
+			     struct ibv_device_attr *attr);
+extern int amso_query_port(struct ibv_context *context, uint8_t port,
+			   struct ibv_port_attr *attr);
+
+extern struct ibv_pd *amso_alloc_pd(struct ibv_context *context);
+extern int amso_free_pd(struct ibv_pd *pd);
+
+extern struct ibv_mr *amso_reg_mr(struct ibv_pd *pd, void *addr,
+				  size_t length, enum ibv_access_flags access);
+extern int amso_dereg_mr(struct ibv_mr *mr);
+
+struct ibv_cq *amso_create_cq(struct ibv_context *context, int cqe,
+			      struct ibv_comp_channel *channel,
+			      int comp_vector);
+extern int amso_resize_cq(struct ibv_cq *cq, int cqe);
+extern int amso_destroy_cq(struct ibv_cq *cq);
+extern int amso_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
+extern int amso_arm_cq(struct ibv_cq *cq, int solicited);
+extern void amso_cq_event(struct ibv_cq *cq);
+extern void amso_init_cq_buf(struct amso_cq *cq, int nent);
+
+extern struct ibv_srq *amso_create_srq(struct ibv_pd *pd,
+				       struct ibv_srq_init_attr *attr);
+extern int amso_modify_srq(struct ibv_srq *srq,
+			   struct ibv_srq_attr *attr,
+			   enum ibv_srq_attr_mask mask);
+extern int amso_destroy_srq(struct ibv_srq *srq);
+extern int amso_post_srq_recv(struct ibv_srq *ibsrq,
+			      struct ibv_recv_wr *wr,
+			      struct ibv_recv_wr **bad_wr);
+
+extern struct ibv_qp *amso_create_qp(struct ibv_pd *pd,
+				     struct ibv_qp_init_attr *attr);
+extern int amso_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+			  enum ibv_qp_attr_mask attr_mask);
+extern int amso_destroy_qp(struct ibv_qp *qp);
+extern int amso_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
+			  struct ibv_send_wr **bad_wr);
+extern int amso_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
+			  struct ibv_recv_wr **bad_wr);
+extern struct ibv_ah *amso_create_ah(struct ibv_pd *pd,
+			     struct ibv_ah_attr *ah_attr);
+extern int amso_destroy_ah(struct ibv_ah *ah);
+extern int amso_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid,
+			     uint16_t lid);
+extern int amso_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid,
+			     uint16_t lid);
+
+#endif				/* AMSO_H */
diff --git a/libamso/src/amso.map b/libamso/src/amso.map
new file mode 100644
index 0000000..59a8bae
--- /dev/null
+++ b/libamso/src/amso.map
@@ -0,0 +1,6 @@
+{
+	global:
+		ibv_driver_init;
+		openib_driver_init;
+	local: *;
+};
diff --git a/libamso/src/cq.c b/libamso/src/cq.c
new file mode 100644
index 0000000..65360ce
--- /dev/null
+++ b/libamso/src/cq.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif				/* HAVE_CONFIG_H */
+
+#include <stdio.h>
+#include <netinet/in.h>
+#include <pthread.h>
+
+#include <infiniband/opcode.h>
+
+#include "amso.h"
+#include "amso-abi.h"
+
+
+int amso_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc)
+{
+	return ibv_cmd_poll_cq(ibcq, ne, wc);
+}
+
+
+int amso_arm_cq(struct ibv_cq *cq, int solicited)
+{
+	return ibv_cmd_req_notify_cq(cq, solicited);
+}
+
+
diff --git a/libamso/src/qp.c b/libamso/src/qp.c
new file mode 100644
index 0000000..e0d99bb
--- /dev/null
+++ b/libamso/src/qp.c
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif				/* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <netinet/in.h>
+#include <pthread.h>
+
+#include "amso.h"
+#include <stdio.h>
+
+int amso_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
+		   struct ibv_send_wr **bad_wr)
+{
+	return ibv_cmd_post_send(ibqp, wr, bad_wr);
+}
+
+int amso_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
+		   struct ibv_recv_wr **bad_wr)
+{
+	return ibv_cmd_post_recv(ibqp, wr, bad_wr);
+}
+
diff --git a/libamso/src/verbs.c b/libamso/src/verbs.c
new file mode 100644
index 0000000..1cd79d8
--- /dev/null
+++ b/libamso/src/verbs.c
@@ -0,0 +1,303 @@
+/*
+ * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif				/* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <strings.h>
+#include <errno.h>
+#include <pthread.h>
+#include <sys/mman.h>
+#include <netinet/in.h>
+
+#include "amso.h"
+#include "amso-abi.h"
+
+
+int amso_query_device(struct ibv_context *context, struct ibv_device_attr *attr)
+{
+	struct ibv_query_device cmd;
+	uint64_t raw_fw_ver;
+	unsigned major, minor, sub_minor;
+	int ret;
+
+	ret =
+	    ibv_cmd_query_device(context, attr, &raw_fw_ver, &cmd, sizeof cmd);
+	if (ret)
+		return ret;
+
+	major = (raw_fw_ver >> 32) & 0xffff;
+	minor = (raw_fw_ver >> 16) & 0xffff;
+	sub_minor = raw_fw_ver & 0xffff;
+
+	snprintf(attr->fw_ver, sizeof attr->fw_ver,
+		 "%d.%d.%d", major, minor, sub_minor);
+
+	return 0;
+}
+
+int amso_query_port(struct ibv_context *context, uint8_t port,
+		    struct ibv_port_attr *attr)
+{
+	struct ibv_query_port cmd;
+
+	return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
+}
+
+struct ibv_pd *amso_alloc_pd(struct ibv_context *context)
+{
+	struct ibv_alloc_pd cmd;
+	struct amso_alloc_pd_resp resp;
+	struct amso_pd *pd;
+
+	pd = malloc(sizeof *pd);
+	if (!pd)
+		return NULL;
+
+	if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof cmd,
+			     &resp.ibv_resp, sizeof resp)) {
+		free(pd);
+		return NULL;
+	}
+
+	return &pd->ibv_pd;
+}
+
+int amso_free_pd(struct ibv_pd *pd)
+{
+	int ret;
+
+	ret = ibv_cmd_dealloc_pd(pd);
+	if (ret)
+		return ret;
+
+	free(pd);
+	return 0;
+}
+
+static struct ibv_mr *__amso_reg_mr(struct ibv_pd *pd, void *addr,
+				    size_t length, uint64_t hca_va,
+				    enum ibv_access_flags access)
+{
+	struct ibv_mr *mr;
+	struct ibv_reg_mr cmd;
+
+	mr = malloc(sizeof *mr);
+	if (!mr)
+		return NULL;
+
+	if (ibv_cmd_reg_mr(pd, addr, length, hca_va,
+			   access, mr, &cmd, sizeof cmd)) {
+		free(mr);
+		return NULL;
+	}
+
+	return mr;
+}
+
+struct ibv_mr *amso_reg_mr(struct ibv_pd *pd, void *addr,
+			   size_t length, enum ibv_access_flags access)
+{
+	return __amso_reg_mr(pd, addr, length, (uintptr_t) addr, access);
+}
+
+int amso_dereg_mr(struct ibv_mr *mr)
+{
+	int ret;
+
+	ret = ibv_cmd_dereg_mr(mr);
+	if (ret)
+		return ret;
+
+	free(mr);
+	return 0;
+}
+
+struct ibv_cq *amso_create_cq(struct ibv_context *context, int cqe,
+			      struct ibv_comp_channel *channel, int comp_vector)
+{
+	struct amso_create_cq cmd;
+	struct amso_create_cq_resp resp;
+	struct amso_cq *cq;
+	int ret;
+
+	cq = malloc(sizeof *cq);
+	if (!cq) {
+		goto err;
+	}
+
+	ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
+				&cq->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
+				&resp.ibv_resp, sizeof resp);
+	if (ret)
+		goto err;
+
+#if 0 /* A reminder for bypass functionality */
+	cq->physaddr = resp.physaddr;
+	cq->queue =
+	    (unsigned long) mmap(NULL, cqe * sizeof(struct t3_cqe), PROT_WRITE,
+				 MAP_SHARED, context->cmd_fd, cq->physaddr);
+#endif
+
+	return &cq->ibv_cq;
+
+
+err:
+	free(cq);
+
+	return NULL;
+}
+
+int amso_resize_cq(struct ibv_cq *cq, int cqe)
+{
+	int ret;
+	struct ibv_resize_cq cmd;
+
+	ret = ibv_cmd_resize_cq(cq, cqe, &cmd, sizeof cmd);
+	if (ret)
+		return ret;
+	/* We will need to unmap and remap when we implement user mode */
+
+	return 0;
+}
+
+int amso_destroy_cq(struct ibv_cq *cq)
+{
+	int ret;
+
+	ret = ibv_cmd_destroy_cq(cq);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+struct ibv_srq *amso_create_srq(struct ibv_pd *pd,
+				struct ibv_srq_init_attr *attr)
+{
+	return (void *) -ENOSYS;
+}
+
+int amso_modify_srq(struct ibv_srq *srq,
+		    struct ibv_srq_attr *attr, enum ibv_srq_attr_mask attr_mask)
+{
+	return -ENOSYS;
+}
+
+int amso_destroy_srq(struct ibv_srq *srq)
+{
+	return -ENOSYS;
+}
+
+int amso_post_srq_recv(struct ibv_srq *ibsrq,
+                       struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr)
+{
+	return -ENOSYS;
+}
+
+struct ibv_qp *amso_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
+{
+	struct amso_create_qp cmd;
+	struct amso_create_qp_resp resp;
+	struct amso_qp *qp;
+	int ret;
+
+	/* Sanity check QP size before proceeding */
+	if (attr->cap.max_send_wr > 65536 ||
+	    attr->cap.max_recv_wr > 65536 ||
+	    attr->cap.max_send_sge > 4 ||
+	    attr->cap.max_recv_sge > 4 || attr->cap.max_inline_data > 1024)
+		return NULL;
+
+	qp = malloc(sizeof *qp);
+	if (!qp)
+		return NULL;
+
+	ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd,
+				&resp.ibv_resp, sizeof resp);
+	if (ret)
+		return NULL;
+
+#if 0 /* A reminder for bypass functionality */
+	qp->physaddr = resp.physaddr;
+#endif
+
+	return &qp->ibv_qp;
+
+
+	return NULL;
+}
+
+int amso_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+		   enum ibv_qp_attr_mask attr_mask)
+{
+	struct ibv_modify_qp cmd;
+
+	return ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof cmd);
+}
+
+int amso_destroy_qp(struct ibv_qp *qp)
+{
+	int ret;
+
+	ret = ibv_cmd_destroy_qp(qp);
+	if (ret)
+		return ret;
+
+	free(qp);
+
+	return 0;
+}
+
+struct ibv_ah *amso_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
+{
+	return (void *) -ENOSYS;
+}
+
+int amso_destroy_ah(struct ibv_ah *ah)
+{
+	return -ENOSYS;
+}
+
+int amso_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
+{
+	return -ENOSYS;
+}
+
+int amso_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
+{
+	return -ENOSYS;
+}
+




More information about the general mailing list