[ewg] [PATCH 4/5 v2] libnes: OpenFabrics userspace verbs

ggrundstrom at neteffect.com ggrundstrom at neteffect.com
Fri Oct 19 13:36:24 PDT 2007


OpenFabrics userspace verbs provider routines.

Signed-off-by: Glenn Grundstrom <ggrundstrom at neteffect.com>
---

--- NULL	1969-12-31 18:00:00.000000000 -0600
+++ libnes/src/nes_uverbs.c	2007-10-19 11:07:15.000000000 -0500
@@ -0,0 +1,918 @@
+/*
+ * Copyright (c) 2006 - 2007 NetEffect, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif				/* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <signal.h>
+#include <errno.h>
+#include <pthread.h>
+#include <malloc.h>
+#include <sys/mman.h>
+#include <netinet/in.h>
+
+#include "nes_umain.h"
+#include "nes-abi.h"
+
+extern long int page_size;
+
+
+/**
+ * nes_uquery_device
+ */
+int nes_uquery_device(struct ibv_context *context, struct ibv_device_attr *attr)
+{
+	struct ibv_query_device cmd;
+	uint64_t reserved;
+	int ret;
+
+	ret = ibv_cmd_query_device(context, attr, &reserved, &cmd, sizeof cmd);
+	return ret;
+}
+
+
+/**
+ * nes_uquery_port
+ */
+int nes_uquery_port(struct ibv_context *context, uint8_t port,
+		struct ibv_port_attr *attr)
+{
+	struct ibv_query_port cmd;
+
+	return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
+}
+
+
+/**
+ * nes_ualloc_pd
+ */
+struct ibv_pd *nes_ualloc_pd(struct ibv_context *context)
+{
+	struct ibv_alloc_pd cmd;
+	struct nes_ualloc_pd_resp resp;
+	struct nes_upd *nesupd;
+
+	nesupd = malloc(sizeof *nesupd);
+	if (!nesupd)
+		return NULL;
+
+	if (ibv_cmd_alloc_pd(context, &nesupd->ibv_pd, &cmd, sizeof cmd,
+			&resp.ibv_resp, sizeof resp)) {
+		free(nesupd);
+		return NULL;
+	}
+	nesupd->pd_id = resp.pd_id;
+	nesupd->db_index = resp.db_index;
+
+	nesupd->udoorbell = mmap(NULL, page_size, PROT_WRITE | PROT_READ, MAP_SHARED,
+			context->cmd_fd, nesupd->db_index * page_size);
+
+	if (((void *)-1) == nesupd->udoorbell) {
+		free(nesupd);
+		return NULL;
+	}
+
+	return &nesupd->ibv_pd;
+}
+
+
+/**
+ * nes_ufree_pd
+ */
+int nes_ufree_pd(struct ibv_pd *pd)
+{
+	int ret;
+	struct nes_upd *nesupd;
+
+	nesupd = to_nes_upd(pd);
+
+	ret = ibv_cmd_dealloc_pd(pd);
+	if (ret)
+		return ret;
+
+	munmap((void *)nesupd->udoorbell, page_size);
+	free(nesupd);
+
+	return 0;
+}
+
+
+/**
+ * nes_ureg_mr
+ */
+struct ibv_mr *nes_ureg_mr(struct ibv_pd *pd, void *addr,
+		size_t length, enum ibv_access_flags access)
+{
+	struct ibv_mr *mr;
+	struct nes_ureg_mr cmd;
+#ifdef IBV_CMD_REG_MR_HAS_RESP_PARAMS
+	struct ibv_reg_mr_resp resp;
+#endif
+
+	mr = malloc(sizeof *mr);
+	if (!mr)
+		return NULL;
+
+	cmd.reg_type = NES_UMEMREG_TYPE_MEM;
+#ifdef IBV_CMD_REG_MR_HAS_RESP_PARAMS
+	if (ibv_cmd_reg_mr(pd, addr, length, (uintptr_t) addr,
+			access, mr, &cmd.ibv_cmd, sizeof cmd,
+			&resp, sizeof resp)) {
+#else
+	if (ibv_cmd_reg_mr(pd, addr, length, (uintptr_t) addr,
+			access, mr, &cmd.ibv_cmd, sizeof cmd)) {
+#endif
+		free(mr);
+
+		return NULL;
+	}
+
+	return mr;
+}
+
+
+/**
+ * nes_udereg_mr
+ */
+int nes_udereg_mr(struct ibv_mr *mr)
+{
+	int ret;
+
+	ret = ibv_cmd_dereg_mr(mr);
+	if (ret)
+		return ret;
+
+	free(mr);
+	return 0;
+}
+
+
+/**
+ * nes_ucreate_cq
+ */
+struct ibv_cq *nes_ucreate_cq(struct ibv_context *context, int cqe,
+		struct ibv_comp_channel *channel, int comp_vector)
+{
+	struct nes_ucq *nesucq;
+	struct nes_ureg_mr reg_mr_cmd;
+#ifdef IBV_CMD_REG_MR_HAS_RESP_PARAMS
+	struct ibv_reg_mr_resp reg_mr_resp;
+#endif
+	struct nes_ucreate_cq cmd;
+	struct nes_ucreate_cq_resp resp;
+	int ret;
+	struct nes_uvcontext *nesvctx = to_nes_uctx(context);
+
+	nesucq = malloc(sizeof *nesucq);
+	if (!nesucq) {
+		return NULL;
+	}
+	memset(nesucq, 0, sizeof(*nesucq));
+
+	if (pthread_spin_init(&nesucq->lock, PTHREAD_PROCESS_PRIVATE)) {
+		free(nesucq);
+		return NULL;
+	}
+
+	if (cqe < 4) 	/* a reasonable minimum */
+		cqe = 4;
+	nesucq->size = cqe + 1;
+
+	nesucq->cqes = memalign(page_size, nesucq->size*sizeof(struct nes_hw_cqe));
+	if (!nesucq->cqes)
+		goto err;
+
+	/* Register the memory for the CQ */
+	reg_mr_cmd.reg_type = NES_UMEMREG_TYPE_CQ;
+
+#ifdef IBV_CMD_REG_MR_HAS_RESP_PARAMS
+	ret = ibv_cmd_reg_mr(&nesvctx->nesupd->ibv_pd, (void *)nesucq->cqes,
+			(nesucq->size*sizeof(struct nes_hw_cqe)),
+			(uintptr_t)nesucq->cqes, IBV_ACCESS_LOCAL_WRITE, &nesucq->mr,
+			&reg_mr_cmd.ibv_cmd, sizeof reg_mr_cmd,
+			&reg_mr_resp, sizeof reg_mr_resp);
+#else
+	ret = ibv_cmd_reg_mr(&nesvctx->nesupd->ibv_pd, (void *)nesucq->cqes,
+			(nesucq->size*sizeof(struct nes_hw_cqe)),
+			(uintptr_t)nesucq->cqes, IBV_ACCESS_LOCAL_WRITE, &nesucq->mr,
+			&reg_mr_cmd.ibv_cmd, sizeof reg_mr_cmd);
+#endif
+	if (ret) {
+		/* fprintf(stderr, "ibv_cmd_reg_mr failed (ret = %d).\n", ret); */
+		free((struct nes_hw_cqe *)nesucq->cqes);
+		goto err;
+	}
+
+	/* Create the CQ */
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.user_cq_buffer = (__u64)((uintptr_t)nesucq->cqes);
+
+	ret = ibv_cmd_create_cq(context, nesucq->size-1, channel, comp_vector,
+			&nesucq->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
+			&resp.ibv_resp, sizeof resp);
+	if (ret)
+		goto err;
+
+	nesucq->cq_id = (uint16_t)resp.cq_id;
+
+	/* Zero out the CQ */
+	memset((struct nes_hw_cqe *)nesucq->cqes, 0, nesucq->size*sizeof(struct nes_hw_cqe));
+
+	return &nesucq->ibv_cq;
+
+err:
+ 	/* fprintf(stderr, PFX "%s: Error Creating CQ.\n", __FUNCTION__); */
+	pthread_spin_destroy(&nesucq->lock);
+	free(nesucq);
+
+	return NULL;
+}
+
+
+/**
+ * nes_uresize_cq
+ */
+int nes_uresize_cq(struct ibv_cq *cq, int cqe)
+{
+ 	/* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+	return -ENOSYS;
+}
+
+
+/**
+ * nes_udestroy_cq
+ */
+int nes_udestroy_cq(struct ibv_cq *cq)
+{
+	struct nes_ucq *nesucq = to_nes_ucq(cq);
+	int ret;
+
+	ret = ibv_cmd_destroy_cq(cq);
+	if (ret)
+		return ret;
+
+	/* Free CQ the memory */
+	free((struct nes_hw_cqe *)nesucq->cqes);
+	pthread_spin_destroy(&nesucq->lock);
+	free(nesucq);
+
+	return 0;
+}
+
+
+/**
+ * nes_upoll_cq
+ */
+int nes_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry)
+{
+	uint64_t wrid;
+	struct nes_ucq *nesucq;
+	struct nes_uvcontext *nesvctx = NULL;
+	struct nes_uqp *nesuqp;
+	int cqe_count=0;
+	uint32_t head;
+	uint32_t wq_tail;
+	uint32_t cq_size;
+	uint32_t wqe_index;
+	struct nes_hw_cqe cqe;
+	uint32_t tmp;
+	unsigned long u64temp;
+
+	nesucq = to_nes_ucq(cq);
+	nesvctx = to_nes_uctx(cq->context);
+
+	pthread_spin_lock(&nesucq->lock);
+
+	head = nesucq->head;
+	cq_size = nesucq->size;
+
+	while (cqe_count<num_entries) {
+		if (le32_to_cpu(nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
+			cqe = (volatile struct nes_hw_cqe)nesucq->cqes[head];
+
+			memset(entry, 0, sizeof *entry);
+			/* this is for both the cqe copy and the zeroing of entry */
+			asm __volatile__("": : :"memory");
+
+			nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
+
+			/* parse CQE, get completion context from WQE (either rq or sq */
+			wqe_index = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]) & 511;
+			u64temp = ((uint64_t) (le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]))) |
+				(((uint64_t) (le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32);
+			nesuqp = *((struct nes_uqp **)&u64temp);
+			nesuqp = (struct nes_uqp *)((uintptr_t)nesuqp & (~1023));
+			if (0 == le32_to_cpu(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX])) {
+				entry->status = IBV_WC_SUCCESS;
+			} else {
+				/* TODO: other errors? */
+				entry->status = IBV_WC_WR_FLUSH_ERR;
+			}
+			entry->qp_num = nesuqp->qp_id;
+			entry->src_qp = nesuqp->qp_id;
+
+			if (le32_to_cpu(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) {
+				/* Working on a SQ Completion*/
+				wq_tail = wqe_index;
+				nesuqp->sq_tail = (wqe_index+1)&(nesuqp->sq_size - 1);
+				wrid = ((uint64_t) le32_to_cpu(nesuqp->sq_vbase[wq_tail].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX])) |
+					(((uint64_t) le32_to_cpu(nesuqp->sq_vbase[wq_tail].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
+				entry->byte_len = le32_to_cpu(nesuqp->sq_vbase[wq_tail].wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]);
+
+				switch (le32_to_cpu(nesuqp->sq_vbase[wq_tail].
+						wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) {
+					case NES_IWARP_SQ_OP_RDMAW:
+						/* fprintf(stderr, PFX "%s: Operation = RDMA WRITE.\n",
+								__FUNCTION__ ); */
+						entry->opcode = IBV_WC_RDMA_WRITE;
+						break;
+					case NES_IWARP_SQ_OP_RDMAR:
+						/* fprintf(stderr, PFX "%s: Operation = RDMA READ.\n",
+								__FUNCTION__ ); */
+						entry->opcode = IBV_WC_RDMA_READ;
+						entry->byte_len = le32_to_cpu(nesuqp->sq_vbase[wq_tail].
+								wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]);
+						break;
+					case NES_IWARP_SQ_OP_SENDINV:
+					case NES_IWARP_SQ_OP_SENDSEINV:
+					case NES_IWARP_SQ_OP_SEND:
+					case NES_IWARP_SQ_OP_SENDSE:
+						/* fprintf(stderr, PFX "%s: Operation = Send.\n",
+								__FUNCTION__ ); */
+						entry->opcode = IBV_WC_SEND;
+						break;
+				}
+			} else {
+				/* Working on a RQ Completion*/
+				wq_tail = wqe_index;
+				nesuqp->rq_tail = (wqe_index+1)&(nesuqp->rq_size - 1);
+				entry->byte_len = le32_to_cpu(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]);
+				wrid = ((uint64_t) le32_to_cpu(nesuqp->rq_vbase[wq_tail].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX])) |
+					(((uint64_t) le32_to_cpu(nesuqp->rq_vbase[wq_tail].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
+				entry->opcode = IBV_WC_RECV;
+			}
+			entry->wr_id = wrid;
+
+			if (++head >= cq_size)
+				head = 0;
+			cqe_count++;
+			nesucq->polled_completions++;
+
+			/* TODO: find a better number...if there is one */
+			if ((nesucq->polled_completions > (cq_size/2)) ||
+					(nesucq->polled_completions == 255)) {
+				if (NULL == nesvctx)
+					nesvctx = to_nes_uctx(cq->context);
+				nesvctx->nesupd->udoorbell->cqe_alloc = cpu_to_le32(nesucq->cq_id |
+						(nesucq->polled_completions << 16));
+				tmp = nesvctx->nesupd->udoorbell->cqe_alloc;
+				nesucq->polled_completions = 0;
+			}
+			entry++;
+		} else
+			break;
+	}
+
+	if (nesucq->polled_completions) {
+		if (NULL == nesvctx)
+			nesvctx = to_nes_uctx(cq->context);
+		nesvctx->nesupd->udoorbell->cqe_alloc = cpu_to_le32(nesucq->cq_id |
+				(nesucq->polled_completions << 16));
+		tmp = nesvctx->nesupd->udoorbell->cqe_alloc;
+		nesucq->polled_completions = 0;
+	}
+	nesucq->head = head;
+
+	pthread_spin_unlock(&nesucq->lock);
+	return cqe_count;
+}
+
+
+/**
+ * nes_uarm_cq
+ */
+int nes_uarm_cq(struct ibv_cq *cq, int solicited)
+{
+	struct nes_ucq *nesucq;
+	struct nes_uvcontext *nesvctx;
+	uint32_t cq_arm;
+	uint32_t tmp;
+
+	nesucq = to_nes_ucq(cq);
+	nesvctx = to_nes_uctx(cq->context);
+
+	cq_arm = nesucq->cq_id;
+
+	if (solicited)
+		cq_arm |= NES_CQE_ALLOC_NOTIFY_SE;
+	else
+		cq_arm |= NES_CQE_ALLOC_NOTIFY_NEXT;
+
+	nesvctx->nesupd->udoorbell->cqe_alloc = cpu_to_le32(cq_arm);
+	tmp = nesvctx->nesupd->udoorbell->cqe_alloc;
+
+	return 0;
+}
+
+
+/**
+ * nes_ucreate_srq
+ */
+struct ibv_srq *nes_ucreate_srq(struct ibv_pd *pd, struct ibv_srq_init_attr *attr)
+{
+	/* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+	return (void *)-ENOSYS;
+}
+
+
+/**
+ * nes_umodify_srq
+ */
+int nes_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr,
+		enum ibv_srq_attr_mask attr_mask)
+{
+	/* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+	return -ENOSYS;
+}
+
+
+/**
+ * nes_udestroy_srq
+ */
+int nes_udestroy_srq(struct ibv_srq *srq)
+{
+	/* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+	return -ENOSYS;
+}
+
+
+/**
+ * nes_upost_srq_recv
+ */
+int nes_upost_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr,
+		struct ibv_recv_wr **bad_wr)
+{
+	/* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+	return -ENOSYS;
+}
+
+
+/**
+ * nes_ucreate_qp
+ */
+struct ibv_qp *nes_ucreate_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
+{
+	struct nes_uqp *nesuqp;
+	struct nes_uvcontext *nesvctx = to_nes_uctx(pd->context);
+	struct nes_ucreate_qp cmd;
+	struct nes_ucreate_qp_resp resp;
+	unsigned long mmap_offset;
+	int ret;
+
+	/* Sanity check QP size before proceeding */
+	if (attr->cap.max_send_wr > 510 ||
+			attr->cap.max_recv_wr > 510 ||
+			attr->cap.max_send_sge > 4 ||
+			attr->cap.max_recv_sge > 4 )
+		return NULL;
+
+	nesuqp = memalign(1024, sizeof(*nesuqp));
+	if (!nesuqp)
+		return NULL;
+	memset(nesuqp, 0, sizeof(*nesuqp));
+
+	if (pthread_spin_init(&nesuqp->lock, PTHREAD_PROCESS_PRIVATE)) {
+		free(nesuqp);
+		return NULL;
+	}
+
+	ret = ibv_cmd_create_qp(pd, &nesuqp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd,
+			&resp.ibv_resp, sizeof resp);
+	if (ret) {
+		pthread_spin_destroy(&nesuqp->lock);
+		free(nesuqp);
+		return NULL;
+	}
+
+	nesuqp->qp_id = resp.qp_id;
+	nesuqp->sq_db_index = resp.mmap_sq_db_index;
+	nesuqp->rq_db_index = resp.mmap_rq_db_index;
+	nesuqp->sq_size = resp.actual_sq_size;
+	nesuqp->rq_size = resp.actual_rq_size;
+	nesuqp->nes_drv_opt = resp.nes_drv_opt;
+	/* Account for LSMM, in theory, could get overrun if app preposts to SQ */
+	nesuqp->sq_head = 1;
+	nesuqp->sq_tail = 1;
+
+	/* Map the SQ/RQ buffers */
+	mmap_offset = ((nesvctx->max_pds*4096) + page_size-1) & (~(page_size-1));
+	mmap_offset += (((sizeof(struct nes_hw_qp_wqe) * nesvctx->wq_size) + page_size-1) &
+			(~(page_size-1)))*nesuqp->sq_db_index;
+
+	nesuqp->sq_vbase = mmap(NULL, (nesuqp->sq_size+nesuqp->rq_size) *
+			sizeof(struct nes_hw_qp_wqe), PROT_WRITE | PROT_READ,
+			MAP_SHARED, pd->context->cmd_fd, mmap_offset);
+
+	if (((void *)-1) == nesuqp->sq_vbase) {
+		pthread_spin_destroy(&nesuqp->lock);
+		free(nesuqp);
+		return NULL;
+	}
+	nesuqp->rq_vbase = (struct nes_hw_qp_wqe *)(((char *)nesuqp->sq_vbase) +
+			(nesuqp->sq_size*sizeof(struct nes_hw_qp_wqe)));
+	*((unsigned int *)nesuqp->sq_vbase) = 0;
+
+	return &nesuqp->ibv_qp;
+}
+
+
+/**
+ * nes_uquery_qp
+ */
+int nes_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+		enum ibv_qp_attr_mask attr_mask, struct ibv_qp_init_attr *init_attr)
+{
+	struct ibv_query_qp cmd;
+
+	/* fprintf(stderr, PFX "nes_uquery_qp: calling ibv_cmd_query_qp\n"); */
+
+	return ibv_cmd_query_qp(qp, attr, attr_mask, init_attr, &cmd, sizeof(cmd));
+}
+
+
+/**
+ * nes_umodify_qp
+ */
+int nes_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+		enum ibv_qp_attr_mask attr_mask)
+{
+	struct ibv_modify_qp cmd;
+
+	/* fprintf(stderr, PFX "%s, QP State = %u, attr_mask = 0x%X.\n", __FUNCTION__,
+			(unsigned int)attr->qp_state, (unsigned int)attr_mask ); */
+	return ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof cmd);
+}
+
+
+/**
+ * nes_udestroy_qp
+ */
+int nes_udestroy_qp(struct ibv_qp *qp)
+{
+	struct nes_uqp *nesuqp = to_nes_uqp(qp);
+	int ret;
+
+	munmap((void *) nesuqp->sq_vbase, (nesuqp->sq_size+nesuqp->rq_size) *
+			sizeof(struct nes_hw_qp_wqe));
+
+	ret = ibv_cmd_destroy_qp(qp);
+	if (ret)
+		return ret;
+
+	pthread_spin_destroy(&nesuqp->lock);
+	free(nesuqp);
+
+	return 0;
+}
+
+
+/**
+ * nes_upost_send
+ */
+int nes_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
+		struct ibv_send_wr **bad_wr)
+{
+	uint64_t u64temp;
+	struct nes_uqp *nesuqp = to_nes_uqp(ib_qp);
+	struct nes_upd *nesupd = to_nes_upd(ib_qp->pd);
+	struct nes_hw_qp_wqe volatile *wqe;
+	uint32_t head = nesuqp->sq_head;
+	uint32_t qsize = nesuqp->sq_size;
+	uint32_t counter;
+	uint32_t err = 0;
+	uint32_t wqe_count = 0;
+	uint32_t outstanding_wqes;
+	uint32_t total_payload_length = 0;
+	int sge_index;
+
+	pthread_spin_lock(&nesuqp->lock);
+
+	while (ib_wr) {
+		/* Check for SQ overflow */
+		outstanding_wqes = head + (2 * qsize) - nesuqp->sq_tail;
+		outstanding_wqes &= qsize - 1;
+		if (unlikely(outstanding_wqes == (qsize - 1))) {
+			err = -EINVAL;
+			break;
+		}
+		if (unlikely(ib_wr->num_sge > 4)) {
+			err = -EINVAL;
+			break;
+		}
+
+		wqe = (struct nes_hw_qp_wqe *)&nesuqp->sq_vbase[head];
+		/* fprintf(stderr, PFX "%s: QP%u: processing sq wqe at %p, head = %u.\n",
+				__FUNCTION__, nesuqp->qp_id, wqe, head);  */
+		u64temp = (uint64_t) ib_wr->wr_id;
+		wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX] = cpu_to_le32((uint32_t)u64temp);
+		wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX] = cpu_to_le32((uint32_t)(u64temp>>32));
+		u64temp = (uint64_t)((uintptr_t)nesuqp);
+		wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX] = cpu_to_le32((uint32_t)u64temp);
+		wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX] = cpu_to_le32((uint32_t)(u64temp>>32));
+		asm __volatile__("": : :"memory");
+		wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX] |= cpu_to_le32(head);
+
+		switch (ib_wr->opcode) {
+		case IBV_WR_SEND:
+		case IBV_WR_SEND_WITH_IMM:
+			/* fprintf(stderr, PFX "%s: QP%u: processing sq wqe%u. Opcode = %s\n",
+					__FUNCTION__, nesuqp->qp_id, head, "Send"); */
+			if (ib_wr->send_flags & IBV_SEND_SOLICITED) {
+				wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(NES_IWARP_SQ_OP_SENDSE);
+			} else {
+				wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(NES_IWARP_SQ_OP_SEND);
+			}
+
+			if (ib_wr->send_flags & IBV_SEND_FENCE) {
+				wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= cpu_to_le32(NES_IWARP_SQ_WQE_LOCAL_FENCE);
+			}
+
+			/* if (ib_wr->send_flags & IBV_SEND_INLINE) {
+				fprintf(stderr, PFX "%s: Send SEND_INLINE, length=%d\n",
+						__FUNCTION__, ib_wr->sg_list[0].length);
+			} */
+			if ((ib_wr->send_flags & IBV_SEND_INLINE) && (ib_wr->sg_list[0].length <= 64) &&
+				(0 == (nesuqp->nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA)) &&
+				(ib_wr->num_sge == 1)) {
+				memcpy((void *)&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
+						(void *)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
+				wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = cpu_to_le32(ib_wr->sg_list[0].length);
+				wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= cpu_to_le32(NES_IWARP_SQ_WQE_IMM_DATA);
+			} else {
+				total_payload_length = 0;
+				for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
+					wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] =
+							cpu_to_le32((uint32_t)ib_wr->sg_list[sge_index].addr);
+					wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] =
+							cpu_to_le32((uint32_t)(ib_wr->sg_list[sge_index].addr>>32));
+					wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX+(sge_index*4)] =
+							cpu_to_le32(ib_wr->sg_list[sge_index].length);
+					wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX+(sge_index*4)] =
+							cpu_to_le32(ib_wr->sg_list[sge_index].lkey);
+					total_payload_length += ib_wr->sg_list[sge_index].length;
+				}
+				wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] =
+						cpu_to_le32(total_payload_length);
+			}
+
+			break;
+		case IBV_WR_RDMA_WRITE:
+		case IBV_WR_RDMA_WRITE_WITH_IMM:
+			/* fprintf(stderr, PFX "%s:QP%u: processing sq wqe%u. Opcode = %s\n",
+					__FUNCTION__, nesuqp->qp_id, head, "Write"); */
+			wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(NES_IWARP_SQ_OP_RDMAW);
+
+			if (ib_wr->send_flags & IBV_SEND_FENCE) {
+				wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= cpu_to_le32(NES_IWARP_SQ_WQE_LOCAL_FENCE);
+			}
+			wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = cpu_to_le32(ib_wr->wr.rdma.rkey);
+			wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = cpu_to_le32(
+					(uint32_t)ib_wr->wr.rdma.remote_addr);
+			wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = cpu_to_le32(
+					(uint32_t)(ib_wr->wr.rdma.remote_addr>>32));
+
+			/* if (ib_wr->send_flags & IBV_SEND_INLINE) {
+				fprintf(stderr, PFX "%s: Write SEND_INLINE, length=%d\n",
+						__FUNCTION__, ib_wr->sg_list[0].length);
+			} */
+			if ((ib_wr->send_flags & IBV_SEND_INLINE) && (ib_wr->sg_list[0].length <= 64) &&
+				(0 == (nesuqp->nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA)) &&
+				(ib_wr->num_sge == 1)) {
+				memcpy((void *)&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
+						(void *)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
+				wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = cpu_to_le32(ib_wr->sg_list[0].length);
+				wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= cpu_to_le32(NES_IWARP_SQ_WQE_IMM_DATA);
+			} else {
+				total_payload_length = 0;
+				for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
+					wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] = cpu_to_le32(
+							(uint32_t)ib_wr->sg_list[sge_index].addr);
+					wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] = cpu_to_le32(
+							(uint32_t)(ib_wr->sg_list[sge_index].addr>>32));
+					wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX+(sge_index*4)] = cpu_to_le32(
+							ib_wr->sg_list[sge_index].length);
+					wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX+(sge_index*4)] = cpu_to_le32(
+							ib_wr->sg_list[sge_index].lkey);
+					total_payload_length += ib_wr->sg_list[sge_index].length;
+				}
+				wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = cpu_to_le32(total_payload_length);
+			}
+			wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] =
+					wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX];
+			break;
+		case IBV_WR_RDMA_READ:
+			/* fprintf(stderr, PFX "%s:QP%u:processing sq wqe%u. Opcode = %s\n",
+					__FUNCTION__, nesuqp->qp_id, head, "Read"); */
+			/* IWarp only supports 1 sge for RDMA reads */
+			if (ib_wr->num_sge > 1) {
+				err = -EINVAL;
+				break;
+			}
+			wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(NES_IWARP_SQ_OP_RDMAR);
+			wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = cpu_to_le32((uint32_t)ib_wr->wr.rdma.remote_addr);
+			wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = cpu_to_le32((uint32_t)(ib_wr->wr.rdma.remote_addr>>32));
+			wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = cpu_to_le32(ib_wr->wr.rdma.rkey);
+			wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = cpu_to_le32(ib_wr->sg_list->length);
+			wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((uint32_t)ib_wr->sg_list->addr);
+			wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((uint32_t)(ib_wr->sg_list->addr>>32));
+			wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = cpu_to_le32(ib_wr->sg_list->lkey);
+			break;
+		default:
+			/* error */
+			err = -EINVAL;
+			break;
+		}
+
+		if (ib_wr->send_flags & IBV_SEND_SIGNALED) {
+			/* fprintf(stderr, PFX "%s:sq wqe%u is signalled\n", __FUNCTION__, head); */
+			wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= cpu_to_le32(NES_IWARP_SQ_WQE_SIGNALED_COMPL);
+		}
+		ib_wr = ib_wr->next;
+		head++;
+		wqe_count++;
+		if (head >= qsize)
+			head = 0;
+	}
+
+	nesuqp->sq_head = head;
+	asm __volatile__("": : :"memory");
+	while (wqe_count) {
+		counter = (wqe_count<(uint32_t)255) ? wqe_count : 255;
+		wqe_count -= counter;
+		nesupd->udoorbell->wqe_alloc =  cpu_to_le32((counter<<24) | 0x00800000 | nesuqp->qp_id);
+	}
+
+	if (err)
+		*bad_wr = ib_wr;
+
+	pthread_spin_unlock(&nesuqp->lock);
+
+	return err;
+}
+
+
+/**
+ * nes_upost_recv
+ */
+int nes_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr,
+		struct ibv_recv_wr **bad_wr)
+{
+	uint64_t u64temp;
+	struct nes_uqp *nesuqp = to_nes_uqp(ib_qp);
+	struct nes_upd *nesupd = to_nes_upd(ib_qp->pd);
+	struct nes_hw_qp_wqe *wqe;
+	uint32_t head = nesuqp->rq_head;
+	uint32_t qsize = nesuqp->rq_size;
+	uint32_t counter;
+	uint32_t err = 0;
+	uint32_t wqe_count = 0;
+	uint32_t outstanding_wqes;
+	uint32_t total_payload_length;
+	int sge_index;
+
+	pthread_spin_lock(&nesuqp->lock);
+
+	while (ib_wr) {
+		/* Check for RQ overflow */
+		outstanding_wqes = head + (2 * qsize) - nesuqp->rq_tail;
+		outstanding_wqes &= qsize - 1;
+		if (unlikely(outstanding_wqes == (qsize - 1))) {
+			err = -EINVAL;
+			break;
+		}
+
+		wqe = (struct nes_hw_qp_wqe *)&nesuqp->rq_vbase[head];
+		u64temp = ib_wr->wr_id;
+		wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX] =
+				cpu_to_le32((uint32_t)u64temp);
+		wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX] =
+				cpu_to_le32((uint32_t)(u64temp >> 32));
+		u64temp = (uint64_t)((uintptr_t)nesuqp);
+		wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX] =
+				cpu_to_le32((uint32_t)u64temp);
+		wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_HIGH_IDX] =
+				cpu_to_le32((uint32_t)(u64temp >> 32));
+		asm __volatile__("": : :"memory");
+		wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX] |= cpu_to_le32(head);
+
+		total_payload_length = 0;
+		for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
+			wqe->wqe_words[NES_IWARP_RQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] =
+					cpu_to_le32((uint32_t)ib_wr->sg_list[sge_index].addr);
+			wqe->wqe_words[NES_IWARP_RQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] =
+					cpu_to_le32((uint32_t)(ib_wr->sg_list[sge_index].addr>>32));
+			wqe->wqe_words[NES_IWARP_RQ_WQE_LENGTH0_IDX+(sge_index*4)] =
+					cpu_to_le32(ib_wr->sg_list[sge_index].length);
+			wqe->wqe_words[NES_IWARP_RQ_WQE_STAG0_IDX+(sge_index*4)] =
+					cpu_to_le32(ib_wr->sg_list[sge_index].lkey);
+			total_payload_length += ib_wr->sg_list->length;
+		}
+		wqe->wqe_words[NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX] = cpu_to_le32(total_payload_length);
+
+		ib_wr = ib_wr->next;
+		head++;
+		wqe_count++;
+		if (head >= qsize)
+			head = 0;
+	}
+
+	nesuqp->rq_head = head;
+	asm __volatile__("": : :"memory");
+	while (wqe_count) {
+		counter = (wqe_count<(uint32_t)255) ? wqe_count : 255;
+		wqe_count -= counter;
+		nesupd->udoorbell->wqe_alloc = cpu_to_le32((counter << 24) | nesuqp->qp_id);
+	}
+
+	if (err)
+		*bad_wr = ib_wr;
+
+	pthread_spin_unlock(&nesuqp->lock);
+
+	return err;
+}
+
+
+/**
+ * nes_ucreate_ah
+ */
+struct ibv_ah *nes_ucreate_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
+{
+	/* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+	return (void *)-ENOSYS;
+}
+
+
+/**
+ * nes_udestroy_ah
+ */
+int nes_udestroy_ah(struct ibv_ah *ah)
+{
+	/* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+	return -ENOSYS;
+}
+
+
+/**
+ * nes_uattach_mcast
+ */
+int nes_uattach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
+{
+	/* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+	return -ENOSYS;
+}
+
+
+/**
+ * nes_udetach_mcast
+ */
+int nes_udetach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
+{
+	/* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+	return -ENOSYS;
+}
+



More information about the ewg mailing list