[ofw] [RFC] [PATCH 11/12] winverbs\uvp:add poll_cq_array() to interface

[Sean Hefty]

Add a new call to the UVP: poll_cq_array().  This exports functionality
in the existing mellanox provider (poll_cq).  It allows the user to
pass in an array of work completions, versus the linked-list API exposed
by IBAL.

This call aligns with the WinVerbs API, avoids the memory dereference
associated with walking a completion list, and returns a user's qp_context
information with the work completion.  This latter is useful for processing
receive completions when a shared receive queue is used.

The call is added to the end of the uvp_interface_t, which should preserve
binary compatibility with existing applications.  A new work completion
structure is defined that can be cast to the existing ib_wc_t completion
structure.  This is done to reduce the changes required of the Mellanox
UVP.

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
The definition of struct uvp_wc was a compromise between minimizing code
changes and exporting the needed qp_context information for winverbs.  It
could be viewed as being really clever or really dumb...


Index: hw/mthca/user/mlnx_ual_osbypass.c
===================================================================
--- hw/mthca/user/mlnx_ual_osbypass.c	(revision 960)
+++ hw/mthca/user/mlnx_ual_osbypass.c	(working copy)
@@ -176,6 +176,25 @@
 }
 
 
+static int
+__poll_cq_array (
+	IN		const	void*						h_cq,
+	IN		const	int							num_entries,
+	IN	OUT			uvp_wc_t*	const			wc )
+{
+	int ne;
+	struct mthca_cq *cq = (struct mthca_cq *) h_cq;
+
+	UVP_ENTER(UVP_DBG_CQ);
+	CL_ASSERT (cq);
+
+	ne = cq->ibv_cq.context->ops.poll_cq(&cq->ibv_cq, num_entries, wc);
+
+	UVP_EXIT(UVP_DBG_CQ);
+	return ne;
+}
+
+
 static ib_api_status_t
 __enable_cq_notify (
 	IN		const	void*		__ptr64			h_cq,
@@ -236,6 +255,7 @@
      * Should the types be same as Verbs?
      */
     p_uvp->poll_cq  = __poll_cq;
+	p_uvp->poll_cq_array = __poll_cq_array;
     p_uvp->rearm_cq = __enable_cq_notify;
     p_uvp->rearm_n_cq = NULL; /* __enable_ncomp_cq_notify: Not implemented */;
     p_uvp->peek_cq  = NULL; /* __peek_cq: Not implemented */
Index: hw/mthca/user/mlnx_uvp.h
===================================================================
--- hw/mthca/user/mlnx_uvp.h	(revision 960)
+++ hw/mthca/user/mlnx_uvp.h	(working copy)
@@ -277,7 +277,7 @@
 struct ibv_cq *mthca_create_cq_post(struct ibv_context *context, 
 				 struct ibv_create_cq_resp *resp);
 int mthca_destroy_cq(struct ibv_cq *cq);
-int mthca_poll_cq(struct ibv_cq *cq, int ne, struct _ib_wc *wc);
+int mthca_poll_cq(struct ibv_cq *cq, int ne, struct _uvp_wc *wc);
 int mthca_poll_cq_list(struct ibv_cq *ibcq, 
 	struct _ib_wc** const pp_free_wclist,
 	struct _ib_wc** const pp_done_wclist );
Index: hw/mthca/user/mlnx_uvp_cq.c
===================================================================
--- hw/mthca/user/mlnx_uvp_cq.c	(revision 960)
+++ hw/mthca/user/mlnx_uvp_cq.c	(working copy)
@@ -37,6 +37,7 @@
 #include <opcode.h>
 #include "mlnx_uvp.h"
 #include "mlnx_uvp_doorbell.h"
+#include <iba\ib_uvp.h>
 
 #if defined(EVENT_TRACING)
 #include "mlnx_uvp_cq.tmh"
@@ -442,7 +443,7 @@
 	return err;
 }
 
-int mthca_poll_cq(struct ibv_cq *ibcq, int num_entries, struct _ib_wc *entry)
+int mthca_poll_cq(struct ibv_cq *ibcq, int num_entries, struct _uvp_wc *entry)
 {
 	struct mthca_cq *cq = to_mcq(ibcq);
 	struct mthca_qp *qp = NULL;
@@ -453,9 +454,10 @@
 	cl_spinlock_acquire(&cq->lock);
 
 	for (npolled = 0; npolled < num_entries; ++npolled) {
-		err = mthca_poll_one(cq, &qp, &freed, entry + npolled);
+		err = mthca_poll_one(cq, &qp, &freed, (struct _ib_wc *) (entry + npolled));
 		if (err)
 			break;
+		entry[npolled].qp_context = qp->ibv_qp.context;
 	}
 
 	if (freed) {
Index: hw/mthca/user/mlnx_uvp_verbs.h
===================================================================
--- hw/mthca/user/mlnx_uvp_verbs.h	(revision 960)
+++ hw/mthca/user/mlnx_uvp_verbs.h	(working copy)
@@ -446,7 +446,7 @@
 			       struct ibv_create_cq *req);
 	struct ibv_cq * (*create_cq_post)(struct ibv_context *context, 
 			       struct ibv_create_cq_resp *resp);
-	int			(*poll_cq)(struct ibv_cq *cq, int num_entries, struct _ib_wc *wc);
+	int			(*poll_cq)(struct ibv_cq *cq, int num_entries, struct _uvp_wc *wc);
 	int 			(*poll_cq_list)( struct ibv_cq *ibcq, 
 		struct _ib_wc** const 			pp_free_wclist,
 		struct _ib_wc** const 			pp_done_wclist );
Index: inc/user/iba/ib_uvp.h
===================================================================
--- inc/user/iba/ib_uvp.h	(revision 960)
+++ inc/user/iba/ib_uvp.h	(working copy)
@@ -2992,6 +2992,53 @@
 
 /********/
 
+/*
+ * Define uvp_wc_t so that we can cast directly to ib_wc_t.
+ *
+ * This allows support for WinVerbs CQ::Poll() with minimal changes to the UVP.
+ */
+typedef struct _uvp_wc
+{
+	void*					qp_context;
+	/* If pointer size is 32-bits, then compiler will pad before uint64_t */
+	uint64_t				wr_id;
+	ib_wc_type_t			wc_type;
+
+	uint32_t				length;
+	ib_wc_status_t			status;
+	uint64_t				vendor_specific;
+
+	union _uvp_wc_recv
+	{
+		struct _uvp_wc_conn
+		{
+			ib_recv_opt_t	recv_opt;
+			ib_net32_t		immediate_data;
+
+		}	conn;
+
+		struct _uvp_wc_ud
+		{
+			ib_recv_opt_t	recv_opt;
+			ib_net32_t		immediate_data;
+			ib_net32_t		remote_qp;
+			uint16_t		pkey_index;
+			ib_net16_t		remote_lid;
+			uint8_t			remote_sl;
+			uint8_t			path_bits;
+
+		}	ud;
+	}	recv;
+}	uvp_wc_t;
+
+typedef int
+(AL_API *uvp_poll_cq_array) (
+	IN		const	void*						h_cq,
+	IN		const	int							num_entries,
+	IN	OUT			uvp_wc_t*	const			wc );
+
+/********/
+
 /****f* user-mode Verbs/uvp_rearm_cq
 * NAME
 *	uvp_rearm_cq -- Invoke the Completion handler, on next entry added.
@@ -3435,6 +3482,8 @@
 	 */
 	uvp_nd_modify_qp_t			nd_modify_qp;
 	uvp_nd_get_qp_state_t		nd_get_qp_state;
+
+	uvp_poll_cq_array			poll_cq_array;
 	
 } uvp_interface_t;