[openib-general] [PATCH] for discussion - combine ib_send_mad_wr with ib_send_wr

[Sean Hefty]

Here's a patch that would combine ib_send_mad_wr with ib_send_wr, to optimize sending and completing MADs.  As part of the change, there is a minor adjustment to the ib_wc structure that allows casting it to an ib_mad_send_wc structure to enable an additional optimization on MAD send completion processing.

This patch is simply for discussion purposes at this point, to see if this is desirable.

Thanks,
- Sean

-- Index: ib_verbs.h
===================================================================
--- ib_verbs.h	(revision 724)
+++ ib_verbs.h	(working copy)
@@ -504,7 +504,8 @@
 	IB_SEND_FENCE		= 1,
 	IB_SEND_SIGNALED	= (1<<1),
 	IB_SEND_SOLICITED	= (1<<2),
-	IB_SEND_INLINE		= (1<<3)
+	IB_SEND_INLINE		= (1<<3),
+	IB_SEND_MAD_GRH_VALID	= (1<<4)
 };
 
 struct ib_sge {
@@ -537,6 +538,7 @@
 			u32	remote_qpn;
 			u32	remote_qkey;
 			u16	pkey_index; /* valid for GSI only */
+			int	timeout_ms; /* valid for MADs only */
 		} ud;
 	} wr;
 };
@@ -591,8 +593,8 @@
 struct ib_wc {
 	u64			wr_id;
 	enum ib_wc_status	status;
-	enum ib_wc_opcode	opcode;
 	u32			vendor_err;
+	enum ib_wc_opcode	opcode;
 	u32			byte_len;
 	u32			imm_data;
 	u32			src_qp;
Index: ib_mad.h
===================================================================
--- ib_mad.h	(revision 712)
+++ ib_mad.h	(working copy)
@@ -128,56 +128,23 @@
 	u32			hi_tid;
 };
 
-enum ib_mad_flags {
-	IB_MAD_GRH_VALID	= 1
-};
-
-/**
- * ib_mad_send_wr - send MAD work request.
- * @list - Allows chaining together multiple requests.
- * @context - User-controlled work request context.
- * @sg_list - An array of scatter-gather entries, referencing the MAD's
- *   data buffer(s).  The first entry must reference the standard MAD
- *   header, plus any RMPP header, if used.
- * @num_sge - The number of scatter-gather entries.
- * @mad_flags - Flags used to control the send operation.
- * @ah - Address handle for the destination.
- * @timeout_ms - Timeout value, in milliseconds, to wait for a response
- *   message.  Set to 0 if no response is expected.
- * @remote_qpn - Destination QP.
- * @remote_qkey - Specifies the qkey used by remote QP.
- * @pkey_index - Pkey index to use.  Required when sending on QP1 only.
- */
-struct ib_mad_send_wr {
-	struct list_head	list;
-	void			*context;
-	struct ib_sge		*sg_list;
-	int			num_sge;
-	int			mad_flags;
-	struct ib_ah		*ah;
-	int			timeout_ms;
-	u32			remote_qpn;
-	u32			remote_qkey;
-	u16			pkey_index;
-};
-
 /**
  * ib_mad_send_wc - MAD send completion information.
- * @context - Context associated with the send MAD request.
+ * @wr_id - Work request identifier associated with the send MAD request.
  * @status - Completion status.
  * @vendor_err - Optional vendor error information returned with a failed
  *   request.
  */
 struct ib_mad_send_wc {
-	void			*context;
+	u64			wr_id;
 	enum ib_wc_status	status;
 	u32			vendor_err;
 };
 
 /**
  * ib_mad_recv_wc - received MAD information.
- * @context - For received response, set to the context specified for
- *   the corresponding send request.
+ * @wr_id - For received response, set to the work request identifier specified
+ *   for the corresponding send request.
  * @grh - References a data buffer containing the global route header.
  *   The data refereced by this buffer is only valid if the GRH is
  *   valid.
@@ -194,7 +161,7 @@
  * An RMPP receive will be coalesced into a single data buffer.
  */
 struct ib_mad_recv_wc {
-	void			*context;
+	u64			wr_id;
 	struct ib_grh		*grh;
 	struct ib_mad		*mad;
 	u32			length;
@@ -263,10 +230,10 @@
  * ib_mad_post_send - Posts a MAD to the send queue of the QP associated
  *   with the registered client.
  * @mad_agent - Specifies the associated registration to post the send to.
- * @mad_send_wr - Specifies the information needed to send the MAD.
+ * @send_wr - Specifies the information needed to send the MAD.
  */
 int ib_mad_post_send(struct ib_mad_agent *mad_agent,
-		     struct ib_mad_send_wr *mad_send_wr);
+		     struct ib_send_wr *send_wr);
 
 /**
  * ib_mad_qp_redir - Registers a QP for MAD services.