[openib-general] [PATCH] SEND_INLINE support in libmthca

[Michael S. Tsirkin]

Quoting r. Roland Dreier <roland at topspin.com>:
> Subject: Re: [openib-general] [PATCH] SEND_INLINE support in libmthca
> 
> Is the test here correct?
> 
> +				if (s + sizeof *seg > (1 << qp->sq.wqe_shift)) {
> It seems we need to take into account the size of next segment and any
> RDMA segment that we may be posting as well.

Right. Fixed (below).

> Also does it make sense to put the code for gathering inline data
> segments and writing gather lists into an inline function that can be
> called from both the tavor and arbel post send function?  Will gcc
> actually inline this function?
> 
>  - R.
> 

It does get inlined, but the function would have to return
both size and status, so however I rearrange it I get either extra loads/stores
or extra branches.

Inline data support for libmthca (important for latency).
Signed-off-by: Michael S. Tsirkin <mst at mellanox.co.il>

Index: src/qp.c
===================================================================
--- src/qp.c	(revision 2096)
+++ src/qp.c	(working copy)
@@ -57,6 +57,10 @@ enum {
 	MTHCA_NEXT_SOLICIT   = 1 << 1,
 };
 
+enum {
+	MTHCA_INLINE_SEG = 1<<31
+};
+
 struct mthca_next_seg {
 	uint32_t	nda_op;	/* [31:6] next WQE [4:0] next opcode */
 	uint32_t	ee_nds;	/* [31:8] next EE  [7] DBD [6] F [5:0] next WQE size */
@@ -107,6 +111,10 @@ struct mthca_data_seg {
 	uint64_t	addr;
 };
 
+struct mthca_inline_seg {
+	uint32_t	byte_count;
+};
+
 static const uint8_t mthca_opcode[] = {
 	[IBV_WR_SEND]                 = MTHCA_OPCODE_SEND,
 	[IBV_WR_SEND_WITH_IMM]        = MTHCA_OPCODE_SEND_IMM,
@@ -255,15 +263,39 @@ int mthca_tavor_post_send(struct ibv_qp 
 			goto out;
 		}
 
-		for (i = 0; i < wr->num_sge; ++i) {
-			((struct mthca_data_seg *) wqe)->byte_count =
-				htonl(wr->sg_list[i].length);
-			((struct mthca_data_seg *) wqe)->lkey =
-				htonl(wr->sg_list[i].lkey);
-			((struct mthca_data_seg *) wqe)->addr =
-				htonll(wr->sg_list[i].addr);
-			wqe += sizeof (struct mthca_data_seg);
-			size += sizeof (struct mthca_data_seg) / 16;
+		if (wr->send_flags & IBV_SEND_INLINE) {
+			struct mthca_inline_seg *seg = wqe;
+			int wqe_size = 1 << qp->sq.wqe_shift;
+			int s = 0;
+			wqe += sizeof *seg;
+			for (i = 0; i < wr->num_sge; ++i) {
+				struct ibv_sge *sge = &wr->sg_list[i];
+				int l;
+				l = sge->length;
+				s += l;
+
+				if (s + sizeof *seg + size * 16 > wqe_size) {
+					ret = -1;
+					*bad_wr = wr;
+					goto out;
+				}
+
+				memcpy(wqe, (void*)(intptr_t)sge->addr, l);
+				wqe += l;
+			}
+			seg->byte_count = htonl(MTHCA_INLINE_SEG | s);
+
+			size += align(s + sizeof *seg, 16) / 16;
+		} else {
+			struct mthca_data_seg *seg;
+			for (i = 0; i < wr->num_sge; ++i) {
+				seg = wqe;
+				seg->byte_count = htonl(wr->sg_list[i].length);
+				seg->lkey = htonl(wr->sg_list[i].lkey);
+				seg->addr = htonll(wr->sg_list[i].addr);
+				wqe += sizeof *seg;
+			}
+			size += wr->num_sge * sizeof *seg / 16;
 		}
 
 		qp->wrid[ind + qp->rq.max] = wr->wr_id;
@@ -512,15 +544,39 @@ int mthca_arbel_post_send(struct ibv_qp 
 			goto out;
 		}
 
-		for (i = 0; i < wr->num_sge; ++i) {
-			((struct mthca_data_seg *) wqe)->byte_count =
-				htonl(wr->sg_list[i].length);
-			((struct mthca_data_seg *) wqe)->lkey =
-				htonl(wr->sg_list[i].lkey);
-			((struct mthca_data_seg *) wqe)->addr =
-				htonll(wr->sg_list[i].addr);
-			wqe += sizeof (struct mthca_data_seg);
-			size += sizeof (struct mthca_data_seg) / 16;
+		if (wr->send_flags & IBV_SEND_INLINE) {
+			struct mthca_inline_seg *seg = wqe;
+			int wqe_size = 1 << qp->sq.wqe_shift;
+			int s = 0;
+			wqe += sizeof *seg;
+			for (i = 0; i < wr->num_sge; ++i) {
+				struct ibv_sge *sge = &wr->sg_list[i];
+				int l;
+				l = sge->length;
+				s += l;
+
+				if (s + sizeof *seg + size * 16 > wqe_size) {
+					ret = -1;
+					*bad_wr = wr;
+					goto out;
+				}
+
+				memcpy(wqe, (void*)(intptr_t)sge->addr, l);
+				wqe += l;
+			}
+			seg->byte_count = htonl(MTHCA_INLINE_SEG | s);
+
+			size += align(s + sizeof *seg, 16) / 16;
+		} else {
+			struct mthca_data_seg *seg;
+			for (i = 0; i < wr->num_sge; ++i) {
+				seg = wqe;
+				seg->byte_count = htonl(wr->sg_list[i].length);
+				seg->lkey = htonl(wr->sg_list[i].lkey);
+				seg->addr = htonll(wr->sg_list[i].addr);
+				wqe += sizeof *seg;
+			}
+			size += wr->num_sge * sizeof *seg / 16;
 		}
 
 		qp->wrid[ind + qp->rq.max] = wr->wr_id;

-- 
MST - Michael S. Tsirkin