[openib-general] [PATCH 1/2] mthca: max_inline_data support

[Michael S. Tsirkin]

Kernel part of a patch: support max_inline_data capability in mthca.

Signed-off-by: Michael S. Tsirkin <mst at mellanox.co.il>

Index: hw/mthca/mthca_dev.h
===================================================================
--- hw/mthca/mthca_dev.h	(revision 2281)
+++ hw/mthca/mthca_dev.h	(working copy)
@@ -439,12 +439,14 @@ int mthca_alloc_qp(struct mthca_dev *dev
 		   struct mthca_cq *recv_cq,
 		   enum ib_qp_type type,
 		   enum ib_sig_type send_policy,
+		   struct ib_qp_cap *cap,
 		   struct mthca_qp *qp);
 int mthca_alloc_sqp(struct mthca_dev *dev,
 		    struct mthca_pd *pd,
 		    struct mthca_cq *send_cq,
 		    struct mthca_cq *recv_cq,
 		    enum ib_sig_type send_policy,
+		    struct ib_qp_cap *cap,
 		    int qpn,
 		    int port,
 		    struct mthca_sqp *sqp);
Index: hw/mthca/mthca_provider.c
===================================================================
--- hw/mthca/mthca_provider.c	(revision 2281)
+++ hw/mthca/mthca_provider.c	(working copy)
@@ -486,16 +486,11 @@ static struct ib_qp *mthca_create_qp(str
 			qp->rq.db_index  = ucmd.rq_db_index;
 		}
 
-		qp->sq.max    = init_attr->cap.max_send_wr;
-		qp->rq.max    = init_attr->cap.max_recv_wr;
-		qp->sq.max_gs = init_attr->cap.max_send_sge;
-		qp->rq.max_gs = init_attr->cap.max_recv_sge;
-
 		err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd),
 				     to_mcq(init_attr->send_cq),
 				     to_mcq(init_attr->recv_cq),
 				     init_attr->qp_type, init_attr->sq_sig_type,
-				     qp);
+				     &init_attr->cap, qp);
 
 		if (err && pd->uobject) {
 			context = to_mucontext(pd->uobject->context);
@@ -524,17 +519,12 @@ static struct ib_qp *mthca_create_qp(str
 		if (!qp)
 			return ERR_PTR(-ENOMEM);
 
-		qp->sq.max    = init_attr->cap.max_send_wr;
-		qp->rq.max    = init_attr->cap.max_recv_wr;
-		qp->sq.max_gs = init_attr->cap.max_send_sge;
-		qp->rq.max_gs = init_attr->cap.max_recv_sge;
-
 		qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
 
 		err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
 				      to_mcq(init_attr->send_cq),
 				      to_mcq(init_attr->recv_cq),
-				      init_attr->sq_sig_type,
+				      init_attr->sq_sig_type, &init_attr->cap,
 				      qp->ibqp.qp_num, init_attr->port_num,
 				      to_msqp(qp));
 		break;
@@ -549,10 +539,6 @@ static struct ib_qp *mthca_create_qp(str
 		return ERR_PTR(err);
 	}
 
-	init_attr->cap.max_inline_data = 0;
-	init_attr->cap.max_send_wr     = qp->sq.max;
-	init_attr->cap.max_recv_wr     = qp->rq.max;
-
 	return &qp->ibqp;
 }
 
Index: hw/mthca/mthca_qp.c
===================================================================
--- hw/mthca/mthca_qp.c	(revision 2281)
+++ hw/mthca/mthca_qp.c	(working copy)
@@ -46,7 +46,9 @@ enum {
 	MTHCA_MAX_DIRECT_QP_SIZE = 4 * PAGE_SIZE,
 	MTHCA_ACK_REQ_FREQ       = 10,
 	MTHCA_FLIGHT_LIMIT       = 9,
-	MTHCA_UD_HEADER_SIZE     = 72 /* largest UD header possible */
+	MTHCA_UD_HEADER_SIZE     = 72, /* largest UD header possible */
+	MTHCA_INLINE_HEADER_SIZE = 4,  /* data segment overhead for inline */
+	MTHCA_INLINE_CHUNK_SIZE  = 16  /* inline data segment chunk */
 };
 
 enum {
@@ -1205,22 +1207,42 @@ static int mthca_alloc_qp_common(struct 
 	return 0;
 }
 
-static void mthca_align_qp_size(struct mthca_dev *dev, struct mthca_qp *qp)
+static void mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap* cap,
+			       	struct mthca_qp *qp)
 {
 	int i;
+	u32 d;
 
-	if (!mthca_is_memfree(dev))
-		return;
+	if (mthca_is_memfree(dev)) {
+		for (i = 0; 1 << i < cap->max_recv_wr; ++i)
+			; /* nothing */
 
-	for (i = 0; 1 << i < qp->rq.max; ++i)
-		; /* nothing */
+		qp->rq.max = 1 << i;
 
-	qp->rq.max = 1 << i;
+		for (i = 0; 1 << i < cap->max_send_wr; ++i)
+			; /* nothing */
 
-	for (i = 0; 1 << i < qp->sq.max; ++i)
-		; /* nothing */
+		qp->sq.max = 1 << i;
+	} else {
+		qp->rq.max = cap->max_recv_wr;
+		qp->sq.max = cap->max_send_wr;
+	}
+
+	qp->rq.max_gs = cap->max_recv_sge;
+
+	d = ALIGN(cap->max_inline_data + MTHCA_INLINE_HEADER_SIZE,
+		  MTHCA_INLINE_CHUNK_SIZE) / sizeof (struct mthca_data_seg);
+	qp->sq.max_gs = max(cap->max_send_sge, d);
+
+	if (qp->transport == MLX) {
+		/* For MLX transport we need 2 extra S/G entries:
+		 * for header and checksum */
+		qp->sq.max_gs += 2;
+	}
 
-	qp->sq.max = 1 << i;
+	cap->max_inline_data = qp->sq.max_gs * sizeof (struct mthca_data_seg);
+	cap->max_send_wr     = qp->sq.max;
+	cap->max_recv_wr     = qp->rq.max;
 }
 
 int mthca_alloc_qp(struct mthca_dev *dev,
@@ -1229,11 +1251,12 @@ int mthca_alloc_qp(struct mthca_dev *dev
 		   struct mthca_cq *recv_cq,
 		   enum ib_qp_type type,
 		   enum ib_sig_type send_policy,
+		   struct ib_qp_cap *cap,
 		   struct mthca_qp *qp)
 {
 	int err;
 
-	mthca_align_qp_size(dev, qp);
+	mthca_set_qp_size(dev, cap, qp);
 
 	switch (type) {
 	case IB_QPT_RC: qp->transport = RC; break;
@@ -1266,6 +1289,7 @@ int mthca_alloc_sqp(struct mthca_dev *de
 		    struct mthca_cq *send_cq,
 		    struct mthca_cq *recv_cq,
 		    enum ib_sig_type send_policy,
+		    struct ib_qp_cap *cap,
 		    int qpn,
 		    int port,
 		    struct mthca_sqp *sqp)
@@ -1273,7 +1297,7 @@ int mthca_alloc_sqp(struct mthca_dev *de
 	int err = 0;
 	u32 mqpn = qpn * 2 + dev->qp_table.sqp_start + port - 1;
 
-	mthca_align_qp_size(dev, &sqp->qp);
+	mthca_set_qp_size(dev, cap, &sqp->qp);
 
 	sqp->header_buf_size = sqp->qp.sq.max * MTHCA_UD_HEADER_SIZE;
 	sqp->header_buf = dma_alloc_coherent(&dev->pdev->dev, sqp->header_buf_size,

-- 
MST - Michael S. Tsirkin