[ofa-general] [PATCH] libmlx4: revert commit which eliminated extra CQE.

[Jack Morgenstein]

commit 078f2170e768e707b8c71eae315f87e2a0c3ab12
Author: Jack Morgenstein <jackm at mellanox.co.il>
Date:   Tue Jan 8 09:53:35 2008 +0200

    Revert "Don't add an extra entry to CQs"
    
    It turns out that this entry is needed for the resize-cq implementation.
    
    This reverts commit 216b90eac10cc8e11b9abaa710385986e26fbf85.
    
    Signed-off-by: Jack Morgenstein <jackm at dev.mellanox.co.il>

---
Roland,

We need to return the extra CQE -- it is needed for a special CQE (opcode = 16h) which
denotes that the resizing operation has completed.  This CQE is placed in the
old CQ buffer, and indicates that it is no longer used by the HCA.

Sorry about that!

- Jack

diff --git a/src/cq.c b/src/cq.c
index 06ae9e2..d9ebff1 100644
--- a/src/cq.c
+++ b/src/cq.c
@@ -114,10 +114,10 @@ static struct mlx4_cqe *get_cqe(struct mlx4_cq *cq, int entry)
 
 static void *get_sw_cqe(struct mlx4_cq *cq, int n)
 {
-	struct mlx4_cqe *cqe = get_cqe(cq, n & (cq->ibv_cq.cqe - 1));
+	struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibv_cq.cqe);
 
 	return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
-		!!(n & cq->ibv_cq.cqe)) ? NULL : cqe;
+		!!(n & (cq->ibv_cq.cqe + 1))) ? NULL : cqe;
 }
 
 static struct mlx4_cqe *next_cqe_sw(struct mlx4_cq *cq)
@@ -398,7 +398,7 @@ void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)
 	 * from our QP and therefore don't need to be checked.
 	 */
 	for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
-		if (prod_index == cq->cons_index + cq->ibv_cq.cqe - 1)
+		if (prod_index == cq->cons_index + cq->ibv_cq.cqe)
 			break;
 
 	/*
@@ -406,13 +406,13 @@ void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)
 	 * that match our QP by copying older entries on top of them.
 	 */
 	while ((int) --prod_index - (int) cq->cons_index >= 0) {
-		cqe = get_cqe(cq, prod_index & (cq->ibv_cq.cqe - 1));
+		cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
 		if ((ntohl(cqe->my_qpn) & 0xffffff) == qpn) {
 			if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
 				mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
 			++nfreed;
 		} else if (nfreed) {
-			dest = get_cqe(cq, (prod_index + nfreed) & (cq->ibv_cq.cqe - 1));
+			dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe);
 			owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
 			memcpy(dest, cqe, sizeof *cqe);
 			dest->owner_sr_opcode = owner_bit |
diff --git a/src/verbs.c b/src/verbs.c
index 0bbab57..50e0947 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -182,11 +182,7 @@ struct ibv_cq *mlx4_create_cq(struct ibv_context *context, int cqe,
 	if (pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE))
 		goto err;
 
-	cqe = align_queue_size(cqe);
-
-	/* Always allocate at least two CQEs to keep things simple */
-	if (cqe < 2)
-		cqe = 2;
+	cqe = align_queue_size(cqe + 1);
 
 	if (mlx4_alloc_buf(&cq->buf, cqe * MLX4_CQ_ENTRY_SIZE,
 			   to_mdev(context->device)->page_size))
@@ -206,8 +202,6 @@ struct ibv_cq *mlx4_create_cq(struct ibv_context *context, int cqe,
 	cmd.buf_addr = (uintptr_t) cq->buf.buf;
 	cmd.db_addr  = (uintptr_t) cq->set_ci_db;
 
-	/* Subtract 1 from the number of entries we pass into the
-	 * kernel because the kernel mlx4_ib driver will add 1 again. */
 	ret = ibv_cmd_create_cq(context, cqe - 1, channel, comp_vector,
 				&cq->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
 				&resp.ibv_resp, sizeof resp);
@@ -215,8 +209,6 @@ struct ibv_cq *mlx4_create_cq(struct ibv_context *context, int cqe,
 		goto err_db;
 
 	cq->cqn = resp.cqn;
-	/* Bump the number of entries to make up for subtracting 1 above */
-	++cq->ibv_cq.cqe;
 
 	return &cq->ibv_cq;