[ofa-general] ***SPAM*** [PATCH 2/3 v2] libmlx4: add support for create_qp_expanded

[ronli at voltaire.com]

Adds support for the new verb: create_qp_expanded
in libmlx4.

Changes in v2:
Minimized code duplications by calling mlx4_create_qp_expanded
from mlx4_create_qp.

Signed-off-by: Ron Livne <ronli at voltaire.com>

diff --git a/src/mlx4.c b/src/mlx4.c
index 1f23144..bfcb7ec 100644
--- a/src/mlx4.c
+++ b/src/mlx4.c
@@ -79,6 +79,7 @@ static struct ibv_more_ops mlx4_more_ops = {
 	.query_xrc_rcv_qp = mlx4_query_xrc_rcv_qp,
 	.reg_xrc_rcv_qp   = mlx4_reg_xrc_rcv_qp,
 	.unreg_xrc_rcv_qp = mlx4_unreg_xrc_rcv_qp,
+	.create_qp_expanded = mlx4_create_qp_expanded,
 #endif
 };
 #endif
diff --git a/src/mlx4.h b/src/mlx4.h
index a2e3aec..7ed3ac8 100644
--- a/src/mlx4.h
+++ b/src/mlx4.h
@@ -427,6 +427,9 @@ int mlx4_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
 			uint32_t xrc_qp_num);
 int mlx4_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
 			uint32_t xrc_qp_num);
+struct ibv_qp *mlx4_create_qp_expanded(struct ibv_pd *pd,
+					struct ibv_qp_init_attr *attr,
+					uint32_t create_flags);
 #endif


diff --git a/src/verbs.c b/src/verbs.c
index b1bad3c..c50e455 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -404,110 +404,7 @@ int mlx4_destroy_srq(struct ibv_srq *ibsrq)

 struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
 {
-	struct mlx4_create_qp     cmd;
-	struct ibv_create_qp_resp resp;
-	struct mlx4_qp		 *qp;
-	int			  ret;
-
-	/* Sanity check QP size before proceeding */
-	if (attr->cap.max_send_wr     > 65536 ||
-	    attr->cap.max_recv_wr     > 65536 ||
-	    attr->cap.max_send_sge    > 64    ||
-	    attr->cap.max_recv_sge    > 64    ||
-	    attr->cap.max_inline_data > 1024)
-		return NULL;
-
-	qp = malloc(sizeof *qp);
-	if (!qp)
-		return NULL;
-
-	mlx4_calc_sq_wqe_size(&attr->cap, attr->qp_type, qp);
-
-	/*
-	 * We need to leave 2 KB + 1 WQE of headroom in the SQ to
-	 * allow HW to prefetch.
-	 */
-	qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + 1;
-	qp->sq.wqe_cnt = align_queue_size(attr->cap.max_send_wr + qp->sq_spare_wqes);
-	qp->rq.wqe_cnt = align_queue_size(attr->cap.max_recv_wr);
-
-	if (attr->srq || attr->qp_type == IBV_QPT_XRC)
-		attr->cap.max_recv_wr = qp->rq.wqe_cnt = 0;
-	else {
-		if (attr->cap.max_recv_sge < 1)
-			attr->cap.max_recv_sge = 1;
-		if (attr->cap.max_recv_wr < 1)
-			attr->cap.max_recv_wr = 1;
-	}
-
-	if (mlx4_alloc_qp_buf(pd, &attr->cap, attr->qp_type, qp))
-		goto err;
-
-	mlx4_init_qp_indices(qp);
-
-	if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE) ||
-	    pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE))
-		goto err_free;
-
-	if (!attr->srq && attr->qp_type != IBV_QPT_XRC) {
-		qp->db = mlx4_alloc_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ);
-		if (!qp->db)
-			goto err_free;
-
-		*qp->db = 0;
-	}
-
-	cmd.buf_addr	    = (uintptr_t) qp->buf.buf;
-	if (attr->srq || attr->qp_type == IBV_QPT_XRC)
-		cmd.db_addr = 0;
-	else
-		cmd.db_addr = (uintptr_t) qp->db;
-	cmd.log_sq_stride   = qp->sq.wqe_shift;
-	for (cmd.log_sq_bb_count = 0;
-	     qp->sq.wqe_cnt > 1 << cmd.log_sq_bb_count;
-	     ++cmd.log_sq_bb_count)
-		; /* nothing */
-	cmd.sq_no_prefetch = 0;	/* OK for ABI 2: just a reserved field */
-	memset(cmd.reserved, 0, sizeof cmd.reserved);
-
-	ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd,
-				&resp, sizeof resp);
-	if (ret)
-		goto err_rq_db;
-
-	ret = mlx4_store_qp(to_mctx(pd->context), qp->ibv_qp.qp_num, qp);
-	if (ret)
-		goto err_destroy;
-
-	qp->rq.wqe_cnt = qp->rq.max_post = attr->cap.max_recv_wr;
-	qp->rq.max_gs  = attr->cap.max_recv_sge;
-	mlx4_set_sq_sizes(qp, &attr->cap, attr->qp_type);
-
-	qp->doorbell_qpn    = htonl(qp->ibv_qp.qp_num << 8);
-	if (attr->sq_sig_all)
-		qp->sq_signal_bits = htonl(MLX4_WQE_CTRL_CQ_UPDATE);
-	else
-		qp->sq_signal_bits = 0;
-
-	return &qp->ibv_qp;
-
-err_destroy:
-	ibv_cmd_destroy_qp(&qp->ibv_qp);
-
-err_rq_db:
-	if (!attr->srq && attr->qp_type != IBV_QPT_XRC)
-		mlx4_free_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ, qp->db);
-
-err_free:
-	free(qp->sq.wrid);
-	if (qp->rq.wqe_cnt)
-		free(qp->rq.wrid);
-	mlx4_free_buf(&qp->buf);
-
-err:
-	free(qp);
-
-	return NULL;
+	return mlx4_create_qp_expanded(pd, attr, 0);
 }

 int mlx4_query_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr,
@@ -825,4 +722,115 @@ int mlx4_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
 	return ibv_cmd_unreg_xrc_rcv_qp(xrc_domain, xrc_qp_num);
 }

+struct ibv_qp *mlx4_create_qp_expanded(struct ibv_pd *pd,
+					struct ibv_qp_init_attr *attr,
+					uint32_t create_flags)
+{
+	struct mlx4_create_qp	cmd;
+	struct ibv_create_qp_resp resp;
+	struct mlx4_qp		 *qp;
+	int			  ret;
+
+	/* Sanity check QP size before proceeding */
+	if (attr->cap.max_send_wr     > 65536 ||
+	    attr->cap.max_recv_wr     > 65536 ||
+	    attr->cap.max_send_sge    > 64    ||
+	    attr->cap.max_recv_sge    > 64    ||
+	    attr->cap.max_inline_data > 1024)
+		return NULL;
+
+	qp = malloc(sizeof *qp);
+	if (!qp)
+		return NULL;
+
+	mlx4_calc_sq_wqe_size(&attr->cap, attr->qp_type, qp);
+
+	/*
+	 * We need to leave 2 KB + 1 WQE of headroom in the SQ to
+	 * allow HW to prefetch.
+	 */
+	qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + 1;
+	qp->sq.wqe_cnt = align_queue_size(attr->cap.max_send_wr + qp->sq_spare_wqes);
+	qp->rq.wqe_cnt = align_queue_size(attr->cap.max_recv_wr);
+
+	if (attr->srq || attr->qp_type == IBV_QPT_XRC)
+		attr->cap.max_recv_wr = qp->rq.wqe_cnt = 0;
+	else {
+		if (attr->cap.max_recv_sge < 1)
+			attr->cap.max_recv_sge = 1;
+		if (attr->cap.max_recv_wr < 1)
+			attr->cap.max_recv_wr = 1;
+	}
+
+	if (mlx4_alloc_qp_buf(pd, &attr->cap, attr->qp_type, qp))
+		goto err;
+
+	mlx4_init_qp_indices(qp);
+
+	if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE) ||
+	    pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE))
+		goto err_free;
+
+	if (!attr->srq && attr->qp_type != IBV_QPT_XRC) {
+		qp->db = mlx4_alloc_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ);
+		if (!qp->db)
+			goto err_free;
+
+		*qp->db = 0;
+	}
+
+	cmd.buf_addr	    = (uintptr_t) qp->buf.buf;
+	if (attr->srq || attr->qp_type == IBV_QPT_XRC)
+		cmd.db_addr = 0;
+	else
+		cmd.db_addr = (uintptr_t) qp->db;
+	cmd.log_sq_stride   = qp->sq.wqe_shift;
+	for (cmd.log_sq_bb_count = 0;
+	     qp->sq.wqe_cnt > 1 << cmd.log_sq_bb_count;
+	     ++cmd.log_sq_bb_count)
+		; /* nothing */
+	cmd.sq_no_prefetch = 0;
+
+	ret = ibv_cmd_create_qp_expanded(pd, &qp->ibv_qp, attr, create_flags,
+					&cmd.ibv_cmd, sizeof cmd,
+					&resp, sizeof resp);
+	if (ret)
+		goto err_rq_db;
+
+	ret = mlx4_store_qp(to_mctx(pd->context), qp->ibv_qp.qp_num, qp);
+	if (ret)
+		goto err_destroy;
+
+	qp->rq.wqe_cnt = qp->rq.max_post = attr->cap.max_recv_wr;
+	qp->rq.max_gs  = attr->cap.max_recv_sge;
+	mlx4_set_sq_sizes(qp, &attr->cap, attr->qp_type);
+
+	qp->doorbell_qpn    = htonl(qp->ibv_qp.qp_num << 8);
+	if (attr->sq_sig_all)
+		qp->sq_signal_bits = htonl(MLX4_WQE_CTRL_CQ_UPDATE);
+	else
+		qp->sq_signal_bits = 0;
+
+	return &qp->ibv_qp;
+
+err_destroy:
+	ibv_cmd_destroy_qp(&qp->ibv_qp);
+
+err_rq_db:
+	if (!attr->srq && attr->qp_type != IBV_QPT_XRC)
+		mlx4_free_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ, qp->db);
+
+err_free:
+	free(qp->sq.wrid);
+	if (qp->rq.wqe_cnt)
+		free(qp->rq.wrid);
+	mlx4_free_buf(&qp->buf);
+
+err:
+	free(qp);
+
+	return NULL;
+}
+
+
 #endif