[ofa-general] [PATCH 2/3] libmlx4: add support for create_qp_expanded

[Jack Morgenstein]

You need to modify mlx4_create_qp() as well, so that it calls
mlx4_create_qp_expanded with create_flags = 0 (and get rid of
all the excess code).

On Sunday 20 July 2008 22:15, Ron Livne wrote:
> Adds support for the new verb: create_qp_expanded
> in libmlx4.
> 
> Signed-off-by: Ron Livne <ronli at voltaire.com>
> 
> diff --git a/src/mlx4.c b/src/mlx4.c
> index 1f23144..bfcb7ec 100644
> --- a/src/mlx4.c
> +++ b/src/mlx4.c
> @@ -79,6 +79,7 @@ static struct ibv_more_ops mlx4_more_ops = {
>  	.query_xrc_rcv_qp = mlx4_query_xrc_rcv_qp,
>  	.reg_xrc_rcv_qp   = mlx4_reg_xrc_rcv_qp,
>  	.unreg_xrc_rcv_qp = mlx4_unreg_xrc_rcv_qp,
> +	.create_qp_expanded = mlx4_create_qp_expanded,
>  #endif
>  };
>  #endif
> diff --git a/src/mlx4.h b/src/mlx4.h
> index a2e3aec..7ed3ac8 100644
> --- a/src/mlx4.h
> +++ b/src/mlx4.h
> @@ -427,6 +427,9 @@ int mlx4_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
>  			uint32_t xrc_qp_num);
>  int mlx4_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
>  			uint32_t xrc_qp_num);
> +struct ibv_qp *mlx4_create_qp_expanded(struct ibv_pd *pd,
> +					struct ibv_qp_init_attr *attr,
> +					uint32_t create_flags);
>  #endif
> 
> 
> diff --git a/src/verbs.c b/src/verbs.c
> index b1bad3c..9a99658 100644
> --- a/src/verbs.c
> +++ b/src/verbs.c
> @@ -825,4 +825,115 @@ int mlx4_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
>  	return ibv_cmd_unreg_xrc_rcv_qp(xrc_domain, xrc_qp_num);
>  }
> 
> +struct ibv_qp *mlx4_create_qp_expanded(struct ibv_pd *pd,
> +					struct ibv_qp_init_attr *attr,
> +					uint32_t create_flags)
> +{
> +	struct mlx4_create_qp	cmd;
> +	struct ibv_create_qp_resp resp;
> +	struct mlx4_qp		 *qp;
> +	int			  ret;
> +
> +	/* Sanity check QP size before proceeding */
> +	if (attr->cap.max_send_wr     > 65536 ||
> +	    attr->cap.max_recv_wr     > 65536 ||
> +	    attr->cap.max_send_sge    > 64    ||
> +	    attr->cap.max_recv_sge    > 64    ||
> +	    attr->cap.max_inline_data > 1024)
> +		return NULL;
> +
> +	qp = malloc(sizeof *qp);
> +	if (!qp)
> +		return NULL;
> +
> +	mlx4_calc_sq_wqe_size(&attr->cap, attr->qp_type, qp);
> +
> +	/*
> +	 * We need to leave 2 KB + 1 WQE of headroom in the SQ to
> +	 * allow HW to prefetch.
> +	 */
> +	qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + 1;
> +	qp->sq.wqe_cnt = align_queue_size(attr->cap.max_send_wr + qp->sq_spare_wqes);
> +	qp->rq.wqe_cnt = align_queue_size(attr->cap.max_recv_wr);
> +
> +	if (attr->srq || attr->qp_type == IBV_QPT_XRC)
> +		attr->cap.max_recv_wr = qp->rq.wqe_cnt = 0;
> +	else {
> +		if (attr->cap.max_recv_sge < 1)
> +			attr->cap.max_recv_sge = 1;
> +		if (attr->cap.max_recv_wr < 1)
> +			attr->cap.max_recv_wr = 1;
> +	}
> +
> +	if (mlx4_alloc_qp_buf(pd, &attr->cap, attr->qp_type, qp))
> +		goto err;
> +
> +	mlx4_init_qp_indices(qp);
> +
> +	if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE) ||
> +	    pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE))
> +		goto err_free;
> +
> +	if (!attr->srq && attr->qp_type != IBV_QPT_XRC) {
> +		qp->db = mlx4_alloc_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ);
> +		if (!qp->db)
> +			goto err_free;
> +
> +		*qp->db = 0;
> +	}
> +
> +	cmd.buf_addr	    = (uintptr_t) qp->buf.buf;
> +	if (attr->srq || attr->qp_type == IBV_QPT_XRC)
> +		cmd.db_addr = 0;
> +	else
> +		cmd.db_addr = (uintptr_t) qp->db;
> +	cmd.log_sq_stride   = qp->sq.wqe_shift;
> +	for (cmd.log_sq_bb_count = 0;
> +	     qp->sq.wqe_cnt > 1 << cmd.log_sq_bb_count;
> +	     ++cmd.log_sq_bb_count)
> +		; /* nothing */
> +	cmd.sq_no_prefetch = 0;
> +
> +	ret = ibv_cmd_create_qp_expanded(pd, &qp->ibv_qp, attr, create_flags,
> +					&cmd.ibv_cmd, sizeof cmd,
> +					&resp, sizeof resp);
> +	if (ret)
> +		goto err_rq_db;
> +
> +	ret = mlx4_store_qp(to_mctx(pd->context), qp->ibv_qp.qp_num, qp);
> +	if (ret)
> +		goto err_destroy;
> +
> +	qp->rq.wqe_cnt = qp->rq.max_post = attr->cap.max_recv_wr;
> +	qp->rq.max_gs  = attr->cap.max_recv_sge;
> +	mlx4_set_sq_sizes(qp, &attr->cap, attr->qp_type);
> +
> +	qp->doorbell_qpn    = htonl(qp->ibv_qp.qp_num << 8);
> +	if (attr->sq_sig_all)
> +		qp->sq_signal_bits = htonl(MLX4_WQE_CTRL_CQ_UPDATE);
> +	else
> +		qp->sq_signal_bits = 0;
> +
> +	return &qp->ibv_qp;
> +
> +err_destroy:
> +	ibv_cmd_destroy_qp(&qp->ibv_qp);
> +
> +err_rq_db:
> +	if (!attr->srq && attr->qp_type != IBV_QPT_XRC)
> +		mlx4_free_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ, qp->db);
> +
> +err_free:
> +	free(qp->sq.wrid);
> +	if (qp->rq.wqe_cnt)
> +		free(qp->rq.wrid);
> +	mlx4_free_buf(&qp->buf);
> +
> +err:
> +	free(qp);
> +
> +	return NULL;
> +}
> +
> +
>  #endif
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
> 
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
>