[ofa-general] [PATCHv2] IB/mad: Allow tuning of QP0 and QP1 sizes

[Suresh Shelvapille]

Hal:

1. Aren't you going to remove the power_of_two?
2. Also, don't you need permissions to be 644?

-Suri

> -----Original Message-----
> From: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] On Behalf
> Of Hal Rosenstock
> Sent: Wednesday, August 12, 2009 1:23 PM
> To: rdreier at cisco.com; sean.hefty at intel.com
> Cc: general at lists.openfabrics.org
> Subject: [ofa-general] [PATCHv2] IB/mad: Allow tuning of QP0 and QP1 sizes
> 
> 
> IB/mad: Allow tuning of QP0 and QP1 sizes
> 
> MADs are UD and can be dropped if there are no receives posted.
> Send side tuning is done for symmetry with receive.
> 
> Signed-off-by: Hal Rosenstock <hal.rosenstock at gmail.com>
> ---
> Changes since v1:
> Added changelog
> 
> diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
> index de922a0..7e553c3 100644
> --- a/drivers/infiniband/core/mad.c
> +++ b/drivers/infiniband/core/mad.c
> @@ -2,6 +2,7 @@
>   * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
>   * Copyright (c) 2005 Intel Corporation.  All rights reserved.
>   * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>   *
>   * This software is available to you under a choice of one of two
>   * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -45,6 +46,14 @@ MODULE_DESCRIPTION("kernel IB MAD API");
>  MODULE_AUTHOR("Hal Rosenstock");
>  MODULE_AUTHOR("Sean Hefty");
> 
> +int mad_sendq_size = IB_MAD_QP_SEND_SIZE;
> +int mad_recvq_size = IB_MAD_QP_RECV_SIZE;
> +
> +module_param_named(send_queue_size, mad_sendq_size, int, 0444);
> +MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests");
> +module_param_named(recv_queue_size, mad_recvq_size, int, 0444);
> +MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests");
> +
>  static struct kmem_cache *ib_mad_cache;
> 
>  static struct list_head ib_mad_port_list;
> @@ -2736,8 +2745,8 @@ static int create_mad_qp(struct ib_mad_qp_info *qp_info,
>  	qp_init_attr.send_cq = qp_info->port_priv->cq;
>  	qp_init_attr.recv_cq = qp_info->port_priv->cq;
>  	qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
> -	qp_init_attr.cap.max_send_wr = IB_MAD_QP_SEND_SIZE;
> -	qp_init_attr.cap.max_recv_wr = IB_MAD_QP_RECV_SIZE;
> +	qp_init_attr.cap.max_send_wr = mad_sendq_size;
> +	qp_init_attr.cap.max_recv_wr = mad_recvq_size;
>  	qp_init_attr.cap.max_send_sge = IB_MAD_SEND_REQ_MAX_SG;
>  	qp_init_attr.cap.max_recv_sge = IB_MAD_RECV_REQ_MAX_SG;
>  	qp_init_attr.qp_type = qp_type;
> @@ -2752,8 +2761,8 @@ static int create_mad_qp(struct ib_mad_qp_info *qp_info,
>  		goto error;
>  	}
>  	/* Use minimum queue sizes unless the CQ is resized */
> -	qp_info->send_queue.max_active = IB_MAD_QP_SEND_SIZE;
> -	qp_info->recv_queue.max_active = IB_MAD_QP_RECV_SIZE;
> +	qp_info->send_queue.max_active = mad_sendq_size;
> +	qp_info->recv_queue.max_active = mad_recvq_size;
>  	return 0;
> 
>  error:
> @@ -2792,7 +2801,7 @@ static int ib_mad_port_open(struct ib_device *device,
>  	init_mad_qp(port_priv, &port_priv->qp_info[0]);
>  	init_mad_qp(port_priv, &port_priv->qp_info[1]);
> 
> -	cq_size = (IB_MAD_QP_SEND_SIZE + IB_MAD_QP_RECV_SIZE) * 2;
> +	cq_size = (mad_sendq_size + mad_recvq_size) * 2;
>  	port_priv->cq = ib_create_cq(port_priv->device,
>  				     ib_mad_thread_completion_handler,
>  				     NULL, port_priv, cq_size, 0);
> @@ -2984,6 +2993,14 @@ static int __init ib_mad_init_module(void)
>  {
>  	int ret;
> 
> +	mad_recvq_size = roundup_pow_of_two(mad_recvq_size);
> +	mad_recvq_size = min(mad_recvq_size, IB_MAD_QP_MAX_SIZE);
> +	mad_recvq_size = max(mad_recvq_size, IB_MAD_QP_MIN_SIZE);
> +
> +	mad_sendq_size = roundup_pow_of_two(mad_sendq_size);
> +	mad_sendq_size = min(mad_sendq_size, IB_MAD_QP_MAX_SIZE);
> +	mad_sendq_size = max(mad_sendq_size, IB_MAD_QP_MIN_SIZE);
> +
>  	spin_lock_init(&ib_mad_port_list_lock);
> 
>  	ib_mad_cache = kmem_cache_create("ib_mad",
> diff --git a/drivers/infiniband/core/mad_priv.h b/drivers/infiniband/core/mad_priv.h
> index 05ce331..9430ab4 100644
> --- a/drivers/infiniband/core/mad_priv.h
> +++ b/drivers/infiniband/core/mad_priv.h
> @@ -2,6 +2,7 @@
>   * Copyright (c) 2004, 2005, Voltaire, Inc. All rights reserved.
>   * Copyright (c) 2005 Intel Corporation. All rights reserved.
>   * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>   *
>   * This software is available to you under a choice of one of two
>   * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -49,6 +50,8 @@
>  /* QP and CQ parameters */
>  #define IB_MAD_QP_SEND_SIZE	128
>  #define IB_MAD_QP_RECV_SIZE	512
> +#define IB_MAD_QP_MIN_SIZE	64
> +#define IB_MAD_QP_MAX_SIZE	8192
>  #define IB_MAD_SEND_REQ_MAX_SG	2
>  #define IB_MAD_RECV_REQ_MAX_SG	1
> 
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
> 
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general