[openib-general] [PATCH 1 of 2] mthca: qp size calculations
Michael S. Tsirkin
mst at mellanox.co.il
Tue Nov 8 09:46:44 PST 2005
Hello, Roland!
So here's the kernel part of the patch that fixes the qp creation
by moving the common part of the math to kernel, where we know the
device limits.
This is built along the lines that we agreed upon: kernel does all the
checks and returns the actual capabilities to userspace.
With this patch in place, it should be quite easy now to add inline
data support for kernel, but thats a separate issue.
Please comment.
---
1. Check that descriptor size does now exceed the value supported by hardware.
2. Set max_send_sge/max_recv_wr (and max_inline_data for userspace)
in the create qp attributes structure to max values supported
by qp, as opposed to (safe, but smaller) values requested by user.
Signed-off-by: Michael S. Tsirkin <mst at mellanox.co.il>
Signed-off-by: Jack MorgensteinMichael <jackm at mellanox.co.il>
Index: linux-2.6.14/drivers/infiniband/include/rdma/ib_user_verbs.h
===================================================================
--- linux-2.6.14.orig/drivers/infiniband/include/rdma/ib_user_verbs.h 2005-11-02 14:24:11.000000000 +0200
+++ linux-2.6.14/drivers/infiniband/include/rdma/ib_user_verbs.h 2005-11-08 19:29:37.000000000 +0200
@@ -43,7 +43,7 @@
* Increment this value if any changes that break userspace ABI
* compatibility are made.
*/
-#define IB_USER_VERBS_ABI_VERSION 3
+#define IB_USER_VERBS_ABI_VERSION 4
enum {
IB_USER_VERBS_CMD_GET_CONTEXT,
@@ -333,6 +333,11 @@ struct ib_uverbs_create_qp {
struct ib_uverbs_create_qp_resp {
__u32 qp_handle;
__u32 qpn;
+ __u32 max_send_wr;
+ __u32 max_recv_wr;
+ __u32 max_send_sge;
+ __u32 max_recv_sge;
+ __u32 max_inline_data;
};
/*
Index: linux-2.6.14/drivers/infiniband/core/uverbs_cmd.c
===================================================================
--- linux-2.6.14.orig/drivers/infiniband/core/uverbs_cmd.c 2005-11-02 14:24:11.000000000 +0200
+++ linux-2.6.14/drivers/infiniband/core/uverbs_cmd.c 2005-11-08 19:29:37.000000000 +0200
@@ -908,7 +908,12 @@ retry:
if (ret)
goto err_destroy;
- resp.qp_handle = uobj->uobject.id;
+ resp.qp_handle = uobj->uobject.id;
+ resp.max_recv_sge = attr.cap.max_recv_sge;
+ resp.max_send_sge = attr.cap.max_send_sge;
+ resp.max_recv_wr = attr.cap.max_recv_wr;
+ resp.max_send_wr = attr.cap.max_send_wr;
+ resp.max_inline_data = attr.cap.max_inline_data;
if (copy_to_user((void __user *) (unsigned long) cmd.response,
&resp, sizeof resp)) {
Index: linux-2.6.14/drivers/infiniband/hw/mthca/mthca_provider.c
===================================================================
--- linux-2.6.14.orig/drivers/infiniband/hw/mthca/mthca_provider.c 2005-11-06 10:30:43.000000000 +0200
+++ linux-2.6.14/drivers/infiniband/hw/mthca/mthca_provider.c 2005-11-08 19:29:37.000000000 +0200
@@ -604,11 +604,11 @@ static struct ib_qp *mthca_create_qp(str
return ERR_PTR(err);
}
- init_attr->cap.max_inline_data = 0;
init_attr->cap.max_send_wr = qp->sq.max;
init_attr->cap.max_recv_wr = qp->rq.max;
init_attr->cap.max_send_sge = qp->sq.max_gs;
init_attr->cap.max_recv_sge = qp->rq.max_gs;
+ init_attr->cap.max_inline_data = qp->max_inline_data;
return &qp->ibqp;
}
Index: linux-2.6.14/drivers/infiniband/hw/mthca/mthca_provider.h
===================================================================
--- linux-2.6.14.orig/drivers/infiniband/hw/mthca/mthca_provider.h 2005-09-12 10:50:00.000000000 +0300
+++ linux-2.6.14/drivers/infiniband/hw/mthca/mthca_provider.h 2005-11-08 19:29:37.000000000 +0200
@@ -251,6 +251,7 @@ struct mthca_qp {
struct mthca_wq sq;
enum ib_sig_type sq_policy;
int send_wqe_offset;
+ int max_inline_data;
u64 *wrid;
union mthca_buf queue;
Index: linux-2.6.14/drivers/infiniband/hw/mthca/mthca_cmd.c
===================================================================
--- linux-2.6.14.orig/drivers/infiniband/hw/mthca/mthca_cmd.c 2005-11-02 14:24:16.000000000 +0200
+++ linux-2.6.14/drivers/infiniband/hw/mthca/mthca_cmd.c 2005-11-08 19:29:37.000000000 +0200
@@ -1060,6 +1060,8 @@ int mthca_QUERY_DEV_LIM(struct mthca_dev
dev_lim->hca.arbel.resize_srq = field & 1;
MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SG_RQ_OFFSET);
dev_lim->max_sg = min_t(int, field, dev_lim->max_sg);
+ MTHCA_GET(size, outbox, QUERY_DEV_LIM_MAX_DESC_SZ_RQ_OFFSET);
+ dev_lim->max_desc_sz = min_t(int, size, dev_lim->max_desc_sz);
MTHCA_GET(size, outbox, QUERY_DEV_LIM_MPT_ENTRY_SZ_OFFSET);
dev_lim->mpt_entry_sz = size;
MTHCA_GET(field, outbox, QUERY_DEV_LIM_PBL_SZ_OFFSET);
Index: linux-2.6.14/drivers/infiniband/hw/mthca/mthca_dev.h
===================================================================
--- linux-2.6.14.orig/drivers/infiniband/hw/mthca/mthca_dev.h 2005-11-06 10:30:43.000000000 +0200
+++ linux-2.6.14/drivers/infiniband/hw/mthca/mthca_dev.h 2005-11-08 19:29:37.000000000 +0200
@@ -131,6 +131,7 @@ struct mthca_limits {
int max_sg;
int num_qps;
int max_wqes;
+ int max_desc_sz;
int max_qp_init_rdma;
int reserved_qps;
int num_srqs;
Index: linux-2.6.14/drivers/infiniband/hw/mthca/mthca_main.c
===================================================================
--- linux-2.6.14.orig/drivers/infiniband/hw/mthca/mthca_main.c 2005-11-06 10:30:43.000000000 +0200
+++ linux-2.6.14/drivers/infiniband/hw/mthca/mthca_main.c 2005-11-08 19:29:37.000000000 +0200
@@ -168,6 +168,7 @@ static int __devinit mthca_dev_lim(struc
mdev->limits.max_srq_wqes = dev_lim->max_srq_sz;
mdev->limits.reserved_srqs = dev_lim->reserved_srqs;
mdev->limits.reserved_eecs = dev_lim->reserved_eecs;
+ mdev->limits.max_desc_sz = dev_lim->max_desc_sz;
/*
* Subtract 1 from the limit because we need to allocate a
* spare CQE so the HCA HW can tell the difference between an
Index: linux-2.6.14/drivers/infiniband/hw/mthca/mthca_qp.c
===================================================================
--- linux-2.6.14.orig/drivers/infiniband/hw/mthca/mthca_qp.c 2005-11-06 10:30:43.000000000 +0200
+++ linux-2.6.14/drivers/infiniband/hw/mthca/mthca_qp.c 2005-11-08 19:36:37.000000000 +0200
@@ -883,6 +883,46 @@ int mthca_modify_qp(struct ib_qp *ibqp,
return err;
}
+static void mthca_adjust_qp_caps(struct mthca_dev *dev,
+ struct mthca_pd *pd,
+ struct mthca_qp *qp)
+{
+ int max_data_size;
+ /*
+ * Calculate the maximum size of WQE s/g segments, excluding
+ * the next segment and other non-data segments.
+ */
+
+ max_data_size = min(dev->limits.max_desc_sz, 1 << qp->sq.wqe_shift) -
+ sizeof (struct mthca_next_seg);
+
+ switch (qp->transport) {
+ case MLX:
+ max_data_size -= 2 * sizeof (struct mthca_data_seg);
+ break;
+ case UD:
+ if (mthca_is_memfree(dev))
+ max_data_size -= sizeof (struct mthca_arbel_ud_seg);
+ else
+ max_data_size -= sizeof (struct mthca_tavor_ud_seg);
+ break;
+
+ default:
+ max_data_size -= sizeof (struct mthca_bind_seg);
+ break;
+ }
+
+ if (!pd->ibpd.uobject)
+ qp->max_inline_data = 0;
+ else
+ qp->max_inline_data = max_data_size - MTHCA_INLINE_HEADER_SIZE;
+
+ qp->sq.max_gs = max_data_size / sizeof (struct mthca_data_seg);
+ qp->rq.max_gs = (min(dev->limits.max_desc_sz, 1 << qp->rq.wqe_shift) -
+ sizeof (struct mthca_next_seg)) /
+ sizeof (struct mthca_data_seg);
+}
+
/*
* Allocate and register buffer for WQEs. qp->rq.max, sq.max,
* rq.max_gs and sq.max_gs must all be assigned.
@@ -900,6 +940,9 @@ static int mthca_alloc_wqe_buf(struct mt
size = sizeof (struct mthca_next_seg) +
qp->rq.max_gs * sizeof (struct mthca_data_seg);
+ if (size > dev->limits.max_desc_sz)
+ return -EINVAL;
+
for (qp->rq.wqe_shift = 6; 1 << qp->rq.wqe_shift < size;
qp->rq.wqe_shift++)
; /* nothing */
@@ -921,6 +964,9 @@ static int mthca_alloc_wqe_buf(struct mt
size += sizeof (struct mthca_bind_seg);
}
+ if (size > dev->limits.max_desc_sz)
+ return -EINVAL;
+
for (qp->sq.wqe_shift = 6; 1 << qp->sq.wqe_shift < size;
qp->sq.wqe_shift++)
; /* nothing */
@@ -1064,6 +1110,7 @@ static int mthca_alloc_qp_common(struct
return ret;
}
+ mthca_adjust_qp_caps(dev, pd, qp);
/*
* If this is a userspace QP, we're done now. The doorbells
* will be allocated and buffers will be initialized in
--
MST
More information about the general
mailing list