[openib-general] [RFC] [PATCH] IB/uverbs: Don't serialize with ib_uverbs_idr_mutex
Roland Dreier
rdreier at cisco.com
Fri Jun 16 15:05:54 PDT 2006
OK, I think that the modify_qp, modify_srq and resize_cq calls all
need to be serialized. Unfortunately the only good way I could see to
serialize these calls is to add a mutex to mthca's CQ, QP and SRQ
structures (which bloats the structures somewhat). The patch I
committed is below -- with this change I think we're OK even if
userspace does crazy multithreaded stuff.
IB/mthca: Make all device methods truly reentrant
Documentation/infiniband/core_locking.txt says:
All of the methods in struct ib_device exported by a low-level
driver must be fully reentrant. The low-level driver is required to
perform all synchronization necessary to maintain consistency, even
if multiple function calls using the same object are run
simultaneously.
However, mthca's modify_qp, modify_srq and resize_cq methods are
currently not reentrant. Add a mutex to the QP, SRQ and CQ structures
so that these calls can be properly serialized.
Signed-off-by: Roland Dreier <rolandd at cisco.com>
diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index 205854e..f20a5b6 100644
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -813,6 +813,7 @@ int mthca_init_cq(struct mthca_dev *dev,
spin_lock_init(&cq->lock);
cq->refcount = 1;
init_waitqueue_head(&cq->wait);
+ mutex_init(&cq->mutex);
memset(cq_context, 0, sizeof *cq_context);
cq_context->flags = cpu_to_be32(MTHCA_CQ_STATUS_OK |
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 8f89ba7..230ae21 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -793,18 +793,24 @@ static int mthca_resize_cq(struct ib_cq
if (entries < 1 || entries > dev->limits.max_cqes)
return -EINVAL;
+ mutex_lock(&cq->mutex);
+
entries = roundup_pow_of_two(entries + 1);
- if (entries == ibcq->cqe + 1)
- return 0;
+ if (entries == ibcq->cqe + 1) {
+ ret = 0;
+ goto out;
+ }
if (cq->is_kernel) {
ret = mthca_alloc_resize_buf(dev, cq, entries);
if (ret)
- return ret;
+ goto out;
lkey = cq->resize_buf->buf.mr.ibmr.lkey;
} else {
- if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
- return -EFAULT;
+ if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
+ ret = -EFAULT;
+ goto out;
+ }
lkey = ucmd.lkey;
}
@@ -821,7 +827,7 @@ static int mthca_resize_cq(struct ib_cq
cq->resize_buf = NULL;
spin_unlock_irq(&cq->lock);
}
- return ret;
+ goto out;
}
if (cq->is_kernel) {
@@ -848,7 +854,10 @@ static int mthca_resize_cq(struct ib_cq
} else
ibcq->cqe = entries - 1;
- return 0;
+out:
+ mutex_unlock(&cq->mutex);
+
+ return ret;
}
static int mthca_destroy_cq(struct ib_cq *cq)
diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index 322bc32..16c387d 100644
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -536,6 +536,8 @@ int mthca_modify_qp(struct ib_qp *ibqp,
u8 status;
int err = -EINVAL;
+ mutex_lock(&qp->mutex);
+
if (attr_mask & IB_QP_CUR_STATE) {
cur_state = attr->cur_qp_state;
} else {
@@ -553,39 +555,41 @@ int mthca_modify_qp(struct ib_qp *ibqp,
"%d->%d with attr 0x%08x\n",
qp->transport, cur_state, new_state,
attr_mask);
- return -EINVAL;
+ goto out;
}
if ((attr_mask & IB_QP_PKEY_INDEX) &&
attr->pkey_index >= dev->limits.pkey_table_len) {
mthca_dbg(dev, "P_Key index (%u) too large. max is %d\n",
attr->pkey_index, dev->limits.pkey_table_len-1);
- return -EINVAL;
+ goto out;
}
if ((attr_mask & IB_QP_PORT) &&
(attr->port_num == 0 || attr->port_num > dev->limits.num_ports)) {
mthca_dbg(dev, "Port number (%u) is invalid\n", attr->port_num);
- return -EINVAL;
+ goto out;
}
if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
attr->max_rd_atomic > dev->limits.max_qp_init_rdma) {
mthca_dbg(dev, "Max rdma_atomic as initiator %u too large (max is %d)\n",
attr->max_rd_atomic, dev->limits.max_qp_init_rdma);
- return -EINVAL;
+ goto out;
}
if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
attr->max_dest_rd_atomic > 1 << dev->qp_table.rdb_shift) {
mthca_dbg(dev, "Max rdma_atomic as responder %u too large (max %d)\n",
attr->max_dest_rd_atomic, 1 << dev->qp_table.rdb_shift);
- return -EINVAL;
+ goto out;
}
mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
- if (IS_ERR(mailbox))
- return PTR_ERR(mailbox);
+ if (IS_ERR(mailbox)) {
+ err = PTR_ERR(mailbox);
+ goto out;
+ }
qp_param = mailbox->buf;
qp_context = &qp_param->context;
memset(qp_param, 0, sizeof *qp_param);
@@ -618,7 +622,7 @@ int mthca_modify_qp(struct ib_qp *ibqp,
if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_2048) {
mthca_dbg(dev, "path MTU (%u) is invalid\n",
attr->path_mtu);
- goto out;
+ goto out_mailbox;
}
qp_context->mtu_msgmax = (attr->path_mtu << 5) | 31;
}
@@ -672,7 +676,7 @@ int mthca_modify_qp(struct ib_qp *ibqp,
if (attr_mask & IB_QP_AV) {
if (mthca_path_set(dev, &attr->ah_attr, &qp_context->pri_path,
attr_mask & IB_QP_PORT ? attr->port_num : qp->port))
- goto out;
+ goto out_mailbox;
qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH);
}
@@ -686,18 +690,18 @@ int mthca_modify_qp(struct ib_qp *ibqp,
if (attr->alt_pkey_index >= dev->limits.pkey_table_len) {
mthca_dbg(dev, "Alternate P_Key index (%u) too large. max is %d\n",
attr->alt_pkey_index, dev->limits.pkey_table_len-1);
- goto out;
+ goto out_mailbox;
}
if (attr->alt_port_num == 0 || attr->alt_port_num > dev->limits.num_ports) {
mthca_dbg(dev, "Alternate port number (%u) is invalid\n",
attr->alt_port_num);
- goto out;
+ goto out_mailbox;
}
if (mthca_path_set(dev, &attr->alt_ah_attr, &qp_context->alt_path,
attr->alt_ah_attr.port_num))
- goto out;
+ goto out_mailbox;
qp_context->alt_path.port_pkey |= cpu_to_be32(attr->alt_pkey_index |
attr->alt_port_num << 24);
@@ -793,12 +797,12 @@ int mthca_modify_qp(struct ib_qp *ibqp,
err = mthca_MODIFY_QP(dev, cur_state, new_state, qp->qpn, 0,
mailbox, sqd_event, &status);
if (err)
- goto out;
+ goto out_mailbox;
if (status) {
mthca_warn(dev, "modify QP %d->%d returned status %02x.\n",
cur_state, new_state, status);
err = -EINVAL;
- goto out;
+ goto out_mailbox;
}
qp->state = new_state;
@@ -853,8 +857,11 @@ int mthca_modify_qp(struct ib_qp *ibqp,
}
}
-out:
+out_mailbox:
mthca_free_mailbox(dev, mailbox);
+
+out:
+ mutex_unlock(&qp->mutex);
return err;
}
@@ -1100,6 +1107,7 @@ static int mthca_alloc_qp_common(struct
qp->refcount = 1;
init_waitqueue_head(&qp->wait);
+ mutex_init(&qp->mutex);
qp->state = IB_QPS_RESET;
qp->atomic_rd_en = 0;
qp->resp_depth = 0;
diff --git a/drivers/infiniband/hw/mthca/mthca_srq.c b/drivers/infiniband/hw/mthca/mthca_srq.c
index b292fef..fab417c 100644
--- a/drivers/infiniband/hw/mthca/mthca_srq.c
+++ b/drivers/infiniband/hw/mthca/mthca_srq.c
@@ -243,6 +243,7 @@ int mthca_alloc_srq(struct mthca_dev *de
spin_lock_init(&srq->lock);
srq->refcount = 1;
init_waitqueue_head(&srq->wait);
+ mutex_init(&srq->mutex);
if (mthca_is_memfree(dev))
mthca_arbel_init_srq_context(dev, pd, srq, mailbox->buf);
@@ -371,7 +372,11 @@ int mthca_modify_srq(struct ib_srq *ibsr
if (attr_mask & IB_SRQ_LIMIT) {
if (attr->srq_limit > srq->max)
return -EINVAL;
+
+ mutex_lock(&srq->mutex);
ret = mthca_ARM_SRQ(dev, srq->srqn, attr->srq_limit, &status);
+ mutex_unlock(&srq->mutex);
+
if (ret)
return ret;
if (status)
More information about the general
mailing list