[openib-general] Re: [PATCH] libmthca: fix wqe post (was Re: strange mem-free bug)

[Viswanath Krishnamurthy]

Michael,

Thanks..

Roland,

Once you generate a kernel patch, I can test out both user and kernel mthca 
since I have the tests
ready..

-Viswa


On 9/13/05, Michael S. Tsirkin <mst at mellanox.co.il> wrote:
> 
> Quoting r. Roland Dreier <rolandd at cisco.com>:
> > Subject: strange mem-free bug (was: [openib-general] completion Q 
> overflow error/panic)
> >
> > While looking at Viswa's example, I've found what seems to be a
> > problem using lots of QPs on mem-free HCAs.
> 
> Hi, Roland!
> This seems to be a bug in libmthca. Patch below.
> 
> We probably need a similiar fix for kernel mthca - let me know if
> you plan to work on that, otherwise I'll look into it tomorrow.
> And its probably something we want fixed for 2.6.14, right?
> Let me know.
> 
> With regard to the test code that you posted - I also have some small
> comments. If you plan to use it in the future, you can stick it
> in svn somewhere and I'll send patches.
> 
> ---
> 
> Fix posting of the first work request for memfree hardware.
> Simplify code for tavor mode hardware.
> 
> Signed-off-by: Michael S. Tsirkin <mst at mellanox.co.il>
> 
> Index: userspace/libmthca/src/qp.c
> ===================================================================
> --- userspace.orig/libmthca/src/qp.c 2005-09-13 17:17:58.000000000 +0300
> +++ userspace/libmthca/src/qp.c 2005-09-13 17:26:23.000000000 +0300
> @@ -259,15 +259,13 @@ int mthca_tavor_post_send(struct ibv_qp
> goto out;
> }
> 
> - if (prev_wqe) {
> - ((struct mthca_next_seg *) prev_wqe)->nda_op =
> - htonl(((ind << qp->sq.wqe_shift) +
> - qp->send_wqe_offset) |
> - mthca_opcode[wr->opcode]);
> + ((struct mthca_next_seg *) prev_wqe)->nda_op =
> + htonl(((ind << qp->sq.wqe_shift) +
> + qp->send_wqe_offset) |
> + mthca_opcode[wr->opcode]);
> 
> - ((struct mthca_next_seg *) prev_wqe)->ee_nds =
> - htonl((size0 ? 0 : MTHCA_NEXT_DBD) | size);
> - }
> + ((struct mthca_next_seg *) prev_wqe)->ee_nds =
> + htonl((size0 ? 0 : MTHCA_NEXT_DBD) | size);
> 
> if (!size0) {
> size0 = size;
> @@ -353,12 +351,10 @@ int mthca_tavor_post_recv(struct ibv_qp
> 
> qp->wrid[ind] = wr->wr_id;
> 
> - if (prev_wqe) {
> - ((struct mthca_next_seg *) prev_wqe)->nda_op =
> - htonl((ind << qp->rq.wqe_shift) | 1);
> - ((struct mthca_next_seg *) prev_wqe)->ee_nds =
> - htonl(MTHCA_NEXT_DBD | size);
> - }
> + ((struct mthca_next_seg *) prev_wqe)->nda_op =
> + htonl((ind << qp->rq.wqe_shift) | 1);
> + ((struct mthca_next_seg *) prev_wqe)->ee_nds =
> + htonl(MTHCA_NEXT_DBD | size);
> 
> if (!size0)
> size0 = size;
> @@ -562,15 +558,13 @@ int mthca_arbel_post_send(struct ibv_qp
> goto out;
> }
> 
> - if (prev_wqe) {
> - ((struct mthca_next_seg *) prev_wqe)->nda_op =
> - htonl(((ind << qp->sq.wqe_shift) +
> - qp->send_wqe_offset) |
> - mthca_opcode[wr->opcode]);
> - mb();
> - ((struct mthca_next_seg *) prev_wqe)->ee_nds =
> - htonl(MTHCA_NEXT_DBD | size);
> - }
> + ((struct mthca_next_seg *) prev_wqe)->nda_op =
> + htonl(((ind << qp->sq.wqe_shift) +
> + qp->send_wqe_offset) |
> + mthca_opcode[wr->opcode]);
> + mb();
> + ((struct mthca_next_seg *) prev_wqe)->ee_nds =
> + htonl(MTHCA_NEXT_DBD | size);
> 
> if (!size0) {
> size0 = size;
> @@ -767,6 +761,8 @@ int mthca_alloc_qp_buf(struct ibv_pd *pd
> }
> }
> 
> + qp->sq.last = get_send_wqe(qp, qp->sq.max - 1);
> + qp->rq.last = get_recv_wqe(qp, qp->sq.max - 1);
> return 0;
> }
> 
> Index: userspace/libmthca/src/srq.c
> ===================================================================
> --- userspace.orig/libmthca/src/srq.c 2005-09-13 17:25:41.000000000 +0300
> +++ userspace/libmthca/src/srq.c 2005-09-13 17:25:51.000000000 +0300
> @@ -142,13 +142,11 @@ int mthca_tavor_post_srq_recv(struct ibv
> ((struct mthca_data_seg *) wqe)->addr = 0;
> }
> 
> - if (prev_wqe) {
> - ((struct mthca_next_seg *) prev_wqe)->nda_op =
> - htonl((ind << srq->wqe_shift) | 1);
> - mb();
> - ((struct mthca_next_seg *) prev_wqe)->ee_nds =
> - htonl(MTHCA_NEXT_DBD);
> - }
> + ((struct mthca_next_seg *) prev_wqe)->nda_op =
> + htonl((ind << srq->wqe_shift) | 1);
> + mb();
> + ((struct mthca_next_seg *) prev_wqe)->ee_nds =
> + htonl(MTHCA_NEXT_DBD);
> 
> srq->wrid[ind] = wr->wr_id;
> srq->first_free = next_ind;
> @@ -294,6 +292,7 @@ int mthca_alloc_srq_buf(struct ibv_pd *p
> 
> srq->first_free = 0;
> srq->last_free = srq->max - 1;
> + srq->last = get_wqe(srq, srq->max - 1);
> 
> return 0;
> }
> Index: userspace/libmthca/src/verbs.c
> ===================================================================
> --- userspace.orig/libmthca/src/verbs.c 2005-08-23 14:03:12.000000000+0300
> +++ userspace/libmthca/src/verbs.c 2005-09-13 17:25:14.000000000 +0300
> @@ -306,7 +306,6 @@ struct ibv_srq *mthca_create_srq(struct
> 
> srq->max = align_queue_size(pd->context, attr->attr.max_wr, 1);
> srq->max_gs = attr->attr.max_sge;
> - srq->last = NULL;
> srq->counter = 0;
> 
> if (mthca_alloc_srq_buf(pd, &attr->attr, srq))
> @@ -413,14 +412,12 @@ struct ibv_qp *mthca_create_qp(struct ib
> qp->sq.last_comp = qp->sq.max - 1;
> qp->sq.head = 0;
> qp->sq.tail = 0;
> - qp->sq.last = NULL;
> 
> qp->rq.max = align_queue_size(pd->context, attr->cap.max_recv_wr, 0);
> qp->rq.next_ind = 0;
> qp->rq.last_comp = qp->rq.max - 1;
> qp->rq.head = 0;
> qp->rq.tail = 0;
> - qp->rq.last = NULL;
> 
> if (mthca_alloc_qp_buf(pd, &attr->cap, qp))
> goto err;
> 
> 
> --
> MST
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/general/attachments/20050913/6aed674a/attachment.html>