[ofa-general] [RFC 1/1] libmthca: CQ/DMA race on Altix

[akepner at sgi.com]

The libmthca-specific changes for this RFC follow.

 mthca-abi.h |    9 ++++++++-
 verbs.c     |   22 +++++++++++++---------
 2 files changed, 21 insertions(+), 10 deletions(-)

-- 

diff -rup ofa_1_2_user-20070623-0200.orig/src/userspace/libmthca/src/mthca-abi.h ofa_1_2_user-20070623-0200/src/userspace/libmthca/src/mthca-abi.h
--- ofa_1_2_user-20070623-0200.orig/src/userspace/libmthca/src/mthca-abi.h	2007-06-23 02:00:34.000000000 -0700
+++ ofa_1_2_user-20070623-0200/src/userspace/libmthca/src/mthca-abi.h	2007-07-15 12:18:54.505352246 -0700
@@ -36,7 +36,7 @@
 
 #include <infiniband/kern-abi.h>
 
-#define MTHCA_UVERBS_ABI_VERSION	1
+#define MTHCA_UVERBS_ABI_VERSION	2
 
 struct mthca_alloc_ucontext_resp {
 	struct ibv_get_context_resp	ibv_resp;
@@ -50,6 +50,13 @@ struct mthca_alloc_pd_resp {
 	__u32				reserved;
 };
 
+struct mthca_reg_mr {
+	struct ibv_reg_mr		ibv_cmd;
+	__u32				mr_attrs;
+#define MTHCA_MR_COHERENT		0x1
+	__u32				reserved;
+};
+
 struct mthca_create_cq {
 	struct ibv_create_cq		ibv_cmd;
 	__u32				lkey;
diff -rup ofa_1_2_user-20070623-0200.orig/src/userspace/libmthca/src/verbs.c ofa_1_2_user-20070623-0200/src/userspace/libmthca/src/verbs.c
--- ofa_1_2_user-20070623-0200.orig/src/userspace/libmthca/src/verbs.c	2007-06-23 02:00:34.000000000 -0700
+++ ofa_1_2_user-20070623-0200/src/userspace/libmthca/src/verbs.c	2007-07-15 13:26:24.371410587 -0700
@@ -117,26 +117,30 @@ int mthca_free_pd(struct ibv_pd *pd)
 
 static struct ibv_mr *__mthca_reg_mr(struct ibv_pd *pd, void *addr,
 				     size_t length, uint64_t hca_va,
-				     enum ibv_access_flags access)
+				     enum ibv_access_flags access, 
+				     int coherent)
 {
 	struct ibv_mr *mr;
-	struct ibv_reg_mr cmd;
+	struct mthca_reg_mr cmd;
 	int ret;
 
 	mr = malloc(sizeof *mr);
 	if (!mr)
 		return NULL;
 
+	cmd.mr_attrs |= (__u32) coherent ? MTHCA_MR_COHERENT: 0;
+
 #ifdef IBV_CMD_REG_MR_HAS_RESP_PARAMS
 	{
 		struct ibv_reg_mr_resp resp;
 
 		ret = ibv_cmd_reg_mr(pd, addr, length, hca_va, access, mr,
-				     &cmd, sizeof cmd, &resp, sizeof resp);
+				     &cmd.ibv_cmd, sizeof cmd, &resp, 
+				     sizeof resp);
 	}
 #else
 	ret = ibv_cmd_reg_mr(pd, addr, length, hca_va, access, mr,
-			     &cmd, sizeof cmd);
+			     &cmd.ibv_cmd, sizeof cmd);
 #endif
 	if (ret) {
 		free(mr);
@@ -149,7 +153,7 @@ static struct ibv_mr *__mthca_reg_mr(str
 struct ibv_mr *mthca_reg_mr(struct ibv_pd *pd, void *addr,
 			    size_t length, enum ibv_access_flags access)
 {
-	return __mthca_reg_mr(pd, addr, length, (uintptr_t) addr, access);
+	return __mthca_reg_mr(pd, addr, length, (uintptr_t) addr, access, 0);
 }
 
 int mthca_dereg_mr(struct ibv_mr *mr)
@@ -202,7 +206,7 @@ struct ibv_cq *mthca_create_cq(struct ib
 
 	cq->mr = __mthca_reg_mr(to_mctx(context)->pd, cq->buf.buf,
 				cqe * MTHCA_CQ_ENTRY_SIZE,
-				0, IBV_ACCESS_LOCAL_WRITE);
+				0, IBV_ACCESS_LOCAL_WRITE, 1);
 	if (!cq->mr)
 		goto err_buf;
 
@@ -294,7 +298,7 @@ int mthca_resize_cq(struct ibv_cq *ibcq,
 
 	mr = __mthca_reg_mr(to_mctx(ibcq->context)->pd, buf.buf,
 			    cqe * MTHCA_CQ_ENTRY_SIZE,
-			    0, IBV_ACCESS_LOCAL_WRITE);
+			    0, IBV_ACCESS_LOCAL_WRITE, 1);
 	if (!mr) {
 		mthca_free_buf(&buf);
 		ret = ENOMEM;
@@ -402,7 +406,7 @@ struct ibv_srq *mthca_create_srq(struct 
 	if (mthca_alloc_srq_buf(pd, &attr->attr, srq))
 		goto err;
 
-	srq->mr = __mthca_reg_mr(pd, srq->buf.buf, srq->buf_size, 0, 0);
+	srq->mr = __mthca_reg_mr(pd, srq->buf.buf, srq->buf_size, 0, 0, 0);
 	if (!srq->mr)
 		goto err_free;
 
@@ -520,7 +524,7 @@ struct ibv_qp *mthca_create_qp(struct ib
 	    pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE))
 		goto err_free;
 
-	qp->mr = __mthca_reg_mr(pd, qp->buf.buf, qp->buf_size, 0, 0);
+	qp->mr = __mthca_reg_mr(pd, qp->buf.buf, qp->buf_size, 0, 0, 0);
 	if (!qp->mr)
 		goto err_free;
 
-- 
Arthur