[ofw] [Patch 33/62] Reference implementation of NDv2

Fab Tillier ftillier at microsoft.com
Wed Feb 20 18:09:04 PST 2013


Migrate HCA driver to ND_MAPPINGS and ND IOCTLs.

This is a pretty large patch, though entirely contained within the HCA driver:
- Use Windows types (UINT64) rather than abstracted (_u64, uint64_t, etc.)
- Trim unused parameters from req/resp structures
- Don't duplicate parameters that come via the IBAL structures in the req/resp structures
- use ND_MAPPINGS for all resource mappings
- output handles are not optional for kernel callers, don't conditionally set them as this leaks resources

Signed-off-by: Fab Tillier <ftillier at microsoft.com>

diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\inc\mx_abi.h .\hw\mlx4\inc\mx_abi.h
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\inc\mx_abi.h	Tue Aug 07 12:37:23 2012
+++ .\hw\mlx4\inc\mx_abi.h	Thu May 31 12:35:10 2012
@@ -35,7 +35,7 @@
 #ifndef MX_ABI_H
 #define MX_ABI_H
 
-#include <complib/cl_types_osd.h>
+#include <ndioctl.h>
 #include "user.h"
 
 // {697925A7-84F0-4ED9-A9E8-76A941E72EB1}
@@ -49,19 +49,29 @@ DEFINE_GUID(GUID_MLX4_DRIVER, 
  * that they pack the same way on 32-bit and 64-bit architectures (to
  * avoid incompatibility between 32-bit userspace and 64-bit kernels).
  * Specifically:
- *  - Do not use pointer types -- pass pointers in uint64_t instead.
+ *  - Do not use pointer types -- pass pointers in UINT64 instead.
  *  - Make sure that any structure larger than 4 bytes is padded to a
  *    multiple of 8 bytes.  Otherwise the structure size will be
  *    different between 32-bit and 64-bit architectures.
  */
 
+enum ibv_get_context_mappings {
+    ibv_get_context_uar,
+    ibv_get_context_bf,
+    ibv_get_context_mapping_max
+};
+
+struct ibv_get_context_req {
+
+    ND_MAPPING mappings[ibv_get_context_mapping_max];
+};
+
 struct ibv_get_context_resp {
 
-	// mmap UAR
-	uint64_t uar_addr;
+	// mmap UAR and BF
+    ND_MAPPING_RESULT mapping_results[ibv_get_context_mapping_max];
 	
 	// mmap Blue Flame
-	uint64_t bf_page;
 	int bf_buf_size;
 	int bf_offset;
 	
@@ -71,110 +81,106 @@ struct ibv_get_context_resp {
 	int max_cqe;
 
 	// general parameters
-	uint32_t cqe_size;
-	uint32_t vend_id;
-	uint16_t dev_id;
-	uint16_t bf_reg_size;
-	uint16_t bf_regs_per_page;
-	uint16_t reserved1;
+	UINT32 cqe_size;
+	UINT32 vend_id;
+	UINT16 dev_id;
+	UINT16 bf_reg_size;
+	UINT16 bf_regs_per_page;
+	UINT16 reserved1;
 
 	// ibv_cmd_get_context result 
-	uint32_t qp_tab_size;
+	UINT32 qp_tab_size;
 
-	uint32_t reserved2;
+	UINT32 reserved2;
 };
 
 struct ibv_alloc_pd_resp {
-	uint64_t pd_handle;
-	uint32_t pdn;
-	uint32_t reserved;
+	UINT64 pd_handle;
+	UINT32 pdn;
+	UINT32 reserved;
 };
 
 struct ibv_reg_mr {
-	uint64_t start;
-	uint64_t length;
-	uint64_t hca_va;
-	uint32_t access_flags;
-	uint32_t pdn;
-	uint64_t pd_handle;
+	UINT64 start;
+	UINT64 length;
+	UINT64 hca_va;
+	UINT32 access_flags;
+	UINT32 pdn;
+	UINT64 pd_handle;
 };
 
 struct ibv_reg_mr_resp {
-	uint64_t mr_handle;
-	uint32_t lkey;
-	uint32_t rkey;
+	UINT64 mr_handle;
+	UINT32 lkey;
+	UINT32 rkey;
 };
 
 
+enum mlx4_ib_create_cq_mapping {
+    mlx4_ib_create_cq_buf,
+    mlx4_ib_create_cq_db,
+    mlx4_ib_create_cq_arm_sn,   // Windows specific
+    mlx4_ib_create_cq_mapping_max
+};
+
 struct ibv_create_cq {
-	// struct ib_uverbs_create_cq
-	__declspec(align(8)) uint32_t cqe;
-	uint32_t reserved;
-	struct mlx4_ib_create_cq;
+    ND_MAPPING mappings[mlx4_ib_create_cq_mapping_max];
 };
 
 struct ibv_create_cq_resp {
-	// struct ib_uverbs_create_cq_resp
-	uint64_t cq_handle;
-	uint32_t cqe;
-	struct mlx4_ib_create_cq_resp;
+    ND_MAPPING_RESULT mapping_results[mlx4_ib_create_cq_mapping_max];
+    UINT32  cqn;
+    UINT32  cqe;
+};
+
+enum mlx4_ib_create_srq_mappings {
+    mlx4_ib_create_srq_buf,
+    mlx4_ib_create_srq_db,
+    mlx4_ib_create_srq_mappings_max
 };
 
 struct ibv_create_srq {
-	// struct ib_uverbs_create_srq
-	uint64_t pd_handle;
-	uint32_t max_wr;
-	uint32_t max_sge;
-	uint32_t srq_limit;
-	uint32_t reserved;
-	struct mlx4_ib_create_srq;
+    ND_MAPPING mappings[mlx4_ib_create_srq_mappings_max];
 };
 
 struct ibv_create_srq_resp {
-	// struct ib_uverbs_create_srq_resp
-	uint64_t srq_handle;
-	uint32_t max_wr;
-	uint32_t max_sge;
-	struct mlx4_ib_create_srq_resp;
+    ND_MAPPING_RESULT mapping_results[mlx4_ib_create_srq_mappings_max];
+};
+
+enum mlx4_ib_create_qp_mappings {
+    mlx4_ib_create_qp_buf,
+    mlx4_ib_create_qp_db,
+    mlx4_ib_create_qp_mappings_max
 };
 
 struct ibv_create_qp {
-	// struct ib_uverbs_create_qp
-	uint64_t pd_handle;
-	uint64_t send_cq_handle;
-	uint64_t recv_cq_handle;
-	uint64_t srq_handle;
-	uint32_t max_send_wr;
-	uint32_t max_recv_wr;
-	uint32_t max_send_sge;
-	uint32_t max_recv_sge;
-	uint32_t max_inline_data;
-	uint8_t  sq_sig_all;
-	uint8_t  qp_type;
-	uint8_t  is_srq;
-	uint8_t  reserved0;
-	struct mlx4_ib_create_qp;
+    ND_MAPPING mappings[mlx4_ib_create_qp_mappings_max];
+    UINT8	log_sq_bb_count;
+    UINT8	log_sq_stride;
+    UINT8	sq_no_prefetch;
+    UINT8	reserved;
 };
 
 struct ibv_create_qp_resp {
+    ND_MAPPING_RESULT mapping_results[mlx4_ib_create_qp_mappings_max];
 	// struct ib_uverbs_create_qp_resp
-	uint64_t qp_handle;
-	uint32_t qpn;
-	uint32_t max_send_wr;
-	uint32_t max_recv_wr;
-	uint32_t max_send_sge;
-	uint32_t max_recv_sge;
-	uint32_t max_inline_data;
+	UINT64 qp_handle;
+	UINT32 qpn;
+	UINT32 max_send_wr;
+	UINT32 max_recv_wr;
+	UINT32 max_send_sge;
+	UINT32 max_recv_sge;
+	UINT32 max_inline_data;
 };
 
 struct ibv_modify_qp_resp {
 	enum ibv_qp_attr_mask attr_mask;
-	uint8_t qp_state;
-	uint8_t reserved[3];
+	UINT8 qp_state;
+	UINT8 reserved[3];
 };
 
 struct ibv_create_ah_resp {
-	uint64_t start;
+	UINT64 start;
 };
 
 #pragma warning( default  : 4201)
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\inc\user.h .\hw\mlx4\inc\user.h
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\inc\user.h	Thu Mar 29 00:15:30 2012
+++ .\hw\mlx4\inc\user.h	Thu May 31 12:35:11 2012
@@ -36,60 +36,27 @@
 /*
  * Increment this value if any changes that break userspace ABI
  * compatibility are made.
+ *
+ * TODO: !!!THIS IS NEVER USED!!!
  */
-#define MLX4_IB_UVERBS_ABI_VERSION	5
+#define MLX4_IB_UVERBS_ABI_VERSION	6
 
 /*
  * Make sure that all structs defined in this file remain laid out so
  * that they pack the same way on 32-bit and 64-bit architectures (to
  * avoid incompatibility between 32-bit userspace and 64-bit kernels).
- * In particular do not use pointer types -- pass pointers in __u64
+ * In particular do not use pointer types -- pass pointers in UINT64
  * instead.
  */
 
-struct mlx4_ib_alloc_ucontext_resp {
-	__u32	qp_tab_size;
-	__u16	bf_reg_size;
-	__u16	bf_regs_per_page;
-	__u32	cqe_size;
-};
-
-struct mlx4_ib_alloc_pd_resp {
-	__u32	pdn;
-	__u32	reserved;
-};
-
-struct mlx4_ib_create_cq {
-	__u64	buf_addr;
-	__u64	db_addr;
-	__u64	arm_sn_addr;	// Windows specific
-};
-
-struct mlx4_ib_create_cq_resp {
-	__u32	cqn;
+enum mlx4_ib_resize_cq_mapping {
+    mlx4_ib_resize_cq_map,
+    mlx4_ib_resize_cq_unmap,
+    mlx4_ib_resize_cq_mapping_max
 };
 
 struct mlx4_ib_resize_cq {
-	__u64	buf_addr;
-};
-
-struct mlx4_ib_create_srq {
-	__u64	buf_addr;
-	__u64	db_addr;
-};
-
-struct mlx4_ib_create_srq_resp {
-	__u32	srqn;
-	__u32	reserved;
-};
-
-struct mlx4_ib_create_qp {
-	__u64	buf_addr;
-	__u64	db_addr;
-	__u8	log_sq_bb_count;
-	__u8	log_sq_stride;
-	__u8	sq_no_prefetch;
-	__u8	reserved[5];
+    ND_MAPPING mappings[mlx4_ib_resize_cq_mapping_max];
 };
 
 #endif /* MLX4_IB_USER_H */
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\drv\precomp.h .\hw\mlx4\kernel\bus\drv\precomp.h
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\drv\precomp.h	Thu Mar 29 00:15:28 2012
+++ .\hw\mlx4\kernel\bus\drv\precomp.h	Thu Jul 26 15:31:14 2012
@@ -2,6 +2,7 @@
 
 #include <ntddk.h>
 #include <wdf.h>
+#include <ndioctl.h>
 #define NTSTRSAFE_LIB
 #include <ntstrsafe.h>
 #include <initguid.h> // required for GUID definitions
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\ib\cq.c .\hw\mlx4\kernel\bus\ib\cq.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\ib\cq.c	Fri Aug 03 12:14:07 2012
+++ .\hw\mlx4\kernel\bus\ib\cq.c	Sat Jun 23 14:15:09 2012
@@ -140,7 +140,7 @@ struct ib_cq *mlx4_ib_create_cq_internal
 	struct mlx4_ib_dev *dev = to_mdev(ibdev);
 	struct mlx4_ib_cq *cq;
 	struct mlx4_uar *uar;
-	int buf_size;
+	ULONG buf_size;
 	int err;
 	int print_cqe = 0;
 
@@ -180,8 +180,10 @@ struct ib_cq *mlx4_ib_create_cq_internal
 	spin_lock_init(&cq->lock);
 	cq->print_cqe = print_cqe;
 
-	if (context && udata) {
-		struct mlx4_ib_create_cq ucmd;
+	if (context != NULL) {
+		struct ibv_create_cq ucmd;
+
+        NT_ASSERT(udata != NULL);
 
 		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
 			MLX4_PRINT(TRACE_LEVEL_WARNING, MLX4_DBG_DRV,
@@ -192,7 +194,29 @@ struct ib_cq *mlx4_ib_create_cq_internal
 			goto err_cq;
 		}
 
-		cq->umem = ib_umem_get(context, ucmd.buf_addr, buf_size,
+        if( NdValidateMemoryMapping( &ucmd.mappings[mlx4_ib_create_cq_buf],
+                                     NdModifyAccess, buf_size ) != STATUS_SUCCESS )
+        {
+            err = -EFAULT;
+            goto err_cq;
+        }
+
+        if( NdValidateCoallescedMapping( &ucmd.mappings[mlx4_ib_create_cq_db],
+                                         NdWriteAccess, sizeof(UINT64) ) != STATUS_SUCCESS )
+        {
+            err = -EFAULT;
+            goto err_cq;
+        }
+
+        if( NdValidateMemoryMapping( &ucmd.mappings[mlx4_ib_create_cq_arm_sn],
+                                     NdModifyAccess, sizeof(int) ) != STATUS_SUCCESS )
+        {
+            err = -EFAULT;
+            goto err_cq;
+        }
+
+		cq->umem = ib_umem_get(context, ucmd.mappings[mlx4_ib_create_cq_buf].MapMemory.Address,
+                               ucmd.mappings[mlx4_ib_create_cq_buf].MapMemory.CbLength,
 				               (ib_access_flags)(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_NO_SECURE));
 		if (IS_ERR(cq->umem)) {
 			err = PTR_ERR(cq->umem);
@@ -222,7 +246,7 @@ struct ib_cq *mlx4_ib_create_cq_internal
 			goto err_mtt;
 		}
 
-		err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr,
+		err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.mappings[mlx4_ib_create_cq_db].MapMemory.Address,
 					  &cq->db);
 		if (err) {
 			MLX4_PRINT(TRACE_LEVEL_ERROR, MLX4_DBG_DRV,
@@ -235,7 +259,8 @@ struct ib_cq *mlx4_ib_create_cq_internal
 		// add mapping to user's arm_sn variable
 		// we have no way pass the completion event to provider library
 		// so we'll increment user's arm_sn in kernel
-		err = ib_umem_map( context, ucmd.arm_sn_addr, sizeof(int),
+		err = ib_umem_map( context, ucmd.mappings[mlx4_ib_create_cq_arm_sn].MapMemory.Address, 
+            ucmd.mappings[mlx4_ib_create_cq_arm_sn].MapMemory.CbLength, 
 			IB_ACCESS_LOCAL_WRITE, &cq->mcq.mdl, &cq->mcq.p_u_arm_sn );
 		if (err) {
 			MLX4_PRINT(TRACE_LEVEL_ERROR, MLX4_DBG_DRV,
@@ -307,16 +332,19 @@ struct ib_cq *mlx4_ib_create_cq_internal
 	cq->mcq.comp  = mlx4_ib_cq_comp;
 	cq->mcq.event = mlx4_ib_cq_event;
 
-	if (context && udata) 
-		if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {
+	if (context != NULL) {
+        struct ibv_create_cq_resp resp;
+        RtlZeroMemory(&resp, sizeof(resp));
+        resp.cqn = cq->mcq.cqn;
+        resp.cqe = cq->ibcq.cqe;
+		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
 			MLX4_PRINT(TRACE_LEVEL_WARNING, MLX4_DBG_DRV,
 				( "%s: ib_copy_to_udata() failed, trying to copy more mem to user than possible.\n",
 				 	ibdev->name));	
-
 			err = -EFAULT;
 			goto err_dbmap;
 		}
-
+    }
 	return &cq->ibcq;
 
 err_dbmap:
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\ib\mlx4_ib.h .\hw\mlx4\kernel\bus\ib\mlx4_ib.h
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\ib\mlx4_ib.h	Tue Aug 07 12:46:47 2012
+++ .\hw\mlx4\kernel\bus\ib\mlx4_ib.h	Thu Jul 26 15:31:14 2012
@@ -35,6 +35,7 @@
 
 #include "l2w.h"
 
+#include <ndioctl.h>
 #include "ib_verbs.h"
 // TODO: do we need this file
 //#include "ib_umem.h"
@@ -77,6 +78,13 @@ struct mlx4_ib_db {
 	int			order;
 };
 
+struct mlx4_ib_alloc_ucontext_resp {
+    UINT32	qp_tab_size;
+    UINT16	bf_reg_size;
+    UINT16	bf_regs_per_page;
+    UINT32	cqe_size;
+};
+
 struct mlx4_ib_ucontext {
 	struct ib_ucontext	ibucontext;
 	struct mlx4_uar		uar;
@@ -167,7 +175,7 @@ struct mlx4_ib_qp {
 
 	struct ib_umem	       *umem;
 	struct mlx4_mtt		mtt;
-	int			buf_size;
+	ULONG			buf_size;
 	struct mutex		mutex;
 	u32			flags;
 	struct list_head	xrc_reg_list;
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\ib\qp.c .\hw\mlx4\kernel\bus\ib\qp.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\ib\qp.c	Tue Aug 07 12:46:47 2012
+++ .\hw\mlx4\kernel\bus\ib\qp.c	Thu Jul 26 15:31:14 2012
@@ -34,7 +34,7 @@
 #include "ib_cache.h"
 #include "ib_pack.h"
 #include "qp.h"
-#include "user.h"
+#include "mx_abi.h"
 #include "mlx4_debug.h"
 #include "mmintrin.h"
 
@@ -490,7 +490,7 @@ static int set_kernel_sq_size(struct mlx
 
 static int set_user_sq_size(struct mlx4_ib_dev *dev,
 			    struct mlx4_ib_qp *qp,
-			    struct mlx4_ib_create_qp *ucmd)
+			    struct ibv_create_qp *ucmd)
 {
 	/* Sanity check SQ size before proceeding */
 	if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes	 ||
@@ -557,7 +557,9 @@ static int create_qp_common(struct mlx4_
 	}
 
 	if (pd->p_uctx) {
-		struct mlx4_ib_create_qp ucmd;
+		struct ibv_create_qp ucmd;
+
+        NT_ASSERT( udata != NULL);
 
 		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
 			MLX4_PRINT(TRACE_LEVEL_WARNING, MLX4_DBG_DRV,
@@ -579,8 +581,22 @@ static int create_qp_common(struct mlx4_
 			goto err;
 		}
 		
-		qp->umem = ib_umem_get(pd->p_uctx, ucmd.buf_addr,
-				       qp->buf_size, IB_ACCESS_NO_SECURE);
+        if( NdValidateMemoryMapping( &ucmd.mappings[mlx4_ib_create_qp_buf],
+                                     NdModifyAccess, qp->buf_size ) != STATUS_SUCCESS )
+        {
+            err = -EFAULT;
+            goto err;
+        }
+
+        if( NdValidateCoallescedMapping( &ucmd.mappings[mlx4_ib_create_qp_db],
+                                         NdWriteAccess, sizeof(UINT32) ) != STATUS_SUCCESS )
+        {
+            err = -EFAULT;
+            goto err;
+        }
+
+		qp->umem = ib_umem_get(pd->p_uctx, ucmd.mappings[mlx4_ib_create_qp_buf].MapMemory.Address,
+				       ucmd.mappings[mlx4_ib_create_qp_buf].MapMemory.CbLength, IB_ACCESS_NO_SECURE);
 		if (IS_ERR(qp->umem)) {
 			err = PTR_ERR(qp->umem);
 			MLX4_PRINT(TRACE_LEVEL_WARNING, MLX4_DBG_DRV,
@@ -611,7 +627,7 @@ static int create_qp_common(struct mlx4_
 
 		if (!init_attr->srq) {
 			err = mlx4_ib_db_map_user(to_mucontext(pd->p_uctx),
-						  ucmd.db_addr, &qp->db);
+						  ucmd.mappings[mlx4_ib_create_qp_db].MapMemory.Address, &qp->db);
 			if (err){
 				MLX4_PRINT(TRACE_LEVEL_WARNING, MLX4_DBG_DRV,
 					( "%s: mlx4_ib_db_map_user() failed with error:%d.\n", 
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\ib\srq.c .\hw\mlx4\kernel\bus\ib\srq.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\ib\srq.c	Thu Aug 02 13:08:44 2012
+++ .\hw\mlx4\kernel\bus\ib\srq.c	Thu Jul 26 15:31:14 2012
@@ -33,7 +33,7 @@
 #include "mlx4_ib.h"
 #include "qp.h"
 #include "srq.h"
-#include "user.h"
+#include "mx_abi.h"
 
 #include "mlx4_debug.h"
 
@@ -85,8 +85,8 @@ struct ib_srq *mlx4_ib_create_srq(struct
 	struct mlx4_ib_dev *dev = to_mdev(pd->device);
 	struct mlx4_ib_srq *srq;
 	struct mlx4_wqe_srq_next_seg *next;
-	int desc_size;
-	int buf_size;
+	ULONG desc_size;
+	ULONG buf_size;
 	int err;
 	int i;
 	u32 cqn = 0;
@@ -132,20 +132,32 @@ struct ib_srq *mlx4_ib_create_srq(struct
 
 	buf_size = srq->msrq.max * desc_size;
 
-	if (pd->p_uctx) {
-		struct mlx4_ib_create_srq ucmd;
+	if (pd->p_uctx != NULL) {
+		struct ibv_create_srq ucmd;
+
+        NT_ASSERT(udata != NULL);
 
 		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
-			MLX4_PRINT(TRACE_LEVEL_WARNING, MLX4_DBG_DRV,
-				("%s: ib_copy_from_udata() failed, trying to copy more mem from user than availiable.\n"
-					, pd->device->dma_device->pdev->name));	
+			err = -EFAULT;
+			goto err_srq;
+		}
 
+        if( NdValidateMemoryMapping( &ucmd.mappings[mlx4_ib_create_srq_buf],
+                                     NdModifyAccess, buf_size ) != STATUS_SUCCESS )
+        {
 			err = -EFAULT;
 			goto err_srq;
 		}
 
-		srq->umem = ib_umem_get(pd->p_uctx, ucmd.buf_addr,
-					buf_size, IB_ACCESS_NO_SECURE);
+        if( NdValidateCoallescedMapping( &ucmd.mappings[mlx4_ib_create_srq_db],
+                                         NdWriteAccess, sizeof(UINT32) ) != STATUS_SUCCESS )
+        {
+            err = -EFAULT;
+            goto err_srq;
+        }
+
+		srq->umem = ib_umem_get(pd->p_uctx, ucmd.mappings[mlx4_ib_create_srq_buf].MapMemory.Address,
+					ucmd.mappings[mlx4_ib_create_srq_buf].MapMemory.CbLength, IB_ACCESS_NO_SECURE);
 		if (IS_ERR(srq->umem)) {
 			err = PTR_ERR(srq->umem);
 			MLX4_PRINT(TRACE_LEVEL_WARNING, MLX4_DBG_DRV,
@@ -174,7 +186,7 @@ struct ib_srq *mlx4_ib_create_srq(struct
 		}
 
 		err = mlx4_ib_db_map_user(to_mucontext(pd->p_uctx),
-					  ucmd.db_addr, &srq->db);
+					  ucmd.mappings[mlx4_ib_create_srq_db].MapMemory.Address, &srq->db);
 		if (err){
 			MLX4_PRINT(TRACE_LEVEL_WARNING, MLX4_DBG_DRV,
 				( "%s: mlx4_ib_db_map_user() failed with error:%d.\n", 
@@ -257,16 +269,18 @@ struct ib_srq *mlx4_ib_create_srq(struct
 
 	srq->msrq.event = mlx4_ib_srq_event;
 
-	if (pd->p_uctx) {
-		if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) {
+	if (pd->p_uctx != NULL) {
+        struct ibv_create_srq_resp resp;
+        RtlZeroMemory(&resp, sizeof(resp));
+		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
 			err = -EFAULT;
 			goto err_wrid;
 		}
-	} 
 #ifdef XRC_SUPPORT	
-	else
+	} else {
 		srq->ibsrq.xrc_srq_num = srq->msrq.srqn;
 #endif	
+    }
 
 	init_attr->attr.max_wr = srq->msrq.max - 1;
 
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\cq.c .\hw\mlx4\kernel\hca\cq.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\cq.c	Fri Aug 03 12:58:34 2012
+++ .\hw\mlx4\kernel\hca\cq.c	Thu May 31 12:35:11 2012
@@ -95,10 +95,7 @@ mlnx_create_cq (
 		goto err_create_cq;
 	}
 
-	// return the result
-	*p_size = p_ib_cq->cqe;
-
-	if (ph_cq) *ph_cq = (ib_cq_handle_t)p_ib_cq;
+	*ph_cq = (ib_cq_handle_t)p_ib_cq;
 
 	status = IB_SUCCESS;
 	
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\hverbs.c .\hw\mlx4\kernel\hca\hverbs.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\hverbs.c	Thu Aug 02 18:02:17 2012
+++ .\hw\mlx4\kernel\hca\hverbs.c	Fri Aug 03 14:44:15 2012
@@ -213,11 +213,12 @@ struct ib_cq *ibv_create_cq(struct ib_de
 		p_req = (struct ibv_create_cq*)(ULONG_PTR)p_umv_buf->p_inout_buf;
 		p_resp = (struct ibv_create_cq_resp*)(ULONG_PTR)
 			p_umv_buf->p_inout_buf;
-		INIT_UDATA(&udata, &p_req->buf_addr, &p_resp->cqn, 
-			sizeof(struct mlx4_ib_create_cq), sizeof(struct mlx4_ib_create_cq_resp));
+		INIT_UDATA(&udata, p_req, p_resp, 
+			sizeof(struct ibv_create_cq), sizeof(struct ibv_create_cq_resp));
 	}
-	else 
+	else {
 		p_udata = NULL;
+    }
 
 	// create cq
 	cq = p_ibdev->create_cq(p_ibdev, cqe, 0, p_uctx, p_udata);
@@ -241,7 +242,6 @@ struct ib_cq *ibv_create_cq(struct ib_de
 
 	// fill results
 	if (p_umv_buf) {
-		p_resp->cq_handle = (u64)(ULONG_PTR)cq;
 		p_resp->cqe = cq->cqe;
 		p_umv_buf->output_size = sizeof(struct ibv_create_cq_resp);
 	}
@@ -271,8 +271,8 @@ struct ib_cq *ibv_create_cq_ex(struct ib
 		p_req = (struct ibv_create_cq*)(ULONG_PTR)p_umv_buf->p_inout_buf;
 		p_resp = (struct ibv_create_cq_resp*)(ULONG_PTR)
 			p_umv_buf->p_inout_buf;
-		INIT_UDATA(&udata, &p_req->buf_addr, &p_resp->cqn, 
-			sizeof(struct mlx4_ib_create_cq), sizeof(struct mlx4_ib_create_cq_resp));
+		INIT_UDATA(&udata, p_req, p_resp, 
+			sizeof(struct ibv_create_cq), sizeof(struct ibv_create_cq_resp));
 	}
 	else 
 		p_udata = NULL;
@@ -299,7 +299,6 @@ struct ib_cq *ibv_create_cq_ex(struct ib
 
 	// fill results
 	if (p_umv_buf) {
-		p_resp->cq_handle = (u64)(ULONG_PTR)cq;
 		p_resp->cqe = cq->cqe;
 		p_umv_buf->output_size = sizeof(struct ibv_create_cq_resp);
 	}
@@ -364,11 +363,12 @@ struct ib_qp *ibv_create_qp(struct ib_pd
 		// prepare user parameters
 		p_req = (struct ibv_create_qp*)(ULONG_PTR)p_umv_buf->p_inout_buf;
 		p_resp = (struct ibv_create_qp_resp*)(ULONG_PTR)p_umv_buf->p_inout_buf;
-		INIT_UDATA(&udata, &p_req->buf_addr, NULL, 
-			sizeof(struct mlx4_ib_create_qp), 0);
+		INIT_UDATA(&udata, p_req, NULL, 
+			sizeof(struct ibv_create_qp), 0);
 	}
-	else 
+	else {
 		p_udata = NULL;
+    }
 
 	p_ib_qp = pd->device->create_qp( pd, qp_init_attr, p_udata );
 
@@ -480,11 +480,12 @@ struct ib_srq *ibv_create_srq(struct ib_
 		// prepare user parameters
 		p_req = (struct ibv_create_srq*)(ULONG_PTR)p_umv_buf->p_inout_buf;
 		p_resp = (struct ibv_create_srq_resp*)(ULONG_PTR)p_umv_buf->p_inout_buf;
-		INIT_UDATA(&udata, &p_req->buf_addr, &p_resp->srqn, 
+		INIT_UDATA(&udata, p_req, p_resp, 
 			sizeof(struct ibv_create_srq), sizeof(struct ibv_create_srq_resp));
 	}
-	else 
+	else {
 		p_udata = NULL;
+    }
 
 	p_ib_srq = pd->device->create_srq( pd, srq_init_attr, p_udata );
 	if (IS_ERR(p_ib_srq)) {
@@ -512,10 +513,6 @@ struct ib_srq *ibv_create_srq(struct ib_
 
 	// fill results for user
 	if (p_uctx && p_umv_buf && p_umv_buf->p_inout_buf) {
-		struct mlx4_ib_srq* p_mib_srq = (struct mlx4_ib_srq*)p_ib_srq;
-		p_resp->srq_handle = (__u64)(ULONG_PTR)p_ib_srq;
-		p_resp->max_wr = p_mib_srq->msrq.max - 1;
-		p_resp->max_sge = p_mib_srq->msrq.max_gs;
 		p_umv_buf->output_size = sizeof(struct ibv_create_srq_resp);
 		HCA_PRINT(TRACE_LEVEL_INFORMATION ,HCA_DBG_QP ,("PD%d use cnt %d \n", 
 			((struct mlx4_ib_pd*)pd)->pdn, pd->usecnt));
@@ -639,6 +635,7 @@ ib_api_status_t ibv_um_open(	
 	int err;
 	ib_api_status_t		status;
 	struct mlx4_ib_ucontext *p_muctx;
+    struct ibv_get_context_req *p_ureq;
 	struct ibv_get_context_resp *p_uresp;
 	struct mlx4_ib_alloc_ucontext_resp ib_alloc_ucontext_resp;
 	struct ib_ucontext 		*p_uctx;
@@ -646,6 +643,20 @@ ib_api_status_t ibv_um_open(	
 
 	HCA_ENTER(HCA_DBG_SHIM);
 
+    p_ureq = (struct ibv_get_context_req*)(ULONG_PTR)p_umv_buf->p_inout_buf;
+    if( NdValidateIoSpaceMapping( &p_ureq->mappings[ibv_get_context_uar],
+                                  NdNonCached, PAGE_SIZE ) != STATUS_SUCCESS )
+    {
+        status = IB_INVALID_PARAMETER;
+        goto end;
+    }
+    if( NdValidateIoSpaceMapping( &p_ureq->mappings[ibv_get_context_bf],
+                                  NdWriteCombined, PAGE_SIZE ) != STATUS_SUCCESS )
+    {
+        status = IB_INVALID_PARAMETER;
+        goto end;
+    }
+
 	// create user context in kernel
 	INIT_UDATA(&udata, NULL, &ib_alloc_ucontext_resp, 
 		0, sizeof(struct mlx4_ib_alloc_ucontext_resp));
@@ -673,7 +684,8 @@ ib_api_status_t ibv_um_open(	
 	if( !NT_SUCCESS(status) ) {
 		goto err_map_uar;
 	}
-	p_uresp->uar_addr	 = (u64)(ULONG_PTR)p_uctx->x.uar.uva;
+    p_uresp->mapping_results[ibv_get_context_uar].Id = 0;
+	p_uresp->mapping_results[ibv_get_context_uar].Information = (u64)(ULONG_PTR)p_uctx->x.uar.uva;
 	HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM,
 		("UAR: pa 0x%I64x, size %#x, kva 0x%I64x, uva 0x%I64x, num_uars %#x \n", 
 			(io_addr_t)p_muctx->uar.pfn << PAGE_SHIFT,
@@ -684,19 +696,20 @@ ib_api_status_t ibv_um_open(	
 			));
 
 	// map BF to user space
+	p_uresp->mapping_results[ibv_get_context_bf].Id = 0;
 	if (ib_alloc_ucontext_resp.bf_reg_size) {
 		status = __map_memory_for_user( 
 			(io_addr_t)(p_muctx->uar.pfn + 
 			to_mdev(p_ibdev)->dev->caps.num_uars) << PAGE_SHIFT, 
 			PAGE_SIZE, MmWriteCombined, mode, &p_uctx->x.bf );
 		if( !NT_SUCCESS(status) ) {
 			HCA_PRINT(TRACE_LEVEL_WARNING ,HCA_DBG_SHIM,
 				("BlueFlame available, but failed to be mapped (%#x)\n", status));
-			p_uresp->bf_page	 = 0;
+			p_uresp->mapping_results[ibv_get_context_bf].Information = 0;
 			p_uresp->bf_buf_size = 0;
 		} 
 		else {
-			p_uresp->bf_page	 = (u64)(ULONG_PTR)p_uctx->x.bf.uva;
+			p_uresp->mapping_results[ibv_get_context_bf].Information = (u64)(ULONG_PTR)p_uctx->x.bf.uva;
 			p_uresp->bf_buf_size = ib_alloc_ucontext_resp.bf_reg_size / 2;
 			p_uresp->bf_offset	 = 0;
 			HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM,
@@ -710,7 +723,7 @@ ib_api_status_t ibv_um_open(	
 		}
 	}
 	else {
-			p_uresp->bf_page	 = 0;
+			p_uresp->mapping_results[ibv_get_context_bf].Information = 0;
 			p_uresp->bf_buf_size = 0;
 	}
 
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\precomp.h .\hw\mlx4\kernel\hca\precomp.h
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\precomp.h	Thu May 31 11:22:19 2012
+++ .\hw\mlx4\kernel\hca\precomp.h	Wed May 23 18:26:47 2012
@@ -31,6 +31,7 @@
 
 #include <ntddk.h>
 #include <wdf.h>
+#include <ndioctl.h>
 #define NTSTRSAFE_LIB
 #include <ntstrsafe.h>
 #include <initguid.h> // required for GUID definitions
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\qp.c .\hw\mlx4\kernel\hca\qp.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\qp.c	Thu May 31 11:22:19 2012
+++ .\hw\mlx4\kernel\hca\qp.c	Thu Jul 26 15:31:14 2012
@@ -122,21 +122,12 @@ __create_qp (
 	qp_init_attr.send_cq = (struct ib_cq *)p_create_attr->h_sq_cq;
 	qp_init_attr.recv_cq = (struct ib_cq *)p_create_attr->h_rq_cq;
 	qp_init_attr.srq = (struct ib_srq *)p_create_attr->h_srq;
-	if( p_umv_buf && p_umv_buf->command ) {
-		qp_init_attr.cap.max_recv_sge = p_req->max_recv_sge;
-		qp_init_attr.cap.max_send_sge = p_req->max_send_sge;
-		qp_init_attr.cap.max_recv_wr = p_req->max_recv_wr;
-		qp_init_attr.cap.max_send_wr = p_req->max_send_wr;
-		qp_init_attr.cap.max_inline_data = p_req->max_inline_data;
-	}
-	else {
 		qp_init_attr.cap.max_recv_sge = p_create_attr->rq_sge;
 		qp_init_attr.cap.max_send_sge = p_create_attr->sq_sge;
 		qp_init_attr.cap.max_recv_wr = p_create_attr->rq_depth;
 		qp_init_attr.cap.max_send_wr = p_create_attr->sq_depth;
 		qp_init_attr.cap.max_inline_data = p_create_attr->sq_max_inline;
 		
-	}
 	qp_init_attr.sq_sig_type = (p_create_attr->sq_signaled) ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
 	qp_init_attr.qp_type = to_qp_type(p_create_attr->qp_type);
 	qp_init_attr.port_num = port_num;
@@ -167,7 +158,7 @@ __create_qp (
 	}
 	
 	// return the results
-	if (ph_qp) *ph_qp = (ib_qp_handle_t)p_ib_qp;
+	*ph_qp = (ib_qp_handle_t)p_ib_qp;
 
 	status = IB_SUCCESS;
 	goto end;
@@ -365,11 +356,10 @@ mlnx_ndi_modify_qp (
 	ci_umv_buf_t umv_buf;
 	ib_api_status_t status;
 	struct ibv_modify_qp_resp resp;
-	void *buf = &resp;
 
 	HCA_ENTER(HCA_DBG_QP);
 
-	if (buf_size < sizeof(resp.qp_state)) {
+	if (buf_size != 0 && buf_size < sizeof(resp.qp_state)) {
 		status = IB_INVALID_PARAMETER;
 		goto out;
 	}
@@ -377,12 +367,12 @@ mlnx_ndi_modify_qp (
 	/* imitate umv_buf */
 	umv_buf.command = TRUE;	/* special case for NDI. Usually it's TRUE */
 	umv_buf.input_size = 0;
-	umv_buf.output_size = sizeof(struct ibv_modify_qp_resp);
-	umv_buf.p_inout_buf = (ULONG_PTR)buf;
+	umv_buf.output_size = sizeof(resp);
+	umv_buf.p_inout_buf = (ULONG_PTR)&resp;
 
 	status = mlnx_modify_qp ( h_qp, p_modify_attr, p_qp_attr, &umv_buf );
 
-	if (status == IB_SUCCESS) {
+	if (status == IB_SUCCESS && buf_size != 0) {
 		cl_memclr( p_outbuf, buf_size );
 		*p_outbuf = resp.qp_state;
 	}
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\vp.c .\hw\mlx4\kernel\hca\vp.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\vp.c	Thu Aug 02 16:18:42 2012
+++ .\hw\mlx4\kernel\hca\vp.c	Thu Jul 26 15:31:14 2012
@@ -73,7 +73,8 @@ mlnx_um_open(
 	else
 	{
 	// sanity check
-	if ( p_umv_buf->output_size < sizeof(struct ibv_get_context_resp) ||
+		if ( p_umv_buf->input_size < sizeof(struct ibv_get_context_req) ||
+		     p_umv_buf->output_size < sizeof(struct ibv_get_context_resp) ||
 		!p_umv_buf->p_inout_buf) {
 		status = IB_INVALID_PARAMETER;
 		goto err_inval_params;
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\mlx4.c .\hw\mlx4\user\hca\mlx4.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\mlx4.c	Thu Mar 29 00:15:30 2012
+++ .\hw\mlx4\user\hca\mlx4.c	Wed May 23 15:14:37 2012
@@ -161,8 +161,8 @@ found:
 	for (i = 0; i < MLX4_NUM_DB_TYPE; ++i)
 		context->db_list[i] = NULL;
 
-	context->uar 			= (uint8_t *)(uintptr_t)p_resp->uar_addr;
-	context->bf_page		= (uint8_t *)(uintptr_t)p_resp->bf_page;
+	context->uar 			= (uint8_t *)(uintptr_t)p_resp->mapping_results[ibv_get_context_uar].Information;
+	context->bf_page		= (uint8_t *)(uintptr_t)p_resp->mapping_results[ibv_get_context_bf].Information;
 	context->bf_buf_size	= p_resp->bf_buf_size;
 	context->bf_offset		= p_resp->bf_offset;
 	context->cqe_size		= p_resp->cqe_size;
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\mlx4.h .\hw\mlx4\user\hca\mlx4.h
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\mlx4.h	Wed Aug 01 17:16:57 2012
+++ .\hw\mlx4\user\hca\mlx4.h	Thu May 31 12:35:11 2012
@@ -35,6 +35,7 @@
 
 #include <iba\ib_types.h>
 #include <iba\ib_uvp.h>
+#include <ndioctl.h>
 
 #include "verbs.h"
 #include "mx_abi.h"
@@ -186,7 +187,6 @@ struct mlx4_srq {
 	struct mlx4_buf			buf;
 	pthread_spinlock_t		lock;
 	uint64_t		       *wrid;
-	uint32_t			srqn;
 	int				max;
 	int				max_gs;
 	int				wqe_shift;
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\qp.c .\hw\mlx4\user\hca\qp.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\qp.c	Wed Aug 01 17:17:03 2012
+++ .\hw\mlx4\user\hca\qp.c	Wed May 23 18:40:41 2012
@@ -660,6 +660,8 @@ int mlx4_alloc_qp_buf(struct ibv_pd *pd,
 		qp->sq.offset = 0;
 	}
 
+    qp->buf_size = align(qp->buf_size, pd->context->page_size);
+
 	if (mlx4_alloc_buf(&qp->buf, qp->buf_size, pd->context->page_size)) {
 		free(qp->sq.wrid);
 		if (qp->rq.wqe_cnt)
Only in c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca: sources.props
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\verbs.c .\hw\mlx4\user\hca\verbs.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\verbs.c	Thu Mar 29 00:15:30 2012
+++ .\hw\mlx4\user\hca\verbs.c	Mon Jul 30 16:04:30 2012
@@ -49,6 +49,7 @@ mlx4_pre_open_ca (
 		OUT			ib_ca_handle_t			*ph_uvp_ca )
 {
 	struct ibv_context	*context;
+    struct ibv_get_context_req *req;
 	ib_api_status_t status = IB_SUCCESS;
 
 	UNREFERENCED_PARAMETER(ca_guid);
@@ -61,13 +62,24 @@ mlx4_pre_open_ca (
 	
 	if( p_umv_buf )
 	{
-		p_umv_buf->p_inout_buf = (ULONG_PTR)cl_zalloc( sizeof(struct ibv_get_context_resp) );
-		if( !p_umv_buf->p_inout_buf )
+		req = (struct ibv_get_context_req*)cl_zalloc(
+            max(sizeof(struct ibv_get_context_req), sizeof(struct ibv_get_context_resp)) );
+		if( req == NULL )
 		{
 			status = IB_INSUFFICIENT_MEMORY;
 			goto end;
 		}
-		p_umv_buf->input_size = 0;
+
+        req->mappings[ibv_get_context_uar].MapType = NdMapIoSpace;
+        req->mappings[ibv_get_context_uar].MapIoSpace.CacheType = NdNonCached;
+        req->mappings[ibv_get_context_uar].MapIoSpace.CbLength = 4096;
+
+        req->mappings[ibv_get_context_bf].MapType = NdMapIoSpace;
+        req->mappings[ibv_get_context_bf].MapIoSpace.CacheType = NdWriteCombined;
+        req->mappings[ibv_get_context_bf].MapIoSpace.CbLength = 4096;
+
+        p_umv_buf->p_inout_buf = (ULONG_PTR)req;
+		p_umv_buf->input_size = sizeof(struct ibv_get_context_req);
 		p_umv_buf->output_size = sizeof(struct ibv_get_context_resp);
 		p_umv_buf->command = TRUE;
 	}
@@ -234,6 +246,7 @@ mlx4_pre_create_cq (
 	ib_api_status_t		status = IB_SUCCESS;
 	int size = max( sizeof(struct ibv_create_cq), sizeof(struct ibv_create_cq_resp) );
 	int cqe_size = to_mctx(context)->cqe_size;
+    uint32_t cqe;
 
 	CL_ASSERT(h_uvp_ca && p_umv_buf);
 
@@ -268,10 +281,11 @@ mlx4_pre_create_cq (
 		goto err_lock;
 	}
 
-	*p_size = __align_queue_size(*p_size + 1);
+	cqe = __align_queue_size(*p_size + 1);
 
-	if (mlx4_alloc_buf(&cq->buf, *p_size * cqe_size, 
-						context->page_size)){
+	if (mlx4_alloc_buf(&cq->buf, cqe * cqe_size, 
+						context->page_size))
+    {
 		status = IB_INSUFFICIENT_MEMORY;
 		goto err_alloc_buf;
 	}
@@ -293,10 +307,22 @@ mlx4_pre_create_cq (
 	cq->arm_sn = 1;
 	*cq->set_ci_db = 0;
 
-	p_create_cq->buf_addr = (uintptr_t) cq->buf.buf;
-	p_create_cq->db_addr  = (uintptr_t) cq->set_ci_db;
-	p_create_cq->arm_sn_addr  = (uintptr_t) &cq->arm_sn;
-	p_create_cq->cqe = --(*p_size);
+	p_create_cq->mappings[mlx4_ib_create_cq_buf].MapMemory.MapType = NdMapMemory;
+	p_create_cq->mappings[mlx4_ib_create_cq_buf].MapMemory.AccessType = NdModifyAccess;
+	p_create_cq->mappings[mlx4_ib_create_cq_buf].MapMemory.Address = (uintptr_t) cq->buf.buf;
+	p_create_cq->mappings[mlx4_ib_create_cq_buf].MapMemory.CbLength = cqe * cqe_size;
+
+	p_create_cq->mappings[mlx4_ib_create_cq_db].MapMemory.MapType = NdMapMemoryCoallesce;
+	p_create_cq->mappings[mlx4_ib_create_cq_db].MapMemory.AccessType = NdWriteAccess;
+	p_create_cq->mappings[mlx4_ib_create_cq_db].MapMemory.Address = (uintptr_t) cq->set_ci_db;
+	p_create_cq->mappings[mlx4_ib_create_cq_db].MapMemory.CbLength = sizeof(*cq->set_ci_db) + sizeof(*cq->arm_db);
+	
+	p_create_cq->mappings[mlx4_ib_create_cq_arm_sn].MapMemory.MapType = NdMapMemory;
+	p_create_cq->mappings[mlx4_ib_create_cq_arm_sn].MapMemory.AccessType = NdModifyAccess;
+	p_create_cq->mappings[mlx4_ib_create_cq_arm_sn].MapMemory.Address = (uintptr_t) &cq->arm_sn;
+	p_create_cq->mappings[mlx4_ib_create_cq_arm_sn].MapMemory.CbLength = sizeof(cq->arm_sn);
+
+    *p_size = --cqe;
 
 	*ph_uvp_cq = (ib_cq_handle_t)&cq->ibv_cq;
 	goto end;
@@ -339,7 +365,6 @@ mlx4_post_create_cq (
 		
 		to_mcq(cq)->cqn	= p_resp->cqn;
 		cq->cqe			= p_resp->cqe;
-		cq->handle		= p_resp->cq_handle;
 	}
 	else
 	{
@@ -452,12 +477,15 @@ mlx4_pre_create_srq (
 	*srq->db = 0;
 	
 	// fill the parameters for ioctl
-	p_create_srq->buf_addr = (uintptr_t) srq->buf.buf;
-	p_create_srq->db_addr  = (uintptr_t) srq->db;
-	p_create_srq->pd_handle = pd->handle;
-	p_create_srq->max_wr = p_srq_attr->max_wr;
-	p_create_srq->max_sge = p_srq_attr->max_sge;
-	p_create_srq->srq_limit = p_srq_attr->srq_limit;
+	p_create_srq->mappings[mlx4_ib_create_srq_buf].MapMemory.MapType = NdMapMemory;
+	p_create_srq->mappings[mlx4_ib_create_srq_buf].MapMemory.AccessType = NdModifyAccess;
+	p_create_srq->mappings[mlx4_ib_create_srq_buf].MapMemory.Address = (uintptr_t) srq->buf.buf;
+	p_create_srq->mappings[mlx4_ib_create_srq_buf].MapMemory.CbLength = srq->max << srq->wqe_shift;
+
+	p_create_srq->mappings[mlx4_ib_create_srq_db].MapMemory.MapType = NdMapMemoryCoallesce;
+	p_create_srq->mappings[mlx4_ib_create_srq_db].MapMemory.AccessType = NdWriteAccess;
+	p_create_srq->mappings[mlx4_ib_create_srq_db].MapMemory.Address = (uintptr_t) srq->db;
+	p_create_srq->mappings[mlx4_ib_create_srq_db].MapMemory.CbLength = sizeof(*srq->db);
 
 	*ph_uvp_srq = (ib_srq_handle_t)&srq->ibv_srq;
 	goto end;
@@ -483,8 +511,6 @@ mlx4_post_create_srq (
 	IN	OUT			ib_srq_handle_t			*ph_uvp_srq,
 	IN				ci_umv_buf_t				*p_umv_buf )
 {
-	struct ibv_srq *ibsrq = (struct ibv_srq *)*ph_uvp_srq;
-	struct mlx4_srq *srq = to_msrq(ibsrq);
 	struct ibv_create_srq_resp *p_resp;
 
 	UNREFERENCED_PARAMETER(h_uvp_pd);
@@ -493,17 +519,7 @@ mlx4_post_create_srq (
 	
 	p_resp = (struct ibv_create_srq_resp*)(ULONG_PTR)p_umv_buf->p_inout_buf;
 
-	if (IB_SUCCESS == ioctl_status)
-	{
-		// Mlx4 code:
-
-		srq->srqn	= p_resp->srqn;
-		ibsrq->handle	= p_resp->srq_handle;
-		
-		srq->max		= p_resp->max_wr;
-		srq->max_gs	= p_resp->max_sge;
-	}
-	else
+	if (IB_SUCCESS != ioctl_status)
 	{
 		mlx4_post_destroy_srq (*ph_uvp_srq, IB_SUCCESS);
 		*ph_uvp_srq = NULL;
@@ -705,34 +721,28 @@ mlx4_pre_create_qp (
 	qp->ibv_qp.qp_type = attr.qp_type;
 
 	// fill request fields
-	p_create_qp->buf_addr = (uintptr_t) qp->buf.buf;
-	if (!attr.srq && attr.qp_type != IBV_QPT_XRC)
-		p_create_qp->db_addr = (uintptr_t) qp->db;
-	else
-		p_create_qp->db_addr = 0;
-
-	p_create_qp->pd_handle = pd->handle;
-	p_create_qp->send_cq_handle = attr.send_cq->handle;
-	p_create_qp->recv_cq_handle = attr.recv_cq->handle;
-	p_create_qp->srq_handle = attr.qp_type == IBV_QPT_XRC ?
-		(attr.xrc_domain ? attr.xrc_domain->handle : 0) :
-		(attr.srq ? attr.srq->handle : 0);
+	p_create_qp->mappings[mlx4_ib_create_qp_buf].MapMemory.MapType = NdMapMemory;
+	p_create_qp->mappings[mlx4_ib_create_qp_buf].MapMemory.AccessType = NdModifyAccess;
+	p_create_qp->mappings[mlx4_ib_create_qp_buf].MapMemory.Address = (uintptr_t) qp->buf.buf;
+	p_create_qp->mappings[mlx4_ib_create_qp_buf].MapMemory.CbLength = qp->buf_size;
 
-	p_create_qp->max_send_wr = attr.cap.max_send_wr;
-	p_create_qp->max_recv_wr = attr.cap.max_recv_wr;
-	p_create_qp->max_send_sge = attr.cap.max_send_sge;
-	p_create_qp->max_recv_sge = attr.cap.max_recv_sge;
-	p_create_qp->max_inline_data = attr.cap.max_inline_data;
-	p_create_qp->sq_sig_all = (uint8_t)attr.sq_sig_all;
-	p_create_qp->qp_type = attr.qp_type;
-	p_create_qp->is_srq = (uint8_t)(attr.qp_type == IBV_QPT_XRC ?
-									!!attr.xrc_domain : !!attr.srq);
+	p_create_qp->mappings[mlx4_ib_create_qp_db].MapMemory.MapType = NdMapMemoryCoallesce;
+	p_create_qp->mappings[mlx4_ib_create_qp_db].MapMemory.AccessType = NdWriteAccess;
+	if (!attr.srq && attr.qp_type != IBV_QPT_XRC) {
+		p_create_qp->mappings[mlx4_ib_create_qp_db].MapMemory.Address = (uintptr_t) qp->db;
+		p_create_qp->mappings[mlx4_ib_create_qp_db].MapMemory.CbLength = sizeof(*qp->db);
+	} else {
+		p_create_qp->mappings[mlx4_ib_create_qp_db].MapMemory.Address = 0;
+		p_create_qp->mappings[mlx4_ib_create_qp_db].MapMemory.CbLength = 0;
+	}
 
 	p_create_qp->log_sq_stride   = (uint8_t)qp->sq.wqe_shift;
 	for (p_create_qp->log_sq_bb_count = 0;
 	     qp->sq.wqe_cnt > 1 << p_create_qp->log_sq_bb_count;
 	     ++p_create_qp->log_sq_bb_count)
+    {
 		; /* nothing */
+    }
 	p_create_qp->sq_no_prefetch = 0;
 
 	*ph_uvp_qp = (ib_qp_handle_t)&qp->ibv_qp;
@@ -972,17 +982,7 @@ ib_api_status_t
 mlx4_pre_destroy_qp (
 	IN		const	ib_qp_handle_t			h_uvp_qp )
 {
-	struct ibv_qp *qp = (struct ibv_qp*)h_uvp_qp;
-
-	mlx4_cq_clean(to_mcq(qp->recv_cq), qp->qp_num,
-		       		qp->srq ? to_msrq(qp->srq) : NULL);
-	if (qp->send_cq != qp->recv_cq)
-		mlx4_cq_clean(to_mcq(qp->send_cq), qp->qp_num, NULL);
-
-	__mlx4_lock_cqs(qp);
-	mlx4_clear_qp(to_mctx(qp->context), qp->qp_num);
-	__mlx4_unlock_cqs(qp);
-
+    UNREFERENCED_PARAMETER(h_uvp_qp);
 	return IB_SUCCESS;
 }
 
@@ -998,6 +998,15 @@ mlx4_post_destroy_qp (
 
 	if (IB_SUCCESS == ioctl_status)
 	{
+		mlx4_cq_clean(to_mcq(ibqp->recv_cq), ibqp->qp_num,
+					ibqp->srq ? to_msrq(ibqp->srq) : NULL);
+		if (ibqp->send_cq != ibqp->recv_cq)
+			mlx4_cq_clean(to_mcq(ibqp->send_cq), ibqp->qp_num, NULL);
+
+		__mlx4_lock_cqs(ibqp);
+		mlx4_clear_qp(to_mctx(ibqp->context), ibqp->qp_num);
+		__mlx4_unlock_cqs(ibqp);
+
 		if (!ibqp->srq && ibqp->qp_type != IBV_QPT_XRC)
 			mlx4_free_db(to_mctx(ibqp->context), MLX4_DB_TYPE_RQ, qp->db);
 
@@ -1012,12 +1021,6 @@ mlx4_post_destroy_qp (
 			cl_free(qp->rq.wrid);
 		mlx4_free_buf(&qp->buf);
 		cl_free(qp);
-	}
-	else
-	{
-		__mlx4_lock_cqs(ibqp);
-		mlx4_store_qp(to_mctx(ibqp->context), ibqp->qp_num, qp);
-		__mlx4_unlock_cqs(ibqp);		
 	}
 }
 
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\verbs.h .\hw\mlx4\user\hca\verbs.h
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\verbs.h	Thu Mar 29 00:15:30 2012
+++ .\hw\mlx4\user\hca\verbs.h	Thu May 31 12:35:12 2012
@@ -176,7 +176,6 @@ struct ibv_pd {
 struct ibv_srq {
 	struct ibv_context     *context;
 	struct ibv_pd	       *pd;
-	uint64_t		handle;
 	uint32_t		xrc_srq_num;
 	struct ibv_xrc_domain  *xrc_domain;
 	struct ibv_cq	       *xrc_cq;
@@ -198,7 +197,6 @@ struct ibv_qp {
 
 struct ibv_cq {
 	struct ibv_context     *context;
-	uint64_t		handle;
 	int			cqe;
 };
 
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\core\SOURCES .\hw\mlx4\kernel\bus\core\SOURCES
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\core\SOURCES	Wed Aug 01 17:18:35 2012
+++ .\hw\mlx4\kernel\bus\core\SOURCES	Thu Jul 26 15:31:14 2012
@@ -25,7 +25,7 @@ SOURCES= \
 	pa_cash.c		\
 	verbs.c			\
 
-INCLUDES=..;..\inc;..\..\inc;..\net;..\..\..\..\..\inc;..\..\..\..\..\inc\kernel;..\..\..\..\..\core\complib\kernel\$(O)
+INCLUDES=..;..\inc;..\..\inc;..\net;..\..\..\..\..\inc;..\..\..\..\..\inc\kernel;$(ND_SDK_PATH)\include;..\..\..\..\..\core\complib\kernel\$(O)
 
 C_DEFINES=$(C_DEFINES) -DDRIVER -DDEPRECATE_DDK_FUNCTIONS -D__LITTLE_ENDIAN -DUSE_WDM_INTERRUPTS
 
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\drv\sources .\hw\mlx4\kernel\bus\drv\sources
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\drv\sources	Thu Mar 29 00:15:28 2012
+++ .\hw\mlx4\kernel\bus\drv\sources	Thu Jul 26 23:13:52 2012
@@ -47,17 +47,17 @@ KMDF_VERSION_MAJOR=1
 
 C_DEFINES=$(C_DEFINES) -DDRIVER -DDEPRECATE_DDK_FUNCTIONS -D__LITTLE_ENDIAN -DUSE_WDM_INTERRUPTS
 
-INCLUDES=..;..\inc;..\..\inc;..\core;..\..\..\inc;..\..\..\..\..\inc;..\..\..\..\..\inc\kernel;..\core;..\core\$(O);..\..\genutils;..\..\genutils\$(O);..\..\..\..\..\core\complib\kernel\$(O);..\ib\$(O);..\net\$(O);..\..\l2w\$(O);
+INCLUDES=..;..\inc;..\..\inc;..\core;..\..\..\inc;..\..\..\..\..\inc;..\..\..\..\..\inc\kernel;..\core;..\core\$(O);$(ND_SDK_PATH)\include;..\..\genutils;..\..\genutils\$(O);..\..\..\..\..\core\complib\kernel\$(O);..\ib\$(O);..\net\$(O);..\..\l2w\$(O);
 
 TARGETLIBS= $(TARGETLIBS) \
 	$(DDK_LIB_PATH)\ntstrsafe.lib \
-	$(LIBPATH)\*\complib.lib \
-	$(LIBPATH)\*\mlx4_core.lib \
-	$(LIBPATH)\*\mlx4_ib.lib \
-	$(LIBPATH)\*\mlx4_net.lib \
-	$(LIBPATH)\*\mlx4_hca.lib \
-	$(LIBPATH)\*\l2w.lib	\
-	$(LIBPATH)\*\genutils.lib     \
+    $(TARGETPATH)\*\complib.lib \
+    $(TARGETPATH)\*\mlx4_core.lib \
+    $(TARGETPATH)\*\mlx4_ib.lib \
+    $(TARGETPATH)\*\mlx4_net.lib \
+    $(TARGETPATH)\*\mlx4_hca.lib \
+    $(TARGETPATH)\*\l2w.lib \
+    $(TARGETPATH)\*\genutils.lib \
 
 #LINKER_FLAGS=/MAP 
 
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\ib\SOURCES .\hw\mlx4\kernel\bus\ib\SOURCES
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\bus\ib\SOURCES	Wed Aug 01 17:18:35 2012
+++ .\hw\mlx4\kernel\bus\ib\SOURCES	Thu Jul 26 21:39:58 2012
@@ -25,7 +25,7 @@ SOURCES= ib.rc		\
 	qp.c			\
 	srq.c			
 
-INCLUDES=..;..\inc;..\..\inc;..\core\$O;..\..\..\inc;..\..\..\..\..\inc;..\..\..\..\..\inc\kernel;..\..\..\..\..\core\complib\kernel\$(O);
+INCLUDES=..;..\inc;..\..\inc;..\core\$(O);..\..\..\inc;..\..\..\..\..\inc;..\..\..\..\..\inc\kernel;..\..\..\..\..\core\complib\kernel\$(O);$(ND_SDK_PATH)\include;
 
 C_DEFINES=$(C_DEFINES) -DDRIVER -DDEPRECATE_DDK_FUNCTIONS -D__LITTLE_ENDIAN
 
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\SOURCES .\hw\mlx4\kernel\hca\SOURCES
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\SOURCES	Thu Mar 29 00:15:29 2012
+++ .\hw\mlx4\kernel\hca\SOURCES	Thu Jul 26 21:39:28 2012
@@ -25,7 +25,7 @@ SOURCES= \
 	hverbs.c		\
 	vp.c			\
 
-INCLUDES=..;..\inc;..\..\inc;..\bus\inc;..\bus\ib;..\bus\core\$O;..\..\..\..\inc;..\..\..\..\inc\kernel;
+INCLUDES=..;..\inc;..\..\inc;..\bus\inc;..\bus\ib;..\bus\core\$O;..\..\..\..\inc;..\..\..\..\inc\kernel;$(ND_SDK_PATH)\include;
 
 PRECOMPILED_INCLUDE=precomp.h
 
diff -dwup3 -X excl.txt -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\SOURCES .\hw\mlx4\user\hca\SOURCES
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\user\hca\SOURCES	Wed Aug 01 17:18:35 2012
+++ .\hw\mlx4\user\hca\SOURCES	Thu Jul 26 21:29:20 2012
@@ -37,8 +37,8 @@ SOURCES= \
 	$(TRUNK)\inc\user; \
 	$(TRUNK)\inc\complib; \
 	$(TRUNK)\inc\user\complib; \
-	$(TRUNK)\core\complib\user\$(O); \
 	$(TRUNK)\inc;	\
+    $(ND_SDK_PATH)\include; \
 
 USER_C_FLAGS=$(USER_C_FLAGS) /DCL_NO_TRACK_MEM

-------------- next part --------------
A non-text attachment was scrubbed...
Name: ndv2.33.patch
Type: application/octet-stream
Size: 45136 bytes
Desc: ndv2.33.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20130221/bbc81b43/attachment.obj>


More information about the ofw mailing list