[ofw] [Patch 17/62] Reference implementation of NDv2
Fab Tillier
ftillier at microsoft.com
Wed Feb 20 17:36:26 PST 2013
Add API to rebase memory registrations.
The HCA driver already supports specifying an arbitrary starting address for an MR. This patch expose that functionality as a new entry point.
Signed-off-by: Fab Tillier <ftillier at microsoft.com>
diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_mr.h .\core\al\al_mr.h
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_mr.h Thu May 31 11:22:16 2012
+++ .\core\al\al_mr.h Wed May 23 18:26:47 2012
@@ -83,6 +83,7 @@ ib_api_status_t
reg_mem(
IN const ib_pd_handle_t h_pd,
IN const ib_mr_create_t* const p_mr_create,
+ IN const uint64_t mapaddr,
OUT net32_t* const p_lkey,
OUT net32_t* const p_rkey,
OUT ib_mr_handle_t* const ph_mr,
diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_mr_shared.c .\core\al\al_mr_shared.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_mr_shared.c Thu May 31 11:22:16 2012
+++ .\core\al\al_mr_shared.c Wed May 23 18:26:47 2012
@@ -172,7 +172,14 @@ ib_reg_mem(
return IB_INVALID_PD_HANDLE;
}
- status = reg_mem( h_pd, p_mr_create, p_lkey, p_rkey, ph_mr, FALSE );
+ if( !p_mr_create )
+ {
+ AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("IB_INVALID_PARAMETER\n") );
+ return IB_INVALID_PARAMETER;
+ }
+
+ status = reg_mem( h_pd, p_mr_create, (ULONG_PTR)p_mr_create->vaddr,
+ p_lkey, p_rkey, ph_mr, FALSE );
/* Release the reference taken in alloc_mr for initialization. */
if( status == IB_SUCCESS )
@@ -188,6 +195,7 @@ ib_api_status_t
reg_mem(
IN const ib_pd_handle_t h_pd,
IN const ib_mr_create_t* const p_mr_create,
+ IN const uint64_t mapaddr,
OUT net32_t* const p_lkey,
OUT net32_t* const p_rkey,
OUT ib_mr_handle_t* const ph_mr,
@@ -198,7 +206,7 @@ reg_mem(
AL_ENTER( AL_DBG_MR );
- if( !p_mr_create || !p_lkey || !p_rkey || !ph_mr )
+ if( !p_lkey || !p_rkey || !ph_mr )
{
AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("IB_INVALID_PARAMETER\n") );
return IB_INVALID_PARAMETER;
diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_verbs.h .\core\al\al_verbs.h
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\al_verbs.h Thu Aug 02 12:53:26 2012
+++ .\core\al\al_verbs.h Wed May 23 18:26:47 2012
@@ -116,8 +116,8 @@ verbs_create_cq(
IB_UNSUPPORTED )
#define verbs_register_mr(h_pd, p_mr_create, p_lkey, p_rkey, h_mr) \
- h_mr->obj.p_ci_ca->verbs.register_mr( h_pd->h_ci_pd,\
- p_mr_create, p_lkey, p_rkey, &h_mr->h_ci_mr, um_call )
+ h_mr->obj.p_ci_ca->verbs.register_mr_remap( h_pd->h_ci_pd,\
+ p_mr_create, mapaddr, p_lkey, p_rkey, &h_mr->h_ci_mr, um_call )
#define verbs_register_pmr(h_pd, p_phys_create, p_vaddr,\
p_lkey, p_rkey, h_mr) \
@@ -462,7 +462,8 @@ verbs_create_cq(
#define verbs_register_mr(h_pd, p_mr_create, p_lkey, p_rkey, h_mr) \
ual_reg_mem(h_pd, p_mr_create, p_lkey, p_rkey, h_mr); \
- UNUSED_PARAM( um_call )
+ UNUSED_PARAM( um_call ); \
+ UNUSED_PARAM( mapaddr )
#define verbs_register_pmr(h_pd, p_phys_create, p_vaddr, p_lkey, p_rkey, h_mr) \
IB_UNSUPPORTED; \
diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\kernel\al_proxy_verbs.c .\core\al\kernel\al_proxy_verbs.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\core\al\kernel\al_proxy_verbs.c Thu Aug 02 16:18:42 2012
+++ .\core\al\kernel\al_proxy_verbs.c Fri Aug 03 14:21:53 2012
@@ -2967,7 +2967,7 @@ proxy_register_mr(
goto proxy_register_mr_err;
}
- status = reg_mem( h_pd, &p_ioctl->in.mem_create, &p_ioctl->out.lkey,
+ status = reg_mem( h_pd, &p_ioctl->in.mem_create, (ULONG_PTR)p_ioctl->in.mem_create.vaddr, &p_ioctl->out.lkey,
&p_ioctl->out.rkey, &h_mr, TRUE );
if( status == IB_SUCCESS )
diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\mr.c .\hw\mlx4\kernel\hca\mr.c
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\hw\mlx4\kernel\hca\mr.c Thu Aug 02 13:58:26 2012
+++ .\hw\mlx4\kernel\hca\mr.c Thu May 31 12:35:11 2012
@@ -42,9 +42,10 @@
*/
ib_api_status_t
-mlnx_register_mr (
+mlnx_register_mr_remap (
IN const ib_pd_handle_t h_pd,
IN const ib_mr_create_t *p_mr_create,
+ IN const uint64_t mapaddr,
OUT net32_t* const p_lkey,
OUT net32_t* const p_rkey,
OUT ib_mr_handle_t *ph_mr,
@@ -69,6 +71,14 @@ mlnx_register_mr (
status = IB_INVALID_PARAMETER;
goto err_invalid_parm;
}
+ if( BYTE_OFFSET( p_mr_create->vaddr ) != BYTE_OFFSET( mapaddr ) )
+ {
+ HCA_PRINT(TRACE_LEVEL_WARNING, HCA_DBG_MEMORY,
+ ("mapaddr offset != vaddr offset\n"));
+ status = IB_INVALID_PARAMETER;
+ goto err_invalid_parm;
+ }
+
/*
* Local write permission is required if remote write or
* remote atomic permission is also requested.
@@ -88,7 +98,7 @@ mlnx_register_mr (
// register mr
p_ib_mr = ibv_reg_mr(p_ib_pd, (ULONG_PTR)p_mr_create->vaddr,
- p_mr_create->length, (ULONG_PTR)p_mr_create->vaddr,
+ p_mr_create->length, mapaddr,
access_ctrl);
if (IS_ERR(p_ib_mr)) {
err = PTR_ERR(p_ib_mr);
@@ -116,6 +126,35 @@ err_unsupported:
}
ib_api_status_t
+mlnx_register_mr (
+ IN const ib_pd_handle_t h_pd,
+ IN const ib_mr_create_t *p_mr_create,
+ OUT net32_t* const p_lkey,
+ OUT net32_t* const p_rkey,
+ OUT ib_mr_handle_t *ph_mr,
+ IN boolean_t um_call )
+{
+ ib_api_status_t status;
+
+ HCA_ENTER(HCA_DBG_MEMORY);
+
+ // sanity checks
+ if (!p_mr_create) {
+ HCA_PRINT(TRACE_LEVEL_WARNING ,HCA_DBG_MEMORY,
+ ("invalid attributes\n"));
+ status = IB_INVALID_PARAMETER;
+ goto err_invalid_parm;
+ }
+
+ status = mlnx_register_mr_remap(
+ h_pd, p_mr_create, (ULONG_PTR)p_mr_create->vaddr, p_lkey, p_rkey, ph_mr, um_call );
+
+err_invalid_parm:
+ HCA_EXIT(HCA_DBG_MEMORY);
+ return status;
+}
+
+ib_api_status_t
mlnx_register_pmr (
IN const ib_pd_handle_t h_pd,
IN const ib_phys_create_t* const p_pmr_create,
@@ -662,6 +701,8 @@ mlnx_mr_if(
p_interface->alloc_fast_reg_mr = mlnx_alloc_fast_reg_mr;
p_interface->alloc_fast_reg_page_list = mlnx_alloc_fast_reg_page_list;
p_interface->free_fast_reg_page_list = mlnx_free_fast_reg_page_list;
+
+ p_interface->register_mr_remap = mlnx_register_mr_remap;
}
void
diff -dwup3 -x *svn* -x makefile -x sources -r c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\inc\iba\ib_ci.h .\inc\iba\ib_ci.h
--- c:\dev\openib\ofw\gen1\branches\mlx4_30\trunk\inc\iba\ib_ci.h Thu Aug 02 16:18:42 2012
+++ .\inc\iba\ib_ci.h Fri Aug 03 14:21:53 2012
@@ -1622,6 +1622,78 @@ typedef ib_api_status_t
******
*/
+
+/****f* Verbs/ci_register_mr
+* NAME
+* ci_register_mr -- Register a memory region with the HCA.
+* SYNOPSIS
+*/
+typedef ib_api_status_t
+(*ci_register_mr_remap) (
+ IN const ib_pd_handle_t h_pd,
+ IN const ib_mr_create_t* const p_mr_create,
+ IN const uint64_t mapaddr,
+ OUT net32_t* const p_lkey,
+ OUT net32_t* const p_rkey,
+ OUT ib_mr_handle_t * const ph_mr,
+ IN boolean_t um_call );
+/*
+* DESCRIPTION
+* This routine registers a virtually contiguous region of memory with the
+* HCA. All memory regions that need to be used by the HCA must be registered
+* prior to use in data transfer operations. On successful completion
+* the region handle, lkey are returned. If remote access rights are specified
+* then the rkey is also returned.
+* PARAMETERS
+* h_pd
+* [in] Handle to the PD on which memory is being registered
+* p_mr_create
+* [in] Holds attributes for the region being registered. Look at
+* ib_mr_create_t for more details.
+* mapaddr
+* [in] Holds the requested address to use as the base for the MR. Must have the same
+* page offset as the address provided in p_mr_create.
+* p_lkey
+* [out] Local Key Attributes of the registered memory region
+* p_rkey
+* [out] Remote key of the registered memory region. The verbs provider
+* is required to give this in the expected ordering on the wire. When
+* rkey's are exchanged between remote nodes, no swapping of this data
+* will be performed.
+* ph_mr
+* [out] Handle to the registered memory region. This handle is used when
+* submitting work requests to refer to this region of memory.
+* um_call
+* [in] Boolean indicating whether the registration originated in user-mode.
+* RETURN VALUE
+* IB_SUCCESS
+* Registration with the adapter was successful.
+* IB_INSUFFICIENT_RESOURCES
+* Insufficient resources to satisfy request.
+* IB_INVALID_PARAMETER
+* One of the input pointers was NULL.
+* IB_INVALID_PD_HANDLE
+* Invalid mr_pdhandle
+* IB_INVALID_PERMISSION
+* Invalid access rights.
+* NOTES
+* In addition to registration, the routine also pins memory, so that the
+* physical page associated with the virtual address does not get swapped
+* out during the time the HCA is attempting to transfer data to this
+* address. If the memory is not pinned, this could lead to data-corruption
+* and unpredictable behavior by the operating environment.
+*
+* This routine allows mapping an arbitrary address to the MR, as long as the
+* offset into the first page of the buffer is identical between the real
+* virtual address and the requested mapping address.
+*
+* SEE ALSO
+* ci_deregister_mr, ci_query_mr, ci_register_pmr, ci_modify_mr,
+* ci_register_smr
+******
+*/
+
+
/****f* Verbs/ci_register_pmr
* NAME
* ci_register_pmr -- Register a physical memory region with the HCA.
@@ -3269,6 +3341,11 @@ typedef struct _ci_interface
/* 2.5 verbs */
ci_create_qp_ex create_qp_ex;
+ /*
+ * Extended functionality to support kernel clients.
+ */
+ ci_register_mr_remap register_mr_remap;
+
} ci_interface_t;
/********/
-------------- next part --------------
A non-text attachment was scrubbed...
Name: ndv2.17.patch
Type: application/octet-stream
Size: 9782 bytes
Desc: ndv2.17.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20130221/4c86ee64/attachment.obj>
More information about the ofw
mailing list