[ofw] [PATCH 8/12] verbs: optimize work request/completion structures
Sean Hefty
sean.hefty at intel.com
Mon Jul 21 09:31:12 PDT 2008
Change work completion types to quickly allow a user to determine if a
completions is for a send or receive. Align work request and work
completion type values to be the same for consistency.
Update work request structure to avoid unnecessary internal padding and
allocating extra unused space for pointers and structures that can be
part of a union. Arrange structure layout to allow casting between
the UVP, IBAL, WinVerbs, and libibverbs work requests.
Update debug print functions to match changes.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
As far as I could tell, there is no code that ever results in passing the work
request structure between userspace and the kernel. We can have a cleaner,
more efficient interface if we do not try to maintain a single structure that
is the same for a 32-bit userspace application and a 64-bit kernel, especially
if the wr structure is not used in that context in practice.
In fact it appears that there's a lot of kernel 'proxy' code that is unreachable
given the existing libraries and drivers that we should consider removing.
diff -up -r -X trunk\docs\dontdiff.txt -I '\$Id:' trunk/core/al/ib_statustext.c branches\winverbs/core/al/ib_statustext.c
--- trunk/core/al/ib_statustext.c 2008-04-30 12:42:31.556750000 -0700
+++ branches\winverbs/core/al/ib_statustext.c 2008-06-03 21:46:04.479125000 -0700
@@ -186,39 +186,46 @@ ib_get_wc_status_str(
}
-static const char* const __ib_wc_type_str[] =
+static const char* const __ib_wc_send_type_str[] =
{
"IB_WC_SEND",
"IB_WC_RDMA_WRITE",
- "IB_WC_RECV",
"IB_WC_RDMA_READ",
- "IB_WC_MW_BIND",
- "IB_WC_FETCH_ADD",
"IB_WC_COMPARE_SWAP",
- "IB_WC_RECV_RDMA_WRITE"
- "IB_WC_UNKNOWN"
+ "IB_WC_FETCH_ADD",
+ "IB_WC_MW_BIND",
};
+static const char* const __ib_wc_recv_type_str[] =
+{
+ "IB_WC_RECV",
+ "IB_WC_RECV_RDMA_WRITE"
+};
const char*
ib_get_wc_type_str(
IN ib_wc_type_t wc_type )
{
- if( wc_type > IB_WC_UNKNOWN )
- wc_type = IB_WC_UNKNOWN;
- return( __ib_wc_type_str[wc_type] );
+ if ( wc_type & IB_WC_RECV )
+ if ( wc_type - IB_WC_RECV >= IB_WC_UNKNOWN2)
+ return "IB_WC_UNKNOWN";
+ else
+ return __ib_wc_recv_type_str[wc_type - IB_WC_RECV];
+ else
+ if ( wc_type >= IB_WC_UNKNOWN1 )
+ return "IB_WC_UNKNOWN";
+ else
+ return __ib_wc_send_type_str[wc_type];
}
static const char* const __ib_wr_type_str[] =
{
- "WR_DUMMY", /*place holder*/
"WR_SEND",
"WR_RDMA_WRITE",
"WR_RDMA_READ",
"WR_COMPARE_SWAP",
- "WR_FETCH_ADD",
- "WR_UNKNOWN"
+ "WR_FETCH_ADD"
};
@@ -226,8 +233,8 @@ const char*
ib_get_wr_type_str(
IN uint8_t wr_type )
{
- if( wr_type > WR_UNKNOWN )
- wr_type = WR_UNKNOWN;
+ if( wr_type >= WR_UNKNOWN )
+ return "WR_UNKNOWN";
return( __ib_wr_type_str[wr_type] );
}
diff -up -r -X trunk\docs\dontdiff.txt -I '\$Id:' trunk/inc/iba/ib_types.h branches\winverbs/inc/iba/ib_types.h
--- trunk/inc/iba/ib_types.h 2008-06-03 21:13:53.666625000 -0700
+++ branches\winverbs/inc/iba/ib_types.h 2008-06-03 21:46:04.510375000 -0700
@@ -10010,7 +10010,7 @@ typedef struct _ib_qp_mod
*/
typedef enum _ib_wr_type_t
{
- WR_SEND = 1,
+ WR_SEND,
WR_RDMA_WRITE,
WR_RDMA_READ,
WR_COMPARE_SWAP,
@@ -10142,65 +10142,66 @@ typedef uint32_t ib_send_opt_t;
*/
typedef struct _ib_send_wr
{
- TO_LONG_PTR(struct _ib_send_wr*, p_next);
uint64_t wr_id;
- ib_wr_type_t wr_type;
- ib_send_opt_t send_opt;
+ struct _ib_send_wr* p_next;
+ ib_local_ds_t* ds_array;
uint32_t num_ds;
- TO_LONG_PTR(ib_local_ds_t*, ds_array);
+ ib_wr_type_t wr_type;
+ ib_send_opt_t send_opt;
ib_net32_t immediate_data;
- union _send_dgrm
+ union
{
- struct _send_ud
+ union _send_dgrm
{
- ib_net32_t remote_qp;
- ib_net32_t remote_qkey;
- TO_LONG_PTR(ib_av_handle_t, h_av);
- uint16_t pkey_index;
- TO_LONG_PTR(void*, rsvd);
-
- } ud;
-
- struct _send_rd
- {
- ib_net32_t remote_qp;
- ib_net32_t remote_qkey;
- ib_net32_t eecn;
+ struct _send_ud
+ {
+ ib_av_handle_t h_av;
+ ib_net32_t remote_qp;
+ ib_net32_t remote_qkey;
+ void* rsvd;
+ uint16_t pkey_index;
- } rd;
+ } ud;
- struct _send_raw_ether
- {
- ib_net16_t dest_lid;
- uint8_t path_bits;
- uint8_t sl;
- uint8_t max_static_rate;
- ib_net16_t ether_type;
+ struct _send_rd
+ {
+ ib_net32_t remote_qp;
+ ib_net32_t remote_qkey;
+ ib_net32_t eecn;
- } raw_ether;
+ } rd;
- struct _send_raw_ipv6
- {
- ib_net16_t dest_lid;
- uint8_t path_bits;
- uint8_t sl;
- uint8_t max_static_rate;
+ struct _send_raw_ether
+ {
+ ib_net16_t dest_lid;
+ uint8_t path_bits;
+ uint8_t sl;
+ uint8_t max_static_rate;
+ ib_net16_t ether_type;
- } raw_ipv6;
+ } raw_ether;
- } dgrm;
+ struct _send_raw_ipv6
+ {
+ ib_net16_t dest_lid;
+ uint8_t path_bits;
+ uint8_t sl;
+ uint8_t max_static_rate;
- struct _send_remote_ops
- {
- uint64_t vaddr;
- net32_t rkey;
+ } raw_ipv6;
- ib_net64_t atomic1;
- ib_net64_t atomic2;
+ } dgrm;
- } remote_ops;
+ struct _send_remote_ops
+ {
+ uint64_t vaddr;
+ net32_t rkey;
+ ib_net64_t atomic1;
+ ib_net64_t atomic2;
+ } remote_ops;
+ };
} ib_send_wr_t;
/*
* FIELDS
@@ -10553,13 +10554,14 @@ typedef enum _ib_wc_type_t
{
IB_WC_SEND,
IB_WC_RDMA_WRITE,
- IB_WC_RECV,
IB_WC_RDMA_READ,
- IB_WC_MW_BIND,
- IB_WC_FETCH_ADD,
IB_WC_COMPARE_SWAP,
+ IB_WC_FETCH_ADD,
+ IB_WC_MW_BIND,
+ IB_WC_UNKNOWN1,
+ IB_WC_RECV = (1 << 7),
IB_WC_RECV_RDMA_WRITE,
- IB_WC_UNKNOWN
+ IB_WC_UNKNOWN2
} ib_wc_type_t;
/*****/
@@ -10640,8 +10642,8 @@ typedef struct _ib_wc
ib_wc_type_t wc_type;
uint32_t length;
+ uint64_t vendor_specific;
ib_wc_status_t status;
- uint64_t vendor_specific;
union _wc_recv
{
diff -up -r -X trunk\docs\dontdiff.txt -I '\$Id:' trunk/core/al/kernel/al_proxy_verbs.c
branches\winverbs/core/al/kernel/al_proxy_verbs.c
--- trunk/core/al/kernel/al_proxy_verbs.c 2008-07-16 08:53:56.437875000 -0700
+++ branches\winverbs/core/al/kernel/al_proxy_verbs.c 2008-07-16 22:53:35.671875000 -0700
@@ -2356,7 +2356,7 @@ proxy_post_send(
{
/* Validate the AV handle for UD */
h_av = (ib_av_handle_t)al_hdl_ref( p_context->h_al,
- p_wr[i].dgrm.ud.h_av_padding, AL_OBJ_TYPE_H_AV );
+ (ULONG_PTR) p_wr[i].dgrm.ud.h_av, AL_OBJ_TYPE_H_AV );
if( !h_av )
{
status = IB_INVALID_AV_HANDLE;
diff -up -r -X trunk\docs\dontdiff.txt -I '\$Id:' trunk/core/al/user/ual_qp.c branches\winverbs/core/al/user/ual_qp.c
--- trunk/core/al/user/ual_qp.c 2008-07-16 08:53:58.016000000 -0700
+++ branches\winverbs/core/al/user/ual_qp.c 2008-07-16 22:53:17.000000000 -0700
@@ -119,8 +119,8 @@ ual_post_send(
p_qp_ioctl->in.send_wr[num_wr] = *p_wr;
if( h_qp->type == IB_QPT_UNRELIABLE_DGRM )
{
- p_qp_ioctl->in.send_wr[num_wr].dgrm.ud.h_av_padding =
- p_wr->dgrm.ud.h_av->obj.hdl;
+ p_qp_ioctl->in.send_wr[num_wr].dgrm.ud.h_av =
+ (ib_av_handle_t) (ULONG_PTR) p_wr->dgrm.ud.h_av->obj.hdl;
}
num_wr++;
cl_memcpy(
More information about the ofw
mailing list