[ofw] RE: [IPoIB CM] fix for large IP packets fragmentation failure.
Alex Estrin
alex.estrin at qlogic.com
Thu Jan 29 12:22:08 PST 2009
Applied in rev. 1918
Thanks,
Alex.
> -----Original Message-----
> From: Alex Estrin
> Sent: Thursday, January 29, 2009 2:34 PM
> To: Alex Estrin; Alex Naslednikov; Tzachi Dar
> Cc: ofw at lists.openfabrics.org
> Subject: RE: [IPoIB CM] fix for large IP packets
> fragmentation failure.
>
> Hello,
>
> Here is a revised version of the changes.
> Revoked most of previous changes from rev. 1873 and rev.1885
> Used one preallocated send descriptor.
> It is all we need for handling large packets segmentation.
>
> Thanks,
> Alex.
>
> ---
>
> Index: kernel/ipoib_cm.c
> ===================================================================
> --- kernel/ipoib_cm.c (revision 1885)
> +++ kernel/ipoib_cm.c (working copy)
> @@ -796,7 +796,6 @@
> uint32_t total_length = 0;
> ipoib_endpt_t *p_endpt;
> send_buf_t *p_send_buf;
> - ipoib_send_desc_t *p_desc;
>
> size_t i;
> BOOLEAN send_failed = FALSE;
> @@ -832,9 +831,7 @@
> CL_ASSERT( p_packet );
>
> CL_ASSERT( IPOIB_PORT_FROM_PACKET( p_packet )
> == p_port );
> - p_desc = IPOIB_SEND_FROM_PACKET( p_packet );
> - p_send_buf = p_desc->p_buf;
> - ipoib_send_desc_mgr_put( p_port, p_desc );
> + p_send_buf = IPOIB_SEND_FROM_PACKET( p_packet );
>
> switch( p_wc->status )
> {
> Index: kernel/ipoib_log.mc
> ===================================================================
> --- kernel/ipoib_log.mc (revision 1885)
> +++ kernel/ipoib_log.mc (working copy)
> @@ -332,10 +332,3 @@
> %2: Connected Mode initialized and operational.
> .
>
> -MessageId=0x005E
> -Facility=IPoIB
> -Severity=Error
> -SymbolicName=EVENT_IPOIB_SEND_DESC_POOL
> -Language=English
> -%2: Failed to create send descriptors pool.
> -.
> Index: kernel/ipoib_port.c
> ===================================================================
> --- kernel/ipoib_port.c (revision 1885)
> +++ kernel/ipoib_port.c (working copy)
> @@ -380,24 +380,6 @@
> IN uint8_t* p_options,
> IN uint32_t options_len,
> IN BOOLEAN copy_all );
> -
> -/************************************************************
> *****************
> - Send Descriptors management
> -*************************************************************
> ****************/
> -static ib_api_status_t
> -__send_desc_mgr_init(
> - IN ipoib_port_t* const
> p_port );
> -
> -static void
> -__send_desc_mgr_destroy(
> - IN ipoib_port_t* const
> p_port );
> -
> -static cl_status_t
> -__send_desc_ctor(
> - IN void* const
> p_object,
> - IN void*
> context,
> - OUT cl_pool_item_t** const pp_pool_item );
> -
>
> /*************************************************************
> *****************
> *
> * Endpoint manager operations
> @@ -689,16 +671,6 @@
> ("cl_spinlock_init returned %#x\n",
> cl_status) );
> return IB_ERROR;
> }
> -
> - /* initialize send descriptors manager. */
> - status = __send_desc_mgr_init( p_port );
> - if( status != IB_SUCCESS )
> - {
> - IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
> - ("__send_desc_mgr_init returned %s\n",
> - p_adapter->p_ifc->get_err_str( status )) );
> - return status;
> - }
>
> /* Initialize the IB resource manager. */
> status = __ib_mgr_init( p_port );
> @@ -829,8 +801,7 @@
> /* Wait for all sends and receives to get flushed. */
> while( p_port->send_mgr.depth || p_port->recv_mgr.depth )
> cl_thread_suspend( 0 );
> -
> - __send_desc_mgr_destroy( p_port );
> +
> /* Destroy the send and receive managers before closing
> the CA. */
> __ib_mgr_destroy( p_port );
>
> @@ -4181,8 +4152,7 @@
> {
> IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
> (" ARP %s SEND to ENDPT[%p] State: %d flag:
> %#x, QPN: %#x MAC %02x:%02x:%02x:%02x:%02x:%02x\n",
> - ( p_ib_arp->op == ARP_OP_REP ? "REP": "REQ"),
> - p_desc->p_endpt,
> + ( p_ib_arp->op == ARP_OP_REP ? "REP":
> "REQ"), p_desc->p_endpt,
> endpt_cm_get_state( p_desc->p_endpt ),
> p_desc->p_endpt->cm_flag,
> cl_ntoh32( ipoib_addr_get_qpn(
> &p_ib_arp->dst_hw )),
> @@ -4421,7 +4391,7 @@
> /* Store context in our reserved area of the packet. */
> IPOIB_PORT_FROM_PACKET( p_desc->p_pkt ) = p_port;
> IPOIB_ENDPT_FROM_PACKET( p_desc->p_pkt ) = p_desc->p_endpt;
> - IPOIB_SEND_FROM_PACKET( p_desc->p_pkt ) = p_desc;
> + IPOIB_SEND_FROM_PACKET( p_desc->p_pkt ) = p_desc->p_buf;
>
> IPOIB_EXIT( IPOIB_DBG_SEND );
> return NDIS_STATUS_SUCCESS;
> @@ -4525,7 +4495,7 @@
> ExFreeToNPagedLookasideList(
> &p_port->buf_mgr.send_buf_list, p_desc->p_buf );
> }
> - ipoib_send_desc_mgr_put( p_port, p_desc );
> +
> IPOIB_EXIT( IPOIB_DBG_SEND );
> }
>
> @@ -4578,24 +4548,13 @@
> cl_spinlock_acquire( &p_port->send_lock );
> for( i = 0; i < num_packets; i++ )
> {
> - p_desc = ipoib_send_desc_mgr_get( p_port );
> - if( !p_desc )
> - {
> - IPOIB_PRINT( TRACE_LEVEL_WARNING,
> IPOIB_DBG_SEND,
> - ("No available Send Descriptors.\n") );
> - while( i < num_packets )
> - {
> - cl_qlist_insert_tail(
> &p_port->send_mgr.pending_list,
> -
> IPOIB_LIST_ITEM_FROM_PACKET( p_packet_array[i++] ) );
> - }
> - break;
> - }
> -
> + p_desc = &p_port->send_mgr.desc;
> p_desc->p_pkt = p_packet_array[i];
> p_desc->p_endpt = NULL;
> p_desc->p_buf = NULL;
> p_desc->send_qp = NULL;
> p_desc->num_wrs = 1;
> + p_desc->send_dir = 0;
>
> /* Get the ethernet header so we can find the
> endpoint. */
> cl_perf_start( GetEthHdr );
> @@ -4661,7 +4620,6 @@
> cl_perf_stop( &p_port->p_adapter->perf, SendMgrQueue );
> if( status == NDIS_STATUS_PENDING )
> {
> - ipoib_send_desc_mgr_put( p_port, p_desc );
> /* Queue all remaining packets. */
> cl_perf_start( QueuePacket );
> while( i < num_packets )
> @@ -4693,7 +4651,6 @@
> if( status == NDIS_STATUS_PENDING )
> {
> ipoib_endpt_deref( p_desc->p_endpt );
> - ipoib_send_desc_mgr_put(
> p_port, p_desc );
> break;
> }
> cl_perf_start( ProcessFailedSends );
> @@ -4771,19 +4728,14 @@
> ("No available WQEs.\n") );
> break;
> }
> - p_desc = ipoib_send_desc_mgr_get( p_port );
> - if( !p_desc )
> - {
> - IPOIB_PRINT( TRACE_LEVEL_WARNING,
> IPOIB_DBG_SEND,
> - ("No available Send Descriptors.\n") );
> - break;
> - }
> + p_desc = &p_port->send_mgr.desc;
> p_desc->p_pkt = IPOIB_PACKET_FROM_LIST_ITEM(
> cl_qlist_remove_head(
> &p_port->send_mgr.pending_list ) );
> p_desc->p_endpt = NULL;
> p_desc->p_buf = NULL;
> p_desc->send_qp = NULL;
> p_desc->num_wrs = 1;
> + p_desc->send_dir = 0;
>
> /* Get the ethernet header so we can find the
> endpoint. */
> status = __send_mgr_get_eth_hdr(
> @@ -4804,7 +4756,6 @@
> CL_ASSERT( p_desc->p_endpt == NULL );
> cl_qlist_insert_head(
> &p_port->send_mgr.pending_list,
> IPOIB_LIST_ITEM_FROM_PACKET(
> p_desc->p_pkt ) );
> - ipoib_send_desc_mgr_put( p_port, p_desc );
> break;
> }
> else if( status != NDIS_STATUS_SUCCESS )
> @@ -4821,7 +4772,6 @@
> ("Multicast Mac
> - trying to join.\n") );
> cl_qlist_insert_head(
> &p_port->send_mgr.pending_list,
>
> IPOIB_LIST_ITEM_FROM_PACKET( p_desc->p_pkt ) );
> -
> ipoib_send_desc_mgr_put( p_port, p_desc );
> break;
> }
> }
> @@ -4845,7 +4795,6 @@
> {
> /* ARP REPLY packet queued */
> ipoib_endpt_deref( p_desc->p_endpt );
> - ipoib_send_desc_mgr_put(
> p_port, p_desc );
> break;
> }
> cl_perf_start( ProcessFailedSends );
> @@ -4891,7 +4840,6 @@
> uint32_t length;
> ipoib_endpt_t *p_endpt;
> send_buf_t *p_send_buf = NULL;
> - ipoib_send_desc_t *p_desc;
> ip_stat_sel_t type;
> size_t i;
> PERF_DECLARE( SendCompBundle );
> @@ -4941,9 +4889,8 @@
> CL_ASSERT( p_packet );
> CL_ASSERT(
> IPOIB_PORT_FROM_PACKET( p_packet ) == p_port );
> p_endpt =
> IPOIB_ENDPT_FROM_PACKET( p_packet );
> - p_desc =
> IPOIB_SEND_FROM_PACKET( p_packet );
> - p_send_buf = p_desc->p_buf;
> - ipoib_send_desc_mgr_put(
> p_port, p_desc );
> + p_send_buf =
> IPOIB_SEND_FROM_PACKET( p_packet );
> +
> if( p_endpt->h_mcast )
> {
> if(
> p_endpt->dgid.multicast.raw_group_id[11] == 0xFF &&
> @@ -4977,9 +4924,7 @@
> if( p_packet )
> {
> p_endpt =
> IPOIB_ENDPT_FROM_PACKET( p_packet );
> - p_desc =
> IPOIB_SEND_FROM_PACKET( p_packet );
> - p_send_buf = p_desc->p_buf;
> -
> ipoib_send_desc_mgr_put( p_port, p_desc );
> + p_send_buf =
> IPOIB_SEND_FROM_PACKET( p_packet );
> ipoib_inc_send_stat(
> p_port->p_adapter, IP_STAT_DROPPED, 0 );
> NdisMSendCompleteX(
> p_port->p_adapter->h_adapter,
> p_packet,
> NDIS_STATUS_RESET_IN_PROGRESS );
> @@ -4996,9 +4941,7 @@
> if( p_packet )
> {
> p_endpt =
> IPOIB_ENDPT_FROM_PACKET( p_packet );
> - p_desc =
> IPOIB_SEND_FROM_PACKET( p_packet );
> - p_send_buf = p_desc->p_buf;
> -
> ipoib_send_desc_mgr_put( p_port, p_desc );
> + p_send_buf =
> IPOIB_SEND_FROM_PACKET( p_packet );
> ipoib_inc_send_stat(
> p_port->p_adapter, IP_STAT_ERROR, 0 );
> NdisMSendCompleteX(
> p_port->p_adapter->h_adapter,
> p_packet,
> NDIS_STATUS_FAILURE );
> @@ -7570,117 +7513,3 @@
> }
> return;
> }
> -
> -
> -/***************************************************
> - Send desriptors pool management
> -**************************************************/
> -
> -static ib_api_status_t
> -__send_desc_mgr_init(
> - IN ipoib_port_t* const p_port )
> -{
> - cl_status_t cl_status;
> -
> - IPOIB_ENTER( IPOIB_DBG_INIT );
> -
> - cl_qpool_construct( &p_port->send_mgr.desc_pool );
> - cl_spinlock_init( &p_port->send_mgr.desc_lock );
> -
> - /* Allocate send descriptors pool */
> - cl_status = cl_qpool_init( &p_port->send_mgr.desc_pool,
> - p_port->p_adapter->params.sq_depth,
> - 0,
> - 10,
> - sizeof( ipoib_send_desc_t ),
> - __send_desc_ctor,
> - NULL,
> - p_port );
> -
> - if( cl_status != CL_SUCCESS )
> - {
> - NdisWriteErrorLogEntry( p_port->p_adapter->h_adapter,
> - EVENT_IPOIB_SEND_DESC_POOL, 1, cl_status );
> -
> - IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
> - ("cl_qpool_init for send descriptors
> pool returned %#x\n", cl_status) );
> -
> - return IB_INSUFFICIENT_MEMORY;
> - }
> -
> - IPOIB_EXIT( IPOIB_DBG_INIT );
> - return IB_SUCCESS;
> -}
> -
> -static void
> -__send_desc_mgr_destroy(
> - IN ipoib_port_t* const p_port )
> -{
> -
> - IPOIB_ENTER(IPOIB_DBG_INIT );
> -
> - CL_ASSERT( p_port );
> - cl_spinlock_destroy( &p_port->send_mgr.desc_lock );
> - if( cl_is_qpool_inited( &p_port->send_mgr.desc_pool ) )
> - {
> - cl_qpool_destroy( &p_port->send_mgr.desc_pool );
> - }
> - IPOIB_EXIT( IPOIB_DBG_INIT );
> -}
> -
> -static cl_status_t
> -__send_desc_ctor(
> - IN void* const
> p_object,
> - IN void*
> context,
> - OUT cl_pool_item_t** const
> pp_pool_item )
> -{
> - ipoib_send_desc_t* p_desc;
> - ipoib_port_t* p_port;
> -
> - CL_ASSERT( p_object );
> - CL_ASSERT( context );
> -
> - p_desc = (ipoib_send_desc_t*)p_object;
> - p_port = (ipoib_port_t*)context;
> - p_desc->p_endpt = NULL;
> - p_desc->p_buf = NULL;
> - p_desc->send_qp = NULL;
> - p_desc->num_wrs = 1;
> -
> - *pp_pool_item = &p_desc->pool_item;
> - return CL_SUCCESS;
> -}
> -
> -inline ipoib_send_desc_t*
> -ipoib_send_desc_mgr_get(
> - IN ipoib_port_t* const p_port )
> -{
> - ipoib_send_desc_t *p_desc;
> - cl_spinlock_acquire( &p_port->send_mgr.desc_lock );
> - p_desc = (ipoib_send_desc_t*)cl_qpool_get(
> &p_port->send_mgr.desc_pool );
> - cl_spinlock_release( &p_port->send_mgr.desc_lock );
> -
> - return p_desc;
> -}
> -
> -void
> -ipoib_send_desc_mgr_put(
> - IN ipoib_port_t* const p_port,
> - IN ipoib_send_desc_t* const p_desc )
> -{
> - /* Return the descriptor to it's pool. */
> - cl_spinlock_acquire( &p_port->send_mgr.desc_lock );
> - cl_qpool_put( &p_port->send_mgr.desc_pool, &p_desc->pool_item );
> - cl_spinlock_release( &p_port->send_mgr.desc_lock );
> -
> -}
> -
> -void
> -ipoib_send_desc_mgr_put_list(
> - IN ipoib_port_t* const p_port,
> - IN cl_qlist_t* const p_list )
> -{
> - cl_spinlock_acquire( &p_port->send_mgr.desc_lock );
> - cl_qpool_put_list( &p_port->send_mgr.desc_pool, p_list );
> - cl_spinlock_release( &p_port->send_mgr.desc_lock );
> -}
> Index: kernel/ipoib_port.h
> ===================================================================
> --- kernel/ipoib_port.h (revision 1885)
> +++ kernel/ipoib_port.h (working copy)
> @@ -72,7 +72,7 @@
> #define IPOIB_RECV_FROM_PACKET( P ) \
> (((void **)P->MiniportReservedEx)[1])
> #define IPOIB_SEND_FROM_PACKET( P ) \
> - (((void **)P->MiniportReservedEx)[2])
> + (((send_buf_t**)P->MiniportReservedEx)[2])
> #define IPOIB_PACKET_FROM_LIST_ITEM( I ) \
> (PARENT_STRUCT( I, NDIS_PACKET, MiniportReservedEx ))
> #define IPOIB_LIST_ITEM_FROM_PACKET( P ) \
> @@ -401,7 +401,6 @@
>
> typedef struct _ipoib_send_desc
> {
> - cl_pool_item_t pool_item;
> NDIS_PACKET *p_pkt;
> ipoib_endpt_t *p_endpt;
> send_buf_t *p_buf;
> @@ -470,10 +469,8 @@
> {
> atomic32_t depth;
> cl_qlist_t pending_list;
> - cl_spinlock_t desc_lock;
> - cl_qpool_t desc_pool;
> + ipoib_send_desc_t desc;
>
> -
> } ipoib_send_mgr_t;
> /*
> * FIELDS
> @@ -796,19 +793,4 @@
> return 256;
> }
> }
> -
> -inline ipoib_send_desc_t*
> -ipoib_send_desc_mgr_get(
> - IN ipoib_port_t* const
> p_port );
> -
> -void
> -ipoib_send_desc_mgr_put(
> - IN ipoib_port_t* const p_port,
> - IN ipoib_send_desc_t* const p_desc );
> -
> -void
> -ipoib_send_desc_mgr_put_list(
> - IN ipoib_port_t* const p_port,
> - IN cl_qlist_t* const
> p_list );
> -
> #endif /* _IPOIB_PORT_H_ */
>
More information about the ofw
mailing list